ucc-1.8.0/0000775000175000017500000000000015211535630012457 5ustar alastairalastairucc-1.8.0/greptile.json0000664000175000017500000000122215211535620015161 0ustar alastairalastair{ "labels": ["Ready-For-Review", "ai-review"], "triggerOnUpdates": true, "shouldUpdateDescription": false, "strictness": 2, "fixWithAI": false, "includeBranches": ["master"], "summarySection": { "included": true, "collapsible": true, "defaultOpen": false }, "issuesTableSection": { "included": true, "collapsible": true, "defaultOpen": false }, "confidenceScoreSection": { "included": true, "collapsible": true, "defaultOpen": false }, "sequenceDiagramSection": { "included": false, "collapsible": false, "defaultOpen": false } } ucc-1.8.0/autogen.sh0000775000175000017500000000242215211535620014457 0ustar alastairalastair#!/bin/sh # Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE:-$0}" )" &> /dev/null && pwd )" if [ $SCRIPT_DIR != `pwd` ]; then echo "autogen.sh script must be launched from the top of the source tree" exit 1 fi rm -rf config/m4/tl_coll_plugins_list.m4 rm -rf config/m4/tls_list.m4 rm -rf src/components/tl/makefile.am touch config/m4/tl_coll_plugins_list.m4 touch config/m4/tls_list.m4 # Detect and generate TLs makefiles for t in $(ls -d src/components/tl/*/); do echo "m4_include([$t/configure.m4])" >> config/m4/tls_list.m4 plugin=$(basename $t) echo "SUBDIRS += components/tl/$plugin" >> src/components/tl/makefile.am done # Detect and generate TL coll plugins makefiles for t in $(ls -d src/components/tl/*/); do if [ -d $t/coll_plugins ]; then rm -rf $t/makefile.coll_plugins.am for cp in $(ls -d $t/coll_plugins/*/); do echo "m4_include([$cp/configure.m4])" >> config/m4/tl_coll_plugins_list.m4 plugin=$(basename $cp) echo "SUBDIRS += coll_plugins/$plugin" >> $t/makefile.coll_plugins.am done fi done rm -rf autom4te.cache mkdir -p config/m4 config/aux autoreconf -f -v --install || exit 1 rm -rf autom4te.cache exit 0 ucc-1.8.0/NEWS0000664000175000017500000004207015211535620013160 0ustar alastairalastair/** * @copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ ## Current ## New Features and Enhancements ### Core - Ported UCS logger from libucs to UCC, enabling file filtering and log-to-file features {PR #1191} ### TL/CUDA - Added multinode NVLS support using CUDA fabric handles for cross-node allreduce {PR #1185} - Added NVLS reduce_scatterv with BF16 datatype support and kernel-based synchronization {PR #1211} - Added ptrace permissions for NVLS POSIX handle sharing via pidfd_getfd {PR #1218} - Added NVLS allgatherv using multimem.st instructions with 16-byte alignment {PR #1240} ### TL/UCP - Added memory type parameter to tl_ucp_put/get for GPU memory in onesided collectives {PR #1253} - Fixed crashes in inplace mode for allgather, alltoall, and alltoallv {PR #1254} - Fixed onesided alltoall algorithm selection to default to PUT for 1 PPN {PR #1247} ### TL/NCCL - Added native ncclAlltoAll support for NCCL 2.28.3+ {PR #1244} ### Build and Test - Bumped version to v1.7 {PR #1225} - Updated clang-format rules for function wrapping and comment reflow {PR #1192} - Added Greptile AI code review configuration {PR #1208} - Improved configure status reporting for CUDA/NVML detection {PR #1239} - Fixed m4 configuration syntax for CUDA {PR #1252} - Fixed uninitialized variable warning in MLX5 UMR WQE test {PR #1195} - Added multinode NVLS tests on GB300 Slurm clusters {PR #1235} - Added timeout to MPI and DLRM tests to prevent hung jobs {PR #1226} - Added 90-minute timeout to torch UCC tests {PR #1204} - Added Blossom CI Jenkins dispatcher job for /build trigger {PR #1229} - Added GitHub Action workflow for Blossom pipeline initialization {PR #1227} - Migrated Jenkins credentials to swx-hpcx service account for SSH key rotation {PR #1233} - Added separate GitHub UI checks for each Jenkins job via Blossom {PR #1237} - Added Blossom CI separated checks and job output upload to GitHub {PR #1238} - Fixed Jenkins job folder name and email in CI configuration {PR #1236} - Fixed clang-format command to use git-clang-format-21 for Ubuntu 22.04 {PR #1212} - Migrated hpcsdk build from GitHub workflow to Jenkins + CI-DEMO {PR #1215} - Set Coverity aggressiveness level to medium for better issue detection {PR #1207} - Fixed parallel GPU tests with CUDA context creation and IB port validation {PR #1209} - Enabled parallel UCC test execution in CI {PR #1206} - Fixed Jenkins JJB YAML variable syntax for check separation {PR #1246} ### Documentation - Fixed various typos throughout comments and outputs {PR #1228} ### Tools - Added matrix generator for alltoallv traffic patterns (uniform, biased, random) {PR #1220} - Fixed segfault in scatterv perftest inplace mode due to early memory free {PR #1234} - Optimized perftest traffic matrix to reuse displacements for same-size messages {PR #1250} ## 1.6.0 (November 14th, 2025) ## New Features and Enhancements ### Core - Added UCC_DEBUGGER_WAIT environment variable {PR #1130} ### CL/HIER - Fixed Wlto-type-mismatch {PR #1179} ### TL/CUDA - Fixed printing of device PCI id {PR #1053} - Added NVLS improvements and bfloat16 data type support {PR #1162} - Added NVLS barrier {PR #1180} - Added Alltoall(v) copy engine {PR #1138} ### TL/UCP - Removed a debug print statement {PR #1177} - Added knomial allgather with mapped buffers {PR #1176} - Added node local id config {PR #1189} - Enable knomial allgatherv {PR #1188} - Added congestion avoidant onesided Alltoall {PR #1096} ### EC/CUDA - Fixed cuctx creation in EC CUDA {PR #1219} ### Build and Test - Added check to see if target exists in CMAKE {PR #1173} - Fixed build with GCC 14 {PR #1190} - Added gtest and mpi test for ucc_mem_map and ucc_mem_unmap {PR #1165} - Check for CX7 in wait_on_data gtest {PR #1127} ### Tools - Updated perftest to print BusBW {PR #1186} - Added support for onesided alltoall in perftest {PR #1194} - Added CUDA managed memory type to ucc_perftest {PR #1199} - Fixes for onesided alltoall in perftest {PR #1216} ## 1.5.0 (July 31st, 2025) ## New Features and Enhancements ### Core - Enhanced error logs in context creation {PR #1135} - Added ucc net devices configuration {PR #1141} - Enhanced error logging in collective initialization {PR #1104} - Added support for ucc_mem_map and ucc_mem_unmap {PR #1070} ### CL/HIER - Added flag for nonroot info {PR #1123} - Removed per node leader, fixed double free {PR #1126} ### TL/UCP - Fixed allreduce knomial data consistency {PR #1145} - Fixed allgather oneshot {PR #1134} - Added allgather linear implementation {PR #1122} - Added fallback if memh not passed {PR #1136} ### TL/MLX5 - Added CUDA support for zero-copy multicast {PR #1118} - Added configuration to set IB QP SL {PR #1057} - Fixed segfault in multicast team creation {PR #1150} - Recovered from IPoIB issue in multicast init {PR #1140} - Added HCA-assisted copy & CUDA scratch design {PR #1154} - Added logging for multicast FORCE/TRY modes {PR #1156} - Fixed reliability initialization after multicast setup {PR #1163} - Added global status check {PR #1113} ### TL/CUDA - Added NVLink SHARP (NVLS) Allreduce {PR #1148} - Added topology cache {PR #1137} - Added NVLink SHARP (NVLS) Reduce Scatter {PR #1144} ### EC/CUDA - Linked with stdc++ {PR #1168} ### EC/ROCM - Included stdbool.h for new versions of ROCm {PR #1146} ### Build and Test - Updated CUDA architecture {PR #1143} - Changed to CUDA 12.9 {PR #1155} - Fixed coverity issues {PR #1152} - Added buffers for onesided tests {PR #1100} - Added missing progress calls {PR #1151} ### Documentation - Updated component image 1.4.4 {PR #1153} ### Tools - Added perftest generator {PR #1147} ## 1.4.4 (April 25th, 2025) ## New Features and Enhancements ### Core - Implemented asymmetric memory support {PR #1000} - Enhanced error handling and resource cleanup {PR #960, #951} - Improved service team handling {PR #1046} - Fixed triggered post for zero size collectives {PR #960} ### CL/HIER - Added allgatherv support {PR #1111} - Implemented node subgroup unpacking {PR #1103} - Added reduce to supported collectives {PR #997} - Fixed integer overflow in alltoall {PR #944} ### TL/UCP - Split single and multithreaded send/receive operations {PR #1109} - Added knomial allgather with CUDA memory support {PR #1095} - Implemented reduce SRG knomial algorithm {PR #1058} - Added radix selection to knomial operations {PR #1072} - Added sliding window allreduce implementation {PR #958} - Added knomial allgatherv support {PR #1008} - Added sparbit algorithm for allgather {PR #940} - Extended broadcast active set support for size > 2 {PR #926} - Added knomial algorithm for reduce-scatter {PR #970} ### TL/MLX5 - Added multicast-based zero-copy broadcast {PR #1087} - Implemented mcast multi-group support {PR #1060} - Added non-blocking CUDA memory copy support {PR #1040} - Added device memory multicast broadcast {PR #989} - Enhanced mcast allgather staging-based algorithm {PR #994} - Improved one-sided mcast reliability initialization {PR #980} - Various performance optimizations in alltoall {PR #1067} - Fixed fences in all-to-all WQEs {PR #1069} - Added context option to disable all-to-all operations {PR #1062} - Improved error handling and device checks {PR #1102} - Disabled mcast for thread multiple mode {PR #961} ### TL/SHARP - Added support for allgather operation {PR #1081} - Enabled reduce-scatter with SAT support {PR #1084} - Added SHARP multi-channel support {PR #1049} - Fixed service team OOB handling {PR #1001} - Improved internal OOB usage {PR #986} ### CUDA - Added linear broadcast implementation {PR #948} - Batch CUDA stream memory operations, reduced CPU and GPU execution overhead {PR #1093} - Enhanced error handling for CUDA context operations {PR #1025} - Fixed context cleanup in CUDA operations {PR #954} ### Build and Test - Added support for specific GPU architectures with ROCM {PR #987} - Added UCC pkg-config support {PR #1036} - Fixed build compatibility with NVC compiler {PR #1052} - Enhanced config parser functionality {PR #1092} - Enhanced ASAN/LSAN memory leak detection {PR #1074} - Added error checking and exit handling in gtests {PR #1083} ### Documentation - Updated README with UCC publication information {PR #1028} - Added DOCA_UROM documentation {PR #999} - Fixed Doxygen documentation issues {PR #1038} - Enhanced code style consistency {PR #1020} ### CL/DOCA_UROM - Implemented new DOCA UROM plugin {PR #978} - Added support for offloading collective operations to DPUs - Implemented allreduce collective ## 1.3.0 (April 18th, 2024) ## New Features and Enhancements ### CL/HIER - Disable onesided alltoallv {PR #875} ### TL/CUDA - Initialize remote CUDA scratch to NULL {PR #911} ### TL/UCP - Enable hybrid alltoallv {PR #781} - Avoid copy in knomial scatter {PR #771} - Enable reorder ranks to reduce_scatter, Knomial Allreduce, Ring Allgather/v {PR #819} - Remove memcpy in last SRA step {PR #743} - Fix sparse pack in hybrid a2av {PR #825} - Fix recycle in hybrid a2av {PR #827} - Reorder ranks for SRA {PR #834} - Use ring allgather when reordering needed {PR #879} - Use pipelining in SRA allreduce for CUDA {PR #873} - Poll for onesided alltoall completion {PR #876} - Add support for non-host buffers in bruck alltoall {PR #852} - Added Neighbor Exchange Allgather{PR #822} ### TL/SHARP - Enable bcast for any predefined dt {PR #774} - Don't print team create error {PR #777} - Check datasize supported {PR #776} - Fix sharp context cleanup {PR #843} ### API - Remove duplicate get_version_string {PR #933} ### TL/NCCL - Make team init non-blocking {PR #772} - Add CUDA managed to score {PR #793} - Make ncclGroupEnd nb {PR #798} - Lazy init nccl comm {PR #851} ### TL/MLX5 - Share ib_ctx and pd {PR #749} - Rcache {PR #753} - Device memory and topo init {PR #780} - Adding mcast interface {PR #784} - A2A part 1 -- coll init {PR #790} - A2A part 2 -- full collective {PR #802} - Revisit team and ctx init {PR #815} - Fix context create hang {PR #887} - Add librdmacm linkage {PR #910} ### CORE - Fix score update when only score given {PR #779} - Coverity fixes {PR #809} - Additional coverty fixes {PR #813} - Fix error handling for ctx create epilog {PR #818} - Skip zero size collectives {PR #787} ### DOCS - Updating NEWS for v1.2 {PR #791} - Updating NEWS for v1.3 {PR #937} ### BUILD and TEST - Updated build system to enable UCC with ROCm 6.x {PR #906 and #917} - Check op and dt compatibility {PR #773} - Fix barrier test {PR #799} - Propagate HIP_CXXFLAGS to gtest and mpi {PR #803} ## 1.2.0 (June 6th, 2023) ## New Features and Enhancements ## CL/HIER - Fixed single proc on node issue in alltoall ([#658](https://github.com/openucx/ucc/pull/658)) - Implemented allreduce rab pipelined ([#608](https://github.com/openucx/ucc/pull/608)) - Added bcast 2step algorithm ([#620](https://github.com/openucx/ucc/pull/620)) - Fixed allreduce rab pipeline ([#759](https://github.com/openucx/ucc/pull/759)) ## TL/CUDA - Support for CUDA 12 - Fixed cache unmap issue ([#642](https://github.com/openucx/ucc/pull/642)) - Implemented reduce scatter linear ([#669](https://github.com/openucx/ucc/pull/669)) - Added algorithm selection based on topology ([#688](https://github.com/openucx/ucc/pull/688)) - Fixed linear algorithms ([#751](https://github.com/openucx/ucc/pull/751)) - Fixed pipelining in linear rs ([#770](https://github.com/openucx/ucc/pull/770)) ## TL/UCP - Added special service worker ([#560](https://github.com/openucx/ucc/pull/560)) - Added scatterv ([#663](https://github.com/openucx/ucc/pull/663)) - Added gatherv ([#664](https://github.com/openucx/ucc/pull/664)) - Fixed running with npolls 0 ([#695](https://github.com/openucx/ucc/pull/695)) - Added knomial allgather ([#729](https://github.com/openucx/ucc/pull/729)) - Fixed bug for triggered colls ([#757](https://github.com/openucx/ucc/pull/757)) - Added bruck alltoall ([#756](https://github.com/openucx/ucc/pull/756)) - Added SLOAV alltoallv ([#687](https://github.com/openucx/ucc/pull/687)) - Large message broadcast optimizations ([#738](https://github.com/openucx/ucc/pull/738)) - Ranks reordering in ring allgather for better locality([#69](https://github.com/openucx/ucc/pull/698)) ## TL/SHARP - Fixed memory type check in allreduce ([#662](https://github.com/openucx/ucc/pull/662)) - Added support for sharpv3 dt ([#661](https://github.com/openucx/ucc/pull/661)) - Fixed assert check ([#686](https://github.com/openucx/ucc/pull/686)) - Implemented SHARP OOB fixes ([#746](https://github.com/openucx/ucc/pull/746)) - Fixed local rank when NODE SBGP not enabled ([#760](https://github.com/openucx/ucc/pull/760)) - Prevented sharp team with team max ppn > 1 ([#761](https://github.com/openucx/ucc/pull/761)) ## CORE - Fixed memory type score update ([#650](https://github.com/openucx/ucc/pull/650)) - Fixed ucc parser build ([#666](https://github.com/openucx/ucc/pull/666)) - Implemented ucc_pipeline_params ([#675](https://github.com/openucx/ucc/pull/675)) - Changed log level of config_modify ([#667](https://github.com/openucx/ucc/pull/667)) - Fixed timeout handle for triggered post ([#679](https://github.com/openucx/ucc/pull/679)) ## DOCS - Added User Guide ([#720](https://github.com/openucx/ucc/pull/720)) ## 1.1.0 (October 7th, 2022) ## Features ## API - Added float 128 and float 32, 64, 128 (complex) data types - Added Active Sets based collectives to support dynamic groups as well as point-to-point messaging - Added ucc_team_get_attr interface ## Core - Config file support - Fixed component search ## CL - Added split rail allreduce collective implementation - Enable hierarchical alltoallv and barrier - Fixed cleanup bugs ## TL - Added SELF TL supporting team size one ### UCP - Added service broadcast - Added reduce_scatterv ring algorithm - Added k-nomial based gather collective implementation - Added one-sided get based algorithms ### SHARP - Fixed SHARP OOB - Added SHARP broadcast ### GPU Collectives (CUDA, NCCL TL and RCCL TL) - Added support for CUDA TL (intranode collectives for NVIDIA GPUs) - Added multiring allgatherv, alltoall, reduce-scatter, and reduce-scatterv multiring in CUDA TL - Added topo based ring construction in CUDA TL to maximize bandwidth - Added NCCL gather, scatter and its vector variant - Enable using multiple streams for collectives - Added support for RCCL gather (v), scatter (v), broadcast, allgather (v), barrier, alltoall (v) and all reduce collectives - Added ROCm memory component - Adapted all GPU collectives to executor design ### Tests - Added tests for triggered collectives in perftests - Fixed bugs in multi-threading tests ### Utils - Added CPU model and vendor detection - Several bug fixes in all components ## 1.0.0 (April 19th, 2022) ### Features #### API - Added Avg reduce operation - Added nonblocking team destroy option - Added user-defined datatype definitions - Added Bfloat16 type - Clarify semantics of core abstractions including teams and context - Added timeout option #### Core - Added coll scoring and selection support - Added support for Triggered collectives - Added support for timeouts in collectives - Added support for team create without ep in post - Added support for multithreaded context progress - Added support for nonblocking team destroy #### CL - Added support for hierarchical collectives - Added support for hierarchical allreduce collective operation - Added support for collectives based on one-sided communication routines #### TL - Added SHARP TL ##### UCP - Added Bcast SAG algorithm for large messages - Added Knomial based reduce algorithm - Making allgather and alltoall agree with the API - Added SRA knomial allreduce algorithm - Added pairwise alltoall and alltoallv algorithms - Added allgather and allgatherv ring algorithms - Added support for collective operations based on one-sided semantics - Added support for alltoall with one-sided transfer semantics - Bug fixes ##### SHARP - Added support for switch based hardware collectives (SHARP) #### NCCL - Add support for NCCL allreduce, alltoall, alltoallv, barrier, reduce, reduce scatter, bcast, allgather and allgatherv #### Tests - Updated tests to test the newly added algorithms and operations ## 0.1.0 (TBD) ### Features #### API - UCC API to support library, contexts, teams, collective operations, execution engine, memory types, and triggered operations #### Core - Added implementation for UCC abstractions - library, context, team, collective operations, execution engine, memory types, and triggered operations - Added support for memory types - CUDA, and CPU - Added support for configuring UCC library and contexts #### CL - Added support for collectives, while the source and destination is either in CPU or device (GPU) - Added support for UCC_THREAD_MULTIPLE - Added support for CUDA stream-based collectives #### TL - Added support for send/receive based collectives using UCX/UCP as a transport layer - Support for basic collectives types including barrier, alltoall, alltoallv, broadcast, allgather, allgatherv, allreduce was added in the UCP TL - Added support using NCCL as a transport layer - Support for collectives types including alltoall, alltoallv, allgather, allgatherv, allreduce, and broadcast #### Tests - Added support for unit testing (gtest) infrastructure - Added support for MPI tests ucc-1.8.0/src/0000775000175000017500000000000015211535620013245 5ustar alastairalastairucc-1.8.0/src/components/0000775000175000017500000000000015211535620015432 5ustar alastairalastairucc-1.8.0/src/components/cl/0000775000175000017500000000000015211535620016030 5ustar alastairalastairucc-1.8.0/src/components/cl/ucc_cl.h0000664000175000017500000001124715211535620017436 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #ifndef UCC_CL_H_ #define UCC_CL_H_ #include "components/base/ucc_base_iface.h" #include "ucc_cl_type.h" #include "utils/ucc_parser.h" #include "core/ucc_context.h" /** CL (collective layer) is an internal collective interface reflecting the public UCC API and extensions to support modularity, the composition of multiple collective implementations, and functionality that bridges the gap between hardware implementation of communication primitives and the programming models. The CL layer will build upon TL for the communication transport requirements. The CL can include a basic implementation, which provides minimal functionality over the TL, or can provide more optimized implementation such as hierarchical implementation that leverages multiple TL components. The different implementations of CL are realized as different CL components. The CL components are loaded dynamically, and their names should match the predefined pattern “ucc_cl_.so”. The CL that is used for a given application invocation can be selected with the UCC_CLS lib parameter. */ typedef struct ucc_cl_lib ucc_cl_lib_t; typedef struct ucc_cl_iface ucc_cl_iface_t; typedef struct ucc_cl_context ucc_cl_context_t; typedef struct ucc_cl_team ucc_cl_team_t; typedef struct ucc_tl_context ucc_tl_context_t; typedef struct ucc_cl_lib_config { ucc_base_lib_config_t super; ucc_cl_iface_t *iface; ucc_config_allow_list_t tls; } ucc_cl_lib_config_t; extern ucc_config_field_t ucc_cl_lib_config_table[]; typedef struct ucc_cl_context_config { ucc_base_ctx_config_t super; ucc_cl_lib_t *cl_lib; } ucc_cl_context_config_t; extern ucc_config_field_t ucc_cl_context_config_table[]; ucc_status_t ucc_cl_context_config_read(ucc_cl_lib_t *cl_lib, const ucc_context_config_t *config, ucc_cl_context_config_t **cl_config); ucc_status_t ucc_cl_lib_config_read(ucc_cl_iface_t *iface, const char *full_prefix, ucc_cl_lib_config_t **cl_config); typedef struct ucc_cl_iface { ucc_component_iface_t super; ucc_cl_type_t type; ucc_config_global_list_entry_t cl_lib_config; ucc_config_global_list_entry_t cl_context_config; ucc_base_lib_iface_t lib; ucc_base_context_iface_t context; ucc_base_team_iface_t team; ucc_base_coll_iface_t coll; ucc_base_coll_alg_info_t * alg_info[UCC_COLL_TYPE_NUM]; } ucc_cl_iface_t; typedef struct ucc_cl_lib { ucc_base_lib_t super; ucc_cl_iface_t *iface; ucc_config_names_list_t tls; ucc_config_names_array_t tls_forced; /*< set of TLs that are requested explicitly */ } ucc_cl_lib_t; UCC_CLASS_DECLARE(ucc_cl_lib_t, ucc_cl_iface_t *, const ucc_cl_lib_config_t *); typedef struct ucc_cl_context { ucc_base_context_t super; ucc_tl_context_t **tl_ctxs; unsigned n_tl_ctxs; } ucc_cl_context_t; UCC_CLASS_DECLARE(ucc_cl_context_t, const ucc_cl_context_config_t *, ucc_context_t *); typedef struct ucc_cl_team { ucc_base_team_t super; } ucc_cl_team_t; UCC_CLASS_DECLARE(ucc_cl_team_t, ucc_cl_context_t *, const ucc_base_team_params_t *); typedef struct ucc_cl_lib_attr { ucc_base_lib_attr_t super; ucc_config_names_array_t *tls; ucc_config_names_array_t *tls_forced; } ucc_cl_lib_attr_t; #define UCC_CL_IFACE_DECLARE(_name, _NAME) \ UCC_BASE_IFACE_DECLARE(CL_, cl_, _name, _NAME) \ __attribute__((constructor)) static void ucc_cl_ ## _name ## \ _iface_construct(void) { \ ucc_cl_ ## _name .super.type = UCC_CL_ ## _NAME; \ } \ #define UCC_CL_CTX_IFACE(_cl_ctx) \ (ucc_derived_of((_cl_ctx)->super.lib, ucc_cl_lib_t))->iface #define UCC_CL_TEAM_IFACE(_cl_team) \ (ucc_derived_of((_cl_team)->super.context->lib, ucc_cl_lib_t))->iface #define UCC_CL_TEAM_LIB(_cl_team) (_cl_team)->super.super.context->lib #define UCC_CL_TEAM_CTX(_cl_team) (_cl_team)->super.super.context #define UCC_CL_TEAM_SIZE(_cl_team) (_cl_team)->super.super.params.size #define UCC_CL_TEAM_RANK(_cl_team) (_cl_team)->super.super.params.rank #endif ucc-1.8.0/src/components/cl/doca_urom/0000775000175000017500000000000015211535620020000 5ustar alastairalastairucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom.h0000664000175000017500000000703415211535620022603 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_CL_DOCA_UROM_H_ #define UCC_CL_DOCA_UROM_H_ #include "components/cl/ucc_cl.h" #include "components/cl/ucc_cl_log.h" #include "components/tl/ucc_tl.h" #include "coll_score/ucc_coll_score.h" #include "utils/ucc_mpool.h" #include #include #include #include #include #include "cl_doca_urom_common.h" #include "cl_doca_urom_worker_ucc.h" #include #ifndef UCC_CL_DOCA_UROM_DEFAULT_SCORE #define UCC_CL_DOCA_UROM_DEFAULT_SCORE 100 #endif #define UCC_CL_DOCA_UROM_ADDR_MAX_LEN 1024 #define UCC_CL_DOCA_UROM_MAX_TEAMS 16 typedef struct ucc_cl_doca_urom_iface { ucc_cl_iface_t super; } ucc_cl_doca_urom_iface_t; // Extern iface should follow the pattern: ucc_cl_ extern ucc_cl_doca_urom_iface_t ucc_cl_doca_urom; typedef struct ucc_cl_doca_urom_lib_config { ucc_cl_lib_config_t super; } ucc_cl_doca_urom_lib_config_t; typedef struct ucc_cl_doca_urom_context_config { ucc_cl_context_config_t super; ucs_config_names_array_t plugin_envs; char *device; char *plugin_name; int doca_log_level; } ucc_cl_doca_urom_context_config_t; typedef struct ucc_cl_doca_urom_ctx { struct doca_urom_service *urom_service; struct doca_urom_worker *urom_worker; struct doca_urom_domain *urom_domain; struct doca_pe *urom_pe; const struct doca_urom_service_plugin_info *ucc_info; void *urom_worker_addr; size_t urom_worker_len; uint64_t worker_id; void *urom_ucc_context; ucc_rank_t ctx_rank; struct doca_dev *dev; } ucc_cl_doca_urom_ctx_t; typedef struct ucc_cl_doca_urom_lib { ucc_cl_lib_t super; ucc_cl_doca_urom_lib_config_t cfg; int tl_ucp_index; } ucc_cl_doca_urom_lib_t; UCC_CLASS_DECLARE(ucc_cl_doca_urom_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); typedef struct ucc_cl_doca_urom_context { ucc_cl_context_t super; ucc_mpool_t sched_mp; ucc_cl_doca_urom_ctx_t urom_ctx; ucc_cl_doca_urom_context_config_t cfg; } ucc_cl_doca_urom_context_t; UCC_CLASS_DECLARE(ucc_cl_doca_urom_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); typedef struct ucc_cl_doca_urom_team { ucc_cl_team_t super; ucc_team_h **teams; unsigned n_teams; ucc_coll_score_t *score; ucc_score_map_t *score_map; struct ucc_cl_doca_urom_result res; // used for the cookie } ucc_cl_doca_urom_team_t; UCC_CLASS_DECLARE(ucc_cl_doca_urom_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_cl_doca_urom_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); #define UCC_CL_DOCA_UROM_TEAM_CTX(_team) \ (ucc_derived_of((_team)->super.super.context, ucc_cl_doca_urom_context_t)) #endif ucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom_team.c0000664000175000017500000001264515211535620023610 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_doca_urom.h" #include "utils/ucc_malloc.h" #include "core/ucc_team.h" UCC_CLASS_INIT_FUNC(ucc_cl_doca_urom_team_t, ucc_base_context_t *cl_context, const ucc_base_team_params_t *params) { union doca_data cookie = {0}; doca_error_t result = DOCA_SUCCESS; ucc_cl_doca_urom_context_t *ctx = ucc_derived_of(cl_context, ucc_cl_doca_urom_context_t); ucc_status_t status; UCC_CLASS_CALL_SUPER_INIT(ucc_cl_team_t, &ctx->super, params); self->teams = (ucc_team_h **) ucc_malloc( sizeof(ucc_team_h *) * UCC_CL_DOCA_UROM_MAX_TEAMS); if (!self->teams) { cl_error(cl_context->lib, "failed to allocate %zd bytes for doca_urom teams", sizeof(ucc_team_h *) * UCC_CL_DOCA_UROM_MAX_TEAMS); status = UCC_ERR_NO_MEMORY; return status; } self->n_teams = 0; self->score_map = NULL; cookie.ptr = &self->res; /* Send the command to create a team on the DPU */ result = ucc_cl_doca_urom_task_team_create( ctx->urom_ctx.urom_worker, cookie, ctx->urom_ctx.ctx_rank, 0 /* start */, 1 /* stride */, params->params.oob.n_oob_eps /* size */, ctx->urom_ctx.urom_ucc_context, ucc_cl_doca_urom_team_create_finished); if (result != DOCA_SUCCESS) { cl_error(cl_context->lib, "failed to create UCC team task"); return UCC_ERR_NO_RESOURCE; } self->res.result = DOCA_ERROR_IN_PROGRESS; cl_debug(cl_context->lib, "posted cl team: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_doca_urom_team_t) { cl_debug(self->super.super.context->lib, "finalizing cl team: %p", self); } UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_doca_urom_team_t, ucc_base_team_t); UCC_CLASS_DEFINE(ucc_cl_doca_urom_team_t, ucc_cl_team_t); ucc_status_t ucc_cl_doca_urom_team_destroy(ucc_base_team_t *cl_team) { return UCC_OK; } ucc_status_t ucc_cl_doca_urom_team_create_test(ucc_base_team_t *cl_team) { ucc_cl_doca_urom_team_t *team = ucc_derived_of(cl_team, ucc_cl_doca_urom_team_t); ucc_cl_doca_urom_context_t *ctx = UCC_CL_DOCA_UROM_TEAM_CTX(team); ucc_memory_type_t mem_types[2] = {UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA}; struct ucc_cl_doca_urom_result *res = &team->res; struct ucc_cl_doca_urom_team_create_result *team_create = &res->team_create; ucc_coll_score_t *score = NULL; int mt_n = 2; ucc_status_t ucc_status; int ret; ret = doca_pe_progress(ctx->urom_ctx.urom_pe); if (ret == 0 && res->result == DOCA_ERROR_IN_PROGRESS) { return UCC_INPROGRESS; } if (res->result != DOCA_SUCCESS) { cl_error(ctx->super.super.lib, "UCC team create task failed: DOCA status %d", res->result); return UCC_ERR_NO_MESSAGE; } team->teams[team->n_teams] = team_create->team; ++team->n_teams; ucc_status = ucc_coll_score_build_default( cl_team, UCC_CL_DOCA_UROM_DEFAULT_SCORE, ucc_cl_doca_urom_coll_init, UCC_COLL_TYPE_ALLREDUCE, mem_types, mt_n, &score); if (UCC_OK != ucc_status) { return ucc_status; } ucc_status = ucc_coll_score_build_map(score, &team->score_map); if (UCC_OK != ucc_status) { cl_error(ctx->super.super.lib, "failed to build score map"); } team->score = score; ucc_coll_score_set(team->score, UCC_CL_DOCA_UROM_DEFAULT_SCORE); return UCC_OK; } ucc_status_t ucc_cl_doca_urom_team_get_scores(ucc_base_team_t *cl_team, ucc_coll_score_t **score) { ucc_cl_doca_urom_team_t *team = ucc_derived_of(cl_team, ucc_cl_doca_urom_team_t); ucc_base_context_t *ctx = UCC_CL_TEAM_CTX(team); ucc_coll_score_team_info_t team_info; ucc_status_t status; status = ucc_coll_score_dup(team->score, score); if (UCC_OK != status) { return status; } if (strlen(ctx->score_str) > 0) { team_info.alg_fn = NULL; team_info.default_score = UCC_CL_DOCA_UROM_DEFAULT_SCORE; team_info.init = NULL; team_info.num_mem_types = 0; team_info.supported_mem_types = NULL; /* all memory types supported*/ team_info.supported_colls = UCC_COLL_TYPE_ALLREDUCE; team_info.size = UCC_CL_TEAM_SIZE(team); status = ucc_coll_score_update_from_str(ctx->score_str, &team_info, &team->super.super, *score); /* If INVALID_PARAM - User provided incorrect input - try to proceed */ if ((status < 0) && (status != UCC_ERR_INVALID_PARAM) && (status != UCC_ERR_NOT_SUPPORTED)) { goto err; } } return UCC_OK; err: ucc_coll_score_free(*score); *score = NULL; return status; } ucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom_common.c0000664000175000017500000002056415211535620024151 0ustar alastairalastair/* * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES, ALL RIGHTS RESERVED. * * This software product is a proprietary product of NVIDIA CORPORATION & * AFFILIATES (the "Company") and all right, title, and interest in and to the * software product, including all associated intellectual property rights, are * and shall remain exclusively with the Company. * * This software product is governed by the End User License Agreement * provided with the software product. * */ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include #include "cl_doca_urom_common.h" DOCA_LOG_REGISTER(UCC::DOCA_CL : UROM_COMMON); doca_error_t ucc_cl_doca_urom_start_urom_service( struct doca_pe *pe, struct doca_dev *dev, uint64_t nb_workers, struct doca_urom_service **service) { enum doca_ctx_states state; struct doca_urom_service *inst; doca_error_t result, tmp_result; /* Create service context */ result = doca_urom_service_create(&inst); if (result != DOCA_SUCCESS) return result; result = doca_pe_connect_ctx(pe, doca_urom_service_as_ctx(inst)); if (result != DOCA_SUCCESS) goto service_cleanup; result = doca_urom_service_set_max_workers(inst, nb_workers); if (result != DOCA_SUCCESS) goto service_cleanup; result = doca_urom_service_set_dev(inst, dev); if (result != DOCA_SUCCESS) goto service_cleanup; result = doca_ctx_start(doca_urom_service_as_ctx(inst)); if (result != DOCA_SUCCESS) goto service_cleanup; result = doca_ctx_get_state(doca_urom_service_as_ctx(inst), &state); if (result != DOCA_SUCCESS || state != DOCA_CTX_STATE_RUNNING) goto service_stop; *service = inst; return DOCA_SUCCESS; service_stop: tmp_result = doca_ctx_stop(doca_urom_service_as_ctx(inst)); if (tmp_result != DOCA_SUCCESS) { DOCA_LOG_ERR("failed to stop UROM service"); DOCA_ERROR_PROPAGATE(result, tmp_result); } service_cleanup: tmp_result = doca_urom_service_destroy(inst); if (tmp_result != DOCA_SUCCESS) { DOCA_LOG_ERR("failed to destroy UROM service"); DOCA_ERROR_PROPAGATE(result, tmp_result); } return result; } doca_error_t ucc_cl_doca_urom_start_urom_worker( struct doca_pe *pe, struct doca_urom_service *service, uint64_t worker_id, uint32_t *gid, uint64_t nb_tasks, doca_cpu_set_t *cpuset, char **env, size_t env_count, uint64_t plugins, struct doca_urom_worker **worker) { enum doca_ctx_states state; struct doca_urom_worker *inst; doca_error_t result, tmp_result; result = doca_urom_worker_create(&inst); if (result != DOCA_SUCCESS) return result; result = doca_urom_worker_set_service(inst, service); if (result != DOCA_SUCCESS) goto worker_cleanup; result = doca_pe_connect_ctx(pe, doca_urom_worker_as_ctx(inst)); if (result != DOCA_SUCCESS) goto worker_cleanup; result = doca_urom_worker_set_id(inst, worker_id); if (result != DOCA_SUCCESS) goto worker_cleanup; if (gid != NULL) { result = doca_urom_worker_set_gid(inst, *gid); if (result != DOCA_SUCCESS) goto worker_cleanup; } if (env != NULL) { result = doca_urom_worker_set_env(inst, env, env_count); if (result != DOCA_SUCCESS) goto worker_cleanup; } result = doca_urom_worker_set_max_inflight_tasks(inst, nb_tasks); if (result != DOCA_SUCCESS) goto worker_cleanup; result = doca_urom_worker_set_plugins(inst, plugins); if (result != DOCA_SUCCESS) goto worker_cleanup; if (cpuset != NULL) { result = doca_urom_worker_set_cpuset(inst, *cpuset); if (result != DOCA_SUCCESS) goto worker_cleanup; } result = doca_ctx_start(doca_urom_worker_as_ctx(inst)); if (result != DOCA_ERROR_IN_PROGRESS) goto worker_cleanup; result = doca_ctx_get_state(doca_urom_worker_as_ctx(inst), &state); if (result != DOCA_SUCCESS) goto worker_stop; if (state != DOCA_CTX_STATE_STARTING) { result = DOCA_ERROR_BAD_STATE; goto worker_stop; } *worker = inst; return DOCA_SUCCESS; worker_stop: tmp_result = doca_ctx_stop(doca_urom_worker_as_ctx(inst)); if (tmp_result != DOCA_SUCCESS && tmp_result != DOCA_ERROR_IN_PROGRESS) { DOCA_LOG_ERR("failed to request stop UROM worker"); DOCA_ERROR_PROPAGATE(result, tmp_result); } do { doca_pe_progress(pe); doca_ctx_get_state(doca_urom_worker_as_ctx(inst), &state); } while (state != DOCA_CTX_STATE_IDLE); worker_cleanup: tmp_result = doca_urom_worker_destroy(inst); if (tmp_result != DOCA_SUCCESS) { DOCA_LOG_ERR("failed to destroy UROM worker"); DOCA_ERROR_PROPAGATE(result, tmp_result); } return result; } doca_error_t ucc_cl_doca_urom_start_urom_domain( struct doca_pe *pe, struct doca_urom_domain_oob_coll *oob, uint64_t *worker_ids, struct doca_urom_worker **workers, size_t nb_workers, struct ucc_cl_doca_urom_domain_buffer_attrs *buffers, size_t nb_buffers, struct doca_urom_domain **domain) { struct doca_urom_domain *inst; enum doca_ctx_states state; doca_error_t result, tmp_result; size_t i; result = doca_urom_domain_create(&inst); if (result != DOCA_SUCCESS) { DOCA_LOG_ERR("failed to create domain"); return result; } result = doca_pe_connect_ctx(pe, doca_urom_domain_as_ctx(inst)); if (result != DOCA_SUCCESS) goto domain_destroy; result = doca_urom_domain_set_oob(inst, oob); if (result != DOCA_SUCCESS) goto domain_destroy; result = doca_urom_domain_set_workers(inst, worker_ids, workers, nb_workers); if (result != DOCA_SUCCESS) goto domain_destroy; /* The buffers in the domain are used for gets/puts from the host without XGVMI. Also, the domain is used for the OOB exchange given to the DPU- side UCC instance */ if (nb_workers != 0 && buffers != NULL) { result = doca_urom_domain_set_buffers_count(inst, nb_buffers); if (result != DOCA_SUCCESS) goto domain_destroy; for (i = 0; i < nb_buffers; i++) { result = doca_urom_domain_add_buffer(inst, buffers[i].buffer, buffers[i].buf_len, buffers[i].memh, buffers[i].memh_len, buffers[i].mkey, buffers[i].mkey_len); if (result != DOCA_SUCCESS) goto domain_destroy; } } result = doca_ctx_start(doca_urom_domain_as_ctx(inst)); if (result != DOCA_ERROR_IN_PROGRESS) goto domain_stop; result = doca_ctx_get_state(doca_urom_domain_as_ctx(inst), &state); if (result != DOCA_SUCCESS) goto domain_stop; if (state != DOCA_CTX_STATE_STARTING) { result = DOCA_ERROR_BAD_STATE; goto domain_stop; } *domain = inst; return DOCA_SUCCESS; domain_stop: tmp_result = doca_ctx_stop(doca_urom_domain_as_ctx(inst)); if (tmp_result != DOCA_SUCCESS) { DOCA_LOG_ERR("failed to stop UROM domain"); DOCA_ERROR_PROPAGATE(result, tmp_result); } domain_destroy: tmp_result = doca_urom_domain_destroy(inst); if (tmp_result != DOCA_SUCCESS) { DOCA_LOG_ERR("failed to destroy UROM domain"); DOCA_ERROR_PROPAGATE(result, tmp_result); } return result; } doca_error_t ucc_cl_doca_urom_open_doca_device_with_ibdev_name( const uint8_t *value, size_t val_size, ucc_cl_doca_urom_tasks_check func, struct doca_dev **retval) { char buf[DOCA_DEVINFO_IBDEV_NAME_SIZE] = {}; char val_copy[DOCA_DEVINFO_IBDEV_NAME_SIZE] = {}; struct doca_devinfo **dev_list; uint32_t nb_devs; int res; size_t i; /* Set default return value */ *retval = NULL; /* Setup */ if (val_size > DOCA_DEVINFO_IBDEV_NAME_SIZE) { DOCA_LOG_ERR("Value size too large. failed to locate device"); return DOCA_ERROR_INVALID_VALUE; } memcpy(val_copy, value, val_size); res = doca_devinfo_create_list(&dev_list, &nb_devs); if (res != DOCA_SUCCESS) { DOCA_LOG_ERR("failed to load doca devices list: %s", doca_error_get_descr(res)); return res; } /* Search */ for (i = 0; i < nb_devs; i++) { res = doca_devinfo_get_ibdev_name(dev_list[i], buf, DOCA_DEVINFO_IBDEV_NAME_SIZE); if (res == DOCA_SUCCESS && strncmp(buf, val_copy, val_size) == 0) { /* If any special capabilities are needed */ if (func != NULL && func(dev_list[i]) != DOCA_SUCCESS) continue; /* if device can be opened */ res = doca_dev_open(dev_list[i], retval); if (res == DOCA_SUCCESS) { doca_devinfo_destroy_list(dev_list); return res; } } } DOCA_LOG_WARN("Matching device not found"); res = DOCA_ERROR_NOT_FOUND; doca_devinfo_destroy_list(dev_list); return res; } ucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom_coll.c0000664000175000017500000002356515211535620023616 0ustar alastairalastair/** * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_doca_urom.h" #include "cl_doca_urom_coll.h" #include "utils/ucc_coll_utils.h" #include #include static ucc_status_t ucc_cl_doca_urom_triggered_post_setup(ucc_coll_task_t *task) { return UCC_OK; } static ucc_status_t ucc_cl_doca_urom_coll_full_start(ucc_coll_task_t *task) { ucc_cl_doca_urom_team_t *cl_team = ucc_derived_of(task->team, ucc_cl_doca_urom_team_t); ucc_cl_doca_urom_context_t *ctx = UCC_CL_DOCA_UROM_TEAM_CTX(cl_team); ucc_cl_doca_urom_lib_t *cl_lib = ucc_derived_of(ctx->super.super.lib, ucc_cl_doca_urom_lib_t); ucc_coll_args_t *coll_args = &task->bargs.args; union doca_data cookie = {0}; int use_xgvmi = 0; int in_place = UCC_IS_INPLACE(*coll_args); ucc_rank_t rank = UCC_CL_TEAM_RANK(cl_team); ucc_cl_doca_urom_schedule_t *schedule = ucc_derived_of(task, ucc_cl_doca_urom_schedule_t); struct export_buf *src_ebuf = &schedule->src_ebuf; struct export_buf *dst_ebuf = &schedule->dst_ebuf; doca_error_t result; ucc_worker_key_buf keys; cookie.ptr = &schedule->res; switch (coll_args->coll_type) { case UCC_COLL_TYPE_ALLREDUCE: { if (!in_place) { keys.src_len = src_ebuf->packed_memh_len; memcpy(keys.rkeys, src_ebuf->packed_memh, keys.src_len); } else { keys.src_len = 0; } keys.dst_len = dst_ebuf->packed_memh_len; memcpy(keys.rkeys + keys.src_len, dst_ebuf->packed_memh, keys.dst_len); use_xgvmi = 1; } break; default: { cl_error(&cl_lib->super, "coll_type %s is not supported", ucc_coll_type_str(coll_args->coll_type)); return UCC_ERR_NOT_IMPLEMENTED; } } coll_args->mask |= UCC_COLL_ARGS_FIELD_GLOBAL_WORK_BUFFER; // Submit the offload coll cmd to the DPU plugin result = ucc_cl_doca_urom_task_collective(ctx->urom_ctx.urom_worker, cookie, rank, coll_args, cl_team->teams[0], use_xgvmi, &keys, sizeof(ucc_worker_key_buf), 0, ucc_cl_doca_urom_collective_finished); if (result != DOCA_SUCCESS) { cl_error(&cl_lib->super, "failed to create UCC collective task"); } task->status = UCC_INPROGRESS; cl_debug(&cl_lib->super, "pushed the collective to urom"); return ucc_progress_queue_enqueue(ctx->super.super.ucc_context->pq, task); } static ucc_status_t ucc_cl_doca_urom_coll_full_finalize(ucc_coll_task_t *task) { ucc_cl_doca_urom_schedule_t *schedule = ucc_derived_of(task, ucc_cl_doca_urom_schedule_t); ucc_cl_doca_urom_team_t *cl_team = ucc_derived_of(task->team, ucc_cl_doca_urom_team_t); ucc_cl_doca_urom_context_t *ctx = UCC_CL_DOCA_UROM_TEAM_CTX(cl_team); ucc_cl_doca_urom_lib_t *cl_lib = ucc_derived_of(ctx->super.super.lib, ucc_cl_doca_urom_lib_t); int ucp_index = cl_lib->tl_ucp_index; ucc_tl_ucp_context_t *tl_ctx = ucc_derived_of( ctx->super.tl_ctxs[ucp_index], ucc_tl_ucp_context_t); struct export_buf *src_ebuf = &schedule->src_ebuf; struct export_buf *dst_ebuf = &schedule->dst_ebuf; ucc_status_t status; // Deregister buffers and free the schedule if (src_ebuf->memh) { ucp_mem_unmap(tl_ctx->worker.ucp_context, src_ebuf->memh); } ucp_mem_unmap(tl_ctx->worker.ucp_context, dst_ebuf->memh); status = ucc_schedule_finalize(task); ucc_cl_doca_urom_put_schedule(&schedule->super.super); return status; } static void ucc_cl_doca_urom_coll_full_progress(ucc_coll_task_t *ctask) { ucc_cl_doca_urom_team_t *cl_team = ucc_derived_of(ctask->team, ucc_cl_doca_urom_team_t); ucc_cl_doca_urom_context_t *ctx = UCC_CL_DOCA_UROM_TEAM_CTX(cl_team); ucc_cl_doca_urom_lib_t *cl_lib = ucc_derived_of( ctx->super.super.lib, ucc_cl_doca_urom_lib_t); ucc_cl_doca_urom_schedule_t *schedule = ucc_derived_of(ctask, ucc_cl_doca_urom_schedule_t); int ucp_index = cl_lib->tl_ucp_index; ucc_tl_ucp_context_t *tl_ctx = ucc_derived_of( ctx->super.tl_ctxs[ucp_index], ucc_tl_ucp_context_t); struct ucc_cl_doca_urom_result *res = &schedule->res; int ret; if (res == NULL) { cl_error(cl_lib, "error in UROM"); ctask->status = UCC_ERR_NO_MESSAGE; return; } ucp_worker_progress(tl_ctx->worker.ucp_worker); // Poll the DOCA PE for completions ret = doca_pe_progress(ctx->urom_ctx.urom_pe); if (ret == 0 && res->result == DOCA_SUCCESS) { ctask->status = UCC_INPROGRESS; return; } if (res->result != DOCA_SUCCESS) { cl_error(&cl_lib->super, "error in DOCA_UROM, UCC collective task failed"); } ctask->status = res->collective.status; cl_debug(&cl_lib->super, "completed the collective from urom"); } ucc_status_t ucc_cl_doca_urom_coll_full_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_doca_urom_team_t *cl_team = ucc_derived_of(team, ucc_cl_doca_urom_team_t); ucc_cl_doca_urom_context_t *ctx = UCC_CL_DOCA_UROM_TEAM_CTX(cl_team); ucc_cl_doca_urom_lib_t *cl_lib = ucc_derived_of(ctx->super.super.lib, ucc_cl_doca_urom_lib_t); int ucp_idx = cl_lib->tl_ucp_index; ucc_tl_ucp_context_t *tl_ctx = ucc_derived_of( ctx->super.tl_ctxs[ucp_idx], ucc_tl_ucp_context_t); int in_place = UCC_IS_INPLACE(coll_args->args); struct export_buf *src_ebuf; struct export_buf *dst_ebuf; ucc_status_t status; ucc_cl_doca_urom_schedule_t *cl_schedule; ucc_schedule_t *schedule; // Allocate and initialize schedule cl_schedule = ucc_cl_doca_urom_get_schedule(cl_team); if (ucc_unlikely(!cl_schedule)) { return UCC_ERR_NO_MEMORY; } schedule = &cl_schedule->super.super; status = ucc_schedule_init(schedule, coll_args, team); if (UCC_OK != status) { ucc_cl_doca_urom_put_schedule(schedule); return status; } schedule->super.post = ucc_cl_doca_urom_coll_full_start; schedule->super.progress = ucc_cl_doca_urom_coll_full_progress; schedule->super.finalize = ucc_cl_doca_urom_coll_full_finalize; schedule->super.triggered_post = ucc_triggered_post; schedule->super.triggered_post_setup = ucc_cl_doca_urom_triggered_post_setup; *task = &schedule->super; src_ebuf = &cl_schedule->src_ebuf; dst_ebuf = &cl_schedule->dst_ebuf; src_ebuf->memh = NULL; dst_ebuf->memh = NULL; // Register the src buf to get the exported memh if (!in_place) { ucc_cl_doca_urom_buffer_export_ucc( tl_ctx->worker.ucp_context, coll_args->args.src.info.buffer, coll_args->args.src.info.count * ucc_dt_size(coll_args->args.src.info.datatype), src_ebuf); } // Register the dst buf to get the exported memh ucc_cl_doca_urom_buffer_export_ucc( tl_ctx->worker.ucp_context, coll_args->args.dst.info.buffer, coll_args->args.dst.info.count * ucc_dt_size(coll_args->args.dst.info.datatype), dst_ebuf); cl_debug(cl_lib, "cl doca urom coll initialized"); return UCC_OK; } ucc_status_t ucc_cl_doca_urom_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_doca_urom_team_t *cl_team = ucc_derived_of(team, ucc_cl_doca_urom_team_t); ucc_cl_doca_urom_context_t *ctx = UCC_CL_DOCA_UROM_TEAM_CTX(cl_team); ucc_cl_doca_urom_lib_t *doca_urom_lib = ucc_derived_of( ctx->super.super.lib, ucc_cl_doca_urom_lib_t); switch (coll_args->args.coll_type) { case UCC_COLL_TYPE_ALLREDUCE: case UCC_COLL_TYPE_ALLGATHER: case UCC_COLL_TYPE_ALLTOALL: return ucc_cl_doca_urom_coll_full_init(coll_args, team, task); default: cl_error(doca_urom_lib, "coll_type %s is not supported", ucc_coll_type_str(coll_args->args.coll_type)); } return UCC_OK; } ucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom_common.h0000664000175000017500000000733015211535620024152 0ustar alastairalastair/* * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES, ALL RIGHTS RESERVED. * * This software product is a proprietary product of NVIDIA CORPORATION & * AFFILIATES (the "Company") and all right, title, and interest in and to the * software product, including all associated intellectual property rights, are * and shall remain exclusively with the Company. * * This software product is governed by the End User License Agreement * provided with the software product. * */ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #ifndef UCC_CL_DOCA_UROM_COMMON_H_ #define UCC_CL_DOCA_UROM_COMMON_H_ #include #include #include #include /* Function to check if a given device is capable of executing some task */ typedef doca_error_t (*ucc_cl_doca_urom_tasks_check)(struct doca_devinfo *); /* * Struct contains domain shared buffer details */ struct ucc_cl_doca_urom_domain_buffer_attrs { void *buffer; /* Buffer address */ size_t buf_len; /* Buffer length */ void *memh; /* Buffer packed memory handle */ size_t memh_len; /* Buffer packed memory handle length */ void *mkey; /* Buffer packed memory key */ size_t mkey_len; /* Buffer packed memory key length*/ }; /* * Open a DOCA device according to a given IB device name * * @value [in]: IB device name * @val_size [in]: input length, in bytes * @func [in]: pointer to a function that checks if the device have some task capabilities (Ignored if set to NULL) * @retval [out]: pointer to doca_dev struct, NULL if not found * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_open_doca_device_with_ibdev_name( const uint8_t *value, size_t val_size, ucc_cl_doca_urom_tasks_check func, struct doca_dev **retval); /* * Start UROM service context * * @pe [in]: Progress engine * @dev [in]: service DOCA device * @nb_workers [in]: number of workers * @service [out]: service context * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_start_urom_service(struct doca_pe *pe, struct doca_dev *dev, uint64_t nb_workers, struct doca_urom_service **service); /* * Start UROM worker context * * @pe [in]: Progress engine * @service [in]: service context * @worker_id [in]: Worker id * @gid [in]: worker group id (optional attribute) * @nb_tasks [in]: number of tasks * @cpuset [in]: worker CPU affinity to set * @env [in]: worker environment variables array * @env_count [in]: worker environment variables array size * @plugins [in]: worker plugins * @worker [out]: set worker context * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_start_urom_worker(struct doca_pe *pe, struct doca_urom_service *service, uint64_t worker_id, uint32_t *gid, uint64_t nb_tasks, doca_cpu_set_t *cpuset, char **env, size_t env_count, uint64_t plugins, struct doca_urom_worker **worker); /* * Start UROM domain context * * @pe [in]: Progress engine * @oob [in]: OOB allgather operations * @worker_ids [in]: workers ids participate in domain * @workers [in]: workers participate in domain * @nb_workers [in]: number of workers in domain * @buffers [in]: shared buffers * @nb_buffers [out]: number of shared buffers * @domain [out]: domain context * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_start_urom_domain(struct doca_pe *pe, struct doca_urom_domain_oob_coll *oob, uint64_t *worker_ids, struct doca_urom_worker **workers, size_t nb_workers, struct ucc_cl_doca_urom_domain_buffer_attrs *buffers, size_t nb_buffers, struct doca_urom_domain **domain); #endif /* UCC_CL_DOCA_UROM_COMMON_H_ */ ucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom_worker_ucc.h0000664000175000017500000002523315211535620025027 0ustar alastairalastair/* * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES, ALL RIGHTS RESERVED. * * This software product is a proprietary product of NVIDIA CORPORATION & * AFFILIATES (the "Company") and all right, title, and interest in and to the * software product, including all associated intellectual property rights, are * and shall remain exclusively with the Company. * * This software product is governed by the End User License Agreement * provided with the software product. * */ #ifndef UCC_CL_DOCA_UROM_WORKER_UCC_H_ #define UCC_CL_DOCA_UROM_WORKER_UCC_H_ #include #include #include #include #include "urom_ucc.h" struct export_buf { ucp_context_h ucp_context; ucp_mem_h memh; void *packed_memh; size_t packed_memh_len; void *packed_key; size_t packed_key_len; uint64_t memh_id; }; /* UCC context create result */ struct ucc_cl_doca_urom_context_create_result { void *context; /* Pointer to UCC context */ }; /* UCC team create result */ struct ucc_cl_doca_urom_team_create_result { void *team; /* Pointer to UCC team */ }; /* UCC collective result */ struct ucc_cl_doca_urom_collective_result { ucc_status_t status; /* UCC collective status */ }; /* UCC passive data channel result */ struct ucc_cl_doca_urom_pass_dc_result { ucc_status_t status; /* UCC data channel status */ }; /* UCC task result structure */ struct ucc_cl_doca_urom_result { doca_error_t result; /* Task result */ uint64_t dpu_worker_id; /* DPU worker id */ union { struct ucc_cl_doca_urom_context_create_result context_create; /* Context create result */ struct ucc_cl_doca_urom_team_create_result team_create; /* Team create result */ struct ucc_cl_doca_urom_collective_result collective; /* Collective result */ struct ucc_cl_doca_urom_pass_dc_result pass_dc; /* Passive data channel result */ }; }; void ucc_cl_doca_urom_collective_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, ucc_status_t status); ucc_status_t ucc_cl_doca_urom_buffer_export_ucc( ucp_context_h ucp_context, void *buf, size_t len, struct export_buf *ebuf); /* * UCC team create callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id * @team [in]: pointer to UCC team */ void ucc_cl_doca_urom_team_create_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, void *team); /* * UCC lib create callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id */ void ucc_cl_doca_urom_lib_create_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id); /* * UCC passive data channel callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id * @status [in]: channel creation status */ void ucc_cl_doca_urom_pss_dc_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, ucc_status_t status); /* * UCC lib destroy callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id */ void ucc_cl_doca_urom_lib_destroy_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id); /* * UCC context create callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id * @context [in]: pointer to UCC context */ void ucc_cl_doca_urom_ctx_create_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, void *context); /* * UCC lib create task callback function, will be called once the task is finished * * @result [in]: task status * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id */ typedef void (*ucc_cl_doca_urom_lib_create_finished_cb)( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id); /* * UCC lib destroy task callback function, will be called once the task is finished * * @result [in]: task status * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id */ typedef void (*ucc_cl_doca_urom_lib_destroy_finished_cb)( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id); /* * UCC context create task callback function, will be called once the task is finished * * @result [in]: task status * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @context [in]: pointer to UCC context */ typedef void (*ucc_cl_doca_urom_ctx_create_finished_cb)( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, void *context); /* * UCC context destroy task callback function, will be called once the task is finished * * @result [in]: task status * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id */ typedef void (*ucc_cl_doca_urom_ctx_destroy_finished_cb)( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id); /* * UCC team create task callback function, will be called once the task is finished * * @result [in]: task status * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @team [in]: pointer to UCC team */ typedef void (*ucc_cl_doca_urom_team_create_finished_cb)( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, void *team); /* * UCC collective task callback function, will be called once the task is finished * * @result [in]: task status * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @status [in]: UCC status */ typedef void (*ucc_cl_doca_urom_collective_finished_cb)( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, ucc_status_t status); /* * UCC passive data channel task callback function, will be called once the task is finished * * @result [in]: task status * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @status [in]: UCC status */ typedef void (*ucc_cl_doca_urom_pd_channel_finished_cb)( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, ucc_status_t status); /* * Create UCC library task * * @worker_ctx [in]: DOCA UROM worker context * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @params [in]: UCC team parameters * @cb [in]: program callback to call once the task is finished * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_task_lib_create( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, void *params, ucc_cl_doca_urom_lib_create_finished_cb cb); /* * Create UCC library destroy task * * @worker_ctx [in]: DOCA UROM worker context * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @cb [in]: program callback to call once the task is finished * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_task_lib_destroy( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, ucc_cl_doca_urom_lib_destroy_finished_cb cb); /* * Create UCC context task * * @worker_ctx [in]: DOCA UROM worker context * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @start [in]: the started index * @array [in]: array of indexes, set stride to <= 0 if array is used * @stride [in]: number of strides * @size [in]: collective context world size * @base_va [in]: shared buffer address * @len [in]: buffer length * @cb [in]: program callback to call once the task is finished * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_task_ctx_create( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, int64_t start, int64_t *array, int64_t stride, int64_t size, void *base_va, uint64_t len, ucc_cl_doca_urom_ctx_create_finished_cb cb); /* * Create UCC context destroy task * * @worker_ctx [in]: DOCA UROM worker context * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @context [in]: pointer of UCC context * @cb [in]: program callback to call once the task is finished * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_task_ctx_destroy( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, void *context, ucc_cl_doca_urom_ctx_destroy_finished_cb cb); /* * Create UCC team task * * @worker_ctx [in]: DOCA UROM worker context * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @start [in]: team start index * @stride [in]: number of strides * @size [in]: stride size * @context [in]: UCC context * @cb [in]: program callback to call once the task is finished * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_task_team_create( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, int64_t start, int64_t stride, int64_t size, void *context, ucc_cl_doca_urom_team_create_finished_cb cb); /* * Create UCC collective task * * @worker_ctx [in]: DOCA UROM worker context * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @coll_args [in]: collective arguments * @team [in]: UCC team * @use_xgvmi [in]: if operation uses XGVMI * @work_buffer [in]: work buffer * @work_buffer_size [in]: buffer size * @team_size [in]: team size * @cb [in]: program callback to call once the task is finished * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_task_collective( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, void *coll_args, void *team, int use_xgvmi, void *work_buffer, size_t work_buffer_size, size_t team_size, ucc_cl_doca_urom_collective_finished_cb cb); /* * Create UCC passive data channel task * * @worker_ctx [in]: DOCA UROM worker context * @cookie [in]: user cookie * @dpu_worker_id [in]: UCC DPU worker id * @ucp_addr [in]: UCP worker address on host * @addr_len [in]: UCP worker address length * @cb [in]: program callback to call once the task is finished * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_task_pd_channel( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, void *ucp_addr, size_t addr_len, ucc_cl_doca_urom_pd_channel_finished_cb cb); /* * This method inits UCC plugin. * * @plugin_id [in]: UROM plugin ID * @version [in]: plugin version on DPU side * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ doca_error_t ucc_cl_doca_urom_save_plugin_id( uint64_t plugin_id, uint64_t version); #endif /* UCC_CL_DOCA_UROM_WORKER_UCC_H_ */ ucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom_context.c0000664000175000017500000005201115211535620024335 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_doca_urom.h" #include "cl_doca_urom_coll.h" #include "utils/ucc_malloc.h" static doca_error_t ucc_status_to_doca_error(ucc_status_t status) { doca_error_t doca_err = DOCA_ERROR_UNKNOWN; switch (status) { case UCC_OK: doca_err = DOCA_SUCCESS; break; case UCC_INPROGRESS: case UCC_OPERATION_INITIALIZED: doca_err = DOCA_ERROR_IN_PROGRESS; break; case UCC_ERR_NOT_SUPPORTED: case UCC_ERR_NOT_IMPLEMENTED: doca_err = DOCA_ERROR_NOT_SUPPORTED; break; case UCC_ERR_INVALID_PARAM: doca_err = DOCA_ERROR_INVALID_VALUE; break; case UCC_ERR_NO_MEMORY: doca_err = DOCA_ERROR_NO_MEMORY; break; case UCC_ERR_NO_RESOURCE: doca_err = DOCA_ERROR_FULL; break; case UCC_ERR_NO_MESSAGE: case UCC_ERR_LAST: doca_err = DOCA_ERROR_UNKNOWN; break; case UCC_ERR_NOT_FOUND: doca_err = DOCA_ERROR_NOT_FOUND; break; case UCC_ERR_TIMED_OUT: doca_err = DOCA_ERROR_TIME_OUT; break; } return doca_err; } // Convert the ucc oob allgather test to work with doca_error_t. // The problem this solves is that DOCA_ERROR_IN_PROGRESS is numerically // equivalent to 26 while UCC_INPROGRESS is equal to 1 ucc_status_t (*params_oob_allgather_test)(void *req); static doca_error_t oob_allgather_test_docafied(void *req) { return ucc_status_to_doca_error(params_oob_allgather_test(req)); } ucc_status_t (*params_oob_allgather_free)(void *req); static doca_error_t oob_allgather_free_docafied(void *req) { return ucc_status_to_doca_error(params_oob_allgather_free(req)); } ucc_status_t (*params_oob_allgather)(void *, void *, size_t, void *, void **); static doca_error_t oob_allgather_docafied(void * s, void * r, size_t z, void * i, void **req_p) { return ucc_status_to_doca_error(params_oob_allgather(s,r,z,i,req_p)); } UCC_CLASS_INIT_FUNC(ucc_cl_doca_urom_context_t, const ucc_base_context_params_t *params, const ucc_base_config_t *config) { struct ucc_cl_doca_urom_domain_buffer_attrs buf_attrs = {0}; struct doca_urom_domain_oob_coll oob_coll = {0}; doca_error_t tmp_result = DOCA_SUCCESS; union doca_data cookie = {0}; struct ucc_cl_doca_urom_result res = {0}; doca_error_t result = DOCA_SUCCESS; size_t length = 4096; int ucp_index = -1; int num_envs = 0; char **envs = NULL; size_t plugins_count = 0; struct doca_log_backend *sdk_log = NULL; const ucc_cl_doca_urom_context_config_t *cl_config = ucc_derived_of(config, ucc_cl_doca_urom_context_config_t); ucc_cl_doca_urom_lib_t *doca_urom_lib = ucc_derived_of(cl_config->super.cl_lib, ucc_cl_doca_urom_lib_t); ucc_config_names_array_t *tls = &cl_config->super.cl_lib->tls.array; ucc_lib_params_t lib_params = { .mask = UCC_LIB_PARAM_FIELD_THREAD_MODE, .thread_mode = UCC_THREAD_SINGLE, }; const struct doca_urom_service_plugin_info *plugins; ucc_tl_ucp_context_t *tl_ctx; enum doca_ctx_states state; struct export_buf ebuf; ucc_status_t status; ucs_status_t ucs_status; ucc_rank_t rank; uint64_t rank_u64; size_t i; void *buffer; int ret; char *plugin_name; char *device; UCC_CLASS_CALL_SUPER_INIT(ucc_cl_context_t, &cl_config->super, params->context); memcpy(&self->cfg, cl_config, sizeof(*cl_config)); if (tls->count == 1 && !strcmp(tls->names[0], "all")) { tls = ¶ms->context->all_tls; } self->super.tl_ctxs = ucc_malloc(sizeof(ucc_tl_context_t*) * tls->count, "cl_doca_urom_tl_ctxs"); if (!self->super.tl_ctxs) { cl_error(cl_config->super.cl_lib, "failed to allocate %zd bytes for tl_ctxs", sizeof(ucc_tl_context_t**) * tls->count); return UCC_ERR_NO_MEMORY; } self->super.n_tl_ctxs = 0; for (i = 0; i < tls->count; i++) { ucc_debug("TL NAME[%zu]: %s", i, tls->names[i]); if (strcmp(tls->names[i], "ucp") == 0) { status = ucc_tl_context_get(params->context, tls->names[i], &self->super.tl_ctxs[self->super.n_tl_ctxs]); if (UCC_OK != status) { cl_error(cl_config->super.cl_lib, "TL ucp not available"); } ucp_index = self->super.n_tl_ctxs; doca_urom_lib->tl_ucp_index = ucp_index; self->super.n_tl_ctxs++; } } if (0 == self->super.n_tl_ctxs) { cl_error(cl_config->super.cl_lib, "no TL contexts are available"); ucc_free(self->super.tl_ctxs); self->super.tl_ctxs = NULL; return UCC_ERR_NOT_FOUND; } ucc_assert(ucp_index != -1); tl_ctx = ucc_derived_of(self->super.tl_ctxs[ucp_index], ucc_tl_ucp_context_t); memset(&self->urom_ctx, 0, sizeof(ucc_cl_doca_urom_ctx_t)); ucc_assert(params->params.mask | UCC_CONTEXT_PARAM_FIELD_OOB); self->urom_ctx.ctx_rank = params->params.oob.oob_ep; rank = self->urom_ctx.ctx_rank; if (self->cfg.plugin_envs.count > 0) { num_envs = self->cfg.plugin_envs.count; envs = self->cfg.plugin_envs.names; } plugin_name = self->cfg.plugin_name; device = self->cfg.device; result = doca_log_backend_create_with_file_sdk(stderr, &sdk_log); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to create DOCA log backend"); return UCC_ERR_NO_RESOURCE; } result = doca_log_backend_set_sdk_level(sdk_log, cl_config->doca_log_level); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to set backend sdk level"); return UCC_ERR_NO_RESOURCE; } result = ucc_cl_doca_urom_open_doca_device_with_ibdev_name( (uint8_t *)device, strlen(device), NULL, &self->urom_ctx.dev); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to open device %s", device); return UCC_ERR_NO_RESOURCE; } result = doca_pe_create(&self->urom_ctx.urom_pe); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to create DOCA PE"); goto dev_close; } result = ucc_cl_doca_urom_start_urom_service( self->urom_ctx.urom_pe, self->urom_ctx.dev, 2, &self->urom_ctx.urom_service); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to create UROM service context"); goto pe_destroy; } result = doca_urom_service_get_plugins_list(self->urom_ctx.urom_service, &plugins, &plugins_count); if (result != DOCA_SUCCESS || plugins_count == 0) { cl_error(cl_config->super.cl_lib, "failed to get UROM plugins list. plugins_count: %ld", plugins_count); goto service_stop; } for (i = 0; i < plugins_count; i++) { if (strcmp(plugin_name, plugins[i].plugin_name) == 0) { self->urom_ctx.ucc_info = &plugins[i]; break; } } if (self->urom_ctx.ucc_info == NULL) { cl_error(cl_config->super.cl_lib, "failed to match UCC plugin"); result = DOCA_ERROR_INVALID_VALUE; goto service_stop; } /* Each command requires a plugin id--save it in the worker */ result = ucc_cl_doca_urom_save_plugin_id(self->urom_ctx.ucc_info->id, self->urom_ctx.ucc_info->version); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to init UCC worker plugin"); goto service_stop; } self->urom_ctx.urom_worker_addr = ucc_calloc(1, UCC_CL_DOCA_UROM_ADDR_MAX_LEN, "doca_urom worker addr"); if (!self->urom_ctx.urom_worker_addr) { cl_error(cl_config->super.cl_lib, "failed to allocate %d bytes", UCC_CL_DOCA_UROM_ADDR_MAX_LEN); return UCC_ERR_NO_MEMORY; } /* Create and start worker context */ result = ucc_cl_doca_urom_start_urom_worker(self->urom_ctx.urom_pe, self->urom_ctx.urom_service, rank, NULL, 16, NULL, envs, num_envs, self->urom_ctx.ucc_info->id, &self->urom_ctx.urom_worker); if (result != DOCA_SUCCESS) cl_error(cl_config->super.cl_lib, "failed to start urom worker"); /* Loop till worker state changes to running */ do { doca_pe_progress(self->urom_ctx.urom_pe); result = doca_ctx_get_state( doca_urom_worker_as_ctx(self->urom_ctx.urom_worker), &state); } while (state == DOCA_CTX_STATE_STARTING && result == DOCA_SUCCESS); if (state != DOCA_CTX_STATE_RUNNING || result != DOCA_SUCCESS) { goto worker_cleanup; } /* Start the UROM domain */ buffer = ucc_calloc(1, length, "doca_urom domain buffer"); if (buffer == NULL) { cl_error(cl_config->super.cl_lib, "failed to allocate urom domain buffer"); result = DOCA_ERROR_NO_MEMORY; goto worker_cleanup; } params_oob_allgather = params->params.oob.allgather; oob_coll.allgather = oob_allgather_docafied; params_oob_allgather_test = params->params.oob.req_test; oob_coll.req_test = oob_allgather_test_docafied; params_oob_allgather_free = params->params.oob.req_free; oob_coll.req_free = oob_allgather_free_docafied; oob_coll.coll_info = params->params.oob.coll_info; oob_coll.n_oob_indexes = params->params.oob.n_oob_eps; oob_coll.oob_index = rank; ucs_status = ucp_worker_get_address(tl_ctx->worker.ucp_worker, &tl_ctx->worker.worker_address, &tl_ctx->worker.ucp_addrlen); if (ucs_status != UCS_OK) { cl_error(cl_config->super.cl_lib, "failed to get ucp worker address"); goto worker_cleanup; } result = (doca_error_t) ucc_cl_doca_urom_buffer_export_ucc( tl_ctx->worker.ucp_context, buffer, length, &ebuf); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to export buffer"); goto worker_cleanup; } /* The buffers in the domain are used for gets/puts from the host without XGVMI. Also, the domain is used for the OOB exchange given to the DPU- side UCC instance */ buf_attrs.buffer = buffer; buf_attrs.buf_len = length; buf_attrs.memh = ebuf.packed_memh; buf_attrs.memh_len = ebuf.packed_memh_len; buf_attrs.mkey = ebuf.packed_key; buf_attrs.mkey_len = ebuf.packed_key_len; /* Create domain context */ rank_u64 = (uint64_t)rank; result = ucc_cl_doca_urom_start_urom_domain( self->urom_ctx.urom_pe, &oob_coll, &rank_u64, &self->urom_ctx.urom_worker, 1, &buf_attrs, 1, &self->urom_ctx.urom_domain); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to start domain"); goto worker_unmap; } /* Loop till domain state changes to running */ do { doca_pe_progress(self->urom_ctx.urom_pe); result = doca_ctx_get_state( doca_urom_domain_as_ctx( self->urom_ctx.urom_domain), &state); } while (state == DOCA_CTX_STATE_STARTING && result == DOCA_SUCCESS); if (state != DOCA_CTX_STATE_RUNNING || result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to start domain"); result = DOCA_ERROR_BAD_STATE; goto worker_unmap; } /* Create lib */ cookie.ptr = &res; result = ucc_cl_doca_urom_task_lib_create( self->urom_ctx.urom_worker, cookie, rank, &lib_params, ucc_cl_doca_urom_lib_create_finished); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to create lib creation task"); goto domain_stop; } do { ret = doca_pe_progress(self->urom_ctx.urom_pe); } while (ret == 0 && res.result == DOCA_SUCCESS); if (res.result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to finish lib create task"); result = res.result; goto domain_stop; } cl_debug(cl_config->super.cl_lib, "UCC lib create is done"); cl_debug(cl_config->super.cl_lib, "Creating pd channel"); result = ucc_cl_doca_urom_task_pd_channel(self->urom_ctx.urom_worker, cookie, rank, tl_ctx->worker.worker_address, tl_ctx->worker.ucp_addrlen, ucc_cl_doca_urom_pss_dc_finished); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to create data channel task"); goto lib_destroy; } do { ret = doca_pe_progress(self->urom_ctx.urom_pe); } while (ret == 0 && res.result == DOCA_SUCCESS); if (res.result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "passive data channel task failed"); result = res.result; goto lib_destroy; } cl_debug(cl_config->super.cl_lib, "passive data channel is done"); cl_debug(cl_config->super.cl_lib, "creating task ctx"); result = ucc_cl_doca_urom_task_ctx_create(self->urom_ctx.urom_worker, cookie, rank, 0, NULL, 1, params->params.oob.n_oob_eps, 0x0, length, ucc_cl_doca_urom_ctx_create_finished); if (result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to create UCC context task"); goto lib_destroy; } do { ret = doca_pe_progress(self->urom_ctx.urom_pe); } while (ret == 0 && res.result == DOCA_SUCCESS); if (res.result != DOCA_SUCCESS || res.context_create.context == NULL) { cl_error(cl_config->super.cl_lib, "UCC context create task failed"); result = res.result; goto lib_destroy; } cl_debug(cl_config->super.cl_lib, "UCC context create is done, ucc_context: %p", res.context_create.context); self->urom_ctx.urom_ucc_context = res.context_create.context; status = ucc_mpool_init(&self->sched_mp, 0, sizeof(ucc_cl_doca_urom_schedule_t), 0, UCC_CACHE_LINE_SIZE, 2, UINT_MAX, &ucc_coll_task_mpool_ops, params->thread_mode, "cl_doca_urom_sched_mp"); if (UCC_OK != status) { cl_error(cl_config->super.cl_lib, "failed to initialize cl_doca_urom_sched mpool"); goto lib_destroy; } cl_debug(cl_config->super.cl_lib, "initialized cl context: %p", self); return UCC_OK; lib_destroy: result = ucc_cl_doca_urom_task_lib_destroy(self->urom_ctx.urom_worker, cookie, rank, ucc_cl_doca_urom_lib_destroy_finished); if (result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "failed to create UCC lib destroy task"); } do { ret = doca_pe_progress(self->urom_ctx.urom_pe); } while (ret == 0 && res.result == DOCA_SUCCESS); if (res.result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "UCC lib destroy failed"); result = res.result; } domain_stop: result = doca_ctx_stop( doca_urom_domain_as_ctx(self->urom_ctx.urom_domain)); if (result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "failed to stop UROM domain"); } result = doca_urom_domain_destroy(self->urom_ctx.urom_domain); if (result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "failed to destroy UROM domain"); } worker_unmap: ucs_status = ucp_mem_unmap(tl_ctx->worker.ucp_context, ebuf.memh); if (ucs_status != UCS_OK) { cl_error(cl_config->super.cl_lib, "failed to unmap memh"); } free(buffer); worker_cleanup: tmp_result = doca_urom_worker_destroy(self->urom_ctx.urom_worker); if (tmp_result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to destroy UROM worker"); } service_stop: tmp_result = doca_ctx_stop( doca_urom_service_as_ctx(self->urom_ctx.urom_service)); if (tmp_result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to stop UROM service"); } tmp_result = doca_urom_service_destroy(self->urom_ctx.urom_service); if (tmp_result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to destroy UROM service"); } pe_destroy: tmp_result = doca_pe_destroy(self->urom_ctx.urom_pe); if (tmp_result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to destroy PE"); } dev_close: tmp_result = doca_dev_close(self->urom_ctx.dev); if (tmp_result != DOCA_SUCCESS) { cl_error(cl_config->super.cl_lib, "failed to close device"); } return UCC_ERR_NO_MESSAGE; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_doca_urom_context_t) { struct ucc_cl_doca_urom_result res = {0}; union doca_data cookie = {0}; doca_error_t result = DOCA_SUCCESS; ucc_rank_t rank; int i, ret; rank = self->urom_ctx.ctx_rank; cookie.ptr = &res; result = ucc_cl_doca_urom_task_lib_destroy(self->urom_ctx.urom_worker, cookie, rank, ucc_cl_doca_urom_lib_destroy_finished); if (result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "failed to create UCC lib destroy task"); } do { ret = doca_pe_progress(self->urom_ctx.urom_pe); } while (ret == 0 && res.result == DOCA_SUCCESS); if (res.result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "UCC lib destroy failed"); result = res.result; } result = doca_ctx_stop( doca_urom_domain_as_ctx(self->urom_ctx.urom_domain)); if (result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "failed to stop UROM domain"); } result = doca_urom_domain_destroy(self->urom_ctx.urom_domain); if (result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "failed to destroy UROM domain"); } result = doca_ctx_stop( doca_urom_service_as_ctx(self->urom_ctx.urom_service)); if (result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "failed to stop UROM service"); } result = doca_urom_service_destroy(self->urom_ctx.urom_service); if (result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "failed to destroy UROM service"); } result = doca_pe_destroy(self->urom_ctx.urom_pe); if (result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "failed to destroy PE"); } result = doca_dev_close(self->urom_ctx.dev); if (result != DOCA_SUCCESS) { cl_error(self->super.super.lib, "failed to close device"); } cl_debug(self->super.super.lib, "finalizing cl context: %p", self); for (i = 0; i < self->super.n_tl_ctxs; i++) { ucc_tl_context_put(self->super.tl_ctxs[i]); } ucc_free(self->super.tl_ctxs); } UCC_CLASS_DEFINE(ucc_cl_doca_urom_context_t, ucc_cl_context_t); ucc_status_t ucc_cl_doca_urom_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *attr) { ucc_base_ctx_attr_clear(attr); return UCC_OK; } ucc_status_t ucc_cl_doca_urom_mem_map(const ucc_base_context_t *context, int type, void *memh, void *tl_h) { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_cl_doca_urom_mem_unmap(const ucc_base_context_t *context, int type, void *tl_h) { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_cl_doca_urom_memh_pack(const ucc_base_context_t *context, int type, void *memh, void **packed_buffer) { return UCC_ERR_NOT_SUPPORTED; } ucc-1.8.0/src/components/cl/doca_urom/Makefile.am0000664000175000017500000000170415211535620022036 0ustar alastairalastair# # Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # sources = \ cl_doca_urom.h \ cl_doca_urom.c \ cl_doca_urom_lib.c \ cl_doca_urom_context.c \ cl_doca_urom_team.c \ cl_doca_urom_common.c \ cl_doca_urom_common.h \ cl_doca_urom_worker_ucc.c \ cl_doca_urom_worker_ucc.h \ cl_doca_urom_coll.c module_LTLIBRARIES = libucc_cl_doca_urom.la libucc_cl_doca_urom_la_SOURCES = $(sources) libucc_cl_doca_urom_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(DOCA_UROM_CPPFLAGS) -I$(top_srcdir)/contrib/doca_urom_ucc_plugin/common libucc_cl_doca_urom_la_CFLAGS = $(BASE_CFLAGS) libucc_cl_doca_urom_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(DOCA_UROM_LDFLAGS) libucc_cl_doca_urom_la_LIBADD = $(DOCA_UROM_LIBADD) $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am ucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom_worker_ucc.c0000664000175000017500000010555215211535620025025 0ustar alastairalastair/* * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES, ALL RIGHTS RESERVED. * * This software product is a proprietary product of NVIDIA CORPORATION & * AFFILIATES (the "Company") and all right, title, and interest in and to the * software product, including all associated intellectual property rights, are * and shall remain exclusively with the Company. * * This software product is governed by the End User License Agreement * provided with the software product. * */ #include #include #include #include #include "cl_doca_urom_worker_ucc.h" DOCA_LOG_REGISTER(UCC::DOCA_CL : WORKER_UCC); static uint64_t ucc_id; /* UCC plugin id, id is generated by UROM lib and * will be updated in init function */ static uint64_t ucc_version = 0x01; /* UCC plugin host version */ /* UCC task metadata */ struct ucc_cl_doca_urom_task_data { union doca_data cookie; /* User cookie */ union { ucc_cl_doca_urom_lib_create_finished_cb lib_create; /* User lib create task callback */ ucc_cl_doca_urom_lib_destroy_finished_cb lib_destroy; /* User lib destroy task callback */ ucc_cl_doca_urom_ctx_create_finished_cb ctx_create; /* User context create task callback */ ucc_cl_doca_urom_ctx_destroy_finished_cb ctx_destroy; /* User context destroy task callback */ ucc_cl_doca_urom_team_create_finished_cb team_create; /* User UCC team create task callback */ ucc_cl_doca_urom_collective_finished_cb collective; /* User UCC collective task callback */ ucc_cl_doca_urom_pd_channel_finished_cb pd_channel; /* User passive data channel task callback */ }; }; /* * UCC notification unpack function * * @packed_notif [in]: packed UCC notification buffer * @ucc_notif [out]: set unpacked UCC notification * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ static doca_error_t notif_unpack(void *packed_notif, struct urom_worker_notify_ucc **ucc_notif) { *ucc_notif = packed_notif; return DOCA_SUCCESS; } /* * UCC common command's completion callback function * * @task [in]: UROM worker task * @type [in]: UCC task type */ static void completion(struct doca_urom_worker_cmd_task *task, enum urom_worker_ucc_notify_type type) { struct urom_worker_notify_ucc notify_error = {0}; struct urom_worker_notify_ucc *ucc_notify = ¬ify_error; struct ucc_cl_doca_urom_task_data *task_data; struct doca_buf *response; doca_error_t result; size_t data_len; notify_error.notify_type = type; task_data = (struct ucc_cl_doca_urom_task_data *) doca_urom_worker_cmd_task_get_user_data(task); if (task_data == NULL) { DOCA_LOG_ERR("Failed to get task data buffer"); goto task_release; } response = doca_urom_worker_cmd_task_get_response(task); if (response == NULL) { DOCA_LOG_ERR("Failed to get task response buffer"); result = DOCA_ERROR_INVALID_VALUE; goto error_exit; } result = doca_buf_get_data(response, (void **)&ucc_notify); if (result != DOCA_SUCCESS) goto error_exit; result = notif_unpack((void *)ucc_notify, &ucc_notify); if (result != DOCA_SUCCESS) goto error_exit; result = doca_buf_get_data_len(response, &data_len); if (result != DOCA_SUCCESS) { DOCA_LOG_ERR("Failed to get response data length"); goto error_exit; } result = doca_task_get_status(doca_urom_worker_cmd_task_as_task(task)); if (result != DOCA_SUCCESS) goto error_exit; if (data_len != sizeof(*ucc_notify)) { DOCA_LOG_ERR("Task response data length is different" "from notification expected length"); result = DOCA_ERROR_INVALID_VALUE; goto error_exit; } error_exit: switch (ucc_notify->notify_type) { case UROM_WORKER_NOTIFY_UCC_LIB_CREATE_COMPLETE: (task_data->lib_create)(result, task_data->cookie, ucc_notify->dpu_worker_id); break; case UROM_WORKER_NOTIFY_UCC_LIB_DESTROY_COMPLETE: (task_data->lib_destroy)(result, task_data->cookie, ucc_notify->dpu_worker_id); break; case UROM_WORKER_NOTIFY_UCC_CONTEXT_CREATE_COMPLETE: (task_data->ctx_create)(result, task_data->cookie, ucc_notify->dpu_worker_id, ucc_notify->context_create_nqe.context); break; case UROM_WORKER_NOTIFY_UCC_CONTEXT_DESTROY_COMPLETE: (task_data->ctx_destroy)(result, task_data->cookie, ucc_notify->dpu_worker_id); break; case UROM_WORKER_NOTIFY_UCC_TEAM_CREATE_COMPLETE: (task_data->team_create)(result, task_data->cookie, ucc_notify->dpu_worker_id, ucc_notify->team_create_nqe.team); break; case UROM_WORKER_NOTIFY_UCC_COLLECTIVE_COMPLETE: (task_data->collective)(result, task_data->cookie, ucc_notify->dpu_worker_id, ucc_notify->coll_nqe.status); break; case UROM_WORKER_NOTIFY_UCC_PASSIVE_DATA_CHANNEL_COMPLETE: (task_data->pd_channel)(result, task_data->cookie, ucc_notify->dpu_worker_id, ucc_notify->pass_dc_nqe.status); break; default: DOCA_LOG_ERR("Invalid UCC notification type %u", ucc_notify->notify_type); break; } task_release: result = doca_urom_worker_cmd_task_release(task); if (result != DOCA_SUCCESS) DOCA_LOG_ERR("Failed to release worker command task %s", doca_error_get_descr(result)); } /* * Pack UCC command * * @ucc_cmd [in]: ucc command * @packed_cmd_len [in/out]: packed command buffer size * @packed_cmd [out]: packed command buffer * @return: DOCA_SUCCESS on success and DOCA_ERROR otherwise */ static doca_error_t cmd_pack(struct urom_worker_ucc_cmd *ucc_cmd, size_t *packed_cmd_len, void *packed_cmd) { void *pack_tail = packed_cmd; void *pack_head; size_t pack_len; size_t team_size; size_t disp_pack_size; size_t count_pack_size; ucc_coll_args_t *coll_args; int is_count_64, is_disp_64; pack_len = sizeof(struct urom_worker_ucc_cmd); if (pack_len > *packed_cmd_len) return DOCA_ERROR_INITIALIZATION; /* Pack base command */ pack_head = urom_ucc_serialize_next_raw(&pack_tail, void, pack_len); memcpy(pack_head, ucc_cmd, pack_len); *packed_cmd_len = pack_len; switch (ucc_cmd->cmd_type) { case UROM_WORKER_CMD_UCC_LIB_CREATE: pack_len = sizeof(ucc_lib_params_t); pack_head = urom_ucc_serialize_next_raw(&pack_tail, void, pack_len); memcpy(pack_head, ucc_cmd->lib_create_cmd.params, pack_len); *packed_cmd_len += pack_len; break; case UROM_WORKER_CMD_UCC_CONTEXT_CREATE: if (ucc_cmd->context_create_cmd.stride <= 0) { pack_len = sizeof(int64_t) * ucc_cmd->context_create_cmd.size; pack_head = urom_ucc_serialize_next_raw(&pack_tail, void, pack_len); memcpy(pack_head, ucc_cmd->context_create_cmd.array, pack_len); *packed_cmd_len += pack_len; } break; case UROM_WORKER_CMD_UCC_COLL: coll_args = ucc_cmd->coll_cmd.coll_args; pack_len = sizeof(ucc_coll_args_t); pack_head = urom_ucc_serialize_next_raw(&pack_tail, void, pack_len); memcpy(pack_head, ucc_cmd->coll_cmd.coll_args, pack_len); *packed_cmd_len += pack_len; pack_len = ucc_cmd->coll_cmd.work_buffer_size; if (pack_len > 0 && ucc_cmd->coll_cmd.work_buffer) { pack_head = urom_ucc_serialize_next_raw(&pack_tail, void, pack_len); memcpy(pack_head, ucc_cmd->coll_cmd.work_buffer, pack_len); *packed_cmd_len += pack_len; } if (coll_args->coll_type == UCC_COLL_TYPE_ALLTOALLV || coll_args->coll_type == UCC_COLL_TYPE_ALLGATHERV || coll_args->coll_type == UCC_COLL_TYPE_GATHERV || coll_args->coll_type == UCC_COLL_TYPE_REDUCE_SCATTERV || coll_args->coll_type == UCC_COLL_TYPE_SCATTERV) { team_size = ucc_cmd->coll_cmd.team_size; is_count_64 = ((coll_args->mask & UCC_COLL_ARGS_FIELD_FLAGS) && (coll_args->flags & UCC_COLL_ARGS_FLAG_COUNT_64BIT)); is_disp_64 = ((coll_args->mask & UCC_COLL_ARGS_FIELD_FLAGS) && (coll_args->flags & UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT)); count_pack_size = team_size * ((is_count_64) ? sizeof(uint64_t) : sizeof(uint32_t)); disp_pack_size = team_size * ((is_disp_64) ? sizeof(uint64_t) : sizeof(uint32_t)); pack_len = count_pack_size; pack_head = urom_ucc_serialize_next_raw(&pack_tail, void, pack_len); memcpy(pack_head, coll_args->src.info_v.counts, pack_len); *packed_cmd_len += pack_len; pack_head = urom_ucc_serialize_next_raw(&pack_tail, void, pack_len); memcpy(pack_head, coll_args->dst.info_v.counts, pack_len); *packed_cmd_len += pack_len; pack_len = disp_pack_size; pack_head = urom_ucc_serialize_next_raw(&pack_tail, void, pack_len); memcpy(pack_head, coll_args->src.info_v.displacements, pack_len); *packed_cmd_len += pack_len; pack_head = urom_ucc_serialize_next_raw(&pack_tail, void, pack_len); memcpy(pack_head, coll_args->dst.info_v.displacements, pack_len); *packed_cmd_len += pack_len; } break; case UROM_WORKER_CMD_UCC_CREATE_PASSIVE_DATA_CHANNEL: pack_len = ucc_cmd->pass_dc_create_cmd.addr_len; pack_head = urom_ucc_serialize_next_raw(&pack_tail, void, pack_len); memcpy(pack_head, ucc_cmd->pass_dc_create_cmd.ucp_addr, pack_len); *packed_cmd_len += pack_len; break; default: DOCA_LOG_ERR("Invalid UCC cmd type %u", ucc_cmd->cmd_type); break; } return DOCA_SUCCESS; } /* * UCC library create command completion callback function, user callback will be called inside the function * * @task [in]: UROM worker task * @task_user_data [in]: task user data * @ctx_user_data [in]: worker context user data */ static void lib_create_completed(struct doca_urom_worker_cmd_task *task, union doca_data task_user_data, union doca_data ctx_user_data) { (void)task_user_data; (void)ctx_user_data; completion(task, UROM_WORKER_NOTIFY_UCC_LIB_CREATE_COMPLETE); } doca_error_t ucc_cl_doca_urom_task_lib_create( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, void *params, ucc_cl_doca_urom_lib_create_finished_cb cb) { size_t pack_len = 0; struct doca_buf *payload; struct doca_urom_worker_cmd_task *task; struct ucc_cl_doca_urom_task_data *task_data; struct urom_worker_ucc_cmd *ucc_cmd; doca_error_t result; /* Allocate task */ result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, ucc_id, &task); if (result != DOCA_SUCCESS) return result; payload = doca_urom_worker_cmd_task_get_payload(task); result = doca_buf_get_data(payload, (void **)&ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_get_data_len(payload, &pack_len); if (result != DOCA_SUCCESS) goto task_destroy; /* Populate commands attributes */ ucc_cmd->cmd_type = UROM_WORKER_CMD_UCC_LIB_CREATE; ucc_cmd->dpu_worker_id = dpu_worker_id; ucc_cmd->lib_create_cmd.params = params; result = cmd_pack(ucc_cmd, &pack_len, (void *)ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_set_data(payload, ucc_cmd, pack_len); if (result != DOCA_SUCCESS) goto task_destroy; task_data = (struct ucc_cl_doca_urom_task_data *) doca_urom_worker_cmd_task_get_user_data(task); task_data->lib_create = cb; task_data->cookie = cookie; doca_urom_worker_cmd_task_set_cb(task, lib_create_completed); result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task)); if (result != DOCA_SUCCESS) goto task_destroy; return DOCA_SUCCESS; task_destroy: doca_urom_worker_cmd_task_release(task); return result; } /* * UCC library destroy command completion callback function, * user callback will be called inside the function * * @task [in]: UROM worker task * @task_user_data [in]: task user data * @ctx_user_data [in]: worker context user data */ static void lib_destroy_completed(struct doca_urom_worker_cmd_task *task, union doca_data task_user_data, union doca_data ctx_user_data) { (void)task_user_data; (void)ctx_user_data; completion(task, UROM_WORKER_NOTIFY_UCC_LIB_DESTROY_COMPLETE); } doca_error_t doca_urom_ucc_task_lib_destroy( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, ucc_cl_doca_urom_lib_destroy_finished_cb cb) { size_t pack_len = 0; struct doca_buf *payload; struct doca_urom_worker_cmd_task *task; struct ucc_cl_doca_urom_task_data *task_data; struct urom_worker_ucc_cmd *ucc_cmd; doca_error_t result; /* Allocate task */ result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, ucc_id, &task); if (result != DOCA_SUCCESS) return result; payload = doca_urom_worker_cmd_task_get_payload(task); result = doca_buf_get_data(payload, (void **)&ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_get_data_len(payload, &pack_len); if (result != DOCA_SUCCESS) goto task_destroy; /* Populate commands attributes */ ucc_cmd->cmd_type = UROM_WORKER_CMD_UCC_LIB_DESTROY; ucc_cmd->dpu_worker_id = dpu_worker_id; result = cmd_pack(ucc_cmd, &pack_len, (void *)ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_set_data(payload, ucc_cmd, pack_len); if (result != DOCA_SUCCESS) goto task_destroy; task_data = (struct ucc_cl_doca_urom_task_data *) doca_urom_worker_cmd_task_get_user_data(task); task_data->lib_destroy = cb; task_data->cookie = cookie; doca_urom_worker_cmd_task_set_cb(task, lib_destroy_completed); result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task)); if (result != DOCA_SUCCESS) goto task_destroy; return DOCA_SUCCESS; task_destroy: doca_urom_worker_cmd_task_release(task); return result; } /* * UCC context create command completion callback function, * user callback will be called inside the function * * @task [in]: UROM worker task * @task_user_data [in]: task user data * @ctx_user_data [in]: worker context user data */ static void ctx_create_completed(struct doca_urom_worker_cmd_task *task, union doca_data task_user_data, union doca_data ctx_user_data) { (void)task_user_data; (void)ctx_user_data; completion(task, UROM_WORKER_NOTIFY_UCC_CONTEXT_CREATE_COMPLETE); } doca_error_t ucc_cl_doca_urom_task_ctx_create( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, int64_t start, int64_t *array, int64_t stride, int64_t size, void *base_va, uint64_t len, ucc_cl_doca_urom_ctx_create_finished_cb cb) { size_t pack_len = 0; struct ucc_cl_doca_urom_task_data *task_data; struct doca_urom_worker_cmd_task *task; struct urom_worker_ucc_cmd *ucc_cmd; struct doca_buf *payload; doca_error_t result; /* Allocate task */ result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, ucc_id, &task); if (result != DOCA_SUCCESS) return result; payload = doca_urom_worker_cmd_task_get_payload(task); result = doca_buf_get_data(payload, (void **)&ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_get_data_len(payload, &pack_len); if (result != DOCA_SUCCESS) goto task_destroy; /* Populate commands attributes */ ucc_cmd->cmd_type = UROM_WORKER_CMD_UCC_CONTEXT_CREATE; ucc_cmd->dpu_worker_id = dpu_worker_id; if (array == NULL) { ucc_cmd->context_create_cmd.start = start; } else { ucc_cmd->context_create_cmd.array = array; } ucc_cmd->context_create_cmd.stride = stride; ucc_cmd->context_create_cmd.size = size; ucc_cmd->context_create_cmd.base_va = base_va; ucc_cmd->context_create_cmd.len = len; result = cmd_pack(ucc_cmd, &pack_len, (void *)ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_set_data(payload, ucc_cmd, pack_len); if (result != DOCA_SUCCESS) goto task_destroy; task_data = (struct ucc_cl_doca_urom_task_data *) doca_urom_worker_cmd_task_get_user_data(task); task_data->ctx_create = cb; task_data->cookie = cookie; doca_urom_worker_cmd_task_set_cb(task, ctx_create_completed); result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task)); if (result != DOCA_SUCCESS) goto task_destroy; return DOCA_SUCCESS; task_destroy: doca_urom_worker_cmd_task_release(task); return result; } /* * UCC context destroy command completion callback function, * user callback will be called inside the function * * @task [in]: UROM worker task * @task_user_data [in]: task user data * @ctx_user_data [in]: worker context user data */ static void ctx_destroy_completed(struct doca_urom_worker_cmd_task *task, union doca_data task_user_data, union doca_data ctx_user_data) { (void)task_user_data; (void)ctx_user_data; completion(task, UROM_WORKER_NOTIFY_UCC_CONTEXT_DESTROY_COMPLETE); } doca_error_t ucc_cl_doca_urom_task_ctx_destroy( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, void *context, ucc_cl_doca_urom_ctx_destroy_finished_cb cb) { size_t pack_len = 0; struct ucc_cl_doca_urom_task_data *task_data; struct doca_urom_worker_cmd_task *task; struct urom_worker_ucc_cmd *ucc_cmd; struct doca_buf *payload; doca_error_t result; /* Allocate task */ result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, ucc_id, &task); if (result != DOCA_SUCCESS) return result; payload = doca_urom_worker_cmd_task_get_payload(task); result = doca_buf_get_data(payload, (void **)&ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_get_data_len(payload, &pack_len); if (result != DOCA_SUCCESS) goto task_destroy; /* Populate commands attributes */ ucc_cmd->cmd_type = UROM_WORKER_CMD_UCC_CONTEXT_DESTROY; ucc_cmd->dpu_worker_id = dpu_worker_id; ucc_cmd->context_destroy_cmd.context_h = context; result = cmd_pack(ucc_cmd, &pack_len, (void *)ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_set_data(payload, ucc_cmd, pack_len); if (result != DOCA_SUCCESS) goto task_destroy; task_data = (struct ucc_cl_doca_urom_task_data *) doca_urom_worker_cmd_task_get_user_data(task); task_data->ctx_destroy = cb; task_data->cookie = cookie; doca_urom_worker_cmd_task_set_cb(task, ctx_destroy_completed); result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task)); if (result != DOCA_SUCCESS) goto task_destroy; return DOCA_SUCCESS; task_destroy: doca_urom_worker_cmd_task_release(task); return result; } /* * UCC team create command completion callback function, user callback will be called inside the function * * @task [in]: UROM worker task * @task_user_data [in]: task user data * @ctx_user_data [in]: worker context user data */ static void team_create_completed(struct doca_urom_worker_cmd_task *task, union doca_data task_user_data, union doca_data ctx_user_data) { (void)task_user_data; (void)ctx_user_data; completion(task, UROM_WORKER_NOTIFY_UCC_TEAM_CREATE_COMPLETE); } doca_error_t ucc_cl_doca_urom_task_team_create( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, int64_t start, int64_t stride, int64_t size, void *context, ucc_cl_doca_urom_team_create_finished_cb cb) { size_t pack_len = 0; struct ucc_cl_doca_urom_task_data *task_data; struct doca_urom_worker_cmd_task *task; struct urom_worker_ucc_cmd *ucc_cmd; struct doca_buf *payload; doca_error_t result; /* Allocate task */ result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, ucc_id, &task); if (result != DOCA_SUCCESS) return result; payload = doca_urom_worker_cmd_task_get_payload(task); result = doca_buf_get_data(payload, (void **)&ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_get_data_len(payload, &pack_len); if (result != DOCA_SUCCESS) goto task_destroy; /* Populate commands attributes */ ucc_cmd->cmd_type = UROM_WORKER_CMD_UCC_TEAM_CREATE; ucc_cmd->dpu_worker_id = dpu_worker_id; ucc_cmd->team_create_cmd.start = start; ucc_cmd->team_create_cmd.stride = stride; ucc_cmd->team_create_cmd.size = size; ucc_cmd->team_create_cmd.context_h = context; result = cmd_pack(ucc_cmd, &pack_len, (void *)ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_set_data(payload, ucc_cmd, pack_len); if (result != DOCA_SUCCESS) goto task_destroy; task_data = (struct ucc_cl_doca_urom_task_data *) doca_urom_worker_cmd_task_get_user_data(task); task_data->team_create = cb; task_data->cookie = cookie; doca_urom_worker_cmd_task_set_cb(task, team_create_completed); result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task)); if (result != DOCA_SUCCESS) goto task_destroy; return DOCA_SUCCESS; task_destroy: doca_urom_worker_cmd_task_release(task); return result; } /* * UCC collective command completion callback function, * user callback will be called inside the function * * @task [in]: UROM worker task * @task_user_data [in]: task user data * @ctx_user_data [in]: worker context user data */ static void collective_completed(struct doca_urom_worker_cmd_task *task, union doca_data task_user_data, union doca_data ctx_user_data) { (void)task_user_data; (void)ctx_user_data; completion(task, UROM_WORKER_NOTIFY_UCC_COLLECTIVE_COMPLETE); } doca_error_t ucc_cl_doca_urom_task_collective( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, void *coll_args, void *team, int use_xgvmi, void *work_buffer, size_t work_buffer_size, size_t team_size, ucc_cl_doca_urom_collective_finished_cb cb) { size_t pack_len = 0; struct ucc_cl_doca_urom_task_data *task_data; struct doca_urom_worker_cmd_task *task; struct urom_worker_ucc_cmd *ucc_cmd; struct doca_buf *payload; doca_error_t result; /* Allocate task */ result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, ucc_id, &task); if (result != DOCA_SUCCESS) return result; payload = doca_urom_worker_cmd_task_get_payload(task); result = doca_buf_get_data(payload, (void **)&ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_get_data_len(payload, &pack_len); if (result != DOCA_SUCCESS) goto task_destroy; /* Populate commands attributes */ ucc_cmd->cmd_type = UROM_WORKER_CMD_UCC_COLL; ucc_cmd->dpu_worker_id = dpu_worker_id; ucc_cmd->coll_cmd.coll_args = coll_args; ucc_cmd->coll_cmd.team = team; ucc_cmd->coll_cmd.use_xgvmi = use_xgvmi; ucc_cmd->coll_cmd.work_buffer = work_buffer; ucc_cmd->coll_cmd.work_buffer_size = work_buffer_size; ucc_cmd->coll_cmd.team_size = team_size; result = cmd_pack(ucc_cmd, &pack_len, (void *)ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_set_data(payload, ucc_cmd, pack_len); if (result != DOCA_SUCCESS) goto task_destroy; task_data = (struct ucc_cl_doca_urom_task_data *) doca_urom_worker_cmd_task_get_user_data(task); task_data->collective = cb; task_data->cookie = cookie; doca_urom_worker_cmd_task_set_cb(task, collective_completed); result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task)); if (result != DOCA_SUCCESS) goto task_destroy; return DOCA_SUCCESS; task_destroy: doca_urom_worker_cmd_task_release(task); return result; } /* * UCC passive data channel command completion callback function, * user callback will be called inside the function * * @task [in]: UROM worker task * @task_user_data [in]: task user data * @ctx_user_data [in]: worker context user data */ static void pd_channel_completed(struct doca_urom_worker_cmd_task *task, union doca_data task_user_data, union doca_data ctx_user_data) { (void)task_user_data; (void)ctx_user_data; completion(task, UROM_WORKER_NOTIFY_UCC_PASSIVE_DATA_CHANNEL_COMPLETE); } doca_error_t ucc_cl_doca_urom_task_pd_channel( struct doca_urom_worker *worker_ctx, union doca_data cookie, uint64_t dpu_worker_id, void *ucp_addr, size_t addr_len, ucc_cl_doca_urom_pd_channel_finished_cb cb) { size_t pack_len = 0; struct ucc_cl_doca_urom_task_data *task_data; struct doca_urom_worker_cmd_task *task; struct urom_worker_ucc_cmd *ucc_cmd; struct doca_buf *payload; doca_error_t result; /* Allocate task */ result = doca_urom_worker_cmd_task_allocate_init(worker_ctx, ucc_id, &task); if (result != DOCA_SUCCESS) return result; payload = doca_urom_worker_cmd_task_get_payload(task); result = doca_buf_get_data(payload, (void **)&ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_get_data_len(payload, &pack_len); if (result != DOCA_SUCCESS) goto task_destroy; /* Populate commands attributes */ ucc_cmd->cmd_type = UROM_WORKER_CMD_UCC_CREATE_PASSIVE_DATA_CHANNEL; ucc_cmd->dpu_worker_id = dpu_worker_id; ucc_cmd->pass_dc_create_cmd.ucp_addr = ucp_addr; ucc_cmd->pass_dc_create_cmd.addr_len = addr_len; result = cmd_pack(ucc_cmd, &pack_len, (void *)ucc_cmd); if (result != DOCA_SUCCESS) goto task_destroy; result = doca_buf_set_data(payload, ucc_cmd, pack_len); if (result != DOCA_SUCCESS) goto task_destroy; task_data = (struct ucc_cl_doca_urom_task_data *) doca_urom_worker_cmd_task_get_user_data(task); task_data->pd_channel = cb; task_data->cookie = cookie; doca_urom_worker_cmd_task_set_cb(task, pd_channel_completed); result = doca_task_submit(doca_urom_worker_cmd_task_as_task(task)); if (result != DOCA_SUCCESS) goto task_destroy; return DOCA_SUCCESS; task_destroy: doca_urom_worker_cmd_task_release(task); return result; } doca_error_t ucc_cl_doca_urom_save_plugin_id(uint64_t plugin_id, uint64_t version) { if (version != ucc_version) { return DOCA_ERROR_UNSUPPORTED_VERSION; } ucc_id = plugin_id; return DOCA_SUCCESS; } /* * UCC lib create callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id */ void ucc_cl_doca_urom_lib_create_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id) { struct ucc_cl_doca_urom_result *res = (struct ucc_cl_doca_urom_result *)cookie.ptr; if (res == NULL) { return; } res->dpu_worker_id = dpu_worker_id; res->result = result; } /* * UCC passive data channel callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id * @status [in]: channel creation status */ void ucc_cl_doca_urom_pss_dc_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, ucc_status_t status) { struct ucc_cl_doca_urom_result *res = (struct ucc_cl_doca_urom_result *)cookie.ptr; if (res == NULL) { return; } res->dpu_worker_id = dpu_worker_id; res->result = result; res->pass_dc.status = status; } /* * UCC lib destroy callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id */ void ucc_cl_doca_urom_lib_destroy_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id) { struct ucc_cl_doca_urom_result *res = (struct ucc_cl_doca_urom_result *)cookie.ptr; if (res == NULL) { return; } res->dpu_worker_id = dpu_worker_id; res->result = result; } /* * UCC context create callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id * @context [in]: pointer to UCC context */ void ucc_cl_doca_urom_ctx_create_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, void *context) { struct ucc_cl_doca_urom_result *res = (struct ucc_cl_doca_urom_result *)cookie.ptr; if (res == NULL) { return; } res->dpu_worker_id = dpu_worker_id; res->result = result; res->context_create.context = context; } /* * UCC collective callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id * @status [in]: collective status */ void ucc_cl_doca_urom_collective_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, ucc_status_t status) { struct ucc_cl_doca_urom_result *res = (struct ucc_cl_doca_urom_result *)cookie.ptr; if (res == NULL) { return; } res->dpu_worker_id = dpu_worker_id; res->result = result; res->collective.status = status; } /* * UCC team create callback * * @result [in]: task result * @cookie [in]: program cookie * @dpu_worker_id [in]: UROM DPU worker id * @team [in]: pointer to UCC team */ void ucc_cl_doca_urom_team_create_finished( doca_error_t result, union doca_data cookie, uint64_t dpu_worker_id, void *team) { struct ucc_cl_doca_urom_result *res = (struct ucc_cl_doca_urom_result *)cookie.ptr; if (res == NULL) { return; } res->dpu_worker_id = dpu_worker_id; res->result = result; res->team_create.team = team; } ucc_status_t ucc_cl_doca_urom_buffer_export_ucc( ucp_context_h ucp_context, void *buf, size_t len, struct export_buf *ebuf) { ucs_status_t ucs_status; ucp_mem_map_params_t params; ucp_memh_pack_params_t pack_params; ebuf->ucp_context = ucp_context; params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH; params.address = buf; params.length = len; ucs_status = ucp_mem_map(ucp_context, ¶ms, &ebuf->memh); assert(ucs_status == UCS_OK); pack_params.field_mask = UCP_MEMH_PACK_PARAM_FIELD_FLAGS; pack_params.flags = UCP_MEMH_PACK_FLAG_EXPORT; ucs_status = ucp_memh_pack(ebuf->memh, &pack_params, &ebuf->packed_memh, &ebuf->packed_memh_len); if (ucs_status != UCS_OK) { printf("ucp_memh_pack() returned error: %s\n", ucs_status_string(ucs_status)); ebuf->packed_memh = NULL; ebuf->packed_memh_len = 0; return UCC_ERR_NO_RESOURCE; } ucs_status = ucp_rkey_pack(ucp_context, ebuf->memh, &ebuf->packed_key, &ebuf->packed_key_len); if (UCS_OK != ucs_status) { printf("ucp_rkey_pack() returned error: %s\n", ucs_status_string(ucs_status)); return UCC_ERR_NO_RESOURCE; } return UCC_OK; } ucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom.c0000664000175000017500000000636215211535620022601 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_doca_urom.h" #include "utils/ucc_malloc.h" ucc_status_t ucc_cl_doca_urom_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr); ucc_status_t ucc_cl_doca_urom_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *base_attr); ucc_status_t ucc_cl_doca_urom_mem_map(const ucc_base_context_t *context, int type, void *memh, void *tl_h); ucc_status_t ucc_cl_doca_urom_mem_unmap(const ucc_base_context_t *context, int type, void *tl_h); ucc_status_t ucc_cl_doca_urom_memh_pack(const ucc_base_context_t *context, int type, void *memh, void **packed_buffer); ucc_status_t ucc_cl_doca_urom_get_lib_properties(ucc_base_lib_properties_t *prop); static ucc_config_field_t ucc_cl_doca_urom_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_cl_doca_urom_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_cl_lib_config_table)}, {NULL}}; static ucs_config_field_t ucc_cl_doca_urom_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_cl_doca_urom_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_cl_context_config_table)}, {"PLUGIN_ENVS", "", "Comma separated envs to pass to the worker plugin", ucc_offsetof(ucc_cl_doca_urom_context_config_t, plugin_envs), UCC_CONFIG_TYPE_STRING_ARRAY}, {"DEVICE", "mlx5_0", "DPU device", ucc_offsetof(ucc_cl_doca_urom_context_config_t, device), UCC_CONFIG_TYPE_STRING}, {"PLUGIN_NAME", "libucc_doca_urom_plugin", "Name of plugin library", ucc_offsetof(ucc_cl_doca_urom_context_config_t, plugin_name), UCC_CONFIG_TYPE_STRING}, {"DOCA_LOG_LEVEL", "10", "DOCA log level", ucc_offsetof(ucc_cl_doca_urom_context_config_t, doca_log_level), UCC_CONFIG_TYPE_INT}, {NULL}}; UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_doca_urom_lib_t, ucc_base_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_doca_urom_lib_t, ucc_base_lib_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_doca_urom_context_t, ucc_base_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_doca_urom_context_t, ucc_base_context_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_doca_urom_team_t, ucc_base_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_cl_doca_urom_team_create_test(ucc_base_team_t *cl_team); ucc_status_t ucc_cl_doca_urom_team_destroy(ucc_base_team_t *cl_team); ucc_status_t ucc_cl_doca_urom_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); ucc_status_t ucc_cl_doca_urom_team_get_scores(ucc_base_team_t *cl_team, ucc_coll_score_t **score); UCC_CL_IFACE_DECLARE(doca_urom, DOCA_UROM); ucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom_coll.h0000664000175000017500000000216515211535620023614 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_CL_DOCA_UROM_COLL_H_ #define UCC_CL_DOCA_UROM_COLL_H_ #include "cl_doca_urom.h" #include "schedule/ucc_schedule_pipelined.h" #include "components/mc/ucc_mc.h" #include "../../tl/ucp/tl_ucp.h" #define UCC_CL_DOCA_UROM_N_DEFAULT_ALG_SELECT_STR 2 extern const char *ucc_cl_doca_urom_default_alg_select_str[UCC_CL_DOCA_UROM_N_DEFAULT_ALG_SELECT_STR]; typedef struct ucc_cl_doca_urom_schedule_t { ucc_schedule_pipelined_t super; struct ucc_cl_doca_urom_result res; struct export_buf src_ebuf; struct export_buf dst_ebuf; } ucc_cl_doca_urom_schedule_t; static inline ucc_cl_doca_urom_schedule_t * ucc_cl_doca_urom_get_schedule(ucc_cl_doca_urom_team_t *team) { ucc_cl_doca_urom_context_t *ctx = UCC_CL_DOCA_UROM_TEAM_CTX(team); ucc_cl_doca_urom_schedule_t *schedule = ucc_mpool_get(&ctx->sched_mp); return schedule; } static inline void ucc_cl_doca_urom_put_schedule(ucc_schedule_t *schedule) { ucc_mpool_put(schedule); } #endif ucc-1.8.0/src/components/cl/doca_urom/cl_doca_urom_lib.c0000664000175000017500000000565215211535620023430 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_doca_urom.h" #include "utils/ucc_malloc.h" #include "components/tl/ucc_tl.h" #include "core/ucc_global_opts.h" #include "utils/ucc_math.h" #include /* NOLINTNEXTLINE TODO params is not used*/ UCC_CLASS_INIT_FUNC(ucc_cl_doca_urom_lib_t, const ucc_base_lib_params_t *params, const ucc_base_config_t *config) { const ucc_cl_doca_urom_lib_config_t *cl_config = ucc_derived_of(config, ucc_cl_doca_urom_lib_config_t); UCC_CLASS_CALL_SUPER_INIT(ucc_cl_lib_t, &ucc_cl_doca_urom.super, &cl_config->super); memcpy(&self->cfg, cl_config, sizeof(*cl_config)); cl_debug(&self->super, "initialized lib object: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_doca_urom_lib_t) { cl_debug(&self->super, "finalizing lib object: %p", self); } UCC_CLASS_DEFINE(ucc_cl_doca_urom_lib_t, ucc_cl_lib_t); static inline ucc_status_t check_tl_lib_attr(const ucc_base_lib_t *lib, ucc_tl_iface_t *tl_iface, ucc_cl_lib_attr_t *attr) { ucc_tl_lib_attr_t tl_attr; ucc_status_t status; memset(&tl_attr, 0, sizeof(tl_attr)); status = tl_iface->lib.get_attr(NULL, &tl_attr.super); if (UCC_OK != status) { cl_error(lib, "failed to query tl %s lib attributes", tl_iface->super.name); return status; } attr->super.attr.thread_mode = ucc_min(attr->super.attr.thread_mode, tl_attr.super.attr.thread_mode); attr->super.attr.coll_types |= tl_attr.super.attr.coll_types; attr->super.flags |= tl_attr.super.flags; return UCC_OK; } ucc_status_t ucc_cl_doca_urom_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr) { ucc_cl_doca_urom_lib_t *cl_lib = ucc_derived_of(lib, ucc_cl_doca_urom_lib_t); ucc_cl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_cl_lib_attr_t); ucc_status_t status; attr->tls = &cl_lib->super.tls.array; if (cl_lib->super.tls.requested) { status = ucc_config_names_array_dup(&cl_lib->super.tls_forced, &cl_lib->super.tls.array); if (UCC_OK != status) { return status; } } attr->tls_forced = &cl_lib->super.tls_forced; attr->super.attr.thread_mode = UCC_THREAD_MULTIPLE; attr->super.attr.coll_types = UCC_COLL_TYPE_ALLREDUCE; attr->super.flags = 0; ucc_assert(tls->array.count >= 1); return UCC_OK; } ucc_status_t ucc_cl_doca_urom_get_lib_properties(ucc_base_lib_properties_t *prop) { prop->default_team_size = 2; prop->min_team_size = 2; prop->max_team_size = UCC_RANK_MAX; return UCC_OK; } ucc-1.8.0/src/components/cl/basic/0000775000175000017500000000000015211535620017111 5ustar alastairalastairucc-1.8.0/src/components/cl/basic/cl_basic_team.c0000664000175000017500000001760215211535620022030 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_basic.h" #include "utils/ucc_malloc.h" #include "core/ucc_team.h" UCC_CLASS_INIT_FUNC(ucc_cl_basic_team_t, ucc_base_context_t *cl_context, const ucc_base_team_params_t *params) { ucc_cl_basic_context_t *ctx = ucc_derived_of(cl_context, ucc_cl_basic_context_t); unsigned n_tl_ctxs = ctx->super.n_tl_ctxs; int i; ucc_status_t status; UCC_CLASS_CALL_SUPER_INIT(ucc_cl_team_t, &ctx->super, params); self->tl_teams = ucc_malloc(sizeof(ucc_tl_team_t *) * n_tl_ctxs, "cl_basic_tl_teams"); if (!self->tl_teams) { cl_error(cl_context->lib, "failed to allocate %zd bytes for tl_teams", sizeof(ucc_tl_team_t *) * n_tl_ctxs); status = UCC_ERR_NO_MEMORY; goto err; } self->n_tl_teams = 0; self->score_map = NULL; status = ucc_team_multiple_req_alloc(&self->team_create_req, n_tl_ctxs); if (UCC_OK != status) { cl_error(cl_context->lib, "failed to allocate team req multiple"); goto err; } for (i = 0; i < n_tl_ctxs; i++) { memcpy(&self->team_create_req->descs[i].param, params, sizeof(ucc_base_team_params_t)); self->team_create_req->descs[i].ctx = ctx->super.tl_ctxs[i]; self->team_create_req->descs[i].param.scope = UCC_CL_BASIC; self->team_create_req->descs[i].param.scope_id = 0; } self->team_create_req->n_teams = n_tl_ctxs; status = ucc_tl_team_create_multiple(self->team_create_req); if (status < 0) { cl_error(cl_context->lib, "failed to post tl team create (%d)", status); goto err; } cl_debug(cl_context->lib, "posted cl team: %p", self); return UCC_OK; err: ucc_free(self->tl_teams); return status; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_basic_team_t) { cl_debug(self->super.super.context->lib, "finalizing cl team: %p", self); } UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_basic_team_t, ucc_base_team_t); UCC_CLASS_DEFINE(ucc_cl_basic_team_t, ucc_cl_team_t); ucc_status_t ucc_cl_basic_team_destroy(ucc_base_team_t *cl_team) { ucc_cl_basic_team_t *team = ucc_derived_of(cl_team, ucc_cl_basic_team_t); ucc_cl_basic_context_t *ctx = UCC_CL_BASIC_TEAM_CTX(team); ucc_status_t status = UCC_OK; int i; if (NULL == team->team_create_req) { status = ucc_team_multiple_req_alloc(&team->team_create_req, team->n_tl_teams); if (UCC_OK != status) { cl_error(ctx->super.super.lib, "failed to allocate team req multiple"); return status; } team->team_create_req->n_teams = team->n_tl_teams; for (i = 0; i < team->n_tl_teams; i++) { team->team_create_req->descs[i].team = team->tl_teams[i]; } } status = ucc_tl_team_destroy_multiple(team->team_create_req); if (UCC_INPROGRESS == status) { return status; } for (i = 0; i < team->n_tl_teams; i++) { if (team->team_create_req->descs[i].status != UCC_OK) { cl_error(ctx->super.super.lib, "tl team destroy failed (%d)", status); status = team->team_create_req->descs[i].status; } } ucc_team_multiple_req_free(team->team_create_req); if (team->score_map) { ucc_coll_score_free_map(team->score_map); } ucc_free(team->tl_teams); UCC_CLASS_DELETE_FUNC_NAME(ucc_cl_basic_team_t)(cl_team); return status; } ucc_status_t ucc_cl_basic_team_create_test(ucc_base_team_t *cl_team) { ucc_cl_basic_team_t *team = ucc_derived_of(cl_team, ucc_cl_basic_team_t); ucc_cl_basic_context_t *ctx = UCC_CL_BASIC_TEAM_CTX(team); ucc_status_t status; int i; ucc_coll_score_t *score, *score_next, *score_merge; status = ucc_tl_team_create_multiple(team->team_create_req); if (status == UCC_OK) { for (i = 0; i < ctx->super.n_tl_ctxs; i++) { if (team->team_create_req->descs[i].status == UCC_OK) { team->tl_teams[team->n_tl_teams++] = team->team_create_req->descs[i].team; cl_debug(ctx->super.super.lib, "initialized tl %s team", UCC_TL_CTX_IFACE(team->team_create_req->descs[i].ctx)-> super.name); } else { cl_debug(ctx->super.super.lib, "failed to create tl %s team: (%d) %s", UCC_TL_CTX_IFACE(team->team_create_req->descs[i].ctx)-> super.name, team->team_create_req->descs[i].status, ucc_status_string(team->team_create_req->descs[i].status)); } } ucc_team_multiple_req_free(team->team_create_req); team->team_create_req = NULL; if (0 == team->n_tl_teams) { cl_error(ctx->super.super.lib, "no tl teams were created"); return UCC_ERR_NOT_FOUND; } status = UCC_TL_TEAM_IFACE(team->tl_teams[0]) ->team.get_scores(&team->tl_teams[0]->super, &score); if (UCC_OK != status) { cl_error(ctx->super.super.lib, "failed to get tl %s scores", UCC_TL_TEAM_IFACE(team->tl_teams[0])->super.name); return status; } for (i = 1; i < team->n_tl_teams; i++) { status = UCC_TL_TEAM_IFACE(team->tl_teams[i]) ->team.get_scores(&team->tl_teams[i]->super, &score_next); if (UCC_OK != status) { cl_error(ctx->super.super.lib, "failed to get tl %s scores", UCC_TL_TEAM_IFACE(team->tl_teams[i])->super.name); return status; } status = ucc_coll_score_merge(score, score_next, &score_merge, 1); if (UCC_OK != status) { cl_error(ctx->super.super.lib, "failed to merge scores"); return status; } score = score_merge; } status = ucc_coll_score_build_map(score, &team->score_map); if (UCC_OK != status) { cl_error(ctx->super.super.lib, "failed to build score map"); } team->score = score; ucc_coll_score_set(team->score, UCC_CL_BASIC_DEFAULT_SCORE); } return status; } ucc_status_t ucc_cl_basic_team_get_scores(ucc_base_team_t *cl_team, ucc_coll_score_t **score) { ucc_cl_basic_team_t *team = ucc_derived_of(cl_team, ucc_cl_basic_team_t); ucc_base_context_t *ctx = UCC_CL_TEAM_CTX(team); ucc_status_t status; ucc_coll_score_team_info_t team_info; status = ucc_coll_score_dup(team->score, score); if (UCC_OK != status) { return status; } if (strlen(ctx->score_str) > 0) { team_info.alg_fn = NULL; team_info.default_score = UCC_CL_BASIC_DEFAULT_SCORE; team_info.init = NULL; team_info.num_mem_types = 0; team_info.supported_mem_types = NULL; /* all memory types supported*/ team_info.supported_colls = UCC_COLL_TYPE_ALL; team_info.size = UCC_CL_TEAM_SIZE(team); status = ucc_coll_score_update_from_str(ctx->score_str, &team_info, &team->super.super, *score); /* If INVALID_PARAM - User provided incorrect input - try to proceed */ if ((status < 0) && (status != UCC_ERR_INVALID_PARAM) && (status != UCC_ERR_NOT_SUPPORTED)) { goto err; } } return UCC_OK; err: ucc_coll_score_free(*score); *score = NULL; return status; } ucc-1.8.0/src/components/cl/basic/cl_basic.h0000664000175000017500000000341215211535620021021 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_CL_BASIC_H_ #define UCC_CL_BASIC_H_ #include "components/cl/ucc_cl.h" #include "components/cl/ucc_cl_log.h" #include "components/tl/ucc_tl.h" #include "coll_score/ucc_coll_score.h" #ifndef UCC_CL_BASIC_DEFAULT_SCORE #define UCC_CL_BASIC_DEFAULT_SCORE 10 #endif typedef struct ucc_cl_basic_iface { ucc_cl_iface_t super; } ucc_cl_basic_iface_t; /* Extern iface should follow the pattern: ucc_cl_ */ extern ucc_cl_basic_iface_t ucc_cl_basic; typedef struct ucc_cl_basic_lib_config { ucc_cl_lib_config_t super; } ucc_cl_basic_lib_config_t; typedef struct ucc_cl_basic_context_config { ucc_cl_context_config_t super; } ucc_cl_basic_context_config_t; typedef struct ucc_cl_basic_lib { ucc_cl_lib_t super; } ucc_cl_basic_lib_t; UCC_CLASS_DECLARE(ucc_cl_basic_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); typedef struct ucc_cl_basic_context { ucc_cl_context_t super; } ucc_cl_basic_context_t; UCC_CLASS_DECLARE(ucc_cl_basic_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); typedef struct ucc_cl_basic_team { ucc_cl_team_t super; ucc_team_multiple_req_t *team_create_req; ucc_tl_team_t **tl_teams; unsigned n_tl_teams; ucc_coll_score_t *score; ucc_score_map_t *score_map; } ucc_cl_basic_team_t; UCC_CLASS_DECLARE(ucc_cl_basic_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); #define UCC_CL_BASIC_TEAM_CTX(_team) \ (ucc_derived_of((_team)->super.super.context, ucc_cl_basic_context_t)) #endif ucc-1.8.0/src/components/cl/basic/cl_basic_lib.c0000664000175000017500000000670315211535620021650 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_basic.h" #include "utils/ucc_malloc.h" #include "components/tl/ucc_tl.h" #include "core/ucc_global_opts.h" #include "utils/ucc_math.h" /* NOLINTNEXTLINE TODO params is not used*/ UCC_CLASS_INIT_FUNC(ucc_cl_basic_lib_t, const ucc_base_lib_params_t *params, const ucc_base_config_t *config) { const ucc_cl_lib_config_t *cl_config = ucc_derived_of(config, ucc_cl_lib_config_t); UCC_CLASS_CALL_SUPER_INIT(ucc_cl_lib_t, &ucc_cl_basic.super, cl_config); cl_debug(&self->super, "initialized lib object: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_basic_lib_t) { cl_debug(&self->super, "finalizing lib object: %p", self); } UCC_CLASS_DEFINE(ucc_cl_basic_lib_t, ucc_cl_lib_t); static inline ucc_status_t check_tl_lib_attr(const ucc_base_lib_t *lib, ucc_tl_iface_t *tl_iface, ucc_cl_lib_attr_t *attr) { ucc_tl_lib_attr_t tl_attr; ucc_status_t status; memset(&tl_attr, 0, sizeof(tl_attr)); status = tl_iface->lib.get_attr(NULL, &tl_attr.super); if (UCC_OK != status) { cl_error(lib, "failed to query tl %s lib attributes", tl_iface->super.name); return status; } attr->super.attr.thread_mode = ucc_min(attr->super.attr.thread_mode, tl_attr.super.attr.thread_mode); attr->super.attr.coll_types |= tl_attr.super.attr.coll_types; attr->super.flags |= tl_attr.super.flags; return UCC_OK; } ucc_status_t ucc_cl_basic_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr) { ucc_cl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_cl_lib_attr_t); ucc_cl_basic_lib_t *cl_lib = ucc_derived_of(lib, ucc_cl_basic_lib_t); ucc_config_names_list_t *tls = &cl_lib->super.tls; ucc_tl_iface_t *tl_iface; int i; ucc_status_t status; attr->tls = &cl_lib->super.tls.array; if (cl_lib->super.tls.requested) { status = ucc_config_names_array_dup(&cl_lib->super.tls_forced, &cl_lib->super.tls.array); if (UCC_OK != status) { return status; } } attr->tls_forced = &cl_lib->super.tls_forced; attr->super.attr.thread_mode = UCC_THREAD_MULTIPLE; attr->super.attr.coll_types = 0; attr->super.flags = 0; ucc_assert(tls->array.count >= 1); for (i = 0; i < tls->array.count; i++) { /* Check TLs provided in CL_BASIC_TLS. Not all of them could be available, check for NULL. */ tl_iface = ucc_derived_of(ucc_get_component(&ucc_global_config.tl_framework, tls->array.names[i]), ucc_tl_iface_t); if (!tl_iface) { cl_warn(lib, "tl %s is not available", tls->array.names[i]); continue; } if (UCC_OK != (status = check_tl_lib_attr(lib, tl_iface, attr))) { return status; } } return UCC_OK; } ucc_status_t ucc_cl_basic_get_lib_properties(ucc_base_lib_properties_t *prop) { prop->default_team_size = 1; prop->min_team_size = 1; prop->max_team_size = UCC_RANK_MAX; return UCC_OK; } ucc-1.8.0/src/components/cl/basic/cl_basic.c0000664000175000017500000000511015211535620021011 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_basic.h" #include "utils/ucc_malloc.h" ucc_status_t ucc_cl_basic_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr); ucc_status_t ucc_cl_basic_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *base_attr); ucc_status_t ucc_cl_basic_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_cl_basic_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_cl_basic_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **packed_buffer); ucc_status_t ucc_cl_basic_get_lib_properties(ucc_base_lib_properties_t *prop); static ucc_config_field_t ucc_cl_basic_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_cl_basic_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_cl_lib_config_table)}, {NULL} }; static ucs_config_field_t ucc_cl_basic_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_cl_basic_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_cl_context_config_table)}, {NULL} }; UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_basic_lib_t, ucc_base_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_basic_lib_t, ucc_base_lib_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_basic_context_t, ucc_base_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_basic_context_t, ucc_base_context_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_basic_team_t, ucc_base_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_cl_basic_team_create_test(ucc_base_team_t *cl_team); ucc_status_t ucc_cl_basic_team_destroy(ucc_base_team_t *cl_team); ucc_status_t ucc_cl_basic_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); ucc_status_t ucc_cl_basic_team_get_scores(ucc_base_team_t *cl_team, ucc_coll_score_t **score); UCC_CL_IFACE_DECLARE(basic, BASIC); ucc-1.8.0/src/components/cl/basic/cl_basic_context.c0000664000175000017500000000743115211535620022565 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_basic.h" #include "utils/ucc_malloc.h" UCC_CLASS_INIT_FUNC(ucc_cl_basic_context_t, const ucc_base_context_params_t *params, const ucc_base_config_t *config) { const ucc_cl_context_config_t *cl_config = ucc_derived_of(config, ucc_cl_context_config_t); ucc_config_names_array_t *tls = &cl_config->cl_lib->tls.array; ucc_status_t status; int i; UCC_CLASS_CALL_SUPER_INIT(ucc_cl_context_t, cl_config, params->context); if (tls->count == 1 && !strcmp(tls->names[0], "all")) { tls = ¶ms->context->all_tls; } self->super.tl_ctxs = ucc_malloc(sizeof(ucc_tl_context_t*) * tls->count, "cl_basic_tl_ctxs"); if (!self->super.tl_ctxs) { cl_error(cl_config->cl_lib, "failed to allocate %zd bytes for tl_ctxs", sizeof(ucc_tl_context_t**) * tls->count); return UCC_ERR_NO_MEMORY; } self->super.n_tl_ctxs = 0; for (i = 0; i < tls->count; i++) { status = ucc_tl_context_get(params->context, tls->names[i], &self->super.tl_ctxs[self->super.n_tl_ctxs]); if (UCC_OK != status) { cl_debug(cl_config->cl_lib, "TL %s context is not available, skipping", tls->names[i]); } else { self->super.n_tl_ctxs++; } } if (0 == self->super.n_tl_ctxs) { cl_error(cl_config->cl_lib, "no TL contexts are available"); ucc_free(self->super.tl_ctxs); self->super.tl_ctxs = NULL; return UCC_ERR_NOT_FOUND; } cl_debug(cl_config->cl_lib, "initialized cl context: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_basic_context_t) { int i; cl_debug(self->super.super.lib, "finalizing cl context: %p", self); for (i = 0; i < self->super.n_tl_ctxs; i++) { ucc_tl_context_put(self->super.tl_ctxs[i]); } ucc_free(self->super.tl_ctxs); } ucc_status_t ucc_cl_basic_mem_map(const ucc_base_context_t *context, int type, /* NOLINT */ void *memh, void *tl_h) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_cl_basic_mem_unmap(const ucc_base_context_t *context, int type, /* NOLINT */ void *tl_h) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_cl_basic_memh_pack(const ucc_base_context_t *context, /* NOLINT */ int type, void *memh, void **packed_buffer) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } UCC_CLASS_DEFINE(ucc_cl_basic_context_t, ucc_cl_context_t); ucc_status_t ucc_cl_basic_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *attr) { const ucc_cl_basic_context_t *ctx = ucc_derived_of(context, ucc_cl_basic_context_t); ucc_base_ctx_attr_t tl_attr; ucc_status_t status; int i; ucc_base_ctx_attr_clear(attr); /* CL BASIC reports topo_required if any of the TL available * TL contexts needs it */ for (i = 0; i < ctx->super.n_tl_ctxs; i++) { memset(&tl_attr, 0, sizeof(tl_attr)); status = UCC_TL_CTX_IFACE(ctx->super.tl_ctxs[i]) ->context.get_attr(&ctx->super.tl_ctxs[i]->super, &tl_attr); if (UCC_OK != status) { cl_error(ctx->super.super.lib, "failed to get %s ctx attr", UCC_TL_CTX_IFACE(ctx->super.tl_ctxs[i])->super.name); return status; } if (tl_attr.topo_required) { attr->topo_required = 1; break; } } return UCC_OK; } ucc-1.8.0/src/components/cl/basic/Makefile.am0000664000175000017500000000113415211535620021144 0ustar alastairalastair# # Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # sources = \ cl_basic.h \ cl_basic.c \ cl_basic_lib.c \ cl_basic_context.c \ cl_basic_team.c \ cl_basic_coll.c module_LTLIBRARIES = libucc_cl_basic.la libucc_cl_basic_la_SOURCES = $(sources) libucc_cl_basic_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) libucc_cl_basic_la_CFLAGS = $(BASE_CFLAGS) libucc_cl_basic_la_LDFLAGS = -version-info $(SOVERSION) --as-needed libucc_cl_basic_la_LIBADD = $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am ucc-1.8.0/src/components/cl/basic/cl_basic_coll.c0000664000175000017500000000136415211535620022031 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_basic.h" #include "utils/ucc_coll_utils.h" ucc_status_t ucc_cl_basic_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_basic_team_t *cl_team = ucc_derived_of(team, ucc_cl_basic_team_t); ucc_status_t status; status = ucc_coll_init(cl_team->score_map, coll_args, task); if (UCC_ERR_NOT_FOUND == status) { cl_warn(UCC_CL_TEAM_LIB(cl_team), "no TL supporting given coll args is available"); return UCC_ERR_NOT_SUPPORTED; } return status; } ucc-1.8.0/src/components/cl/hier/0000775000175000017500000000000015211535620016757 5ustar alastairalastairucc-1.8.0/src/components/cl/hier/reduce/0000775000175000017500000000000015211535620020226 5ustar alastairalastairucc-1.8.0/src/components/cl/hier/reduce/reduce.h0000664000175000017500000000161315211535620021647 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef REDUCE_H_ #define REDUCE_H_ #include "../cl_hier.h" enum { UCC_CL_HIER_REDUCE_ALG_2STEP, UCC_CL_HIER_REDUCE_ALG_LAST, }; extern ucc_base_coll_alg_info_t ucc_cl_hier_reduce_algs[UCC_CL_HIER_REDUCE_ALG_LAST + 1]; #define UCC_CL_HIER_REDUCE_DEFAULT_ALG_SELECT_STR "reduce:0-4k:@2step" ucc_status_t ucc_cl_hier_reduce_2step_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); static inline int ucc_cl_hier_reduce_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_CL_HIER_REDUCE_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_cl_hier_reduce_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/cl/hier/reduce/reduce_2step.c0000664000175000017500000002707315211535620022767 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce.h" #include "core/ucc_team.h" #include "../cl_hier_coll.h" #define MAX_AR_2STEP_TASKS 3 static ucc_status_t ucc_cl_hier_reduce_2step_start(ucc_coll_task_t *task) { UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_reduce_2step_start", 0); return ucc_schedule_start(task); } static ucc_status_t ucc_cl_hier_reduce_2step_finalize(ucc_coll_task_t *task) { ucc_cl_hier_schedule_t *schedule = ucc_derived_of(task, ucc_cl_hier_schedule_t); ucc_status_t status; UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_reduce_2step_finalize", 0); status = ucc_schedule_finalize(task); if (schedule->scratch) { ucc_mc_free(schedule->scratch); } ucc_cl_hier_put_schedule(&schedule->super.super); return status; } static inline ucc_rank_t find_root_net_rank(ucc_host_id_t root_host_id, ucc_cl_hier_team_t *cl_team) { ucc_rank_t net_rank = UCC_RANK_INVALID; ucc_sbgp_t *sbgp = cl_team->sbgps[UCC_HIER_SBGP_NODE_LEADERS].sbgp; ucc_team_t *core_team = cl_team->super.super.params.team; ucc_rank_t i, rank; for (i = 0; i < sbgp->group_size; i++) { rank = ucc_ep_map_eval(sbgp->map, i); if (ucc_team_rank_host_id(rank, core_team) == root_host_id) { net_rank = i; break; } } return net_rank; } static inline ucc_rank_t find_root_node_rank(ucc_rank_t root, ucc_cl_hier_team_t *cl_team) { ucc_rank_t node_rank = UCC_RANK_INVALID; ucc_sbgp_t *sbgp = cl_team->sbgps[UCC_HIER_SBGP_NODE].sbgp; ucc_rank_t i; for (i = 0; i < sbgp->group_size; i++) { if (ucc_ep_map_eval(sbgp->map, i) == root) { node_rank = i; break; } } return node_rank; } static ucc_status_t ucc_cl_hier_reduce_2step_init_schedule(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_schedule_t **sched_p, int n_frags) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_team_t *core_team = team->params.team; ucc_coll_task_t *tasks[2] = {NULL, NULL}; ucc_rank_t root = coll_args->args.root; ucc_rank_t rank = UCC_TL_TEAM_RANK(cl_team); ucc_base_coll_args_t args = *coll_args; size_t count = (rank == root) ? args.args.dst.info.count : args.args.src.info.count; ucc_cl_hier_schedule_t *cl_schedule; ucc_schedule_t *schedule; ucc_status_t status; int n_tasks, i, first_task, root_on_local_node; root_on_local_node = ucc_team_ranks_on_same_node(root, rank, core_team); n_tasks = 0; first_task = 0; if (root != rank) { args.args.dst.info.count = args.args.src.info.count; args.args.dst.info.mem_type = args.args.src.info.mem_type; args.args.dst.info.datatype = args.args.src.info.datatype; args.args.mask &= (~UCC_COLL_ARGS_FLAG_IN_PLACE); } cl_schedule = ucc_cl_hier_get_schedule(cl_team); if (ucc_unlikely(!cl_schedule)) { return UCC_ERR_NO_MEMORY; } schedule = &cl_schedule->super.super; status = ucc_schedule_init(schedule, &args, team); if (ucc_unlikely(UCC_OK != status)) { goto out; } args.max_frag_count = ucc_buffer_block_count(count, n_frags, 0); if (n_frags > 1) { args.mask |= UCC_BASE_CARGS_MAX_FRAG_COUNT; } ucc_assert(SBGP_ENABLED(cl_team, NODE_LEADERS) || SBGP_ENABLED(cl_team, NODE)); if (SBGP_ENABLED(cl_team, NODE)) { args.args.root = root_on_local_node ? find_root_node_rank(root, cl_team) : 0; if ((root != rank) && SBGP_ENABLED(cl_team, NODE_LEADERS)) { status = ucc_mc_alloc(&cl_schedule->scratch, args.max_frag_count * ucc_dt_size(args.args.src.info.datatype), args.args.src.info.mem_type); if (ucc_unlikely(UCC_OK != status)) { goto out; } args.args.dst.info.buffer = cl_schedule->scratch->addr; if (root_on_local_node) { first_task = 1; args.args.src.info.buffer = cl_schedule->scratch->addr; } } status = ucc_coll_init(SCORE_MAP(cl_team, NODE), &args, &tasks[n_tasks]); if (ucc_unlikely(UCC_OK != status)) { goto out; } n_tasks++; } if (SBGP_ENABLED(cl_team, NODE_LEADERS)) { if (n_tasks == 1) { if (root != rank) { args.args.src.info.buffer = root_on_local_node ? coll_args->args.src.info.buffer : cl_schedule->scratch->addr; } else { args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; } } args.args.root = find_root_net_rank(ucc_team_rank_host_id(root, core_team), cl_team); status = ucc_coll_init(SCORE_MAP(cl_team, NODE_LEADERS), &args, &tasks[n_tasks]); if (ucc_unlikely(UCC_OK != status)) { goto out; } n_tasks++; } ucc_task_subscribe_dep(&schedule->super, tasks[first_task], UCC_EVENT_SCHEDULE_STARTED); ucc_schedule_add_task(schedule, tasks[first_task]); if (n_tasks > 1) { ucc_task_subscribe_dep(tasks[first_task], tasks[(first_task + 1) % 2], UCC_EVENT_COMPLETED); ucc_schedule_add_task(schedule, tasks[(first_task + 1) % 2]); } schedule->super.post = ucc_cl_hier_reduce_2step_start; schedule->super.progress = NULL; schedule->super.finalize = ucc_cl_hier_reduce_2step_finalize; schedule->super.triggered_post = ucc_triggered_post; *sched_p = schedule; return UCC_OK; out: for (i = 0; i < n_tasks; i++) { tasks[i]->finalize(tasks[i]); } ucc_cl_hier_put_schedule(schedule); return status; } static ucc_status_t ucc_cl_hier_reduce_2step_frag_init(ucc_base_coll_args_t *coll_args, ucc_schedule_pipelined_t *sp, ucc_base_team_t *team, ucc_schedule_t **frag_p) { int n_frags = sp->super.n_tasks; ucc_status_t status; status = ucc_cl_hier_reduce_2step_init_schedule(coll_args, team, frag_p, n_frags); return status; } static ucc_status_t ucc_cl_hier_reduce_2step_frag_setup(ucc_schedule_pipelined_t *schedule_p, ucc_schedule_t *frag, int frag_num) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(schedule_p->super.super.team, ucc_cl_hier_team_t); ucc_coll_args_t *args = &schedule_p->super.super.bargs.args; size_t dt_size = ucc_dt_size(args->src.info.datatype); int n_frags = schedule_p->super.n_tasks; ucc_rank_t root = args->root; ucc_rank_t rank = UCC_TL_TEAM_RANK(cl_team); size_t count = (rank == root) ? args->dst.info.count : args->src.info.count; size_t frag_count, frag_offset; ucc_coll_task_t *task; int i; ucc_cl_hier_schedule_t *cl_schedule; void *scratch; cl_schedule = ucc_derived_of(frag, ucc_cl_hier_schedule_t); scratch = cl_schedule->scratch ? cl_schedule->scratch->addr : NULL; frag_count = ucc_buffer_block_count(count, n_frags, frag_num); frag_offset = ucc_buffer_block_offset(count, n_frags, frag_num); for (i = 0; i < frag->n_tasks; i++) { task = frag->tasks[i]; task->bargs.args.src.info.count = frag_count; task->bargs.args.dst.info.count = frag_count; if (task->bargs.args.src.info.buffer != scratch) { task->bargs.args.src.info.buffer = PTR_OFFSET(args->src.info.buffer, frag_offset * dt_size); } if (task->bargs.args.dst.info.buffer != scratch) { task->bargs.args.dst.info.buffer = PTR_OFFSET(args->dst.info.buffer, frag_offset * dt_size); } } return UCC_OK; } static ucc_status_t ucc_cl_hier_reduce_2step_pipelined_start(ucc_coll_task_t *task) { ucc_schedule_pipelined_t *schedule = ucc_derived_of(task, ucc_schedule_pipelined_t); cl_debug(task->team->context->lib, "posting reduce_2step, count %zd, dt %s" " pdepth %d, frags_total %d", task->bargs.args.src.info.count, ucc_datatype_str(task->bargs.args.src.info.datatype), schedule->n_frags, schedule->super.n_tasks); return ucc_schedule_pipelined_post(task); } static ucc_status_t ucc_cl_hier_reduce_2step_pipelined_finalize(ucc_coll_task_t *task) { ucc_cl_hier_schedule_t *schedule = ucc_derived_of(task, ucc_cl_hier_schedule_t); ucc_status_t status; status = ucc_schedule_pipelined_finalize(&schedule->super.super.super); ucc_cl_hier_put_schedule(&schedule->super.super); return status; } UCC_CL_HIER_PROFILE_FUNC(ucc_status_t, ucc_cl_hier_reduce_2step_init, (coll_args, team, task), ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_cl_hier_lib_config_t *cfg = &UCC_CL_HIER_TEAM_LIB(cl_team)->cfg; ucc_cl_hier_schedule_t *schedule; int n_frags, pipeline_depth; ucc_status_t status; if (UCC_IS_PERSISTENT(coll_args->args) || (coll_args->args.op == UCC_OP_AVG)) { return UCC_ERR_NOT_SUPPORTED; } ucc_pipeline_nfrags_pdepth(&cfg->reduce_2step_pipeline, coll_args->args.src.info.count * ucc_dt_size(coll_args->args.src.info.datatype), &n_frags, &pipeline_depth); if (n_frags == 1) { return ucc_cl_hier_reduce_2step_init_schedule( coll_args, team, (ucc_schedule_t **)task, n_frags); } schedule = ucc_cl_hier_get_schedule(cl_team); if (ucc_unlikely(!schedule)) { return UCC_ERR_NO_MEMORY; } status = ucc_schedule_pipelined_init( coll_args, team, ucc_cl_hier_reduce_2step_frag_init, ucc_cl_hier_reduce_2step_frag_setup, pipeline_depth, n_frags, cfg->reduce_2step_pipeline.order, &schedule->super); if (ucc_unlikely(status != UCC_OK)) { cl_error(team->context->lib, "failed to init pipelined 2step ar schedule"); goto err_pipe_init; } schedule->super.super.super.post = ucc_cl_hier_reduce_2step_pipelined_start; schedule->super.super.super.finalize = ucc_cl_hier_reduce_2step_pipelined_finalize; schedule->super.super.super.triggered_post = ucc_triggered_post; *task = &schedule->super.super.super; return UCC_OK; err_pipe_init: ucc_cl_hier_put_schedule(&schedule->super.super); return status; } ucc-1.8.0/src/components/cl/hier/reduce/reduce.c0000664000175000017500000000106115211535620021637 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce.h" #include "../reduce/reduce.h" ucc_base_coll_alg_info_t ucc_cl_hier_reduce_algs[UCC_CL_HIER_REDUCE_ALG_LAST + 1] = { [UCC_CL_HIER_REDUCE_ALG_2STEP] = {.id = UCC_CL_HIER_REDUCE_ALG_2STEP, .name = "2step", .desc = "intra-node and inter-node reduces executed in parallel"}, [UCC_CL_HIER_REDUCE_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc-1.8.0/src/components/cl/hier/cl_hier_coll.h0000664000175000017500000000322615211535620021551 0ustar alastairalastair/** * Copyright (c) 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_CL_HIER_COLL_H_ #define UCC_CL_HIER_COLL_H_ #include "cl_hier.h" #include "schedule/ucc_schedule_pipelined.h" #include "components/mc/ucc_mc.h" #include "allreduce/allreduce.h" #include "alltoallv/alltoallv.h" #include "alltoall/alltoall.h" #include "barrier/barrier.h" #include "bcast/bcast.h" #include "reduce/reduce.h" #include "allgatherv/allgatherv.h" #define UCC_CL_HIER_N_DEFAULT_ALG_SELECT_STR 4 extern const char *ucc_cl_hier_default_alg_select_str[UCC_CL_HIER_N_DEFAULT_ALG_SELECT_STR]; typedef struct ucc_cl_hier_schedule_t { ucc_schedule_pipelined_t super; ucc_mc_buffer_header_t *scratch; union { struct { uint64_t *counts; } allreduce_split_rail; }; } ucc_cl_hier_schedule_t; static inline ucc_cl_hier_schedule_t * ucc_cl_hier_get_schedule(ucc_cl_hier_team_t *team) { ucc_cl_hier_context_t *ctx = UCC_CL_HIER_TEAM_CTX(team); ucc_cl_hier_schedule_t *schedule = ucc_mpool_get(&ctx->sched_mp); schedule->scratch = NULL; UCC_CL_HIER_PROFILE_REQUEST_NEW(schedule, "cl_hier_sched_p", 0); return schedule; } static inline void ucc_cl_hier_put_schedule(ucc_schedule_t *schedule) { UCC_CL_HIER_PROFILE_REQUEST_FREE(schedule); ucc_mpool_put(schedule); } ucc_status_t ucc_cl_hier_alg_id_to_init(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, //NOLINT ucc_base_coll_init_fn_t *init); #endif ucc-1.8.0/src/components/cl/hier/allgatherv/0000775000175000017500000000000015211535620021110 5ustar alastairalastairucc-1.8.0/src/components/cl/hier/allgatherv/unpack.c0000664000175000017500000002001015211535620022526 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "unpack.h" ucc_status_t ucc_cl_hier_allgatherv_unpack_finalize(ucc_coll_task_t *task) { ucc_cl_hier_schedule_t *cl_schedule = ucc_derived_of(task, ucc_cl_hier_schedule_t); ucc_mc_free(cl_schedule->scratch); ucc_cl_hier_put_schedule(&cl_schedule->super.super); return UCC_OK; } void ucc_cl_hier_allgatherv_unpack_progress(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_cl_hier_schedule_t *cl_schedule = ucc_derived_of(schedule, ucc_cl_hier_schedule_t); ucc_rank_t *n_tasks = cl_schedule->scratch->addr; ucc_ee_executor_task_t **tasks = PTR_OFFSET( cl_schedule->scratch->addr, sizeof(ucc_rank_t)); ucc_status_t st = UCC_OK; ucc_rank_t i; ucc_ee_executor_task_t *etask; for (i = 0; i < *n_tasks; i++) { etask = tasks[i]; if (etask != NULL) { st = ucc_ee_executor_task_test(etask); if (st == UCC_OK) { ucc_ee_executor_task_finalize(etask); tasks[i] = NULL; } else { if (ucc_likely(st > 0)) { st = UCC_INPROGRESS; } goto out; } } } out: schedule->super.status = st; } ucc_status_t ucc_cl_hier_allgatherv_unpack_start(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_cl_hier_team_t *cl_team = ucc_derived_of(task->team, ucc_cl_hier_team_t); ucc_rank_t team_size = UCC_CL_TEAM_SIZE(cl_team); ucc_coll_args_t *args = &task->bargs.args; ucc_ee_executor_task_args_t eargs = {0}; ucc_cl_hier_schedule_t *cl_schedule = ucc_derived_of(schedule, ucc_cl_hier_schedule_t); ucc_rank_t *n_tasks = cl_schedule->scratch->addr; ucc_ee_executor_task_t **tasks = PTR_OFFSET( cl_schedule->scratch->addr, sizeof(ucc_rank_t)); size_t src_dt_size = ucc_dt_size( args->src.info_v.datatype); size_t dst_dt_size = ucc_dt_size( args->dst.info_v.datatype); ucc_rank_t *node_leaders = NULL; ucc_sbgp_t *all_nodes = NULL; ucc_sbgp_t *node_leaders_sbgp = NULL; ucc_topo_t *topo = task->team->params.team->topo; ucc_ee_executor_t *exec; ucc_status_t status; ucc_rank_t i; size_t dst_rank_count; size_t dst_rank_disp; ucc_rank_t curr_team_rank; int n_nodes; ucc_rank_t node_leader_idx; size_t src_rank_disp; UCC_CHECK_GOTO( ucc_coll_task_get_executor(&schedule->super, &exec), out, status); eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY; *n_tasks = 0; // Get the node leaders UCC_CHECK_GOTO( ucc_topo_get_node_leaders(topo, &node_leaders), out, status); // Get the node leaders subgroup for proper rank mapping node_leaders_sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); if (!SBGP_EXISTS(cl_team, NODE_LEADERS)) { status = UCC_ERR_NOT_SUPPORTED; goto out; } UCC_CHECK_GOTO( ucc_topo_get_all_nodes(topo, &all_nodes, &n_nodes), out, status); for (i = 0; i < team_size; i++) { // Get destination count and displacement (in team rank order) dst_rank_count = ucc_coll_args_get_count( args, args->dst.info_v.counts, i); dst_rank_disp = ucc_coll_args_get_displacement( args, args->dst.info_v.displacements, i); // Find which node leader this rank belongs to ucc_rank_t leader_team_rank = node_leaders[i]; // Find the position of this leader in the node_leaders_sbgp node_leader_idx = ucc_ep_map_local_rank(node_leaders_sbgp->map, leader_team_rank); // Get source displacement from leader_displacements (passed in src.info_v) src_rank_disp = ucc_coll_args_get_displacement( args, args->src.info_v.displacements, node_leader_idx); for (ucc_rank_t j = 0; j < all_nodes[node_leader_idx].group_size; j++) { if (all_nodes[node_leader_idx].group_size == 1) { // Node sbgp wont exist for just a single rank on a node break; } else { curr_team_rank = ucc_ep_map_eval(all_nodes[node_leader_idx].map, j); if (curr_team_rank == i) { break; // We found our position } } // Add previous ranks' counts from the same node src_rank_disp += ucc_coll_args_get_count( args, args->dst.info_v.counts, curr_team_rank); } eargs.copy.src = PTR_OFFSET(args->src.info_v.buffer, src_rank_disp * src_dt_size); eargs.copy.dst = PTR_OFFSET(args->dst.info_v.buffer, dst_rank_disp * dst_dt_size); eargs.copy.len = dst_rank_count * dst_dt_size; if (eargs.copy.src != eargs.copy.dst) { UCC_CHECK_GOTO( ucc_ee_executor_task_post(exec, &eargs, &tasks[*n_tasks]), out, status); (*n_tasks)++; } } schedule->super.status = UCC_INPROGRESS; ucc_progress_queue_enqueue(cl_team->super.super.context->ucc_context->pq, task); return UCC_OK; out: return status; } ucc_status_t ucc_cl_hier_allgatherv_unpack_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_rank_t team_size = UCC_CL_TEAM_SIZE(cl_team); ucc_status_t status; ucc_schedule_t *schedule; ucc_cl_hier_schedule_t *cl_schedule; size_t scratch_size; schedule = &ucc_cl_hier_get_schedule(cl_team)->super.super; if (ucc_unlikely(!schedule)) { return UCC_ERR_NO_MEMORY; } cl_schedule = ucc_derived_of(schedule, ucc_cl_hier_schedule_t); UCC_CHECK_GOTO( ucc_schedule_init(schedule, coll_args, team), free_schedule, status); /* Holds n_tasks and n_tasks # of ucc_ee_executor_task_t pointers */ scratch_size = sizeof(ucc_rank_t) + team_size * sizeof(ucc_ee_executor_task_t*); UCC_CHECK_GOTO( ucc_mc_alloc(&cl_schedule->scratch, scratch_size, UCC_MEMORY_TYPE_HOST), free_schedule, status); schedule->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; schedule->super.post = ucc_cl_hier_allgatherv_unpack_start; schedule->super.progress = ucc_cl_hier_allgatherv_unpack_progress; schedule->super.finalize = ucc_cl_hier_allgatherv_unpack_finalize; *task_h = &schedule->super; return UCC_OK; free_schedule: ucc_cl_hier_put_schedule(schedule); return status; } ucc-1.8.0/src/components/cl/hier/allgatherv/unpack.h0000664000175000017500000000117615211535620022547 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "../cl_hier_coll.h" #include "core/ucc_team.h" ucc_status_t ucc_cl_hier_allgatherv_unpack_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_cl_hier_allgatherv_unpack_start(ucc_coll_task_t *task); void ucc_cl_hier_allgatherv_unpack_progress(ucc_coll_task_t *task); ucc_status_t ucc_cl_hier_allgatherv_unpack_finalize(ucc_coll_task_t *task); ucc-1.8.0/src/components/cl/hier/allgatherv/allgatherv.h0000775000175000017500000000167215211535620023423 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLGATHERV_H_ #define ALLGATHERV_H_ #include "../cl_hier.h" enum { UCC_CL_HIER_ALLGATHERV_ALG_GAB, UCC_CL_HIER_ALLGATHERV_ALG_LAST, }; extern ucc_base_coll_alg_info_t ucc_cl_hier_allgatherv_algs[UCC_CL_HIER_ALLGATHERV_ALG_LAST + 1]; #define UCC_CL_HIER_ALLGATHERV_DEFAULT_ALG_SELECT_STR "allgatherv:0-8m:host:@gab" ucc_status_t ucc_cl_hier_allgatherv_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); static inline int ucc_cl_hier_allgatherv_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_CL_HIER_ALLGATHERV_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_cl_hier_allgatherv_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/cl/hier/allgatherv/allgatherv.c0000775000175000017500000004345315211535620023421 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allgatherv.h" #include "unpack.h" #include "../cl_hier_coll.h" #include "core/ucc_team.h" #define MAX_ALLGATHERV_TASKS 4 ucc_base_coll_alg_info_t ucc_cl_hier_allgatherv_algs[UCC_CL_HIER_ALLGATHERV_ALG_LAST + 1] = { [UCC_CL_HIER_ALLGATHERV_ALG_GAB] = {.id = UCC_CL_HIER_ALLGATHERV_ALG_GAB, .name = "gab", .desc = "gatherv + allgatherv + bcast"}, [UCC_CL_HIER_ALLGATHERV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; static ucc_status_t ucc_cl_hier_allgatherv_start(ucc_coll_task_t *task) { UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_allgatherv_start", 0); return ucc_schedule_start(task); } static ucc_status_t ucc_cl_hier_allgatherv_finalize(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_cl_hier_schedule_t *cl_schedule = ucc_derived_of(task, ucc_cl_hier_schedule_t); ucc_status_t status; ucc_mc_free(cl_schedule->scratch); UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_allgatherv_finalize", 0); status = ucc_schedule_finalize(task); ucc_cl_hier_put_schedule(schedule); return status; } /* Return team_rank's node leader in team space */ static inline ucc_status_t find_leader_rank(ucc_base_team_t *team, ucc_rank_t team_rank, ucc_rank_t *rank_out) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_team_t *core_team = cl_team->super.super.params.team; ucc_rank_t *node_leaders = NULL; ucc_status_t status; ucc_assert(team_rank < UCC_CL_TEAM_SIZE(cl_team)); ucc_assert(SBGP_EXISTS(cl_team, NODE_LEADERS)); status = ucc_topo_get_node_leaders(core_team->topo, &node_leaders); if (UCC_OK != status) { cl_error(team->context->lib, "Could not get node leaders"); return status; } *rank_out = node_leaders[team_rank]; return UCC_OK; } /* Check if the ranks are block ordered. If they aren't, we'll need to unpack the data after the allgatherv into the right position, even if the dst buffer is contiguous */ static inline ucc_status_t is_block_ordered(ucc_cl_hier_team_t *cl_team, int *ordered) { ucc_topo_t *topo = cl_team->super.super.params.team->topo; ucc_sbgp_t *all_nodes = NULL; int is_block_ordered = 1; int n_nodes; ucc_status_t status; ucc_rank_t prev_rank, curr_rank; ucc_rank_t i, j; if (cl_team->is_block_ordered != -1) { is_block_ordered = cl_team->is_block_ordered; } else { status = ucc_topo_get_all_nodes(topo, &all_nodes, &n_nodes); if (status != UCC_OK) { return status; } // For each node, check if its ranks are consecutive in team order for (i = 0; i < n_nodes; i++) { if (all_nodes[i].status == UCC_SBGP_ENABLED && all_nodes[i].group_size > 1) { // Get first rank in this node prev_rank = ucc_ep_map_eval(all_nodes[i].map, 0); // Check if all other ranks are consecutive for (j = 1; j < all_nodes[i].group_size; j++) { curr_rank = ucc_ep_map_eval(all_nodes[i].map, j); // If ranks are not consecutive, not block ordered if (curr_rank != prev_rank + 1) { is_block_ordered = 0; goto break_outer; } prev_rank = curr_rank; } } } break_outer: // Save for next time cl_team->is_block_ordered = is_block_ordered; } *ordered = is_block_ordered; return UCC_OK; } UCC_CL_HIER_PROFILE_FUNC(ucc_status_t, ucc_cl_hier_allgatherv_init, (coll_args, team, task), ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_coll_task_t *tasks[MAX_ALLGATHERV_TASKS] = {NULL}; ucc_rank_t rank = UCC_CL_TEAM_RANK(cl_team); ucc_rank_t node_sbgp_size = SBGP_SIZE(cl_team, NODE); ucc_rank_t leader_sbgp_size = SBGP_SIZE(cl_team, NODE_LEADERS); ucc_rank_t team_size = UCC_CL_TEAM_SIZE(cl_team); ucc_topo_t *topo = team->params.team->topo; ucc_aint_t *node_disps = NULL; ucc_count_t *node_counts = NULL; ucc_aint_t *leader_disps = NULL; ucc_count_t *leader_counts = NULL; size_t dt_size = ucc_dt_size(coll_args->args. dst.info_v.datatype); int in_place = 0; int is_contig = 1; size_t disp_counter = 0; ucc_schedule_t *schedule; ucc_cl_hier_schedule_t *cl_schedule; ucc_status_t status; ucc_base_coll_args_t args, args_old; int n_tasks, i; size_t scratch_size; size_t node_counts_size; size_t node_disps_size; size_t leader_counts_size; size_t leader_disps_size; size_t total_count; void *buffer; void *node_gathered_data; ucc_rank_t leader_team_rank; ucc_rank_t leader_sbgp_rank; ucc_rank_t team_rank; size_t leader_old_count; size_t add_count, new_count; int block_ordered; int ldr_sbgp_only; memcpy(&args, coll_args, sizeof(args)); memcpy(&args_old, coll_args, sizeof(args)); in_place = UCC_IS_INPLACE(args.args); if (args.args.dst.info_v.mem_type != UCC_MEMORY_TYPE_HOST || (!in_place && args.args.src.info.mem_type != UCC_MEMORY_TYPE_HOST) || UCC_IS_PERSISTENT(args.args)) { return UCC_ERR_NOT_SUPPORTED; } schedule = &ucc_cl_hier_get_schedule(cl_team)->super.super; if (ucc_unlikely(!schedule)) { return UCC_ERR_NO_MEMORY; } cl_schedule = ucc_derived_of(schedule, ucc_cl_hier_schedule_t); n_tasks = 0; UCC_CHECK_GOTO(is_block_ordered(cl_team, &block_ordered), free_sched, status); is_contig = block_ordered && ucc_coll_args_is_disp_contig(&args.args, team_size); /* handle the case where this rank may be the only one on this node */ ldr_sbgp_only = !SBGP_ENABLED(cl_team, NODE) && SBGP_ENABLED(cl_team, NODE_LEADERS); UCC_CHECK_GOTO(ucc_schedule_init(schedule, &args, team), free_sched, status); node_counts_size = node_sbgp_size * sizeof(ucc_count_t); node_disps_size = node_sbgp_size * sizeof(ucc_aint_t); leader_counts_size = leader_sbgp_size * sizeof(ucc_count_t); leader_disps_size = leader_sbgp_size * sizeof(ucc_aint_t); total_count = ucc_coll_args_get_total_count(&args.args, args.args.dst.info_v.counts, team_size); scratch_size = node_counts_size + node_disps_size + leader_counts_size + leader_disps_size; /* If the dst buf isn't contig, allocate and work on a contig scratch buffer */ scratch_size += (is_contig ? 0 : (total_count * dt_size)); UCC_CHECK_GOTO( ucc_mc_alloc(&cl_schedule->scratch, scratch_size, UCC_MEMORY_TYPE_HOST), free_sched, status); memset(cl_schedule->scratch->addr, 0, scratch_size); node_counts = PTR_OFFSET(cl_schedule->scratch->addr, 0); node_disps = PTR_OFFSET(node_counts, node_counts_size); leader_counts = PTR_OFFSET(node_disps, node_disps_size); leader_disps = PTR_OFFSET(leader_counts, leader_counts_size); if (is_contig) { buffer = args.args.dst.info_v.buffer; } else { buffer = PTR_OFFSET(leader_disps, leader_disps_size); } node_gathered_data = NULL; /* If node ldr sbgp exists, calculate leader_counts, leader_disps, and set the dst buffer of the gatherv to the right displacements for the in-place node-leader allgatherv. Calculate this on non-node-leader ranks as well for the unpack phase */ if((SBGP_ENABLED(cl_team, NODE) && SBGP_EXISTS(cl_team, NODE_LEADERS)) || ldr_sbgp_only) { /* Sum up the counts on each node to get the count for each node leader */ for (i = 0; i < team_size; i++) { UCC_CHECK_GOTO( find_leader_rank(team, i, &leader_team_rank), free_scratch, status); leader_sbgp_rank = ucc_ep_map_local_rank( SBGP_MAP(cl_team, NODE_LEADERS), leader_team_rank); leader_old_count = ucc_coll_args_get_count( &args.args, leader_counts, leader_sbgp_rank); add_count = ucc_coll_args_get_count( &args.args, args.args.dst.info_v.counts, i); new_count = add_count + leader_old_count; ucc_coll_args_set_count(&args.args, leader_counts, leader_sbgp_rank, new_count); } /* Calculate leader_disps by adding each count to disp_counter to make a contiguous chunk */ disp_counter = 0; for (i = 0; i < leader_sbgp_size; i++) { ucc_coll_args_set_displacement(&args.args, leader_disps, i, disp_counter); disp_counter += ucc_coll_args_get_count(&args.args, leader_counts, i); } if (SBGP_ENABLED(cl_team, NODE_LEADERS)) { node_gathered_data = PTR_OFFSET(buffer, dt_size * ucc_coll_args_get_displacement( &args.args, leader_disps, SBGP_RANK(cl_team, NODE_LEADERS)) ); } } if (SBGP_ENABLED(cl_team, NODE)) { ucc_assert(n_tasks == 0); if (cl_team->top_sbgp == UCC_HIER_SBGP_NODE) { args.args.coll_type = UCC_COLL_TYPE_ALLGATHERV; } else { disp_counter = 0; for (i = 0; i < node_sbgp_size; i++) { team_rank = ucc_ep_map_eval(SBGP_MAP(cl_team, NODE), i); ucc_coll_args_set_count( &args.args, node_counts, i, ucc_coll_args_get_count(&args.args, args.args.dst.info_v.counts, team_rank)); ucc_coll_args_set_displacement(&args.args, node_disps, i, disp_counter); disp_counter += ucc_coll_args_get_count(&args.args, node_counts, i); } if (in_place) { args.args.src.info.buffer = PTR_OFFSET(args.args.dst.info_v.buffer, dt_size * ucc_coll_args_get_displacement( &args.args, args.args.dst.info_v.displacements, rank)); args.args.src.info.count = ucc_coll_args_get_count(&args.args, args.args.dst.info_v.counts, rank); args.args.src.info.datatype = args.args.dst.info_v.datatype; args.args.src.info.mem_type = args.args.dst.info_v.mem_type; } args.args.coll_type = UCC_COLL_TYPE_GATHERV; args.args.root = topo->node_leader_rank_id; args.args.flags &= ~UCC_COLL_ARGS_FLAG_IN_PLACE; args.args.dst.info_v.displacements = node_disps; args.args.dst.info_v.counts = node_counts; args.args.dst.info_v.buffer = node_gathered_data; /* Indicate to gatherv it can read the dst counts on nonroots */ args.mask |= UCC_BASE_CARGS_NONROOT_INFO; } UCC_CHECK_GOTO( ucc_coll_init(SCORE_MAP(cl_team, NODE), &args, &tasks[n_tasks]), free_scratch, status); n_tasks++; } args = args_old; /* Need to pack in case it's not inplace or the buf isnt contig and we didnt do the gatherv */ if (ldr_sbgp_only) { /* Linter assumes true for ldr_sbgp_only, but takes false branch for if (SBGP_ENABLED(cl_team, NODE_LEADERS)) node_gathered_data = ... */ ucc_assert(node_gathered_data != NULL); if (!in_place) { //NOLINTNEXTLINE memcpy(node_gathered_data, args.args.src.info.buffer, args.args.src.info.count * ucc_dt_size(args.args.src.info.datatype)); } else if (!is_contig) { //NOLINTNEXTLINE memcpy(node_gathered_data, PTR_OFFSET(args.args.dst.info_v.buffer, dt_size * ucc_coll_args_get_displacement(&args.args, args.args.dst.info_v.displacements, rank)), dt_size * ucc_coll_args_get_count(&args.args, args.args.dst.info_v.counts, rank)); } } args = args_old; if (SBGP_ENABLED(cl_team, NODE_LEADERS)) { ucc_assert(cl_team->top_sbgp == UCC_HIER_SBGP_NODE_LEADERS); args.args.coll_type = UCC_COLL_TYPE_ALLGATHERV; args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; args.args.flags |= UCC_COLL_ARGS_FLAG_CONTIG_DST_BUFFER; args.args.dst.info_v.buffer = buffer; args.args.dst.info_v.displacements = leader_disps; args.args.dst.info_v.counts = leader_counts; UCC_CHECK_GOTO(ucc_coll_init(SCORE_MAP(cl_team, NODE_LEADERS), &args, &tasks[n_tasks]), free_scratch, status); n_tasks++; } if (cl_team->top_sbgp != UCC_HIER_SBGP_NODE) { args = args_old; if (SBGP_ENABLED(cl_team, NODE)) { args.args.coll_type = UCC_COLL_TYPE_BCAST; args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; args.args.root = topo->node_leader_rank_id; args.args.src.info.buffer = buffer; args.args.src.info.count = total_count; args.args.src.info.datatype = args_old.args.dst.info_v.datatype; args.args.src.info.mem_type = args_old.args.dst.info_v.mem_type; /* If using tl_shm and the shmem segment size is less than total_count, this node-level bcast will cause the allgatherv to fail and fall back */ UCC_CHECK_GOTO( ucc_coll_init(SCORE_MAP(cl_team, NODE), &args, &tasks[n_tasks]), free_scratch, status); n_tasks++; } if (!is_contig) { args = args_old; args.args.src.info_v.datatype = args.args.dst.info_v.datatype; args.args.src.info_v.mem_type = args.args.dst.info_v.mem_type; args.args.src.info_v.buffer = buffer; // Pass leader_disps and leader_counts through src.info_v for unpack args.args.src.info_v.displacements = leader_disps; args.args.src.info_v.counts = leader_counts; UCC_CHECK_GOTO( ucc_cl_hier_allgatherv_unpack_init(&args, team, &tasks[n_tasks]), free_scratch, status); n_tasks++; } } UCC_CHECK_GOTO(ucc_event_manager_subscribe( &schedule->super, UCC_EVENT_SCHEDULE_STARTED, tasks[0], ucc_task_start_handler), free_scratch, status); UCC_CHECK_GOTO( ucc_schedule_add_task(schedule, tasks[0]), free_scratch, status); for (i = 1; i < n_tasks; i++) { UCC_CHECK_GOTO( ucc_event_manager_subscribe(tasks[i - 1], UCC_EVENT_COMPLETED, tasks[i], ucc_task_start_handler), free_scratch, status); UCC_CHECK_GOTO( ucc_schedule_add_task(schedule, tasks[i]), free_scratch, status); } schedule->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; schedule->super.post = ucc_cl_hier_allgatherv_start; schedule->super.finalize = ucc_cl_hier_allgatherv_finalize; *task = &schedule->super; return UCC_OK; free_scratch: ucc_mc_free(cl_schedule->scratch); free_sched: for (i = 0; i < n_tasks; i++) { tasks[i]->finalize(tasks[i]); } ucc_cl_hier_put_schedule(schedule); cl_error(team->context->lib, "failed to init cl hier allgatherv"); return status; } ucc-1.8.0/src/components/cl/hier/cl_hier.c0000664000175000017500000001230015211535620020524 0ustar alastairalastair/** * Copyright (c) 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_hier.h" #include "utils/ucc_malloc.h" #include "allreduce/allreduce.h" #include "alltoall/alltoall.h" #include "alltoallv/alltoallv.h" ucc_status_t ucc_cl_hier_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr); ucc_status_t ucc_cl_hier_get_lib_properties(ucc_base_lib_properties_t *prop); ucc_status_t ucc_cl_hier_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *base_attr); ucc_status_t ucc_cl_hier_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_cl_hier_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_cl_hier_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **packed_buffer); static ucc_config_field_t ucc_cl_hier_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_cl_hier_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_cl_lib_config_table)}, {"NODE_SBGP_TLS", "ucp", "TLS to be used for NODE subgroup.\n" "NODE subgroup contains processes of a team located on the same node", ucc_offsetof(ucc_cl_hier_lib_config_t, sbgp_tls[UCC_HIER_SBGP_NODE]), UCC_CONFIG_TYPE_ALLOW_LIST}, {"NODE_LEADERS_SBGP_TLS", "ucp", "TLS to be used for NODE_LEADERS subgroup.\n" "NODE_LEADERS subgroup contains processes of a team with local node rank " "equal 0", ucc_offsetof(ucc_cl_hier_lib_config_t, sbgp_tls[UCC_HIER_SBGP_NODE_LEADERS]), UCC_CONFIG_TYPE_ALLOW_LIST}, {"NET_SBGP_TLS", "ucp", "TLS to be used for NET subgroup.\n" "NET subgroup contains processes of a team with identical local node " "rank.\n" "This subgroup only exists for teams with equal number of processes " "across the nodes", ucc_offsetof(ucc_cl_hier_lib_config_t, sbgp_tls[UCC_HIER_SBGP_NET]), UCC_CONFIG_TYPE_ALLOW_LIST}, {"FULL_SBGP_TLS", "ucp", "TLS to be used for FULL subgroup.\n" "FULL subgroup contains all processes of the team", ucc_offsetof(ucc_cl_hier_lib_config_t, sbgp_tls[UCC_HIER_SBGP_FULL]), UCC_CONFIG_TYPE_ALLOW_LIST}, {"ALLTOALLV_SPLIT_NODE_THRESH", "0", "Messages larger than that threshold will be sent via node sbgp tl", ucc_offsetof(ucc_cl_hier_lib_config_t, a2av_node_thresh), UCC_CONFIG_TYPE_MEMUNITS}, {"ALLREDUCE_SPLIT_RAIL_PIPELINE", "n", "Pipelining settings for SplitRail allreduce algorithm", ucc_offsetof(ucc_cl_hier_lib_config_t, allreduce_split_rail_pipeline), UCC_CONFIG_TYPE_PIPELINE_PARAMS}, {"ALLREDUCE_RAB_PIPELINE", "n", "Pipelining settings for RAB allreduce algorithm", ucc_offsetof(ucc_cl_hier_lib_config_t, allreduce_rab_pipeline), UCC_CONFIG_TYPE_PIPELINE_PARAMS}, {"BCAST_2STEP_PIPELINE", "n", "Pipelining settings for RAB bcast algorithm", ucc_offsetof(ucc_cl_hier_lib_config_t, bcast_2step_pipeline), UCC_CONFIG_TYPE_PIPELINE_PARAMS}, {"REDUCE_2STEP_PIPELINE", "n", "Pipelining settings for RAB reduce algorithm", ucc_offsetof(ucc_cl_hier_lib_config_t, reduce_2step_pipeline), UCC_CONFIG_TYPE_PIPELINE_PARAMS}, {NULL}}; static ucs_config_field_t ucc_cl_hier_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_cl_hier_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_cl_context_config_table)}, {NULL}}; UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_hier_lib_t, ucc_base_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_hier_lib_t, ucc_base_lib_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_hier_context_t, ucc_base_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_hier_context_t, ucc_base_context_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_cl_hier_team_t, ucc_base_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_cl_hier_team_create_test(ucc_base_team_t *cl_team); ucc_status_t ucc_cl_hier_team_destroy(ucc_base_team_t *cl_team); ucc_status_t ucc_cl_hier_team_get_scores(ucc_base_team_t *cl_team, ucc_coll_score_t **score); UCC_CL_IFACE_DECLARE(hier, HIER); __attribute__((constructor)) static void cl_hier_iface_init(void) { ucc_cl_hier.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLREDUCE)] = ucc_cl_hier_allreduce_algs; ucc_cl_hier.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLTOALL)] = ucc_cl_hier_alltoall_algs; ucc_cl_hier.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLTOALLV)] = ucc_cl_hier_alltoallv_algs; ucc_cl_hier.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_BCAST)] = ucc_cl_hier_bcast_algs; ucc_cl_hier.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLGATHERV)] = ucc_cl_hier_allgatherv_algs; } ucc-1.8.0/src/components/cl/hier/cl_hier.h0000664000175000017500000001340615211535620020541 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #ifndef UCC_CL_HIER_H_ #define UCC_CL_HIER_H_ #include "components/cl/ucc_cl.h" #include "components/cl/ucc_cl_log.h" #include "components/tl/ucc_tl.h" #include "coll_score/ucc_coll_score.h" #include "utils/ucc_mpool.h" #include "schedule/ucc_schedule_pipelined.h" #ifdef HAVE_PROFILING_CL_HIER #include "utils/profile/ucc_profile_on.h" #else #include "utils/profile/ucc_profile_off.h" #endif #define UCC_CL_HIER_PROFILE_FUNC UCC_PROFILE_FUNC #define UCC_CL_HIER_PROFILE_REQUEST_NEW UCC_PROFILE_REQUEST_NEW #define UCC_CL_HIER_PROFILE_REQUEST_EVENT UCC_PROFILE_REQUEST_EVENT #define UCC_CL_HIER_PROFILE_REQUEST_FREE UCC_PROFILE_REQUEST_FREE #ifndef UCC_CL_HIER_DEFAULT_SCORE #define UCC_CL_HIER_DEFAULT_SCORE 50 #endif typedef struct ucc_cl_hier_iface { ucc_cl_iface_t super; } ucc_cl_hier_iface_t; /* Extern iface should follow the pattern: ucc_cl_ */ extern ucc_cl_hier_iface_t ucc_cl_hier; typedef enum { UCC_HIER_SBGP_NODE, UCC_HIER_SBGP_NODE_LEADERS, UCC_HIER_SBGP_NET, UCC_HIER_SBGP_FULL, UCC_HIER_SBGP_LAST, } ucc_hier_sbgp_type_t; //DO we need it? Potential use case: different hier sbgps over same sbgp typedef struct ucc_cl_hier_lib_config { ucc_cl_lib_config_t super; /* List of TLs corresponding to the sbgp team, which are selected based on the TL scores */ ucc_config_names_list_t sbgp_tls[UCC_HIER_SBGP_LAST]; size_t a2av_node_thresh; ucc_pipeline_params_t allreduce_split_rail_pipeline; ucc_pipeline_params_t allreduce_rab_pipeline; ucc_pipeline_params_t bcast_2step_pipeline; ucc_pipeline_params_t reduce_2step_pipeline; } ucc_cl_hier_lib_config_t; typedef struct ucc_cl_hier_context_config { ucc_cl_context_config_t super; } ucc_cl_hier_context_config_t; typedef struct ucc_cl_hier_lib { ucc_cl_lib_t super; ucc_cl_hier_lib_config_t cfg; ucc_config_allow_list_t tls; /*< Intersection of UCC_CL_HIER_TLS vs sbgp_tls */ } ucc_cl_hier_lib_t; UCC_CLASS_DECLARE(ucc_cl_hier_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); typedef struct ucc_cl_hier_context { ucc_cl_context_t super; ucc_mpool_t sched_mp; } ucc_cl_hier_context_t; UCC_CLASS_DECLARE(ucc_cl_hier_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); typedef enum { UCC_HIER_SBGP_DISABLED, UCC_HIER_SBGP_ENABLED } ucc_hier_sbgp_state_t; #define CL_HIER_MAX_SBGP_TLS 4 /* This struct represents a unit of hierarchy: a combination of (i) subgroup (sbgp) of processes from the original ucc_team and (ii) communication TL backend initialized for that subgroup. The backend is a superposition of at least one or several TL teams. Their selection is stored in score_map. */ typedef struct ucc_hier_sbgp { ucc_hier_sbgp_state_t state; ucc_sbgp_type_t sbgp_type; ucc_sbgp_t *sbgp; ucc_score_map_t *score_map; ucc_coll_score_t *score; ucc_tl_team_t *tl_teams[CL_HIER_MAX_SBGP_TLS]; ucc_tl_context_t *tl_ctxs[CL_HIER_MAX_SBGP_TLS]; int n_tls; } ucc_hier_sbgp_t; typedef struct ucc_cl_hier_team { ucc_cl_team_t super; ucc_team_multiple_req_t *team_create_req; unsigned n_tl_teams; ucc_coll_score_t *score; ucc_hier_sbgp_t sbgps[UCC_HIER_SBGP_LAST]; ucc_hier_sbgp_type_t top_sbgp; int is_block_ordered; } ucc_cl_hier_team_t; UCC_CLASS_DECLARE(ucc_cl_hier_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); #define UCC_CL_HIER_SUPPORTED_COLLS \ (UCC_COLL_TYPE_ALLTOALL | \ UCC_COLL_TYPE_ALLTOALLV | \ UCC_COLL_TYPE_ALLGATHERV | \ UCC_COLL_TYPE_ALLREDUCE | \ UCC_COLL_TYPE_BARRIER | \ UCC_COLL_TYPE_BCAST | \ UCC_COLL_TYPE_REDUCE) ucc_status_t ucc_cl_hier_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); #define UCC_CL_HIER_TEAM_CTX(_team) \ (ucc_derived_of((_team)->super.super.context, ucc_cl_hier_context_t)) #define UCC_CL_HIER_TEAM_LIB(_team) \ (ucc_derived_of((_team)->super.super.context->lib, ucc_cl_hier_lib_t)) #define SBGP_ENABLED(_team, _sbgp) \ ((_team)->sbgps[UCC_HIER_SBGP_##_sbgp].state == UCC_HIER_SBGP_ENABLED) #define SBGP_RANK(_team, _sbgp) \ ((_team)->sbgps[UCC_HIER_SBGP_##_sbgp].sbgp->group_rank) #define SBGP_SIZE(_team, _sbgp) \ ((_team)->sbgps[UCC_HIER_SBGP_##_sbgp].sbgp->group_size) #define SBGP_MAP(_team, _sbgp) \ ((_team)->sbgps[UCC_HIER_SBGP_##_sbgp].sbgp->map) #define SBGP_EXISTS(_team, _sbgp) \ ((NULL != (_team)->sbgps[UCC_HIER_SBGP_##_sbgp].sbgp) && \ ((_team)->sbgps[UCC_HIER_SBGP_##_sbgp].sbgp->status != \ UCC_SBGP_NOT_EXISTS)) #define SCORE_MAP(_team, _sbgp) (_team)->sbgps[UCC_HIER_SBGP_##_sbgp].score_map #endif ucc-1.8.0/src/components/cl/hier/cl_hier_coll.c0000664000175000017500000001100615211535620021537 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_hier.h" #include "components/mc/ucc_mc.h" #include "core/ucc_team.h" #include "utils/ucc_coll_utils.h" #include "cl_hier_coll.h" const char * ucc_cl_hier_default_alg_select_str[UCC_CL_HIER_N_DEFAULT_ALG_SELECT_STR] = { UCC_CL_HIER_ALLREDUCE_DEFAULT_ALG_SELECT_STR, UCC_CL_HIER_BCAST_DEFAULT_ALG_SELECT_STR, UCC_CL_HIER_REDUCE_DEFAULT_ALG_SELECT_STR, UCC_CL_HIER_ALLGATHERV_DEFAULT_ALG_SELECT_STR}; ucc_status_t ucc_cl_hier_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { switch (coll_args->args.coll_type) { case UCC_COLL_TYPE_ALLREDUCE: return ucc_cl_hier_allreduce_rab_init(coll_args, team, task); case UCC_COLL_TYPE_ALLTOALL: return ucc_cl_hier_alltoall_init(coll_args, team, task); case UCC_COLL_TYPE_ALLTOALLV: return ucc_cl_hier_alltoallv_init(coll_args, team, task); case UCC_COLL_TYPE_BARRIER: return ucc_cl_hier_barrier_init(coll_args, team, task); case UCC_COLL_TYPE_BCAST: return ucc_cl_hier_bcast_2step_init(coll_args, team, task); case UCC_COLL_TYPE_REDUCE: return ucc_cl_hier_reduce_2step_init(coll_args, team, task); default: cl_error(team->context->lib, "coll_type %s is not supported", ucc_coll_type_str(coll_args->args.coll_type)); break; } return UCC_ERR_NOT_SUPPORTED; } static inline int alg_id_from_str(ucc_coll_type_t coll_type, const char *str) { switch (coll_type) { case UCC_COLL_TYPE_ALLREDUCE: return ucc_cl_hier_allreduce_alg_from_str(str); case UCC_COLL_TYPE_ALLTOALLV: return ucc_cl_hier_alltoallv_alg_from_str(str); case UCC_COLL_TYPE_ALLTOALL: return ucc_cl_hier_alltoall_alg_from_str(str); case UCC_COLL_TYPE_BCAST: return ucc_cl_hier_bcast_alg_from_str(str); case UCC_COLL_TYPE_REDUCE: return ucc_cl_hier_reduce_alg_from_str(str); case UCC_COLL_TYPE_ALLGATHERV: return ucc_cl_hier_allgatherv_alg_from_str(str); default: break; } return -1; } ucc_status_t ucc_cl_hier_alg_id_to_init(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, //NOLINT ucc_base_coll_init_fn_t *init) { ucc_status_t status = UCC_OK; if (alg_id_str) { alg_id = alg_id_from_str(coll_type, alg_id_str); } switch (coll_type) { case UCC_COLL_TYPE_ALLREDUCE: switch (alg_id) { case UCC_CL_HIER_ALLREDUCE_ALG_RAB: *init = ucc_cl_hier_allreduce_rab_init; break; case UCC_CL_HIER_ALLREDUCE_ALG_SPLIT_RAIL: *init = ucc_cl_hier_allreduce_split_rail_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_ALLTOALLV: switch (alg_id) { case UCC_CL_HIER_ALLTOALLV_ALG_NODE_SPLIT: *init = ucc_cl_hier_alltoallv_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_ALLTOALL: switch (alg_id) { case UCC_CL_HIER_ALLTOALL_ALG_NODE_SPLIT: *init = ucc_cl_hier_alltoall_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_BCAST: switch (alg_id) { case UCC_CL_HIER_BCAST_ALG_2STEP: *init = ucc_cl_hier_bcast_2step_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_REDUCE: switch(alg_id) { case UCC_CL_HIER_REDUCE_ALG_2STEP: *init = ucc_cl_hier_reduce_2step_init; break; default: status = UCC_ERR_INVALID_PARAM; break; } break; case UCC_COLL_TYPE_ALLGATHERV: switch(alg_id) { case UCC_CL_HIER_ALLGATHERV_ALG_GAB: *init = ucc_cl_hier_allgatherv_init; break; default: status = UCC_ERR_INVALID_PARAM; break; } break; default: status = UCC_ERR_NOT_SUPPORTED; break; } return status; } ucc-1.8.0/src/components/cl/hier/cl_hier_context.c0000664000175000017500000000730315211535620022277 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_hier.h" #include "utils/ucc_malloc.h" #include "utils/arch/cpu.h" #include "cl_hier_coll.h" UCC_CLASS_INIT_FUNC(ucc_cl_hier_context_t, const ucc_base_context_params_t *params, const ucc_base_config_t *config) { const ucc_cl_context_config_t *cl_config = ucc_derived_of(config, ucc_cl_context_config_t); ucc_cl_hier_lib_t *lib = ucc_derived_of(cl_config->cl_lib, ucc_cl_hier_lib_t); ucc_config_names_array_t *tls = &lib->tls.array; ucc_status_t status; int i; UCC_CLASS_CALL_SUPER_INIT(ucc_cl_context_t, cl_config, params->context); if (tls->count == 1 && !strcmp(tls->names[0], "all")) { tls = ¶ms->context->all_tls; } self->super.tl_ctxs = ucc_malloc(sizeof(ucc_tl_context_t *) * tls->count, "cl_hier_tl_ctxs"); if (!self->super.tl_ctxs) { cl_error(cl_config->cl_lib, "failed to allocate %zd bytes for tl_ctxs", sizeof(ucc_tl_context_t **) * tls->count); return UCC_ERR_NO_MEMORY; } self->super.n_tl_ctxs = 0; for (i = 0; i < tls->count; i++) { status = ucc_tl_context_get(params->context, tls->names[i], &self->super.tl_ctxs[self->super.n_tl_ctxs]); if (UCC_OK != status) { cl_debug(cl_config->cl_lib, "TL %s context is not available, skipping", tls->names[i]); } else { self->super.n_tl_ctxs++; } } if (0 == self->super.n_tl_ctxs) { cl_error(cl_config->cl_lib, "no TL contexts are available"); status = UCC_ERR_NOT_FOUND; goto out; } status = ucc_mpool_init(&self->sched_mp, 0, sizeof(ucc_cl_hier_schedule_t), 0, UCC_CACHE_LINE_SIZE, 2, UINT_MAX, &ucc_coll_task_mpool_ops, params->thread_mode, "cl_hier_sched_mp"); if (UCC_OK != status) { cl_error(cl_config->cl_lib, "failed to initialize cl_hier_sched mpool"); goto out; } cl_debug(cl_config->cl_lib, "initialized cl context: %p", self); return UCC_OK; out: ucc_free(self->super.tl_ctxs); return status; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_hier_context_t) { int i; cl_debug(self->super.super.lib, "finalizing cl context: %p", self); ucc_mpool_cleanup(&self->sched_mp, 1); for (i = 0; i < self->super.n_tl_ctxs; i++) { ucc_tl_context_put(self->super.tl_ctxs[i]); } ucc_free(self->super.tl_ctxs); } UCC_CLASS_DEFINE(ucc_cl_hier_context_t, ucc_cl_context_t); ucc_status_t ucc_cl_hier_get_context_attr(const ucc_base_context_t *context, /* NOLINT */ ucc_base_ctx_attr_t *attr) { ucc_base_ctx_attr_clear(attr); attr->topo_required = 1; return UCC_OK; } /* NOLINTBEGIN */ ucc_status_t ucc_cl_hier_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h) { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_cl_hier_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h) { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_cl_hier_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **packed_buffer) { return UCC_ERR_NOT_SUPPORTED; } /* NOLINTEND */ ucc-1.8.0/src/components/cl/hier/alltoallv/0000775000175000017500000000000015211535620020751 5ustar alastairalastairucc-1.8.0/src/components/cl/hier/alltoallv/alltoallv.c0000664000175000017500000003151515211535620023114 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "alltoallv.h" #include "../cl_hier_coll.h" #include "core/ucc_team.h" ucc_base_coll_alg_info_t ucc_cl_hier_alltoallv_algs[UCC_CL_HIER_ALLTOALLV_ALG_LAST + 1] = { [UCC_CL_HIER_ALLTOALLV_ALG_NODE_SPLIT] = {.id = UCC_CL_HIER_ALLTOALLV_ALG_NODE_SPLIT, .name = "node_split", .desc = "splitting alltoallv into two concurrent a2av calls" " within the node and outside of it"}, [UCC_CL_HIER_ALLTOALLV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; static ucc_status_t ucc_cl_hier_alltoallv_start(ucc_coll_task_t *task) { UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_alltoallv_start", 0); return ucc_schedule_start(task); } static ucc_status_t ucc_cl_hier_alltoallv_finalize(ucc_coll_task_t *task) { ucc_cl_hier_schedule_t *schedule = ucc_derived_of(task, ucc_cl_hier_schedule_t); ucc_status_t status; UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_alltoallv_finalize", 0); ucc_assert(schedule->super.super.n_tasks == 1 || schedule->super.super.n_tasks == 2); if (schedule->scratch) { ucc_mc_free(schedule->scratch); } status = ucc_schedule_finalize(task); ucc_cl_hier_put_schedule(&schedule->super.super); return status; } #define SET_FULL_COUNTS(_type, _sbgp, _coll_args, _team, _node_thresh, \ _sdt_size, _rdt_size, _sc_full, _sd_full, _rc_full, \ _rd_full) \ do { \ int _i, _is_local; \ _type _scount, _rcount; \ \ for (_i = 0; _i < (_sbgp)->group_size; _i++) { \ _scount = ((_type *)(_coll_args)->args.src.info_v.counts)[_i]; \ _rcount = ((_type *)(_coll_args)->args.dst.info_v.counts)[_i]; \ _is_local = \ ucc_rank_on_local_node(_i, (_team)->params.team->topo); \ if ((_scount * _sdt_size > (_node_thresh)) && _is_local) { \ ((_type *)_sc_full)[_i] = 0; \ } else { \ ((_type *)_sc_full)[_i] = _scount; \ ((_type *)_sd_full)[_i] = \ ((_type *)(_coll_args) \ ->args.src.info_v.displacements)[_i]; \ } \ if ((_rcount * _rdt_size > (_node_thresh)) && _is_local) { \ ((_type *)_rc_full)[_i] = 0; \ } else { \ ((_type *)_rc_full)[_i] = _rcount; \ ((_type *)_rd_full)[_i] = \ ((_type *)(_coll_args) \ ->args.dst.info_v.displacements)[_i]; \ } \ } \ } while (0) #define SET_NODE_COUNTS(_type, _sbgp, _coll_args, _node_thresh, _sdt_size, \ _rdt_size, _sc_node, _sd_node, _rc_node, _rd_node) \ do { \ int _i; \ _type _scount, _rcount; \ \ for (_i = 0; _i < (_sbgp)->group_size; _i++) { \ ucc_rank_t r = ucc_ep_map_eval((_sbgp)->map, _i); \ _scount = ((_type *)(_coll_args)->args.src.info_v.counts)[r]; \ _rcount = ((_type *)(_coll_args)->args.dst.info_v.counts)[r]; \ if (_scount * _sdt_size <= (_node_thresh)) { \ ((_type *)_sc_node)[_i] = 0; \ } else { \ ((_type *)_sc_node)[_i] = _scount; \ ((_type *)_sd_node)[_i] = \ ((_type *)(_coll_args)->args.src.info_v.displacements)[r]; \ } \ if (_rcount * _rdt_size <= (_node_thresh)) { \ ((_type *)_rc_node)[_i] = 0; \ } else { \ ((_type *)_rc_node)[_i] = _rcount; \ ((_type *)_rd_node)[_i] = \ ((_type *)(_coll_args)->args.dst.info_v.displacements)[r]; \ } \ } \ } while (0) ucc_status_t ucc_cl_hier_alltoallv_triggered_post_setup(ucc_coll_task_t *task) { ucc_cl_hier_schedule_t *schedule = ucc_derived_of(task, ucc_cl_hier_schedule_t); ucc_status_t status = UCC_OK; int n_tasks = schedule->super.super.n_tasks; int i = 0; for (i = 0; i < n_tasks; ++i) { ucc_coll_task_t *sub_task = schedule->super.super.tasks[i]; if (sub_task->triggered_post_setup != NULL) { sub_task->ee = task->ee; sub_task->triggered_post_setup(sub_task); } } return status; } UCC_CL_HIER_PROFILE_FUNC(ucc_status_t, ucc_cl_hier_alltoallv_init, (coll_args, team, task), ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_cl_hier_lib_t *cl_lib = UCC_CL_HIER_TEAM_LIB(cl_team); int full_only = 0; ucc_cl_hier_schedule_t *cl_schedule; ucc_schedule_t *schedule; ucc_status_t status; ucc_base_coll_args_t args; ucc_coll_task_t *task_node, *task_full; int c64, d64; void *sc_full, *sd_full, *rc_full, *rd_full; void *sc_node, *sd_node, *rc_node, *rd_node; ucc_rank_t full_size, node_size; size_t sdt_size, rdt_size; ucc_sbgp_t *sbgp; size_t elem_size; if (UCC_IS_INPLACE(coll_args->args)) { cl_debug(team->context->lib, "inplace alltoallv is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (coll_args->args.mask & UCC_COLL_ARGS_FIELD_GLOBAL_WORK_BUFFER) { cl_debug(team->context->lib, "onesided alltoallv is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (!SBGP_ENABLED(cl_team, FULL)) { cl_debug(team->context->lib, "alltoallv requires FULL sbgp"); return UCC_ERR_NOT_SUPPORTED; } c64 = UCC_COLL_ARGS_COUNT64(&coll_args->args); d64 = UCC_COLL_ARGS_DISPL64(&coll_args->args); if (c64 ^ d64) { cl_debug(team->context->lib, "mixed 64 bit count/displ mode is not supported"); return UCC_ERR_NOT_SUPPORTED; } cl_schedule = ucc_cl_hier_get_schedule(cl_team); if (ucc_unlikely(!cl_schedule)) { return UCC_ERR_NO_MEMORY; } schedule = &cl_schedule->super.super; memcpy(&args, coll_args, sizeof(args)); UCC_CHECK_GOTO(ucc_schedule_init(schedule, &args, team), error, status); full_size = cl_team->sbgps[UCC_HIER_SBGP_FULL].sbgp->group_size; node_size = cl_team->sbgps[UCC_HIER_SBGP_NODE].sbgp->group_size; elem_size = c64 ? 8 : 4; if (!SBGP_ENABLED(cl_team, NODE)) { full_only = 1; UCC_CHECK_GOTO(ucc_coll_init( cl_team->sbgps[UCC_HIER_SBGP_FULL].score_map, &args, &task_full), error, status); goto full; } status = ucc_mc_alloc(&cl_schedule->scratch, elem_size * (full_size + node_size) * 4, UCC_MEMORY_TYPE_HOST); if (ucc_unlikely(UCC_OK != status)) { cl_error(team->context->lib, "failed to allocate %zd bytes for full counts", elem_size * (full_size + node_size) * 4); goto error; } sc_full = cl_schedule->scratch->addr; sd_full = PTR_OFFSET(sc_full, full_size * elem_size); rc_full = PTR_OFFSET(sc_full, full_size * elem_size * 2); rd_full = PTR_OFFSET(sc_full, full_size * elem_size * 3); sc_node = PTR_OFFSET(sc_full, full_size * elem_size * 4); sd_node = PTR_OFFSET(sc_node, node_size * elem_size); rc_node = PTR_OFFSET(sc_node, node_size * elem_size * 2); rd_node = PTR_OFFSET(sc_node, node_size * elem_size * 3); /* Duplicate FULL a2av info and alloc task */ sbgp = cl_team->sbgps[UCC_HIER_SBGP_FULL].sbgp; ucc_assert(sbgp->group_size == team->params.size); sdt_size = ucc_dt_size(coll_args->args.src.info_v.datatype); rdt_size = ucc_dt_size(coll_args->args.dst.info_v.datatype); if (c64) { SET_FULL_COUNTS(uint64_t, sbgp, coll_args, team, cl_lib->cfg.a2av_node_thresh, sdt_size, rdt_size, sc_full, sd_full, rc_full, rd_full); } else { SET_FULL_COUNTS(uint32_t, sbgp, coll_args, team, cl_lib->cfg.a2av_node_thresh, sdt_size, rdt_size, sc_full, sd_full, rc_full, rd_full); } args.args.src.info_v.counts = (ucc_count_t *)sc_full; args.args.dst.info_v.counts = (ucc_count_t *)rc_full; args.args.src.info_v.displacements = (ucc_aint_t *)sd_full; args.args.dst.info_v.displacements = (ucc_aint_t *)rd_full; UCC_CHECK_GOTO(ucc_coll_init(cl_team->sbgps[UCC_HIER_SBGP_FULL].score_map, &args, &task_full), err_init_1, status); /* Setup NODE a2av */ sbgp = cl_team->sbgps[UCC_HIER_SBGP_NODE].sbgp; if (c64) { SET_NODE_COUNTS(uint64_t, sbgp, coll_args, cl_lib->cfg.a2av_node_thresh, sdt_size, rdt_size, sc_node, sd_node, rc_node, rd_node); } else { SET_NODE_COUNTS(uint32_t, sbgp, coll_args, cl_lib->cfg.a2av_node_thresh, sdt_size, rdt_size, sc_node, sd_node, rc_node, rd_node); } args.args.src.info_v.counts = (ucc_count_t *)sc_node; args.args.dst.info_v.counts = (ucc_count_t *)rc_node; args.args.src.info_v.displacements = (ucc_aint_t *)sd_node; args.args.dst.info_v.displacements = (ucc_aint_t *)rd_node; UCC_CHECK_GOTO(ucc_coll_init(cl_team->sbgps[UCC_HIER_SBGP_NODE].score_map, &args, &task_node), err_init_2, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, task_node), err_init_3, status); UCC_CHECK_GOTO(ucc_task_subscribe_dep(&schedule->super, task_node, UCC_EVENT_SCHEDULE_STARTED), err_init_3, status); full: UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, task_full), err_init_3, status); UCC_CHECK_GOTO(ucc_task_subscribe_dep(&schedule->super, task_full, UCC_EVENT_SCHEDULE_STARTED), err_init_3, status); schedule->super.post = ucc_cl_hier_alltoallv_start; schedule->super.progress = NULL; schedule->super.finalize = ucc_cl_hier_alltoallv_finalize; schedule->super.triggered_post_setup = ucc_cl_hier_alltoallv_triggered_post_setup; *task = &schedule->super; return UCC_OK; err_init_3: if (!full_only) { ucc_collective_finalize(&task_node->super); } err_init_2: ucc_collective_finalize(&task_full->super); err_init_1: if (!full_only) { ucc_mc_free(cl_schedule->scratch); } error: ucc_cl_hier_put_schedule(schedule); return status; } ucc-1.8.0/src/components/cl/hier/alltoallv/alltoallv.h0000664000175000017500000000153515211535620023120 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLTOALLV_H_ #define ALLTOALLV_H_ #include "../cl_hier.h" enum { UCC_CL_HIER_ALLTOALLV_ALG_NODE_SPLIT, UCC_CL_HIER_ALLTOALLV_ALG_LAST, }; extern ucc_base_coll_alg_info_t ucc_cl_hier_alltoallv_algs[UCC_CL_HIER_ALLTOALLV_ALG_LAST + 1]; ucc_status_t ucc_cl_hier_alltoallv_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); static inline int ucc_cl_hier_alltoallv_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_CL_HIER_ALLTOALLV_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_cl_hier_alltoallv_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/cl/hier/cl_hier_team.c0000664000175000017500000004143315211535620021543 0ustar alastairalastair/** * Copyright (c) 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_hier.h" #include "utils/ucc_malloc.h" #include "core/ucc_team.h" #include "core/ucc_service_coll.h" #include "cl_hier_coll.h" #define SBGP_SET(_team, _sbgp, _enable) \ _team->sbgps[UCC_HIER_SBGP_##_sbgp].sbgp_type = UCC_SBGP_##_sbgp; \ _team->sbgps[UCC_HIER_SBGP_##_sbgp].state = UCC_HIER_SBGP_##_enable; #define N_MT 3 /* The function below must enable/disable those hier sbgps that will be * used to construct hierarchical schedules. * Currently just enable two sbgps as example and for testing purposes. * Next step is to enable sbgps based on the requested hierarchical algs. */ static void ucc_cl_hier_enable_sbgps(ucc_cl_hier_team_t *team) { SBGP_SET(team, NET, ENABLED); SBGP_SET(team, NODE, ENABLED); SBGP_SET(team, NODE_LEADERS, ENABLED); SBGP_SET(team, FULL, ENABLED); /* TODO: parse score if a2av is enabled */ } UCC_CLASS_INIT_FUNC(ucc_cl_hier_team_t, ucc_base_context_t *cl_context, const ucc_base_team_params_t *params) { ucc_cl_hier_context_t *ctx = ucc_derived_of(cl_context, ucc_cl_hier_context_t); ucc_cl_hier_lib_t *lib = ucc_derived_of(cl_context->lib, ucc_cl_hier_lib_t); int i, j, t, n_sbgp_teams; ucc_status_t status; ucc_hier_sbgp_t *hs; ucc_config_names_array_t *tls; ucc_subset_t subset; struct ucc_team_team_desc *d; ucc_tl_context_t *tl_ctx; ucc_tl_lib_t *tl_lib; ucc_base_lib_attr_t attr; if (!params->team->topo) { cl_debug(cl_context->lib, "can't create hier team without topology data"); return UCC_ERR_INVALID_PARAM; } if (ucc_topo_is_single_node(params->team->topo)) { cl_debug(cl_context->lib, "skipping single node team"); return UCC_ERR_INVALID_PARAM; } UCC_CLASS_CALL_SUPER_INIT(ucc_cl_team_t, &ctx->super, params); memset(self->sbgps, 0, sizeof(self->sbgps)); ucc_cl_hier_enable_sbgps(self); n_sbgp_teams = 0; for (i = 0; i < UCC_HIER_SBGP_LAST; i++) { hs = &self->sbgps[i]; if (hs->state == UCC_HIER_SBGP_ENABLED) { hs->sbgp = ucc_topo_get_sbgp(params->team->topo, hs->sbgp_type); if (hs->sbgp->status != UCC_SBGP_ENABLED) { /* SBGP of that type either not exists or the calling process * is not part of subgroup */ cl_debug(cl_context->lib, "sbgp %s is not enabled", ucc_sbgp_str(hs->sbgp_type)); hs->state = UCC_HIER_SBGP_DISABLED; continue; } hs->n_tls = 0; tls = &lib->cfg.sbgp_tls[i].array; for (j = 0; j < tls->count; j++) { if (hs->n_tls == CL_HIER_MAX_SBGP_TLS) { cl_debug(cl_context->lib, "skipping tl context %s for %s sbgp: " "max number of TLs per SBGP is reached", tls->names[j], ucc_sbgp_str(hs->sbgp_type)); continue; } status = ucc_tl_context_get(ctx->super.super.ucc_context, tls->names[j], &hs->tl_ctxs[hs->n_tls]); if (UCC_OK != status) { cl_debug(cl_context->lib, "tl context %s is not available for sbgp %s", tls->names[j], ucc_sbgp_str(hs->sbgp_type)); continue; } attr.mask = UCC_BASE_LIB_ATTR_FIELD_MIN_TEAM_SIZE | UCC_BASE_LIB_ATTR_FIELD_MAX_TEAM_SIZE; tl_ctx = hs->tl_ctxs[hs->n_tls]; tl_lib = ucc_derived_of(tl_ctx->super.lib, ucc_tl_lib_t); status = tl_lib->iface->lib.get_attr(tl_ctx->super.lib, &attr); if (status != UCC_OK) { cl_debug(cl_context->lib, "failed to get attributes for tl context %s", tls->names[j]); ucc_tl_context_put(tl_ctx); continue; } if (hs->sbgp->group_size < attr.min_team_size || hs->sbgp->group_size > attr.max_team_size) { cl_debug(cl_context->lib, "tl context %s is not suitable for sbgp %s" "sbgp: sbgp size %d is not in range [%d; %d]", tls->names[j], ucc_sbgp_str(hs->sbgp_type), hs->sbgp->group_size, attr.min_team_size, attr.max_team_size); ucc_tl_context_put(tl_ctx); continue; } hs->n_tls++; n_sbgp_teams++; ucc_assert(hs->n_tls <= CL_HIER_MAX_SBGP_TLS); } } } status = ucc_team_multiple_req_alloc(&self->team_create_req, n_sbgp_teams); if (UCC_OK != status) { cl_error(cl_context->lib, "failed to allocate team req multiple"); goto err; } /* initialize base params for ALL tl teams we need to create: * for each hier sbgp we have n_tls potentially requested tl teams */ j = 0; for (i = 0; i < UCC_HIER_SBGP_LAST; i++) { hs = &self->sbgps[i]; if (hs->state == UCC_HIER_SBGP_ENABLED) { for (t = 0; t < hs->n_tls; t++) { d = &self->team_create_req->descs[j]; d->param.params.mask = UCC_TEAM_PARAM_FIELD_EP_RANGE | UCC_TEAM_PARAM_FIELD_EP | UCC_TEAM_PARAM_FIELD_TEAM_SIZE | UCC_TEAM_PARAM_FIELD_OOB; d->param.team = params->team; d->param.rank = hs->sbgp->group_rank; d->param.size = hs->sbgp->group_size; d->param.params.ep = (uint64_t)hs->sbgp->group_rank; d->param.params.ep_range = UCC_COLLECTIVE_EP_RANGE_CONTIG; d->param.scope = UCC_CL_HIER; d->param.id = params->id; d->param.scope_id = i; d->param.map = hs->sbgp->map; d->ctx = hs->tl_ctxs[t]; subset.myrank = hs->sbgp->group_rank; subset.map = hs->sbgp->map; /* internal oob will execute allgather over subset */ status = ucc_internal_oob_init(params->team, subset, &d->param.params.oob); if (UCC_OK != status) { cl_error(cl_context->lib, "failed to init oob for sbgp %s", ucc_sbgp_str(hs->sbgp->type)); goto err; } d->args[0] = i; d->args[1] = t; j++; } } } status = ucc_tl_team_create_multiple(self->team_create_req); if (status < 0) { cl_error(cl_context->lib, "failed to post tl team create (%d)", status); goto err; } cl_debug(cl_context->lib, "posted cl team: %p", self); return UCC_OK; err: ucc_team_multiple_req_free(self->team_create_req); return status; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_hier_team_t) { cl_debug(self->super.super.context->lib, "finalizing cl team: %p", self); } UCC_CLASS_DEFINE_DELETE_FUNC(ucc_cl_hier_team_t, ucc_base_team_t); UCC_CLASS_DEFINE(ucc_cl_hier_team_t, ucc_cl_team_t); ucc_status_t ucc_cl_hier_team_destroy(ucc_base_team_t *cl_team) { ucc_cl_hier_team_t *team = ucc_derived_of(cl_team, ucc_cl_hier_team_t); ucc_cl_hier_context_t *ctx = UCC_CL_HIER_TEAM_CTX(team); ucc_status_t status = UCC_OK; int i, j; ucc_hier_sbgp_t *hs; struct ucc_team_team_desc *d; if (NULL == team->team_create_req) { status = ucc_team_multiple_req_alloc(&team->team_create_req, team->n_tl_teams); if (UCC_OK != status) { cl_error(ctx->super.super.lib, "failed to allocate team req multiple"); return status; } team->team_create_req->n_teams = 0; for (i = 0; i < UCC_HIER_SBGP_LAST; i++) { hs = &team->sbgps[i]; if (hs->state == UCC_HIER_SBGP_ENABLED) { if (hs->score_map) { ucc_coll_score_free_map(hs->score_map); } for (j = 0; j < hs->n_tls; j++) { if (hs->tl_teams[j]) { ucc_tl_context_put(hs->tl_ctxs[j]); d = &team->team_create_req->descs[ team->team_create_req->n_teams++]; d->team = hs->tl_teams[j]; d->param.params.oob = d->team->super.params.params.oob; } } } } } status = ucc_tl_team_destroy_multiple(team->team_create_req); if (UCC_INPROGRESS == status) { return status; } for (i = 0; i < team->team_create_req->n_teams; i++) { ucc_internal_oob_finalize(&team->team_create_req-> descs[i].param.params.oob); if (team->team_create_req->descs[i].status != UCC_OK) { cl_error(ctx->super.super.lib, "tl team destroy failed (%d)", status); status = team->team_create_req->descs[i].status; } } ucc_team_multiple_req_free(team->team_create_req); UCC_CLASS_DELETE_FUNC_NAME(ucc_cl_hier_team_t)(cl_team); return status; } ucc_status_t ucc_cl_hier_team_create_test(ucc_base_team_t *cl_team) { ucc_cl_hier_team_t *team = ucc_derived_of(cl_team, ucc_cl_hier_team_t); ucc_cl_hier_context_t *ctx = UCC_CL_HIER_TEAM_CTX(team); ucc_status_t status; int i; ucc_coll_score_t *score, *score_merge; struct ucc_team_team_desc *d; ucc_hier_sbgp_t *hs; status = ucc_tl_team_create_multiple(team->team_create_req); if (status != UCC_OK) { return status; } team->is_block_ordered = -1; team->n_tl_teams = 0; /* TL teams are created: get scores and merge them to produce * score map for each sbgp */ for (i = 0; i < team->team_create_req->n_teams; i++) { d = &team->team_create_req->descs[i]; ucc_hier_sbgp_type_t st = (ucc_hier_sbgp_type_t)d->args[0]; int tl = (int)d->args[1]; hs = &team->sbgps[st]; if (d->status == UCC_OK) { hs->tl_teams[tl] = d->team; team->n_tl_teams++; status = UCC_TL_TEAM_IFACE(d->team)->team.get_scores( &d->team->super, &score); if (UCC_OK != status) { cl_warn(ctx->super.super.lib, "failed to get tl %s scores", UCC_TL_TEAM_IFACE(d->team)->super.name); continue; /* TODO: goto cleanup? */ } if (hs->score == NULL) { hs->score = score; } else { status = ucc_coll_score_merge(hs->score, score, &score_merge, 1); if (UCC_OK != status) { cl_warn(ctx->super.super.lib, "failed to merge scores"); } else { hs->score = score_merge; } } cl_debug(ctx->super.super.lib, "initialized tl %s team for sbgp %s", UCC_TL_CTX_IFACE(d->ctx)->super.name, ucc_sbgp_str(hs->sbgp_type)); } else { cl_debug(ctx->super.super.lib, "failed to create tl %s team", UCC_TL_CTX_IFACE(d->ctx)->super.name); hs->tl_teams[tl] = NULL; hs->tl_ctxs[tl] = NULL; ucc_tl_context_put(d->ctx); } } for (i = 0; i < UCC_HIER_SBGP_LAST; i++) { hs = &team->sbgps[i]; if (hs->score == NULL) { if (hs->state == UCC_HIER_SBGP_ENABLED) { /* we tried to enable that sbgp, which means the subgroup * exists. however failed to create a single team there */ cl_error(ctx->super.super.lib, "no tl teams were created for sbgp %s", ucc_sbgp_str(hs->sbgp_type)); status = UCC_ERR_NO_RESOURCE; break; } hs->state = UCC_HIER_SBGP_DISABLED; } else { status = ucc_coll_score_build_map(hs->score, &hs->score_map); if (UCC_OK != status) { cl_error(ctx->super.super.lib, "failed to build score map"); hs->state = UCC_HIER_SBGP_DISABLED; status = UCC_ERR_NO_RESOURCE; break; } } } ucc_team_multiple_req_free(team->team_create_req); team->team_create_req = NULL; if (SBGP_EXISTS(team, NODE_LEADERS)) { team->top_sbgp = UCC_HIER_SBGP_NODE_LEADERS; } else { ucc_assert(SBGP_EXISTS(team, NODE)); team->top_sbgp = UCC_HIER_SBGP_NODE; } return status; } ucc_status_t ucc_cl_hier_team_get_scores(ucc_base_team_t *cl_team, ucc_coll_score_t **score_p) { ucc_cl_hier_team_t *team = ucc_derived_of(cl_team, ucc_cl_hier_team_t); ucc_base_lib_t *lib = UCC_CL_TEAM_LIB(team); ucc_base_context_t *ctx = UCC_CL_TEAM_CTX(team); ucc_memory_type_t mt[N_MT] = {UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED}; ucc_coll_score_t *score; ucc_status_t status; int i; ucc_coll_score_team_info_t team_info; team_info.alg_fn = ucc_cl_hier_alg_id_to_init; team_info.default_score = UCC_CL_HIER_DEFAULT_SCORE; team_info.init = ucc_cl_hier_coll_init; team_info.num_mem_types = 0; team_info.supported_mem_types = NULL; /* all memory types supported*/ team_info.supported_colls = UCC_CL_HIER_SUPPORTED_COLLS; team_info.size = UCC_CL_TEAM_SIZE(team); status = ucc_coll_score_alloc(&score); if (UCC_OK != status) { cl_error(lib, "faild to alloc score_t"); return status; } for (i = 0; i < N_MT; i++) { status = ucc_coll_score_add_range( score, UCC_COLL_TYPE_ALLTOALLV, mt[i], 0, UCC_MSG_MAX, /* low priority 1: to be enabled manually */ 1, ucc_cl_hier_alltoallv_init, cl_team); if (UCC_OK != status) { cl_error(lib, "failed to add range to score_t"); return status; } status = ucc_coll_score_add_range( score, UCC_COLL_TYPE_ALLTOALL, mt[i], 0, UCC_MSG_MAX, /* low priority 1: to be enabled manually */ 1, ucc_cl_hier_alltoall_init, cl_team); if (UCC_OK != status) { cl_error(lib, "failed to add range to score_t"); return status; } } status = ucc_coll_score_add_range( score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, 0, UCC_MSG_MAX, UCC_CL_HIER_DEFAULT_SCORE, ucc_cl_hier_barrier_init, cl_team); if (UCC_OK != status) { cl_error(lib, "failed to add range to score_t"); return status; } for (i = 0; i < UCC_CL_HIER_N_DEFAULT_ALG_SELECT_STR; i++) { status = ucc_coll_score_update_from_str( ucc_cl_hier_default_alg_select_str[i], &team_info, &team->super.super, score); if (UCC_OK != status) { cl_error(lib, "failed to apply default coll select setting: %s", ucc_cl_hier_default_alg_select_str[i]); goto err; } } if (strlen(ctx->score_str) > 0) { status = ucc_coll_score_update_from_str(ctx->score_str, &team_info, &team->super.super, score); /* if INVALID_PARAM - user provided incorrect input - try to proceed */ if ((status < 0) && (status != UCC_ERR_INVALID_PARAM) && (status != UCC_ERR_NOT_SUPPORTED)) { goto err; } } *score_p = score; return UCC_OK; err: ucc_coll_score_free(score); *score_p = NULL; return status; } ucc-1.8.0/src/components/cl/hier/barrier/0000775000175000017500000000000015211535620020405 5ustar alastairalastairucc-1.8.0/src/components/cl/hier/barrier/barrier.c0000664000175000017500000000713315211535620022203 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "barrier.h" #include "../cl_hier_coll.h" #define MAX_BARRIER_TASKS 3 static ucc_status_t ucc_cl_hier_barrier_start(ucc_coll_task_t *task) { UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_barrier_start", 0); return ucc_schedule_start(task); } static ucc_status_t ucc_cl_hier_barrier_finalize(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_status_t status; UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_barrier_finalize", 0); status = ucc_schedule_finalize(task); ucc_cl_hier_put_schedule(schedule); return status; } UCC_CL_HIER_PROFILE_FUNC(ucc_status_t, ucc_cl_hier_barrier_init, (coll_args, team, task), ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_coll_task_t *tasks[MAX_BARRIER_TASKS] = {NULL}; ucc_schedule_t *schedule; ucc_status_t status; ucc_base_coll_args_t args; int n_tasks, i; schedule = &ucc_cl_hier_get_schedule(cl_team)->super.super; if (ucc_unlikely(!schedule)) { return UCC_ERR_NO_MEMORY; } memcpy(&args, coll_args, sizeof(args)); args.args.root = 0; /* TODO: we can select the rank closest to HCA */ n_tasks = 0; UCC_CHECK_GOTO(ucc_schedule_init(schedule, &args, team), out, status); if (SBGP_ENABLED(cl_team, NODE)) { ucc_assert(n_tasks == 0); if (cl_team->top_sbgp == UCC_HIER_SBGP_NODE) { args.args.coll_type = UCC_COLL_TYPE_BARRIER; } else { args.args.coll_type = UCC_COLL_TYPE_FANIN; } UCC_CHECK_GOTO( ucc_coll_init(SCORE_MAP(cl_team, NODE), &args, &tasks[n_tasks]), out, status); n_tasks++; } if (SBGP_ENABLED(cl_team, NODE_LEADERS)) { ucc_assert(cl_team->top_sbgp == UCC_HIER_SBGP_NODE_LEADERS); args.args.coll_type = UCC_COLL_TYPE_BARRIER; UCC_CHECK_GOTO(ucc_coll_init(SCORE_MAP(cl_team, NODE_LEADERS), &args, &tasks[n_tasks]), out, status); n_tasks++; } if (SBGP_ENABLED(cl_team, NODE) && cl_team->top_sbgp != UCC_HIER_SBGP_NODE) { args.args.coll_type = UCC_COLL_TYPE_FANOUT; UCC_CHECK_GOTO( ucc_coll_init(SCORE_MAP(cl_team, NODE), &args, &tasks[n_tasks]), out, status); n_tasks++; } UCC_CHECK_GOTO(ucc_event_manager_subscribe( &schedule->super, UCC_EVENT_SCHEDULE_STARTED, tasks[0], ucc_task_start_handler), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, tasks[0]), out, status); for (i = 1; i < n_tasks; i++) { UCC_CHECK_GOTO( ucc_event_manager_subscribe(tasks[i - 1], UCC_EVENT_COMPLETED, tasks[i], ucc_task_start_handler), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, tasks[i]), out, status); } schedule->super.post = ucc_cl_hier_barrier_start; schedule->super.finalize = ucc_cl_hier_barrier_finalize; *task = &schedule->super; return UCC_OK; out: for (i = 0; i < n_tasks; i++) { tasks[i]->finalize(tasks[i]); } ucc_cl_hier_put_schedule(schedule); return status; } ucc-1.8.0/src/components/cl/hier/barrier/barrier.h0000664000175000017500000000061315211535620022204 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef BARRIER_H_ #define BARRIER_H_ #include "../cl_hier.h" ucc_status_t ucc_cl_hier_barrier_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); #endif ucc-1.8.0/src/components/cl/hier/alltoall/0000775000175000017500000000000015211535620020563 5ustar alastairalastairucc-1.8.0/src/components/cl/hier/alltoall/alltoall.h0000664000175000017500000000152615211535620022544 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLTOALL_H_ #define ALLTOALL_H_ #include "../cl_hier_coll.h" enum { UCC_CL_HIER_ALLTOALL_ALG_NODE_SPLIT, UCC_CL_HIER_ALLTOALL_ALG_LAST, }; extern ucc_base_coll_alg_info_t ucc_cl_hier_alltoall_algs[UCC_CL_HIER_ALLTOALL_ALG_LAST + 1]; ucc_status_t ucc_cl_hier_alltoall_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task); static inline int ucc_cl_hier_alltoall_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_CL_HIER_ALLTOALL_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_cl_hier_alltoall_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/cl/hier/alltoall/alltoall.c0000664000175000017500000000717015211535620022540 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "alltoall.h" #include "../alltoallv/alltoallv.h" ucc_base_coll_alg_info_t ucc_cl_hier_alltoall_algs[UCC_CL_HIER_ALLTOALL_ALG_LAST + 1] = { [UCC_CL_HIER_ALLTOALL_ALG_NODE_SPLIT] = {.id = UCC_CL_HIER_ALLTOALL_ALG_NODE_SPLIT, .name = "node_split", .desc = "splitting alltoall into two concurrent a2av calls" " within the node and outside of it"}, [UCC_CL_HIER_ALLTOALL_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; UCC_CL_HIER_PROFILE_FUNC(ucc_status_t, ucc_cl_hier_alltoall_init, (coll_args, team, task), ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_rank_t tsize = UCC_CL_TEAM_SIZE(cl_team); uint64_t *counts, *displs; ucc_status_t status; ucc_base_coll_args_t args; int i; size_t count; ucc_mc_buffer_header_t *h; if (UCC_IS_INPLACE(coll_args->args)) { cl_debug(team->context->lib, "inplace alltoall is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (!SBGP_ENABLED(cl_team, FULL)) { cl_debug(team->context->lib, "alltoall requires FULL sbgps"); return UCC_ERR_NOT_SUPPORTED; } memcpy(&args, coll_args, sizeof(args)); args.args.coll_type = UCC_COLL_TYPE_ALLTOALLV; if (!(args.args.mask & UCC_COLL_ARGS_FIELD_FLAGS)) { args.args.mask = UCC_COLL_ARGS_FIELD_FLAGS; args.args.flags = 0; } args.args.flags |= UCC_COLL_ARGS_FLAG_CONTIG_SRC_BUFFER | UCC_COLL_ARGS_FLAG_CONTIG_DST_BUFFER | UCC_COLL_ARGS_FLAG_COUNT_64BIT | UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT; status = ucc_mc_alloc(&h, sizeof(uint64_t) * tsize * 2, UCC_MEMORY_TYPE_HOST); if (ucc_unlikely(UCC_OK != status)) { cl_error(team->context->lib, "failed to allocate %zd bytes for full counts", sizeof(uint64_t) * tsize * 2); return status; } args.args.src.info_v.buffer = coll_args->args.src.info.buffer; args.args.dst.info_v.buffer = coll_args->args.dst.info.buffer; args.args.src.info_v.datatype = coll_args->args.src.info.datatype; args.args.dst.info_v.datatype = coll_args->args.dst.info.datatype; args.args.src.info_v.mem_type = coll_args->args.src.info.mem_type; args.args.dst.info_v.mem_type = coll_args->args.dst.info.mem_type; args.args.src.info_v.counts = h->addr; args.args.src.info_v.displacements = PTR_OFFSET(h->addr, sizeof(uint64_t) * tsize); args.args.dst.info_v.counts = args.args.src.info_v.counts; args.args.dst.info_v.displacements = args.args.src.info_v.displacements; counts = (uint64_t *)args.args.src.info_v.counts; displs = (uint64_t *)args.args.src.info_v.displacements; count = coll_args->args.src.info.count / tsize; counts[0] = count; displs[0] = 0; for (i = 1; i < tsize; i++) { counts[i] = count; displs[i] = displs[i - 1] + count; } status = ucc_cl_hier_alltoallv_init(&args, team, task); if (UCC_OK != status) { cl_error(team->context->lib, "failed to init split node a2av task"); } ucc_mc_free(h); return status; } ucc-1.8.0/src/components/cl/hier/allreduce/0000775000175000017500000000000015211535620020717 5ustar alastairalastairucc-1.8.0/src/components/cl/hier/allreduce/allreduce.h0000664000175000017500000000227015211535620023031 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLREDUCE_H_ #define ALLREDUCE_H_ #include "../cl_hier.h" enum { UCC_CL_HIER_ALLREDUCE_ALG_RAB, UCC_CL_HIER_ALLREDUCE_ALG_SPLIT_RAIL, UCC_CL_HIER_ALLREDUCE_ALG_LAST, }; extern ucc_base_coll_alg_info_t ucc_cl_hier_allreduce_algs[UCC_CL_HIER_ALLREDUCE_ALG_LAST + 1]; #define UCC_CL_HIER_ALLREDUCE_DEFAULT_ALG_SELECT_STR "allreduce:0-4k:@rab" ucc_status_t ucc_cl_hier_allreduce_rab_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); ucc_status_t ucc_cl_hier_allreduce_split_rail_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); static inline int ucc_cl_hier_allreduce_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_CL_HIER_ALLREDUCE_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_cl_hier_allreduce_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/cl/hier/allreduce/allreduce_split_rail.c0000664000175000017500000003160615211535620025253 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allreduce.h" #include "../cl_hier_coll.h" #include "core/ucc_team.h" static ucc_status_t ucc_cl_hier_allreduce_split_rail_frag_finalize(ucc_coll_task_t *task) { ucc_status_t status = UCC_OK; ucc_cl_hier_schedule_t *schedule = ucc_derived_of(task, ucc_cl_hier_schedule_t); status = ucc_schedule_finalize(&schedule->super.super.super); ucc_free(schedule->allreduce_split_rail.counts); ucc_cl_hier_put_schedule(&schedule->super.super); return status; } static ucc_status_t ucc_cl_hier_ar_split_rail_schedule_finalize(ucc_coll_task_t *task) { ucc_cl_hier_schedule_t *schedule = ucc_derived_of(task, ucc_cl_hier_schedule_t); ucc_status_t status; status = ucc_schedule_pipelined_finalize(&schedule->super.super.super); ucc_cl_hier_put_schedule(&schedule->super.super); return status; } static ucc_status_t ucc_cl_hier_allreduce_split_rail_frag_setup( ucc_schedule_pipelined_t *schedule_p, ucc_schedule_t *frag, int frag_num) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(schedule_p->super.super.team, ucc_cl_hier_team_t); ucc_coll_args_t *args = &schedule_p->super.super.bargs.args; size_t dt_size = ucc_dt_size(args->dst.info.datatype); int n_frags = schedule_p->super.n_tasks; int inplace = UCC_IS_INPLACE(*args); size_t frag_count, frag_offset, ar_count, ar_offset; ucc_rank_t node_size, node_rank; ucc_coll_task_t *task_rs, *task_ar, *task_ag; int i; uint64_t * counts, *displs; node_size = cl_team->sbgps[UCC_HIER_SBGP_NODE].sbgp->group_size; node_rank = cl_team->sbgps[UCC_HIER_SBGP_NODE].sbgp->group_rank; frag_count = ucc_buffer_block_count(args->dst.info.count, n_frags, frag_num); frag_offset = ucc_buffer_block_offset(args->dst.info.count, n_frags, frag_num); counts = ucc_derived_of(frag, ucc_cl_hier_schedule_t) ->allreduce_split_rail.counts; displs = counts + node_size; for (i = 0; i < node_size; i++) { counts[i] = ucc_buffer_block_count(frag_count, node_size, i); displs[i] = ucc_buffer_block_offset(frag_count, node_size, i); } ar_count = counts[node_rank]; ar_offset = displs[node_rank]; task_rs = frag->tasks[0]; task_ar = frag->tasks[1]; task_ag = frag->tasks[2]; ucc_assert(task_rs->bargs.args.dst.info_v.counts == counts); if (inplace) { task_rs->bargs.args.dst.info_v.buffer = PTR_OFFSET( args->dst.info.buffer, frag_offset * dt_size); } else { task_rs->bargs.args.src.info.buffer = PTR_OFFSET(args->src.info.buffer, frag_offset * dt_size); task_rs->bargs.args.src.info.count = frag_count; task_rs->bargs.args.dst.info_v.buffer = PTR_OFFSET( args->dst.info.buffer, (frag_offset + ar_offset) * dt_size); } task_ar->bargs.args.dst.info.count = ar_count; task_ar->bargs.args.dst.info.buffer = PTR_OFFSET( args->dst.info.buffer, (frag_offset + ar_offset) * dt_size); ucc_assert(UCC_IS_INPLACE(task_ag->bargs.args)); task_ag->bargs.args.dst.info_v.buffer = PTR_OFFSET( args->dst.info.buffer, frag_offset * dt_size); //only dst since inplace task_ag->bargs.args.src.info.count = frag_count; ucc_assert(task_ag->bargs.args.dst.info_v.counts == counts); ucc_assert(task_ag->bargs.args.dst.info_v.displacements == displs); return UCC_OK; } static ucc_status_t ucc_cl_hier_allreduce_split_rail_frag_init( ucc_base_coll_args_t *coll_args, ucc_schedule_pipelined_t *sp, ucc_base_team_t *team, ucc_schedule_t **frag_p) { ucc_cl_hier_team_t * cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); size_t dt_size = ucc_dt_size(coll_args->args.dst.info.datatype); ucc_status_t status = UCC_OK; int inplace = UCC_IS_INPLACE(coll_args->args); int n_frags = sp->super.n_tasks; ucc_coll_task_t *task_rs, *task_ag, *task_ar; ucc_base_coll_args_t rs_args, ar_args, ag_args; ucc_cl_hier_schedule_t *cl_schedule; ucc_schedule_t * schedule; size_t total_count; ucc_rank_t node_size, node_rank; int i; uint64_t *counts, *displs; cl_schedule = ucc_cl_hier_get_schedule(cl_team); if (ucc_unlikely(!cl_schedule)) { return UCC_ERR_NO_MEMORY; } schedule = &cl_schedule->super.super; status = ucc_schedule_init(schedule, coll_args, team); if (UCC_OK != status) { return status; } node_size = cl_team->sbgps[UCC_HIER_SBGP_NODE].sbgp->group_size; if (ucc_unlikely(0 == node_size)) { /* this check is only needed to suppress clang-tidy linter which assumes potential devision-by-zero when node_size is passed to ucc_buffer_block_count */ goto err_rs; } node_rank = cl_team->sbgps[UCC_HIER_SBGP_NODE].sbgp->group_rank; total_count = coll_args->args.dst.info.count; cl_schedule->allreduce_split_rail.counts = ucc_malloc(node_size * 2 * sizeof(uint64_t), "counts"); if (ucc_unlikely(!cl_schedule->allreduce_split_rail.counts)) { cl_error(team->context->lib, "failed to allocate %zd bytes for counts array", node_size * 2 * sizeof(uint64_t)); goto err_rs; } counts = cl_schedule->allreduce_split_rail.counts; displs = counts + node_size; for (i = 0; i < node_size; i++) { counts[i] = ucc_buffer_block_count(total_count, node_size, i); displs[i] = ucc_buffer_block_offset(total_count, node_size, i); } memcpy(&rs_args, coll_args, sizeof(rs_args)); memcpy(&ar_args, coll_args, sizeof(ar_args)); memcpy(&ag_args, coll_args, sizeof(ag_args)); rs_args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; rs_args.args.flags &= (~UCC_COLL_ARGS_FLAG_IN_PLACE); rs_args.args.flags |= (UCC_COLL_ARGS_FLAG_COUNT_64BIT | UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT); /* REDUCE-SCATTER */ rs_args.args.coll_type = UCC_COLL_TYPE_REDUCE_SCATTERV; rs_args.args.dst.info_v.counts = counts; rs_args.args.dst.info_v.mem_type = coll_args->args.dst.info.mem_type; rs_args.args.dst.info_v.datatype = coll_args->args.dst.info.datatype; /* linter thinks node_size can be 0 - false positive */ rs_args.max_frag_count = ucc_buffer_block_count( ucc_buffer_block_count(total_count, n_frags, 0), node_size, 0); //NOLINT rs_args.mask |= UCC_BASE_CARGS_MAX_FRAG_COUNT; if (inplace) { rs_args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; rs_args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; rs_args.args.src.info.buffer = coll_args->args.dst.info.buffer; rs_args.args.src.info.datatype = coll_args->args.dst.info.datatype; rs_args.args.dst.info_v.buffer = coll_args->args.dst.info.buffer; } else { rs_args.args.dst.info_v.buffer = PTR_OFFSET( coll_args->args.dst.info.buffer, displs[node_rank] * dt_size); rs_args.args.src.info.count = coll_args->args.dst.info.count; } status = ucc_coll_init(SCORE_MAP(cl_team, NODE), &rs_args, &task_rs); if (ucc_unlikely(UCC_OK != status)) { cl_error(team->context->lib, "failed to init rs task"); goto err_rs; } /* ALLREDUCE */ ar_args.mask |= UCC_BASE_CARGS_MAX_FRAG_COUNT; ar_args.max_frag_count = ucc_buffer_block_count(total_count, n_frags, 0); ar_args.args.coll_type = UCC_COLL_TYPE_ALLREDUCE; ar_args.args.src.info.count = counts[node_rank]; ar_args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; ar_args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; ar_args.args.dst.info.count = counts[node_rank]; status = ucc_coll_init(SCORE_MAP(cl_team, NET), &ar_args, &task_ar); if (ucc_unlikely(UCC_OK != status)) { cl_error(team->context->lib, "failed to init ar task"); goto err_ar; } /* ALLGATHER */ ag_args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; ag_args.args.flags |= (UCC_COLL_ARGS_FLAG_COUNT_64BIT | UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT); ag_args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; ag_args.args.coll_type = UCC_COLL_TYPE_ALLGATHERV; ag_args.args.dst.info_v.buffer = coll_args->args.dst.info.buffer; ag_args.args.dst.info_v.mem_type = coll_args->args.dst.info.mem_type; ag_args.args.dst.info_v.datatype = coll_args->args.dst.info.datatype; ag_args.args.dst.info_v.counts = counts; ag_args.args.dst.info_v.displacements = displs; status = ucc_coll_init(SCORE_MAP(cl_team, NODE), &ag_args, &task_ag); if (ucc_unlikely(UCC_OK != status)) { cl_error(team->context->lib, "failed to init ag task"); goto err_ag; } UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, task_rs), err_ag, status); UCC_CHECK_GOTO(ucc_task_subscribe_dep(&schedule->super, task_rs, UCC_EVENT_SCHEDULE_STARTED), err_ag, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, task_ar), err_ag, status); UCC_CHECK_GOTO(ucc_task_subscribe_dep(task_rs, task_ar, UCC_EVENT_COMPLETED), err_ag, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, task_ag), err_ag, status); UCC_CHECK_GOTO(ucc_task_subscribe_dep(task_ar, task_ag, UCC_EVENT_COMPLETED), err_ag, status); schedule->super.post = ucc_schedule_start; schedule->super.progress = NULL; schedule->super.finalize = ucc_cl_hier_allreduce_split_rail_frag_finalize; *frag_p = schedule; return status; err_ag: if (task_ar) { ucc_collective_finalize(&task_ar->super); } err_ar: if (task_rs) { ucc_collective_finalize(&task_rs->super); } err_rs: ucc_cl_hier_put_schedule(schedule); return status; } static ucc_status_t ucc_cl_hier_split_rail_allreduce_start(ucc_coll_task_t *task) { ucc_schedule_pipelined_t *schedule = ucc_derived_of(task, ucc_schedule_pipelined_t); cl_debug(task->team->context->lib, "posting split_rail ar, sbuf %p, rbuf %p, count %zd, dt %s, op %s, " "inplace %d, pdepth %d, frags_total %d", task->bargs.args.src.info.buffer, task->bargs.args.dst.info.buffer, task->bargs.args.dst.info.count, ucc_datatype_str(task->bargs.args.src.info.datatype), ucc_reduction_op_str(task->bargs.args.op), UCC_IS_INPLACE(task->bargs.args), schedule->n_frags, schedule->super.n_tasks); return ucc_schedule_pipelined_post(task); } UCC_CL_HIER_PROFILE_FUNC(ucc_status_t, ucc_cl_hier_allreduce_split_rail_init, (coll_args, team, task), ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_cl_hier_lib_config_t *cfg = &UCC_CL_HIER_TEAM_LIB(cl_team)->cfg; ucc_cl_hier_schedule_t *schedule; int n_frags, pipeline_depth; ucc_status_t status; if (coll_args->args.op == UCC_OP_AVG) { return UCC_ERR_NOT_SUPPORTED; } if (!SBGP_ENABLED(cl_team, NODE) || !SBGP_ENABLED(cl_team, NET)) { return UCC_ERR_NOT_SUPPORTED; } if (!ucc_topo_isoppn(team->params.team->topo)) { cl_debug(team->context->lib, "split_rail algorithm does not support " "teams with non-uniform ppn across nodes"); return UCC_ERR_NOT_SUPPORTED; } schedule = ucc_cl_hier_get_schedule(cl_team); if (ucc_unlikely(!schedule)) { return UCC_ERR_NO_MEMORY; } ucc_pipeline_nfrags_pdepth(&cfg->allreduce_split_rail_pipeline, coll_args->args.dst.info.count * ucc_dt_size(coll_args->args.dst.info.datatype), &n_frags, &pipeline_depth); status = ucc_schedule_pipelined_init( coll_args, team, ucc_cl_hier_allreduce_split_rail_frag_init, ucc_cl_hier_allreduce_split_rail_frag_setup, pipeline_depth, n_frags, cfg->allreduce_split_rail_pipeline.order, &schedule->super); if (ucc_unlikely(status != UCC_OK)) { cl_error(team->context->lib, "failed to init pipelined split_rail ar schedule"); goto err_pipe_init; } schedule->super.super.super.post = ucc_cl_hier_split_rail_allreduce_start; schedule->super.super.super.finalize = ucc_cl_hier_ar_split_rail_schedule_finalize; *task = &schedule->super.super.super; return UCC_OK; err_pipe_init: ucc_cl_hier_put_schedule(&schedule->super.super); return status; } ucc-1.8.0/src/components/cl/hier/allreduce/allreduce.c0000664000175000017500000000164315211535620023027 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allreduce.h" #include "../allreduce/allreduce.h" ucc_base_coll_alg_info_t ucc_cl_hier_allreduce_algs[UCC_CL_HIER_ALLREDUCE_ALG_LAST + 1] = { [UCC_CL_HIER_ALLREDUCE_ALG_RAB] = {.id = UCC_CL_HIER_ALLREDUCE_ALG_RAB, .name = "rab", .desc = "intra-node reduce, followed by inter-node allreduce," " followed by intra-node broadcast"}, [UCC_CL_HIER_ALLREDUCE_ALG_SPLIT_RAIL] = {.id = UCC_CL_HIER_ALLREDUCE_ALG_SPLIT_RAIL, .name = "split_rail", .desc = "intra-node reduce_scatter, followed by PPN concurrent " " inter-node allreduces, followed by intra-node allgather"}, [UCC_CL_HIER_ALLREDUCE_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc-1.8.0/src/components/cl/hier/allreduce/allreduce_rab.c0000664000175000017500000002403415211535620023652 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allreduce.h" #include "../cl_hier_coll.h" #define MAX_AR_RAB_TASKS 3 static ucc_status_t ucc_cl_hier_allreduce_rab_start(ucc_coll_task_t *task) { UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_allreduce_rab_start", 0); return ucc_schedule_start(task); } static ucc_status_t ucc_cl_hier_allreduce_rab_finalize(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_status_t status; UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_allreduce_rab_finalize", 0); status = ucc_schedule_finalize(task); ucc_cl_hier_put_schedule(schedule); return status; } static ucc_status_t ucc_cl_hier_ar_rab_schedule_finalize(ucc_coll_task_t *task) { ucc_cl_hier_schedule_t *schedule = ucc_derived_of(task, ucc_cl_hier_schedule_t); ucc_status_t status; status = ucc_schedule_pipelined_finalize(&schedule->super.super.super); ucc_cl_hier_put_schedule(&schedule->super.super); return status; } static ucc_status_t ucc_cl_hier_allreduce_rab_frag_setup(ucc_schedule_pipelined_t *schedule_p, ucc_schedule_t *frag, int frag_num) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(schedule_p->super.super.team, ucc_cl_hier_team_t); ucc_coll_args_t *args = &schedule_p->super.super.bargs.args; size_t dt_size = ucc_dt_size(args->dst.info.datatype); int n_frags = schedule_p->super.n_tasks; int inplace = UCC_IS_INPLACE(*args); size_t frag_count, frag_offset; ucc_coll_task_t *task; int i; frag_count = ucc_buffer_block_count(args->dst.info.count, n_frags, frag_num); frag_offset = ucc_buffer_block_offset(args->dst.info.count, n_frags, frag_num); for (i = 0; i < frag->n_tasks; i++) { task = frag->tasks[i]; task->bargs.args.src.info.count = frag_count; task->bargs.args.dst.info.count = frag_count; task->bargs.args.dst.info.buffer = PTR_OFFSET(args->dst.info.buffer, frag_offset * dt_size); if ((task->bargs.args.coll_type == UCC_COLL_TYPE_BCAST) || (task->bargs.args.coll_type == UCC_COLL_TYPE_REDUCE && inplace && (SBGP_RANK(cl_team, NODE) != args->root))) { task->bargs.args.src.info.buffer = PTR_OFFSET(args->dst.info.buffer, frag_offset * dt_size); } else { task->bargs.args.src.info.buffer = PTR_OFFSET(args->src.info.buffer, frag_offset * dt_size); } } return UCC_OK; } static ucc_status_t ucc_cl_hier_allreduce_rab_init_schedule(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_schedule_t **sched_p, int n_frags) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_coll_task_t *tasks[MAX_AR_RAB_TASKS] = {NULL}; ucc_schedule_t *schedule; ucc_status_t status; ucc_base_coll_args_t args; int n_tasks, i; schedule = &ucc_cl_hier_get_schedule(cl_team)->super.super; if (ucc_unlikely(!schedule)) { return UCC_ERR_NO_MEMORY; } args = *coll_args; args.args.root = 0; /* TODO: we can select the rank closest to HCA */ n_tasks = 0; UCC_CHECK_GOTO(ucc_schedule_init(schedule, &args, team), out, status); if (n_frags > 1) { args.max_frag_count = ucc_buffer_block_count(args.args.dst.info.count, n_frags, 0); args.mask |= UCC_BASE_CARGS_MAX_FRAG_COUNT; } ucc_assert(SBGP_ENABLED(cl_team, NODE) || SBGP_ENABLED(cl_team, NODE_LEADERS)); if (SBGP_ENABLED(cl_team, NODE)) { ucc_assert(n_tasks == 0); if (cl_team->top_sbgp == UCC_HIER_SBGP_NODE) { args.args.coll_type = UCC_COLL_TYPE_ALLREDUCE; } else { args.args.coll_type = UCC_COLL_TYPE_REDUCE; if (UCC_IS_INPLACE(args.args) && (SBGP_RANK(cl_team, NODE) != args.args.root)) { args.args.src.info = args.args.dst.info; } } UCC_CHECK_GOTO( ucc_coll_init(SCORE_MAP(cl_team, NODE), &args, &tasks[n_tasks]), out, status); n_tasks++; args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; } if (SBGP_ENABLED(cl_team, NODE_LEADERS)) { ucc_assert(cl_team->top_sbgp == UCC_HIER_SBGP_NODE_LEADERS); args.args.coll_type = UCC_COLL_TYPE_ALLREDUCE; UCC_CHECK_GOTO(ucc_coll_init(SCORE_MAP(cl_team, NODE_LEADERS), &args, &tasks[n_tasks]), out, status); n_tasks++; } if (SBGP_ENABLED(cl_team, NODE) && cl_team->top_sbgp != UCC_HIER_SBGP_NODE) { /* For bcast src should point to origin dst of allreduce */ args.args.src.info = args.args.dst.info; args.args.coll_type = UCC_COLL_TYPE_BCAST; UCC_CHECK_GOTO( ucc_coll_init(SCORE_MAP(cl_team, NODE), &args, &tasks[n_tasks]), out, status); n_tasks++; } /* subscription logic is different depending on top level schedule type * being used */ UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, tasks[0]), out, status); if (n_frags > 1) { UCC_CHECK_GOTO(ucc_task_subscribe_dep(&schedule->super, tasks[0], UCC_EVENT_SCHEDULE_STARTED), out, status); for (i = 1; i < n_tasks; i++) { UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, tasks[i]), out, status); UCC_CHECK_GOTO(ucc_task_subscribe_dep(tasks[i-1], tasks[i], UCC_EVENT_COMPLETED), out, status); } } else { UCC_CHECK_GOTO(ucc_event_manager_subscribe( &schedule->super, UCC_EVENT_SCHEDULE_STARTED, tasks[0], ucc_task_start_handler), out, status); for (i = 1; i < n_tasks; i++) { UCC_CHECK_GOTO( ucc_event_manager_subscribe(tasks[i - 1], UCC_EVENT_COMPLETED, tasks[i], ucc_task_start_handler), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, tasks[i]), out, status); } } schedule->super.post = ucc_cl_hier_allreduce_rab_start; schedule->super.progress = NULL; schedule->super.finalize = ucc_cl_hier_allreduce_rab_finalize; *sched_p = schedule; return UCC_OK; out: for (i = 0; i < n_tasks; i++) { tasks[i]->finalize(tasks[i]); } ucc_cl_hier_put_schedule(schedule); return status; } static ucc_status_t ucc_cl_hier_allreduce_rab_frag_init( ucc_base_coll_args_t *coll_args, ucc_schedule_pipelined_t *sp, ucc_base_team_t *team, ucc_schedule_t **frag_p) { int n_frags = sp->super.n_tasks; ucc_status_t status; status = ucc_cl_hier_allreduce_rab_init_schedule(coll_args, team, frag_p, n_frags); return status; } static ucc_status_t ucc_cl_hier_rab_allreduce_start(ucc_coll_task_t *task) { ucc_schedule_pipelined_t *schedule = ucc_derived_of(task, ucc_schedule_pipelined_t); cl_debug(task->team->context->lib, "posting rab ar, sbuf %p, rbuf %p, count %zd, dt %s, op %s, " "inplace %d, pdepth %d, frags_total %d", task->bargs.args.src.info.buffer, task->bargs.args.dst.info.buffer, task->bargs.args.dst.info.count, ucc_datatype_str(task->bargs.args.src.info.datatype), ucc_reduction_op_str(task->bargs.args.op), UCC_IS_INPLACE(task->bargs.args), schedule->n_frags, schedule->super.n_tasks); return ucc_schedule_pipelined_post(task); } UCC_CL_HIER_PROFILE_FUNC(ucc_status_t, ucc_cl_hier_allreduce_rab_init, (coll_args, team, task), ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_cl_hier_lib_config_t *cfg = &UCC_CL_HIER_TEAM_LIB(cl_team)->cfg; ucc_cl_hier_schedule_t *schedule; int n_frags, pipeline_depth; ucc_status_t status; if (coll_args->args.op == UCC_OP_AVG) { return UCC_ERR_NOT_SUPPORTED; } ucc_pipeline_nfrags_pdepth(&cfg->allreduce_rab_pipeline, coll_args->args.dst.info.count * ucc_dt_size(coll_args->args.dst.info.datatype), &n_frags, &pipeline_depth); if (n_frags == 1) { return ucc_cl_hier_allreduce_rab_init_schedule( coll_args, team, (ucc_schedule_t **)task, n_frags); } schedule = ucc_cl_hier_get_schedule(cl_team); if (ucc_unlikely(!schedule)) { return UCC_ERR_NO_MEMORY; } status = ucc_schedule_pipelined_init( coll_args, team, ucc_cl_hier_allreduce_rab_frag_init, ucc_cl_hier_allreduce_rab_frag_setup, pipeline_depth, n_frags, cfg->allreduce_rab_pipeline.order, &schedule->super); if (ucc_unlikely(status != UCC_OK)) { cl_error(team->context->lib, "failed to init pipelined rab ar schedule"); goto err_pipe_init; } schedule->super.super.super.post = ucc_cl_hier_rab_allreduce_start; schedule->super.super.super.finalize = ucc_cl_hier_ar_rab_schedule_finalize; *task = &schedule->super.super.super; return UCC_OK; err_pipe_init: ucc_cl_hier_put_schedule(&schedule->super.super); return status; } ucc-1.8.0/src/components/cl/hier/Makefile.am0000664000175000017500000000352515211535620021020 0ustar alastairalastair# # Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # allgatherv = \ allgatherv/unpack.h \ allgatherv/unpack.c \ allgatherv/allgatherv.h \ allgatherv/allgatherv.c allreduce = \ allreduce/allreduce.h \ allreduce/allreduce.c \ allreduce/allreduce_rab.c \ allreduce/allreduce_split_rail.c alltoallv = \ alltoallv/alltoallv.h \ alltoallv/alltoallv.c alltoall = \ alltoall/alltoall.h \ alltoall/alltoall.c barrier = \ barrier/barrier.h \ barrier/barrier.c bcast = \ bcast/bcast.h \ bcast/bcast.c \ bcast/bcast_2step.c reduce = \ reduce/reduce.h \ reduce/reduce.c \ reduce/reduce_2step.c sources = \ cl_hier.h \ cl_hier.c \ cl_hier_lib.c \ cl_hier_context.c \ cl_hier_team.c \ cl_hier_coll.c \ cl_hier_coll.h \ $(allgatherv) \ $(allreduce) \ $(alltoallv) \ $(alltoall) \ $(barrier) \ $(bcast) \ $(reduce) module_LTLIBRARIES = libucc_cl_hier.la libucc_cl_hier_la_SOURCES = $(sources) libucc_cl_hier_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) libucc_cl_hier_la_CFLAGS = $(BASE_CFLAGS) libucc_cl_hier_la_LDFLAGS = -version-info $(SOVERSION) --as-needed libucc_cl_hier_la_LIBADD = $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am ucc-1.8.0/src/components/cl/hier/bcast/0000775000175000017500000000000015211535620020053 5ustar alastairalastairucc-1.8.0/src/components/cl/hier/bcast/bcast.c0000664000175000017500000000104315211535620021311 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "bcast.h" #include "../bcast/bcast.h" ucc_base_coll_alg_info_t ucc_cl_hier_bcast_algs[UCC_CL_HIER_BCAST_ALG_LAST + 1] = { [UCC_CL_HIER_BCAST_ALG_2STEP] = {.id = UCC_CL_HIER_BCAST_ALG_2STEP, .name = "2step", .desc = "intra-node and inter-node bcasts executed in parallel"}, [UCC_CL_HIER_BCAST_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc-1.8.0/src/components/cl/hier/bcast/bcast.h0000664000175000017500000000160115211535620021316 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef BCAST_H_ #define BCAST_H_ #include "../cl_hier.h" enum { UCC_CL_HIER_BCAST_ALG_2STEP, UCC_CL_HIER_BCAST_ALG_LAST, }; extern ucc_base_coll_alg_info_t ucc_cl_hier_bcast_algs[UCC_CL_HIER_BCAST_ALG_LAST + 1]; #define UCC_CL_HIER_BCAST_DEFAULT_ALG_SELECT_STR "bcast:0-4k:@2step" ucc_status_t ucc_cl_hier_bcast_2step_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); static inline int ucc_cl_hier_bcast_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_CL_HIER_BCAST_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_cl_hier_bcast_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/cl/hier/bcast/bcast_2step.c0000664000175000017500000002255415211535620022440 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "bcast.h" #include "core/ucc_team.h" #include "../cl_hier_coll.h" static ucc_status_t ucc_cl_hier_bcast_2step_start(ucc_coll_task_t *task) { UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_bcast_2step_start", 0); return ucc_schedule_start(task); } static ucc_status_t ucc_cl_hier_bcast_2step_finalize(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_status_t status; UCC_CL_HIER_PROFILE_REQUEST_EVENT(task, "cl_hier_bcast_2step_finalize", 0); status = ucc_schedule_finalize(task); ucc_cl_hier_put_schedule(schedule); return status; } static ucc_status_t ucc_cl_hier_bcast_2step_schedule_finalize(ucc_coll_task_t *task) { ucc_cl_hier_schedule_t *schedule = ucc_derived_of(task, ucc_cl_hier_schedule_t); ucc_status_t status; status = ucc_schedule_pipelined_finalize(&schedule->super.super.super); ucc_cl_hier_put_schedule(&schedule->super.super); return status; } static ucc_status_t ucc_cl_hier_bcast_2step_frag_setup(ucc_schedule_pipelined_t *schedule_p, ucc_schedule_t *frag, int frag_num) { ucc_coll_args_t *args = &schedule_p->super.super.bargs.args; size_t dt_size = ucc_dt_size(args->src.info.datatype); int n_frags = schedule_p->super.n_tasks; size_t frag_count, frag_offset; ucc_coll_task_t *task; int i; frag_count = ucc_buffer_block_count(args->src.info.count, n_frags, frag_num); frag_offset = ucc_buffer_block_offset(args->src.info.count, n_frags, frag_num); for (i = 0; i < frag->n_tasks; i++) { task = frag->tasks[i]; task->bargs.args.src.info.count = frag_count; task->bargs.args.src.info.buffer = PTR_OFFSET(args->src.info.buffer, frag_offset * dt_size); } return UCC_OK; } static inline ucc_rank_t find_root_net_rank(ucc_host_id_t root_host_id, ucc_cl_hier_team_t *cl_team) { ucc_sbgp_t *sbgp = cl_team->sbgps[UCC_HIER_SBGP_NODE_LEADERS].sbgp; ucc_team_t *core_team = cl_team->super.super.params.team; ucc_rank_t i, rank; for (i = 0; i < sbgp->group_size; i++) { rank = ucc_ep_map_eval(sbgp->map, i); if (ucc_team_rank_host_id(rank, core_team) == root_host_id) { return i; } } return UCC_RANK_INVALID; } static inline ucc_rank_t find_root_node_rank(ucc_rank_t root, ucc_cl_hier_team_t *cl_team) { ucc_sbgp_t *sbgp = cl_team->sbgps[UCC_HIER_SBGP_NODE].sbgp; ucc_rank_t i; for (i = 0; i < sbgp->group_size; i++) { if (ucc_ep_map_eval(sbgp->map, i) == root) { return i; } } return UCC_RANK_INVALID; } static ucc_status_t ucc_cl_hier_bcast_2step_init_schedule(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_schedule_t **sched_p, int n_frags) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_team_t *core_team = team->params.team; ucc_coll_task_t *tasks[2] = {NULL, NULL}; ucc_rank_t root = coll_args->args.root; ucc_rank_t rank = UCC_TL_TEAM_RANK(cl_team); int root_on_local_node = ucc_team_ranks_on_same_node(root, rank, core_team); ucc_base_coll_args_t args = *coll_args; int n_tasks = 0; int first_task = 0; ucc_schedule_t *schedule; ucc_status_t status; int i; schedule = &ucc_cl_hier_get_schedule(cl_team)->super.super; if (ucc_unlikely(!schedule)) { return UCC_ERR_NO_MEMORY; } status = ucc_schedule_init(schedule, &args, team); if (ucc_unlikely(UCC_OK != status)) { goto out; } if (n_frags > 1) { args.max_frag_count = ucc_buffer_block_count(args.args.src.info.count, n_frags, 0); args.mask |= UCC_BASE_CARGS_MAX_FRAG_COUNT; } ucc_assert(SBGP_ENABLED(cl_team, NODE_LEADERS) || SBGP_ENABLED(cl_team, NODE)); if (SBGP_ENABLED(cl_team, NODE_LEADERS)) { args.args.root = find_root_net_rank( ucc_team_rank_host_id(root, core_team), cl_team); status = ucc_coll_init(SCORE_MAP(cl_team, NODE_LEADERS), &args, &tasks[n_tasks]); if (ucc_unlikely(UCC_OK != status)) { goto out; } n_tasks++; if (root_on_local_node && (root != rank)) { first_task = 1; } } if (SBGP_ENABLED(cl_team, NODE)) { args.args.root = root_on_local_node ? find_root_node_rank(root, cl_team) : core_team->topo->node_leader_rank_id; status = ucc_coll_init(SCORE_MAP(cl_team, NODE), &args, &tasks[n_tasks]); if (ucc_unlikely(UCC_OK != status)) { goto out; } n_tasks++; } UCC_CHECK_GOTO(ucc_task_subscribe_dep(&schedule->super, tasks[first_task], UCC_EVENT_SCHEDULE_STARTED), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, tasks[first_task]), out, status); if (n_tasks > 1) { if (root == rank) { UCC_CHECK_GOTO(ucc_task_subscribe_dep(&schedule->super, tasks[(first_task + 1) % 2], UCC_EVENT_SCHEDULE_STARTED), out, status); } else { UCC_CHECK_GOTO(ucc_task_subscribe_dep(tasks[first_task], tasks[(first_task + 1) % 2], UCC_EVENT_COMPLETED), out, status); } UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, tasks[(first_task + 1) % 2]), out, status); } schedule->super.post = ucc_cl_hier_bcast_2step_start; schedule->super.progress = NULL; schedule->super.finalize = ucc_cl_hier_bcast_2step_finalize; *sched_p = schedule; return UCC_OK; out: for (i = 0; i < n_tasks; i++) { tasks[i]->finalize(tasks[i]); } ucc_cl_hier_put_schedule(schedule); return status; } static ucc_status_t ucc_cl_hier_bcast_2step_frag_init( ucc_base_coll_args_t *coll_args, ucc_schedule_pipelined_t *sp, ucc_base_team_t *team, ucc_schedule_t **frag_p) { int n_frags = sp->super.n_tasks; return ucc_cl_hier_bcast_2step_init_schedule(coll_args, team, frag_p, n_frags); } static ucc_status_t ucc_cl_hier_2step_bcast_start(ucc_coll_task_t *task) { ucc_schedule_pipelined_t *schedule = ucc_derived_of(task, ucc_schedule_pipelined_t); cl_debug(task->team->context->lib, "posting 2step bcast, buf %p, count %zd, dt %s" " pdepth %d, frags_total %d", task->bargs.args.src.info.buffer, task->bargs.args.src.info.count, ucc_datatype_str(task->bargs.args.src.info.datatype), schedule->n_frags, schedule->super.n_tasks); return ucc_schedule_pipelined_post(task); } UCC_CL_HIER_PROFILE_FUNC(ucc_status_t, ucc_cl_hier_bcast_2step_init, (coll_args, team, task), ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task) { ucc_cl_hier_team_t *cl_team = ucc_derived_of(team, ucc_cl_hier_team_t); ucc_cl_hier_lib_config_t *cfg = &UCC_CL_HIER_TEAM_LIB(cl_team)->cfg; ucc_cl_hier_schedule_t * schedule; int n_frags, pipeline_depth; ucc_status_t status; if (UCC_IS_PERSISTENT(coll_args->args)) { return UCC_ERR_NOT_SUPPORTED; } ucc_pipeline_nfrags_pdepth(&cfg->bcast_2step_pipeline, coll_args->args.src.info.count * ucc_dt_size(coll_args->args.src.info.datatype), &n_frags, &pipeline_depth); if (n_frags == 1) { return ucc_cl_hier_bcast_2step_init_schedule( coll_args, team, (ucc_schedule_t **)task, n_frags); } schedule = ucc_cl_hier_get_schedule(cl_team); if (ucc_unlikely(!schedule)) { return UCC_ERR_NO_MEMORY; } status = ucc_schedule_pipelined_init( coll_args, team, ucc_cl_hier_bcast_2step_frag_init, ucc_cl_hier_bcast_2step_frag_setup, pipeline_depth, n_frags, cfg->bcast_2step_pipeline.order, &schedule->super); if (ucc_unlikely(status != UCC_OK)) { cl_error(team->context->lib, "failed to init pipelined 2step bcast schedule"); goto err_pipe_init; } schedule->super.super.super.post = ucc_cl_hier_2step_bcast_start; schedule->super.super.super.finalize = ucc_cl_hier_bcast_2step_schedule_finalize; *task = &schedule->super.super.super; return UCC_OK; err_pipe_init: ucc_cl_hier_put_schedule(&schedule->super.super); return status; } ucc-1.8.0/src/components/cl/hier/cl_hier_lib.c0000664000175000017500000001171715211535620021365 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "cl_hier.h" #include "utils/ucc_malloc.h" #include "components/tl/ucc_tl.h" #include "core/ucc_global_opts.h" #include "utils/ucc_math.h" /* NOLINTNEXTLINE TODO params is not used*/ UCC_CLASS_INIT_FUNC(ucc_cl_hier_lib_t, const ucc_base_lib_params_t *params, const ucc_base_config_t *config) { const ucc_cl_hier_lib_config_t *cl_hier_config = ucc_derived_of(config, ucc_cl_hier_lib_config_t); ucc_config_names_array_t requested_sbgp_tls; ucc_status_t status; int i; UCC_CLASS_CALL_SUPER_INIT(ucc_cl_lib_t, &ucc_cl_hier.super, &cl_hier_config->super); memcpy(&self->cfg, cl_hier_config, sizeof(*cl_hier_config)); requested_sbgp_tls.count = 0; for (i = 0; i < UCC_HIER_SBGP_LAST; i++) { status = ucc_config_allow_list_process(&cl_hier_config->sbgp_tls[i].list, &self->super.tls.array, &self->cfg.sbgp_tls[i]); if (UCC_OK != status) { cl_error(&self->super, "failed to process sbgp tls array"); return status; } if (self->cfg.sbgp_tls[i].array.count == 0) { cl_error(&self->super, "no TLs are available for sbgp"); return UCC_ERR_INVALID_PARAM; } status = ucc_config_names_array_merge(&requested_sbgp_tls, &self->cfg.sbgp_tls[i].array); if (ucc_unlikely(UCC_OK != status)) { cl_error(&self->super, "failed to merge tls config names arrays"); return status; } } ucc_assert(requested_sbgp_tls.count > 0); self->tls.array.count = requested_sbgp_tls.count; self->tls.array.names = requested_sbgp_tls.names; cl_debug(&self->super, "initialized lib object: %p", self); return status; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_hier_lib_t) { int i; cl_debug(&self->super, "finalizing lib object: %p", self); for (i = 0; i < UCC_HIER_SBGP_LAST; i++) { ucc_config_names_array_free(&self->cfg.sbgp_tls[i].array); } ucc_config_names_array_free(&self->tls.array); } UCC_CLASS_DEFINE(ucc_cl_hier_lib_t, ucc_cl_lib_t); static inline ucc_status_t check_tl_lib_attr(const ucc_base_lib_t *lib, ucc_tl_iface_t * tl_iface, ucc_cl_lib_attr_t * attr) { ucc_tl_lib_attr_t tl_attr; ucc_status_t status; memset(&tl_attr, 0, sizeof(tl_attr)); status = tl_iface->lib.get_attr(NULL, &tl_attr.super); if (UCC_OK != status) { cl_error(lib, "failed to query tl %s lib attributes", tl_iface->super.name); return status; } attr->super.attr.thread_mode = ucc_min(attr->super.attr.thread_mode, tl_attr.super.attr.thread_mode); attr->super.flags |= tl_attr.super.flags; return UCC_OK; } ucc_status_t ucc_cl_hier_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr) { ucc_cl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_cl_lib_attr_t); ucc_cl_hier_lib_t *cl_lib = ucc_derived_of(lib, ucc_cl_hier_lib_t); ucc_config_allow_list_t *tls = &cl_lib->tls; ucc_tl_iface_t *tl_iface; int i; ucc_status_t status; attr->tls = &cl_lib->tls.array; for (i = 0; i < UCC_HIER_SBGP_LAST; i++) { if (cl_lib->cfg.sbgp_tls[i].requested) { status = ucc_config_names_array_merge( &cl_lib->super.tls_forced, &cl_lib->cfg.sbgp_tls[i].array); if (ucc_unlikely(UCC_OK != status)) { return status; } } } attr->tls_forced = &cl_lib->super.tls_forced; attr->super.attr.thread_mode = UCC_THREAD_MULTIPLE; attr->super.attr.coll_types = UCC_CL_HIER_SUPPORTED_COLLS; attr->super.flags = 0; ucc_assert(tls->array.count >= 1); for (i = 0; i < tls->array.count; i++) { /* Check TLs proveded in CL_HIER_TLS. Not all of them could be available, check for NULL. */ tl_iface = ucc_derived_of(ucc_get_component(&ucc_global_config.tl_framework, tls->array.names[i]), ucc_tl_iface_t); if (!tl_iface) { cl_warn(lib, "tl %s is not available", tls->array.names[i]); continue; } if (UCC_OK != (status = check_tl_lib_attr(lib, tl_iface, attr))) { return status; } } return UCC_OK; } ucc_status_t ucc_cl_hier_get_lib_properties(ucc_base_lib_properties_t *prop) { prop->default_team_size = 2; prop->min_team_size = 2; prop->max_team_size = UCC_RANK_MAX; return UCC_OK; } ucc-1.8.0/src/components/cl/ucc_cl.c0000664000175000017500000001464515211535620017436 0ustar alastairalastair/** * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_cl.h" #include "utils/ucc_log.h" #include "utils/ucc_malloc.h" #include "core/ucc_global_opts.h" static char * ucc_cl_tls_doc_str = "List of TLs used by a given CL component.\n" "Allowed values: either \"all\" or comma-separated list of: "; #define TLS_CONFIG_ENTRY 1 ucc_config_field_t ucc_cl_lib_config_table[] = { [0] = {"", "", NULL, ucc_offsetof(ucc_cl_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_base_lib_config_table)}, [TLS_CONFIG_ENTRY] = {"TLS", "all", NULL, ucc_offsetof(ucc_cl_lib_config_t, tls), UCC_CONFIG_TYPE_ALLOW_LIST}, {NULL}}; ucc_config_field_t ucc_cl_context_config_table[] = { [0] = {"", "", NULL, ucc_offsetof(ucc_cl_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_base_ctx_config_table)}, {NULL} }; const char *ucc_cl_names[] = { [UCC_CL_BASIC] = "basic", [UCC_CL_HIER] = "hier", [UCC_CL_DOCA_UROM] = "doca_urom", [UCC_CL_ALL] = "all", [UCC_CL_LAST] = NULL }; UCC_CLASS_INIT_FUNC(ucc_cl_lib_t, ucc_cl_iface_t *cl_iface, const ucc_cl_lib_config_t *cl_config) { ucc_status_t status; UCC_CLASS_CALL_BASE_INIT(); self->iface = cl_iface; self->super.use_tuning = cl_config->super.use_tuning; self->super.log_component = cl_config->super.log_component; ucc_strncpy_safe(self->super.log_component.name, cl_iface->cl_lib_config.name, sizeof(self->super.log_component.name)); status = ucc_config_allow_list_process( &cl_config->tls, &ucc_global_config.tl_framework.names, &self->tls); if (status != UCC_OK) { return status; } if (self->tls.array.count == 0) { ucc_error("no TLs are selected for %s", cl_iface->cl_lib_config.name); ucc_free(self->tls.array.names); return UCC_ERR_NOT_FOUND; } self->tls_forced.count = 0; self->tls_forced.names = NULL; return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_lib_t) { ucc_config_names_array_free(&self->tls.array); ucc_config_names_array_free(&self->tls_forced); } UCC_CLASS_DEFINE(ucc_cl_lib_t, void); ucc_status_t ucc_parse_cls_string(const char *cls_str, ucc_cl_type_t **cls_array, int *n_cls) { int cls_selected[UCC_CL_LAST] = {0}; int n_cls_selected = 0; char *cls_copy, *saveptr, *cl; ucc_cl_type_t *cls; ucc_cl_type_t cl_type; cls_copy = strdup(cls_str); if (!cls_copy) { ucc_error("failed to create a copy cls string"); return UCC_ERR_NO_MEMORY; } cl = strtok_r(cls_copy, ",", &saveptr); while (NULL != cl) { cl_type = ucc_cl_name_to_type(cl); if (cl_type == UCC_CL_LAST) { ucc_error("incorrect value is passed as part of UCC_CLS list: %s", cl); ucc_free(cls_copy); return UCC_ERR_INVALID_PARAM; } n_cls_selected++; cls_selected[cl_type] = 1; cl = strtok_r(NULL, ",", &saveptr); } ucc_free(cls_copy); if (n_cls_selected == 0) { ucc_error("incorrect value is passed as part of UCC_CLS list: %s", cls_str); return UCC_ERR_INVALID_PARAM; } cls = ucc_malloc(n_cls_selected * sizeof(ucc_cl_type_t), "cls_array"); if (!cls) { ucc_error("failed to allocate %zd bytes for cls_array", n_cls_selected * sizeof(int)); return UCC_ERR_NO_MEMORY; } n_cls_selected = 0; for (cl_type = (ucc_cl_type_t)0; cl_type < UCC_CL_LAST; cl_type++) { if (cls_selected[cl_type]) { cls[n_cls_selected++] = cl_type; } } *cls_array = cls; *n_cls = n_cls_selected; return UCC_OK; } UCC_CLASS_INIT_FUNC(ucc_cl_context_t, const ucc_cl_context_config_t *cl_config, ucc_context_t *ucc_context) { UCC_CLASS_CALL_BASE_INIT(); self->super.lib = &cl_config->cl_lib->super; self->super.ucc_context = ucc_context; if (0 == strcmp(cl_config->super.score_str, "0")) { return UCC_ERR_LAST; } self->super.score_str = strdup(cl_config->super.score_str); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_context_t) { ucc_free(self->super.score_str); } UCC_CLASS_DEFINE(ucc_cl_context_t, void); ucc_status_t ucc_cl_context_config_read(ucc_cl_lib_t *cl_lib, const ucc_context_config_t *config, ucc_cl_context_config_t **cl_config) { ucc_status_t status; status = ucc_base_config_read(config->lib->full_prefix, &cl_lib->iface->cl_context_config, (ucc_base_config_t **)cl_config); if (UCC_OK == status) { (*cl_config)->cl_lib = cl_lib; } return status; } ucc_status_t ucc_cl_lib_config_read(ucc_cl_iface_t *iface, const char *full_prefix, ucc_cl_lib_config_t **cl_config) { char *tls_list; char *doc_str; size_t doc_len; if (!ucc_cl_lib_config_table[TLS_CONFIG_ENTRY].doc) { tls_list = ucc_get_framework_components_list( &ucc_global_config.tl_framework, ","); if (tls_list) { doc_len = strlen(ucc_cl_tls_doc_str) + strlen(tls_list) + 1; doc_str = ucc_malloc(doc_len, "cl_tls_doc"); if (!doc_str) { ucc_error("failed to allocate %zd bytes for cl_tls_doc", doc_len); } else { ucc_snprintf_safe(doc_str, doc_len, "%s%s", ucc_cl_tls_doc_str, tls_list); ucc_cl_lib_config_table[TLS_CONFIG_ENTRY].doc = doc_str; } ucc_free(tls_list); } } return ucc_base_config_read(full_prefix, &iface->cl_lib_config, (ucc_base_config_t **)cl_config); } UCC_CLASS_INIT_FUNC(ucc_cl_team_t, ucc_cl_context_t *cl_context, const ucc_base_team_params_t *params) { UCC_CLASS_CALL_BASE_INIT(); self->super.context = &cl_context->super; self->super.params = *params; return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_cl_team_t) { } UCC_CLASS_DEFINE(ucc_cl_team_t, void); ucc-1.8.0/src/components/cl/ucc_cl_log.h0000664000175000017500000000130115211535620020265 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_CL_LOG_H_ #define UCC_CL_LOG_H_ #include "components/base/ucc_base_iface.h" #define cl_error(_cl_lib, _fmt, ...) base_error((ucc_base_lib_t*)(_cl_lib), _fmt, ## __VA_ARGS__) #define cl_warn(_cl_lib, _fmt, ...) base_warn((ucc_base_lib_t*)(_cl_lib), _fmt, ## __VA_ARGS__) #define cl_info(_cl_lib, _fmt, ...) base_info((ucc_base_lib_t*)(_cl_lib), _fmt, ## __VA_ARGS__) #define cl_debug(_cl_lib, _fmt, ...) base_debug((ucc_base_lib_t*)(_cl_lib), _fmt, ## __VA_ARGS__) #define cl_trace(_cl_lib, _fmt, ...) base_trace((ucc_base_lib_t*)(_cl_lib), _fmt, ## __VA_ARGS__) #endif ucc-1.8.0/src/components/cl/ucc_cl_type.h0000664000175000017500000000157515211535620020502 0ustar alastairalastair/** * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_CL_TYPE_H_ #define UCC_CL_TYPE_H_ #include typedef enum { UCC_CL_BASIC, UCC_CL_HIER, UCC_CL_DOCA_UROM, UCC_CL_ALL, UCC_CL_LAST } ucc_cl_type_t; extern const char *ucc_cl_names[]; static inline ucc_cl_type_t ucc_cl_name_to_type(const char *cl_name) { int i; for (i = 0; i < UCC_CL_LAST; i++) { if (0 == strcmp(cl_name, ucc_cl_names[i])) { break; } } return (ucc_cl_type_t)i; } /* takes string of comma separated cls and returns and array of ucc_cl_type_t. the checks for correct input are done. the allocated should be freed by the user. */ ucc_status_t ucc_parse_cls_string(const char *cls_str, ucc_cl_type_t **cls_array, int *n_cls); #endif ucc-1.8.0/src/components/base/0000775000175000017500000000000015211535620016344 5ustar alastairalastairucc-1.8.0/src/components/base/ucc_base_iface.c0000664000175000017500000000615215211535620021407 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_base_iface.h" #include "utils/ucc_malloc.h" #include "utils/ucc_log.h" ucc_config_field_t ucc_base_lib_config_table[] = { {"LOG_LEVEL", "warn", "UCC logging level. Messages with a level higher or equal to the " "selected will be printed.\n" "Possible values are: fatal, error, warn, info, debug, trace, data, func, " "poll.", ucc_offsetof(ucc_base_lib_config_t, log_component), UCC_CONFIG_TYPE_LOG_COMP}, {"USE_TUNING", "y", "Use perf tuning", ucc_offsetof(ucc_base_lib_config_t, use_tuning), UCC_CONFIG_TYPE_BOOL}, {"MIN_TEAM_SIZE", "auto", "minimal team size for which component can be used", ucc_offsetof(ucc_base_lib_config_t, min_team_size), UCC_CONFIG_TYPE_ULUNITS}, {NULL}}; ucc_config_field_t ucc_base_ctx_config_table[] = { {"TUNE", "", "Collective tuning modifier for a CL/TL component\n" "format: token1#token2#...#tokenn - '#' separated list of tokens where\n" " token=coll_type:msg_range:mem_type:team_size:score:alg - ':' separated\n" " list of qualifiers. Each qualifier is optional. The only requirement\n" " is that either \"score\" or \"alg\" is provided.\n" "qualifiers:\n" " coll_type=coll_type_1,coll_type_2,...,coll_type_n - ',' separated\n" " list of coll_types\n" " msg_range=m_start_1-m_end_1,m_start_2-m_end_2,..,m_start_n-m_end_n -\n" " ',' separated list of msg ranges, where each range is\n" " represented by \"start\" and \"end\" values separated by \"-\".\n" " Special value \"inf\" means MAX msg size.\n" " mem_type=m1,m2,..,mn - ',' separated list of memory types\n" " team_size=[t_start_1-t_end_1,t_start_2-t_end_2,...,t_start_n-t_end_n] -\n" " ',' separated list of team size ranges enclosed with [].\n" " score=value - int value from 0 to \"inf\"\n" " 0 - disables the CL/TL in the given range for a given coll\n" " inf - forces the CL/TL in the given range for a given coll\n" " alg=@ - character @ followed by either int number or string\n" " representing the collective algorithm.", ucc_offsetof(ucc_base_ctx_config_t, score_str), UCC_CONFIG_TYPE_STRING}, {NULL}}; ucc_status_t ucc_base_config_read(const char *full_prefix, ucc_config_global_list_entry_t *cfg_entry, ucc_base_config_t **config) { ucc_base_config_t *cfg; ucc_status_t status; cfg = ucc_malloc(cfg_entry->size, "cl_ctx_cfg"); if (!cfg) { ucc_error("failed to allocate %zd bytes for cl context config", cfg_entry->size); return UCC_ERR_NO_MEMORY; } cfg->cfg_entry = cfg_entry; status = ucc_config_parser_fill_opts(cfg, cfg_entry, full_prefix, 0); if (UCC_OK != status) { ucc_free(cfg); *config = NULL; } else { *config = cfg; } return status; } ucc-1.8.0/src/components/base/ucc_base_iface.h0000664000175000017500000003122715211535620021415 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #ifndef UCC_BASE_IFACE_H_ #define UCC_BASE_IFACE_H_ #include "ucc/api/ucc.h" #include "core/ucc_lib.h" #include "utils/ucc_component.h" #include "utils/ucc_parser.h" #include "utils/ucc_class.h" #include "utils/ucc_malloc.h" #include "utils/ucc_log.h" #include "utils/ucc_coll_utils.h" typedef struct ucc_team ucc_team_t; typedef struct ucc_context ucc_context_t; typedef struct ucc_coll_score ucc_coll_score_t; typedef struct ucc_coll_task ucc_coll_task_t; typedef struct ucc_mem_map_memh_t ucc_mem_map_memh_t; typedef struct ucc_mem_map_tl_t ucc_mem_map_tl_t; typedef struct ucc_base_lib { ucc_log_component_config_t log_component; int use_tuning; unsigned long min_team_size; } ucc_base_lib_t; typedef struct ucc_base_config { ucc_config_global_list_entry_t *cfg_entry; } ucc_base_config_t; typedef struct ucc_base_lib_config { ucc_base_config_t super; ucc_log_component_config_t log_component; int use_tuning; unsigned long min_team_size; } ucc_base_lib_config_t; typedef struct ucc_base_ctx_config { ucc_base_config_t super; char *score_str; } ucc_base_ctx_config_t; enum { UCC_BASE_LIB_FLAG_TEAM_ID_REQUIRED = UCC_BIT(1), UCC_BASE_LIB_FLAG_SERVICE_TEAM_REQUIRED = UCC_BIT(2), UCC_BASE_LIB_FLAG_CTX_SERVICE_TEAM_REQUIRED = UCC_BIT(3) }; enum { UCC_BASE_LIB_ATTR_FIELD_MIN_TEAM_SIZE = UCC_BIT(0), UCC_BASE_LIB_ATTR_FIELD_MAX_TEAM_SIZE = UCC_BIT(1), UCC_BASE_LIB_ATTR_FIELD_FLAGS = UCC_BIT(2), }; typedef struct ucc_base_lib_attr_t { uint64_t mask; ucc_lib_attr_t attr; ucc_rank_t min_team_size; ucc_rank_t max_team_size; uint64_t flags; } ucc_base_lib_attr_t; typedef struct ucc_base_lib_properties { ucc_rank_t min_team_size; ucc_rank_t max_team_size; ucc_rank_t default_team_size; } ucc_base_lib_properties_t; typedef struct ucc_base_lib_params { ucc_lib_params_t params; char *full_prefix; } ucc_base_lib_params_t; extern ucc_config_field_t ucc_base_lib_config_table[]; extern ucc_config_field_t ucc_base_ctx_config_table[]; typedef struct ucc_base_lib_iface { ucc_status_t (*init)(const ucc_base_lib_params_t *params, const ucc_base_config_t *config, ucc_base_lib_t **lib); void (*finalize)(ucc_base_lib_t *lib); ucc_status_t (*get_attr)(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *attr); ucc_status_t (*get_properties)(ucc_base_lib_properties_t *prop); } ucc_base_lib_iface_t; typedef struct ucc_base_context_params { ucc_context_params_t params; int estimated_num_eps; int estimated_num_ppn; int node_local_id; ucc_thread_mode_t thread_mode; const char *prefix; ucc_context_t *context; } ucc_base_context_params_t; typedef struct ucc_base_context { ucc_context_t *ucc_context; ucc_base_lib_t *lib; char *score_str; } ucc_base_context_t; typedef struct ucc_base_ctx_attr_t { ucc_context_attr_t attr; uint32_t topo_required; } ucc_base_ctx_attr_t; static inline void ucc_base_ctx_attr_clear(ucc_base_ctx_attr_t *attr) { uint64_t mask = attr->attr.mask; memset(attr, 0, sizeof(ucc_base_ctx_attr_t)); attr->attr.mask = mask; } typedef struct ucc_base_context_iface { ucc_status_t (*create)(const ucc_base_context_params_t *params, const ucc_base_config_t *config, ucc_base_context_t **ctx); ucc_status_t (*create_epilog)(ucc_base_context_t *ctx); void (*destroy)(ucc_base_context_t *ctx); ucc_status_t (*get_attr)(const ucc_base_context_t *context, ucc_base_ctx_attr_t *attr); /* maps a memory-region specified by memory handle, memh, to a tl specific handle, tl_h, based on the mapping mode defined by mode. For the export mode, the TL will map a local memory-region memory and store the necessary information in the tl_h. For the import mode, the TL will map, if necessary, memory handles provided by a peer and store the necessary information in the tl_h. */ ucc_status_t (*mem_map)(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h); /* unmaps a memory-region previously mapped to a specific TL pointed to by tl_h with a mode of mapping by mode. */ ucc_status_t (*mem_unmap)(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h); /* packs necessary TL specific elements for a mapped memory-region to a packed buffer. Each TL implementing this function should set the packed_size member of the tl_h, allocate memory for the pack_buffer, and pack data in the buffer. */ ucc_status_t (*memh_pack)(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **pack_buffer); } ucc_base_context_iface_t; typedef struct ucc_base_team_params { ucc_team_params_t params; int scope; /* Scope that allocates the team. When TL team is created the scope would be a CL_TYPE. This provides a separation of teams created from different CLs with the same TL_TYPE */ int scope_id; /* The id of the base_team in the specified scope. Use case: a single CL team (e.g. basic) creates multiple TL teams of the same type (e.g. several tl_ucp teams). In this case CL would give those teams different scope_id. */ ucc_rank_t rank; /* Rank of a calling process in the TL/CL team. It is a uniq process identifier within a team (not job) but it has the property: it is always contig and in the range [0, team_size).*/ ucc_rank_t size; /* Size of the TL team. size <= team->size (tl can be a subset of the core team) */ uint16_t id; /* core level team id */ ucc_team_t * team; /* core team pointer */ ucc_ep_map_t map; /* ranks map to the core ucc team */ } ucc_base_team_params_t; typedef struct ucc_base_team { ucc_base_context_t *context; ucc_base_team_params_t params; } ucc_base_team_t; typedef ucc_status_t (*ucc_get_coll_scores_fn_t)(ucc_base_team_t *team, ucc_coll_score_t **score); typedef struct ucc_base_team_iface { ucc_status_t (*create_post)(ucc_base_context_t *context, const ucc_base_team_params_t *params, ucc_base_team_t **team); ucc_status_t (*create_test)(ucc_base_team_t *team); ucc_status_t (*destroy)(ucc_base_team_t *team); ucc_get_coll_scores_fn_t get_scores; } ucc_base_team_iface_t; enum { UCC_BASE_CARGS_MAX_FRAG_COUNT = UCC_BIT(0), /* Info is available on nonroot ranks */ UCC_BASE_CARGS_NONROOT_INFO = UCC_BIT(1) }; typedef struct ucc_buffer_info_asymmetric_memtype { union { ucc_coll_buffer_info_t info; ucc_coll_buffer_info_v_t info_v; } old_asymmetric_buffer; ucc_mc_buffer_header_t *scratch; } ucc_buffer_info_asymmetric_memtype_t; typedef struct ucc_base_coll_args { uint64_t mask; ucc_coll_args_t args; ucc_team_t *team; size_t max_frag_count; /* For asymmetric mem types across ranks */ ucc_buffer_info_asymmetric_memtype_t asymmetric_save_info; } ucc_base_coll_args_t; typedef ucc_status_t (*ucc_base_coll_init_fn_t)(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); typedef struct ucc_base_coll_iface { ucc_base_coll_init_fn_t init; } ucc_base_coll_iface_t; ucc_status_t ucc_base_config_read(const char *full_prefix, ucc_config_global_list_entry_t *cfg_entry, ucc_base_config_t **config); static inline void ucc_base_config_release(ucc_base_config_t *config) { ucc_config_parser_release_opts(config, config->cfg_entry->table); ucc_free(config); } typedef struct ucc_base_coll_alg_info { unsigned id; const char *name; const char *desc; } ucc_base_coll_alg_info_t; #define UCC_IFACE_NAME_PREFIX(_F, _NAME, _cfg) \ .name = UCC_PP_MAKE_STRING(_F##_NAME), \ .prefix = UCC_PP_MAKE_STRING(_F##_NAME##_) #define UCC_IFACE_CFG(_F, _f, _cfg, _name, _NAME) \ .super._f##_cfg##_config = { \ UCC_IFACE_NAME_PREFIX(_F, _NAME, _cfg), \ .table = ucc_##_f##_name##_##_cfg##_config_table, \ .size = sizeof(ucc_##_f##_name##_##_cfg##_config_t)} #define UCC_BASE_IFACE_DECLARE(_F, _f, _name, _NAME) \ ucc_##_f##_name##_iface_t ucc_##_f##_name = { \ UCC_IFACE_CFG(_F, _f, lib, _name, _NAME), \ UCC_IFACE_CFG(_F, _f, context, _name, _NAME), \ .super.super.score = UCC_##_F##_NAME##_DEFAULT_SCORE, \ .super.super.name = UCC_PP_MAKE_STRING(_name), \ .super.lib.init = UCC_CLASS_NEW_FUNC_NAME(ucc_##_f##_name##_lib_t), \ .super.lib.finalize = \ UCC_CLASS_DELETE_FUNC_NAME(ucc_##_f##_name##_lib_t), \ .super.lib.get_attr = ucc_##_f##_name##_get_lib_attr, \ .super.lib.get_properties = ucc_##_f##_name##_get_lib_properties, \ .super.context.create = \ UCC_CLASS_NEW_FUNC_NAME(ucc_##_f##_name##_context_t), \ .super.context.create_epilog = NULL, \ .super.context.destroy = \ UCC_CLASS_DELETE_FUNC_NAME(ucc_##_f##_name##_context_t), \ .super.context.get_attr = ucc_##_f##_name##_get_context_attr, \ .super.context.mem_map = ucc_##_f##_name##_mem_map, \ .super.context.mem_unmap = ucc_##_f##_name##_mem_unmap, \ .super.context.memh_pack = ucc_##_f##_name##_memh_pack, \ .super.team.create_post = \ UCC_CLASS_NEW_FUNC_NAME(ucc_##_f##_name##_team_t), \ .super.team.create_test = ucc_##_f##_name##_team_create_test, \ .super.team.destroy = ucc_##_f##_name##_team_destroy, \ .super.team.get_scores = ucc_##_f##_name##_team_get_scores, \ .super.coll.init = ucc_##_f##_name##_coll_init, \ .super.alg_info = {NULL}}; \ UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_##_f##_name.super._f##lib_config, \ &ucc_config_global_list); \ UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_##_f##_name.super._f##context_config, \ &ucc_config_global_list) #define ucc_base_log(_lib, _level, fmt, ...) \ ucc_log_component(_level, &(_lib)->log_component, fmt, \ ##__VA_ARGS__) #define base_error(_lib, _fmt, ...) \ ucc_base_log(_lib, UCC_LOG_LEVEL_ERROR, _fmt, ##__VA_ARGS__) #define base_warn(_lib, _fmt, ...) \ ucc_base_log(_lib, UCC_LOG_LEVEL_WARN, _fmt, ##__VA_ARGS__) #define base_info(_lib, _fmt, ...) \ ucc_base_log(_lib, UCC_LOG_LEVEL_INFO, _fmt, ##__VA_ARGS__) #define base_debug(_lib, _fmt, ...) \ ucc_base_log(_lib, UCC_LOG_LEVEL_DEBUG, _fmt, ##__VA_ARGS__) #define base_trace(_lib, _fmt, ...) \ ucc_base_log(_lib, UCC_LOG_LEVEL_TRACE, _fmt, ##__VA_ARGS__) #endif ucc-1.8.0/src/components/tl/0000775000175000017500000000000015211535620016051 5ustar alastairalastairucc-1.8.0/src/components/tl/nccl/0000775000175000017500000000000015211535620016770 5ustar alastairalastairucc-1.8.0/src/components/tl/nccl/tl_nccl_coll.c0000664000175000017500000012014015211535620021561 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Facebook, Inc. and its affiliates. 2021. * * See file LICENSE for terms. */ #include "tl_nccl_coll.h" #include "components/mc/ucc_mc.h" #include "components/ec/ucc_ec.h" #include "core/ucc_ee.h" #include "utils/ucc_compiler_def.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "utils/arch/cuda_def.h" #include "allgatherv/allgatherv.h" #define ncclOpUnsupported (ncclNumOps + 1) #define ncclDataTypeUnsupported (ncclNumTypes + 1) ncclDataType_t ucc_to_nccl_dtype[] = { [UCC_DT_PREDEFINED_ID(UCC_DT_INT8)] = (ncclDataType_t)ncclInt8, [UCC_DT_PREDEFINED_ID(UCC_DT_INT16)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_INT32)] = (ncclDataType_t)ncclInt32, [UCC_DT_PREDEFINED_ID(UCC_DT_INT64)] = (ncclDataType_t)ncclInt64, [UCC_DT_PREDEFINED_ID(UCC_DT_INT128)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT8)] = (ncclDataType_t)ncclUint8, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT16)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT32)] = (ncclDataType_t)ncclUint32, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT64)] = (ncclDataType_t)ncclUint64, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT128)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT16)] = (ncclDataType_t)ncclFloat16, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT32)] = (ncclDataType_t)ncclFloat32, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT64)] = (ncclDataType_t)ncclFloat64, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT128)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT32_COMPLEX)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT64_COMPLEX)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT128_COMPLEX)] = (ncclDataType_t)ncclDataTypeUnsupported, #if (CUDART_VERSION >= 11000) && (NCCL_VERSION_CODE >= NCCL_VERSION(2,10,3)) [UCC_DT_PREDEFINED_ID(UCC_DT_BFLOAT16)] = (ncclDataType_t)ncclBfloat16, #else [UCC_DT_PREDEFINED_ID(UCC_DT_BFLOAT16)] = (ncclDataType_t)ncclDataTypeUnsupported, #endif }; ncclRedOp_t ucc_to_nccl_reduce_op[] = { [UCC_OP_SUM] = (ncclRedOp_t)ncclSum, [UCC_OP_PROD] = (ncclRedOp_t)ncclProd, [UCC_OP_MAX] = (ncclRedOp_t)ncclMax, [UCC_OP_MIN] = (ncclRedOp_t)ncclMin, #if NCCL_VERSION_CODE < NCCL_VERSION(2,10,3) [UCC_OP_AVG] = (ncclRedOp_t)ncclOpUnsupported, #else [UCC_OP_AVG] = (ncclRedOp_t)ncclAvg, #endif [UCC_OP_LAND] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_LOR] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_LXOR] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_BAND] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_BOR] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_BXOR] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_MAXLOC] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_MINLOC] = (ncclRedOp_t)ncclOpUnsupported, }; const char *ucc_tl_nccl_default_alg_select_str[UCC_TL_NCCL_N_DEFAULT_ALG_SELECT_STR] = { UCC_TL_NCCL_ALLGATHERV_DEFAULT_ALG_SELECT_STR}; static inline void ucc_tl_nccl_check_and_convert_buffer(ucc_coll_buffer_info_t *buffer_info, ucc_datatype_t new_datatype) { if (ucc_to_nccl_dtype[UCC_DT_PREDEFINED_ID(buffer_info->datatype)] == ncclDataTypeUnsupported) { ucc_assert(ucc_dt_size(buffer_info->datatype) % ucc_dt_size(new_datatype) == 0); buffer_info->count *= ucc_dt_size(buffer_info->datatype) / ucc_dt_size(new_datatype); buffer_info->datatype = new_datatype; } } static inline ucc_status_t ucc_tl_nccl_check_and_convert_buffer_reduction( ucc_coll_buffer_info_t *buffer_info, ucc_tl_nccl_task_t *task) { ucc_reduction_op_t op = TASK_ARGS(task).op; if (ucc_to_nccl_reduce_op[op] == ncclOpUnsupported) { tl_debug(UCC_TASK_LIB(task), "reduction operation %s is not supported", ucc_reduction_op_str(op)); return UCC_ERR_NOT_SUPPORTED; } if (ucc_to_nccl_dtype[UCC_DT_PREDEFINED_ID(buffer_info->datatype)] == ncclDataTypeUnsupported) { if (op == UCC_OP_SUM) { switch (buffer_info->datatype) { case UCC_DT_FLOAT32_COMPLEX: ucc_tl_nccl_check_and_convert_buffer(buffer_info, UCC_DT_FLOAT32); return UCC_OK; case UCC_DT_FLOAT64_COMPLEX: ucc_tl_nccl_check_and_convert_buffer(buffer_info, UCC_DT_FLOAT64); return UCC_OK; default: break; } } tl_debug(UCC_TASK_LIB(task), "datatype %s is not supported for reduction operation %s", ucc_datatype_str(buffer_info->datatype), ucc_reduction_op_str(op)); return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } ucc_status_t ucc_tl_nccl_init_task(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_tl_nccl_task_t **coll_task) { ucc_tl_nccl_team_t *nccl_team = ucc_derived_of(team, ucc_tl_nccl_team_t); ucc_tl_nccl_context_t *nccl_ctx = ucc_derived_of(team->context, ucc_tl_nccl_context_t); ucc_tl_nccl_task_t *task; ucc_status_t status; ucc_coll_progress_fn_t progress_fn; if (!ucc_coll_args_is_predefined_dt(&coll_args->args, team->params.rank)) { tl_error(team->context->lib, "user defined datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (ucc_unlikely(nccl_team->comm_state != TL_NCCL_COMM_STATE_READY)) { if (UCC_COLL_ARGS_ACTIVE_SET(&coll_args->args)) { /* active set is not supported with lazy comm init*/ return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_nccl_comm_init(nccl_team); if (ucc_unlikely(status != UCC_OK)) { return status; } } task = ucc_mpool_get(&nccl_ctx->req_mp); if (ucc_unlikely(!task)) { tl_error(team->context->lib, "failed to get task from mpool"); return UCC_ERR_NO_MEMORY; } progress_fn = task->super.progress; ucc_coll_task_init(&task->super, coll_args, team); UCC_TL_NCCL_PROFILE_REQUEST_NEW(task, "tl_nccl_task", 0); task->super.finalize = ucc_tl_nccl_coll_finalize; task->super.triggered_post = ucc_tl_nccl_triggered_post; task->super.progress = progress_fn; task->completed = NULL; if (nccl_ctx->cfg.sync_type == UCC_TL_NCCL_COMPLETION_SYNC_TYPE_EVENT) { status = ucc_ec_create_event(&task->completed, UCC_EE_CUDA_STREAM); if (ucc_unlikely(status != UCC_OK)) { ucc_mpool_put(task); return status; } } *coll_task = task; return UCC_OK; } void ucc_tl_nccl_free_task(ucc_tl_nccl_task_t *task) { UCC_TL_NCCL_PROFILE_REQUEST_FREE(task); if (task->completed) { ucc_ec_destroy_event(task->completed, UCC_EE_CUDA_STREAM); } ucc_mpool_put(task); } ucc_status_t ucc_tl_nccl_triggered_post(ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_status_t status; ucc_ev_t post_event; ucc_assert(ee->ee_type == UCC_EE_CUDA_STREAM); coll_task->ee = ee; tl_debug(UCC_TASK_LIB(task), "triggered post. task:%p", coll_task); status = coll_task->post(coll_task); if (ucc_likely(status == UCC_OK)) { post_event.ev_type = UCC_EVENT_COLLECTIVE_POST; post_event.ev_context_size = 0; post_event.ev_context = NULL; post_event.req = &coll_task->super; ucc_ee_set_event_internal(coll_task->ee, &post_event, &coll_task->ee->event_out_queue); } return status; } ucc_status_t ucc_tl_nccl_coll_finalize(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_status_t status = UCC_OK; if (ucc_unlikely(task->super.super.status != UCC_OK)) { team->comm_state = TL_NCCL_COMM_STATE_ERROR; } tl_debug(UCC_TASK_LIB(task), "finalizing coll task %p", task); ucc_tl_nccl_free_task(task); return status; } ucc_status_t ucc_tl_nccl_collective_sync(ucc_tl_nccl_task_t *task, cudaStream_t stream) { ucc_tl_nccl_context_t *ctx = TASK_CTX(task); ucc_status_t status = UCC_OK; enum cudaStreamCaptureStatus capture_st; CUresult cu_status; cudaError_t cuda_st; if (task->super.ee) { cuda_st =cudaStreamIsCapturing((cudaStream_t)task->super.ee->ee_context, &capture_st); if ((cuda_st == cudaSuccess) && (capture_st != cudaStreamCaptureStatusNone)) { task->super.status = UCC_OK; return ucc_task_complete(&task->super); } } task->host_status = task->super.status; if (ctx->cfg.sync_type == UCC_TL_NCCL_COMPLETION_SYNC_TYPE_EVENT) { status = ucc_ec_event_post(stream, task->completed, UCC_EE_CUDA_STREAM); if (ucc_unlikely(status != UCC_OK)) { return status; } } else { cu_status = cuStreamWriteValue32(stream, (CUdeviceptr)task->dev_status, UCC_OK, 0); if (ucc_unlikely(cu_status != CUDA_SUCCESS)) { return UCC_ERR_NO_MESSAGE; } } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(TASK_TEAM(task))->pq, &task->super); } ucc_status_t ucc_tl_nccl_alltoall_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); ucc_status_t status = UCC_OK; ptrdiff_t sbuf = (ptrdiff_t)args->src.info.buffer; ptrdiff_t rbuf = (ptrdiff_t)args->dst.info.buffer; size_t data_size; task->super.status = UCC_INPROGRESS; data_size = (size_t)(args->src.info.count / gsize) * ucc_dt_size(args->src.info.datatype); ucc_assert(args->src.info.count % gsize == 0); if (data_size == 0) { task->super.status = UCC_OK; return ucc_task_complete(&task->super); } UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_alltoall_start", 0); #if NCCL_VERSION_CODE >= NCCL_VERSION(2,28,0) NCCLCHECK_GOTO(ncclAlltoAll((void *)sbuf, (void *)rbuf, data_size, ncclChar, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); #else NCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); for (ucc_rank_t peer = 0; peer < gsize; peer++) { NCCLCHECK_GOTO(ncclSend((void *)(sbuf + peer * data_size), data_size, ncclChar, peer, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); NCCLCHECK_GOTO(ncclRecv((void *)(rbuf + peer * data_size), data_size, ncclChar, peer, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } NCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); #endif status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_alltoall_init(ucc_tl_nccl_task_t *task) { if (UCC_IS_INPLACE(TASK_ARGS(task))) { tl_error(UCC_TASK_LIB(task), "inplace alltoallv is not supported"); return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_nccl_alltoall_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_alltoallv_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; ucc_status_t status = UCC_OK; ptrdiff_t sbuf = (ptrdiff_t)args->src.info_v.buffer; ptrdiff_t rbuf = (ptrdiff_t)args->dst.info_v.buffer; size_t sdt_size, rdt_size, count, displ; ucc_rank_t peer; task->super.status = UCC_INPROGRESS; sdt_size = ucc_dt_size(args->src.info_v.datatype); rdt_size = ucc_dt_size(args->dst.info_v.datatype); UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_alltoallv_start", 0); NCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); for (peer = 0; peer < UCC_TL_TEAM_SIZE(team); peer++) { count = ucc_coll_args_get_count(args, args->src.info_v.counts, peer); if (count != 0) { displ = ucc_coll_args_get_displacement( args, args->src.info_v.displacements, peer); NCCLCHECK_GOTO(ncclSend((void *)(sbuf + displ * sdt_size), count * sdt_size, ncclChar, peer, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } count = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); if (count != 0) { displ = ucc_coll_args_get_displacement( args, args->dst.info_v.displacements, peer); NCCLCHECK_GOTO(ncclRecv((void *)(rbuf + displ * rdt_size), count * rdt_size, ncclChar, peer, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } } NCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_alltoallv_init(ucc_tl_nccl_task_t *task) { if (UCC_IS_INPLACE(TASK_ARGS(task))) { tl_error(UCC_TASK_LIB(task), "inplace alltoall is not supported"); return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_nccl_alltoallv_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_allreduce_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = UCC_IS_INPLACE(*args) ? args->dst.info.buffer : args->src.info.buffer; ucc_status_t status = UCC_OK; ncclRedOp_t op = ucc_to_nccl_reduce_op[args->op]; size_t count = args->dst.info.count; ncclDataType_t dt; dt = ucc_to_nccl_dtype[UCC_DT_PREDEFINED_ID(args->dst.info.datatype)]; task->super.status = UCC_INPROGRESS; UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, args->coll_type == UCC_COLL_TYPE_BARRIER ? "nccl_barrier_start" : "nccl_allreduce_start", 0); NCCLCHECK_GOTO(ncclAllReduce(src, dst, count, dt, op, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_allreduce_init(ucc_tl_nccl_task_t *task) { ucc_coll_args_t *args = &TASK_ARGS(task); if (!UCC_IS_INPLACE(*args)) { if (ucc_tl_nccl_check_and_convert_buffer_reduction(&args->src.info, task) != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } } if (ucc_tl_nccl_check_and_convert_buffer_reduction(&args->dst.info, task) != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_nccl_allreduce_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_allgather_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; size_t count = args->dst.info.count; ncclDataType_t dt; dt = ucc_to_nccl_dtype[UCC_DT_PREDEFINED_ID(args->dst.info.datatype)]; if (UCC_IS_INPLACE(*args)) { src = (void *)((ptrdiff_t)args->dst.info.buffer + (count / size) * ucc_dt_size(args->dst.info.datatype) * rank); } task->super.status = UCC_INPROGRESS; UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_allgather_start", 0); NCCLCHECK_GOTO(ncclAllGather(src, dst, count / size, dt, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_allgather_init(ucc_tl_nccl_task_t *task) { ucc_coll_args_t *args = &TASK_ARGS(task); if (!UCC_IS_INPLACE(*args)) { ucc_tl_nccl_check_and_convert_buffer( &args->src.info, UCC_TL_NCCL_DT_FOR_UNSUPPORTED); } ucc_tl_nccl_check_and_convert_buffer(&args->dst.info, UCC_TL_NCCL_DT_FOR_UNSUPPORTED); task->super.post = ucc_tl_nccl_allgather_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_allgatherv_init(ucc_tl_nccl_task_t *task) { task->super.post = ucc_tl_nccl_allgatherv_p2p_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_bcast_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; size_t count = args->src.info.count; ucc_rank_t root = args->root; ucc_rank_t peer, rank, size; ncclDataType_t dt; ucc_ep_map_t map; dt = ucc_to_nccl_dtype[UCC_DT_PREDEFINED_ID(args->src.info.datatype)]; task->super.status = UCC_INPROGRESS; UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_bcast_start", 0); if (UCC_COLL_ARGS_ACTIVE_SET(args)) { map = ucc_active_set_to_ep_map(args); rank = UCC_TL_TEAM_RANK(team); size = (ucc_rank_t)args->active_set.size; if (root == rank) { NCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); for (peer = 0; peer < size; peer++) { if (ucc_ep_map_eval(map, peer) == rank) { continue; } NCCLCHECK_GOTO(ncclSend(src, count, dt, ucc_ep_map_eval(map, peer), team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } NCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); } else { NCCLCHECK_GOTO(ncclRecv(src, count, dt, root, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); } } else { NCCLCHECK_GOTO(ncclBroadcast(src, src, count, dt, root, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_bcast_init(ucc_tl_nccl_task_t *task) { ucc_tl_nccl_check_and_convert_buffer(&TASK_ARGS(task).src.info, UCC_TL_NCCL_DT_FOR_UNSUPPORTED); task->super.post = ucc_tl_nccl_bcast_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_reduce_scatter_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; ncclRedOp_t op = ucc_to_nccl_reduce_op[args->op]; size_t count = args->dst.info.count; ncclDataType_t dt; dt = ucc_to_nccl_dtype[UCC_DT_PREDEFINED_ID(args->dst.info.datatype)]; task->super.status = UCC_INPROGRESS; UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_reduce_scatter_start", 0); if (UCC_IS_INPLACE(*args)) { count /= UCC_TL_TEAM_SIZE(team); src = args->dst.info.buffer; dst = PTR_OFFSET(src, UCC_TL_TEAM_RANK(team) * count * ucc_dt_size(args->dst.info.datatype)); } NCCLCHECK_GOTO(ncclReduceScatter(src, dst, count, dt, op, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_reduce_scatter_init(ucc_tl_nccl_task_t *task) { ucc_coll_args_t *args = &TASK_ARGS(task); if (!UCC_IS_INPLACE(*args)) { if (ucc_tl_nccl_check_and_convert_buffer_reduction(&args->src.info, task) != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } } if (ucc_tl_nccl_check_and_convert_buffer_reduction(&args->dst.info, task) != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_nccl_reduce_scatter_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_reduce_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info.buffer; ucc_datatype_t ucc_dt = args->src.info.datatype; size_t count = args->src.info.count; ncclRedOp_t op = ucc_to_nccl_reduce_op[args->op]; ucc_status_t status = UCC_OK; ncclDataType_t nccl_dt; UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_reduce_start", 0); if (args->root == UCC_TL_TEAM_RANK(team)) { ucc_dt = TASK_ARGS(task).dst.info.datatype; count = TASK_ARGS(task).dst.info.count; if (UCC_IS_INPLACE(*args)) { src = args->dst.info.buffer; } } nccl_dt = ucc_to_nccl_dtype[UCC_DT_PREDEFINED_ID(ucc_dt)]; task->super.status = UCC_INPROGRESS; NCCLCHECK_GOTO(ncclReduce(src, dst, count, nccl_dt, op, args->root, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_reduce_init(ucc_tl_nccl_task_t *task) { ucc_coll_args_t *args = &TASK_ARGS(task); int is_root = UCC_IS_ROOT(TASK_ARGS(task), UCC_TL_TEAM_RANK(TASK_TEAM(task))); if (is_root) { if (ucc_tl_nccl_check_and_convert_buffer_reduction(&args->dst.info, task) != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } } if (!(is_root && UCC_IS_INPLACE(*args))) { if (ucc_tl_nccl_check_and_convert_buffer_reduction(&args->src.info, task) != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } } task->super.post = ucc_tl_nccl_reduce_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_barrier_init(ucc_tl_nccl_task_t *task) { /* use 4-byte allreduce to accomplish barrier */ ucc_coll_args_t *args = &TASK_ARGS(task); args->mask |= UCC_COLL_ARGS_FIELD_FLAGS; args->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; args->op = UCC_OP_SUM; args->dst.info.buffer = TASK_CTX(task)->scratch_buf; args->src.info.buffer = args->dst.info.buffer; args->dst.info.datatype = args->src.info.datatype = UCC_DT_FLOAT32; args->dst.info.count = args->src.info.count = 1; task->super.post = ucc_tl_nccl_allreduce_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_gather_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; size_t send_size; ucc_rank_t peer; if (rank == args->root) { send_size = ucc_dt_size(args->dst.info.datatype) * args->dst.info.count / size; } else { send_size = ucc_dt_size(args->src.info.datatype) * args->src.info.count; } UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_gather_start", 0); if (rank == args->root) { if (!UCC_IS_INPLACE(*args)) { CUDA_CHECK_GOTO(cudaMemcpyAsync(PTR_OFFSET(dst, rank * send_size), src, send_size, cudaMemcpyDeviceToDevice, stream), exit_coll, status); } NCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); for (peer = 0; peer < size; peer++) { if (peer == args->root) { continue; } NCCLCHECK_GOTO(ncclRecv(PTR_OFFSET(dst, peer * send_size), send_size, ncclChar, peer, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } NCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); } else { NCCLCHECK_GOTO(ncclSend(src, send_size, ncclChar, args->root, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); } task->super.status = UCC_INPROGRESS; status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_gather_init(ucc_tl_nccl_task_t *task) { task->super.post = ucc_tl_nccl_gather_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_gatherv_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; void *dst = args->dst.info_v.buffer; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; size_t count, displ, dt_size; ucc_rank_t peer; if (rank == args->root) { dt_size = ucc_dt_size(args->dst.info_v.datatype); } else { dt_size = ucc_dt_size(args->src.info.datatype); } UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_gatherv_start", 0); if (rank == args->root) { if (!UCC_IS_INPLACE(*args)) { count = ucc_coll_args_get_count(args, args->dst.info_v.counts, rank); displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, rank); CUDA_CHECK_GOTO(cudaMemcpyAsync(PTR_OFFSET(dst, displ * dt_size), src, count * dt_size, cudaMemcpyDeviceToDevice, stream), exit_coll, status); } NCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); for (peer = 0; peer < size; peer++) { if (peer == args->root) { continue; } count = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, peer); NCCLCHECK_GOTO(ncclRecv(PTR_OFFSET(dst, displ * dt_size), count * dt_size, ncclChar, peer,team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } NCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); } else { NCCLCHECK_GOTO(ncclSend(src, args->src.info.count * dt_size, ncclChar, args->root, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); } task->super.status = UCC_INPROGRESS; status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_gatherv_init(ucc_tl_nccl_task_t *task) { task->super.post = ucc_tl_nccl_gatherv_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_scatter_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; size_t send_size; ucc_rank_t peer; if (rank == args->root) { send_size = ucc_dt_size(args->src.info.datatype) * args->src.info.count / size; } else { send_size = ucc_dt_size(args->dst.info.datatype) * args->dst.info.count; } UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_scatter_start", 0); if (rank == args->root) { if (!UCC_IS_INPLACE(*args)) { CUDA_CHECK_GOTO(cudaMemcpyAsync(dst, PTR_OFFSET(src, rank * send_size), send_size, cudaMemcpyDeviceToDevice, stream), exit_coll, status); } NCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); for (peer = 0; peer < size; peer++) { if (peer == args->root) { continue; } NCCLCHECK_GOTO(ncclSend(PTR_OFFSET(src, peer * send_size), send_size, ncclChar, peer, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } NCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); } else { NCCLCHECK_GOTO(ncclRecv(dst, send_size, ncclChar, args->root, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); } task->super.status = UCC_INPROGRESS; status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_scatter_init(ucc_tl_nccl_task_t *task) { task->super.post = ucc_tl_nccl_scatter_start; return UCC_OK; } ucc_status_t ucc_tl_nccl_scatterv_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info_v.buffer; ucc_status_t status = UCC_OK; size_t count, displ, dt_size; ucc_rank_t peer; if (rank == args->root) { dt_size = ucc_dt_size(args->src.info_v.datatype); } else { dt_size = ucc_dt_size(args->dst.info.datatype); } UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_scatterv_start", 0); if (rank == args->root) { if (!UCC_IS_INPLACE(*args)) { count = ucc_coll_args_get_count(args, args->src.info_v.counts, rank); displ = ucc_coll_args_get_displacement(args, args->src.info_v.displacements, rank); CUDA_CHECK_GOTO(cudaMemcpyAsync(dst, PTR_OFFSET(src, displ * dt_size), count * dt_size, cudaMemcpyDeviceToDevice, stream), exit_coll, status); } NCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); for (peer = 0; peer < size; peer++) { if (peer == args->root) { continue; } count = ucc_coll_args_get_count(args, args->src.info_v.counts, peer); displ = ucc_coll_args_get_displacement(args, args->src.info_v.displacements, peer); NCCLCHECK_GOTO(ncclSend(PTR_OFFSET(src, displ * dt_size), count * dt_size, ncclChar, peer, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } NCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); } else { NCCLCHECK_GOTO(ncclRecv(dst, args->dst.info.count * dt_size, ncclChar, args->root, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); } task->super.status = UCC_INPROGRESS; status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_scatterv_init(ucc_tl_nccl_task_t *task) { task->super.post = ucc_tl_nccl_scatterv_start; return UCC_OK; } static inline int alg_id_from_str(ucc_coll_type_t coll_type, const char *str) { switch (coll_type) { case UCC_COLL_TYPE_ALLGATHERV: return ucc_tl_nccl_allgatherv_alg_from_str(str); default: break; } return -1; } ucc_status_t ucc_tl_nccl_alg_id_to_init(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, //NOLINT ucc_base_coll_init_fn_t *init) { ucc_status_t status = UCC_OK; if (alg_id_str) { alg_id = alg_id_from_str(coll_type, alg_id_str); } switch (coll_type) { case UCC_COLL_TYPE_ALLGATHERV: switch (alg_id) { case UCC_TL_NCCL_ALLGATHERV_ALG_P2P: *init = ucc_tl_nccl_allgatherv_p2p_init; break; case UCC_TL_NCCL_ALLGATHERV_ALG_BCOPY: *init = ucc_tl_nccl_allgatherv_bcopy_init; break; case UCC_TL_NCCL_ALLGATHERV_ALG_BCAST: *init = ucc_tl_nccl_allgatherv_bcast_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; default: status = UCC_ERR_NOT_SUPPORTED; break; } return status; } ucc-1.8.0/src/components/tl/nccl/allgatherv/0000775000175000017500000000000015211535620021121 5ustar alastairalastairucc-1.8.0/src/components/tl/nccl/allgatherv/allgatherv.h0000664000175000017500000000347615211535620023435 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLGATHERV_H_ #define ALLGATHERV_H_ #include "tl_nccl_coll.h" enum { UCC_TL_NCCL_ALLGATHERV_ALG_P2P, UCC_TL_NCCL_ALLGATHERV_ALG_BCOPY, UCC_TL_NCCL_ALLGATHERV_ALG_BCAST, UCC_TL_NCCL_ALLGATHERV_ALG_LAST }; #define UCC_TL_NCCL_ALLGATHERV_DEFAULT_ALG_SELECT_STR \ "allgatherv:cuda:0-16k:@0#allgatherv:cuda:16k-1M:@1#allgatherv:cuda:1M-inf:@2#allgatherv:cuda_managed:0-inf:@0" extern ucc_base_coll_alg_info_t ucc_tl_nccl_allgatherv_algs[UCC_TL_NCCL_ALLGATHERV_ALG_LAST + 1]; ucc_status_t ucc_tl_nccl_allgatherv_p2p_start(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_nccl_allgatherv_p2p_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h); ucc_status_t ucc_tl_nccl_allgatherv_bcopy_start(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_nccl_allgatherv_bcopy_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h); ucc_status_t ucc_tl_nccl_allgatherv_bcast_start(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_nccl_allgatherv_bcast_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h); static inline int ucc_tl_nccl_allgatherv_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_NCCL_ALLGATHERV_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_nccl_allgatherv_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/nccl/allgatherv/allgatherv.c0000664000175000017500000003064015211535620023421 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_nccl.h" #include "allgatherv.h" #include "components/mc/ucc_mc.h" #include "core/ucc_ee.h" #include "utils/arch/cuda_def.h" ucc_base_coll_alg_info_t ucc_tl_nccl_allgatherv_algs[UCC_TL_NCCL_ALLGATHERV_ALG_LAST + 1] = { [UCC_TL_NCCL_ALLGATHERV_ALG_P2P] = {.id = UCC_TL_NCCL_ALLGATHERV_ALG_P2P, .name = "p2p", .desc = "allgatherv based on nccl point-to-point"}, [UCC_TL_NCCL_ALLGATHERV_ALG_BCOPY] = {.id = UCC_TL_NCCL_ALLGATHERV_ALG_BCOPY, .name = "bcopy", .desc = "allgatherv with buffered copy"}, [UCC_TL_NCCL_ALLGATHERV_ALG_BCAST] = {.id = UCC_TL_NCCL_ALLGATHERV_ALG_BCAST, .name = "bcast", .desc = "allgatherv based on nccl bcast"}, [UCC_TL_NCCL_ALLGATHERV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; #define CHECK_INPLACE(_args, _team) \ do { \ if (UCC_IS_INPLACE((_args))) { \ tl_error(UCC_TL_TEAM_LIB((_team)), \ "inplace allgatherv is not supported"); \ status = UCC_ERR_NOT_SUPPORTED; \ goto out; \ } \ } while(0) ucc_status_t ucc_tl_nccl_allgatherv_p2p_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; ucc_status_t status = UCC_OK; void *rbuf = args->dst.info_v.buffer; void *sbuf; size_t sdt_size, rdt_size, count, displ; ucc_rank_t peer; rdt_size = ucc_dt_size(args->dst.info_v.datatype); if (UCC_IS_INPLACE(*args)) { displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, trank); sbuf = PTR_OFFSET(rbuf, displ * rdt_size); sdt_size = rdt_size; count = ucc_coll_args_get_count(args, args->dst.info_v.counts, trank); } else { sbuf = args->src.info.buffer; sdt_size = ucc_dt_size(args->src.info.datatype); count = args->src.info.count; } task->super.status = UCC_INPROGRESS; UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_allgatherv_start", 0); NCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); if (count != 0) { for (peer = 0; peer < size; peer++) { NCCLCHECK_GOTO(ncclSend(sbuf, count * sdt_size, ncclChar, peer, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } } for (peer = 0; peer < size; peer++) { count = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); if (count != 0) { displ = ucc_coll_args_get_displacement( args, args->dst.info_v.displacements, peer); NCCLCHECK_GOTO(ncclRecv(PTR_OFFSET(rbuf, displ * rdt_size), count * rdt_size, ncclChar, peer, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } } NCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_allgatherv_p2p_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h) { ucc_status_t status = UCC_OK; ucc_tl_nccl_task_t *task; status = ucc_tl_nccl_init_task(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->super.post = ucc_tl_nccl_allgatherv_p2p_start; *task_h = &task->super; return status; } ucc_status_t ucc_tl_nccl_allgatherv_bcopy_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; ucc_status_t status = UCC_OK; void *sbuf = args->src.info.buffer; void *rbuf = args->dst.info_v.buffer; void *scratch = task->allgatherv_bcopy.scratch->addr; size_t max_count, rdt_size, sdt_size, displ, scount, rcount; ucc_rank_t peer; task->super.status = UCC_INPROGRESS; UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_allgatherv_start", 0); max_count = task->allgatherv_bcopy.max_count; scount = args->src.info.count; rdt_size = ucc_dt_size(args->dst.info_v.datatype); sdt_size = ucc_dt_size(args->src.info.datatype); if (max_count * rdt_size > scount * sdt_size) { CUDA_CHECK_GOTO(cudaMemcpyAsync(PTR_OFFSET(scratch, max_count * rdt_size * size), sbuf, scount * sdt_size, cudaMemcpyDeviceToDevice, stream), exit_coll, status); sbuf = PTR_OFFSET(scratch, max_count * rdt_size * size); } NCCLCHECK_GOTO(ncclAllGather(sbuf, scratch, max_count * rdt_size, ncclChar, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); for (peer = 0; peer < size; peer++) { rcount = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, peer); CUDA_CHECK_GOTO(cudaMemcpyAsync(PTR_OFFSET(rbuf, displ * rdt_size), PTR_OFFSET(scratch, peer * max_count * rdt_size), rcount * rdt_size, cudaMemcpyDeviceToDevice, stream), exit_coll, status); } status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_allgatherv_bcopy_finalize(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_mc_free(task->allgatherv_bcopy.scratch); return ucc_tl_nccl_coll_finalize(coll_task); } ucc_status_t ucc_tl_nccl_allgatherv_bcopy_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h) { ucc_coll_args_t *args = &coll_args->args; ucc_status_t status = UCC_OK; ucc_tl_nccl_task_t *task; size_t max_count, sdt_size, rdt_size; ucc_rank_t peer; if (UCC_IS_INPLACE(*args)) { tl_debug(team->context->lib, "fallback to bcast based inplace allgatherv"); return ucc_tl_nccl_allgatherv_bcast_init(coll_args, team, task_h); } status = ucc_tl_nccl_init_task(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } sdt_size = ucc_dt_size(args->src.info.datatype); rdt_size = ucc_dt_size(args->dst.info_v.datatype); max_count = ucc_coll_args_get_count(args, args->dst.info_v.counts, 0); for (peer = 1; peer < team->params.size; peer++) { max_count = ucc_max(ucc_coll_args_get_count(args, args->dst.info_v.counts, peer), max_count); } task->allgatherv_bcopy.max_count = max_count; if (max_count * rdt_size > args->src.info.count * sdt_size) { status = ucc_mc_alloc(&task->allgatherv_bcopy.scratch, (team->params.size + 1) * max_count * ucc_dt_size(args->dst.info_v.datatype), UCC_MEMORY_TYPE_CUDA); } else { status = ucc_mc_alloc(&task->allgatherv_bcopy.scratch, max_count * team->params.size * ucc_dt_size(args->dst.info_v.datatype), UCC_MEMORY_TYPE_CUDA); } if (ucc_unlikely(status != UCC_OK)) { ucc_tl_nccl_free_task(task); return status; } task->super.post = ucc_tl_nccl_allgatherv_bcopy_start; task->super.finalize = ucc_tl_nccl_allgatherv_bcopy_finalize; *task_h = &task->super; return status; } ucc_status_t ucc_tl_nccl_allgatherv_bcast_start(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_nccl_team_t *team = TASK_TEAM(task); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t) ee->ee_context : team->stream; ucc_status_t status = UCC_OK; void *sbuf = args->src.info.buffer; ptrdiff_t rbuf = (ptrdiff_t)args->dst.info_v.buffer; size_t rdt_size, count, displ; ucc_rank_t peer; task->super.status = UCC_INPROGRESS; rdt_size = ucc_dt_size(args->dst.info_v.datatype); UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_allgatherv_start", 0); NCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); for (peer = 0; peer < size; peer++) { count = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, peer); if (UCC_IS_INPLACE(*args)) { sbuf = PTR_OFFSET(rbuf, displ * rdt_size); } NCCLCHECK_GOTO(ncclBroadcast(sbuf, PTR_OFFSET(rbuf, displ * rdt_size), count * rdt_size, ncclChar, peer, team->nccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 0); } NCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team), &task->nccl_progress_st, team->nccl_comm, 1); status = ucc_tl_nccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_nccl_allgatherv_bcast_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h) { ucc_status_t status = UCC_OK; ucc_tl_nccl_task_t *task; status = ucc_tl_nccl_init_task(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->super.post = ucc_tl_nccl_allgatherv_bcast_start; *task_h = &task->super; return status; } ucc-1.8.0/src/components/tl/nccl/configure.m40000664000175000017500000000136415211535620021217 0ustar alastairalastair# # Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # tl_nccl_enabled=n CHECK_TLS_REQUIRED(["nccl"]) AS_IF([test "$CHECKED_TL_REQUIRED" = "y"], [ CHECK_NCCL AC_MSG_RESULT([NCCL support: $nccl_happy]) if test $nccl_happy = "yes"; then tl_modules="${tl_modules}:nccl" tl_nccl_enabled=y CHECK_NEED_TL_PROFILING(["tl_nccl"]) AS_IF([test "$TL_PROFILING_REQUIRED" = "y"], [ AC_DEFINE([HAVE_PROFILING_TL_NCCL], [1], [Enable profiling for TL NCCL]) prof_modules="${prof_modules}:tl_nccl" ], []) fi ], []) AM_CONDITIONAL([TL_NCCL_ENABLED], [test "$tl_nccl_enabled" = "y"]) AC_CONFIG_FILES([src/components/tl/nccl/Makefile]) ucc-1.8.0/src/components/tl/nccl/tl_nccl.c0000664000175000017500000000723215211535620020556 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_nccl.h" #include "components/mc/base/ucc_mc_base.h" #include "allgatherv/allgatherv.h" ucc_status_t ucc_tl_nccl_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr); ucc_status_t ucc_tl_nccl_get_lib_properties(ucc_base_lib_properties_t *prop); ucc_status_t ucc_tl_nccl_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *base_attr); ucc_status_t ucc_tl_nccl_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_nccl_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_nccl_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **pack_buffer); static ucc_config_field_t ucc_tl_nccl_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_nccl_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_lib_config_table)}, {NULL}}; const char* ucc_tl_nccl_completion_sync_names[] = { [UCC_TL_NCCL_COMPLETION_SYNC_TYPE_EVENT] = "event", [UCC_TL_NCCL_COMPLETION_SYNC_TYPE_MEMOPS] = "driver", [UCC_TL_NCCL_COMPLETION_SYNC_TYPE_AUTO] = "auto", [UCC_TL_NCCL_COMPLETION_SYNC_TYPE_LAST] = NULL }; static ucs_config_field_t ucc_tl_nccl_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_nccl_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_context_config_table)}, {"SYNC", "auto", "Determines how UCC tests completion of NCCL collective", ucs_offsetof(ucc_tl_nccl_context_config_t, sync_type), UCS_CONFIG_TYPE_ENUM(ucc_tl_nccl_completion_sync_names) }, {"BLOCKING", "yes", "If set to no will use non-blocking mode communicator behavior, " "if set to yes will use blocking mode", ucs_offsetof(ucc_tl_nccl_context_config_t, nccl_cfg_blocking), UCS_CONFIG_TYPE_BOOL}, {"LAZY_INIT", "yes", "Initialize NCCL communicator on first collective", ucc_offsetof(ucc_tl_nccl_context_config_t, nccl_lazy_init), UCC_CONFIG_TYPE_BOOL}, {NULL}}; UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_nccl_lib_t, ucc_base_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_nccl_lib_t, ucc_base_lib_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_nccl_context_t, ucc_base_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_nccl_context_t, ucc_base_context_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_nccl_team_t, ucc_base_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_tl_nccl_team_create_test(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_nccl_team_destroy(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_nccl_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); ucc_status_t ucc_tl_nccl_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p); UCC_TL_IFACE_DECLARE(nccl, NCCL); __attribute__((constructor)) static void tl_nccl_iface_init(void) { ucc_tl_nccl.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLGATHERV)] = ucc_tl_nccl_allgatherv_algs; } ucc-1.8.0/src/components/tl/nccl/tl_nccl_coll.h0000664000175000017500000000425015211535620021571 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Facebook, Inc. and its affiliates. 2021. * * See file LICENSE for terms. */ #ifndef UCC_TL_NCCL_COLL_H_ #define UCC_TL_NCCL_COLL_H_ #include "tl_nccl.h" #define UCC_TL_NCCL_DT_FOR_UNSUPPORTED UCC_DT_INT8 #define UCC_TL_NCCL_N_DEFAULT_ALG_SELECT_STR 1 extern const char *ucc_tl_nccl_default_alg_select_str[UCC_TL_NCCL_N_DEFAULT_ALG_SELECT_STR]; ucc_status_t ucc_tl_nccl_alg_id_to_init(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, ucc_base_coll_init_fn_t *init); ucc_status_t ucc_tl_nccl_init_task(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_tl_nccl_task_t **coll_task); void ucc_tl_nccl_free_task(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_triggered_post(ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_nccl_coll_finalize(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_nccl_collective_sync(ucc_tl_nccl_task_t *task, cudaStream_t stream); ucc_status_t ucc_tl_nccl_allgather_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_allgatherv_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_allreduce_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_alltoall_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_alltoallv_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_bcast_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_reduce_scatter_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_reduce_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_barrier_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_gather_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_gatherv_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_scatter_init(ucc_tl_nccl_task_t *task); ucc_status_t ucc_tl_nccl_scatterv_init(ucc_tl_nccl_task_t *task); #endif ucc-1.8.0/src/components/tl/nccl/tl_nccl_lib.c0000664000175000017500000000324515211535620021404 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_nccl.h" /* NOLINTNEXTLINE params is not used*/ UCC_CLASS_INIT_FUNC(ucc_tl_nccl_lib_t, const ucc_base_lib_params_t *params, const ucc_base_config_t *config) { const ucc_tl_lib_config_t *tl_config = ucc_derived_of(config, ucc_tl_lib_config_t); UCC_CLASS_CALL_SUPER_INIT(ucc_tl_lib_t, &ucc_tl_nccl.super, tl_config); tl_debug(&self->super, "initialized lib object: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_nccl_lib_t) { tl_debug(&self->super, "finalizing lib object: %p", self); } UCC_CLASS_DEFINE(ucc_tl_nccl_lib_t, ucc_tl_lib_t); ucc_status_t ucc_tl_nccl_get_lib_attr(const ucc_base_lib_t *lib, /* NOLINT */ ucc_base_lib_attr_t *base_attr) { ucc_tl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_tl_lib_attr_t); attr->super.attr.thread_mode = UCC_THREAD_MULTIPLE; attr->super.attr.coll_types = UCC_TL_NCCL_SUPPORTED_COLLS; attr->super.flags = 0; attr->super.min_team_size = 2; attr->super.max_team_size = UCC_RANK_MAX; if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MIN_TEAM_SIZE) { attr->super.min_team_size = lib->min_team_size; } if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MAX_TEAM_SIZE) { attr->super.max_team_size = UCC_RANK_MAX; } return UCC_OK; } ucc_status_t ucc_tl_nccl_get_lib_properties(ucc_base_lib_properties_t *prop) { prop->default_team_size = 2; prop->min_team_size = 2; prop->max_team_size = UCC_RANK_MAX; return UCC_OK; } ucc-1.8.0/src/components/tl/nccl/tl_nccl_team.c0000664000175000017500000003004615211535620021563 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Facebook, Inc. and its affiliates. 2021. * * See file LICENSE for terms. */ #include "tl_nccl.h" #include "tl_nccl_coll.h" #include "components/mc/ucc_mc.h" #include "core/ucc_ee.h" #include "coll_score/ucc_coll_score.h" #include "utils/arch/cuda_def.h" UCC_CLASS_INIT_FUNC(ucc_tl_nccl_team_t, ucc_base_context_t *tl_context, const ucc_base_team_params_t *params) { ucc_tl_nccl_context_t *ctx = ucc_derived_of(tl_context, ucc_tl_nccl_context_t); ucc_team_oob_coll_t *oob; ucc_status_t status; ucc_rank_t size; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_team_t, &ctx->super, params); oob = &(UCC_TL_TEAM_OOB(self)); size = UCC_TL_TEAM_SIZE(self); self->stream = NULL; self->nccl_comm = NULL; self->unique_id = ucc_malloc(sizeof(ncclUniqueId) * (size + 1), "tl_nccl_unique_id"); if (!self->unique_id) { tl_error(ctx->super.super.lib, "failed to allocate %zd bytes for unique_id array", sizeof(ncclUniqueId) * (size + 1)); return UCC_ERR_NO_MEMORY; } if (UCC_TL_TEAM_RANK(self) == 0) { ncclResult_t st; st = ncclGetUniqueId(&self->unique_id[size]); if (st != ncclSuccess) { tl_error(ctx->super.super.lib, "failed to get unique id"); memset(&self->unique_id[size], 0, sizeof(ncclUniqueId)); } } status = oob->allgather(&self->unique_id[size], self->unique_id, sizeof(ncclUniqueId), oob->coll_info, &self->oob_req); if (status != UCC_OK) { tl_error(ctx->super.super.lib, "failed to start oob allgather"); goto free_unique_id; } self->comm_state = TL_NCCL_COMM_STATE_OOB; return UCC_OK; free_unique_id: ucc_free(self->unique_id); return status; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_nccl_team_t) { tl_debug(self->super.super.context->lib, "finalizing tl team: %p", self); } UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_nccl_team_t, ucc_base_team_t); UCC_CLASS_DEFINE(ucc_tl_nccl_team_t, ucc_tl_team_t); ucc_status_t ucc_tl_nccl_team_destroy(ucc_base_team_t *tl_team) { ucc_tl_nccl_team_t *team = ucc_derived_of(tl_team, ucc_tl_nccl_team_t); #if NCCL_USE_NON_BLOCKING ncclResult_t nccl_status, st; if (team->comm_state == TL_NCCL_COMM_STATE_DESTROY_COMM) { goto check_finalize; } #endif if (team->stream) { cudaStreamDestroy(team->stream); team->stream = NULL; } if (team->nccl_comm) { if (team->comm_state == TL_NCCL_COMM_STATE_ERROR) { ncclCommAbort(team->nccl_comm); } else { #if NCCL_USE_NON_BLOCKING ncclCommFinalize(team->nccl_comm); check_finalize: st = ncclCommGetAsyncError(team->nccl_comm, &nccl_status); if (st != ncclSuccess || (nccl_status != ncclSuccess)) { tl_debug(tl_team->context->lib, "NCCL error %d %s", st != ncclSuccess ? st : nccl_status, ncclGetErrorString(st != ncclSuccess ? st : nccl_status)); ncclCommAbort(team->nccl_comm); return UCC_ERR_NO_MESSAGE; } else if (nccl_status == ncclInProgress) { team->comm_state = TL_NCCL_COMM_STATE_DESTROY_COMM; return UCC_INPROGRESS; } else { ncclCommDestroy(team->nccl_comm); } team->comm_state = UCC_OK; #else ncclCommDestroy(team->nccl_comm); #endif } } UCC_CLASS_DELETE_FUNC_NAME(ucc_tl_nccl_team_t)(tl_team); return UCC_OK; } ucc_status_t ucc_tl_nccl_comm_init(ucc_tl_nccl_team_t *team) { ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_status_t status; ncclResult_t nccl_status; #if NCCL_USE_NON_BLOCKING ncclConfig_t nccl_cfg = NCCL_CONFIG_INITIALIZER; ncclResult_t async_status; #endif if (team->comm_state == TL_NCCL_COMM_STATE_READY) { return UCC_OK; } else if (team->comm_state == TL_NCCL_COMM_STATE_ERROR) { return UCC_ERR_NOT_SUPPORTED; } else if (team->comm_state == TL_NCCL_COMM_STATE_INIT_COMM) { #if NCCL_USE_NON_BLOCKING goto nccl_async_init; #else ucc_assert_always(0); #endif } CUDA_CHECK_GOTO(cudaStreamCreateWithFlags(&team->stream, cudaStreamNonBlocking), exit_err, status); #if NCCL_USE_NON_BLOCKING /* * if NCCL comm initialized during first call to collective init a.k.a lazy init * we need to use blocking init to correctly fallback to other TL in case of error */ nccl_cfg.blocking = (UCC_TL_NCCL_TEAM_CTX(team)->cfg.nccl_cfg_blocking || UCC_TL_NCCL_TEAM_CTX(team)->cfg.nccl_lazy_init) ? 1: 0; nccl_status = ncclCommInitRankConfig(&team->nccl_comm, tsize, team->unique_id[0], trank, &nccl_cfg); if ((nccl_status != ncclInProgress) && (nccl_status != ncclSuccess)) { goto nccl_comm_init_err; } nccl_async_init: nccl_status = ncclCommGetAsyncError(team->nccl_comm, &async_status); if (nccl_status != ncclSuccess) { goto nccl_comm_init_err; } if (async_status == ncclInProgress) { team->comm_state = TL_NCCL_COMM_STATE_INIT_COMM; } #else nccl_status = ncclCommInitRank(&team->nccl_comm, tsize, team->unique_id[0], trank); if (nccl_status != ncclSuccess) { goto nccl_comm_init_err; } #endif team->comm_state = TL_NCCL_COMM_STATE_READY; return UCC_OK; nccl_comm_init_err: tl_debug(team->super.super.context->lib, "NCCL error %d %s", nccl_status, ncclGetErrorString(nccl_status)); if (nccl_status == ncclInvalidUsage) { /* * handles the case when trying to inititize multiple ranks * on the same GPU. Return "not supported" and fallback to other TL */ status = UCC_ERR_NOT_SUPPORTED; } else { status = UCC_ERR_NO_RESOURCE; } team->comm_state = TL_NCCL_COMM_STATE_ERROR; exit_err: return status; } ucc_status_t ucc_tl_nccl_team_create_test(ucc_base_team_t *tl_team) { ucc_tl_nccl_team_t *team = ucc_derived_of(tl_team, ucc_tl_nccl_team_t); ucc_team_oob_coll_t *oob = &(UCC_TL_TEAM_OOB(team)); ncclUniqueId errorid; ucc_status_t status; if (team->comm_state == TL_NCCL_COMM_STATE_OOB) { status = oob->req_test(team->oob_req); if (status == UCC_INPROGRESS) { return UCC_INPROGRESS; } oob->req_free(team->oob_req); if (status != UCC_OK) { tl_error(tl_team->context->lib, "oob req test failed"); return status; } /* check unique id is valid */ memset(&errorid, 0, sizeof(errorid)); if (!memcmp(&errorid, team->unique_id, sizeof(errorid))) { tl_error(tl_team->context->lib, "incorrect unique id"); return status; } team->comm_state = TL_NCCL_COMM_STATE_INIT_TEAM; } if (UCC_TL_NCCL_TEAM_CTX(team)->cfg.nccl_lazy_init) { return UCC_OK; } return ucc_tl_nccl_comm_init(team); } ucc_status_t ucc_tl_nccl_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_nccl_task_t *task; ucc_status_t status; status = ucc_tl_nccl_init_task(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } switch (coll_args->args.coll_type) { case UCC_COLL_TYPE_ALLGATHER: status = ucc_tl_nccl_allgather_init(task); break; case UCC_COLL_TYPE_ALLGATHERV: status = ucc_tl_nccl_allgatherv_init(task); break; case UCC_COLL_TYPE_ALLREDUCE: status = ucc_tl_nccl_allreduce_init(task); break; case UCC_COLL_TYPE_ALLTOALL: status = ucc_tl_nccl_alltoall_init(task); break; case UCC_COLL_TYPE_ALLTOALLV: status = ucc_tl_nccl_alltoallv_init(task); break; case UCC_COLL_TYPE_BCAST: status = ucc_tl_nccl_bcast_init(task); break; case UCC_COLL_TYPE_REDUCE_SCATTER: status = ucc_tl_nccl_reduce_scatter_init(task); break; case UCC_COLL_TYPE_REDUCE: status = ucc_tl_nccl_reduce_init(task); break; case UCC_COLL_TYPE_BARRIER: status = ucc_tl_nccl_barrier_init(task); break; case UCC_COLL_TYPE_GATHER: status = ucc_tl_nccl_gather_init(task); break; case UCC_COLL_TYPE_GATHERV: status = ucc_tl_nccl_gatherv_init(task); break; case UCC_COLL_TYPE_SCATTER: status = ucc_tl_nccl_scatter_init(task); break; case UCC_COLL_TYPE_SCATTERV: status = ucc_tl_nccl_scatterv_init(task); break; default: tl_debug(UCC_TASK_LIB(task), "collective %s is not supported by nccl tl", ucc_coll_type_str(coll_args->args.coll_type)); status = UCC_ERR_NOT_SUPPORTED; } if (ucc_unlikely(status != UCC_OK)) { goto free_task; } tl_debug(UCC_TASK_LIB(task), "init coll task %p", task); *task_h = &task->super; return status; free_task: ucc_tl_nccl_free_task(task); return status; } ucc_status_t ucc_tl_nccl_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p) { ucc_tl_nccl_team_t *team = ucc_derived_of(tl_team, ucc_tl_nccl_team_t); ucc_base_context_t *ctx = UCC_TL_TEAM_CTX(team); ucc_memory_type_t mts[2] = {UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED}; ucc_coll_score_t *score; ucc_status_t status; int i; ucc_coll_score_team_info_t team_info; team_info.alg_fn = ucc_tl_nccl_alg_id_to_init; team_info.default_score = UCC_TL_NCCL_DEFAULT_SCORE; team_info.init = ucc_tl_nccl_coll_init; team_info.num_mem_types = 2; team_info.supported_mem_types = mts; team_info.supported_colls = UCC_TL_NCCL_SUPPORTED_COLLS; team_info.size = UCC_TL_TEAM_SIZE(team); /* There can be a different logic for different coll_type/mem_type. Right now just init everything the same way. */ status = ucc_coll_score_build_default(tl_team, UCC_TL_NCCL_DEFAULT_SCORE, ucc_tl_nccl_coll_init, UCC_TL_NCCL_SUPPORTED_COLLS, mts, 2, &score); if (ucc_unlikely(UCC_OK != status)) { return status; } for (i = 0; i < UCC_TL_NCCL_N_DEFAULT_ALG_SELECT_STR; i++) { status = ucc_coll_score_update_from_str( ucc_tl_nccl_default_alg_select_str[i], &team_info, &team->super.super, score); if (ucc_unlikely(UCC_OK != status)) { tl_error(tl_team->context->lib, "failed to apply default coll select setting: %s", ucc_tl_nccl_default_alg_select_str[i]); goto err; } } // add barrier, which might be triggered from host memory type // use lower score status = ucc_coll_score_add_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, 0, UCC_MSG_MAX, 1, ucc_tl_nccl_coll_init, tl_team); if (ucc_unlikely(UCC_OK != status)) { return status; } if (strlen(ctx->score_str) > 0) { status = ucc_coll_score_update_from_str(ctx->score_str, &team_info, &team->super.super, score); /* If INVALID_PARAM - User provided incorrect input - try to proceed */ if ((status < 0) && (status != UCC_ERR_INVALID_PARAM) && (status != UCC_ERR_NOT_SUPPORTED)) { goto err; } } *score_p = score; return UCC_OK; err: ucc_coll_score_free(score); return status; } ucc-1.8.0/src/components/tl/nccl/tl_nccl.h0000664000175000017500000001533715211535620020570 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Facebook, Inc. and its affiliates. 2021. * * See file LICENSE for terms. */ #ifndef UCC_TL_NCCL_H_ #define UCC_TL_NCCL_H_ #include "components/mc/base/ucc_mc_base.h" #include "components/tl/ucc_tl.h" #include "components/tl/ucc_tl_log.h" #include "utils/ucc_mpool.h" #include #if CUDART_VERSION >= 11000 #ifndef __CUDA_BF16_TYPES_EXIST__ #define __CUDA_BF16_TYPES_EXIST__ #endif #endif // Until NCCL will provide run-time method to query ncclBfloat16 support, we will use this check. // In corner case NCCL was compiled with older version of CUDA than UCC, that doesn't support Bfloat16, // The program will crash. In case UCC was compiled with older version of CUDA that doesn't support // Bfloat16, but NCCL was compiled with newer version of CUDA that does support, we won't support // this NCCL datatype. #include #include #ifndef UCC_TL_NCCL_DEFAULT_SCORE #define UCC_TL_NCCL_DEFAULT_SCORE 20 #endif #ifdef HAVE_PROFILING_TL_NCCL #include "utils/profile/ucc_profile.h" #else #include "utils/profile/ucc_profile_off.h" #endif #define UCC_TL_NCCL_PROFILE_FUNC UCC_PROFILE_FUNC #define UCC_TL_NCCL_PROFILE_FUNC_VOID UCC_PROFILE_FUNC_VOID #define UCC_TL_NCCL_PROFILE_REQUEST_NEW UCC_PROFILE_REQUEST_NEW #define UCC_TL_NCCL_PROFILE_REQUEST_EVENT UCC_PROFILE_REQUEST_EVENT #define UCC_TL_NCCL_PROFILE_REQUEST_FREE UCC_PROFILE_REQUEST_FREE #define NCCL_VERSION_COMM_INIT_NB NCCL_VERSION(2,14,3) #define NCCL_USE_NON_BLOCKING NCCL_VERSION_CODE >= NCCL_VERSION_COMM_INIT_NB enum { TL_NCCL_COMM_STATE_ERROR, TL_NCCL_COMM_STATE_OOB, TL_NCCL_COMM_STATE_INIT_TEAM, TL_NCCL_COMM_STATE_INIT_COMM, TL_NCCL_COMM_STATE_DESTROY_COMM, TL_NCCL_COMM_STATE_READY, }; typedef struct ucc_tl_nccl_iface { ucc_tl_iface_t super; } ucc_tl_nccl_iface_t; extern ucc_tl_nccl_iface_t ucc_tl_nccl; typedef struct ucc_tl_nccl_lib_config { ucc_tl_lib_config_t super; } ucc_tl_nccl_lib_config_t; typedef enum ucc_tl_nccl_completion_sync_type { UCC_TL_NCCL_COMPLETION_SYNC_TYPE_EVENT, UCC_TL_NCCL_COMPLETION_SYNC_TYPE_MEMOPS, UCC_TL_NCCL_COMPLETION_SYNC_TYPE_AUTO, UCC_TL_NCCL_COMPLETION_SYNC_TYPE_LAST } ucc_tl_nccl_completion_sync_type_t; typedef struct ucc_tl_nccl_context_config { ucc_tl_context_config_t super; ucc_tl_nccl_completion_sync_type_t sync_type; int nccl_cfg_blocking; int nccl_lazy_init; } ucc_tl_nccl_context_config_t; typedef struct ucc_tl_nccl_lib { ucc_tl_lib_t super; } ucc_tl_nccl_lib_t; UCC_CLASS_DECLARE(ucc_tl_nccl_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_nccl_context { ucc_tl_context_t super; ucc_tl_nccl_context_config_t cfg; ucc_mpool_t req_mp; void *scratch_buf; } ucc_tl_nccl_context_t; UCC_CLASS_DECLARE(ucc_tl_nccl_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_nccl_team { ucc_tl_team_t super; int comm_state; ncclUniqueId *unique_id; void *oob_req; ncclComm_t nccl_comm; cudaStream_t stream; } ucc_tl_nccl_team_t; typedef struct ucc_tl_nccl_task { ucc_coll_task_t super; ucc_status_t host_status; ucc_status_t nccl_progress_st; ucc_status_t *dev_status; void *completed; union { struct { ucc_mc_buffer_header_t *scratch; size_t max_count; } allgatherv_bcopy; }; } ucc_tl_nccl_task_t; #define TASK_TEAM(_task) \ (ucc_derived_of((_task)->super.team, ucc_tl_nccl_team_t)) #define TASK_CTX(_task) \ (ucc_derived_of((_task)->super.team->context, ucc_tl_nccl_context_t)) #define TASK_LIB(_task) \ (ucc_derived_of((_task)->super.team->context->lib, ucc_tl_nccl_lib_t)) #define TASK_ARGS(_task) (_task)->super.bargs.args #define UCC_TL_NCCL_SUPPORTED_COLLS \ (UCC_COLL_TYPE_ALLTOALL | UCC_COLL_TYPE_ALLTOALLV | \ UCC_COLL_TYPE_ALLGATHER | UCC_COLL_TYPE_ALLGATHERV | \ UCC_COLL_TYPE_ALLREDUCE | UCC_COLL_TYPE_BCAST | \ UCC_COLL_TYPE_REDUCE_SCATTER | UCC_COLL_TYPE_REDUCE | \ UCC_COLL_TYPE_BARRIER | UCC_COLL_TYPE_GATHER | \ UCC_COLL_TYPE_GATHERV | UCC_COLL_TYPE_SCATTER | \ UCC_COLL_TYPE_SCATTERV) UCC_CLASS_DECLARE(ucc_tl_nccl_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); static inline ucc_status_t ucc_tl_nccl_check_nb(ncclResult_t *nccl_status, // NOLINT ucc_status_t *task_st, // NOLINT ncclComm_t nccl_comm, //NOLINT int check_nb) { //NOLINT #if NCCL_USE_NON_BLOCKING if (check_nb && (*nccl_status == ncclSuccess || *nccl_status == ncclInProgress)) { ncclResult_t st = ncclCommGetAsyncError(nccl_comm, nccl_status); if (st != ncclSuccess) { return UCC_ERR_NO_MESSAGE; } if (ncclInProgress == *nccl_status) { *task_st = UCC_INPROGRESS; return UCC_INPROGRESS; } } #endif return UCC_OK; } ucc_status_t ucc_tl_nccl_comm_init(ucc_tl_nccl_team_t *team); #define NCCLCHECK_GOTO(_cmd, _label, _st, _lib, _task_st, _comm, _check_nb) \ do { \ ncclResult_t e = _cmd; \ _st = ucc_tl_nccl_check_nb(&e, _task_st, _comm, _check_nb); \ if (_st != UCC_INPROGRESS && ncclSuccess != e) { \ tl_error(_lib, "NCCL error %d %s", e, ncclGetErrorString(e)); \ _st = UCC_ERR_NO_MESSAGE; \ goto _label; \ } \ } while (0) #define UCC_TL_NCCL_TEAM_LIB(_team) \ (ucc_derived_of((_team)->super.super.context->lib, ucc_tl_nccl_lib_t)) #define UCC_TL_NCCL_TEAM_CTX(_team) \ (ucc_derived_of((_team)->super.super.context, ucc_tl_nccl_context_t)) #endif ucc-1.8.0/src/components/tl/nccl/tl_nccl_context.c0000664000175000017500000002045415211535620022323 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Facebook, Inc. and its affiliates. 2021. * * See file LICENSE for terms. */ #include "tl_nccl.h" #include "components/mc/ucc_mc.h" #include "components/ec/ucc_ec.h" #include "core/ucc_ee.h" #include "utils/arch/cpu.h" static ucc_status_t ucc_tl_nccl_nb_progress(ucc_tl_nccl_task_t *task) { #if NCCL_USE_NON_BLOCKING ucc_tl_nccl_team_t *team = TASK_TEAM(task); ncclResult_t nccl_status, st; if (task->nccl_progress_st == UCC_INPROGRESS) { st = ncclCommGetAsyncError(team->nccl_comm, &nccl_status); if (st != ncclSuccess || (nccl_status != ncclSuccess && nccl_status != ncclInProgress)) { tl_error(UCC_TL_TEAM_LIB(team), "NCCL error %d %s", st != ncclSuccess ? st : nccl_status, ncclGetErrorString(st != ncclSuccess ? st : nccl_status)); return UCC_ERR_NO_MESSAGE; } if (nccl_status == ncclInProgress) { return UCC_INPROGRESS; } } #endif return UCC_OK; } void ucc_tl_nccl_event_collective_progress(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_status_t status; status = ucc_tl_nccl_nb_progress(task); if (status != UCC_OK) { coll_task->status = status; return; } ucc_assert(task->completed != NULL); status = ucc_ec_event_test(task->completed, UCC_EE_CUDA_STREAM); coll_task->status = status; #ifdef HAVE_PROFILING_TL_NCCL if (coll_task->status == UCC_OK) { UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_coll_done", 0); } #endif } void ucc_tl_nccl_driver_collective_progress(ucc_coll_task_t *coll_task) { ucc_tl_nccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_nccl_task_t); ucc_status_t status; status = ucc_tl_nccl_nb_progress(task); if (status != UCC_OK) { coll_task->status = status; return; } coll_task->status = task->host_status; #ifdef HAVE_PROFILING_TL_NCCL if (coll_task->status == UCC_OK) { UCC_TL_NCCL_PROFILE_REQUEST_EVENT(coll_task, "nccl_coll_done", 0); } #endif } static void ucc_tl_nccl_req_mpool_obj_cleanup(ucc_mpool_t *mp, void *obj) { ucc_coll_task_destruct(obj); } static void ucc_tl_nccl_req_mpool_obj_init(ucc_mpool_t *mp, void *obj, void *chunk) { ucc_tl_nccl_task_t *req = (ucc_tl_nccl_task_t*) obj; ucc_coll_task_construct(&req->super); req->super.progress = ucc_tl_nccl_event_collective_progress; req->nccl_progress_st = UCC_OK; } static ucc_mpool_ops_t ucc_tl_nccl_req_mpool_ops = { .chunk_alloc = ucc_mpool_hugetlb_malloc, .chunk_release = ucc_mpool_hugetlb_free, .obj_init = ucc_tl_nccl_req_mpool_obj_init, .obj_cleanup = ucc_tl_nccl_req_mpool_obj_cleanup }; static ucc_status_t ucc_tl_nccl_req_mapped_mpool_chunk_malloc(ucc_mpool_t *mp, size_t *size_p, void ** chunk_p) { cudaError_t cu_st; cu_st = cudaHostAlloc((void**)chunk_p, *size_p, cudaHostAllocMapped); if (cu_st != cudaSuccess) { return UCC_ERR_NO_MEMORY; } return UCC_OK; } static void ucc_tl_nccl_req_mapped_mpool_chunk_free(ucc_mpool_t *mp, void *chunk) { cudaFreeHost(chunk); } static void ucc_tl_nccl_req_mapped_mpool_obj_init(ucc_mpool_t *mp, void *obj, void *chunk) { ucc_tl_nccl_task_t *req = (ucc_tl_nccl_task_t*) obj; cudaError_t st; st = cudaHostGetDevicePointer((void **)(&req->dev_status), (void *)&req->host_status, 0); if (st != cudaSuccess) { req->super.status = UCC_ERR_NO_MESSAGE; } ucc_coll_task_construct(&req->super); req->super.progress = ucc_tl_nccl_driver_collective_progress; req->nccl_progress_st = UCC_OK; } static ucc_mpool_ops_t ucc_tl_nccl_req_mapped_mpool_ops = { .chunk_alloc = ucc_tl_nccl_req_mapped_mpool_chunk_malloc, .chunk_release = ucc_tl_nccl_req_mapped_mpool_chunk_free, .obj_init = ucc_tl_nccl_req_mapped_mpool_obj_init, .obj_cleanup = ucc_tl_nccl_req_mpool_obj_cleanup }; UCC_CLASS_INIT_FUNC(ucc_tl_nccl_context_t, const ucc_base_context_params_t *params, const ucc_base_config_t *config) { ucc_tl_nccl_context_config_t *tl_nccl_config = ucc_derived_of(config, ucc_tl_nccl_context_config_t); int mem_ops_attr = 0; ucc_status_t status; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_context_t, &tl_nccl_config->super, params->context); memcpy(&self->cfg, tl_nccl_config, sizeof(*tl_nccl_config)); if (self->cfg.sync_type != UCC_TL_NCCL_COMPLETION_SYNC_TYPE_EVENT) { #if CUDA_VERSION < 12000 CUresult cu_st; CUdevice cu_dev; cu_st = cuCtxGetDevice(&cu_dev); if (cu_st == CUDA_SUCCESS) { cu_st = cuDeviceGetAttribute(&mem_ops_attr, CU_DEVICE_ATTRIBUTE_CAN_USE_STREAM_MEM_OPS, cu_dev); } else { tl_debug(self->super.super.lib, "failed to get cuda device"); } #else mem_ops_attr = 1; #endif if (mem_ops_attr == 0) { if (self->cfg.sync_type == UCC_TL_NCCL_COMPLETION_SYNC_TYPE_MEMOPS) { tl_error(self->super.super.lib, "memops not supported"); return UCC_ERR_NOT_SUPPORTED; } tl_debug(self->super.super.lib, "fallback to event completion sync"); self->cfg.sync_type = UCC_TL_NCCL_COMPLETION_SYNC_TYPE_EVENT; } else { self->cfg.sync_type = UCC_TL_NCCL_COMPLETION_SYNC_TYPE_MEMOPS; } } ucc_assert(self->cfg.sync_type == UCC_TL_NCCL_COMPLETION_SYNC_TYPE_MEMOPS || self->cfg.sync_type == UCC_TL_NCCL_COMPLETION_SYNC_TYPE_EVENT); if (self->cfg.sync_type == UCC_TL_NCCL_COMPLETION_SYNC_TYPE_MEMOPS) { tl_debug(self->super.super.lib, "using memops completion sync"); status = ucc_mpool_init(&self->req_mp, 0, sizeof(ucc_tl_nccl_task_t), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_tl_nccl_req_mapped_mpool_ops, params->thread_mode, "tl_nccl_req_mp"); } else { tl_debug(self->super.super.lib, "using event completion sync"); status = ucc_mpool_init(&self->req_mp, 0, sizeof(ucc_tl_nccl_task_t), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_tl_nccl_req_mpool_ops, params->thread_mode, "tl_nccl_req_mp"); } if (status != UCC_OK) { tl_error(self->super.super.lib, "failed to initialize tl_nccl_req mpool"); return status; } // scratch buffer for barrier cudaError_t cuda_st = cudaMalloc(&self->scratch_buf, sizeof(float)); if (cuda_st != cudaSuccess) { return UCC_ERR_NO_MEMORY; } tl_debug(self->super.super.lib, "initialized tl context: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_nccl_context_t) { tl_debug(self->super.super.lib, "finalizing tl context: %p", self); ucc_mpool_cleanup(&self->req_mp, 1); cudaFree(self->scratch_buf); self->scratch_buf = NULL; } UCC_CLASS_DEFINE(ucc_tl_nccl_context_t, ucc_tl_context_t); ucc_status_t ucc_tl_nccl_get_context_attr(const ucc_base_context_t *context, /* NOLINT */ ucc_base_ctx_attr_t *attr) { ucc_base_ctx_attr_clear(attr); return UCC_OK; } ucc_status_t ucc_tl_nccl_mem_map(const ucc_base_context_t *context, int type, /* NOLINT */ void *memh, void *tl_h) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_tl_nccl_mem_unmap(const ucc_base_context_t *context, int type, /* NOLINT */ void *memh) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_tl_nccl_memh_pack(const ucc_base_context_t *context, /* NOLINT */ int type, void *memh, void **pack_buffer) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc-1.8.0/src/components/tl/nccl/Makefile.am0000664000175000017500000000150615211535620021026 0ustar alastairalastair# # Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # if TL_NCCL_ENABLED allgatherv = \ allgatherv/allgatherv.h \ allgatherv/allgatherv.c sources = \ tl_nccl.h \ tl_nccl.c \ tl_nccl_lib.c \ tl_nccl_context.c \ tl_nccl_team.c \ tl_nccl_coll.h \ tl_nccl_coll.c \ $(allgatherv) module_LTLIBRARIES = libucc_tl_nccl.la libucc_tl_nccl_la_SOURCES = $(sources) libucc_tl_nccl_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) $(NCCL_CPPFLAGS) libucc_tl_nccl_la_CFLAGS = $(BASE_CFLAGS) libucc_tl_nccl_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(CUDA_LDFLAGS) $(NCCL_LDFLAGS) libucc_tl_nccl_la_LIBADD = $(CUDA_LIBS) $(NCCL_LIBADD) $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/tl/ucc_tl.h0000664000175000017500000001437115211535620017501 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #ifndef UCC_TL_H_ #define UCC_TL_H_ #include "components/base/ucc_base_iface.h" #include "core/ucc_context.h" #include "utils/ucc_sys.h" /** TL (transport layer) is an internal interface that provides a basic implementation of collectives and p2p primitives. It differs from CL in that it focuses on the abstraction over hardware transport rather than the programming model. These collectives can leverage either p2p communication transport or collective transport. The primary example of p2p communication transport is UCX, and the primary example of collective transport is SHARP, MCAST, and shared memory. */ typedef struct ucc_tl_lib ucc_tl_lib_t; typedef struct ucc_tl_iface ucc_tl_iface_t; typedef struct ucc_tl_context ucc_tl_context_t; typedef struct ucc_tl_team ucc_tl_team_t; typedef struct ucc_tl_lib_config { ucc_base_lib_config_t super; ucc_tl_iface_t *iface; } ucc_tl_lib_config_t; extern ucc_config_field_t ucc_tl_lib_config_table[]; typedef struct ucc_tl_context_config { ucc_base_ctx_config_t super; ucc_tl_lib_t *tl_lib; } ucc_tl_context_config_t; extern ucc_config_field_t ucc_tl_context_config_table[]; ucc_status_t ucc_tl_context_config_read(ucc_tl_lib_t *tl_lib, const ucc_context_config_t *config, ucc_tl_context_config_t **cl_config); ucc_status_t ucc_tl_lib_config_read(ucc_tl_iface_t *iface, const char *full_prefix, ucc_tl_lib_config_t **cl_config); typedef struct ucc_tl_service_coll { ucc_status_t (*allreduce)(ucc_base_team_t *team, void *sbuf, void *rbuf, ucc_datatype_t dt, size_t count, ucc_reduction_op_t op, ucc_subset_t subset, ucc_coll_task_t **task); ucc_status_t (*allgather)(ucc_base_team_t *team, void *sbuf, void *rbuf, size_t msgsize, ucc_subset_t subset, ucc_coll_task_t **task); ucc_status_t (*bcast)(ucc_base_team_t *team, void *buf, size_t msgsize, ucc_rank_t root, ucc_subset_t subset, ucc_coll_task_t **task); void (*update_id)(ucc_base_team_t *team, uint16_t id); } ucc_tl_service_coll_t; typedef struct ucc_tl_coll_plugin_iface { ucc_component_iface_t super; ucs_config_global_list_entry_t config; ucc_get_coll_scores_fn_t get_scores; uint32_t id; } ucc_tl_coll_plugin_iface_t; typedef struct ucc_tl_iface { ucc_component_iface_t super; ucs_config_global_list_entry_t tl_lib_config; ucs_config_global_list_entry_t tl_context_config; ucc_base_lib_iface_t lib; ucc_base_context_iface_t context; ucc_base_team_iface_t team; ucc_base_coll_iface_t coll; ucc_tl_service_coll_t scoll; ucc_base_coll_alg_info_t * alg_info[UCC_COLL_TYPE_NUM]; ucc_component_framework_t coll_plugins; } ucc_tl_iface_t; typedef struct ucc_tl_lib { ucc_base_lib_t super; ucc_tl_iface_t *iface; } ucc_tl_lib_t; UCC_CLASS_DECLARE(ucc_tl_lib_t, ucc_tl_iface_t *, const ucc_tl_lib_config_t *); typedef struct ucc_tl_context { ucc_base_context_t super; int ref_count; } ucc_tl_context_t; UCC_CLASS_DECLARE(ucc_tl_context_t, const ucc_tl_context_config_t *, ucc_context_t *); typedef struct ucc_tl_team { ucc_base_team_t super; } ucc_tl_team_t; UCC_CLASS_DECLARE(ucc_tl_team_t, ucc_tl_context_t *, const ucc_base_team_params_t *); #define UCC_TL_IFACE_DECLARE(_name, _NAME) \ UCC_BASE_IFACE_DECLARE(TL_, tl_, _name, _NAME) ucc_status_t ucc_tl_context_get(ucc_context_t *ctx, const char *name, ucc_tl_context_t **tl_context); ucc_status_t ucc_tl_context_put(ucc_tl_context_t *tl_context); typedef struct ucc_team_multiple_req { int n_teams; int last; struct ucc_team_team_desc { ucc_tl_context_t *ctx; ucc_tl_team_t *team; ucc_base_team_params_t param; ucc_status_t status; uint64_t args[2]; } descs[1]; } ucc_team_multiple_req_t; ucc_status_t ucc_team_multiple_req_alloc(ucc_team_multiple_req_t **req, int n_teams); ucc_status_t ucc_tl_team_create_multiple(ucc_team_multiple_req_t *req); ucc_status_t ucc_tl_team_destroy_multiple(ucc_team_multiple_req_t *req); void ucc_team_multiple_req_free(ucc_team_multiple_req_t *req); typedef struct ucc_tl_lib_attr { ucc_base_lib_attr_t super; } ucc_tl_lib_attr_t; #define UCC_TL_CTX_IFACE(_tl_ctx) \ (ucc_derived_of((_tl_ctx)->super.lib, ucc_tl_lib_t))->iface #define UCC_TL_TEAM_IFACE(_tl_team) \ (ucc_derived_of((_tl_team)->super.context->lib, ucc_tl_lib_t))->iface /** * Get TL team lib * @param [in] _tl_team pointer to TL team object * @return pointer to TL lib object */ #define UCC_TL_TEAM_LIB(_tl_team) (_tl_team)->super.super.context->lib /** * Get TL team context * @param [in] _tl_team pointer to TL team object * @return pointer to TL context object */ #define UCC_TL_TEAM_CTX(_tl_team) (_tl_team)->super.super.context #define UCC_TL_CORE_CTX(_tl_team) ((_tl_team)->super.super.context->ucc_context) #define UCC_TL_CTX_OOB(_ctx) ((_ctx)->super.super.ucc_context->params.oob) #define UCC_TL_TEAM_SIZE(_tl_team) (_tl_team)->super.super.params.size #define UCC_TL_TEAM_RANK(_tl_team) (_tl_team)->super.super.params.rank #define UCC_TL_CORE_TEAM(_tl_team) (_tl_team)->super.super.params.team #define UCC_TL_TEAM_MAP(_tl_team) (_tl_team)->super.super.params.map #define UCC_TL_TEAM_OOB(_tl_team) (_tl_team)->super.super.params.params.oob #define UCC_TL_IS_SERVICE_TEAM(_tl_team) \ ((_tl_team)->super.super.params.scope == UCC_CL_LAST + 1) #endif ucc-1.8.0/src/components/tl/cuda/0000775000175000017500000000000015211535620016765 5ustar alastairalastairucc-1.8.0/src/components/tl/cuda/tl_cuda_cache.c0000664000175000017500000003016315211535620021672 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_cuda.h" #include "tl_cuda_cache.h" #include "tl_cuda_ep_hash.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "utils/arch/cuda_def.h" #include "core/ucc_team.h" #include #define ENABLE_CACHE 1 static ucs_pgt_dir_t* ucc_tl_cuda_cache_pgt_dir_alloc(const ucs_pgtable_t *pgtable) //NOLINT: pgtable is unused { void *ptr; int ret; ret = posix_memalign(&ptr, ucc_max(sizeof(void *), UCS_PGT_ENTRY_MIN_ALIGN), sizeof(ucs_pgt_dir_t)); return (ret == 0) ? ptr : NULL; } static void ucc_tl_cuda_cache_pgt_dir_release(const ucs_pgtable_t *pgtable, //NOLINT: pgtable is unused ucs_pgt_dir_t *dir) { free(dir); } static void ucc_tl_cuda_cache_region_collect_callback(const ucs_pgtable_t *pgtable, //NOLINT: pgtable is unused ucs_pgt_region_t *pgt_region, void *arg) { ucs_list_link_t *list = arg; ucc_tl_cuda_cache_region_t *region; region = ucc_derived_of(pgt_region, ucc_tl_cuda_cache_region_t); ucs_list_add_tail(list, ®ion->list); } static void ucc_tl_cuda_cache_purge(ucc_tl_cuda_cache_t *cache) { ucc_tl_cuda_cache_region_t *region, *tmp; ucs_list_link_t region_list; ucs_list_head_init(®ion_list); ucs_pgtable_purge(&cache->pgtable, ucc_tl_cuda_cache_region_collect_callback, ®ion_list); ucs_list_for_each_safe(region, tmp, ®ion_list, list) { CUDA_FUNC(cudaIpcCloseMemHandle(region->mapped_addr)); free(region); } } ucc_status_t ucc_tl_cuda_create_cache(ucc_tl_cuda_cache_t **cache, const char *name) { ucc_status_t status; ucs_status_t ucs_st; ucc_tl_cuda_cache_t *cache_desc; int ret; cache_desc = ucc_malloc(sizeof(ucc_tl_cuda_cache_t), "ucc_tl_cuda_cache_t"); if (cache_desc == NULL) { ucc_error("failed to allocate memory for tl_cuda cache"); return UCC_ERR_NO_MEMORY; } ret = pthread_rwlock_init(&cache_desc->lock, NULL); if (ret) { ucc_error("pthread_rwlock_init() failed: %m"); status = UCC_ERR_INVALID_PARAM; goto err; } ucs_st = ucs_pgtable_init(&cache_desc->pgtable, ucc_tl_cuda_cache_pgt_dir_alloc, ucc_tl_cuda_cache_pgt_dir_release); if (ucc_unlikely(UCS_OK != ucs_st)) { status = ucs_status_to_ucc_status(ucs_st); goto err_destroy_rwlock; } cache_desc->name = strdup(name); if (cache_desc->name == NULL) { status = UCC_ERR_NO_MEMORY; goto err_destroy_rwlock; } *cache = cache_desc; return UCC_OK; err_destroy_rwlock: pthread_rwlock_destroy(&cache_desc->lock); err: free(cache_desc); return status; } void ucc_tl_cuda_destroy_cache(ucc_tl_cuda_cache_t *cache) { ucc_tl_cuda_cache_purge(cache); ucs_pgtable_cleanup(&cache->pgtable); pthread_rwlock_destroy(&cache->lock); free(cache->name); free(cache); } #if ENABLE_CACHE static void ucc_tl_cuda_cache_invalidate_regions(ucc_tl_cuda_cache_t *cache, void *from, void *to) { ucs_list_link_t region_list; ucs_status_t status; ucc_tl_cuda_cache_region_t *region, *tmp; ucs_list_head_init(®ion_list); ucs_pgtable_search_range(&cache->pgtable, (ucs_pgt_addr_t)from, (ucs_pgt_addr_t)to - 1, ucc_tl_cuda_cache_region_collect_callback, ®ion_list); ucs_list_for_each_safe(region, tmp, ®ion_list, list) { status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); if (status != UCS_OK) { ucc_error("failed to remove address:%p from cache (%s)", (void *)region->d_ptr, ucs_status_string(status)); } CUDA_FUNC(cudaIpcCloseMemHandle(region->mapped_addr)); free(region); } ucc_trace("%s: closed memhandles in the range [%p..%p]", cache->name, from, to); } #endif ucc_status_t ucc_tl_cuda_map_memhandle(const void *d_ptr, size_t size, cudaIpcMemHandle_t mem_handle, void **mapped_addr, ucc_tl_cuda_cache_t *cache) { if (d_ptr == NULL || size == 0) { *mapped_addr = NULL; return UCC_OK; } #if ENABLE_CACHE ucc_status_t status; ucs_status_t ucs_status; ucs_pgt_region_t *pgt_region; ucc_tl_cuda_cache_region_t *region; cudaError_t cuerr; int ret; pthread_rwlock_wrlock(&cache->lock); //todo :is lock needed pgt_region = ucs_pgtable_lookup(&cache->pgtable, (uintptr_t) d_ptr); if (pgt_region != NULL) { region = ucc_derived_of(pgt_region, ucc_tl_cuda_cache_region_t); if (memcmp((const void *)&mem_handle, (const void *)®ion->mem_handle, sizeof(cudaIpcMemHandle_t)) == 0) { /*cache hit */ ucc_debug("%s: tl_cuda cache hit addr:%p size:%lu region:" UCS_PGT_REGION_FMT, cache->name, d_ptr, size, UCS_PGT_REGION_ARG(®ion->super)); *mapped_addr = region->mapped_addr; ucc_assert(region->refcount < UINT64_MAX); region->refcount++; pthread_rwlock_unlock(&cache->lock); return UCC_OK; } else { ucc_debug("%s: tl_cuda cache remove stale region:" UCS_PGT_REGION_FMT " new_addr:%p new_size:%lu", cache->name, UCS_PGT_REGION_ARG(®ion->super), d_ptr, size); ucs_status = ucs_pgtable_remove(&cache->pgtable, ®ion->super); if (ucc_unlikely(ucs_status != UCS_OK)) { ucc_error("%s: failed to remove address:%p from cache", cache->name, d_ptr); status = ucs_status_to_ucc_status(ucs_status); goto err; } /* close memhandle */ cuerr = cudaIpcCloseMemHandle(region->mapped_addr); if (ucc_unlikely(cuerr != cudaSuccess)) { ucc_error("cudaIpcCloseMemHandle error %d %s", cuerr, cudaGetErrorName(cuerr)); status = UCC_ERR_NO_MESSAGE; goto err; } free(region); } } cuerr = cudaIpcOpenMemHandle(mapped_addr, mem_handle, cudaIpcMemLazyEnablePeerAccess); if (cuerr != cudaSuccess) { if (cuerr == cudaErrorAlreadyMapped) { ucc_tl_cuda_cache_invalidate_regions(cache, (void *)ucc_align_down_pow2((uintptr_t)d_ptr, UCS_PGT_ADDR_ALIGN), (void *)ucc_align_up_pow2((uintptr_t)d_ptr + size, UCS_PGT_ADDR_ALIGN)); cuerr = cudaIpcOpenMemHandle(mapped_addr, mem_handle, cudaIpcMemLazyEnablePeerAccess); if (cuerr != cudaSuccess) { if (cuerr == cudaErrorAlreadyMapped) { ucc_tl_cuda_cache_purge(cache); cuerr = cudaIpcOpenMemHandle(mapped_addr, mem_handle, cudaIpcMemLazyEnablePeerAccess); if (ucc_unlikely(cuerr != cudaSuccess)) { ucc_error("cudaIpcOpenMemHandle error %d %s", cuerr, cudaGetErrorString(cuerr)); status = UCC_ERR_INVALID_PARAM; goto err; } } else { ucc_error("%s: failed to open ipc mem handle. addr:%p len:%lu", cache->name, d_ptr, size); } } cudaGetLastError(); } else { ucc_error("%s: failed to open ipc mem handle. addr:%p len:%lu " "err: %d %s", cache->name, d_ptr, size, cuerr, cudaGetErrorString(cuerr)); status = UCC_ERR_NO_MESSAGE; goto err; } } /*create new cache entry */ ret = posix_memalign((void **)®ion, ucc_max(sizeof(void *), UCS_PGT_ENTRY_MIN_ALIGN), sizeof(ucc_tl_cuda_cache_region_t)); if (ret != 0) { ucc_warn("failed to allocate uct_tl_cuda_cache region"); status = UCC_ERR_NO_MEMORY; goto err; } region->super.start = ucc_align_down_pow2((uintptr_t)d_ptr, UCS_PGT_ADDR_ALIGN); region->super.end = ucc_align_up_pow2 ((uintptr_t)d_ptr + size, UCS_PGT_ADDR_ALIGN); region->d_ptr = (void *)d_ptr; region->size = size; region->mem_handle = mem_handle; region->mapped_addr = *mapped_addr; region->refcount = 1; ucs_status = ucs_pgtable_insert(&cache->pgtable, ®ion->super); if (ucs_status == UCS_ERR_ALREADY_EXISTS) { /* overlapped region means memory freed at source. remove and try insert */ ucc_tl_cuda_cache_invalidate_regions(cache, (void *)region->super.start, (void *)region->super.end); ucs_status = ucs_pgtable_insert(&cache->pgtable, ®ion->super); } if (ucs_status != UCS_OK) { ucc_error("%s: failed to insert region:"UCS_PGT_REGION_FMT" size:%lu :%s", cache->name, UCS_PGT_REGION_ARG(®ion->super), size, ucs_status_string(ucs_status)); free(region); status = ucs_status_to_ucc_status(ucs_status); goto err; } ucc_debug("%s: tl_cuda cache new region:"UCS_PGT_REGION_FMT" size:%lu", cache->name, UCS_PGT_REGION_ARG(®ion->super), size); status = UCC_OK; err: pthread_rwlock_unlock(&cache->lock); // coverity[leaked_storage:FALSE] return status; #else CUDA_FUNC(cudaIpcOpenMemHandle(mapped_addr, mem_handle, cudaIpcMemLazyEnablePeerAccess)); return UCC_OK; #endif } ucc_status_t ucc_tl_cuda_unmap_memhandle(uintptr_t d_bptr, void *mapped_addr, ucc_tl_cuda_cache_t *cache, int force) { if ((d_bptr == 0) || (mapped_addr == 0)) { return UCC_OK; } #if ENABLE_CACHE ucs_pgt_region_t *pgt_region; ucc_tl_cuda_cache_region_t *region; /* use write lock because cache maybe modified */ pthread_rwlock_wrlock(&cache->lock); pgt_region = ucs_pgtable_lookup(&cache->pgtable, d_bptr); ucc_debug("%s: tl_cuda unmap addr:%p region:" UCS_PGT_REGION_FMT, cache->name, (void*)d_bptr, UCS_PGT_REGION_ARG(pgt_region)); ucc_assert(pgt_region != NULL); region = ucc_derived_of(pgt_region, ucc_tl_cuda_cache_region_t); ucc_assert(region->refcount >= 1); region->refcount--; if ((region->refcount == 0 ) && (force == 1)) { ucs_pgtable_remove(&cache->pgtable, ®ion->super); CUDA_FUNC(cudaIpcCloseMemHandle(mapped_addr)); } pthread_rwlock_unlock(&cache->lock); #else CUDA_FUNC(cudaIpcCloseMemHandle(mapped_addr)); #endif return UCC_OK; } ucc_tl_cuda_cache_t* ucc_tl_cuda_get_cache(ucc_tl_cuda_team_t *team, ucc_rank_t rank) { ucc_tl_cuda_context_t *ctx = UCC_TL_CUDA_TEAM_CTX(team); ucc_context_addr_header_t *h = NULL; ucc_tl_cuda_cache_t *cache; ucc_status_t status; h = ucc_get_team_ep_header(UCC_TL_CORE_CTX(team), UCC_TL_CORE_TEAM(team), rank); cache = tl_cuda_hash_get(ctx->ipc_cache, h->ctx_id); if (ucc_unlikely(NULL == cache)) { status = ucc_tl_cuda_create_cache(&cache, "ipc-cache"); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to create ipc cache"); return NULL; } tl_cuda_hash_put(ctx->ipc_cache, h->ctx_id, cache); } return cache; } ucc-1.8.0/src/components/tl/cuda/kernels/0000775000175000017500000000000015211535620020430 5ustar alastairalastairucc-1.8.0/src/components/tl/cuda/kernels/allgatherv_kernel.cu0000664000175000017500000000632715211535620024462 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifdef __cplusplus extern "C" { #endif #include "utils/arch/cuda_def.h" #include "../tl_cuda.h" #ifdef __cplusplus } #endif #include "nvls.cuh" __global__ void __launch_bounds__(UCC_TL_CUDA_MAX_NVLS_THREADS) allgatherv_kernel_vec32( ucc_tl_cuda_nvls_control_t *mc_bar, ucc_tl_cuda_nvls_control_t *uc_bar, /* block count per gpu * num gpus in Multicast group */ const uint32_t total_blocks, uint64_t launch_counter, uint32_t *src_u32, uint32_t *base_u32, size_t my_offset, size_t my_count) { // Pre-barrier: ensure all ranks are ready before writing nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 1)); // Each rank copies its data to NVLS mc buffer at its specific offset using multimem store // Datatype agnostic - just copy raw data as uint4 vectors (16 bytes at a time) size_t thread_offset = (threadIdx.x + blockIdx.x * blockDim.x) * 4; size_t stride = blockDim.x * gridDim.x * 4; for (size_t idx = thread_offset; idx < my_count; idx += stride) { // Read 4 uint32_t values (16 bytes) from source uint4 val = reinterpret_cast(src_u32)[idx / 4]; // Write to destination at my_offset using multimem store (datatype agnostic) uint32_t *dst = base_u32 + my_offset + idx; MULTIMEM_ST_U32(val, dst); } // Post-barrier: ensure all ranks have completed writing before reading nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 2)); } #ifdef __cplusplus extern "C" { #endif ucc_status_t post_allgatherv_kernel( cudaStream_t stream, uint32_t sm_count, uint32_t threads, CUdeviceptr src_ptr, CUdeviceptr mc_base_addr, size_t my_offset, size_t my_count, CUdeviceptr mc_control_addr, CUdeviceptr uc_control_addr, uint64_t launch_counter, uint32_t tsize) { ucc_assert(sm_count > 0 && sm_count <= UCC_TL_CUDA_MAX_NVLS_SM_COUNT); ucc_assert(threads > 0 && threads <= UCC_TL_CUDA_MAX_NVLS_THREADS); /* NVLS requires 16-byte alignment */ ucc_assert(my_offset % 4 == 0); ucc_assert(my_count % 4 == 0); ucc_assert(mc_base_addr % 16 == 0); ucc_assert(mc_control_addr % 16 == 0); ucc_assert(src_ptr % 16 == 0); uint32_t *src_u32 = reinterpret_cast(src_ptr); uint32_t *base_u32 = reinterpret_cast(mc_base_addr); ucc_tl_cuda_nvls_control_t *mc_bar = reinterpret_cast( mc_control_addr); ucc_tl_cuda_nvls_control_t *uc_bar = reinterpret_cast( uc_control_addr); /* total num of blocks in the multicast group, used for barrier sync */ uint32_t expected_blocks = sm_count * tsize; allgatherv_kernel_vec32<<>>( mc_bar, uc_bar, expected_blocks, launch_counter, src_u32, base_u32, my_offset, my_count); CUDA_CHECK(cudaGetLastError()); return UCC_OK; } #ifdef __cplusplus } #endif ucc-1.8.0/src/components/tl/cuda/kernels/reduce_scatter_kernel.cu0000664000175000017500000001711615211535620025323 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifdef __cplusplus extern "C" { #endif #include "utils/arch/cuda_def.h" #include "../tl_cuda.h" #ifdef __cplusplus } #endif #include "nvls.cuh" template __global__ void __launch_bounds__(UCC_TL_CUDA_MAX_NVLS_THREADS) reduce_scatter_kernel_vec32( ucc_tl_cuda_nvls_control_t *mc_bar, ucc_tl_cuda_nvls_control_t *uc_bar, const uint32_t total_blocks, uint64_t launch_counter, uint32_t *base_u32, size_t offset, size_t count, uint32_t *dst_u32) { // pre barrier nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 1)); size_t thread_offset = (threadIdx.x + blockIdx.x * blockDim.x) * 4; size_t stride = blockDim.x * gridDim.x * 4; for (size_t idx = offset + thread_offset; idx + 3 < offset + count; idx += stride) { uint4 val; NvlsOps::ld(val, base_u32 + idx); uint32_t *dst = dst_u32 + (idx - offset); reinterpret_cast(dst)[0] = val; } // post barrier nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 2)); } template __global__ void __launch_bounds__(UCC_TL_CUDA_MAX_NVLS_THREADS) reduce_scatter_kernel_scalar32( ucc_tl_cuda_nvls_control_t *mc_bar, ucc_tl_cuda_nvls_control_t *uc_bar, const uint32_t total_blocks, uint64_t launch_counter, uint32_t *base_u32, size_t offset, size_t count, uint32_t *dst_u32) { // pre barrier nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 1)); size_t thread_offset = (threadIdx.x + blockIdx.x * blockDim.x) * 4; size_t stride = blockDim.x * gridDim.x * 4; for (size_t idx = offset + thread_offset; idx + 3 < offset + count; idx += stride) { typename NvlsOps::value_type v0, v1, v2, v3; NvlsOps::ld(v0, base_u32 + idx + 0); NvlsOps::ld(v1, base_u32 + idx + 1); NvlsOps::ld(v2, base_u32 + idx + 2); NvlsOps::ld(v3, base_u32 + idx + 3); dst_u32[idx - offset + 0] = v0; dst_u32[idx - offset + 1] = v1; dst_u32[idx - offset + 2] = v2; dst_u32[idx - offset + 3] = v3; } // post barrier nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 2)); } template __global__ void __launch_bounds__(UCC_TL_CUDA_MAX_NVLS_THREADS) reduce_scatter_kernel_scalar64( ucc_tl_cuda_nvls_control_t *mc_bar, ucc_tl_cuda_nvls_control_t *uc_bar, const uint32_t total_blocks, uint64_t launch_counter, uint64_t *base_u64, size_t offset, size_t count, uint64_t *dst_u64) { // pre barrier nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 1)); size_t thread_offset = (threadIdx.x + blockIdx.x * blockDim.x) * 2; size_t stride = blockDim.x * gridDim.x * 2; for (size_t idx = offset + thread_offset; idx + 1 < offset + count; idx += stride) { typename NvlsOps::value_type v0, v1; NvlsOps::ld(v0, base_u64 + idx + 0); NvlsOps::ld(v1, base_u64 + idx + 1); dst_u64[idx - offset + 0] = v0; dst_u64[idx - offset + 1] = v1; } // post barrier nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 2)); } #ifdef __cplusplus extern "C" { #endif ucc_status_t post_reduce_scatter_kernel( cudaStream_t stream, uint32_t sm_count, uint32_t threads, CUdeviceptr dst_ptr, CUdeviceptr mc_base_addr, CUdeviceptr mc_control_addr, CUdeviceptr uc_control_addr, uint64_t launch_counter, size_t offset, size_t count, ucc_datatype_t datatype, uint32_t tsize) { ucc_assert(sm_count > 0 && sm_count <= UCC_TL_CUDA_MAX_NVLS_SM_COUNT); ucc_assert(threads > 0 && threads <= UCC_TL_CUDA_MAX_NVLS_THREADS); /* NVLS requires 16-byte alignment */ ucc_assert(offset % 4 == 0); ucc_assert(count % 4 == 0); ucc_assert(mc_base_addr % 16 == 0); ucc_assert(mc_control_addr % 16 == 0); ucc_assert(dst_ptr % 16 == 0); uint32_t *base_u32 = reinterpret_cast(mc_base_addr); ucc_tl_cuda_nvls_control_t *mc_bar = reinterpret_cast( mc_control_addr); ucc_tl_cuda_nvls_control_t *uc_bar = reinterpret_cast( uc_control_addr); // total num of blocks in the multicast group, num gpus * num blocks per gpu, used for barrier synchronization uint32_t expected_blocks = sm_count * tsize; switch (datatype) { case UCC_DT_FLOAT32: reduce_scatter_kernel_vec32 <<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u32, offset, count, reinterpret_cast(dst_ptr)); break; case UCC_DT_BFLOAT16: reduce_scatter_kernel_vec32 <<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u32, offset, count, reinterpret_cast(dst_ptr)); break; case UCC_DT_INT32: reduce_scatter_kernel_scalar32 <<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u32, offset, count, reinterpret_cast(dst_ptr)); break; case UCC_DT_UINT32: reduce_scatter_kernel_scalar32 <<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u32, offset, count, reinterpret_cast(dst_ptr)); break; case UCC_DT_INT64: { uint64_t *base_u64 = reinterpret_cast(mc_base_addr); size_t offset_u64 = offset / 2; size_t count_u64 = count / 2; reduce_scatter_kernel_scalar64 <<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u64, offset_u64, count_u64, reinterpret_cast(dst_ptr)); } break; case UCC_DT_UINT64: { uint64_t *base_u64 = reinterpret_cast(mc_base_addr); size_t offset_u64 = offset / 2; size_t count_u64 = count / 2; reduce_scatter_kernel_scalar64 <<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u64, offset_u64, count_u64, reinterpret_cast(dst_ptr)); } break; default: return UCC_ERR_NOT_SUPPORTED; } CUDA_CHECK(cudaGetLastError()); return UCC_OK; } #ifdef __cplusplus } #endif ucc-1.8.0/src/components/tl/cuda/kernels/allgatherv_kernel.h0000664000175000017500000000125715211535620024277 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_ALLGATHERV_KERNEL_H_ #define UCC_TL_CUDA_ALLGATHERV_KERNEL_H_ #include #include "ucc/api/ucc.h" #ifdef __cplusplus extern "C" { #endif /* Kernel function declaration */ ucc_status_t post_allgatherv_kernel( cudaStream_t stream, uint32_t sm_count, uint32_t threads, CUdeviceptr src_ptr, CUdeviceptr mc_base_addr, size_t my_offset, size_t my_count, CUdeviceptr mc_control_addr, CUdeviceptr uc_control_addr, uint64_t launch_counter, uint32_t tsize); #ifdef __cplusplus } #endif #endif // UCC_TL_CUDA_ALLGATHERV_KERNEL_H_ ucc-1.8.0/src/components/tl/cuda/kernels/reduce_scatter_kernel.h0000664000175000017500000000125715211535620025142 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_REDUCE_SCATTER_KERNEL_H_ #define UCC_TL_CUDA_REDUCE_SCATTER_KERNEL_H_ #include #include "ucc/api/ucc.h" #ifdef __cplusplus extern "C" { #endif ucc_status_t post_reduce_scatter_kernel( cudaStream_t stream, uint32_t sm_count, uint32_t threads, CUdeviceptr dst_ptr, CUdeviceptr mc_base_addr, CUdeviceptr mc_control_addr, CUdeviceptr uc_control_addr, uint64_t launch_counter, size_t offset, size_t count, ucc_datatype_t datatype, uint32_t tsize); #ifdef __cplusplus } #endif #endif // UCC_TL_CUDA_REDUCE_SCATTER_KERNEL_H_ ucc-1.8.0/src/components/tl/cuda/kernels/Makefile.am0000664000175000017500000000217715211535620022473 0ustar alastairalastair# # Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # NVCCFLAGS = \ ${AM_CPPFLAGS} \ ${UCS_CPPFLAGS} \ ${NVCC_CFLAGS} \ -I${UCC_TOP_BUILDDIR} \ -I${UCC_TOP_SRCDIR}/src \ -I${UCC_TOP_BUILDDIR}/src \ --compiler-options -fno-rtti,-fno-exceptions LINK = $(LIBTOOL) --mode=link $(CC) -o $@ .cu.o: $(NVCC) -c $< -o $@ $(NVCCFLAGS) $(NVCC_ARCH_NVLS) .cu.lo: /bin/bash $(top_srcdir)/cuda_lt.sh "$(LIBTOOL)" $@ $(NVCC) -c $< $(NVCCFLAGS) $(NVCC_ARCH_NVLS) comp_noinst = libucc_tl_cuda_kernels.la if TL_CUDA_NVLS_ENABLED libucc_tl_cuda_kernels_la_SOURCES = reduce_scatter_kernel.cu \ allreduce_kernel.cu \ allgatherv_kernel.cu else libucc_tl_cuda_kernels_la_SOURCES = endif libucc_tl_cuda_kernels_la_CPPFLAGS = noinst_LTLIBRARIES = $(comp_noinst) # Ensure kernels rebuild when helper header changes reduce_scatter_kernel.lo: nvls.cuh allreduce_kernel.lo: nvls.cuh allgatherv_kernel.lo: nvls.cuh ucc-1.8.0/src/components/tl/cuda/kernels/allreduce_kernel.h0000664000175000017500000000175515211535620024111 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_ALLREDUCE_KERNEL_H_ #define UCC_TL_CUDA_ALLREDUCE_KERNEL_H_ #include #include "ucc/api/ucc.h" #ifdef __cplusplus extern "C" { #endif // Kernel function declaration ucc_status_t post_allreduce_kernel(cudaStream_t stream, uint32_t sm_count, uint32_t threads, CUdeviceptr mc_base_addr, size_t src_size_bytes, CUdeviceptr mc_control_addr, CUdeviceptr uc_control_addr, uint64_t launch_counter, // launch counter for specific NVLS task in flight slot, used for barrier synchronization uint32_t rank, uint32_t tsize, ucc_datatype_t datatype); #ifdef __cplusplus } #endif #endif // UCC_TL_CUDA_ALLREDUCE_KERNEL_H_ ucc-1.8.0/src/components/tl/cuda/kernels/allreduce_kernel.cu0000664000175000017500000002007215211535620024262 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifdef __cplusplus extern "C" { #endif #include "utils/arch/cuda_def.h" #include "../tl_cuda.h" #ifdef __cplusplus } #endif #include "nvls.cuh" // vectorized allreduce kernel for 32-bit lanes template __global__ void __launch_bounds__(UCC_TL_CUDA_MAX_NVLS_THREADS) allreduce_kernel_vec32( ucc_tl_cuda_nvls_control_t *mc_bar, ucc_tl_cuda_nvls_control_t *uc_bar, const uint32_t total_blocks, uint64_t launch_counter, uint32_t *base_u32, size_t count_u32, uint32_t rank, uint32_t tsize) { size_t thread_offset = (threadIdx.x + blockIdx.x * blockDim.x) * 4; size_t stride = blockDim.x * gridDim.x * 4; size_t chunk_start; size_t chunk_end; uint4 val; nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 1)); if (count_u32 >= 4 * tsize) { chunk_start = ((int64_t)count_u32 * (int64_t)rank) / (int64_t)tsize; chunk_end = ((int64_t)count_u32 * (int64_t)(rank + 1)) / (int64_t)tsize; } else { chunk_start = 0; chunk_end = (rank == 0) ? count_u32 : 0; } for (size_t idx = chunk_start + thread_offset; idx < chunk_end; idx += stride) { NvlsOps::ld(val, base_u32 + idx); NvlsOps::st(val, base_u32 + idx); } nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 2)); } template __global__ void __launch_bounds__(UCC_TL_CUDA_MAX_NVLS_THREADS) allreduce_kernel_scalar32( ucc_tl_cuda_nvls_control_t *mc_bar, ucc_tl_cuda_nvls_control_t *uc_bar, const uint32_t total_blocks, uint64_t launch_counter, uint32_t *base_u32, size_t count_u32, uint32_t rank, uint32_t tsize) { size_t thread_offset = (threadIdx.x + blockIdx.x * blockDim.x) * 4; size_t stride = blockDim.x * gridDim.x * 4; size_t chunk_start; size_t chunk_end; typename NvlsOps::value_type v0, v1, v2, v3; nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 1)); if (count_u32 >= 4 * tsize) { chunk_start = ((int64_t)count_u32 * (int64_t)rank) / (int64_t)tsize; chunk_end = ((int64_t)count_u32 * (int64_t)(rank + 1)) / (int64_t)tsize; } else { chunk_start = 0; chunk_end = (rank == 0) ? count_u32 : 0; } for (size_t idx = chunk_start + thread_offset; idx < chunk_end; idx += stride) { NvlsOps::ld(v0, base_u32 + idx + 0); NvlsOps::ld(v1, base_u32 + idx + 1); NvlsOps::ld(v2, base_u32 + idx + 2); NvlsOps::ld(v3, base_u32 + idx + 3); NvlsOps::st(v0, base_u32 + idx + 0); NvlsOps::st(v1, base_u32 + idx + 1); NvlsOps::st(v2, base_u32 + idx + 2); NvlsOps::st(v3, base_u32 + idx + 3); } nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 2)); } template __global__ void __launch_bounds__(UCC_TL_CUDA_MAX_NVLS_THREADS) allreduce_kernel_scalar64( ucc_tl_cuda_nvls_control_t *mc_bar, ucc_tl_cuda_nvls_control_t *uc_bar, const uint32_t total_blocks, uint64_t launch_counter, uint64_t *base_u64, size_t count_u64, uint32_t rank, uint32_t tsize) { size_t thread_offset = (threadIdx.x + blockIdx.x * blockDim.x) * 2; size_t stride = blockDim.x * gridDim.x * 2; size_t chunk_start; size_t chunk_end; typename NvlsOps::value_type v0, v1; nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 1)); if (count_u64 >= 2 * tsize) { chunk_start = ((int64_t)count_u64 * (int64_t)rank) / (int64_t)tsize; chunk_end = ((int64_t)count_u64 * (int64_t)(rank + 1)) / (int64_t)tsize; } else { chunk_start = 0; chunk_end = (rank == 0) ? count_u64 : 0; } for (size_t idx = chunk_start + thread_offset; idx < chunk_end; idx += stride) { NvlsOps::ld(v0, base_u64 + idx + 0); NvlsOps::ld(v1, base_u64 + idx + 1); NvlsOps::st(v0, base_u64 + idx + 0); NvlsOps::st(v1, base_u64 + idx + 1); } nvls_bar( &(mc_bar->arrival_counter), &(uc_bar->arrival_counter), total_blocks * (launch_counter * 2 + 2)); } #ifdef __cplusplus extern "C" { #endif ucc_status_t post_allreduce_kernel(cudaStream_t stream, uint32_t sm_count, uint32_t threads, CUdeviceptr mc_base_addr, size_t src_size_bytes, CUdeviceptr mc_control_addr, CUdeviceptr uc_control_addr, uint64_t launch_counter, uint32_t rank, uint32_t tsize, ucc_datatype_t datatype) { assert(sm_count > 0 && sm_count <= UCC_TL_CUDA_MAX_NVLS_SM_COUNT); assert(threads > 0 && threads <= UCC_TL_CUDA_MAX_NVLS_THREADS); uint32_t *base_u32 = reinterpret_cast(mc_base_addr); size_t count_u32 = src_size_bytes / sizeof(uint32_t); ucc_tl_cuda_nvls_control_t *mc_bar = reinterpret_cast(mc_control_addr); ucc_tl_cuda_nvls_control_t *uc_bar = reinterpret_cast(uc_control_addr); uint32_t expected_blocks = sm_count * tsize; // total num of blocks in the multicast group, num gpus * num blocks per gpu, used for barrier synchronization assert(((uintptr_t)(mc_base_addr) % 16) == 0); if (datatype == UCC_DT_FLOAT32 || datatype == UCC_DT_BFLOAT16) { assert(count_u32 % (4 * tsize) == 0); } switch (datatype) { case UCC_DT_FLOAT32: allreduce_kernel_vec32<<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u32, count_u32, rank, tsize); break; case UCC_DT_BFLOAT16: allreduce_kernel_vec32<<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u32, count_u32, rank, tsize); break; case UCC_DT_INT32: allreduce_kernel_scalar32 <<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u32, count_u32, rank, tsize); break; case UCC_DT_UINT32: allreduce_kernel_scalar32 <<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u32, count_u32, rank, tsize); break; case UCC_DT_INT64: { uint64_t *base_u64 = reinterpret_cast(mc_base_addr); size_t count_u64 = src_size_bytes / sizeof(uint64_t); allreduce_kernel_scalar64 <<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u64, count_u64, rank, tsize); } break; case UCC_DT_UINT64: { uint64_t *base_u64 = reinterpret_cast(mc_base_addr); size_t count_u64 = src_size_bytes / sizeof(uint64_t); allreduce_kernel_scalar64 <<>>( mc_bar, uc_bar, expected_blocks, launch_counter, base_u64, count_u64, rank, tsize); } break; default: return UCC_ERR_NOT_SUPPORTED; } CUDA_CHECK(cudaGetLastError()); return UCC_OK; } #ifdef __cplusplus } #endif ucc-1.8.0/src/components/tl/cuda/kernels/nvls.cuh0000664000175000017500000001250715211535620022120 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_NVLS_CUH_ #define UCC_TL_CUDA_NVLS_CUH_ #include #include #include #define MULTIMEM_ST(val, ptr) \ asm volatile("multimem.st.global.v4.f32 [%0], {%1,%2,%3,%4};" ::"l"(ptr), \ "r"(val.x), "r"(val.y), "r"(val.z), "r"(val.w) \ : "memory"); #define MULTIMEM_ST_U32(val, ptr) \ asm volatile("multimem.st.global.v4.f32 [%0], {%1,%2,%3,%4};" ::"l"(ptr), \ "r"(val.x), "r"(val.y), "r"(val.z), "r"(val.w) \ : "memory"); #define MULTIMEM_LD(val, ptr) \ asm("multimem.ld_reduce.global.add.v4.f32 {%0,%1,%2,%3}, [%4];" \ : "=r"(val.x), "=r"(val.y), "=r"(val.z), "=r"(val.w) \ : "l"(ptr) \ : "memory"); #define MULTIMEM_ST_BF16(val, ptr) \ asm volatile("multimem.st.global.v4.bf16x2 [%0], {%1,%2,%3,%4};" ::"l"(ptr), \ "r"(val.x), "r"(val.y), "r"(val.z), "r"(val.w) \ : "memory"); #define MULTIMEM_LD_BF16(val, ptr) \ asm("multimem.ld_reduce.global.add.v4.bf16x2 {%0,%1,%2,%3}, [%4];" \ : "=r"(val.x), "=r"(val.y), "=r"(val.z), "=r"(val.w) \ : "l"(ptr) \ : "memory"); #ifdef __cplusplus // NVLS global barrier helper used by kernels to synchronize via multicast/unicast counters __device__ __forceinline__ void nvls_bar(uint64_t *mc_arrival_counter, uint64_t *uc_arrival_counter, uint64_t expected_count) { if (threadIdx.x == 0) { // first thread in block increments the multicast arrival counter asm volatile("multimem.red.release.sys.global.add.u64 [%0], %1;" ::"l"( mc_arrival_counter), "n"(1) : "memory"); asm volatile("fence.proxy.alias;" ::: "memory"); // waits others blocks to reach the same phase cuda::atomic_ref ac( *uc_arrival_counter); // sync per block: block 0 on gpu 0 with block 0 on gpu 1, block 1 on gpu 0 with block 1 on gpu 1, etc. while (expected_count > ac.load(cuda::memory_order_acquire)) { } } // all other threads in block wait for the first thread to finish __syncthreads(); } // Traits wrapping NVLS LD/ST variants on 32-bit lanes struct NvlsFp32Ops { __device__ static inline void ld(uint4 &v, const uint32_t *ptr) { MULTIMEM_LD(v, ptr); } __device__ static inline void st(const uint4 &v, uint32_t *ptr) { MULTIMEM_ST(v, ptr); } }; struct NvlsBf16Ops { __device__ static inline void ld(uint4 &v, const uint32_t *ptr) { MULTIMEM_LD_BF16(v, ptr); } __device__ static inline void st(const uint4 &v, uint32_t *ptr) { MULTIMEM_ST_BF16(v, ptr); } }; struct NvlsInt32Ops { using value_type = int; __device__ static inline void ld(int &v, const uint32_t *ptr) { asm("multimem.ld_reduce.global.add.s32 %0, [%1];" : "=r"(v) : "l"(ptr) : "memory"); } __device__ static inline void st(const int &v, uint32_t *ptr) { asm volatile("multimem.st.global.s32 [%0], %1;" ::"l"(ptr), "r"(v) : "memory"); } }; struct NvlsUint32Ops { using value_type = unsigned int; __device__ static inline void ld(unsigned int &v, const uint32_t *ptr) { asm("multimem.ld_reduce.global.add.u32 %0, [%1];" : "=r"(v) : "l"(ptr) : "memory"); } __device__ static inline void st(const unsigned int &v, uint32_t *ptr) { asm volatile("multimem.st.global.u32 [%0], %1;" ::"l"(ptr), "r"(v) : "memory"); } }; // PTX does not support s64 with add operation, so we use u64 instead struct NvlsInt64Ops { using value_type = unsigned long long; __device__ static inline void ld(unsigned long long &v, const uint64_t *ptr) { asm("multimem.ld_reduce.global.add.u64 %0, [%1];" : "=l"(v) : "l"(ptr) : "memory"); } __device__ static inline void st(const unsigned long long &v, uint64_t *ptr) { asm volatile("multimem.st.global.u64 [%0], %1;" ::"l"(ptr), "l"(v) : "memory"); } }; struct NvlsUint64Ops { using value_type = unsigned long long; __device__ static inline void ld(unsigned long long &v, const uint64_t *ptr) { asm("multimem.ld_reduce.global.add.u64 %0, [%1];" : "=l"(v) : "l"(ptr) : "memory"); } __device__ static inline void st(const unsigned long long &v, uint64_t *ptr) { asm volatile("multimem.st.global.u64 [%0], %1;" ::"l"(ptr), "l"(v) : "memory"); } }; #endif // __cplusplus #endif // UCC_TL_CUDA_NVLS_CUH_ ucc-1.8.0/src/components/tl/cuda/tl_cuda_team_topo.h0000664000175000017500000000421615211535620022623 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_TEAM_TOPO_H_ #define UCC_TL_CUDA_TEAM_TOPO_H_ #include "components/tl/ucc_tl.h" #include "tl_cuda_topo.h" typedef struct ucc_tl_cuda_proxy { ucc_rank_t src; /* source rank */ ucc_rank_t dst; /* destination rank */ ucc_rank_t proxy; /* proxy rank */ } ucc_tl_cuda_proxy_t; typedef struct ucc_tl_cuda_ring { ucc_rank_t *ring; /* list of ranks forming ring */ ucc_rank_t *iring; /* inverse of ring */ } ucc_tl_cuda_ring_t; typedef struct ucc_tl_cuda_team_topo { ucc_rank_t *matrix; /* nvlink adjacency matrix */ int proxy_needed; /* is proxy needed for current rank */ int num_proxies; /* number of entries in proxies list */ ucc_tl_cuda_proxy_t *proxies; /* list of pairs where current rank is proxy */ int num_rings; /* number of entries in rings list */ ucc_tl_cuda_ring_t *rings; /* list of rings for ring algorithms */ int is_fully_connected; /* no proxies in team topo */ } ucc_tl_cuda_team_topo_t; ucc_status_t ucc_tl_cuda_team_topo_create(const ucc_tl_team_t *team, ucc_tl_cuda_team_topo_t **team_topo); ucc_status_t ucc_tl_cuda_team_topo_destroy(ucc_tl_cuda_team_topo_t *team_topo); void ucc_tl_cuda_team_topo_print_proxies(const ucc_tl_team_t *team, const ucc_tl_cuda_team_topo_t *topo); void ucc_tl_cuda_team_topo_print_rings(const ucc_tl_team_t *tl_team, const ucc_tl_cuda_team_topo_t *topo); static inline int ucc_tl_cuda_team_topo_is_direct(const ucc_tl_team_t *team, const ucc_tl_cuda_team_topo_t *topo, ucc_rank_t r1, ucc_rank_t r2) { return topo->matrix[r1 * team->super.params.size + r2] != 0; } static inline int ucc_tl_cuda_team_topo_is_fully_connected(const ucc_tl_cuda_team_topo_t *topo) { return topo->is_fully_connected; } #endif ucc-1.8.0/src/components/tl/cuda/tl_cuda_coll.h0000664000175000017500000002366415211535620021575 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_COLL_H_ #define UCC_TL_CUDA_COLL_H_ #include "tl_cuda.h" #include "components/mc/ucc_mc.h" #define UCC_TL_CUDA_N_DEFAULT_ALG_SELECT_STR 6 extern const char *ucc_tl_cuda_default_alg_select_str[UCC_TL_CUDA_N_DEFAULT_ALG_SELECT_STR]; #define TASK_TEAM(_task) \ (ucc_derived_of((_task)->super.team, ucc_tl_cuda_team_t)) #define TASK_ARGS(_task) (_task)->super.bargs.args #define TASK_SYNC(_task, _rank) \ ({ \ ucc_tl_cuda_team_t *_team = TASK_TEAM(_task); \ UCC_TL_CUDA_TEAM_SYNC(_team, _rank, (_task)->coll_id); \ }) #define TASK_BAR(_task) \ ({ \ ucc_tl_cuda_team_t *_team = TASK_TEAM(_task); \ UCC_TL_CUDA_TEAM_BARRIER(_team, (_task)->coll_id); \ }) #define TASK_SCRATCH(_task, _rank) \ ({ \ ucc_tl_cuda_team_t *_team = TASK_TEAM(_task); \ size_t _scratch_size = UCC_TL_CUDA_TEAM_LIB(_team)->cfg.scratch_size; \ void *_scratch; \ if (_rank == UCC_TL_TEAM_RANK(_team)) { \ _scratch = _team->scratch.loc; \ } else { \ _scratch = PTR_OFFSET(_team->scratch.rem[_rank], \ _team->scratch.rem_info[_rank].offset); \ } \ (PTR_OFFSET(_scratch, (_task)->coll_id * _scratch_size)); \ }) #define NVLS_CONTROL_SIZE 1024 #define TASK_SYMMETRIC_MC(_task) \ ({ \ ucc_tl_cuda_team_t *_team = TASK_TEAM(_task); \ size_t _symm_size = UCC_TL_CUDA_TEAM_LIB(_team)->cfg.nvls_symmetric_size + NVLS_CONTROL_SIZE; \ (PTR_OFFSET(_team->nvls.mc_va, (_task)->coll_id * _symm_size)); \ }) #define TASK_SYMMETRIC_UC(_task) \ ({ \ ucc_tl_cuda_team_t *_team = TASK_TEAM(_task); \ size_t _symm_size = UCC_TL_CUDA_TEAM_LIB(_team)->cfg.nvls_symmetric_size + NVLS_CONTROL_SIZE; \ (PTR_OFFSET(_team->nvls.uc_va, (_task)->coll_id * _symm_size)); \ }) #define TASK_NVLS_CONTROL_MC(_task) \ ({ \ ucc_tl_cuda_team_t *_team = TASK_TEAM(_task); \ size_t _symm_payload_size = UCC_TL_CUDA_TEAM_LIB(_team)->cfg.nvls_symmetric_size; \ size_t _symm_size = _symm_payload_size + NVLS_CONTROL_SIZE; \ ((CUdeviceptr) PTR_OFFSET(_team->nvls.mc_va, (_task)->coll_id * _symm_size + _symm_payload_size)); \ }) #define TASK_NVLS_CONTROL_UC(_task) \ ({ \ ucc_tl_cuda_team_t *_team = TASK_TEAM(_task); \ size_t _symm_payload_size = UCC_TL_CUDA_TEAM_LIB(_team)->cfg.nvls_symmetric_size; \ size_t _symm_size = _symm_payload_size + NVLS_CONTROL_SIZE; \ ((CUdeviceptr) PTR_OFFSET(_team->nvls.uc_va, (_task)->coll_id * _symm_size + _symm_payload_size)); \ }) static inline void ucc_tl_cuda_task_reset(ucc_tl_cuda_task_t *task) { task->super.status = UCC_INPROGRESS; } ucc_status_t ucc_tl_cuda_shm_barrier_init_root(ucc_rank_t size, ucc_rank_t rank, ucc_rank_t root, ucc_tl_cuda_shm_barrier_t *barrier); ucc_status_t ucc_tl_cuda_shm_barrier_init(ucc_rank_t size, ucc_rank_t rank, ucc_tl_cuda_shm_barrier_t *barrier); ucc_status_t ucc_tl_cuda_shm_barrier_start(ucc_rank_t rank, ucc_tl_cuda_shm_barrier_t *barrier); ucc_status_t ucc_tl_cuda_shm_barrier_test(ucc_rank_t rank, ucc_tl_cuda_shm_barrier_t *barrier); static inline ucc_tl_cuda_task_t *ucc_tl_cuda_task_get(ucc_tl_cuda_team_t *team) { ucc_tl_cuda_context_t *ctx = UCC_TL_CUDA_TEAM_CTX(team); ucc_tl_cuda_task_t *task = ucc_mpool_get(&ctx->req_mp); if (ucc_unlikely(!task)) { tl_error(UCC_TL_CUDA_TEAM_LIB(team), "failed to get task from mpool"); return NULL; } UCC_TL_CUDA_PROFILE_REQUEST_NEW(task, "tl_cuda_task", 0); task->super.status = UCC_OPERATION_INITIALIZED; task->super.flags = 0; task->super.team = &team->super.super; ucc_tl_cuda_task_reset(task); return task; } static inline void ucc_tl_cuda_task_put(ucc_tl_cuda_task_t *task) { UCC_TL_CUDA_PROFILE_REQUEST_FREE(task); ucc_mpool_put(task); } static inline uint64_t compute_key(ucc_rank_t root, ucc_rank_t peer, uint16_t tag) { assert(peer < (1 << 24)); assert(root < (1 << 24)); return (uint64_t)tag << 48 | root << 24 | peer; } static inline ucc_status_t ucc_tl_cuda_task_init(ucc_base_coll_args_t *coll_args, ucc_tl_cuda_team_t *team, ucc_tl_cuda_task_t **task_h) { ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_tl_cuda_lib_t *lib = UCC_TL_CUDA_TEAM_LIB(team); uint32_t max_concurrent = lib->cfg.max_concurrent; ucc_rank_t peer; ucc_tl_cuda_task_t *task; ucc_status_t status; if (!ucc_coll_args_is_predefined_dt(&coll_args->args, trank)) { return UCC_ERR_NOT_SUPPORTED; } task = ucc_tl_cuda_task_get(team); if (ucc_unlikely(!task)) { return UCC_ERR_NO_MEMORY; } status = ucc_coll_task_init(&task->super, coll_args, &team->super.super); if (ucc_unlikely(status != UCC_OK)) { ucc_tl_cuda_task_put(task); return status; } /* active set */ if (UCC_COLL_ARGS_ACTIVE_SET(&coll_args->args)) { ucc_assert(coll_args->args.coll_type == UCC_COLL_TYPE_BCAST); task->subset.map = ucc_active_set_to_ep_map(&coll_args->args); task->subset.myrank = UCC_TL_TEAM_RANK(team); // currently we support only active set bacst with 2 ranks // so root rank should remap phys rank of peer with rank 1 peer = (task->subset.myrank == coll_args->args.root) ? ucc_ep_map_eval(task->subset.map, 1) : task->subset.myrank; task->bcast_linear.key = compute_key(coll_args->args.root, peer, coll_args->args.tag); task->seq_num = team->seq_num_active_set++; } else { task->seq_num = team->seq_num++; task->coll_id = task->seq_num % max_concurrent; task->bar = TASK_BAR(task); } *task_h = task; return UCC_OK; } // check if segment for current task is available and barrier is available (completed from prev iteration) // and possibly mark the segment as occupied by updating the state counter to the current seq_num static inline ucc_status_t ucc_tl_cuda_get_sync_root(ucc_tl_cuda_task_t *task, ucc_rank_t root) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); volatile ucc_tl_cuda_sync_state_t *state = &team->sync_state[task->coll_id]; if ((UCC_TL_TEAM_RANK(team) == root) && (*state == 0)) { *state = task->seq_num; } if ((*state != task->seq_num) || (task->bar->state[UCC_TL_TEAM_RANK(team)] != UCC_OK)) { return UCC_INPROGRESS; } return UCC_OK; } static inline void ucc_tl_cuda_put_sync_root(ucc_tl_cuda_task_t *task, ucc_rank_t root) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_tl_cuda_sync_state_t *state = &team->sync_state[task->coll_id]; if (UCC_TL_TEAM_RANK(team) == root) { ucc_assert(*state == task->seq_num); *state = 0; } } static inline ucc_status_t ucc_tl_cuda_get_sync(ucc_tl_cuda_task_t *task) { return ucc_tl_cuda_get_sync_root(task, 0); } static inline void ucc_tl_cuda_put_sync(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_put_sync_root(task, 0); } ucc_status_t ucc_tl_cuda_mem_info_get(void *ptr, size_t length, ucc_tl_cuda_mem_info_t *mi); ucc_status_t ucc_tl_cuda_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_cuda_coll_finalize(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_cuda_alg_id_to_init(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, ucc_base_coll_init_fn_t *init); // common utils function for collectives: static inline int get_rank_step(ucc_tl_cuda_task_t *task, ucc_rank_t rank, int step_id) { ucc_tl_cuda_sync_t *sync = TASK_SYNC(task, rank); return sync->seq_num[step_id]; } static inline void set_rank_step(ucc_tl_cuda_task_t *task, ucc_rank_t rank, int step, int step_id) { ucc_tl_cuda_sync_t *sync = TASK_SYNC(task, rank); sync->seq_num[step_id] = step; } #endif ucc-1.8.0/src/components/tl/cuda/allgatherv/0000775000175000017500000000000015211535620021116 5ustar alastairalastairucc-1.8.0/src/components/tl/cuda/allgatherv/allgatherv.h0000664000175000017500000000554615211535620023432 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLGATHERV_H_ #define ALLGATHERV_H_ #include "tl_cuda.h" #include "tl_cuda_coll.h" enum { UCC_TL_CUDA_ALLGATHERV_ALG_AUTO, UCC_TL_CUDA_ALLGATHERV_ALG_RING, UCC_TL_CUDA_ALLGATHERV_ALG_LINEAR, #ifdef HAVE_NVLS UCC_TL_CUDA_ALLGATHERV_ALG_NVLS, #endif /* HAVE_NVLS */ UCC_TL_CUDA_ALLGATHERV_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_cuda_allgatherv_algs[UCC_TL_CUDA_ALLGATHERV_ALG_LAST + 1]; #define UCC_TL_CUDA_ALLGATHERV_DEFAULT_ALG_SELECT_STR "allgatherv:cuda:@0" ucc_status_t ucc_tl_cuda_allgatherv_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p); size_t ucc_tl_cuda_allgatherv_get_count(const ucc_tl_cuda_task_t *task, ucc_rank_t block); size_t ucc_tl_cuda_allgatherv_get_offset(const ucc_tl_cuda_task_t *task, ucc_rank_t block); /* Allgatherv ring */ ucc_status_t ucc_tl_cuda_allgatherv_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p); ucc_status_t ucc_tl_cuda_allgatherv_ring_start(ucc_coll_task_t *task); void ucc_tl_cuda_allgatherv_ring_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_allgatherv_ring_finalize(ucc_coll_task_t *task); /* Allgatherv linear */ ucc_status_t ucc_tl_cuda_allgatherv_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p); ucc_status_t ucc_tl_cuda_allgatherv_linear_start(ucc_coll_task_t *task); void ucc_tl_cuda_allgatherv_linear_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_allgatherv_linear_finalize(ucc_coll_task_t *task); #ifdef HAVE_NVLS /* Allgatherv NVLS */ ucc_status_t ucc_tl_cuda_allgatherv_nvls_init( ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p); ucc_status_t ucc_tl_cuda_allgatherv_nvls_start(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_allgatherv_nvls_triggered_post( ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *coll_task); void ucc_tl_cuda_allgatherv_nvls_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_allgatherv_nvls_finalize(ucc_coll_task_t *task); #endif /* HAVE_NVLS */ static inline int ucc_tl_cuda_allgatherv_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_CUDA_ALLGATHERV_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_cuda_allgatherv_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/cuda/allgatherv/allgatherv_nvls.c0000664000175000017500000002254015211535620024460 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allgatherv/allgatherv.h" #include #include "core/ucc_ee.h" #include "utils/arch/cuda_def.h" #include "tl_cuda_nvls.h" #include "kernels/allgatherv_kernel.h" #include "components/ec/ucc_ec.h" #include "components/ec/cuda/ec_cuda_resources.h" ucc_status_t ucc_tl_cuda_allgatherv_nvls_start(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_datatype_t dt = (args->coll_type == UCC_COLL_TYPE_ALLGATHERV) ? args->dst.info_v.datatype : args->dst.info.datatype; size_t dt_size = ucc_dt_size(dt); ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t)ee->ee_context : team->stream; ucc_ec_cuda_event_t *ec_event = (ucc_ec_cuda_event_t *) task->allgatherv_nvls.evt_completion; cudaEvent_t evt = ec_event->event; CUdeviceptr mc_va = task->allgatherv_nvls.mc_va; CUdeviceptr uc_va = task->allgatherv_nvls.uc_va; uint32_t sm_count = UCC_TL_CUDA_TEAM_LIB(team)->cfg.nvls_sm_count; uint32_t threads = UCC_TL_CUDA_TEAM_LIB(team)->cfg.nvls_threads; ucc_status_t status; void *sbuf; void *rbuf; if (args->coll_type == UCC_COLL_TYPE_ALLGATHERV) { rbuf = args->dst.info_v.buffer; sbuf = UCC_IS_INPLACE(*args) ? PTR_OFFSET( args->dst.info_v.buffer, ucc_tl_cuda_allgatherv_get_offset(task, trank) * dt_size) : args->src.info.buffer; } else { rbuf = args->dst.info.buffer; sbuf = UCC_IS_INPLACE(*args) ? PTR_OFFSET( args->dst.info.buffer, ucc_tl_cuda_allgatherv_get_offset(task, trank) * dt_size) : args->src.info.buffer; } /* Each rank copies its data to the NVLS buffer at its specific offset */ status = post_allgatherv_kernel( stream, sm_count, threads, (CUdeviceptr)sbuf, mc_va, task->allgatherv_nvls.offset, task->allgatherv_nvls.count, TASK_NVLS_CONTROL_MC(task), TASK_NVLS_CONTROL_UC(task), task->allgatherv_nvls.coll_id, UCC_TL_TEAM_SIZE(team)); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to post allgatherv kernel"); return status; } /* Copy gathered data from uc ptr to destination buffer * total_count is in uint32_t units, convert to bytes */ status = CUDA_FUNC(cudaMemcpyAsync( rbuf, (void *)uc_va, task->allgatherv_nvls.total_count * sizeof(uint32_t), cudaMemcpyDeviceToDevice, stream)); if (ucc_unlikely(status != UCC_OK)) { return status; } status = CUDA_FUNC(cudaEventRecord(evt, stream)); if (ucc_unlikely(status != UCC_OK)) { return status; } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } void ucc_tl_cuda_allgatherv_nvls_progress(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_ec_cuda_event_t *ec_event = (ucc_ec_cuda_event_t *) task->allgatherv_nvls.evt_completion; cudaEvent_t evt = ec_event->event; task->super.status = cuda_error_to_ucc_status(cudaEventQuery(evt)); } ucc_status_t ucc_tl_cuda_allgatherv_nvls_triggered_post( ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *coll_task) /* NOLINT */ { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_status_t status; ucc_ev_t post_event; ucc_assert(ee->ee_type == UCC_EE_CUDA_STREAM); coll_task->ee = ee; tl_trace(UCC_TASK_LIB(task), "triggered post. task:%p", coll_task); status = coll_task->post(coll_task); if (ucc_likely(status == UCC_OK)) { post_event.ev_type = UCC_EVENT_COLLECTIVE_POST; post_event.ev_context_size = 0; post_event.ev_context = NULL; post_event.req = &coll_task->super; status = ucc_ee_set_event_internal( coll_task->ee, &post_event, &coll_task->ee->event_out_queue); if (ucc_unlikely(status != UCC_OK)) { tl_error( UCC_TASK_LIB(task), "failed to set EE event: %s", ucc_status_string(status)); return status; } } return status; } ucc_status_t ucc_tl_cuda_allgatherv_nvls_finalize(ucc_coll_task_t *task) { ucc_tl_cuda_task_t *tl_task = ucc_derived_of(task, ucc_tl_cuda_task_t); tl_trace(UCC_TASK_LIB(tl_task), "task: %p allgatherv_nvls_finalize", task); ucc_ec_destroy_event( tl_task->allgatherv_nvls.evt_completion, UCC_EE_CUDA_STREAM); ucc_tl_cuda_task_put(tl_task); return UCC_OK; } ucc_status_t ucc_tl_cuda_allgatherv_nvls_init( ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_datatype_t dt = coll_args->args.dst.info_v.datatype; size_t dt_size = ucc_dt_size(dt); ucc_tl_cuda_task_t *task; ucc_status_t status; size_t total_count_bytes; size_t offset_elements; size_t count_elements; size_t offset_bytes; size_t count_bytes; ucc_rank_t i; status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } status = ucc_ec_create_event( &task->allgatherv_nvls.evt_completion, UCC_EE_CUDA_STREAM); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TL_TEAM_LIB(team), "failed to create CUDA event"); ucc_tl_cuda_task_put(task); return status; } /* Get offset and count in datatype elements, then convert to bytes and * then uint32_t units. Datatype agnostic - we just copy raw bytes. * NVLS requires 16-byte alignment (4 uint32_t = 16 bytes). */ offset_elements = ucc_tl_cuda_allgatherv_get_offset(task, trank); count_elements = ucc_tl_cuda_allgatherv_get_count(task, trank); offset_bytes = offset_elements * dt_size; count_bytes = count_elements * dt_size; /* Calculate total count in bytes */ total_count_bytes = 0; for (i = 0; i < tsize; i++) { total_count_bytes += ucc_tl_cuda_allgatherv_get_count(task, i) * dt_size; } /* Validate total size fits within NVLS symmetric buffer */ if (ucc_unlikely( total_count_bytes > UCC_TL_CUDA_TEAM_LIB(team)->cfg.nvls_symmetric_size)) { tl_debug( UCC_TL_TEAM_LIB(team), "NVLS allgatherv total size %zu bytes exceeds symmetric buffer " "size %zu bytes", total_count_bytes, UCC_TL_CUDA_TEAM_LIB(team)->cfg.nvls_symmetric_size); goto err_cleanup; } /* Convert bytes to uint32_t units for the kernel */ task->allgatherv_nvls.offset = offset_bytes / sizeof(uint32_t); task->allgatherv_nvls.count = count_bytes / sizeof(uint32_t); task->allgatherv_nvls.total_count = total_count_bytes / sizeof(uint32_t); /* NVLS requires 16-byte alignment (4 uint32_t elements) */ if (ucc_unlikely(task->allgatherv_nvls.offset % 4 != 0)) { tl_debug( UCC_TL_TEAM_LIB(team), "NVLS requires 16-byte alignment for offset, got offset=%zu bytes " "(not aligned to 16 bytes). offset_elements=%zu dt_size=%zu", offset_bytes, offset_elements, dt_size); goto err_cleanup; } if (ucc_unlikely(task->allgatherv_nvls.count % 4 != 0)) { tl_debug( UCC_TL_TEAM_LIB(team), "NVLS requires 16-byte alignment for count, got count=%zu bytes " "(not aligned to 16 bytes). count_elements=%zu dt_size=%zu", count_bytes, count_elements, dt_size); goto err_cleanup; } task->allgatherv_nvls.mc_va = (CUdeviceptr)TASK_SYMMETRIC_MC(task); task->allgatherv_nvls.uc_va = (CUdeviceptr)TASK_SYMMETRIC_UC(task); task->allgatherv_nvls.coll_id = team->nvls.coll_ids[task->coll_id]++; task->super.post = ucc_tl_cuda_allgatherv_nvls_start; task->super.triggered_post = ucc_tl_cuda_allgatherv_nvls_triggered_post; task->super.progress = ucc_tl_cuda_allgatherv_nvls_progress; task->super.finalize = ucc_tl_cuda_allgatherv_nvls_finalize; *task_p = &task->super; return UCC_OK; err_cleanup: ucc_ec_destroy_event( task->allgatherv_nvls.evt_completion, UCC_EE_CUDA_STREAM); ucc_tl_cuda_task_put(task); return UCC_ERR_NOT_SUPPORTED; } ucc-1.8.0/src/components/tl/cuda/allgatherv/allgatherv.c0000664000175000017500000000441715211535620023421 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allgatherv.h" #include "components/mc/ucc_mc.h" ucc_base_coll_alg_info_t ucc_tl_cuda_allgatherv_algs[UCC_TL_CUDA_ALLGATHERV_ALG_LAST + 1] = { [UCC_TL_CUDA_ALLGATHERV_ALG_AUTO] = {.id = UCC_TL_CUDA_ALLGATHERV_ALG_AUTO, .name = "auto", .desc = "choose allgatherv algorithm based on CUDA topology"}, [UCC_TL_CUDA_ALLGATHERV_ALG_RING] = {.id = UCC_TL_CUDA_ALLGATHERV_ALG_RING, .name = "ring", .desc = "multiring allgatherv algorithm"}, [UCC_TL_CUDA_ALLGATHERV_ALG_LINEAR] = {.id = UCC_TL_CUDA_ALLGATHERV_ALG_LINEAR, .name = "linear", .desc = "linear allgatherv algorithm"}, #ifdef HAVE_NVLS [UCC_TL_CUDA_ALLGATHERV_ALG_NVLS] = {.id = UCC_TL_CUDA_ALLGATHERV_ALG_NVLS, .name = "nvls", .desc = "NVLS allgatherv algorithm"}, #endif /* HAVE_NVLS */ [UCC_TL_CUDA_ALLGATHERV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; size_t ucc_tl_cuda_allgatherv_get_count(const ucc_tl_cuda_task_t *task, ucc_rank_t block) { const ucc_coll_args_t *args = &TASK_ARGS(task); return ucc_coll_args_get_count(args, args->dst.info_v.counts, block); } size_t ucc_tl_cuda_allgatherv_get_offset(const ucc_tl_cuda_task_t *task, ucc_rank_t block) { const ucc_coll_args_t *args = &TASK_ARGS(task); return ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, block); } ucc_status_t ucc_tl_cuda_allgatherv_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); if (ucc_tl_cuda_team_topo_is_fully_connected(team->topo)) { return ucc_tl_cuda_allgatherv_linear_init(coll_args, tl_team, task_p); } else { return ucc_tl_cuda_allgatherv_ring_init(coll_args, tl_team, task_p); } } ucc-1.8.0/src/components/tl/cuda/allgatherv/allgatherv_linear.c0000664000175000017500000004133415211535620024752 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allgatherv.h" #include "components/ec/ucc_ec.h" #include "tl_cuda_cache.h" #include "utils/arch/cpu.h" #include "utils/arch/cuda_def.h" /* * fragmented buffered copy linear allgatherv algorithm * * Description: * Definitions: * blockI - full send buffer at Rank I * fragI_J - fragment of send buffer at Rank I and step J * NS - number of steps * NF - number of fragments * N - team size * * Setup: * max_frag_size = ucc_min(ucc_max(block1, block2, ..., block N), * scratch_size / 2 / N) * NF = ucc_max(block1, block2, ..., block N) / max_frag_size * NS = 1 + NF * * Algorithm * for rank R * step 0: copy fragR_1 to remote scratch buffers for all ranks * if not inplace copy local src buffer to local dst buffer * * step 1: copy frag1_1, frag2_1, ..., fragN_1 from local scratch buffer * to local dst buffer * copy fragR_2 from local dst buffer to remote scratch buffers * for all ranks * * step NS-1: copy frag1_(NS-2), frag2_(NS-2), ..., fragN_(NS-2) from local * scratch buffer to local dst buffer * copy fragR_NS from local dst buffer to remote scratch buffers * for all ranks * * step NS: copy frag1_(NS-1), frag2_(NS-1), ..., fragN_(NS-1) from local * scratch buffer to local dst buffer */ enum { STAGE_SYNC, /*< Wait for free SYNC segment */ STAGE_SETUP, /*< Wait for memhandle setup to finish */ STAGE_COPIES, /*< Linear algorithm is running */ STAGE_BARRIER, /*< Linear algorithm is done, waiting for * other ranks to finish */ }; ucc_status_t ucc_tl_cuda_allgatherv_linear_finalize(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); tl_trace(UCC_TASK_LIB(task), "finalizing task %p", task); ucc_tl_cuda_task_put(task); return UCC_OK; } ucc_status_t ucc_tl_cuda_allgatherv_linear_setup_start(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_status_t status; set_rank_step(task, trank, 0, 0); ucc_memory_cpu_store_fence(); status = ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), task->bar); if (ucc_unlikely(status != UCC_OK)) { goto exit_err; } return UCC_OK; exit_err: return status; } ucc_status_t ucc_tl_cuda_allgatherv_linear_setup_test(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); return ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); } static inline size_t get_scratch_size(ucc_tl_cuda_team_t *team, ucc_datatype_t dt) { size_t dt_size = ucc_dt_size(dt); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); size_t div; ucc_assert((dt_size > 0) && (tsize > 0)); div = 2 * dt_size * tsize; return (UCC_TL_CUDA_TEAM_LIB(team)->cfg.scratch_size / div) * div; } static inline size_t get_scratch_offset(ucc_tl_cuda_team_t *team, ucc_datatype_t dt, int block) { size_t step_ssize = get_scratch_size(team, dt) / 2; ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); return ucc_buffer_block_offset(step_ssize, tsize, block); } static inline ucc_status_t ecopy(void *dst, void *src, size_t size, ucc_ee_executor_t * exec, ucc_ee_executor_task_t **etask) { ucc_ee_executor_task_args_t exec_args = {0}; exec_args.task_type = UCC_EE_EXECUTOR_TASK_COPY; exec_args.copy.dst = dst; exec_args.copy.src = src; exec_args.copy.len = size; return ucc_ee_executor_task_post(exec, &exec_args, etask); } ucc_status_t ucc_tl_cuda_allgatherv_linear_progress_frag(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t * team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_coll_args_t * args = &TASK_ARGS(task); ucc_datatype_t dt = task->allgatherv_linear.dt; size_t dt_size = ucc_dt_size(dt); size_t ssize = get_scratch_size(team, dt); int nfrags = task->allgatherv_linear.num_frags; int num_steps = nfrags + 1; ucc_ee_executor_task_t *etask; ucc_ee_executor_t * exec; ucc_status_t st; int step, i; void * sbuf, *dbuf; ucc_rank_t peer; size_t send_size, frag_size, frag_offset, local_offset, remote_offset, scratch_offset, rank_offset; ucc_ee_executor_task_args_t eargs; st = ucc_coll_task_get_executor(&task->super, &exec); if (ucc_unlikely(st != UCC_OK)) { return st; } step = get_rank_step(task, trank, 0); while (step < num_steps) { if ((task->allgatherv_linear.exec_task[0] != NULL) || (task->allgatherv_linear.exec_task[1] != NULL)) { for (i = 0; i < 2; i++) { etask = task->allgatherv_linear.exec_task[i]; if (etask != NULL) { st = ucc_ee_executor_task_test(etask); if (st == UCC_OK) { ucc_ee_executor_task_finalize(etask); task->allgatherv_linear.exec_task[i] = NULL; } else { if (ucc_likely(st > 0)) { return UCC_INPROGRESS; } return st; } } } step++; set_rank_step(task, trank, step, 0); continue; } for (i = 0; i < tsize; i++) { if (get_rank_step(task, i, 0) < step) { return UCC_INPROGRESS; } } if (step % 2) { remote_offset = ssize / 2; local_offset = 0; } else { remote_offset = 0; local_offset = ssize / 2; } if (step == 0) { send_size = task->allgatherv_linear.get_count(task, trank); frag_size = ucc_buffer_block_count(send_size, nfrags, step) * dt_size; frag_offset = ucc_buffer_block_offset(send_size, nfrags, step) * dt_size; if (UCC_IS_INPLACE(*args)) { rank_offset = task->allgatherv_linear.get_offset(task, trank) * dt_size; sbuf = PTR_OFFSET(task->allgatherv_linear.rbuf, frag_offset + rank_offset); } else { sbuf = PTR_OFFSET(task->allgatherv_linear.sbuf, frag_offset); } eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY_MULTI; for (i = 0; i < tsize - 1; i++) { peer = (trank + i + 1) % tsize; scratch_offset = get_scratch_offset(team, dt, trank); dbuf = PTR_OFFSET(TASK_SCRATCH(task, peer), remote_offset + scratch_offset); eargs.copy_multi.src[i] = sbuf; eargs.copy_multi.dst[i] = dbuf; eargs.copy_multi.counts[i] = frag_size; } eargs.copy_multi.num_vectors = tsize - 1; st = ucc_ee_executor_task_post(exec, &eargs, &task->allgatherv_linear.exec_task[0]); if (ucc_unlikely(st != UCC_OK)) { return st; } if (!UCC_IS_INPLACE(*args)) { rank_offset = task->allgatherv_linear.get_offset(task, trank) * dt_size; dbuf = PTR_OFFSET(task->allgatherv_linear.rbuf, rank_offset); st = ecopy(dbuf, sbuf, task->allgatherv_linear.get_count(task, trank) * dt_size, exec, &task->allgatherv_linear.exec_task[1]); if (ucc_unlikely(st != UCC_OK)) { return st; } } } else if (step == (num_steps - 1)) { eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY_MULTI; for (i = 0; i < tsize - 1; i++) { peer = (trank + i + 1) % tsize; scratch_offset = get_scratch_offset(team, dt, peer); rank_offset = task->allgatherv_linear.get_offset(task, peer) * dt_size; send_size = task->allgatherv_linear.get_count(task, peer); frag_offset = ucc_buffer_block_offset(send_size, nfrags, step - 1) * dt_size; frag_size = ucc_buffer_block_count(send_size, nfrags, step - 1) * dt_size; sbuf = PTR_OFFSET(TASK_SCRATCH(task, trank), local_offset + scratch_offset); dbuf = PTR_OFFSET(task->allgatherv_linear.rbuf, rank_offset + frag_offset); eargs.copy_multi.src[i] = sbuf; eargs.copy_multi.dst[i] = dbuf; eargs.copy_multi.counts[i] = frag_size; } eargs.copy_multi.num_vectors = tsize - 1; st = ucc_ee_executor_task_post(exec, &eargs, &task->allgatherv_linear.exec_task[0]); if (ucc_unlikely(st != UCC_OK)) { return st; } } else { send_size = task->allgatherv_linear.get_count(task, trank); frag_size = ucc_buffer_block_count(send_size, nfrags, step) * dt_size; frag_offset = ucc_buffer_block_offset(send_size, nfrags, step) * dt_size; rank_offset = task->allgatherv_linear.get_offset(task, trank) * dt_size; sbuf = PTR_OFFSET(task->allgatherv_linear.rbuf, rank_offset + frag_offset); scratch_offset = get_scratch_offset(team, dt, trank); eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY_MULTI; for (i = 0; i < tsize - 1; i++) { peer = (trank + i + 1) % tsize; dbuf = PTR_OFFSET(TASK_SCRATCH(task, peer), remote_offset + scratch_offset); eargs.copy_multi.src[i] = sbuf; eargs.copy_multi.dst[i] = dbuf; eargs.copy_multi.counts[i] = frag_size; } eargs.copy_multi.num_vectors = tsize - 1; st = ucc_ee_executor_task_post(exec, &eargs, &task->allgatherv_linear.exec_task[0]); if (ucc_unlikely(st != UCC_OK)) { return st; } eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY_MULTI; for (i = 0; i < tsize - 1; i++) { peer = (trank + i + 1) % tsize; scratch_offset = get_scratch_offset(team, dt, peer); rank_offset = task->allgatherv_linear.get_offset(task, peer) * dt_size; send_size = task->allgatherv_linear.get_count(task, peer); frag_offset = ucc_buffer_block_offset(send_size, nfrags, step - 1) * dt_size; frag_size = ucc_buffer_block_count(send_size, nfrags, step - 1) * dt_size; sbuf = PTR_OFFSET(TASK_SCRATCH(task, trank), local_offset + scratch_offset); dbuf = PTR_OFFSET(task->allgatherv_linear.rbuf, rank_offset + frag_offset); eargs.copy_multi.src[i] = sbuf; eargs.copy_multi.dst[i] = dbuf; eargs.copy_multi.counts[i] = frag_size; } eargs.copy_multi.num_vectors = tsize - 1; st = ucc_ee_executor_task_post(exec, &eargs, &task->allgatherv_linear.exec_task[1]); if (ucc_unlikely(st != UCC_OK)) { return st; } } } return UCC_OK; } void ucc_tl_cuda_allgatherv_linear_progress(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_status_t st; task->super.status = UCC_INPROGRESS; switch (task->allgatherv_linear.stage) { case STAGE_SYNC: if (ucc_tl_cuda_get_sync(task) != UCC_OK) { task->super.status = UCC_INPROGRESS; return; } st = ucc_tl_cuda_allgatherv_linear_setup_start(task); if (st != UCC_OK) { task->super.status = st; return; } task->allgatherv_linear.stage = STAGE_SETUP; case STAGE_SETUP: st = ucc_tl_cuda_allgatherv_linear_setup_test(task); if (st != UCC_OK) { task->super.status = st; return; } task->allgatherv_linear.stage = STAGE_COPIES; case STAGE_COPIES: st = ucc_tl_cuda_allgatherv_linear_progress_frag(task); if (st != UCC_OK) { task->super.status = st; return; } st = ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), task->bar); if (ucc_unlikely(st != UCC_OK)) { task->super.status = st; return; } task->allgatherv_linear.stage = STAGE_BARRIER; default: ucc_assert(task->allgatherv_linear.stage == STAGE_BARRIER); break; } task->super.status = ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); if (task->super.status == UCC_OK) { ucc_tl_cuda_put_sync(task); } } ucc_status_t ucc_tl_cuda_allgatherv_linear_start(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_coll_args_t * args = &TASK_ARGS(task); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_datatype_t dt = task->allgatherv_linear.dt; ucc_rank_t i; size_t send_size, frag_size, ssize; task->allgatherv_linear.stage = STAGE_SYNC; task->allgatherv_linear.sbuf = args->src.info.buffer; if (args->coll_type == UCC_COLL_TYPE_ALLGATHERV) { task->allgatherv_linear.rbuf = args->dst.info_v.buffer; } else { task->allgatherv_linear.rbuf = args->dst.info.buffer; } send_size = task->allgatherv_linear.get_count(task, 0); for (i = 1; i < tsize; i++) { send_size = ucc_max(send_size, task->allgatherv_linear.get_count(task, i)); } if (send_size == 0) { task->super.status = UCC_OK; return ucc_task_complete(&task->super); } ssize = get_scratch_size(team, dt); frag_size = ucc_min(ssize / 2 / ucc_dt_size(dt) / tsize, send_size); task->allgatherv_linear.num_frags = ucc_div_round_up(send_size, frag_size); task->allgatherv_linear.exec_task[0] = NULL; task->allgatherv_linear.exec_task[1] = NULL; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_cuda_allgatherv_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_task_t *task; ucc_status_t status; if (ucc_unlikely(!ucc_tl_cuda_team_topo_is_fully_connected(team->topo) || UCC_TL_TEAM_SIZE(team) - 1 > UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS)) { return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->allgatherv_linear.get_count = ucc_tl_cuda_allgatherv_get_count; task->allgatherv_linear.get_offset = ucc_tl_cuda_allgatherv_get_offset; task->allgatherv_linear.dt = coll_args->args.dst.info_v.datatype; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_cuda_allgatherv_linear_start; task->super.progress = ucc_tl_cuda_allgatherv_linear_progress; task->super.finalize = ucc_tl_cuda_allgatherv_linear_finalize; task->bar = TASK_BAR(task); *task_p = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/allgatherv/allgatherv_ring.c0000664000175000017500000004327215211535620024442 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allgatherv.h" #include "components/ec/ucc_ec.h" #include "tl_cuda_cache.h" #include "tl_cuda_ring.h" #include "utils/arch/cpu.h" #include "utils/arch/cuda_def.h" ucc_status_t ucc_tl_cuda_allgatherv_ring_finalize(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); tl_trace(UCC_TASK_LIB(task), "finalizing task %p", task); ucc_tl_cuda_task_put(task); return UCC_OK; } ucc_status_t ucc_tl_cuda_allgatherv_ring_setup_start(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_status_t status; int chunk; for (chunk = 0; chunk < task->allgatherv_ring.num_chunks; chunk++) { set_rank_step(task, trank, 0, chunk); } ucc_memory_cpu_store_fence(); status = ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), task->bar); if (ucc_unlikely(status != UCC_OK)) { goto exit_err; } return UCC_OK; exit_err: return status; } ucc_status_t ucc_tl_cuda_allgatherv_ring_setup_test(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); return ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); } static inline void ucc_tl_cuda_allgatherv_ring_size_offset(ucc_tl_cuda_task_t *task, int block, int frag, int chunk, int ring, size_t *ring_frag_size, size_t *block_offset, size_t *frag_offset, size_t *chunk_offset, size_t *ring_frag_offset) { ucc_datatype_t dt_size = ucc_dt_size(task->allgatherv_ring.dt); int nrings = task->allgatherv_ring.num_rings; int nfrags = task->allgatherv_ring.num_frags; int nchunks = task->allgatherv_ring.num_chunks; size_t block_size, frag_size, chunk_size; block_size = task->allgatherv_ring.get_count(task, block) * dt_size; frag_size = ucc_buffer_block_count_aligned(block_size, nfrags, frag, 64); chunk_size = ucc_buffer_block_count_aligned(frag_size, nchunks, chunk, 64); *ring_frag_size = ucc_buffer_block_count_aligned(chunk_size, nrings, ring, 64); *block_offset = task->allgatherv_ring.get_offset(task, block) * dt_size; *frag_offset = ucc_buffer_block_offset_aligned(block_size, nfrags, frag, 64); *chunk_offset = ucc_buffer_block_offset_aligned(frag_size, nchunks, chunk, 64); *ring_frag_offset = ucc_buffer_block_offset_aligned(chunk_size, nrings, ring, 64); } static inline int ucc_tl_cuda_check_peers_ready(ucc_tl_cuda_task_t *task, int chunk, int step) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); int nsteps = (tsize == 2) ? tsize: tsize - 1; int nrings = task->allgatherv_ring.num_rings; int ring, l, ll, r, rr, substep; ucc_rank_t peer; substep = step % nsteps; for (ring = 0; ring < nrings; ring++) { peer = get_send_to(team, trank, tsize, ring); r = get_rank_step(task, peer, chunk); peer = get_send_to(team, peer, tsize, ring); rr = get_rank_step(task, peer, chunk); peer = get_recv_from(team, trank, tsize, ring); l = get_rank_step(task, peer, chunk); peer = get_recv_from(team, peer, tsize, ring); ll = get_rank_step(task, peer, chunk); if (substep == 0) { if ((l < step) || (r < step) || (rr < step)) { return 0; } } else if (substep == nsteps - 1) { if ((l < step) || (r < step) || (ll < step)) { return 0; } } else { if ((l < step) || (r < step)) { return 0; } } } return 1; } static inline size_t ucc_tl_cuda_allgatherv_ring_scratch_offset(ucc_tl_cuda_task_t *task, int chunk, int ring) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); int nrings = task->allgatherv_ring.num_rings; int nchunks = task->allgatherv_ring.num_chunks; size_t ssize = get_scratch_size(team, nrings, nchunks, 1); size_t chunk_size, chunk_offset; chunk_size = ucc_buffer_block_count(ssize/2, nchunks, chunk); chunk_offset = ucc_buffer_block_offset(ssize/2, nchunks, chunk); return ucc_buffer_block_offset(chunk_size, nrings, ring) + chunk_offset; } ucc_status_t ucc_tl_cuda_allgatherv_ring_progress_ring(ucc_tl_cuda_task_t *task, int chunk) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); int tsize = (int)UCC_TL_TEAM_SIZE(team); void *rbuf = task->allgatherv_ring.rbuf; int nsteps = (tsize == 2) ? tsize: tsize - 1; int nrings = task->allgatherv_ring.num_rings; int nchunks = task->allgatherv_ring.num_chunks; size_t ssize = get_scratch_size(team, nrings, nchunks, 1); ucc_ee_executor_t *exec; ucc_ee_executor_task_args_t eargs_loc, eargs_rem; ucc_ee_executor_task_t *etask; void *dbuf1, *dbuf2, *sbuf1, *sbuf2; int step, frag, frag_step, i, ring; ucc_rank_t peer_block, sendto, recvfrom; ucc_status_t st; size_t remote_offset, local_offset, frag_offset, block_offset, ring_frag_offset, chunk_offset, ring_frag_size, frag_count1, frag_count2; size_t ring_scratch_offset; st = ucc_coll_task_get_executor(&task->super, &exec); if (ucc_unlikely(st != UCC_OK)) { return st; } step = get_rank_step(task, trank, chunk); while (step < (nsteps * task->allgatherv_ring.num_frags)) { if ((task->allgatherv_ring.exec_task[2*chunk] != NULL) || (task->allgatherv_ring.exec_task[2*chunk + 1] != NULL)) { for (i = 1; i >= 0; i--) { etask = task->allgatherv_ring.exec_task[2*chunk + i]; if (etask != NULL) { st = ucc_ee_executor_task_test(etask); if (st == UCC_OK) { ucc_ee_executor_task_finalize(etask); task->allgatherv_ring.exec_task[2*chunk + i] = NULL; } else { if (ucc_likely(st > 0)) { return UCC_INPROGRESS; } return st; } } } step++; set_rank_step(task, trank, step, chunk); continue; } frag = step / nsteps; frag_step = step % nsteps; if (!ucc_tl_cuda_check_peers_ready(task, chunk, step)) { return UCC_INPROGRESS; } if (step % 2) { remote_offset = ssize / 2; local_offset = 0; } else { remote_offset = 0; local_offset = ssize / 2; } eargs_loc.copy_multi.num_vectors = 0; eargs_rem.copy_multi.num_vectors = 0; for (ring = 0; ring < nrings; ring++) { ring_scratch_offset = ucc_tl_cuda_allgatherv_ring_scratch_offset(task, chunk, ring); sendto = get_send_to(team, trank, tsize, ring); recvfrom = get_recv_from(team, trank, tsize, ring); if (frag_step == nsteps - 1) { if (tsize != 2) { peer_block = get_recv_block(team, trank, tsize, frag_step, ring); ucc_tl_cuda_allgatherv_ring_size_offset(task, peer_block, frag, chunk, ring, &ring_frag_size, &block_offset, &frag_offset, &chunk_offset, &ring_frag_offset); sbuf1 = PTR_OFFSET(TASK_SCRATCH(task, recvfrom), local_offset + ring_scratch_offset); dbuf1 = PTR_OFFSET(rbuf, block_offset + chunk_offset + frag_offset + ring_frag_offset); frag_count1 = ring_frag_size; } else { sbuf1 = NULL; dbuf1 = NULL; } peer_block = get_send_block(team, trank, tsize, frag_step, ring); ucc_tl_cuda_allgatherv_ring_size_offset(task, peer_block, frag, chunk, ring, &ring_frag_size, &block_offset, &frag_offset, &chunk_offset, &ring_frag_offset); sbuf2 = PTR_OFFSET(TASK_SCRATCH(task, trank), local_offset + ring_scratch_offset); dbuf2 = PTR_OFFSET(rbuf, block_offset + frag_offset + chunk_offset + ring_frag_offset); frag_count2 = ring_frag_size; } else { peer_block = get_send_block(team, trank, tsize, frag_step, ring); ucc_tl_cuda_allgatherv_ring_size_offset(task, peer_block, frag, chunk, ring, &ring_frag_size, &block_offset, &frag_offset, &chunk_offset, &ring_frag_offset); if (frag_step == 0) { if (UCC_IS_INPLACE(*args)) { sbuf1 = PTR_OFFSET(rbuf, block_offset + frag_offset + ring_frag_offset + chunk_offset); sbuf2 = NULL; dbuf2 = NULL; } else { sbuf1 = PTR_OFFSET(task->allgatherv_ring.sbuf, frag_offset + ring_frag_offset + chunk_offset); sbuf2 = sbuf1; dbuf2 = PTR_OFFSET(rbuf, block_offset + frag_offset + ring_frag_offset + chunk_offset); } dbuf1 = PTR_OFFSET(TASK_SCRATCH(task, sendto), remote_offset + ring_scratch_offset); } else { sbuf1 = PTR_OFFSET(TASK_SCRATCH(task, trank), local_offset + ring_scratch_offset); sbuf2 = sbuf1; dbuf1 = PTR_OFFSET(TASK_SCRATCH(task, sendto), remote_offset + ring_scratch_offset); dbuf2 = PTR_OFFSET(rbuf, block_offset + frag_offset + ring_frag_offset + chunk_offset); } frag_count1 = ring_frag_size; frag_count2 = ring_frag_size; } if (dbuf1 != NULL) { eargs_rem.task_type = UCC_EE_EXECUTOR_TASK_COPY_MULTI; eargs_rem.copy_multi.src[ring] = sbuf1; eargs_rem.copy_multi.dst[ring] = dbuf1; eargs_rem.copy_multi.counts[ring] = frag_count1; eargs_rem.copy_multi.num_vectors++; } if (dbuf2 != NULL) { eargs_loc.task_type = UCC_EE_EXECUTOR_TASK_COPY_MULTI; eargs_loc.copy_multi.src[ring] = sbuf2; eargs_loc.copy_multi.dst[ring] = dbuf2; eargs_loc.copy_multi.counts[ring] = frag_count2; eargs_loc.copy_multi.num_vectors++; } } if (eargs_rem.copy_multi.num_vectors != 0) { st = ucc_ee_executor_task_post(exec, &eargs_rem, &task->allgatherv_ring.exec_task[2*chunk]); if (ucc_unlikely(st != UCC_OK)) { return st; } } if (eargs_loc.copy_multi.num_vectors != 0) { st = ucc_ee_executor_task_post(exec, &eargs_loc, &task->allgatherv_ring.exec_task[2*chunk + 1]); if (ucc_unlikely(st != UCC_OK)) { return st; } } } return UCC_OK; } void ucc_tl_cuda_allgatherv_ring_progress(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_status_t st; int chunk, num_done; task->super.status = UCC_INPROGRESS; switch (task->allgatherv_ring.stage) { case RING_STAGE_SYNC: if (ucc_tl_cuda_get_sync(task) != UCC_OK) { task->super.status = UCC_INPROGRESS; return; } st = ucc_tl_cuda_allgatherv_ring_setup_start(task); if (st != UCC_OK) { task->super.status = st; return; } task->allgatherv_ring.stage = RING_STAGE_SETUP; /* fall through */ case RING_STAGE_SETUP: st = ucc_tl_cuda_allgatherv_ring_setup_test(task); if (st != UCC_OK) { task->super.status = st; return; } task->allgatherv_ring.stage = RING_STAGE_RING; /* fall through */ case RING_STAGE_RING: num_done = 0; for (chunk = 0; chunk < task->allgatherv_ring.num_chunks; chunk++) { st = ucc_tl_cuda_allgatherv_ring_progress_ring(task, chunk); if (ucc_unlikely(st < 0)) { task->super.status = st; return; } else if (st == UCC_OK) { num_done++; } } if (num_done != task->allgatherv_ring.num_chunks) { return; } st = ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), task->bar); if (ucc_unlikely(st != UCC_OK)) { task->super.status = st; return; } task->allgatherv_ring.stage = RING_STAGE_BARRIER; /* fall through */ default: ucc_assert(task->allgatherv_ring.stage == RING_STAGE_BARRIER); break; } task->super.status = ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); if (task->super.status == UCC_OK) { ucc_tl_cuda_put_sync(task); UCC_TL_CUDA_PROFILE_REQUEST_EVENT(coll_task, "cuda_agv_ring_done", 0); } } ucc_status_t ucc_tl_cuda_allgatherv_ring_start(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_tl_cuda_lib_t *lib = UCC_TL_CUDA_TEAM_LIB(team); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); int nchunks = lib->cfg.allgather_ring_num_chunks; int nrings; size_t ssize; size_t send_size; size_t frag_size; ucc_rank_t i; UCC_TL_CUDA_PROFILE_REQUEST_EVENT(coll_task, "cuda_agv_ring_start", 0); send_size = task->allgatherv_ring.get_count(task, 0); for (i = 1; i < tsize; i++) { send_size = ucc_max(send_size, task->allgatherv_ring.get_count(task, i)); } send_size = ucc_dt_size(task->allgatherv_ring.dt) * send_size; if (send_size == 0) { task->super.status = UCC_OK; return ucc_task_complete(&task->super); } nrings = get_num_rings(team, send_size, lib->cfg.allgather_ring_max_rings); task->allgatherv_ring.sbuf = args->src.info.buffer; task->allgatherv_ring.num_rings = nrings; task->allgatherv_ring.num_chunks = nchunks; if (args->coll_type == UCC_COLL_TYPE_ALLGATHERV) { task->allgatherv_ring.rbuf = args->dst.info_v.buffer; } else { task->allgatherv_ring.rbuf = args->dst.info.buffer; } ssize = get_scratch_size(team, nrings, nchunks, 1); frag_size = ucc_min(ssize / 2, send_size); task->allgatherv_ring.num_frags = ucc_div_round_up(send_size, frag_size); task->allgatherv_ring.stage = RING_STAGE_SYNC; memset(task->allgatherv_ring.exec_task, 0, 2 * task->allgatherv_ring.num_chunks * sizeof(ucc_ee_executor_task_t*)); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_cuda_allgatherv_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_task_t *task; ucc_status_t status; status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->allgatherv_ring.get_count = ucc_tl_cuda_allgatherv_get_count; task->allgatherv_ring.get_offset = ucc_tl_cuda_allgatherv_get_offset; task->allgatherv_ring.dt = coll_args->args.dst.info_v.datatype; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_cuda_allgatherv_ring_start; task->super.progress = ucc_tl_cuda_allgatherv_ring_progress; task->super.finalize = ucc_tl_cuda_allgatherv_ring_finalize; task->bar = TASK_BAR(task); *task_p = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/tl_cuda_topo.c0000664000175000017500000003267415211535620021621 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_cuda_topo.h" #include "utils/arch/cuda_def.h" #include #include #include pthread_mutex_t nvml_lock = PTHREAD_MUTEX_INITIALIZER; #define MAX_PCI_DEVICES 32 #define NVMLCHECK_GOTO(_cmd, _label, _status, _lib) \ do { \ nvmlReturn_t e = _cmd; \ if (ucc_unlikely(NVML_SUCCESS != e)) { \ tl_error(_lib, "NVML error %d %s", e, nvmlErrorString(e)); \ _status = UCC_ERR_NO_MESSAGE; \ goto _label; \ } \ } while (0) static ucc_status_t ucc_tl_cuda_topo_pci_id_from_str(const char * bus_id_str, ucc_tl_cuda_device_pci_id_t *pci_id) { int n; n = sscanf(bus_id_str, "%hx:%hhx:%hhx.%hhx", &pci_id->domain, &pci_id->bus, &pci_id->device, &pci_id->function); if (n != 4) { return UCC_ERR_INVALID_PARAM; } return UCC_OK; } // TODO: add to topo print void ucc_tl_cuda_topo_pci_id_to_str(const ucc_tl_cuda_device_pci_id_t *pci_id, char *str, size_t max) { ucc_snprintf_safe(str, max, "%04x:%02x:%02x.%d", pci_id->domain, pci_id->bus, pci_id->device, pci_id->function); } ucc_status_t ucc_tl_cuda_topo_get_pci_id(int device, ucc_tl_cuda_device_pci_id_t *pci_id) { char pci_bus_id[MAX_PCI_BUS_ID_STR]; ucc_status_t st; CUDA_CHECK_GOTO(cudaDeviceGetPCIBusId(pci_bus_id, MAX_PCI_BUS_ID_STR, device), exit, st); st = ucc_tl_cuda_topo_pci_id_from_str(pci_bus_id, pci_id); exit: return st; } static uint64_t ucc_tl_cuda_device_pci_id_to_uint64(const ucc_tl_cuda_device_pci_id_t *id) { return (((uint64_t)id->domain << 24) | ((uint64_t)id->bus << 16) | ((uint64_t)id->device << 8) | ((uint64_t)id->function)); } static ucc_status_t ucc_tl_cuda_topo_graph_find_by_id(const ucc_tl_cuda_topo_t *topo, const ucc_tl_cuda_device_pci_id_t *dev_id, ucc_tl_cuda_topo_node_t **node) { uint64_t id; khiter_t iter; id = ucc_tl_cuda_device_pci_id_to_uint64(dev_id); iter = kh_get(bus_to_node, &topo->bus_to_node_hash, id); if (iter == kh_end(&topo->bus_to_node_hash)) { return UCC_ERR_NOT_FOUND; } *node = &topo->graph[kh_value(&topo->bus_to_node_hash, iter)]; return UCC_OK; } static ucc_status_t ucc_tl_cuda_topo_graph_add_device(const ucc_tl_cuda_device_pci_id_t *dev_id, ucc_tl_cuda_topo_dev_type_t dev_type, ucc_tl_cuda_topo_t *topo, ucc_tl_cuda_topo_node_t **node) { uint64_t key; khiter_t iter; int ret; int n; char dev_id_str[MAX_PCI_BUS_ID_STR]; key = ucc_tl_cuda_device_pci_id_to_uint64(dev_id); iter = kh_put(bus_to_node, &topo->bus_to_node_hash, key, &ret); if (ret < 0) { ucc_tl_cuda_topo_pci_id_to_str(dev_id, dev_id_str, MAX_PCI_BUS_ID_STR); tl_error(topo->lib, "failed to add device id %s key %" PRIu64 " to hash", dev_id_str, key); return UCC_ERR_NO_MESSAGE; } else if (ret == 0) { /* device already exists */ *node = &topo->graph[kh_value(&topo->bus_to_node_hash, iter)]; ucc_assert((*node)->type == dev_type); return UCC_OK; } else { n = topo->num_nodes; topo->num_nodes++; kh_value(&topo->bus_to_node_hash, iter) = n; topo->graph[n].pci_id = *dev_id; topo->graph[n].type = dev_type; ucc_list_head_init(&topo->graph[n].link.list_link); *node = &topo->graph[n]; } return UCC_OK; } static ucc_status_t ucc_tl_cuda_topo_graph_add_link(ucc_tl_cuda_topo_t *topo, ucc_tl_cuda_topo_node_t *src, ucc_tl_cuda_topo_node_t *dst) { ucc_tl_cuda_topo_link_t *link; ucc_list_for_each(link, &src->link.list_link, list_link) { if (ucc_tl_cuda_topo_device_id_equal(&link->pci_id, &dst->pci_id)) { link->width += 1; return UCC_OK; } } link = (ucc_tl_cuda_topo_link_t*)ucc_malloc(sizeof(*link), "cuda_topo_link"); if (!link) { tl_error(topo->lib, "failed to allocate topo link"); return UCC_ERR_NO_MEMORY; } link->pci_id = dst->pci_id; link->width = 1; ucc_list_add_tail(&src->link.list_link, &link->list_link); return UCC_OK; } static void ucc_tl_cuda_topo_free_link(ucc_tl_cuda_topo_link_t *link) //NOLINT: link is unused {} static void ucc_tl_cuda_topo_graph_destroy(ucc_tl_cuda_topo_t *topo) { int i; for (i = 0; i < topo->num_nodes; i++) { ucc_list_destruct(&topo->graph[i].link.list_link, ucc_tl_cuda_topo_link_t, ucc_tl_cuda_topo_free_link, list_link); } kh_destroy_inplace(bus_to_node, &topo->bus_to_node_hash); free(topo->graph); } static ucc_status_t ucc_tl_cuda_topo_get_remote_dev_type(ucc_tl_cuda_topo_t *topo, nvmlDevice_t dev, int link, ucc_tl_cuda_topo_dev_type_t *dev_type) { #if HAVE_NVML_REMOTE_DEVICE_TYPE ucc_status_t status = UCC_OK; nvmlIntNvLinkDeviceType_t nvml_dt; NVMLCHECK_GOTO(nvmlDeviceGetNvLinkRemoteDeviceType(dev, link, &nvml_dt), exit, status, topo->lib); switch(nvml_dt) { case NVML_NVLINK_DEVICE_TYPE_GPU: *dev_type = UCC_TL_CUDA_TOPO_DEV_TYPE_GPU; break; case NVML_NVLINK_DEVICE_TYPE_SWITCH: *dev_type = UCC_TL_CUDA_TOPO_DEV_TYPE_SWITCH; break; default: *dev_type = UCC_TL_CUDA_TOPO_DEV_TYPE_LAST; break; } exit: return status; #else nvmlPciInfo_t nvml_pci; nvmlDevice_t nvml_dev; nvmlReturn_t nvml_st; nvml_st = nvmlDeviceGetNvLinkRemotePciInfo_v2(dev, link, &nvml_pci); if (nvml_st != NVML_SUCCESS) { *dev_type = UCC_TL_CUDA_TOPO_DEV_TYPE_LAST; return UCC_OK; } nvml_st = nvmlDeviceGetHandleByPciBusId_v2(nvml_pci.busId, &nvml_dev); if (nvml_st == NVML_SUCCESS) { *dev_type = UCC_TL_CUDA_TOPO_DEV_TYPE_GPU; return UCC_OK; } else if (nvml_st == NVML_ERROR_NOT_FOUND) { *dev_type = UCC_TL_CUDA_TOPO_DEV_TYPE_SWITCH; return UCC_OK; } return UCC_ERR_NOT_SUPPORTED; #endif } static ucc_status_t ucc_tl_cuda_topo_graph_create(ucc_tl_cuda_topo_t *topo) { ucc_status_t status = UCC_OK; nvmlDevice_t nvml_dev; nvmlFieldValue_t nvml_value; nvmlPciInfo_t nvml_pci; ucc_tl_cuda_topo_dev_type_t dev_type; ucc_tl_cuda_device_pci_id_t pci_id; ucc_tl_cuda_topo_node_t *node, *peer_node; int num_nvlinks, link, i; unsigned int num_gpus; nvmlReturn_t nvml_st; nvml_st = nvmlInit_v2(); if (nvml_st != NVML_SUCCESS) { tl_debug(topo->lib, "failed to init NVML: %s", nvmlErrorString(nvml_st)); return UCC_ERR_NO_MESSAGE; } nvml_st = nvmlDeviceGetCount(&num_gpus); if (nvml_st != NVML_SUCCESS) { tl_debug(topo->lib, "nvmlDeviceGetCount failed: %s", nvmlErrorString(nvml_st)); status = UCC_ERR_NO_RESOURCE; goto exit_nvml_shutdown; } if (num_gpus == 0) { tl_debug(topo->lib, "no GPU devices found"); status = UCC_ERR_NO_RESOURCE; goto exit_nvml_shutdown; } topo->num_nodes = 0; topo->graph = (ucc_tl_cuda_topo_node_t*)ucc_malloc(MAX_PCI_DEVICES * sizeof(*topo->graph), "tl cuda topo graph"); if (!topo->graph) { tl_error(topo->lib, "failed to allocate tl cuda topo graph"); return UCC_ERR_NO_MEMORY; } kh_init_inplace(bus_to_node, &topo->bus_to_node_hash); pthread_mutex_lock(&nvml_lock); nvml_value.fieldId = NVML_FI_DEV_NVLINK_LINK_COUNT; for (i = 0; i < num_gpus; i++) { NVMLCHECK_GOTO(nvmlDeviceGetHandleByIndex(i, &nvml_dev), exit_free_graph, status, topo->lib); NVMLCHECK_GOTO(nvmlDeviceGetPciInfo(nvml_dev, &nvml_pci), exit_free_graph, status, topo->lib); pci_id.domain = nvml_pci.domain; pci_id.bus = nvml_pci.bus; pci_id.device = nvml_pci.device; pci_id.function = 0; status = ucc_tl_cuda_topo_graph_add_device(&pci_id, UCC_TL_CUDA_TOPO_DEV_TYPE_GPU, topo, &node); if (status != UCC_OK) { goto exit_free_graph; } NVMLCHECK_GOTO(nvmlDeviceGetFieldValues(nvml_dev, 1, &nvml_value), exit_free_graph, status, topo->lib); num_nvlinks = ((nvml_value.nvmlReturn == NVML_SUCCESS) && (nvml_value.valueType == NVML_VALUE_TYPE_UNSIGNED_INT)) ? nvml_value.value.uiVal : 0; for (link = 0; link < num_nvlinks; link++) { status = ucc_tl_cuda_topo_get_remote_dev_type(topo, nvml_dev, link, &dev_type); if (status != UCC_OK) { tl_debug(topo->lib, "failed to get remote device type"); goto exit_free_graph; } if (dev_type == UCC_TL_CUDA_TOPO_DEV_TYPE_LAST) { /* nvlink connected device is not supported by cuda tl */ continue; } NVMLCHECK_GOTO(nvmlDeviceGetNvLinkRemotePciInfo_v2(nvml_dev, link, &nvml_pci), exit_free_graph, status, topo->lib); pci_id.domain = nvml_pci.domain; pci_id.bus = nvml_pci.bus; pci_id.device = nvml_pci.device; pci_id.function = 0; status = ucc_tl_cuda_topo_graph_add_device(&pci_id, dev_type, topo, &peer_node); if (status != UCC_OK) { goto exit_free_graph; } status = ucc_tl_cuda_topo_graph_add_link(topo, node, peer_node); if (status != UCC_OK) { goto exit_free_graph; } } } nvmlShutdown(); pthread_mutex_unlock(&nvml_lock); return UCC_OK; exit_free_graph: ucc_tl_cuda_topo_graph_destroy(topo); pthread_mutex_unlock(&nvml_lock); exit_nvml_shutdown: nvmlShutdown(); return status; } ucc_status_t ucc_tl_cuda_topo_create(const ucc_base_lib_t *lib, ucc_tl_cuda_topo_t **cuda_topo) { ucc_tl_cuda_topo_t *topo; ucc_status_t status; topo = (ucc_tl_cuda_topo_t*)ucc_malloc(sizeof(*topo), "cuda_topo"); if (!topo) { tl_error(lib, "failed to alloc cuda topo"); status = UCC_ERR_NO_MEMORY; goto exit_err; } topo->lib = lib; status = ucc_tl_cuda_topo_graph_create(topo); if (status != UCC_OK) { goto free_topo; } *cuda_topo = topo; return UCC_OK; free_topo: ucc_free(topo); exit_err: return status; } ucc_status_t ucc_tl_cuda_topo_num_links(const ucc_tl_cuda_topo_t *topo, const ucc_tl_cuda_device_pci_id_t *dev1, const ucc_tl_cuda_device_pci_id_t *dev2, ucc_rank_t *num_links) { ucc_status_t status; ucc_tl_cuda_topo_node_t *dev1_node, *dev2_node; ucc_tl_cuda_topo_node_t *link_node; ucc_tl_cuda_topo_link_t *dev1_link, *dev2_link; *num_links = 0; status = ucc_tl_cuda_topo_graph_find_by_id(topo, dev1, &dev1_node); if (status != UCC_OK) { return status; } status = ucc_tl_cuda_topo_graph_find_by_id(topo, dev2, &dev2_node); if (status != UCC_OK) { return status; } ucc_list_for_each(dev1_link, &dev1_node->link.list_link, list_link) { if (ucc_tl_cuda_topo_device_id_equal(&dev1_link->pci_id, dev2)) { *num_links += dev1_link->width; } else { /* check if peer device is switch and it connects given devices */ status = ucc_tl_cuda_topo_graph_find_by_id(topo, &dev1_link->pci_id, &link_node); if (status != UCC_OK) { return status; } if (link_node->type == UCC_TL_CUDA_TOPO_DEV_TYPE_SWITCH) { ucc_list_for_each(dev2_link, &dev2_node->link.list_link, list_link) { if (ucc_tl_cuda_topo_device_id_equal(&dev2_link->pci_id, &dev1_link->pci_id)) { *num_links += ucc_min(dev1_link->width, dev2_link->width); } } } } } return UCC_OK; } ucc_status_t ucc_tl_cuda_topo_destroy(ucc_tl_cuda_topo_t *cuda_topo) { ucc_tl_cuda_topo_graph_destroy(cuda_topo); ucc_free(cuda_topo); return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/tl_cuda_ep_hash.h0000664000175000017500000000420115211535620022235 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_EP_HASH_H_ #define UCC_TL_CUDA_EP_HASH_H_ #include "config.h" #include "core/ucc_context.h" #include "utils/khash.h" #include /* TODO: common code with TL/UCP */ static inline uint32_t tl_cuda_ctx_id_hash_fn_impl(uint32_t h, uint32_t k) { uint32_t H = h & 0xf8000000; h = h << 5; h = h ^ ( H >> 27 ); h = h ^ k; return h; } /* Collisions are handled in khash implementation */ static inline khint32_t tl_cuda_ctx_id_hash_fn(ucc_context_id_t k) { uint32_t h = 0; ucc_assert(sizeof(k.pi.host_hash) == 8); h = tl_cuda_ctx_id_hash_fn_impl(h, kh_int64_hash_func(k.pi.host_hash)); h = tl_cuda_ctx_id_hash_fn_impl(h, k.pi.pid); h = tl_cuda_ctx_id_hash_fn_impl(h, k.seq_num); return (khint32_t)h; } #define tl_cuda_ctx_id_equal_fn(_a, _b) \ (((_a).pi.host_hash == (_b).pi.host_hash) && \ ((_a).pi.pid == (_b).pi.pid) && ((_a).seq_num == (_b).seq_num)) KHASH_INIT(tl_cuda_ep_hash, ucc_context_id_t, void*, 1, \ tl_cuda_ctx_id_hash_fn, tl_cuda_ctx_id_equal_fn); #define tl_cuda_ep_hash_t khash_t(tl_cuda_ep_hash) static inline void* tl_cuda_hash_get(tl_cuda_ep_hash_t *h, ucc_context_id_t key) { khiter_t k; void *value; k = kh_get(tl_cuda_ep_hash, h , key); if (k == kh_end(h)) { return NULL; } value = kh_value(h, k); return value; } static inline void tl_cuda_hash_put(tl_cuda_ep_hash_t *h, ucc_context_id_t key, void *value) { int ret; khiter_t k; k = kh_put(tl_cuda_ep_hash, h, key, &ret); kh_value(h, k) = value; } static inline void* tl_cuda_hash_pop(tl_cuda_ep_hash_t *h) { void *ep = NULL; khiter_t k; k = kh_begin(h); while (k != kh_end(h)) { if (kh_exist(h, k)) { ep = kh_value(h, k); break; } k++; } if (ep) { kh_del(tl_cuda_ep_hash, h, k); } return ep; } #endif ucc-1.8.0/src/components/tl/cuda/tl_cuda_coll.c0000664000175000017500000002324415211535620021562 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_cuda_coll.h" #include "alltoall/alltoall.h" #include "alltoallv/alltoallv.h" #include "allgather/allgather.h" #include "allgatherv/allgatherv.h" #include "bcast/bcast.h" #include "reduce_scatter/reduce_scatter.h" #include "reduce_scatterv/reduce_scatterv.h" #include "allreduce/allreduce.h" #include "utils/arch/cpu.h" #include "utils/arch/cuda_def.h" #if ENABLE_DEBUG == 1 /* TODO: possible need to check CUDA context */ #define UCC_TL_CUDA_CHECK_DEVICE_MATCH(_team) do { \ int _dev; \ CUDA_CHECK(cudaGetDevice(&_dev)); \ if (_dev != UCC_TL_CUDA_TEAM_CTX(_team)->device) { \ tl_error(UCC_TL_TEAM_LIB(_team), "CUDA device mismatch, " \ "current device %d, team device %d\n", _dev, \ UCC_TL_CUDA_TEAM_CTX(_team)->device); \ return UCC_ERR_INVALID_PARAM; \ } \ } while(0) #else #define UCC_TL_CUDA_CHECK_DEVICE_MATCH(_team) #endif const char * ucc_tl_cuda_default_alg_select_str[UCC_TL_CUDA_N_DEFAULT_ALG_SELECT_STR] = { UCC_TL_CUDA_ALLGATHER_DEFAULT_ALG_SELECT_STR, UCC_TL_CUDA_ALLGATHERV_DEFAULT_ALG_SELECT_STR, UCC_TL_CUDA_ALLREDUCE_DEFAULT_ALG_SELECT_STR, UCC_TL_CUDA_BCAST_DEFAULT_ALG_SELECT_STR, UCC_TL_CUDA_REDUCE_SCATTER_DEFAULT_ALG_SELECT_STR, UCC_TL_CUDA_REDUCE_SCATTERV_DEFAULT_ALG_SELECT_STR}; ucc_status_t ucc_tl_cuda_mem_info_get(void *ptr, size_t length, ucc_tl_cuda_mem_info_t *mi) { ucc_mem_attr_t mem_attr; ucc_status_t status; if (ptr == NULL || length == 0) { mi->ptr = NULL; mi->length = 0; mi->offset = 0; return UCC_OK; } mem_attr.field_mask = UCC_MEM_ATTR_FIELD_BASE_ADDRESS | UCC_MEM_ATTR_FIELD_ALLOC_LENGTH; mem_attr.alloc_length = length; status = ucc_mc_get_mem_attr(ptr, &mem_attr); if (ucc_unlikely(status != UCC_OK)) { return status; } mi->ptr = mem_attr.base_address; mi->length = mem_attr.alloc_length; mi->offset = (ptrdiff_t)ptr - (ptrdiff_t)mi->ptr; CUDA_CHECK_GOTO(cudaIpcGetMemHandle(&mi->handle, mi->ptr), exit, status); exit: return status; } ucc_status_t ucc_tl_cuda_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { UCC_TL_CUDA_CHECK_DEVICE_MATCH(ucc_derived_of(team, ucc_tl_cuda_team_t)); switch (coll_args->args.coll_type) { case UCC_COLL_TYPE_ALLTOALL: return ucc_tl_cuda_alltoall_init(coll_args, team, task_h); case UCC_COLL_TYPE_ALLGATHER: return ucc_tl_cuda_allgather_init(coll_args, team, task_h); case UCC_COLL_TYPE_ALLGATHERV: return ucc_tl_cuda_allgatherv_init(coll_args, team, task_h); case UCC_COLL_TYPE_BCAST: return ucc_tl_cuda_bcast_init(coll_args, team, task_h); case UCC_COLL_TYPE_REDUCE_SCATTER: return ucc_tl_cuda_reduce_scatter_init(coll_args, team, task_h); case UCC_COLL_TYPE_REDUCE_SCATTERV: return ucc_tl_cuda_reduce_scatterv_init(coll_args, team, task_h); case UCC_COLL_TYPE_ALLREDUCE: return ucc_tl_cuda_allreduce_init(coll_args, team, task_h); case UCC_COLL_TYPE_ALLTOALLV: return ucc_tl_cuda_alltoallv_init(coll_args, team, task_h); default: return UCC_ERR_NOT_SUPPORTED; } } ucc_status_t ucc_tl_cuda_shm_barrier_init_root(ucc_rank_t size, ucc_rank_t rank, ucc_rank_t root, ucc_tl_cuda_shm_barrier_t *barrier) { if (rank == root) { barrier->size = size; barrier->count = 0; barrier->sense = 0; } barrier->state[rank] = UCC_OK; barrier->local_sense[rank] = 1; return UCC_OK; } ucc_status_t ucc_tl_cuda_shm_barrier_init(ucc_rank_t size, ucc_rank_t rank, ucc_tl_cuda_shm_barrier_t *barrier) { if (rank == 0) { barrier->size = size; barrier->count = 0; barrier->sense = 0; } barrier->state[rank] = UCC_OK; barrier->local_sense[rank] = 1; return UCC_OK; } ucc_status_t ucc_tl_cuda_shm_barrier_start(ucc_rank_t rank, ucc_tl_cuda_shm_barrier_t *barrier) { ucc_rank_t pos = ucc_atomic_fadd32(&barrier->count, 1); barrier->state[rank] = UCC_INPROGRESS; if (pos == barrier->size - 1) { barrier->count = 0; ucc_memory_cpu_store_fence(); barrier->sense = barrier->local_sense[rank]; } return UCC_OK; } ucc_status_t ucc_tl_cuda_shm_barrier_test(ucc_rank_t rank, ucc_tl_cuda_shm_barrier_t *barrier) { if (barrier->sense != barrier->local_sense[rank]) { return UCC_INPROGRESS; } barrier->state[rank] = UCC_OK; barrier->local_sense[rank] = 1 - barrier->local_sense[rank]; return UCC_OK; } static inline int alg_id_from_str(ucc_coll_type_t coll_type, const char *str) { switch (coll_type) { case UCC_COLL_TYPE_ALLGATHER: return ucc_tl_cuda_allgather_alg_from_str(str); case UCC_COLL_TYPE_ALLGATHERV: return ucc_tl_cuda_allgatherv_alg_from_str(str); case UCC_COLL_TYPE_BCAST: return ucc_tl_cuda_bcast_alg_from_str(str); case UCC_COLL_TYPE_ALLREDUCE: return ucc_tl_cuda_allreduce_alg_from_str(str); default: break; } return -1; } ucc_status_t ucc_tl_cuda_alg_id_to_init(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, ucc_base_coll_init_fn_t *init) { ucc_status_t status = UCC_OK; if (alg_id_str) { alg_id = alg_id_from_str(coll_type, alg_id_str); } if (mem_type != UCC_MEMORY_TYPE_CUDA) { return UCC_ERR_NOT_SUPPORTED; } switch (coll_type) { case UCC_COLL_TYPE_ALLGATHER: switch (alg_id) { case UCC_TL_CUDA_ALLGATHER_ALG_AUTO: *init = ucc_tl_cuda_allgather_init; break; case UCC_TL_CUDA_ALLGATHER_ALG_RING: *init = ucc_tl_cuda_allgather_ring_init; break; case UCC_TL_CUDA_ALLGATHER_ALG_LINEAR: *init = ucc_tl_cuda_allgather_linear_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_ALLGATHERV: switch (alg_id) { case UCC_TL_CUDA_ALLGATHERV_ALG_AUTO: *init = ucc_tl_cuda_allgatherv_init; break; case UCC_TL_CUDA_ALLGATHERV_ALG_RING: *init = ucc_tl_cuda_allgatherv_ring_init; break; case UCC_TL_CUDA_ALLGATHERV_ALG_LINEAR: *init = ucc_tl_cuda_allgatherv_linear_init; break; #ifdef HAVE_NVLS case UCC_TL_CUDA_ALLGATHERV_ALG_NVLS: *init = ucc_tl_cuda_allgatherv_nvls_init; break; #endif /* HAVE_NVLS */ default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_BCAST: switch (alg_id) { case UCC_TL_CUDA_BCAST_ALG_LINEAR: *init = ucc_tl_cuda_bcast_linear_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_ALLREDUCE: switch (alg_id) { #ifdef HAVE_NVLS case UCC_TL_CUDA_ALLREDUCE_ALG_NVLS: *init = ucc_tl_cuda_allreduce_nvls_init; break; default: status = UCC_ERR_INVALID_PARAM; break; #else default: status = UCC_ERR_NOT_SUPPORTED; break; #endif /* HAVE_NVLS */ }; break; case UCC_COLL_TYPE_REDUCE_SCATTER: switch (alg_id) { case UCC_TL_CUDA_REDUCE_SCATTER_ALG_AUTO: *init = ucc_tl_cuda_reduce_scatter_init; break; case UCC_TL_CUDA_REDUCE_SCATTER_ALG_RING: *init = ucc_tl_cuda_reduce_scatter_ring_init; break; case UCC_TL_CUDA_REDUCE_SCATTER_ALG_LINEAR: *init = ucc_tl_cuda_reduce_scatter_linear_init; break; #ifdef HAVE_NVLS case UCC_TL_CUDA_REDUCE_SCATTER_ALG_NVLS: *init = ucc_tl_cuda_reduce_scatter_nvls_init; break; #endif /* HAVE_NVLS */ default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_REDUCE_SCATTERV: switch (alg_id) { case UCC_TL_CUDA_REDUCE_SCATTERV_ALG_AUTO: *init = ucc_tl_cuda_reduce_scatterv_init; break; case UCC_TL_CUDA_REDUCE_SCATTERV_ALG_RING: *init = ucc_tl_cuda_reduce_scatterv_ring_init; break; case UCC_TL_CUDA_REDUCE_SCATTERV_ALG_LINEAR: *init = ucc_tl_cuda_reduce_scatterv_linear_init; break; #ifdef HAVE_NVLS case UCC_TL_CUDA_REDUCE_SCATTERV_ALG_NVLS: *init = ucc_tl_cuda_reduce_scatterv_nvls_init; break; #endif /* HAVE_NVLS */ default: status = UCC_ERR_INVALID_PARAM; break; }; break; default: status = UCC_ERR_NOT_SUPPORTED; break; } return status; } ucc-1.8.0/src/components/tl/cuda/tl_cuda_cache.h0000664000175000017500000000332015211535620021672 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_CACHE_H_ #define UCC_TL_CUDA_CACHE_H_ #include #include #include #include typedef struct ucc_tl_cuda_cache_region { ucs_pgt_region_t super; /**< Base class - page table region */ ucs_list_link_t list; /**< List element */ void *d_ptr; size_t size; cudaIpcMemHandle_t mem_handle; void *mapped_addr; /**< Local mapped address */ uint64_t refcount; /**< Track inflight ops before unmapping*/ } ucc_tl_cuda_cache_region_t; typedef struct ucc_tl_cuda_cache { pthread_rwlock_t lock; /**< Protects the page table */ ucs_pgtable_t pgtable; /**< Page table to hold the regions */ char *name; /**< Name */ } ucc_tl_cuda_cache_t; ucc_status_t ucc_tl_cuda_create_cache(ucc_tl_cuda_cache_t **cache, const char *name); void ucc_tl_cuda_destroy_cache(ucc_tl_cuda_cache_t *cache); ucc_status_t ucc_tl_cuda_map_memhandle(const void *dptr, size_t size, cudaIpcMemHandle_t mem_handle, void **mapped_addr, ucc_tl_cuda_cache_t *cache); ucc_status_t ucc_tl_cuda_unmap_memhandle(uintptr_t d_bptr, void *mapped_addr, ucc_tl_cuda_cache_t *cache, int force); ucc_tl_cuda_cache_t* ucc_tl_cuda_get_cache(ucc_tl_cuda_team_t *team, ucc_rank_t rank); #endif ucc-1.8.0/src/components/tl/cuda/configure.m40000664000175000017500000000335315211535620021214 0ustar alastairalastair# # Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # tl_cuda_enabled=n CHECK_TLS_REQUIRED(["cuda"]) AS_IF([test "$CHECKED_TL_REQUIRED" = "y"], [ if test $cuda_happy = "yes" -a $nvml_happy = "yes"; then tl_modules="${tl_modules}:cuda" tl_cuda_enabled=y AC_MSG_RESULT([TL CUDA: enabled]) CHECK_NEED_TL_PROFILING(["tl_cuda"]) AS_IF([test "$TL_PROFILING_REQUIRED" = "y"], [ AC_DEFINE([HAVE_PROFILING_TL_CUDA], [1], [Enable profiling for TL CUDA]) prof_modules="${prof_modules}:tl_cuda" ], []) else # Provide specific reasons for TL CUDA being disabled AS_IF([test "x$cuda_happy" != "xyes"], [AC_MSG_RESULT([TL CUDA: disabled (CUDA not available)])], [AS_IF([test "x$nvml_happy" != "xyes"], [AC_MSG_RESULT([TL CUDA: disabled (NVML headers/library missing - install cuda-nvml-devel package)])], [AC_MSG_RESULT([TL CUDA: disabled (unknown reason)])])]) fi ], [AC_MSG_RESULT([TL CUDA: not required])]) # Check for NVLS support AS_IF([test "$tl_cuda_enabled" = "y" -a "$nvls_happy" = "yes"], [ AC_DEFINE([HAVE_TL_CUDA_NVLS], [1], [Enable NVLS support in TL CUDA]) AC_MSG_RESULT([TL CUDA NVLS support: enabled]) ], [ AS_IF([test "$tl_cuda_enabled" = "y"], [AC_MSG_RESULT([TL CUDA NVLS support: disabled (NVLS not available)])], [AC_MSG_RESULT([TL CUDA NVLS support: disabled (TL CUDA not enabled)])]) ]) AM_CONDITIONAL([TL_CUDA_ENABLED], [test "$tl_cuda_enabled" = "y"]) AM_CONDITIONAL([TL_CUDA_NVLS_ENABLED], [test "$tl_cuda_enabled" = "y" -a "$nvls_happy" = "yes"]) AC_CONFIG_FILES([src/components/tl/cuda/Makefile]) ucc-1.8.0/src/components/tl/cuda/tl_cuda_topo.h0000664000175000017500000000464715211535620021625 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_TOPO_H_ #define UCC_TL_CUDA_TOPO_H_ #include "components/tl/ucc_tl_log.h" #include "utils/khash.h" #define MAX_PCI_BUS_ID_STR 16 typedef struct ucc_tl_cuda_device_id { uint16_t domain; /* range: 0 to ffff */ uint8_t bus; /* range: 0 to ff */ uint8_t device; /* range: 0 to 1f */ uint8_t function; /* range: 0 to 7 */ } ucc_tl_cuda_device_pci_id_t; typedef enum ucc_tl_cuda_topo_dev_type { UCC_TL_CUDA_TOPO_DEV_TYPE_GPU, UCC_TL_CUDA_TOPO_DEV_TYPE_SWITCH, UCC_TL_CUDA_TOPO_DEV_TYPE_LAST } ucc_tl_cuda_topo_dev_type_t; static inline int ucc_tl_cuda_topo_device_id_equal(const ucc_tl_cuda_device_pci_id_t *id1, const ucc_tl_cuda_device_pci_id_t *id2) { return ((id1->domain == id2->domain) && (id1->bus == id2->bus) && (id1->device == id2->device) && (id1->function == id2->function)); } typedef struct ucc_tl_cuda_topo_link { ucc_list_link_t list_link; ucc_tl_cuda_device_pci_id_t pci_id; int width; } ucc_tl_cuda_topo_link_t; typedef struct ucc_tl_cuda_topo_node { ucc_tl_cuda_device_pci_id_t pci_id; ucc_tl_cuda_topo_dev_type_t type; ucc_tl_cuda_topo_link_t link; } ucc_tl_cuda_topo_node_t; KHASH_MAP_INIT_INT64(bus_to_node, int); typedef struct ucc_tl_cuda_topo { const ucc_base_lib_t *lib; khash_t(bus_to_node) bus_to_node_hash; int num_nodes; ucc_tl_cuda_topo_node_t *graph; } ucc_tl_cuda_topo_t; ucc_status_t ucc_tl_cuda_topo_get_pci_id(int device, ucc_tl_cuda_device_pci_id_t *pci_id); ucc_status_t ucc_tl_cuda_topo_create(const ucc_base_lib_t *lib, ucc_tl_cuda_topo_t **cuda_topo); ucc_status_t ucc_tl_cuda_topo_destroy(ucc_tl_cuda_topo_t *cuda_topo); ucc_status_t ucc_tl_cuda_topo_num_links(const ucc_tl_cuda_topo_t *topo, const ucc_tl_cuda_device_pci_id_t *dev1, const ucc_tl_cuda_device_pci_id_t *dev2, ucc_rank_t *num_links); void ucc_tl_cuda_topo_pci_id_to_str(const ucc_tl_cuda_device_pci_id_t *pci_id, char *str, size_t max); #endif ucc-1.8.0/src/components/tl/cuda/tl_cuda_ring.h0000664000175000017500000000534015211535620021572 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_RING_H_ #define UCC_TL_CUDA_RING_H_ #include "tl_cuda.h" #include "tl_cuda_coll.h" enum { RING_STAGE_SYNC, /*< Wait for free SYNC segment */ RING_STAGE_SETUP, /*< Wait for memhandle setup to finish */ RING_STAGE_RING, /*< Ring algorithm is running */ RING_STAGE_BARRIER, /*< Ring algorithm is done, waiting for * other ranks to finish */ }; #define RING_MIN_MSG_SIZE 8192 /* Min size of msg per ring */ static inline size_t get_scratch_size(ucc_tl_cuda_team_t *team, int nrings, int nchunks, size_t dt_size) { size_t ratio = 2 * nrings * nchunks * dt_size; return ucc_align_down_pow2((UCC_TL_CUDA_TEAM_LIB(team)->cfg.scratch_size / ratio), 64) * ratio; } static inline int get_num_rings(ucc_tl_cuda_team_t *team, size_t msgsize, unsigned long rings_requested) { int nrings; if (rings_requested != UCC_ULUNITS_AUTO) { nrings = ucc_min(rings_requested, team->topo->num_rings); } else { nrings = team->topo->num_rings; } nrings = ucc_min(nrings, UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS); nrings = ucc_min(nrings, ucc_div_round_up(msgsize, RING_MIN_MSG_SIZE)); return nrings; } static inline ucc_rank_t get_send_to(ucc_tl_cuda_team_t *team, ucc_rank_t trank, ucc_rank_t tsize, int ring_id) { ucc_tl_cuda_ring_t *ring = &team->topo->rings[ring_id]; return ring->ring[(ring->iring[trank] + 1) % tsize]; } static inline ucc_rank_t get_recv_from(ucc_tl_cuda_team_t *team, ucc_rank_t trank, ucc_rank_t tsize, int ring_id) { ucc_tl_cuda_ring_t *ring = &team->topo->rings[ring_id]; return ring->ring[(ring->iring[trank] - 1 + tsize) % tsize]; } static inline ucc_rank_t get_send_block(ucc_tl_cuda_team_t *team, ucc_rank_t trank, ucc_rank_t tsize, uint32_t step, int ring_id) { ucc_tl_cuda_ring_t *ring = &team->topo->rings[ring_id]; return ring->ring[(ring->iring[trank] + tsize - step) % tsize]; } static inline ucc_rank_t get_recv_block(ucc_tl_cuda_team_t *team, ucc_rank_t trank, ucc_rank_t tsize, uint32_t step, int ring_id) { ucc_tl_cuda_ring_t *ring = &team->topo->rings[ring_id]; return ring->ring[(ring->iring[trank] + tsize - step - 1) % tsize]; } #endif ucc-1.8.0/src/components/tl/cuda/alltoallv/0000775000175000017500000000000015211535620020757 5ustar alastairalastairucc-1.8.0/src/components/tl/cuda/alltoallv/alltoallv.c0000664000175000017500000000246115211535620023120 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "alltoallv.h" #include "components/mc/ucc_mc.h" ucc_status_t ucc_tl_cuda_alltoallv_ce_init(ucc_tl_cuda_task_t *task); ucc_status_t ucc_tl_cuda_alltoallv_ce_start(ucc_coll_task_t *task); void ucc_tl_cuda_alltoallv_ce_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_alltoallv_ce_finalize(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_alltoallv_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_task_t *task; ucc_status_t status; if (UCC_IS_INPLACE(coll_args->args)) { return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } status = ucc_tl_cuda_alltoallv_ce_init(task); if (ucc_unlikely(status != UCC_OK)) { goto free_task; } *task_p = &task->super; return UCC_OK; free_task: ucc_tl_cuda_task_put(task); return status; } ucc-1.8.0/src/components/tl/cuda/alltoallv/alltoallv_ce.c0000664000175000017500000006500715211535620023574 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "alltoallv.h" #include "core/ucc_ee.h" #include "tl_cuda_cache.h" #include "utils/arch/cpu.h" #include "utils/arch/cuda_def.h" #include "utils/ucc_compiler_def.h" #include "components/ec/ucc_ec.h" #include "components/tl/ucc_tl_log.h" #include "components/ec/cuda/ec_cuda_resources.h" #include // memcpy memset enum { ALLTOALL_CE_STAGE_SYNC, /*< Wait for free SYNC segment */ ALLTOALL_CE_STAGE_SETUP, /*< Wait for memhandle setup to finish */ ALLTOALL_CE_STAGE_POST_COPIES, /*< Post copies */ ALLTOALL_CE_STAGE_COPY, /*< Wait for all copies to finish */ ALLTOALL_CE_STAGE_BAR, /*< Wait for other ranks to finish */ }; //NOLINTNEXTLINE(misc-unused-parameters): stream parameter unused as executor manages execution ucc_status_t ee_copy_post(void *dst, const void *src, size_t len, ucc_ee_executor_t *executor, ucc_ee_executor_task_t **task, cudaStream_t stream) { (void)stream; /* Unused parameter */ ucc_ee_executor_task_args_t exec_args = {0}; exec_args.task_type = UCC_EE_EXECUTOR_TASK_COPY; exec_args.copy.dst = dst; exec_args.copy.src = src; exec_args.copy.len = len; return ucc_ee_executor_task_post(executor, &exec_args, task); } //NOLINTNEXTLINE(misc-unused-parameters): executor and task unused as operation handled by CUDA ucc_status_t cuda_copy_post(void *dst, const void *src, size_t len, ucc_ee_executor_t *executor, ucc_ee_executor_task_t **task, cudaStream_t stream) { (void)executor; /* Unused parameter */ (void)task; /* Unused parameter */ CUDA_CHECK( cudaMemcpyAsync(dst, src, len, cudaMemcpyDeviceToDevice, stream)); return UCC_OK; } ucc_status_t ucc_tl_cuda_alltoallv_ce_finalize(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); tl_trace(UCC_TASK_LIB(task), "finalizing task %p", task); // Clean up completion events if (task->alltoallv_ce.evt_completion) { ucc_ec_destroy_event(task->alltoallv_ce.evt_completion, UCC_EE_CUDA_STREAM); } ucc_tl_cuda_task_put(task); return UCC_OK; } ucc_status_t ucc_tl_cuda_alltoallv_setup_start(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_tl_cuda_sync_t *sync = TASK_SYNC(task, UCC_TL_TEAM_RANK(team)); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_ee_h ee = task->super.ee; cudaStream_t stream = (ee) ? (cudaStream_t)ee->ee_context : team->stream; ucc_status_t status; // For Alltoallv: copy counts and displ. to SHM for remote GPUs to access (if required) if (UCC_COLL_TYPE_ALLTOALLV == args->coll_type) { int i; size_t rdt_size = ucc_dt_size(task->alltoallv_ce.rdt); size_t sdt_size = ucc_dt_size(task->alltoallv_ce.sdt); for (i = 0; i < UCC_TL_TEAM_SIZE(team); ++i) { sync->alltoallv_ce.sbytes[i] = sdt_size * (size_t)ucc_coll_args_get_count( args, task->alltoallv_ce.scnts, i); sync->alltoallv_ce.rbytes[i] = rdt_size * (size_t)ucc_coll_args_get_count( args, task->alltoallv_ce.rcnts, i); sync->alltoallv_ce.sdispl_bytes[i] = sdt_size * (size_t)ucc_coll_args_get_displacement( args, task->alltoallv_ce.sdispl, i); sync->alltoallv_ce.rdispl_bytes[i] = rdt_size * (size_t)ucc_coll_args_get_displacement( args, task->alltoallv_ce.rdispl, i); } } memcpy(&sync->mem_info_src, &task->alltoallv_ce.mem_info_src, sizeof(ucc_tl_cuda_mem_info_t)); memcpy(&sync->mem_info_dst, &task->alltoallv_ce.mem_info_dst, sizeof(ucc_tl_cuda_mem_info_t)); CUDA_CHECK_GOTO(cudaEventRecord(sync->ipc_event_local, stream), exit_err, status); ucc_memory_cpu_store_fence(); status = ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), task->bar); if (ucc_unlikely(status != UCC_OK)) { goto exit_err; } return UCC_OK; exit_err: return status; } ucc_status_t ucc_tl_cuda_alltoallv_setup_test(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_ee_h ee = task->super.ee; cudaStream_t stream = (ee) ? (cudaStream_t)ee->ee_context : team->stream; volatile ucc_tl_cuda_sync_t *peer_sync, *sync; ucc_tl_cuda_cache_t *cache; ucc_status_t status; ucc_rank_t i, dst; status = ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); if (status != UCC_OK) { return status; } sync = TASK_SYNC(task, UCC_TL_TEAM_RANK(team)); for (i = 0; i < UCC_TL_TEAM_SIZE(team); i++) { if (i == UCC_TL_TEAM_RANK(team) || !ucc_tl_cuda_team_topo_is_direct(&team->super, team->topo, UCC_TL_TEAM_RANK(team), i)) { continue; } peer_sync = TASK_SYNC(task, i); cache = ucc_tl_cuda_get_cache(team, i); if (ucc_unlikely(!cache)) { status = UCC_ERR_NO_MESSAGE; goto exit_err; } status = ucc_tl_cuda_map_memhandle( peer_sync->mem_info_src.ptr, peer_sync->mem_info_src.length, peer_sync->mem_info_src.handle, &task->alltoallv_ce.peer_map_addr_src[i], cache); if (UCC_OK != status) { ucc_error("ucc_cuda_ipc_map_memhandle failed"); return UCC_ERR_INVALID_PARAM; } CUDA_CHECK_GOTO( cudaStreamWaitEvent(stream, sync->data[i].ipc_event_remote, 0), exit_err, status); } for (i = 0; i < team->topo->num_proxies; i++) { dst = team->topo->proxies[i].dst; peer_sync = TASK_SYNC(task, dst); cache = ucc_tl_cuda_get_cache(team, dst); if (ucc_unlikely(!cache)) { status = UCC_ERR_NO_MESSAGE; goto exit_err; } status = ucc_tl_cuda_map_memhandle( peer_sync->mem_info_dst.ptr, peer_sync->mem_info_dst.length, peer_sync->mem_info_dst.handle, &task->alltoallv_ce.peer_map_addr_dst[dst], cache); if (UCC_OK != status) { ucc_error("ucc_cuda_ipc_map_memhandle failed"); return UCC_ERR_INVALID_PARAM; } } return UCC_OK; exit_err: return status; } ucc_status_t ucc_tl_cuda_alltoallv_ce_post_copies(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_tl_cuda_sync_t *sync = TASK_SYNC(task, rank); ucc_ee_h ee = task->super.ee; ucc_ec_cuda_event_t *ec_event = NULL; cudaEvent_t evt = NULL; ucc_status_t status = UCC_OK; cudaStream_t stream = 0; ucc_tl_cuda_sync_t *peer_sync; ucc_ee_executor_t *exec; void *src, *dst; size_t data_size, data_displ; ucc_rank_t i, peer, psrc, pdst; if (task->alltoallv_ce.use_copy_engine) { ec_event = (ucc_ec_cuda_event_t *)task->alltoallv_ce.evt_completion; evt = ec_event->event; } if (task->alltoallv_ce.use_copy_engine) { // If triggered post, use the stream from the executor if (ee) { stream = (cudaStream_t)ee->ee_context; } // copy engine is used, so no executor is needed exec = NULL; } else { status = ucc_coll_task_get_executor(&task->super, &exec); if (ucc_unlikely(status != UCC_OK)) { goto exit; } } task->alltoallv_ce.num_posted = 0; for (i = 0; i < UCC_TL_TEAM_SIZE(team); i++) { peer = (rank + i) % UCC_TL_TEAM_SIZE(team); if (!ucc_tl_cuda_team_topo_is_direct(&team->super, team->topo, rank, peer)) { continue; } peer_sync = TASK_SYNC(task, peer); if (peer == rank) { src = task->alltoallv_ce.sbuf; } else { src = PTR_OFFSET(task->alltoallv_ce.peer_map_addr_src[peer], peer_sync->mem_info_src.offset); } data_size = task->alltoallv_ce.get_size( task, peer_sync->alltoallv_ce.sbytes, rank); if (data_size == 0) { continue; } data_displ = task->alltoallv_ce.get_offset( task, peer_sync->alltoallv_ce.sdispl_bytes, rank); src = PTR_OFFSET(src, data_displ); data_displ = task->alltoallv_ce.get_offset( task, sync->alltoallv_ce.rdispl_bytes, peer); dst = PTR_OFFSET(task->alltoallv_ce.rbuf, data_displ); status = task->alltoallv_ce.copy_post( dst, src, data_size, exec, &task->alltoallv_ce.exec_task[task->alltoallv_ce.num_posted], stream); if (ucc_unlikely(status != UCC_OK)) { goto exit; } task->alltoallv_ce.num_posted++; } for (i = 0; i < team->topo->num_proxies; i++) { psrc = team->topo->proxies[i].src; pdst = team->topo->proxies[i].dst; peer_sync = TASK_SYNC(task, psrc); data_size = task->alltoallv_ce.get_size( task, peer_sync->alltoallv_ce.sbytes, pdst); if (data_size == 0) { continue; } data_displ = task->alltoallv_ce.get_offset( task, peer_sync->alltoallv_ce.sdispl_bytes, pdst); src = PTR_OFFSET(task->alltoallv_ce.peer_map_addr_src[psrc], peer_sync->mem_info_src.offset); src = PTR_OFFSET(src, data_displ); peer_sync = TASK_SYNC(task, pdst); dst = PTR_OFFSET(task->alltoallv_ce.peer_map_addr_dst[pdst], peer_sync->mem_info_dst.offset); data_displ = task->alltoallv_ce.get_offset( task, peer_sync->alltoallv_ce.rdispl_bytes, psrc); dst = PTR_OFFSET(dst, data_displ); status = task->alltoallv_ce.copy_post( dst, src, data_size, exec, &task->alltoallv_ce.exec_task[task->alltoallv_ce.num_posted], stream); if (ucc_unlikely(status != UCC_OK)) { goto exit; } task->alltoallv_ce.num_posted++; } if (task->alltoallv_ce.use_copy_engine) { CUDA_CHECK_GOTO(cudaEventRecord(evt, stream), exit, status); } exit: return status; } #if CUDART_VERSION >= 13000 static ucc_status_t ucc_tl_cuda_alltoallv_ce_post_batch_copies( ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_tl_cuda_sync_t *sync = TASK_SYNC(task, rank); ucc_ee_h ee = task->super.ee; ucc_ec_cuda_event_t *ec_event = NULL; cudaEvent_t evt = NULL; cudaStream_t stream = 0; size_t op_count = 0; const void **srcs; void **dsts; size_t *sizes; struct cudaMemcpyAttributes *attrs; size_t *attrsIdxs; ucc_tl_cuda_sync_t *peer_sync; ucc_rank_t i, peer; void *src, *dst; size_t data_size, data_displ; if (task->alltoallv_ce.use_copy_engine) { ec_event = (ucc_ec_cuda_event_t *)task->alltoallv_ce.evt_completion; evt = ec_event->event; } // Count total operations for (i = 0; i < UCC_TL_TEAM_SIZE(team); i++) { peer = (rank + i) % UCC_TL_TEAM_SIZE(team); peer_sync = TASK_SYNC(task, peer); data_size = task->alltoallv_ce.get_size( task, peer_sync->alltoallv_ce.sbytes, rank); if (data_size > 0) { op_count++; } } stream = ee ? (cudaStream_t)ee->ee_context : team->stream; // If there is nothing to copy, just record completion and return if (op_count == 0) { CUDA_CHECK(cudaEventRecord(evt, stream)); task->alltoallv_ce.num_posted = 0; return UCC_OK; } // Allocate arrays on stack (bounded by small team size for single-node NVLink) srcs = alloca(op_count * sizeof(const void *)); dsts = alloca(op_count * sizeof(void *)); sizes = alloca(op_count * sizeof(size_t)); attrs = alloca(op_count * sizeof(struct cudaMemcpyAttributes)); attrsIdxs = alloca(op_count * sizeof(size_t)); op_count = 0; for (i = 0; i < UCC_TL_TEAM_SIZE(team); i++) { peer = (rank + i) % UCC_TL_TEAM_SIZE(team); peer_sync = TASK_SYNC(task, peer); data_size = task->alltoallv_ce.get_size( task, peer_sync->alltoallv_ce.sbytes, rank); if (data_size == 0) { continue; } // Source buffer if (peer == rank) { src = task->alltoallv_ce.sbuf; } else { src = PTR_OFFSET(task->alltoallv_ce.peer_map_addr_src[peer], peer_sync->mem_info_src.offset); } data_displ = task->alltoallv_ce.get_offset( task, peer_sync->alltoallv_ce.sdispl_bytes, rank); src = PTR_OFFSET(src, data_displ); // Destination buffer data_displ = task->alltoallv_ce.get_offset( task, sync->alltoallv_ce.rdispl_bytes, peer); dst = PTR_OFFSET(task->alltoallv_ce.rbuf, data_displ); srcs[op_count] = (const void *)src; dsts[op_count] = dst; sizes[op_count] = data_size; // Setup attributes: leave location hints unset so runtime infers devices memset(&attrs[op_count], 0, sizeof(struct cudaMemcpyAttributes)); attrs[op_count].srcAccessOrder = cudaMemcpySrcAccessOrderAny; attrs[op_count].flags = cudaMemcpyFlagPreferOverlapWithCompute; attrsIdxs[op_count] = op_count; op_count++; } // Launch batch copy CUDA_CHECK( cudaMemcpyBatchAsync( dsts, (const void *const *)srcs, sizes, op_count, attrs, attrsIdxs, op_count, stream)); task->alltoallv_ce.num_posted = op_count; // Record completion events CUDA_CHECK(cudaEventRecord(evt, stream)); return UCC_OK; } #endif ucc_status_t ucc_tl_cuda_alltoallv_unmap(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t i, dst; volatile ucc_tl_cuda_sync_t *peer_sync; ucc_tl_cuda_cache_t *cache; ucc_status_t status; for (i = 0; i < UCC_TL_TEAM_SIZE(team); i++) { if (i == UCC_TL_TEAM_RANK(team) || !ucc_tl_cuda_team_topo_is_direct(&team->super, team->topo, UCC_TL_TEAM_RANK(team), i)) { continue; } peer_sync = TASK_SYNC(task, i); cache = ucc_tl_cuda_get_cache(team, i); status = ucc_tl_cuda_unmap_memhandle( (uintptr_t)peer_sync->mem_info_src.ptr, task->alltoallv_ce.peer_map_addr_src[i], cache, 0); if (ucc_unlikely(status != UCC_OK)) { return status; } } for (i = 0; i < team->topo->num_proxies; i++) { dst = team->topo->proxies[i].dst; peer_sync = TASK_SYNC(task, dst); cache = ucc_tl_cuda_get_cache(team, dst); status = ucc_tl_cuda_unmap_memhandle( (uintptr_t)peer_sync->mem_info_dst.ptr, task->alltoallv_ce.peer_map_addr_dst[dst], cache, 0); if (ucc_unlikely(status != UCC_OK)) { return status; } } return UCC_OK; } void ucc_tl_cuda_alltoallv_ce_progress(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_ec_cuda_event_t *ec_event = NULL; cudaEvent_t evt = NULL; ucc_status_t status; int i; if (task->alltoallv_ce.use_copy_engine) { ec_event = (ucc_ec_cuda_event_t *)task->alltoallv_ce.evt_completion; evt = ec_event->event; } switch (task->alltoallv_ce.stage) { case ALLTOALL_CE_STAGE_SYNC: if (ucc_tl_cuda_get_sync(task) != UCC_OK) { task->super.status = UCC_INPROGRESS; return; } status = ucc_tl_cuda_alltoallv_setup_start(task); if (status != UCC_OK) { task->super.status = status; return; } task->alltoallv_ce.stage = ALLTOALL_CE_STAGE_SETUP; /* fall through */ case ALLTOALL_CE_STAGE_SETUP: status = ucc_tl_cuda_alltoallv_setup_test(task); if (status != UCC_OK) { task->super.status = status; return; } /* fall through */ case ALLTOALL_CE_STAGE_POST_COPIES: /* Use batch copies only on CUDA 13.0+; otherwise fall back */ #if CUDART_VERSION >= 13000 if (task->alltoallv_ce.use_copy_engine) { status = ucc_tl_cuda_alltoallv_ce_post_batch_copies(task); } else { status = ucc_tl_cuda_alltoallv_ce_post_copies(task); } #else status = ucc_tl_cuda_alltoallv_ce_post_copies(task); #endif if (ucc_unlikely(status != UCC_OK)) { task->super.status = status; return; } task->alltoallv_ce.stage = ALLTOALL_CE_STAGE_COPY; /* fall through */ case ALLTOALL_CE_STAGE_COPY: if (task->alltoallv_ce.use_copy_engine) { cudaError_t cuda_status = cudaEventQuery(evt); if (cuda_status == cudaErrorNotReady) { task->super.status = UCC_INPROGRESS; return; } else if (cuda_status != cudaSuccess) { tl_error(UCC_TASK_LIB(task), "error cudaEventQuery %s!", cudaGetErrorString(cuda_status)); task->super.status = UCC_ERR_NO_MESSAGE; return; } tl_trace(UCC_TASK_LIB(task), "all cuda copies finished"); task->super.status = UCC_OK; } else { for (i = 0; i < task->alltoallv_ce.num_posted; i++) { if (!task->alltoallv_ce.exec_task[i]) { continue; } status = ucc_ee_executor_task_test(task->alltoallv_ce.exec_task[i]); if (status != UCC_OK) { if (status == UCC_OPERATION_INITIALIZED) { status = UCC_INPROGRESS; } task->super.status = status; return; } ucc_ee_executor_task_finalize(task->alltoallv_ce.exec_task[i]); task->alltoallv_ce.exec_task[i] = NULL; } } status = ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), task->bar); if (ucc_unlikely(status != UCC_OK)) { task->super.status = status; return; } task->alltoallv_ce.stage = ALLTOALL_CE_STAGE_BAR; /* fall through */ default: ucc_assert(task->alltoallv_ce.stage == ALLTOALL_CE_STAGE_BAR); break; } status = ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); if (status == UCC_OK) { status = ucc_tl_cuda_alltoallv_unmap(task); ucc_tl_cuda_put_sync(task); } task->super.status = status; } ucc_status_t ucc_tl_cuda_alltoallv_ce_start(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); // if not triggered post or copy engine is used, we need to start from sync if (task->alltoallv_ce.stage != ALLTOALL_CE_STAGE_POST_COPIES && task->alltoallv_ce.stage != ALLTOALL_CE_STAGE_COPY) { task->alltoallv_ce.stage = ALLTOALL_CE_STAGE_SYNC; } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_cuda_alltoallv_ce_triggered_post_setup(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_status_t status; do { status = ucc_tl_cuda_get_sync(task); } while (status == UCC_INPROGRESS); if (ucc_unlikely(status != UCC_OK)) { return status; } status = ucc_tl_cuda_alltoallv_setup_start(task); if (ucc_unlikely(status != UCC_OK)) { ucc_tl_cuda_put_sync(task); return status; } do { status = ucc_tl_cuda_alltoallv_setup_test(task); } while (status == UCC_INPROGRESS); if (ucc_unlikely(status != UCC_OK)) { ucc_tl_cuda_put_sync(task); return status; } task->alltoallv_ce.stage = ALLTOALL_CE_STAGE_POST_COPIES; return UCC_OK; } ucc_status_t ucc_tl_cuda_alltoallv_ce_setup_copy_engine( ucc_tl_cuda_task_t *task, ucc_tl_cuda_lib_t *lib, const char *init_func_name) { ucc_status_t status; if (ucc_coll_task_is_cl_hier(&task->super)) { tl_trace( lib, "CL hier does not support copy engine, fallback to executor"); task->alltoallv_ce.use_copy_engine = 0; } else { task->alltoallv_ce.use_copy_engine = lib->cfg.alltoall_use_copy_engine; } if (task->alltoallv_ce.use_copy_engine) { tl_trace(lib, "%s: copy engine", init_func_name); task->super.triggered_post = ucc_tl_cuda_alltoallv_ce_triggered_post; task->alltoallv_ce.copy_post = cuda_copy_post; status = ucc_ec_create_event( &task->alltoallv_ce.evt_completion, UCC_EE_CUDA_STREAM); if (ucc_unlikely(status != UCC_OK)) { return status; } } else { tl_trace(lib, "%s: executor", init_func_name); task->alltoallv_ce.copy_post = ee_copy_post; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; } task->super.post = ucc_tl_cuda_alltoallv_ce_start; task->super .triggered_post_setup = ucc_tl_cuda_alltoallv_ce_triggered_post_setup; task->super.progress = ucc_tl_cuda_alltoallv_ce_progress; task->super.finalize = ucc_tl_cuda_alltoallv_ce_finalize; task->bar = TASK_BAR(task); return UCC_OK; } //NOLINTNEXTLINE: task is unused size_t ucc_tl_cuda_alltoallv_get_size(const ucc_tl_cuda_task_t *task, size_t *sizes, ucc_rank_t block) { return sizes[block]; } //NOLINTNEXTLINE: task is unused size_t ucc_tl_cuda_alltoallv_get_offset(const ucc_tl_cuda_task_t *task, size_t *displ, ucc_rank_t block) { return displ[block]; } //NOLINTNEXTLINE(misc-unused-parameters): ev parameter unused as it's not needed for this implementation ucc_status_t ucc_tl_cuda_alltoallv_ce_triggered_post(ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_status_t status; ucc_ev_t post_event; ucc_assert(ee->ee_type == UCC_EE_CUDA_STREAM); coll_task->ee = ee; tl_trace(UCC_TASK_LIB(task), "triggered post. task:%p", coll_task); coll_task->triggered_post_setup(coll_task); /* Use batch copies only on CUDA 13.0+; otherwise fall back */ #if CUDART_VERSION >= 13000 status = ucc_tl_cuda_alltoallv_ce_post_batch_copies(task); #else status = ucc_tl_cuda_alltoallv_ce_post_copies(task); #endif if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to post copies"); return status; } task->alltoallv_ce.stage = ALLTOALL_CE_STAGE_COPY; status = coll_task->post(coll_task); if (ucc_likely(status == UCC_OK)) { post_event.ev_type = UCC_EVENT_COLLECTIVE_POST; post_event.ev_context_size = 0; post_event.ev_context = NULL; post_event.req = &coll_task->super; ucc_ee_set_event_internal(coll_task->ee, &post_event, &coll_task->ee->event_out_queue); } return status; } ucc_status_t ucc_tl_cuda_alltoallv_ce_init(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_tl_cuda_lib_t *lib = UCC_TL_CUDA_TEAM_LIB(team); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_status_t status; size_t data_len; if (!UCC_COLL_ARGS_CONTIG_BUFFER(args)) { tl_debug(UCC_TL_TEAM_LIB(team), "Do not support non-contiguous buffer"); return UCC_ERR_NOT_SUPPORTED; } task->alltoallv_ce.get_size = ucc_tl_cuda_alltoallv_get_size; task->alltoallv_ce.get_offset = ucc_tl_cuda_alltoallv_get_offset; task->alltoallv_ce.sdt = args->src.info_v.datatype; task->alltoallv_ce.rdt = args->dst.info_v.datatype; task->alltoallv_ce.sbuf = args->src.info_v.buffer; task->alltoallv_ce.rbuf = args->dst.info_v.buffer; task->alltoallv_ce.scnts = args->src.info_v.counts; task->alltoallv_ce.rcnts = args->dst.info_v.counts; task->alltoallv_ce.sdispl = args->src.info_v.displacements; task->alltoallv_ce.rdispl = args->dst.info_v.displacements; task->alltoallv_ce.stage = ALLTOALL_CE_STAGE_SYNC; data_len = ucc_dt_size(args->src.info_v.datatype) * ucc_coll_args_get_total_count(args, args->src.info_v.counts, UCC_TL_TEAM_SIZE(team)); status = ucc_tl_cuda_mem_info_get(args->src.info_v.buffer, data_len, &task->alltoallv_ce.mem_info_src); if (ucc_unlikely(status != UCC_OK)) { return status; } if (team->topo->proxy_needed) { data_len = ucc_dt_size(args->dst.info_v.datatype) * ucc_coll_args_get_total_count( args, args->dst.info_v.counts, UCC_TL_TEAM_SIZE(team)); status = ucc_tl_cuda_mem_info_get( args->dst.info_v.buffer, data_len, &task->alltoallv_ce.mem_info_dst); if (ucc_unlikely(status != UCC_OK)) { return status; } } return ucc_tl_cuda_alltoallv_ce_setup_copy_engine( task, lib, "ucc_tl_cuda_alltoallv_ce_init"); } ucc-1.8.0/src/components/tl/cuda/alltoallv/alltoallv.h0000664000175000017500000000573315211535620023132 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #ifndef ALLTOALLV_H_ #define ALLTOALLV_H_ #include "../tl_cuda.h" #include "../tl_cuda_coll.h" ucc_status_t ucc_tl_cuda_alltoallv_ce_finalize(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_cuda_alltoallv_ce_start(ucc_coll_task_t *coll_task); void ucc_tl_cuda_alltoallv_ce_progress(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_cuda_alltoallv_ce_triggered_post_setup(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_cuda_alltoallv_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p); /** * @brief Post a copy operation using CUDA copy engine * * This function posts a copy operation to be executed directly by the CUDA copy engine. * The executor and task parameters are unused as the operation is handled by CUDA. * The stream parameter is used to specify which CUDA stream should execute the copy. * * @param dst Destination buffer for the copy operation * @param src Source buffer for the copy operation * @param len Length of data to copy in bytes * @param executor Unused - operation handled by CUDA copy engine * @param task Unused - operation handled by CUDA copy engine * @param stream CUDA stream to execute the copy operation * @return UCC_OK on success, error code otherwise */ ucc_status_t cuda_copy_post(void *dst, const void *src, size_t len, ucc_ee_executor_t *executor, ucc_ee_executor_task_t **task, cudaStream_t stream); /** * @brief Post a copy operation using the UCC executor * * This function posts a copy operation to be executed by the UCC executor. * The executor and task parameters are used to track the operation's progress. * The stream parameter is unused as the executor manages its own execution context. * * @param dst Destination buffer for the copy operation * @param src Source buffer for the copy operation * @param len Length of data to copy in bytes * @param executor UCC executor to handle the copy operation * @param task Pointer to store the executor task handle * @param stream Unused - executor manages its own execution context * @return UCC_OK on success, error code otherwise */ ucc_status_t ee_copy_post(void *dst, const void *src, size_t len, ucc_ee_executor_t *executor, ucc_ee_executor_task_t **task, cudaStream_t stream); ucc_status_t ucc_tl_cuda_alltoallv_ce_triggered_post(ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_cuda_alltoallv_ce_setup_copy_engine(ucc_tl_cuda_task_t *task, ucc_tl_cuda_lib_t *lib, const char *init_func_name); #endif ucc-1.8.0/src/components/tl/cuda/tl_cuda.c0000664000175000017500000001257615211535620020557 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_cuda.h" #include "core/ucc_team.h" #include "components/mc/base/ucc_mc_base.h" #include "allgather/allgather.h" #include "allgatherv/allgatherv.h" #ifdef HAVE_NVLS #include "allreduce/allreduce.h" #endif #include "bcast/bcast.h" #include "reduce_scatter/reduce_scatter.h" #include "reduce_scatterv/reduce_scatterv.h" static ucc_config_field_t ucc_tl_cuda_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_cuda_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_lib_config_table)}, {"MAX_CONCURRENT", "8", "Maximum number of outstanding colls", ucc_offsetof(ucc_tl_cuda_lib_config_t, max_concurrent), UCC_CONFIG_TYPE_UINT}, {"SCRATCH_SIZE", "2Mb", "Size of the internal scratch buffer", ucc_offsetof(ucc_tl_cuda_lib_config_t, scratch_size), UCC_CONFIG_TYPE_MEMUNITS}, {"ALLGATHER_RING_MAX_RINGS", "auto", "Max number of rings used in allgather and allgatherv ring algorithms", ucc_offsetof(ucc_tl_cuda_lib_config_t, allgather_ring_max_rings), UCC_CONFIG_TYPE_ULUNITS}, {"ALLGATHER_RING_NUM_CHUNKS", "4", "Number of chunks each ring message will be split into", ucc_offsetof(ucc_tl_cuda_lib_config_t, allgather_ring_num_chunks), UCC_CONFIG_TYPE_UINT}, {"REDUCE_SCATTER_RING_MAX_RINGS", "auto", "Max number of rings used in reduce_scatter and " "reduce_scatterv ring algorithms", ucc_offsetof(ucc_tl_cuda_lib_config_t, reduce_scatter_ring_max_rings), UCC_CONFIG_TYPE_ULUNITS}, {"TOPO_CACHE_ENABLE", "y", "Enable NVLINK topology cache", ucc_offsetof(ucc_tl_cuda_lib_config_t, topo_cache_enable), UCC_CONFIG_TYPE_BOOL}, #ifdef HAVE_NVLS {"NVLS_SYMMETRIC_SIZE", "512Mb", "Size of the symmetric memory for NVLS, for each task", ucc_offsetof(ucc_tl_cuda_lib_config_t, nvls_symmetric_size), UCC_CONFIG_TYPE_MEMUNITS}, {"NVLS_SM_COUNT", "4", "Number of SMs to use for NVLS", ucc_offsetof(ucc_tl_cuda_lib_config_t, nvls_sm_count), UCC_CONFIG_TYPE_UINT}, {"NVLS_THREADS", "1024", "Number of threads per block to use for NVLS", ucc_offsetof(ucc_tl_cuda_lib_config_t, nvls_threads), UCC_CONFIG_TYPE_UINT}, #endif {"ALLTOALL_USE_COPY_ENGINE", "y", "Use copy engine for alltoall(v)", ucc_offsetof(ucc_tl_cuda_lib_config_t, alltoall_use_copy_engine), UCC_CONFIG_TYPE_BOOL}, {NULL}}; UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_cuda_lib_t, ucc_base_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_cuda_lib_t, ucc_base_lib_t); ucc_status_t ucc_tl_cuda_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr); ucc_status_t ucc_tl_cuda_get_lib_properties(ucc_base_lib_properties_t *prop); static ucs_config_field_t ucc_tl_cuda_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_cuda_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_context_config_table)}, {NULL}}; ucc_status_t ucc_tl_cuda_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *base_attr); ucc_status_t ucc_tl_cuda_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_cuda_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_cuda_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **pack_buffer); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_cuda_context_t, ucc_base_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_cuda_context_t, ucc_base_context_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_cuda_team_t, ucc_base_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_tl_cuda_team_create_test(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_cuda_team_destroy(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_cuda_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); ucc_status_t ucc_tl_cuda_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p); UCC_TL_IFACE_DECLARE(cuda, CUDA); __attribute__((constructor)) static void tl_cuda_iface_init(void) { ucc_tl_cuda.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLGATHER)] = ucc_tl_cuda_allgather_algs; ucc_tl_cuda.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLGATHERV)] = ucc_tl_cuda_allgatherv_algs; #ifdef HAVE_NVLS ucc_tl_cuda.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLREDUCE)] = ucc_tl_cuda_allreduce_algs; #endif ucc_tl_cuda.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_BCAST)] = ucc_tl_cuda_bcast_algs; ucc_tl_cuda.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_REDUCE_SCATTER)] = ucc_tl_cuda_reduce_scatter_algs; ucc_tl_cuda.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_REDUCE_SCATTERV)] = ucc_tl_cuda_reduce_scatterv_algs; } ucc-1.8.0/src/components/tl/cuda/tl_cuda_context.c0000664000175000017500000001441415211535620022314 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_cuda.h" #include "utils/arch/cpu.h" #include "utils/arch/cuda_def.h" #include #include #include /** * Initialize CUDA transport layer context * * This function initializes a CUDA TL context which requires an active CUDA context. * It sets up memory pools for CUDA tasks and initializes the topology information. * * @param [in] params Base context initialization parameters * @param [in] config Configuration for CUDA context * * @return UCC_OK on success or error code on failure */ UCC_CLASS_INIT_FUNC(ucc_tl_cuda_context_t, const ucc_base_context_params_t *params, const ucc_base_config_t *config) { ucc_tl_cuda_context_config_t *tl_cuda_config = ucc_derived_of(config, ucc_tl_cuda_context_config_t); ucc_status_t status; ucc_tl_cuda_lib_t *lib; int num_devices; cudaError_t cuda_st; CUcontext cu_ctx; CUresult cu_st; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_context_t, &tl_cuda_config->super, params->context); lib = ucc_derived_of(self->super.super.lib, ucc_tl_cuda_lib_t); memcpy(&self->cfg, tl_cuda_config, sizeof(*tl_cuda_config)); cuda_st = cudaGetDeviceCount(&num_devices); if (cuda_st != cudaSuccess) { tl_debug(self->super.super.lib, "failed to get number of GPU devices: %d (%s)", cuda_st, cudaGetErrorName(cuda_st)); return UCC_ERR_NO_RESOURCE; } else if (num_devices == 0) { tl_debug(self->super.super.lib, "no GPU devices found"); return UCC_ERR_NO_RESOURCE; } cu_st = cuCtxGetCurrent(&cu_ctx); if (cu_ctx == NULL || cu_st != CUDA_SUCCESS) { tl_debug(self->super.super.lib, "cannot create CUDA TL context without active CUDA context"); return UCC_ERR_NO_RESOURCE; } status = ucc_mpool_init(&self->req_mp, 0, sizeof(ucc_tl_cuda_task_t), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_coll_task_mpool_ops, params->thread_mode, "tl_cuda_req_mp"); if (status != UCC_OK) { tl_error(self->super.super.lib, "failed to initialize tl_cuda_req mpool"); return status; } CUDA_CHECK_GOTO(cudaGetDevice(&self->device), free_mpool, status); /* Handle CUDA topology initialization based on caching configuration */ if (lib->cfg.topo_cache_enable && lib->topo != NULL) { /* If topology caching is enabled and a cached topology exists, reuse the existing topology from the library */ self->topo = lib->topo; } else { /* Determine where to store the topology: - If caching is enabled: store in lib->topo for reuse - If caching is disabled: store in self->topo (context-specific) */ ucc_tl_cuda_topo_t **topo_ptr = lib->cfg.topo_cache_enable ? &lib->topo : &self->topo; /* Create new topology instance and store it in the appropriate location */ status = ucc_tl_cuda_topo_create((const ucc_base_lib_t *)&lib->super, topo_ptr); if (status != UCC_OK) { tl_error(self->super.super.lib, "failed to initialize topology"); goto free_mpool; } /* Update the context's topology pointer to point to the newly created topology */ self->topo = *topo_ptr; } status = ucc_tl_cuda_topo_get_pci_id(self->device, &self->device_id); if (status != UCC_OK) { tl_error(self->super.super.lib, "failed to get pci id for device %d, status: %s", self->device, ucc_status_string(status)); goto free_mpool; } self->ipc_cache = kh_init(tl_cuda_ep_hash); tl_debug(self->super.super.lib, "initialized tl context: %p", self); return UCC_OK; free_mpool: ucc_mpool_cleanup(&self->req_mp, 1); return status; } ucc_status_t ucc_tl_cuda_mem_map(const ucc_base_context_t *context, /* NOLINT */ int type, void *memh, void *tl_h) /* NOLINT */ { return UCC_ERR_NOT_IMPLEMENTED; } ucc_status_t ucc_tl_cuda_mem_unmap(const ucc_base_context_t *context, /* NOLINT */ int type, void *tl_h) /* NOLINT */ { return UCC_ERR_NOT_IMPLEMENTED; } ucc_status_t ucc_tl_cuda_memh_pack(const ucc_base_context_t *context, /* NOLINT */ int type, void *memh, void **pack_buffer) /* NOLINT */ { return UCC_ERR_NOT_IMPLEMENTED; } /** * @brief Cleanup function for CUDA TL context * * This function is responsible for cleaning up resources associated with a CUDA TL context. * It performs the following operations: * 1. Logs the context finalization with debug information * 2. Destroys the IPC cache hash table if it exists * 3. Cleans up topology if it's context-specific (not cached) * 4. Cleans up the request memory pool * * @param self Pointer to the CUDA TL context structure to be cleaned up */ UCC_CLASS_CLEANUP_FUNC(ucc_tl_cuda_context_t) { ucc_tl_cuda_lib_t *lib = ucc_derived_of(self->super.super.lib, ucc_tl_cuda_lib_t); // Log context finalization for debugging purposes tl_debug(self->super.super.lib, "finalizing tl context: %p", self); // Clean up IPC cache if it exists if (self->ipc_cache != NULL) { kh_destroy(tl_cuda_ep_hash, self->ipc_cache); self->ipc_cache = NULL; } // Only destroy topology if it's context-specific (not cached) // For cached topology, it will be destroyed when the library is cleaned up if (self->topo != NULL && !lib->cfg.topo_cache_enable) { ucc_tl_cuda_topo_destroy(self->topo); self->topo = NULL; } // Clean up the request memory pool with force leak check ucc_mpool_cleanup(&self->req_mp, 1); } UCC_CLASS_DEFINE(ucc_tl_cuda_context_t, ucc_tl_context_t); ucc_status_t ucc_tl_cuda_get_context_attr(const ucc_base_context_t *context, /* NOLINT */ ucc_base_ctx_attr_t *attr) { ucc_base_ctx_attr_clear(attr); attr->topo_required = 1; return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/alltoall/0000775000175000017500000000000015211535620020571 5ustar alastairalastairucc-1.8.0/src/components/tl/cuda/alltoall/alltoall.h0000664000175000017500000000065515211535620022554 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLTOALL_H_ #define ALLTOALL_H_ #include "tl_cuda.h" #include "tl_cuda_coll.h" ucc_status_t ucc_tl_cuda_alltoall_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p); #endif ucc-1.8.0/src/components/tl/cuda/alltoall/alltoall_ce.c0000664000175000017500000000520115211535620023206 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "../alltoallv/alltoallv.h" #include "alltoall.h" #include "components/ec/ucc_ec.h" #include "tl_cuda_cache.h" #include "utils/arch/cpu.h" #include "utils/arch/cuda_def.h" //NOLINTNEXTLINE(misc-unused-parameters): cnts and block parameters unused in alltoall variant size_t ucc_tl_cuda_alltoall_get_size(const ucc_tl_cuda_task_t *task, size_t *cnts, ucc_rank_t block) { (void)cnts; /* Unused parameter */ (void)block; /* Unused parameter */ return ucc_dt_size(TASK_ARGS(task).dst.info.datatype) * (TASK_ARGS(task).dst.info.count / UCC_TL_TEAM_SIZE(TASK_TEAM(task))); } size_t ucc_tl_cuda_alltoall_get_offset(const ucc_tl_cuda_task_t *task, size_t *displ, ucc_rank_t block) { (void)displ; /* Unused parameter */ return ucc_dt_size(TASK_ARGS(task).dst.info.datatype) * (TASK_ARGS(task).dst.info.count / UCC_TL_TEAM_SIZE(TASK_TEAM(task))) * block; } ucc_status_t ucc_tl_cuda_alltoall_ce_init(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_tl_cuda_lib_t *lib = UCC_TL_CUDA_TEAM_LIB(team); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_status_t status; size_t data_len; task->alltoallv_ce.get_size = ucc_tl_cuda_alltoall_get_size; task->alltoallv_ce.get_offset = ucc_tl_cuda_alltoall_get_offset; task->alltoallv_ce.sdt = args->src.info.datatype; task->alltoallv_ce.rdt = args->dst.info.datatype; task->alltoallv_ce.sbuf = args->src.info.buffer; task->alltoallv_ce.rbuf = args->dst.info.buffer; task->alltoallv_ce.stage = 0; /* NOT used for alltoall */ task->alltoallv_ce.scnts = 0; task->alltoallv_ce.rcnts = 0; task->alltoallv_ce.sdispl = 0; task->alltoallv_ce.rdispl = 0; data_len = ucc_dt_size(args->src.info.datatype) * args->src.info.count; status = ucc_tl_cuda_mem_info_get( args->src.info.buffer, data_len, &task->alltoallv_ce.mem_info_src); if (ucc_unlikely(status != UCC_OK)) { return status; } if (team->topo->proxy_needed) { status = ucc_tl_cuda_mem_info_get( args->dst.info.buffer, data_len, &task->alltoallv_ce.mem_info_dst); if (ucc_unlikely(status != UCC_OK)) { return status; } } return ucc_tl_cuda_alltoallv_ce_setup_copy_engine( task, lib, "ucc_tl_cuda_alltoall_ce_init"); } ucc-1.8.0/src/components/tl/cuda/alltoall/alltoall.c0000664000175000017500000000236415211535620022546 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "alltoall.h" #include "components/mc/ucc_mc.h" ucc_status_t ucc_tl_cuda_alltoall_ce_init(ucc_tl_cuda_task_t *task); ucc_status_t ucc_tl_cuda_alltoall_ce_start(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_alltoall_ce_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_alltoall_ce_finalize(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_alltoall_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_task_t *task; ucc_status_t status; if (UCC_IS_INPLACE(coll_args->args)) { return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } status = ucc_tl_cuda_alltoall_ce_init(task); if (ucc_unlikely(status != UCC_OK)) { goto free_task; } *task_p = &task->super; return UCC_OK; free_task: ucc_tl_cuda_task_put(task); return status; } ucc-1.8.0/src/components/tl/cuda/tl_cuda_nvls.h0000664000175000017500000000542615211535620021622 0ustar alastairalastair/** * Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_NVLS_H_ #define UCC_TL_CUDA_NVLS_H_ #include #include "ucc/api/ucc_status.h" // Forward declaration to avoid circular dependency struct ucc_tl_cuda_lib; struct ucc_tl_cuda_team; struct ucc_base_context; typedef enum { UCC_TL_CUDA_NVLS_HANDLE_TYPE_POSIX, UCC_TL_CUDA_NVLS_HANDLE_TYPE_FABRIC } ucc_tl_cuda_nvls_handle_type_t; typedef struct { ucc_tl_cuda_nvls_handle_type_t type; /* Rank 0 sets this to UCC_ERR_NOT_SUPPORTED when cuMulticastCreate fails. * Non-root ranks read share_data[0].status to propagate the error instead * of relying on garbage-handle import failure. */ ucc_status_t status; union { struct { pid_t pid; int handle; } posix; CUmemFabricHandle fabric; } data; } ucc_tl_cuda_nvls_handle_t; typedef enum { UCC_TL_CUDA_NVLS_STATE_INIT, UCC_TL_CUDA_NVLS_STATE_SHARE_HANDLES, UCC_TL_CUDA_NVLS_STATE_IMPORT_HANDLE, UCC_TL_CUDA_NVLS_STATE_ADD_DEVICE, UCC_TL_CUDA_NVLS_STATE_BARRIER, } ucc_tl_cuda_nvls_state_t; typedef struct ucc_tl_cuda_nvls { // Multicast handle CUmemGenericAllocationHandle mc_handle; // Multicast memory handle CUmemGenericAllocationHandle mc_memhandle; // Device pointer for multicast CUdeviceptr mc_va; // Device pointer for unicast CUdeviceptr uc_va; // Size of multicast memory size_t mc_size; // Offset of multicast memory size_t mc_offset; // Coll id for each task size_t *coll_ids; // Whether the team is multi-node int is_multinode; // Temporary buffer for allgather ucc_tl_cuda_nvls_handle_t *share_data; // State variables for re-entrant initialization ucc_status_t status_supported; // Handle for export (POSIX or fabric) ucc_tl_cuda_nvls_handle_t local_handle; // CUDA device ID int device; // Minimum granularity size_t minGran; // Granularity size_t gran; /* temporary buffer for STATE_BARRIER */ char *barrier_data; } ucc_tl_cuda_nvls_t; typedef struct ucc_tl_cuda_nvls_control { uint64_t arrival_counter; } ucc_tl_cuda_nvls_control_t; ucc_status_t ucc_tl_cuda_nvls_check_support( struct ucc_tl_cuda_lib *lib, int device, int is_multinode); ucc_status_t ucc_tl_cuda_nvls_init( struct ucc_tl_cuda_team *team, struct ucc_base_context *tl_context); ucc_status_t ucc_tl_cuda_nvls_destroy(struct ucc_tl_cuda_team *team); #endif // UCC_TL_CUDA_NVLS_H_ ucc-1.8.0/src/components/tl/cuda/tl_cuda.h0000664000175000017500000003614715211535620020564 0ustar alastairalastair/** * Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #ifndef UCC_TL_CUDA_H_ #define UCC_TL_CUDA_H_ #include "components/tl/ucc_tl.h" #include "components/tl/ucc_tl_log.h" #include "components/mc/ucc_mc.h" #include "components/cl/ucc_cl_type.h" // for UCC_CL_HIER #include "utils/ucc_mpool.h" #include "utils/ucc_datastruct.h" #include "tl_cuda_ep_hash.h" #include "tl_cuda_topo.h" #include "tl_cuda_team_topo.h" #ifdef HAVE_NVLS #include "tl_cuda_nvls.h" #endif #include #include #ifndef UCC_TL_CUDA_DEFAULT_SCORE #define UCC_TL_CUDA_DEFAULT_SCORE 40 #endif #define UCC_TL_CUDA_MAX_PEERS 8 #define UCC_TL_CUDA_MAX_RING_CHUNKS 8 #ifdef HAVE_NVLS #define UCC_TL_CUDA_MAX_NVLS_PEERS (72 * 8) #define UCC_TL_CUDA_MAX_NVLS_SM_COUNT 32 #define UCC_TL_CUDA_MAX_NVLS_THREADS 1024 #endif /* HAVE_NVLS */ #define UCC_TL_CUDA_SUPPORTED_COLLS \ (UCC_COLL_TYPE_ALLTOALL | UCC_COLL_TYPE_ALLTOALLV | \ UCC_COLL_TYPE_ALLGATHER | UCC_COLL_TYPE_ALLGATHERV | \ UCC_COLL_TYPE_BCAST | UCC_COLL_TYPE_REDUCE_SCATTER | \ UCC_COLL_TYPE_REDUCE_SCATTERV) #define UCC_TL_CUDA_TEAM_LIB(_team) \ (ucc_derived_of((_team)->super.super.context->lib, ucc_tl_cuda_lib_t)) #define UCC_TL_CUDA_TEAM_CTX(_team) \ (ucc_derived_of((_team)->super.super.context, ucc_tl_cuda_context_t)) #define UCC_TL_CUDA_TEAM_SYNC(_team, _rank, _id) \ ({ \ size_t _ctrl_size_rank = \ (sizeof(ucc_tl_cuda_sync_t) + \ sizeof(ucc_tl_cuda_sync_data_t) * (UCC_TL_TEAM_SIZE(_team) - 1)); \ size_t _ctrl_size = _ctrl_size_rank * UCC_TL_TEAM_SIZE(_team); \ void *_sync = PTR_OFFSET(_team->sync, _ctrl_size * (_id) + \ _ctrl_size_rank * (_rank)); \ (ucc_tl_cuda_sync_t *)_sync; \ }) #define UCC_TL_CUDA_TEAM_BARRIER(_team, _id) \ ({ \ size_t _bar_size = sizeof(ucc_tl_cuda_shm_barrier_t); \ void *_bar = PTR_OFFSET(_team->bar, _bar_size * (_id)); \ (ucc_tl_cuda_shm_barrier_t *)_bar; \ }) #ifdef HAVE_PROFILING_TL_CUDA #include "utils/profile/ucc_profile.h" #else #include "utils/profile/ucc_profile_off.h" #endif #define UCC_TL_CUDA_PROFILE_FUNC UCC_PROFILE_FUNC #define UCC_TL_CUDA_PROFILE_REQUEST_NEW UCC_PROFILE_REQUEST_NEW #define UCC_TL_CUDA_PROFILE_REQUEST_EVENT UCC_PROFILE_REQUEST_EVENT #define UCC_TL_CUDA_PROFILE_REQUEST_FREE UCC_PROFILE_REQUEST_FREE typedef struct ucc_tl_cuda_iface { ucc_tl_iface_t super; } ucc_tl_cuda_iface_t; extern ucc_tl_cuda_iface_t ucc_tl_cuda; typedef struct ucc_tl_cuda_lib_config { ucc_tl_lib_config_t super; uint32_t max_concurrent; // Maximum number of tasks that can be progressed simultaneously. size_t scratch_size; // Size of the scratch buffer for each task unsigned long allgather_ring_max_rings; uint32_t allgather_ring_num_chunks; unsigned long reduce_scatter_ring_max_rings; int topo_cache_enable; #ifdef HAVE_NVLS size_t nvls_symmetric_size; // Size of the symmetric memory for NVLS, for each task uint32_t nvls_sm_count; // Number of blocks (SMs) to use for NVLS algorithms uint32_t nvls_threads; // Number of threads per block to use for NVLS algorithms #endif int alltoall_use_copy_engine; } ucc_tl_cuda_lib_config_t; typedef struct ucc_tl_cuda_context_config { ucc_tl_context_config_t super; } ucc_tl_cuda_context_config_t; typedef struct ucc_tl_cuda_lib { ucc_tl_lib_t super; ucc_tl_cuda_lib_config_t cfg; ucc_tl_cuda_topo_t *topo; /* Shared topology information */ } ucc_tl_cuda_lib_t; UCC_CLASS_DECLARE(ucc_tl_cuda_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_cuda_context { ucc_tl_context_t super; ucc_tl_cuda_context_config_t cfg; int device; ucc_tl_cuda_device_pci_id_t device_id; ucc_tl_cuda_topo_t *topo; ucc_mpool_t req_mp; tl_cuda_ep_hash_t *ipc_cache; } ucc_tl_cuda_context_t; UCC_CLASS_DECLARE(ucc_tl_cuda_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); typedef uint32_t ucc_tl_cuda_sync_state_t; #define UCC_TL_CUDA_TAG_FREE 0xFFFFFFFFFFFFFFFF typedef struct ucc_tl_cuda_shm_barrier { ucc_rank_t size; ucc_rank_t count; uint64_t tag; int sense; ucc_status_t state[UCC_TL_CUDA_MAX_PEERS]; int local_sense[UCC_TL_CUDA_MAX_PEERS]; } ucc_tl_cuda_shm_barrier_t; typedef struct ucc_tl_cuda_sync_data { cudaEvent_t ipc_event_remote; } ucc_tl_cuda_sync_data_t; typedef struct ucc_tl_cuda_mem_info { void *ptr; size_t length; size_t offset; cudaIpcMemHandle_t handle; } ucc_tl_cuda_mem_info_t; typedef struct ucc_tl_cuda_rank_id { ucc_tl_cuda_device_pci_id_t pci_id; ucc_tl_cuda_mem_info_t scratch_info; int shm; } ucc_tl_cuda_rank_id_t; typedef struct ucc_tl_cuda_sync { int seq_num[UCC_TL_CUDA_MAX_RING_CHUNKS]; ucc_tl_cuda_mem_info_t mem_info_src; ucc_tl_cuda_mem_info_t mem_info_dst; cudaEvent_t ipc_event_local; cudaIpcEventHandle_t ev_handle; union { struct { size_t sbytes[UCC_TL_CUDA_MAX_PEERS]; size_t rbytes[UCC_TL_CUDA_MAX_PEERS]; size_t sdispl_bytes[UCC_TL_CUDA_MAX_PEERS]; size_t rdispl_bytes[UCC_TL_CUDA_MAX_PEERS]; } alltoallv_ce; }; ucc_tl_cuda_sync_data_t data[1]; } ucc_tl_cuda_sync_t; typedef struct ucc_tl_cuda_scratch { void *loc; void *rem[UCC_TL_CUDA_MAX_PEERS]; ucc_tl_cuda_mem_info_t rem_info[UCC_TL_CUDA_MAX_PEERS]; } ucc_tl_cuda_scratch_t; // Team represents a communicator created within the CUDA context, typically using NVLink for inter-GPU communication typedef struct ucc_tl_cuda_team { ucc_tl_team_t super; uint32_t seq_num; // Counter for the number of launched collective tasks for this team uint32_t seq_num_active_set; // Counter for tasks in the active set (subset of tasks requiring special handling) ucc_tl_cuda_team_topo_t *topo; ucc_tl_cuda_sync_t *sync; // Pointer to shared memory segment for synchronization ucc_tl_cuda_sync_state_t *sync_state; // Tracks the task currently using the sync segment of shared memory, if free - 0 ucc_tl_cuda_shm_barrier_t *bar; // Pointer to the first barrier in an array of size [0; 2 * max_concurrent]. First max_concurrent barriers are for normal mode, the second one for active set mode ucc_tl_cuda_scratch_t scratch; cudaStream_t stream; // CUDA stream for the team ucc_tl_cuda_rank_id_t *ids; ucc_team_oob_coll_t oob; void *oob_req; #ifdef HAVE_NVLS ucc_tl_cuda_nvls_t nvls; ucc_tl_cuda_nvls_state_t state; // State of the nvls initialization #endif } ucc_tl_cuda_team_t; UCC_CLASS_DECLARE(ucc_tl_cuda_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); // Task represents a collective operation that runs in the CUDA context, typically using NVLink for inter-GPU communication typedef struct ucc_tl_cuda_task ucc_tl_cuda_task_t; struct ucc_tl_cuda_task { ucc_coll_task_t super; uint32_t seq_num; // Sequential identifier for each task started within the team uint32_t coll_id; // Index of the collective task in flight, within the range [0; max_concurrent) ucc_tl_cuda_shm_barrier_t *bar; // Pointer to the reserved barrier for this task in the CUDA team ucc_subset_t subset; // Mapping information for the active set, if it is present union { struct { int stage; ucc_tl_cuda_mem_info_t mem_info_src; ucc_tl_cuda_mem_info_t mem_info_dst; void *peer_map_addr_src[UCC_TL_CUDA_MAX_PEERS]; void *peer_map_addr_dst[UCC_TL_CUDA_MAX_PEERS]; int num_posted; ucc_datatype_t sdt; ucc_datatype_t rdt; void *sbuf; void *rbuf; ucc_count_t *scnts; ucc_count_t *rcnts; ucc_aint_t *sdispl; ucc_aint_t *rdispl; void *evt_completion; // CUDA event for completion of the task int use_copy_engine; ucc_ee_executor_task_t *exec_task[UCC_TL_CUDA_MAX_PEERS * UCC_TL_CUDA_MAX_PEERS]; size_t (*get_size)(const ucc_tl_cuda_task_t *task, size_t *bytes, ucc_rank_t block); size_t (*get_offset)(const ucc_tl_cuda_task_t *task, size_t *displ_bytes, ucc_rank_t block); ucc_status_t (*copy_post)(void *dst, const void *src, size_t len, ucc_ee_executor_t *executor, ucc_ee_executor_task_t **task, cudaStream_t stream); } alltoallv_ce; struct { int stage; int num_frags; int num_rings; int num_chunks; ucc_datatype_t dt; void *sbuf; void *rbuf; ucc_ee_executor_task_t *exec_task[2 * UCC_TL_CUDA_MAX_RING_CHUNKS]; size_t (*get_count)(const ucc_tl_cuda_task_t *task, ucc_rank_t block); size_t (*get_offset)(const ucc_tl_cuda_task_t *task, ucc_rank_t block); } allgatherv_ring; struct { int stage; int num_frags; ucc_datatype_t dt; void * sbuf; void * rbuf; ucc_ee_executor_task_t *exec_task[2]; size_t (*get_count)(const ucc_tl_cuda_task_t *task, ucc_rank_t block); size_t (*get_offset)(const ucc_tl_cuda_task_t *task, ucc_rank_t block); } allgatherv_linear; struct { int stage; int step; void *sbuf; ucc_datatype_t dt; ucc_rank_t root; size_t size; int num_steps; ucc_ee_executor_task_t *exec_task; uint64_t key; // This is mix of user provided tag, root and peer to be unique for each task, algorithm uses it to mark barrier as used } bcast_linear; struct { int stage; int num_frags; int num_rings; ucc_datatype_t dt; void *sbuf; void *rbuf; ucc_ee_executor_task_t *exec_task[UCC_TL_CUDA_MAX_RING_CHUNKS]; size_t (*get_count)(const ucc_tl_cuda_task_t *task, ucc_rank_t block); size_t (*get_offset)(const ucc_tl_cuda_task_t *task, ucc_rank_t block); } reduce_scatterv_ring; struct { int stage; int num_frags; ucc_datatype_t dt; void * sbuf; void * rbuf; ucc_ee_executor_task_t *exec_task[2]; size_t (*get_count)(const ucc_tl_cuda_task_t *task, ucc_rank_t block); size_t (*get_offset)(const ucc_tl_cuda_task_t *task, ucc_rank_t block); } reduce_scatterv_linear; #ifdef HAVE_NVLS struct { ucc_datatype_t dt; /* Offset of the current rank in the src buffer */ size_t offset; /* Count of the current rank in the src buffer */ size_t count; void *evt_completion; /* Memory handle for MC symmetric memory */ CUdeviceptr mc_va; /* Memory handle for UC symmetric memory */ CUdeviceptr uc_va; /* Coll id for the NVLS task in flight slot */ size_t coll_id; } reduce_scatterv_nvls; struct { int stage; ucc_datatype_t dt; void *sbuf; void *rbuf; size_t buf_size_bytes; /* Padded size launched on the NVLS kernel; >= buf_size_bytes, * rounded up so each rank gets a full vec/scalar unit. The * tail is zero-filled before the kernel; SUM with zeros is * a no-op so user data is unaffected. */ size_t kernel_size_bytes; /* Memory handle for MC symmetric memory */ CUdeviceptr mc_va; /* Memory handle for UC symmetric memory */ CUdeviceptr uc_va; void *evt_completion; /* Coll id for the NVLS task in flight slot */ size_t coll_id; } allreduce_nvls; struct { /* Offset of the current rank in the dst buffer */ size_t offset; /* Count of the current rank's data */ size_t count; /* Total count of all data (sum of all ranks) */ size_t total_count; void *evt_completion; /* Memory handle for MC symmetric memory */ CUdeviceptr mc_va; /* Memory handle for UC symmetric memory */ CUdeviceptr uc_va; /* Coll id for the NVLS task in flight slot */ size_t coll_id; } allgatherv_nvls; #endif }; }; #endif ucc-1.8.0/src/components/tl/cuda/tl_cuda_team_topo.c0000664000175000017500000004360215211535620022620 0ustar alastairalastair/** * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_cuda_team_topo.h" #include "tl_cuda.h" #include "core/ucc_team.h" #define UCC_TL_CUDA_TEAM_TOPO_SAME_DEVICE ((ucc_rank_t)(UCC_RANK_MAX)) static ucc_status_t ucc_tl_cuda_team_topo_add_ring(const ucc_tl_cuda_team_t *team, ucc_tl_cuda_team_topo_t *topo, ucc_tl_cuda_ring_t *ring, int invert, int num_dups) { ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_tl_cuda_ring_t *new_ring; ucc_status_t status; int i, j; ucc_assert(size > 1); for (i = 0; i < num_dups; i++) { topo->rings[topo->num_rings + i].ring = NULL; topo->rings[topo->num_rings + i].iring = NULL; } for (i = 0; i < num_dups; i++) { new_ring = &topo->rings[topo->num_rings + i]; new_ring->ring = (ucc_rank_t*)ucc_malloc(2 * size * sizeof(ucc_rank_t), "cuda_topo_ring"); new_ring->iring = PTR_OFFSET(new_ring->ring, size * sizeof(ucc_rank_t)); if (!new_ring->ring) { tl_error(UCC_TL_TEAM_LIB(team), "failed to allocate topo ring"); status = UCC_ERR_NO_MEMORY; goto free_rings; } for (j = 0; j < size; j++) { if (invert) { new_ring->ring[j] = ring->ring[size - j - 1]; } else { new_ring->ring[j] = ring->ring[j]; } } for (j = 0; j < size; j++) { new_ring->iring[new_ring->ring[j]] = j; } } topo->num_rings += num_dups; return UCC_OK; free_rings: for (i = 0; i < num_dups; i++) { ucc_free(topo->rings[topo->num_rings + i].ring); } return status; } static ucc_status_t ucc_tl_cuda_team_topo_build_ring(const ucc_tl_cuda_team_t *team, const ucc_rank_t *graph, ucc_tl_cuda_ring_t *ring, ucc_rank_t pos, int width) { ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_rank_t i, j; int in_ring; int links; ucc_status_t status; if (pos == size) { links = graph[ring->ring[pos - 1] * size + ring->ring[0]]; if ((links == UCC_TL_CUDA_TEAM_TOPO_SAME_DEVICE) || (links >= width)) { return UCC_OK; } else { return UCC_ERR_NOT_SUPPORTED; } } for (i = 0; i < size; i++) { links = graph[ring->ring[pos - 1] * size + i]; if ((links < width) && (links != UCC_TL_CUDA_TEAM_TOPO_SAME_DEVICE)) { continue; } in_ring = 0; for (j = 0; j < pos; j++) { if (ring->ring[j] == i) { in_ring = 1; break; } } if (in_ring) { continue; } ring->ring[pos] = i; status = ucc_tl_cuda_team_topo_build_ring(team, graph, ring, pos + 1, width); if (status == UCC_OK) { return UCC_OK; } } return UCC_ERR_NOT_SUPPORTED; } /* TODO: simple algorithm to find NVLink rings. * 1. Find number of rings and minimal ring width * 2. Try to find ring of width W in given team topo. * 3. If found * 3.1. Duplicate ring W/min_width times to get rings of width min_width * 3.2. Duplicate inverted ring W times because NVLink is full duplex * 3.3. Remove ring from topology * 3. W = W/2 goto 2 */ static ucc_status_t ucc_tl_cuda_team_topo_init_rings(const ucc_tl_cuda_team_t *team, ucc_tl_cuda_team_topo_t *topo) { ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_tl_cuda_ring_t ring; int i, width, nr, num_rings, min_width; ucc_status_t status; ucc_rank_t *graph; ucc_assert(size > 1); topo->num_rings = 0; ring.ring = (ucc_rank_t*) ucc_malloc(size * sizeof(ucc_rank_t), "cuda_topo_ring"); if (!ring.ring) { tl_error(UCC_TL_TEAM_LIB(team), "failed to allocate topo ring"); return UCC_ERR_NO_MEMORY; } graph = (ucc_rank_t*) ucc_malloc(size * size * sizeof(ucc_rank_t), "cuda_topo_graph"); if (!graph) { status = UCC_ERR_NO_MEMORY; tl_error(UCC_TL_TEAM_LIB(team), "failed to allocate topo graph"); goto free_ring; } memcpy(graph, topo->matrix, size * size * sizeof(ucc_rank_t)); num_rings = 0; min_width = 4; for (width = min_width; width > 0; width >>= 1) { ring.ring[0] = 0; status = ucc_tl_cuda_team_topo_build_ring(team, graph, &ring, 1, width); if (status == UCC_OK) { num_rings += 2*width; if (width < min_width) { min_width = width; } for (i = 0; i < size; i++) { if (graph[ring.ring[i] * size + ring.ring[(i+1)%size]] != UCC_TL_CUDA_TEAM_TOPO_SAME_DEVICE) { graph[ring.ring[i] * size + ring.ring[(i+1)%size]] -= width; graph[ring.ring[(i+1)%size] * size + ring.ring[i]] -= width; } } } } if (num_rings == 0) { status = UCC_ERR_NOT_SUPPORTED; tl_debug(UCC_TL_TEAM_LIB(team), "no rings found"); goto free_graph; } topo->rings = (ucc_tl_cuda_ring_t*)ucc_malloc(num_rings * sizeof(*topo->rings), "cuda_topo_rings"); if (!topo->rings) { status = UCC_ERR_NO_MEMORY; tl_error(UCC_TL_TEAM_LIB(team), "failed to allocate topo rings array"); goto free_graph; } for (i = 0; i < size * size; i++) { graph[i] = topo->matrix[i]; } for (width = 4; width > 0; width >>= 1) { ring.ring[0] = 0; status = ucc_tl_cuda_team_topo_build_ring(team, graph, &ring, 1, width); if (status == UCC_OK) { nr = width / min_width; status = ucc_tl_cuda_team_topo_add_ring(team, topo, &ring, 0, nr); if (status != UCC_OK) { goto free_rings; } if (size > 2) { status = ucc_tl_cuda_team_topo_add_ring(team, topo, &ring, 1, nr); if (status != UCC_OK) { goto free_rings; } } for (i = 0; i < size; i++) { if (graph[ring.ring[i] * size + ring.ring[(i+1)%size]] != UCC_TL_CUDA_TEAM_TOPO_SAME_DEVICE) { graph[ring.ring[i] * size + ring.ring[(i+1)%size]] -= width; graph[ring.ring[(i+1)%size] * size + ring.ring[i]] -= width; } } } } ucc_free(graph); ucc_free(ring.ring); return UCC_OK; free_rings: for (i = 0; i < topo->num_rings; i++) { ucc_free(topo->rings[i].ring); } ucc_free(topo->rings); free_graph: ucc_free(graph); free_ring: ucc_free(ring.ring); return status; } static ucc_status_t ucc_tl_cuda_team_topo_init_proxies(const ucc_tl_cuda_team_t *team, ucc_tl_cuda_team_topo_t *topo) { ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t num_proxies = 0; ucc_rank_t i, j, k, proxy; float *data; float score, min_score; ucc_status_t status; char pci_str[2][MAX_PCI_BUS_ID_STR]; topo->proxy_needed = 0; for (i = 0; i < size * size; i++) { if (topo->matrix[i] == 0) { num_proxies++; } } topo->num_proxies = num_proxies; topo->is_fully_connected = (num_proxies == 0) ? 1 : 0; if (num_proxies == 0) { return UCC_OK; } topo->proxies = (ucc_tl_cuda_proxy_t*)ucc_malloc( num_proxies * sizeof(ucc_tl_cuda_proxy_t), "cuda_topo_proxies"); if (!topo->proxies) { tl_error(UCC_TL_TEAM_LIB(team), "failed to alloc cuda topo proxies"); return UCC_ERR_NO_MEMORY; } data = (float*)ucc_malloc(size * size * sizeof(float), "cuda topo proxies data"); if (!data) { tl_error(UCC_TL_TEAM_LIB(team), "failed to alloc cuda topo work array"); status = UCC_ERR_NO_MEMORY; goto free_proxy; } for (i = 0; i < size; i++) { for (j = 0; j < size; j++) { if (ucc_tl_cuda_team_topo_is_direct(&team->super, topo, i, j)) { data[i * size + j] = 1.0; } else { data[i * size + j] = 0.0; } } } num_proxies = 0; for (i = 0; i < size; i++) { for (j = 0; j < size; j++) { if (ucc_tl_cuda_team_topo_is_direct(&team->super, topo, i, j)) { continue; } if ((i == rank) || (j == rank)) { topo->proxy_needed = 1; } proxy = UCC_RANK_INVALID; min_score = (float)(UCC_RANK_MAX); for (k = 0; k < size; k++) { if (ucc_tl_cuda_team_topo_is_direct(&team->super, topo, i, k) && ucc_tl_cuda_team_topo_is_direct(&team->super, topo, k, j)) { ucc_assert((topo->matrix[i * size + k] > 0) && (topo->matrix[k * size + j] > 0)); score = ucc_max((data[i * size + k] + 1.0) / topo->matrix[i * size + k], (data[k * size + j] + 1.0) / topo->matrix[k * size + j]); if (score >= min_score) { continue; } proxy = k; min_score = score; } } if (proxy == UCC_RANK_INVALID) { ucc_tl_cuda_topo_pci_id_to_str(&team->ids[i].pci_id, pci_str[0], MAX_PCI_BUS_ID_STR); ucc_tl_cuda_topo_pci_id_to_str(&team->ids[j].pci_id, pci_str[1], MAX_PCI_BUS_ID_STR); tl_debug(UCC_TL_TEAM_LIB(team), "no proxy found between " "dev %s (%d) and dev %s (%d), " "cuda topology is not supported", pci_str[0], i, pci_str[1], j); status = UCC_ERR_NOT_SUPPORTED; goto free_data; } if (proxy == rank) { topo->proxies[num_proxies].src = i; topo->proxies[num_proxies].dst = j; topo->proxies[num_proxies].proxy = proxy; num_proxies++; } data[i * size + proxy] += 1.0; data[proxy * size + j] += 1.0; } } topo->num_proxies = num_proxies; ucc_free(data); return UCC_OK; free_data: ucc_free(data); free_proxy: ucc_free(topo->proxies); return status; } static ucc_status_t ucc_tl_cuda_team_topo_init_matrix(const ucc_tl_cuda_team_t *team, ucc_rank_t *matrix) { ucc_topo_t *topo = UCC_TL_CORE_TEAM(team)->topo; ucc_proc_info_t *procs = topo->topo->procs; ucc_device_id_t *dev_ids = topo->device_map.device_ids; int size = UCC_TL_TEAM_SIZE(team); int i, j; ucc_rank_t ci, cj; ucc_device_id_t di, dj; const ucc_host_info_t *host_i; for (i = 0; i < size; i++) { matrix[i + i * size] = UCC_TL_CUDA_TEAM_TOPO_SAME_DEVICE; ci = ucc_ep_map_eval(topo->set.map, i); di = dev_ids[i]; host_i = &topo->topo->hosts[ci]; for (j = i + 1; j < size; j++) { cj = ucc_ep_map_eval(topo->set.map, j); dj = dev_ids[j]; if (procs[ci].host_hash != procs[cj].host_hash) { /* Cross-node pair: no intra-node NVLink */ matrix[i + j * size] = 0; } else if (di == dj) { matrix[i + j * size] = UCC_TL_CUDA_TEAM_TOPO_SAME_DEVICE; } else { /* Same node (host_hash match above): host_i == host_j, * so looking up nvlink_matrix[di][dj] on host_i is valid. */ matrix[i + j * size] = host_i->nvlink_matrix[di][dj]; } matrix[j + i * size] = matrix[i + j * size]; } } return UCC_OK; } ucc_status_t ucc_tl_cuda_team_topo_create(const ucc_tl_team_t *cuda_team, ucc_tl_cuda_team_topo_t **team_topo) { ucc_tl_cuda_team_t *team = ucc_derived_of(cuda_team, ucc_tl_cuda_team_t); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_tl_cuda_team_topo_t *topo; ucc_status_t status; topo = (ucc_tl_cuda_team_topo_t*)ucc_malloc(sizeof(*topo), "cuda_team_topo"); if (!topo) { tl_error(UCC_TL_TEAM_LIB(team), "failed to alloc cuda team topo"); return UCC_ERR_NO_MEMORY; } topo->matrix = (ucc_rank_t*)ucc_malloc(size * size * sizeof(ucc_rank_t), "cuda_topo_matrix"); if (!topo->matrix) { tl_error(UCC_TL_TEAM_LIB(team), "failed to alloc cuda team topo matrix"); status = UCC_ERR_NO_MEMORY; goto free_topo; } status = ucc_tl_cuda_team_topo_init_matrix(team, topo->matrix); if (status != UCC_OK) { goto free_matrix; } /* Use the authoritative ucc_topo NVLink check to determine full * connectivity. This handles NVSwitch, fabric clique, and direct NVLink * connections consistently and avoids rescanning the matrix for zeros. */ { ucc_topo_t *utopo = UCC_TL_CORE_TEAM(team)->topo; ucc_sbgp_t *node_sg = ucc_topo_get_sbgp(utopo, UCC_SBGP_NODE); topo->is_fully_connected = ucc_topo_is_nvlink_fully_connected(utopo, node_sg); } if (topo->is_fully_connected) { topo->num_proxies = 0; topo->proxy_needed = 0; } else { status = ucc_tl_cuda_team_topo_init_proxies(team, topo); if (status != UCC_OK) { if (status != UCC_ERR_NOT_SUPPORTED) { tl_error(UCC_TL_TEAM_LIB(team), "failed to init cuda topo proxy"); } goto free_matrix; } } status = ucc_tl_cuda_team_topo_init_rings(team, topo); if (status != UCC_OK) { if (status != UCC_ERR_NOT_SUPPORTED) { tl_error(UCC_TL_TEAM_LIB(team), "failed to init cuda topo rings"); } goto free_proxy; } *team_topo = topo; return UCC_OK; free_proxy: if (topo->num_proxies > 0) { ucc_free(topo->proxies); } free_matrix: ucc_free(topo->matrix); free_topo: ucc_free(topo); return status; } void ucc_tl_cuda_team_topo_print_proxies(const ucc_tl_team_t *tl_team, const ucc_tl_cuda_team_topo_t *topo) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t i; char pci_str[3][MAX_PCI_BUS_ID_STR]; for (i = 0; i < size; i++) { if (ucc_tl_cuda_team_topo_is_direct(tl_team, topo, rank, i)) { ucc_tl_cuda_topo_pci_id_to_str(&team->ids[rank].pci_id, pci_str[0], MAX_PCI_BUS_ID_STR); ucc_tl_cuda_topo_pci_id_to_str(&team->ids[i].pci_id, pci_str[1], MAX_PCI_BUS_ID_STR); if (topo->matrix[rank * size +i] == UCC_TL_CUDA_TEAM_TOPO_SAME_DEVICE) { tl_debug(UCC_TL_TEAM_LIB(team), "dev %s (%d) to dev %s (%d): same device", pci_str[0], rank, pci_str[1], i); } else { tl_debug(UCC_TL_TEAM_LIB(team), "dev %s (%d) to dev %s (%d): %d direct links", pci_str[0], rank, pci_str[1], i, topo->matrix[rank * size + i]); } } } for (i = 0; i < topo->num_proxies; i++) { ucc_tl_cuda_topo_pci_id_to_str(&team->ids[topo->proxies[i].src].pci_id, pci_str[0], MAX_PCI_BUS_ID_STR); ucc_tl_cuda_topo_pci_id_to_str(&team->ids[topo->proxies[i].dst].pci_id, pci_str[1], MAX_PCI_BUS_ID_STR); ucc_tl_cuda_topo_pci_id_to_str(&team->ids[topo->proxies[i].proxy].pci_id, pci_str[2], MAX_PCI_BUS_ID_STR); tl_debug(UCC_TL_TEAM_LIB(team), "dev %s (%d) to dev %s (%d): proxy dev %s (%d)", pci_str[0], topo->proxies[i].src, pci_str[1], topo->proxies[i].dst, pci_str[2], topo->proxies[i].proxy); } } void ucc_tl_cuda_team_topo_print_rings(const ucc_tl_team_t *tl_team, const ucc_tl_cuda_team_topo_t *topo) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); int i, j; for (i = 0; i < topo->num_rings; i++) { for (j = 0; j < size; j++) { if (topo->rings[i].ring[j] == rank) { tl_debug(UCC_TL_TEAM_LIB(team), "ring %d: %d send to %d", i, rank, topo->rings[i].ring[(j + 1) % size]); } } } } ucc_status_t ucc_tl_cuda_team_topo_destroy(ucc_tl_cuda_team_topo_t *team_topo) { int i; for (i = 0; i < team_topo->num_rings; i++) { ucc_free(team_topo->rings[i].ring); } ucc_free(team_topo->rings); if (team_topo->num_proxies > 0) { ucc_free(team_topo->proxies); } ucc_free(team_topo->matrix); ucc_free(team_topo); return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/tl_cuda_team.c0000664000175000017500000005633115211535620021562 0ustar alastairalastair/** * Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_cuda.h" #include "tl_cuda_coll.h" #include "tl_cuda_topo.h" #include "tl_cuda_cache.h" #ifdef HAVE_NVLS #include "tl_cuda_nvls.h" #endif #include "core/ucc_team.h" #include "coll_score/ucc_coll_score.h" #include "utils/ucc_math.h" #include "utils/arch/cpu.h" #include "utils/arch/cuda_def.h" #include "utils/ucc_sys.h" #include // Returns supported collectives depending on NVLS availability // and whether the team is single-node or multi-node. static uint64_t ucc_tl_cuda_get_supported_colls(const ucc_tl_cuda_team_t *team) { const int is_multinode = !ucc_team_map_is_single_node( team->super.super.params.team, team->super.super.params.map); #ifdef HAVE_NVLS ucc_status_t status; #endif // Base TL/CUDA collectives that are supported without NVLS uint64_t base_tl_cuda_colls = (UCC_COLL_TYPE_ALLTOALL | UCC_COLL_TYPE_ALLTOALLV | UCC_COLL_TYPE_ALLGATHER | UCC_COLL_TYPE_ALLGATHERV | UCC_COLL_TYPE_BCAST | UCC_COLL_TYPE_REDUCE_SCATTER | UCC_COLL_TYPE_REDUCE_SCATTERV); #ifdef HAVE_NVLS // With NVLS compiled in, ALLREDUCE may be supported via NVLS. // For multi-node teams, advertise ONLY NVLS ALLREDUCE if supported; // otherwise advertise nothing for TL/CUDA (prevent non-NVLS colls). // For single-node teams, advertise base TL/CUDA colls and add ALLREDUCE // only if NVLS is supported. status = ucc_tl_cuda_nvls_check_support( ucc_derived_of(team->super.super.context->lib, ucc_tl_cuda_lib_t), UCC_TL_CUDA_TEAM_CTX(team)->device, is_multinode); if (is_multinode) { return (status == UCC_OK) ? UCC_COLL_TYPE_ALLREDUCE : 0; } return (status == UCC_OK) ? (base_tl_cuda_colls | UCC_COLL_TYPE_ALLREDUCE) : base_tl_cuda_colls; #else if (is_multinode) { // TL/CUDA is not supported for multi-node teams return 0; } return base_tl_cuda_colls; #endif } UCC_CLASS_INIT_FUNC(ucc_tl_cuda_team_t, ucc_base_context_t *tl_context, const ucc_base_team_params_t *params) { ucc_tl_cuda_context_t *ctx = ucc_derived_of(tl_context, ucc_tl_cuda_context_t); ucc_tl_cuda_lib_t *lib = ucc_derived_of(tl_context->lib, ucc_tl_cuda_lib_t); // Number of preallocated resource groups for tasks, including the active set. uint32_t resource_num = lib->cfg.max_concurrent * 2; ucc_tl_cuda_shm_barrier_t *bar; ucc_status_t status; cudaError_t st; int shm_id, i, j; size_t ctrl_size, alloc_size, scratch_size; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_team_t, &ctx->super, params); self->oob = params->params.oob; self->oob_req = NULL; self->stream = NULL; self->topo = NULL; self->scratch.loc = NULL; self->ids = NULL; shm_id = -1; self->sync = (void*)-1; /* Defensive init: ensure remote scratch mapping arrays are NULL/zeroed even if we return early (e.g., multinode NVLS regime or failures) */ for (i = 0; i < UCC_TL_CUDA_MAX_PEERS; i++) { self->scratch.rem[i] = NULL; memset(&self->scratch.rem_info[i], 0, sizeof(self->scratch.rem_info[i])); } #ifdef HAVE_NVLS /* Zero the nvls struct so that ucc_tl_cuda_nvls_destroy() is safe even if we return early below (before nvls_init()/team_create_test() can zero it themselves). All fields are guard-checked (if handle/ptr) in destroy. */ memset(&self->nvls, 0, sizeof(self->nvls)); #endif #ifdef HAVE_NVLS self->state = UCC_TL_CUDA_NVLS_STATE_INIT; if (!ucc_team_map_is_single_node(params->team, params->map)) { tl_debug(tl_context->lib, "nvls multinode team regime"); self->seq_num = 1; return UCC_OK; } #else if (!ucc_team_map_is_single_node(params->team, params->map)) { tl_debug(tl_context->lib, "multinode team is not supported"); return UCC_ERR_NOT_SUPPORTED; } #endif if (UCC_TL_TEAM_SIZE(self) > UCC_TL_CUDA_MAX_PEERS) { tl_debug(tl_context->lib, "team size %u exceeds TL/CUDA max supported peers %d", UCC_TL_TEAM_SIZE(self), UCC_TL_CUDA_MAX_PEERS); return UCC_ERR_NOT_SUPPORTED; } self->ids = ucc_malloc((UCC_TL_TEAM_SIZE(self) + 1) * sizeof(*(self->ids)), "ids"); if (!self->ids) { tl_error(tl_context->lib, "failed to alloc ranks id"); return UCC_ERR_NO_MEMORY; } // active set scratch_size = resource_num * lib->cfg.scratch_size; status = CUDA_FUNC(cudaMalloc(&self->scratch.loc, scratch_size)); if (status != UCC_OK) { tl_error( tl_context->lib, "failed to alloc scratch buffer size: %ld bytes for rank: %d", scratch_size, UCC_TL_TEAM_RANK(self)); goto free_ids; } status = ucc_tl_cuda_mem_info_get(self->scratch.loc, scratch_size, &self->ids[UCC_TL_TEAM_SIZE(self)].scratch_info); if (status != UCC_OK) { tl_error(tl_context->lib, "failed to get scratch memory info"); goto free_scratch; } ctrl_size = (sizeof(ucc_tl_cuda_sync_t) + sizeof(ucc_tl_cuda_sync_data_t) * (UCC_TL_TEAM_SIZE(self) - 1)) * UCC_TL_TEAM_SIZE(self) * lib->cfg.max_concurrent + sizeof(ucc_tl_cuda_shm_barrier_t) * lib->cfg.max_concurrent + sizeof(ucc_tl_cuda_sync_state_t) * lib->cfg.max_concurrent; ctrl_size *= 2; // active sets if (UCC_TL_TEAM_RANK(self) == 0) { alloc_size = ctrl_size; status = ucc_sysv_alloc(&alloc_size, (void**)&self->sync, &shm_id); if (status != UCC_OK) { tl_error(tl_context->lib, "failed to alloc sysv segment"); /* proceed and notify other ranks about error */ shm_id = -1; goto ids_exchange; } memset(self->sync, 0, ctrl_size); self->bar = (ucc_tl_cuda_shm_barrier_t *)UCC_TL_CUDA_TEAM_SYNC( self, 0, resource_num); /* active set */ for (i = 0; i < resource_num; i++) { bar = UCC_TL_CUDA_TEAM_BARRIER(self, i); bar->tag = UCC_TL_CUDA_TAG_FREE; // mark as free for (j = 0; j < UCC_TL_TEAM_SIZE(self); j++) { status = ucc_tl_cuda_shm_barrier_init(UCC_TL_TEAM_SIZE(self), j, bar); if (status != UCC_OK) { tl_error(tl_context->lib, "failed to initialize shm barrier"); ucc_sysv_free(self->sync); shm_id = -1; self->sync = (void*)(-1); /* proceed and notify other ranks about error */ goto ids_exchange; } } } } ids_exchange: self->ids[UCC_TL_TEAM_SIZE(self)].pci_id = ctx->device_id; self->ids[UCC_TL_TEAM_SIZE(self)].shm = shm_id; status = self->oob.allgather(&self->ids[UCC_TL_TEAM_SIZE(self)], self->ids, sizeof(ucc_tl_cuda_rank_id_t), self->oob.coll_info, &self->oob_req); if (UCC_OK != status) { tl_error(tl_context->lib, "failed to start oob allgather"); goto free_devices; } tl_debug(tl_context->lib, "posted tl team: %p", self); self->seq_num = 1; self->seq_num_active_set = 1; return UCC_OK; free_devices: if (shm_id != -1) { ucc_sysv_free(self->sync); self->sync = (void*)(-1); } free_scratch: st = cudaFree(self->scratch.loc); if (st != cudaSuccess) { tl_error( tl_context->lib, "cudaFree failed: %d (%s)", st, cudaGetErrorName(st)); } free_ids: ucc_free(self->ids); return status; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_cuda_team_t) { ucc_tl_cuda_lib_t *lib = ucc_derived_of(self->super.super.context->lib, ucc_tl_cuda_lib_t); // Number of preallocated resource groups for tasks, including the active set. uint32_t resource_num = lib->cfg.max_concurrent * 2; ucc_tl_cuda_sync_t *sync; cudaError_t st; int i, j; tl_debug(self->super.super.context->lib, "finalizing tl team: %p", self); if (self->topo) { ucc_tl_cuda_team_topo_destroy(self->topo); } #ifdef HAVE_NVLS // destroy the nvls context ucc_tl_cuda_nvls_destroy(self); #endif if (self->ids) { if (self->sync != (void*)-1) { for (i = 0; i < resource_num; i++) { /* Remote event handles opened by this rank live in MY sync * block (slot MY_RANK), not in peer j's block. Use the * correct base pointer outside the inner loop. */ sync = UCC_TL_CUDA_TEAM_SYNC(self, UCC_TL_TEAM_RANK(self), i); for (j = 0; j < UCC_TL_TEAM_SIZE(self); j++) { if (j == UCC_TL_TEAM_RANK(self)) { continue; } if (sync->data[j].ipc_event_remote) { st = cudaEventDestroy(sync->data[j].ipc_event_remote); if (st != cudaSuccess) { tl_warn(UCC_TL_TEAM_LIB(self), "cudaEventDestroy " "failed: %d (%s)", st, cudaGetErrorName(st)); } } } sync = UCC_TL_CUDA_TEAM_SYNC(self, UCC_TL_TEAM_RANK(self), i); if (sync->ipc_event_local) { st = cudaEventDestroy(sync->ipc_event_local); if (st != cudaSuccess) { tl_warn(UCC_TL_TEAM_LIB(self), "cudaEventDestroy " "failed: %d (%s)", st, cudaGetErrorName(st)); } } } ucc_sysv_free(self->sync); } ucc_free(self->ids); } if (self->stream) { st = cudaStreamDestroy(self->stream); if (st != cudaSuccess) { tl_warn(UCC_TL_TEAM_LIB(self), "cudaStreamDestroy failed: %d (%s)", st, cudaGetErrorName(st)); } } for (i = 0; i < ucc_min(UCC_TL_TEAM_SIZE(self), UCC_TL_CUDA_MAX_PEERS); i++) { if (self->scratch.rem[i] && self->scratch.rem_info[i].ptr) { ucc_tl_cuda_unmap_memhandle((uintptr_t)self->scratch.rem_info[i].ptr, self->scratch.rem[i], ucc_tl_cuda_get_cache(self, i), 1); } } if (self->scratch.loc) { cudaFree(self->scratch.loc); } } UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_cuda_team_t, ucc_base_team_t); UCC_CLASS_DEFINE(ucc_tl_cuda_team_t, ucc_tl_team_t); ucc_status_t ucc_tl_cuda_team_destroy(ucc_base_team_t *tl_team) { UCC_CLASS_DELETE_FUNC_NAME(ucc_tl_cuda_team_t)(tl_team); return UCC_OK; } ucc_status_t ucc_tl_cuda_team_create_test(ucc_base_team_t *tl_team) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_lib_t *lib = ucc_derived_of(tl_team->context->lib, ucc_tl_cuda_lib_t); // Number of preallocated resource groups for tasks, including the active set. uint32_t resource_num = lib->cfg.max_concurrent * 2; ucc_status_t status; ucc_tl_cuda_sync_t *sync; ucc_tl_cuda_shm_barrier_t *bar; volatile ucc_tl_cuda_sync_t *peer_sync; int i, j, shm_id; #ifdef HAVE_NVLS if (team->state != UCC_TL_CUDA_NVLS_STATE_INIT) { // in that case nvls initialization is in progress goto nvls_init; } if (!ucc_team_map_is_single_node(team->super.super.params.team, team->super.super.params.map)) { // for multinode team skip tl/cuda initialization since it is not supported goto nvls_init; } #endif if (team->oob_req == NULL) { return UCC_OK; } else if (team->oob_req == (void*)0x1) { goto barrier; } status = team->oob.req_test(team->oob_req); if (status == UCC_INPROGRESS) { return UCC_INPROGRESS; } else if (status < 0) { tl_error(tl_team->context->lib, "oob allgather failed"); goto exit_err; } team->oob.req_free(team->oob_req); team->oob_req = (void*)0x1; for (i = 0; i < UCC_TL_TEAM_SIZE(team); i++) { team->scratch.rem[i] = NULL; } if (!ucc_topo_has_device_info(UCC_TL_CORE_TEAM(team)->topo)) { tl_debug(tl_team->context->lib, "not all ranks have visible GPU device info; " "skipping TL/CUDA team creation"); status = UCC_ERR_NOT_SUPPORTED; goto exit_err; } status = ucc_tl_cuda_team_topo_create(&team->super, &team->topo); if (status != UCC_OK) { goto exit_err; } for (i = 0; i < UCC_TL_TEAM_SIZE(team); i++) { if (i == UCC_TL_TEAM_RANK(team) || !ucc_tl_cuda_team_topo_is_direct(&team->super, team->topo, i, UCC_TL_TEAM_RANK(team))) { team->scratch.rem[i] = NULL; continue; } status = ucc_tl_cuda_map_memhandle(team->ids[i].scratch_info.ptr, team->ids[i].scratch_info.length, team->ids[i].scratch_info.handle, &team->scratch.rem[i], ucc_tl_cuda_get_cache(team, i)); if (status != UCC_OK) { tl_error(tl_team->context->lib, "failed to map memhandle"); goto exit_err; } memcpy(&team->scratch.rem_info[i], &team->ids[i].scratch_info, sizeof(ucc_tl_cuda_mem_info_t)); } if (UCC_TL_TEAM_LIB(team)->log_component.log_level >= UCC_LOG_LEVEL_DEBUG) { ucc_tl_cuda_team_topo_print_proxies(&team->super, team->topo); ucc_tl_cuda_team_topo_print_rings(&team->super, team->topo); } shm_id = team->ids[0].shm; if (shm_id < 0) { tl_error(tl_team->context->lib, "failed to create shmem region"); status = UCC_ERR_NO_MEMORY; goto exit_err; } if (UCC_TL_TEAM_RANK(team) != 0) { team->sync = shmat(shm_id, NULL, 0); if (team->sync == (void *)-1) { tl_error(tl_team->context->lib, "failed to shmat errno: %d (%s)", errno, strerror(errno)); status = UCC_ERR_NO_MEMORY; goto exit_err; } team->bar = (ucc_tl_cuda_shm_barrier_t*)UCC_TL_CUDA_TEAM_SYNC(team, 0, resource_num); } team->sync_state = (ucc_tl_cuda_sync_state_t*)PTR_OFFSET(team->bar, sizeof(ucc_tl_cuda_shm_barrier_t) * resource_num); CUDA_CHECK_GOTO(cudaStreamCreateWithFlags(&team->stream, cudaStreamNonBlocking), exit_err, status); for (i = 0; i < resource_num; i++) { sync = UCC_TL_CUDA_TEAM_SYNC(team, UCC_TL_TEAM_RANK(team), i); CUDA_CHECK_GOTO(cudaEventCreateWithFlags(&sync->ipc_event_local, cudaEventDisableTiming | cudaEventInterprocess), exit_err, status); CUDA_CHECK_GOTO(cudaIpcGetEventHandle(&sync->ev_handle, sync->ipc_event_local), exit_err, status); } ucc_memory_cpu_store_fence(); bar = UCC_TL_CUDA_TEAM_BARRIER(team, 0); status = ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), bar); if (status != UCC_OK) { tl_error(tl_team->context->lib, "failed to start shm barrier"); goto exit_err; } barrier: bar = UCC_TL_CUDA_TEAM_BARRIER(team, 0); status = ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), bar); if (status == UCC_INPROGRESS) { return status; } else if (status != UCC_OK) { goto exit_err; } for (i = 0; i < resource_num; i++) { sync = UCC_TL_CUDA_TEAM_SYNC(team, UCC_TL_TEAM_RANK(team), i); for (j = 0 ; j < UCC_TL_TEAM_SIZE(team); j++) { if (j == UCC_TL_TEAM_RANK(team)) { continue; } peer_sync = UCC_TL_CUDA_TEAM_SYNC(team, j, i); CUDA_CHECK_GOTO(cudaIpcOpenEventHandle(&sync->data[j].ipc_event_remote, peer_sync->ev_handle), exit_err, status); } } team->oob_req = NULL; tl_debug(lib, "initialized tl team: %p", team); #ifdef HAVE_NVLS nvls_init: if (team->state == UCC_TL_CUDA_NVLS_STATE_INIT) { // zero out the nvls struct memset(&team->nvls, 0, sizeof(team->nvls)); } // initialize the nvls struct status = ucc_tl_cuda_nvls_init(team, tl_team->context); switch (status) { case UCC_ERR_NOT_SUPPORTED: tl_debug(lib, "NVLS is not supported"); if (!ucc_team_map_is_single_node(team->super.super.params.team, team->super.super.params.map)) { tl_debug(lib, "NVLS is not supported for multi-node team"); return UCC_ERR_NOT_SUPPORTED; } break; case UCC_INPROGRESS: return status; case UCC_OK: break; default: tl_error(lib, "failed to initialize NVLS with status (%d) %s", status, ucc_status_string(status)); // For multi-node teams in NVLS-only mode, no IPC resources were allocated // so just return NOT_SUPPORTED to allow fallback to other TLs if (!ucc_team_map_is_single_node(team->super.super.params.team, team->super.super.params.map)) { return UCC_ERR_NOT_SUPPORTED; } // For single-node teams, NVLS failure means cleanup is needed // overwrite status since NVLS related calls might have failed // and we need to set tl/cuda to not supported status = UCC_ERR_NOT_SUPPORTED; goto exit_err; } #endif return UCC_OK; exit_err: // Clean up CUDA stream if created if (team->stream) { cudaError_t st = cudaStreamDestroy(team->stream); if (st != cudaSuccess) { tl_warn( lib, "cudaStreamDestroy failed during " "cleanup: %d (%s)", st, cudaGetErrorName(st)); } team->stream = NULL; } // Clean up partially created IPC events (local events) // Only access sync structures if shared memory was successfully attached if (team->sync != (void *)-1 && team->sync != NULL) { for (i = 0; i < resource_num; i++) { sync = UCC_TL_CUDA_TEAM_SYNC(team, UCC_TL_TEAM_RANK(team), i); if (sync->ipc_event_local) { cudaError_t st = cudaEventDestroy(sync->ipc_event_local); if (st != cudaSuccess) { tl_warn( lib, "cudaEventDestroy failed during " "cleanup: %d (%s)", st, cudaGetErrorName(st)); } sync->ipc_event_local = NULL; } // Clean up opened remote event handles for (j = 0; j < UCC_TL_TEAM_SIZE(team); j++) { if (j == UCC_TL_TEAM_RANK(team)) { continue; } if (sync->data[j].ipc_event_remote) { cudaError_t st = cudaEventDestroy( sync->data[j].ipc_event_remote); if (st != cudaSuccess) { tl_warn( lib, "cudaEventDestroy failed " "during cleanup: %d (%s)", st, cudaGetErrorName(st)); } sync->data[j].ipc_event_remote = NULL; } } } } // Clean up mapped scratch memory for (i = 0; i < ucc_min(UCC_TL_TEAM_SIZE(team), UCC_TL_CUDA_MAX_PEERS); i++) { if (team->scratch.rem[i] && team->scratch.rem_info[i].ptr) { ucc_tl_cuda_unmap_memhandle( (uintptr_t)team->scratch.rem_info[i].ptr, team->scratch.rem[i], ucc_tl_cuda_get_cache(team, i), 1); team->scratch.rem[i] = NULL; } } // Clean up shared memory if attached by non-root if (UCC_TL_TEAM_RANK(team) != 0 && team->sync != (void*)-1) { if (shmdt(team->sync) != 0) { tl_warn(lib, "shmdt failed during cleanup: %s", strerror(errno)); } team->sync = (void*)-1; } // Reset oob_req state to prevent re-entry into partially initialized code // If we set oob_req to (void*)0x1 but then hit an error, we must reset it // so that subsequent calls to this function will return the error immediately // rather than attempting to continue from the barrier checkpoint if (team->oob_req == (void *)0x1) { team->oob_req = NULL; } return status; } ucc_status_t ucc_tl_cuda_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_base_context_t *ctx = UCC_TL_TEAM_CTX(team); ucc_memory_type_t mt = UCC_MEMORY_TYPE_CUDA; ucc_coll_score_t *score; ucc_status_t status; int i; ucc_coll_score_team_info_t team_info; team_info.alg_fn = ucc_tl_cuda_alg_id_to_init; team_info.default_score = UCC_TL_CUDA_DEFAULT_SCORE; team_info.init = ucc_tl_cuda_coll_init; team_info.num_mem_types = 1; team_info.supported_mem_types = &mt; team_info.supported_colls = ucc_tl_cuda_get_supported_colls(team); team_info.size = UCC_TL_TEAM_SIZE(team); status = ucc_coll_score_build_default(tl_team, UCC_TL_CUDA_DEFAULT_SCORE, ucc_tl_cuda_coll_init, team_info.supported_colls, &mt, 1, &score); if (UCC_OK != status) { return status; } for (i = 0; i < UCC_TL_CUDA_N_DEFAULT_ALG_SELECT_STR; i++) { status = ucc_coll_score_update_from_str( ucc_tl_cuda_default_alg_select_str[i], &team_info, &team->super.super, score); if (UCC_OK != status) { tl_error(tl_team->context->lib, "failed to apply default coll select setting: %s", ucc_tl_cuda_default_alg_select_str[i]); goto err; } } if (strlen(ctx->score_str) > 0) { status = ucc_coll_score_update_from_str(ctx->score_str, &team_info, &team->super.super, score); if ((status < 0) && (status != UCC_ERR_INVALID_PARAM) && (status != UCC_ERR_NOT_SUPPORTED)) { goto err; } } *score_p = score; return UCC_OK; err: ucc_coll_score_free(score); return status; } ucc-1.8.0/src/components/tl/cuda/allreduce/0000775000175000017500000000000015211535620020725 5ustar alastairalastairucc-1.8.0/src/components/tl/cuda/allreduce/allreduce_nvls.c0000664000175000017500000002263615211535620024104 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allreduce/allreduce.h" #include "ucc/api/ucc.h" #include "core/ucc_ee.h" #include "utils/arch/cuda_def.h" #include "tl_cuda_nvls.h" #include "../kernels/allreduce_kernel.h" #include "components/ec/ucc_ec.h" #include "components/ec/cuda/ec_cuda_resources.h" enum { STAGE_KERNEL, /*< Post memcpy to symmetric buffer, launch kernel, memcpy to destination */ STAGE_WAIT, /*< Wait for the copies and kernel to complete */ }; ucc_status_t ucc_tl_cuda_allreduce_nvls_start(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_ee_h ee = task->super.ee; cudaStream_t stream = (ee) ? (cudaStream_t)ee->ee_context : team->stream; // stream is used in tl_trace below, but not used in this function (void)stream; task->allreduce_nvls.buf_size_bytes = args->dst.info.count * ucc_dt_size(task->allreduce_nvls.dt); /* Pad to 16*tsize so chunk_start = count_u32*rank/tsize is 4-aligned for * the MULTIMEM v4.f32 16-byte accesses in vec32 kernels. Padding bytes are * uninitialized and never copied back to the user buffer. */ task->allreduce_nvls.kernel_size_bytes = ucc_align_up( task->allreduce_nvls.buf_size_bytes, 16 * UCC_TL_TEAM_SIZE(team)); task->allreduce_nvls.rbuf = args->dst.info.buffer; task->allreduce_nvls.sbuf = UCC_IS_INPLACE(*args) ? args->dst.info.buffer : args->src.info.buffer; tl_trace( UCC_TASK_LIB(task), "task: %p stream: %p allreduce_nvls_start symmetric uc addr: %p " "mc addr: %p " "buf_size_bytes: %zu, is inplace: %d", task, stream, (void *)task->allreduce_nvls.uc_va, (void *)task->allreduce_nvls.mc_va, task->allreduce_nvls.buf_size_bytes, UCC_IS_INPLACE(*args)); task->allreduce_nvls.stage = STAGE_KERNEL; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } void ucc_tl_cuda_allreduce_nvls_progress(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_ec_cuda_event_t *ec_event = (ucc_ec_cuda_event_t *) task->allreduce_nvls.evt_completion; cudaEvent_t evt = ec_event->event; CUdeviceptr mc_va = task->allreduce_nvls.mc_va; CUdeviceptr uc_va = task->allreduce_nvls.uc_va; ucc_ee_h ee = task->super.ee; cudaStream_t stream = (ee) ? (cudaStream_t)ee->ee_context : team->stream; ucc_datatype_t dt = task->allreduce_nvls.dt; uint32_t sm_count = UCC_TL_CUDA_TEAM_LIB(team)->cfg.nvls_sm_count; uint32_t threads = UCC_TL_CUDA_TEAM_LIB(team)->cfg.nvls_threads; ucc_status_t status; cudaError_t cuda_status; switch (task->allreduce_nvls.stage) { case STAGE_KERNEL: /* Copy the user source into the symmetric buffer. */ status = CUDA_FUNC(cudaMemcpyAsync( (void *)uc_va, task->allreduce_nvls.sbuf, task->allreduce_nvls.buf_size_bytes, cudaMemcpyDeviceToDevice, stream)); if (status != UCC_OK) { task->super.status = status; return; } status = post_allreduce_kernel( stream, sm_count, threads, mc_va, task->allreduce_nvls.kernel_size_bytes, TASK_NVLS_CONTROL_MC(task), TASK_NVLS_CONTROL_UC(task), task->allreduce_nvls.coll_id, trank, UCC_TL_TEAM_SIZE(team), dt); if (status != UCC_OK) { tl_error(UCC_TASK_LIB(task), "failed to post allreduce kernel"); task->super.status = status; return; } /* Copy back only the user data range; padding stays in the * symmetric buffer and is overwritten on the next collective. */ status = CUDA_FUNC(cudaMemcpyAsync( (void *)task->allreduce_nvls.rbuf, (void *)uc_va, task->allreduce_nvls.buf_size_bytes, cudaMemcpyDeviceToDevice, stream)); if (status != UCC_OK) { task->super.status = status; return; } status = CUDA_FUNC(cudaEventRecord(evt, stream)); if (status != UCC_OK) { task->super.status = status; return; } task->allreduce_nvls.stage = STAGE_WAIT; // fallthrough case STAGE_WAIT: cuda_status = cudaEventQuery(evt); if (cuda_status == cudaErrorNotReady) { task->super.status = UCC_INPROGRESS; return; } task->super.status = UCC_OK; break; } } ucc_status_t ucc_tl_cuda_allreduce_nvls_finalize(ucc_coll_task_t *task) { ucc_tl_cuda_task_t *tl_task = ucc_derived_of(task, ucc_tl_cuda_task_t); tl_trace(UCC_TASK_LIB(tl_task), "task: %p allreduce_nvls_finalize", task); ucc_ec_destroy_event( tl_task->allreduce_nvls.evt_completion, UCC_EE_CUDA_STREAM); ucc_tl_cuda_task_put(tl_task); return UCC_OK; } // NOLINTNEXTLINE(misc-unused-parameters): ev parameter unused as it's not needed for this implementation ucc_status_t ucc_tl_cuda_allreduce_nvls_triggered_post( ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_status_t status; ucc_ev_t post_event; ucc_assert(ee->ee_type == UCC_EE_CUDA_STREAM); coll_task->ee = ee; tl_trace(UCC_TASK_LIB(task), "triggered post. task:%p", coll_task); task->allreduce_nvls.stage = STAGE_KERNEL; status = coll_task->post(coll_task); if (ucc_likely(status == UCC_OK)) { post_event.ev_type = UCC_EVENT_COLLECTIVE_POST; post_event.ev_context_size = 0; post_event.ev_context = NULL; post_event.req = &coll_task->super; status = ucc_ee_set_event_internal( coll_task->ee, &post_event, &coll_task->ee->event_out_queue); if (ucc_unlikely(status != UCC_OK)) { tl_error( UCC_TASK_LIB(task), "failed to set EE event: %s", ucc_status_string(status)); return status; } } return status; } ucc_status_t ucc_tl_cuda_allreduce_nvls_init( ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); size_t buf_size = coll_args->args.dst.info.count * ucc_dt_size(coll_args->args.dst.info.datatype); size_t tsize = UCC_TL_TEAM_SIZE(team); size_t kernel_size = ucc_align_up(buf_size, 16 * tsize); ucc_tl_cuda_task_t *task; ucc_status_t status; /* kernel_size must be a multiple of 16*tsize; see allreduce_nvls_start. */ if (buf_size == 0 || coll_args->args.op != UCC_OP_SUM || !ucc_tl_cuda_allreduce_nvls_dt_supported( coll_args->args.dst.info.datatype)) { tl_debug( UCC_TL_TEAM_LIB(team), "NVLS allreduce is supported only with SUM operation " "and float32, bfloat16, int32, uint32, int64, or uint64 " "datatype, with non-zero message size"); return UCC_ERR_NOT_SUPPORTED; } if (ucc_unlikely(kernel_size > UCC_TL_CUDA_TEAM_LIB(team)->cfg.nvls_symmetric_size)) { tl_debug( UCC_TL_TEAM_LIB(team), "NVLS allreduce padded buffer size %zu bytes exceeds " "symmetric buffer size %zu bytes", kernel_size, UCC_TL_CUDA_TEAM_LIB(team)->cfg.nvls_symmetric_size); return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TL_TEAM_LIB(team), "failed to initialize CUDA task"); return status; } status = ucc_ec_create_event( &task->allreduce_nvls.evt_completion, UCC_EE_CUDA_STREAM); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TL_TEAM_LIB(team), "failed to create CUDA event"); ucc_tl_cuda_task_put(task); return status; } task->allreduce_nvls.dt = coll_args->args.dst.info.datatype; tl_trace( UCC_TL_TEAM_LIB(team), "NVLS allreduce datatype: %s", ucc_datatype_str(task->allreduce_nvls.dt)); task->super.post = ucc_tl_cuda_allreduce_nvls_start; task->super.triggered_post = ucc_tl_cuda_allreduce_nvls_triggered_post; task->super.progress = ucc_tl_cuda_allreduce_nvls_progress; task->super.finalize = ucc_tl_cuda_allreduce_nvls_finalize; task->bar = TASK_BAR(task); task->allreduce_nvls.uc_va = (CUdeviceptr)TASK_SYMMETRIC_UC(task); task->allreduce_nvls.mc_va = (CUdeviceptr)TASK_SYMMETRIC_MC(task); task->allreduce_nvls.coll_id = team->nvls.coll_ids[task->coll_id]++; *task_p = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/allreduce/allreduce.h0000664000175000017500000000320215211535620023033 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLREDUCE_H_ #define ALLREDUCE_H_ #include "tl_cuda.h" #include "tl_cuda_coll.h" enum { #ifdef HAVE_NVLS UCC_TL_CUDA_ALLREDUCE_ALG_NVLS, #else UCC_TL_CUDA_ALLREDUCE_ALG_AUTO, #endif /* HAVE_NVLS */ UCC_TL_CUDA_ALLREDUCE_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_cuda_allreduce_algs[UCC_TL_CUDA_ALLREDUCE_ALG_LAST + 1]; #ifdef HAVE_NVLS #define UCC_TL_CUDA_ALLREDUCE_DEFAULT_ALG_SELECT_STR "allreduce:cuda:@0" #else #define UCC_TL_CUDA_ALLREDUCE_DEFAULT_ALG_SELECT_STR "" #endif /* HAVE_NVLS */ ucc_status_t ucc_tl_cuda_allreduce_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_cuda_allreduce_nvls_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); static inline int ucc_tl_cuda_allreduce_nvls_dt_supported(ucc_datatype_t dt) { switch (dt) { case UCC_DT_FLOAT32: case UCC_DT_BFLOAT16: case UCC_DT_INT32: case UCC_DT_UINT32: case UCC_DT_INT64: case UCC_DT_UINT64: return 1; default: return 0; } } static inline int ucc_tl_cuda_allreduce_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_CUDA_ALLREDUCE_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_cuda_allreduce_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/cuda/allreduce/allreduce.c0000664000175000017500000000301115211535620023024 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allreduce.h" #include "tl_cuda.h" #include "utils/arch/cuda_def.h" #include "utils/ucc_coll_utils.h" ucc_base_coll_alg_info_t ucc_tl_cuda_allreduce_algs[UCC_TL_CUDA_ALLREDUCE_ALG_LAST + 1] = { #ifdef HAVE_NVLS [UCC_TL_CUDA_ALLREDUCE_ALG_NVLS] = {.id = UCC_TL_CUDA_ALLREDUCE_ALG_NVLS, .name = "nvls", .desc = "NVLINK SHARP allreduce"}, #else [UCC_TL_CUDA_ALLREDUCE_ALG_AUTO] = {.id = UCC_TL_CUDA_ALLREDUCE_ALG_AUTO, .name = "auto", .desc = "allreduce algorithm is not available without NVLS"}, #endif /* HAVE_NVLS */ [UCC_TL_CUDA_ALLREDUCE_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_cuda_allreduce_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_status_t status = UCC_ERR_NOT_IMPLEMENTED; #ifdef HAVE_NVLS /* Use NVLS algorithm as default */ status = ucc_tl_cuda_allreduce_nvls_init(coll_args, team, task_h); #else (void) coll_args; (void) team; (void) task_h; #endif /* HAVE_NVLS */ return status; } ucc-1.8.0/src/components/tl/cuda/Makefile.am0000664000175000017500000000474715211535620021035 0ustar alastairalastair# # Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Copyright (c) Meta Platforms, Inc. and affiliates. 2022. # if TL_CUDA_ENABLED SUBDIRS = kernels allgather = \ allgather/allgather.h \ allgather/allgather.c \ allgather/allgather_ring.c \ allgather/allgather_linear.c allgatherv = \ allgatherv/allgatherv.h \ allgatherv/allgatherv.c \ allgatherv/allgatherv_ring.c \ allgatherv/allgatherv_linear.c alltoall = \ alltoall/alltoall.h \ alltoall/alltoall.c \ alltoall/alltoall_ce.c alltoallv = \ alltoallv/alltoallv.h \ alltoallv/alltoallv.c \ alltoallv/alltoallv_ce.c bcast = \ bcast/bcast.h \ bcast/bcast.c \ bcast/bcast_linear.c reduce_scatter = \ reduce_scatter/reduce_scatter.h \ reduce_scatter/reduce_scatter.c \ reduce_scatter/reduce_scatter_ring.c \ reduce_scatter/reduce_scatter_linear.c reduce_scatterv = \ reduce_scatterv/reduce_scatterv.h \ reduce_scatterv/reduce_scatterv.c \ reduce_scatterv/reduce_scatterv_ring.c \ reduce_scatterv/reduce_scatterv_linear.c allreduce = \ allreduce/allreduce.h \ allreduce/allreduce.c if TL_CUDA_NVLS_ENABLED reduce_scatter += reduce_scatter/reduce_scatter_nvls.c reduce_scatterv += reduce_scatterv/reduce_scatterv_nvls.c allreduce += allreduce/allreduce_nvls.c allgatherv += allgatherv/allgatherv_nvls.c endif sources = \ tl_cuda.h \ tl_cuda.c \ tl_cuda_lib.c \ tl_cuda_context.c \ tl_cuda_team.c \ tl_cuda_coll.c \ tl_cuda_cache.c \ tl_cuda_topo.c \ tl_cuda_team_topo.c \ $(allgather) \ $(allgatherv) \ $(alltoall) \ $(alltoallv) \ $(bcast) \ $(allreduce) \ $(reduce_scatter) \ $(reduce_scatterv) if TL_CUDA_NVLS_ENABLED sources += tl_cuda_nvls.h tl_cuda_nvls.c endif module_LTLIBRARIES = libucc_tl_cuda.la libucc_tl_cuda_la_SOURCES = $(sources) libucc_tl_cuda_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) libucc_tl_cuda_la_CFLAGS = $(BASE_CFLAGS) libucc_tl_cuda_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(CUDA_LDFLAGS) -lstdc++ libucc_tl_cuda_la_LIBADD = $(CUDA_LIBS) $(NVML_LIBS) $(UCC_TOP_BUILDDIR)/src/libucc.la kernels/libucc_tl_cuda_kernels.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/tl/cuda/bcast/0000775000175000017500000000000015211535620020061 5ustar alastairalastairucc-1.8.0/src/components/tl/cuda/bcast/bcast_linear.c0000664000175000017500000004103615211535620022657 0ustar alastairalastair/** * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "bcast.h" enum { // Barrier setup stages STAGE_INIT_BAR_ROOT, // Initial stage for the root rank to identify and claim a free barrier STAGE_FIND_BAR_PEER, // Stage where peer ranks wait while the root rank identifies a free barrier STAGE_SYNC, // Initialize the barrier and synchronize the segment required for the current task STAGE_SETUP, // Verify that all ranks are aligned and have reached the barrier // Stages specific to the root rank STAGE_COPY, // Post copy task: copy data block from src to a scratch buffer STAGE_WAIT_COPY, // The root waits for the completion of its copy operation STAGE_WAIT_ALL, // The root rank waits until all other ranks have reached the same operational step STAGE_WAIT_COMPLETION, // The root rank waits for all other ranks to complete the broadcast operation // non-root STAGE_WAIT_ROOT, // Wait while the root rank writes data to its scratch buffer STAGE_CLIENT_COPY, // Initiate their own copy tasks after the root's operations STAGE_CLIENT_COPY_WAIT, // Wait for the completion of the copy operation from the root's scratch buffer STAGE_CLIENT_WAIT_COMPLETION, // Wait for the completion of algorithm on all ranks, global sync with root }; static inline ucc_status_t ucc_tl_cuda_bcast_linear_setup_start(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); set_rank_step(task, trank, 0, 0); // Initialize rank step tracking ucc_memory_cpu_store_fence(); // initiate barrier wait while all ranks set theirs steps to 0 return ucc_tl_cuda_shm_barrier_start(trank, task->bar); } // Tests if setup is complete for a linear broadcast task static inline ucc_status_t ucc_tl_cuda_bcast_linear_setup_test(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); return ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); } // Returns the size of the scratch buffer used for data transfers static inline size_t get_raw_scratch_size(ucc_tl_cuda_team_t *team) { return UCC_TL_CUDA_TEAM_LIB(team)->cfg.scratch_size; } // Posts a copy task to the CUDA executor static inline ucc_status_t ecopy(void *dst, void *src, size_t size, ucc_ee_executor_t *exec, ucc_ee_executor_task_t **etask) { ucc_ee_executor_task_args_t exec_args = {0}; exec_args.task_type = UCC_EE_EXECUTOR_TASK_COPY; exec_args.copy.dst = dst; exec_args.copy.src = src; exec_args.copy.len = size; return ucc_ee_executor_task_post(exec, &exec_args, etask); } // Root rank searches for and claims a free barrier static inline ucc_status_t root_find_free_barrier(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); uint32_t max_concurrent = UCC_TL_CUDA_TEAM_LIB(team)->cfg.max_concurrent; ucc_tl_cuda_shm_barrier_t *curr_bar; int i; ucc_status_t st; // Iterate over available barriers in active set pool to find a free one for (i = 0; i < max_concurrent; ++i) { curr_bar = UCC_TL_CUDA_TEAM_BARRIER(team, max_concurrent + i); // try to set user specified tag to mark that this barrier is used by this task if (ucc_atomic_cswap64(&curr_bar->tag, UCC_TL_CUDA_TAG_FREE, task->bcast_linear.key) == UCC_TL_CUDA_TAG_FREE) { ucc_debug("Acquire barrier: %p idx: %d marked with tag: %ld", curr_bar, i, curr_bar->tag); task->bar = curr_bar; st = ucc_tl_cuda_shm_barrier_init_root( task->subset.map.ep_num, task->subset.myrank, task->bcast_linear.root, task->bar); if (ucc_unlikely(st != UCC_OK)) { ucc_error("failed to init root barrier"); return UCC_ERR_NO_RESOURCE; } // Assign a collective ID (index of barrier) task->coll_id = i + max_concurrent; return UCC_OK; } } // try next time return UCC_ERR_NOT_FOUND; } // Peer rank searches for a barrier claimed by the root static inline ucc_status_t peer_find_free_barrier(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); uint32_t max_concurrent = UCC_TL_CUDA_TEAM_LIB(team)->cfg.max_concurrent; ucc_tl_cuda_shm_barrier_t *curr_bar; int i; ucc_status_t st; for (i = 0; i < max_concurrent; ++i) { curr_bar = UCC_TL_CUDA_TEAM_BARRIER(team, max_concurrent + i); // Check if the barrier is claimed by the task's root if (curr_bar->tag == task->bcast_linear.key) { task->bar = curr_bar; st = ucc_tl_cuda_shm_barrier_init_root( task->subset.map.ep_num, task->subset.myrank, task->bcast_linear.root, task->bar); if (ucc_unlikely(st != UCC_OK)) { ucc_error("failed to init peer barrier"); return UCC_ERR_NO_RESOURCE; } task->coll_id = i + max_concurrent; return UCC_OK; } } // try next time return UCC_ERR_NOT_FOUND; } static ucc_status_t ucc_tl_cuda_bcast_linear_finalize(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); tl_trace(UCC_TASK_LIB(task), "finalizing task %p", task); ucc_tl_cuda_task_put(task); return UCC_OK; } static void ucc_tl_cuda_bcast_linear_progress(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); size_t half_scratch_size = get_raw_scratch_size(team) / 2; ucc_rank_t tsize = UCC_COLL_ARGS_ACTIVE_SET(&TASK_ARGS(task)) ? (ucc_rank_t)task->subset.map.ep_num : UCC_TL_TEAM_SIZE(team); size_t chunk_size = ucc_min( half_scratch_size, task->bcast_linear.size - task->bcast_linear.step * half_scratch_size); size_t offset_buff = task->bcast_linear.step * half_scratch_size; ucc_ee_executor_t *exec; ucc_ee_executor_task_t *etask; void *sbuf, *dbuf; ucc_rank_t peer; ucc_status_t st; int i; task->super.status = UCC_INPROGRESS; st = ucc_coll_task_get_executor(&task->super, &exec); if (ucc_unlikely(st != UCC_OK)) { task->super.status = st; return; } switch (task->bcast_linear.stage) { case STAGE_INIT_BAR_ROOT: st = root_find_free_barrier(task); if (st == UCC_OK) { task->bcast_linear.stage = STAGE_SYNC; } else if (st != UCC_ERR_NOT_FOUND) { task->super.status = st; } // no free barriers found, try next time return; case STAGE_FIND_BAR_PEER: st = peer_find_free_barrier(task); if (st == UCC_OK) { // barrier found, continue to next stages task->bcast_linear.stage = STAGE_SYNC; } else if (st != UCC_ERR_NOT_FOUND) { task->super.status = st; } // no free barriers found by root, try next time return; case STAGE_SYNC: if (ucc_tl_cuda_get_sync_root(task, task->bcast_linear.root) != UCC_OK) { return; } task->bcast_linear.step = 0; st = ucc_tl_cuda_bcast_linear_setup_start(task); if (st != UCC_OK) { task->super.status = st; return; } task->bcast_linear.stage = STAGE_SETUP; /* fall through */ case STAGE_SETUP: st = ucc_tl_cuda_bcast_linear_setup_test(task); if (st != UCC_OK) { task->super.status = st; return; } if (trank == task->bcast_linear.root) { task->bcast_linear.stage = STAGE_COPY; } else { task->bcast_linear.stage = STAGE_WAIT_ROOT; } /* fall through */ default: break; } if (trank == task->bcast_linear.root) { // Root scenario // fall-through between cases is intentional switch (task->bcast_linear.stage) { case STAGE_COPY: // copy from src buffer to scratch dbuf = PTR_OFFSET(TASK_SCRATCH(task, trank), task->bcast_linear.step % 2 * half_scratch_size); sbuf = PTR_OFFSET(task->bcast_linear.sbuf, offset_buff); st = ecopy(dbuf, sbuf, chunk_size, exec, &task->bcast_linear.exec_task); if (st != UCC_OK) { ucc_error("failed to post ecopy task"); task->super.status = st; return; } task->bcast_linear.stage = STAGE_WAIT_COPY; /* fall through */ case STAGE_WAIT_COPY: etask = task->bcast_linear.exec_task; ucc_assert(NULL != etask); st = ucc_ee_executor_task_test(etask); if (st != UCC_OK) { return; // not ready } ucc_ee_executor_task_finalize(etask); task->bcast_linear.exec_task = NULL; // signal others ++task->bcast_linear.step; set_rank_step(task, task->bcast_linear.root, task->bcast_linear.step, 0); task->bcast_linear.stage = STAGE_WAIT_ALL; /* fall through */ case STAGE_WAIT_ALL: for (i = 0; i < tsize; ++i) { if (UCC_COLL_ARGS_ACTIVE_SET(&TASK_ARGS(task))) { // eval phys rank from virt peer = ucc_ep_map_eval(task->subset.map, i); } else { peer = i; } // need to wait until all ranks complete step - 1, because of double buffering if (get_rank_step(task, peer, 0) < task->bcast_linear.step - 1) { // rank is not ready, lets wait return; } } if (task->bcast_linear.step < task->bcast_linear.num_steps) { // go to next iteration task->bcast_linear.stage = STAGE_COPY; return; } // finish st = ucc_tl_cuda_shm_barrier_start(trank, task->bar); if (ucc_unlikely(st != UCC_OK)) { ucc_error("failed to start barrier from root rank"); task->super.status = st; return; } task->bcast_linear.stage = STAGE_WAIT_COMPLETION; /* fall through */ case STAGE_WAIT_COMPLETION: st = ucc_tl_cuda_shm_barrier_test(trank, task->bar); if (st != UCC_OK) { // peers still working, lets check next time task->super.status = st; return; } // set barrier free to unlock others, this is roots responsibility ucc_debug("Release bar: %p with tag: %ld", task->bar, task->bar->tag); task->bar->tag = UCC_TL_CUDA_TAG_FREE; ucc_tl_cuda_put_sync_root(task, task->bcast_linear.root); task->super.status = UCC_OK; break; default: ucc_assert(0); break; } } else { // clients // fall-through between cases is intentional switch (task->bcast_linear.stage) { case STAGE_WAIT_ROOT: if (get_rank_step(task, task->bcast_linear.root, 0) > task->bcast_linear.step) { task->bcast_linear.stage = STAGE_CLIENT_COPY; break; } else { return; } /* fall through */ case STAGE_CLIENT_COPY: // need to copy from root's scratch buffer dbuf = PTR_OFFSET(task->bcast_linear.sbuf, offset_buff); sbuf = PTR_OFFSET(TASK_SCRATCH(task, task->bcast_linear.root), task->bcast_linear.step % 2 * half_scratch_size); st = ecopy(dbuf, sbuf, chunk_size, exec, &task->bcast_linear.exec_task); if (st != UCC_OK) { ucc_error("failed to post ecopy task at client"); task->super.status = st; return; } task->bcast_linear.stage = STAGE_CLIENT_COPY_WAIT; /* fall through */ case STAGE_CLIENT_COPY_WAIT: etask = task->bcast_linear.exec_task; ucc_assert(NULL != etask); st = ucc_ee_executor_task_test(etask); if (st != UCC_OK) { return; // executor task is not ready } ucc_ee_executor_task_finalize(etask); task->bcast_linear.exec_task = NULL; ++task->bcast_linear.step; set_rank_step(task, trank, task->bcast_linear.step, 0); if (task->bcast_linear.step < task->bcast_linear.num_steps) { task->bcast_linear.stage = STAGE_WAIT_ROOT; return; } // start barrier to sync with root st = ucc_tl_cuda_shm_barrier_start(trank, task->bar); if (ucc_unlikely(st != UCC_OK)) { ucc_error("failed to start barrier from peer rank"); task->super.status = st; return; } task->bcast_linear.stage = STAGE_CLIENT_WAIT_COMPLETION; /* fall through */ case STAGE_CLIENT_WAIT_COMPLETION: st = ucc_tl_cuda_shm_barrier_test(trank, task->bar); if (st != UCC_OK) { // someone still working, lets check next time task->super.status = st; return; } task->super.status = UCC_OK; break; default: ucc_assert(0); break; } } } static ucc_status_t ucc_tl_cuda_bcast_linear_start(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); task->bcast_linear.step = 0; task->bcast_linear.stage = STAGE_SYNC; // in case of active set bcast we need to do additional steps to find free barriers if (UCC_COLL_ARGS_ACTIVE_SET(&TASK_ARGS(task))) { task->bcast_linear.stage = UCC_TL_TEAM_RANK(team) == task->bcast_linear.root ? STAGE_INIT_BAR_ROOT : STAGE_FIND_BAR_PEER; } ucc_debug("bcast linear start dt: %s, buffer size: %ld, num_steps: %d", ucc_datatype_str(task->bcast_linear.dt), task->bcast_linear.size, task->bcast_linear.num_steps); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_cuda_bcast_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_task_t *task; ucc_status_t status; if (!ucc_tl_cuda_team_topo_is_fully_connected(team->topo) || UCC_TL_TEAM_SIZE(team) - 1 > UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS) { return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->bcast_linear.root = coll_args->args.root; task->bcast_linear.dt = coll_args->args.src.info.datatype; task->bcast_linear.sbuf = coll_args->args.src.info.buffer; task->bcast_linear.size = ucc_dt_size(task->bcast_linear.dt) * coll_args->args.src.info.count; task->bcast_linear.num_steps = ucc_div_round_up( task->bcast_linear.size, get_raw_scratch_size(team) / 2); task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_cuda_bcast_linear_start; task->super.progress = ucc_tl_cuda_bcast_linear_progress; task->super.finalize = ucc_tl_cuda_bcast_linear_finalize; *task_p = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/bcast/bcast.c0000664000175000017500000000202415211535620021317 0ustar alastairalastair/** * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "bcast.h" #include "components/mc/ucc_mc.h" ucc_base_coll_alg_info_t ucc_tl_cuda_bcast_algs[UCC_TL_CUDA_BCAST_ALG_LAST + 1] = { [UCC_TL_CUDA_BCAST_ALG_LINEAR] = {.id = UCC_TL_CUDA_BCAST_ALG_LINEAR, .name = "linear", .desc = "linear bcast algorithm"}, [UCC_TL_CUDA_BCAST_ALG_LAST] = {.id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_cuda_bcast_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); if (ucc_tl_cuda_team_topo_is_fully_connected(team->topo)) { return ucc_tl_cuda_bcast_linear_init(coll_args, tl_team, task_p); } else { return UCC_ERR_NOT_SUPPORTED; } } ucc-1.8.0/src/components/tl/cuda/bcast/bcast.h0000664000175000017500000000215515211535620021331 0ustar alastairalastair/** * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef BCAST_H_ #define BCAST_H_ #include "tl_cuda.h" #include "tl_cuda_coll.h" enum { UCC_TL_CUDA_BCAST_ALG_LINEAR, UCC_TL_CUDA_BCAST_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_cuda_bcast_algs[UCC_TL_CUDA_BCAST_ALG_LAST + 1]; #define UCC_TL_CUDA_BCAST_DEFAULT_ALG_SELECT_STR "bcast:cuda:@0" ucc_status_t ucc_tl_cuda_bcast_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p); ucc_status_t ucc_tl_cuda_bcast_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p); static inline int ucc_tl_cuda_bcast_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_CUDA_BCAST_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_cuda_bcast_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/cuda/reduce_scatter/0000775000175000017500000000000015211535620021761 5ustar alastairalastairucc-1.8.0/src/components/tl/cuda/reduce_scatter/reduce_scatter.c0000664000175000017500000000461115211535620025123 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatter.h" #include "components/mc/ucc_mc.h" ucc_base_coll_alg_info_t ucc_tl_cuda_reduce_scatter_algs[UCC_TL_CUDA_REDUCE_SCATTER_ALG_LAST + 1] = { [UCC_TL_CUDA_REDUCE_SCATTER_ALG_AUTO] = {.id = UCC_TL_CUDA_REDUCE_SCATTER_ALG_AUTO, .name = "auto", .desc = "choose reduce scatter algorithm based on CUDA topology"}, [UCC_TL_CUDA_REDUCE_SCATTER_ALG_RING] = {.id = UCC_TL_CUDA_REDUCE_SCATTER_ALG_RING, .name = "ring", .desc = "multiring reduce scatter algorithm"}, [UCC_TL_CUDA_REDUCE_SCATTER_ALG_LINEAR] = {.id = UCC_TL_CUDA_REDUCE_SCATTER_ALG_LINEAR, .name = "linear", .desc = "linear reduce scatter algorithm"}, #ifdef HAVE_NVLS [UCC_TL_CUDA_REDUCE_SCATTER_ALG_NVLS] = {.id = UCC_TL_CUDA_REDUCE_SCATTER_ALG_NVLS, .name = "nvls", .desc = "nvls reduce scatter algorithm"}, #endif /* HAVE_NVLS */ [UCC_TL_CUDA_REDUCE_SCATTER_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; size_t ucc_tl_cuda_reduce_scatter_get_count(const ucc_tl_cuda_task_t *task, ucc_rank_t block) //NOLINT { const ucc_coll_args_t *args = &TASK_ARGS(task); size_t count = args->dst.info.count; if (UCC_IS_INPLACE(*args)) { count = args->dst.info.count / UCC_TL_TEAM_SIZE(TASK_TEAM(task)); } return count; } size_t ucc_tl_cuda_reduce_scatter_get_offset(const ucc_tl_cuda_task_t *task, ucc_rank_t block) { return ucc_tl_cuda_reduce_scatter_get_count(task, block) * block; } ucc_status_t ucc_tl_cuda_reduce_scatter_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); if (ucc_tl_cuda_team_topo_is_fully_connected(team->topo)) { return ucc_tl_cuda_reduce_scatter_linear_init(coll_args, tl_team, task_p); } else { return ucc_tl_cuda_reduce_scatter_ring_init(coll_args, tl_team, task_p); } } ucc-1.8.0/src/components/tl/cuda/reduce_scatter/reduce_scatter_nvls.c0000664000175000017500000000106015211535620026160 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatterv/reduce_scatterv.h" #include "reduce_scatter/reduce_scatter.h" ucc_status_t ucc_tl_cuda_reduce_scatter_nvls_init( ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { return ucc_tl_cuda_reduce_scatterv_nvls_init_common( coll_args, tl_team, task_p, ucc_tl_cuda_reduce_scatter_get_offset, ucc_tl_cuda_reduce_scatter_get_count); } ucc-1.8.0/src/components/tl/cuda/reduce_scatter/reduce_scatter.h0000664000175000017500000000435315211535620025133 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef REDUCE_SCATTER_H_ #define REDUCE_SCATTER_H_ #include "tl_cuda.h" #include "tl_cuda_coll.h" enum { UCC_TL_CUDA_REDUCE_SCATTER_ALG_AUTO, UCC_TL_CUDA_REDUCE_SCATTER_ALG_RING, UCC_TL_CUDA_REDUCE_SCATTER_ALG_LINEAR, #ifdef HAVE_NVLS UCC_TL_CUDA_REDUCE_SCATTER_ALG_NVLS, #endif /* HAVE_NVLS */ UCC_TL_CUDA_REDUCE_SCATTER_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_cuda_reduce_scatter_algs[UCC_TL_CUDA_REDUCE_SCATTER_ALG_LAST + 1]; #define UCC_TL_CUDA_REDUCE_SCATTER_DEFAULT_ALG_SELECT_STR "reduce_scatter:cuda:@0" size_t ucc_tl_cuda_reduce_scatter_get_count(const ucc_tl_cuda_task_t *task, ucc_rank_t block); size_t ucc_tl_cuda_reduce_scatter_get_offset(const ucc_tl_cuda_task_t *task, ucc_rank_t block); ucc_status_t ucc_tl_cuda_reduce_scatter_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p); ucc_status_t ucc_tl_cuda_reduce_scatter_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p); ucc_status_t ucc_tl_cuda_reduce_scatter_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p); #ifdef HAVE_NVLS ucc_status_t ucc_tl_cuda_reduce_scatter_nvls_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p); #endif /* HAVE_NVLS */ static inline int ucc_tl_cuda_reduce_scatter_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_CUDA_REDUCE_SCATTER_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_cuda_reduce_scatter_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/cuda/reduce_scatter/reduce_scatter_ring.c0000664000175000017500000000377315211535620026152 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatterv/reduce_scatterv.h" #include "reduce_scatter/reduce_scatter.h" ucc_status_t ucc_tl_cuda_reduce_scatter_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); size_t ssize = UCC_TL_CUDA_TEAM_LIB(team)->cfg.scratch_size; ucc_datatype_t dt = coll_args->args.dst.info.datatype; ucc_tl_cuda_task_t *task; size_t send_size, frag_size; ucc_status_t status; if (coll_args->args.op == UCC_OP_AVG) { return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->reduce_scatterv_ring.get_count = ucc_tl_cuda_reduce_scatter_get_count; task->reduce_scatterv_ring.get_offset = ucc_tl_cuda_reduce_scatter_get_offset; task->reduce_scatterv_ring.dt = coll_args->args.dst.info.datatype; task->reduce_scatterv_ring.sbuf = coll_args->args.src.info.buffer; task->reduce_scatterv_ring.rbuf = coll_args->args.dst.info.buffer; send_size = task->reduce_scatterv_ring.get_count(task, 0); frag_size = ucc_min(ssize / ucc_dt_size(dt) / 2, send_size); task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->reduce_scatterv_ring.num_frags = ucc_div_round_up(send_size, frag_size); task->super.post = ucc_tl_cuda_reduce_scatterv_ring_start; task->super.progress = ucc_tl_cuda_reduce_scatterv_ring_progress; task->super.finalize = ucc_tl_cuda_reduce_scatterv_ring_finalize; task->bar = TASK_BAR(task); *task_p = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/reduce_scatter/reduce_scatter_linear.c0000664000175000017500000000321415211535620026453 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatterv/reduce_scatterv.h" #include "reduce_scatter/reduce_scatter.h" ucc_status_t ucc_tl_cuda_reduce_scatter_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_task_t *task; ucc_status_t status; if (coll_args->args.op == UCC_OP_AVG) { return UCC_ERR_NOT_SUPPORTED; } if (ucc_unlikely(!ucc_tl_cuda_team_topo_is_fully_connected(team->topo) || UCC_TL_TEAM_SIZE(team) - 1 > UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS)) { return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->reduce_scatterv_linear.get_count = ucc_tl_cuda_reduce_scatter_get_count; task->reduce_scatterv_linear.get_offset = ucc_tl_cuda_reduce_scatter_get_offset; task->reduce_scatterv_linear.dt = coll_args->args.dst.info.datatype; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_cuda_reduce_scatterv_linear_start; task->super.progress = ucc_tl_cuda_reduce_scatterv_linear_progress; task->super.finalize = ucc_tl_cuda_reduce_scatterv_linear_finalize; task->bar = TASK_BAR(task); *task_p = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/allgather/0000775000175000017500000000000015211535620020730 5ustar alastairalastairucc-1.8.0/src/components/tl/cuda/allgather/allgather.c0000664000175000017500000000363315211535620023044 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allgather.h" #include "components/mc/ucc_mc.h" ucc_base_coll_alg_info_t ucc_tl_cuda_allgather_algs[UCC_TL_CUDA_ALLGATHER_ALG_LAST + 1] = { [UCC_TL_CUDA_ALLGATHER_ALG_AUTO] = {.id = UCC_TL_CUDA_ALLGATHER_ALG_AUTO, .name = "auto", .desc = "choose allgather algorithm based on CUDA topology"}, [UCC_TL_CUDA_ALLGATHER_ALG_RING] = {.id = UCC_TL_CUDA_ALLGATHER_ALG_RING, .name = "ring", .desc = "multiring allgather algorithm"}, [UCC_TL_CUDA_ALLGATHER_ALG_LINEAR] = {.id = UCC_TL_CUDA_ALLGATHER_ALG_LINEAR, .name = "linear", .desc = "linear allgather algorithm"}, [UCC_TL_CUDA_ALLGATHER_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; size_t ucc_tl_cuda_allgather_get_count(const ucc_tl_cuda_task_t *task, ucc_rank_t block) //NOLINT: block is unused { return TASK_ARGS(task).dst.info.count / UCC_TL_TEAM_SIZE(TASK_TEAM(task)); } size_t ucc_tl_cuda_allgather_get_offset(const ucc_tl_cuda_task_t *task, ucc_rank_t block) { return (TASK_ARGS(task).dst.info.count / UCC_TL_TEAM_SIZE(TASK_TEAM(task))) * block; } ucc_status_t ucc_tl_cuda_allgather_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); if (ucc_tl_cuda_team_topo_is_fully_connected(team->topo)) { return ucc_tl_cuda_allgather_linear_init(coll_args, tl_team, task_p); } else { return ucc_tl_cuda_allgather_ring_init(coll_args, tl_team, task_p); } } ucc-1.8.0/src/components/tl/cuda/allgather/allgather.h0000664000175000017500000000336615211535620023054 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLGATHER_H_ #define ALLGATHER_H_ #include "tl_cuda.h" #include "tl_cuda_coll.h" enum { UCC_TL_CUDA_ALLGATHER_ALG_AUTO, UCC_TL_CUDA_ALLGATHER_ALG_RING, UCC_TL_CUDA_ALLGATHER_ALG_LINEAR, UCC_TL_CUDA_ALLGATHER_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_cuda_allgather_algs[UCC_TL_CUDA_ALLGATHER_ALG_LAST + 1]; #define UCC_TL_CUDA_ALLGATHER_DEFAULT_ALG_SELECT_STR "allgather:cuda:@0" size_t ucc_tl_cuda_allgather_get_count(const ucc_tl_cuda_task_t *task, ucc_rank_t block); size_t ucc_tl_cuda_allgather_get_offset(const ucc_tl_cuda_task_t *task, ucc_rank_t block); ucc_status_t ucc_tl_cuda_allgather_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p); ucc_status_t ucc_tl_cuda_allgather_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p); ucc_status_t ucc_tl_cuda_allgather_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p); static inline int ucc_tl_cuda_allgather_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_CUDA_ALLGATHER_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_cuda_allgather_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/cuda/allgather/allgather_ring.c0000664000175000017500000000266215211535620024064 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allgatherv/allgatherv.h" #include "allgather/allgather.h" ucc_status_t ucc_tl_cuda_allgather_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_task_t *task; ucc_status_t status; status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->allgatherv_ring.get_count = ucc_tl_cuda_allgather_get_count; task->allgatherv_ring.get_offset = ucc_tl_cuda_allgather_get_offset; task->allgatherv_ring.dt = coll_args->args.dst.info.datatype; task->allgatherv_ring.sbuf = coll_args->args.src.info.buffer; task->allgatherv_ring.rbuf = coll_args->args.dst.info.buffer; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_cuda_allgatherv_ring_start; task->super.progress = ucc_tl_cuda_allgatherv_ring_progress; task->super.finalize = ucc_tl_cuda_allgatherv_ring_finalize; task->bar = TASK_BAR(task); *task_p = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/allgather/allgather_linear.c0000664000175000017500000000315315211535620024373 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allgatherv/allgatherv.h" #include "allgather/allgather.h" ucc_status_t ucc_tl_cuda_allgather_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_task_t *task; ucc_status_t status; if (ucc_unlikely(!ucc_tl_cuda_team_topo_is_fully_connected(team->topo) || UCC_TL_TEAM_SIZE(team) - 1 > UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS)) { return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->allgatherv_linear.get_count = ucc_tl_cuda_allgather_get_count; task->allgatherv_linear.get_offset = ucc_tl_cuda_allgather_get_offset; task->allgatherv_linear.dt = coll_args->args.dst.info.datatype; task->allgatherv_linear.sbuf = coll_args->args.src.info.buffer; task->allgatherv_linear.rbuf = coll_args->args.dst.info.buffer; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_cuda_allgatherv_linear_start; task->super.progress = ucc_tl_cuda_allgatherv_linear_progress; task->super.finalize = ucc_tl_cuda_allgatherv_linear_finalize; task->bar = TASK_BAR(task); *task_p = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/tl_cuda_lib.c0000664000175000017500000001004115211535620021366 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_cuda.h" #ifdef HAVE_NVLS #include "tl_cuda_nvls.h" #endif /* NOLINTNEXTLINE params is not used*/ UCC_CLASS_INIT_FUNC(ucc_tl_cuda_lib_t, const ucc_base_lib_params_t *params, const ucc_base_config_t *config) { const ucc_tl_cuda_lib_config_t *tl_config = ucc_derived_of(config, ucc_tl_cuda_lib_config_t); size_t min_scratch_size; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_lib_t, &ucc_tl_cuda.super, &tl_config->super); memcpy(&self->cfg, tl_config, sizeof(*tl_config)); if (self->cfg.allgather_ring_num_chunks < 1) { self->cfg.allgather_ring_num_chunks = 1; } if (self->cfg.allgather_ring_num_chunks > UCC_TL_CUDA_MAX_RING_CHUNKS) { self->cfg.allgather_ring_num_chunks = UCC_TL_CUDA_MAX_RING_CHUNKS; } #ifdef HAVE_NVLS if (self->cfg.nvls_sm_count < 1) { tl_error( &self->super, "nvls_sm_count is too small, min is 1, please check NVLS_SM_COUNT " "config parameter"); return UCC_ERR_INVALID_PARAM; } if (self->cfg.nvls_sm_count > UCC_TL_CUDA_MAX_NVLS_SM_COUNT) { tl_error( &self->super, "nvls_sm_count is too large, max is %d, please check NVLS_SM_COUNT " "config parameter", UCC_TL_CUDA_MAX_NVLS_SM_COUNT); return UCC_ERR_INVALID_PARAM; } if (self->cfg.nvls_threads < 1) { tl_error( &self->super, "nvls_threads is too small, min is 1, please check NVLS_THREADS " "config parameter"); return UCC_ERR_INVALID_PARAM; } if (self->cfg.nvls_threads > UCC_TL_CUDA_MAX_NVLS_THREADS) { tl_error( &self->super, "nvls_threads is too large, max is %d, please check NVLS_THREADS " "config parameter", UCC_TL_CUDA_MAX_NVLS_THREADS); return UCC_ERR_INVALID_PARAM; } #endif /* min scratch size should be large enough so that * ucc_align_down_pow2(scratch_size / nrings / nchunks / dt_size / 2, 64) > 1 */ min_scratch_size = 128 * 16 * ucc_dt_size(UCC_DT_FLOAT128_COMPLEX) * self->cfg.allgather_ring_num_chunks; if (self->cfg.scratch_size < min_scratch_size) { self->cfg.scratch_size = min_scratch_size; } /* Initialize topology pointer to NULL - will be lazily initialized */ self->topo = NULL; tl_debug(&self->super, "initialized lib object: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_cuda_lib_t) { if (self->topo) { ucc_tl_cuda_topo_destroy(self->topo); } tl_debug(&self->super, "finalizing lib object: %p", self); } UCC_CLASS_DEFINE(ucc_tl_cuda_lib_t, ucc_tl_lib_t); ucc_status_t ucc_tl_cuda_get_lib_attr(const ucc_base_lib_t *lib, /* NOLINT */ ucc_base_lib_attr_t *base_attr) { ucc_tl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_tl_lib_attr_t); attr->super.attr.thread_mode = UCC_THREAD_MULTIPLE; attr->super.attr.coll_types = UCC_TL_CUDA_SUPPORTED_COLLS; #ifdef HAVE_NVLS attr->super.attr.coll_types |= UCC_COLL_TYPE_ALLREDUCE; #endif attr->super.flags = 0; if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MIN_TEAM_SIZE) { attr->super.min_team_size = lib->min_team_size; } if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MAX_TEAM_SIZE) { #ifdef HAVE_NVLS /* Advertise maximum possible team size. Actual NVLS support * will be checked during team creation. */ attr->super.max_team_size = UCC_TL_CUDA_MAX_NVLS_PEERS; #else attr->super.max_team_size = UCC_TL_CUDA_MAX_PEERS; #endif } return UCC_OK; } ucc_status_t ucc_tl_cuda_get_lib_properties(ucc_base_lib_properties_t *prop) { prop->default_team_size = 2; prop->min_team_size = 2; #ifdef HAVE_NVLS prop->max_team_size = UCC_TL_CUDA_MAX_NVLS_PEERS; #else prop->max_team_size = UCC_TL_CUDA_MAX_PEERS; #endif return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/tl_cuda_nvls.c0000664000175000017500000007010615211535620021612 0ustar alastairalastair/** * Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_cuda_nvls.h" #include "tl_cuda.h" #include "tl_cuda_coll.h" #include "core/ucc_team.h" #include "utils/arch/cuda_def.h" #include "utils/ucc_math.h" #include // for pidfd_open and pidfd_getfd #include // for prctl() #include // for close() /* RHEL 8 glibc headers (kernel 4.18) don't define pidfd syscall numbers */ #ifndef SYS_pidfd_open #define SYS_pidfd_open 434 #endif #ifndef SYS_pidfd_getfd #define SYS_pidfd_getfd 438 #endif ucc_status_t ucc_tl_cuda_nvls_check_support( ucc_tl_cuda_lib_t *lib, int device, int is_multinode) { int multicast_supported, fabric_supported; ucc_status_t status; status = CUDADRV_FUNC(cuDeviceGetAttribute( &multicast_supported, CU_DEVICE_ATTRIBUTE_MULTICAST_SUPPORTED, device)); if (status != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } status = CUDADRV_FUNC(cuDeviceGetAttribute( &fabric_supported, CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_FABRIC_SUPPORTED, device)); if (status != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } tl_debug( lib, "MULTICAST_SUPPORTED: %d, HANDLE_TYPE_FABRIC_SUPPORTED: %d\n", multicast_supported, fabric_supported); if (!multicast_supported) { tl_debug(lib, "multicast not supported on device %d", device); return UCC_ERR_NOT_SUPPORTED; } if (is_multinode && !fabric_supported) { tl_debug(lib, "fabric handle not supported on device %d", device); return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } static ucc_status_t ucc_tl_cuda_nvls_get_granularity( CUmulticastObjectProp *mc_prop, size_t *min_gran, size_t *gran) { ucc_status_t status; status = CUDADRV_FUNC(cuMulticastGetGranularity( min_gran, mc_prop, CU_MULTICAST_GRANULARITY_MINIMUM)); if (status != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } status = CUDADRV_FUNC(cuMulticastGetGranularity( gran, mc_prop, CU_MULTICAST_GRANULARITY_RECOMMENDED)); if (status != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } static ucc_status_t ucc_tl_cuda_nvls_create_multicast_object_posix( CUmulticastObjectProp *mc_prop, CUmemGenericAllocationHandle *mc_handle, int *export_handle) { ucc_status_t status; status = CUDADRV_FUNC(cuMulticastCreate(mc_handle, mc_prop)); if (status != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } // For POSIX file descriptors, CUDA API writes the fd to the memory pointed to by the first arg // The API expects void* but will write an int-sized value status = CUDADRV_FUNC(cuMemExportToShareableHandle( (void *)export_handle, *mc_handle, mc_prop->handleTypes, 0)); if (status != UCC_OK) { CUDADRV_FUNC(cuMemRelease(*mc_handle)); return status; } return UCC_OK; } static ucc_status_t ucc_tl_cuda_nvls_create_multicast_object_fabric( CUmulticastObjectProp *mc_prop, CUmemGenericAllocationHandle *mc_handle, CUmemFabricHandle *export_handle) { ucc_status_t status; status = CUDADRV_FUNC(cuMulticastCreate(mc_handle, mc_prop)); if (status != UCC_OK) { return UCC_ERR_NOT_SUPPORTED; } status = CUDADRV_FUNC(cuMemExportToShareableHandle( export_handle, *mc_handle, mc_prop->handleTypes, 0)); if (status != UCC_OK) { CUDADRV_FUNC(cuMemRelease(*mc_handle)); return status; } return UCC_OK; } static ucc_status_t ucc_tl_cuda_nvls_share_handles( ucc_tl_cuda_team_t *team, ucc_tl_cuda_nvls_handle_t *local_handle) { ucc_status_t status; // If oob_req is NULL, initiate the allgather if (team->oob_req == NULL) { // Prepare local data based on handle type // In both cases, only rank 0 creates and exports the handle if (UCC_TL_TEAM_RANK(team) == 0) { // Copy the local handle to the share buffer team->nvls.share_data[0] = *local_handle; } // Initiate single allgather for handle data status = team->oob.allgather( team->nvls.share_data, team->nvls.share_data, sizeof(ucc_tl_cuda_nvls_handle_t), team->oob.coll_info, &team->oob_req); if (status != UCC_OK) { tl_error( UCC_TL_TEAM_LIB(team), "failed to initiate allgather for handles (type: %s)", local_handle->type == UCC_TL_CUDA_NVLS_HANDLE_TYPE_FABRIC ? "fabric" : "posix"); ucc_free(team->nvls.share_data); team->nvls.share_data = NULL; return status; } } // Test the request status status = team->oob.req_test(team->oob_req); // If still in progress, return immediately (non-blocking) if (status > 0) { return UCC_INPROGRESS; } // If error occurred if (status < 0) { tl_error( UCC_TL_TEAM_LIB(team), "failed to test allgather for handles (type: %s)", local_handle->type == UCC_TL_CUDA_NVLS_HANDLE_TYPE_FABRIC ? "fabric" : "posix"); goto cleanup_on_error; } // Clean up team->oob.req_free(team->oob_req); team->oob_req = NULL; return UCC_OK; cleanup_on_error: ucc_free(team->nvls.share_data); team->nvls.share_data = NULL; team->oob.req_free(team->oob_req); team->oob_req = NULL; return status; } static ucc_status_t ucc_tl_cuda_nvls_import_handle_posix( struct ucc_tl_cuda_team *team, ucc_tl_cuda_nvls_handle_t *share_data, CUmemGenericAllocationHandle *mc_handle) { void *os_handle; int pid_fd, peer_fd; ucc_status_t status; int export_handle; pid_t target_pid; if (share_data->type != UCC_TL_CUDA_NVLS_HANDLE_TYPE_POSIX) { tl_error( UCC_TL_TEAM_LIB(team), "expected POSIX handle type but got fabric handle type"); return UCC_ERR_INVALID_PARAM; } export_handle = share_data->data.posix.handle; target_pid = share_data->data.posix.pid; pid_fd = syscall(SYS_pidfd_open, target_pid, 0); if (pid_fd < 0) { tl_error( UCC_TL_TEAM_LIB(team), "failed to open pidfd for pid %d", target_pid); return UCC_ERR_NO_RESOURCE; } peer_fd = syscall(SYS_pidfd_getfd, pid_fd, export_handle, 0); if (peer_fd < 0) { tl_error( UCC_TL_TEAM_LIB(team), "failed to get peer fd: %s (errno=%d)", strerror(errno), errno); close(pid_fd); return UCC_ERR_NO_RESOURCE; } os_handle = (void *)((uint64_t)peer_fd); status = CUDADRV_FUNC(cuMemImportFromShareableHandle( mc_handle, os_handle, CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR)); if (close(peer_fd) != 0) { tl_error( UCC_TL_TEAM_LIB(team), "failed to close peer_fd: %s (errno=%d)", strerror(errno), errno); } if (close(pid_fd) != 0) { tl_error( UCC_TL_TEAM_LIB(team), "failed to close pid_fd: %s (errno=%d)", strerror(errno), errno); } if (status != UCC_OK) { tl_error( UCC_TL_TEAM_LIB(team), "failed to import POSIX file descriptor handle from rank 0"); return status; } return UCC_OK; } static ucc_status_t ucc_tl_cuda_nvls_import_handle_fabric( struct ucc_tl_cuda_team *team, ucc_tl_cuda_nvls_handle_t *share_data, CUmemGenericAllocationHandle *mc_handle) { ucc_status_t status; if (share_data->type != UCC_TL_CUDA_NVLS_HANDLE_TYPE_FABRIC) { tl_error( UCC_TL_TEAM_LIB(team), "expected fabric handle type but got POSIX handle type"); return UCC_ERR_INVALID_PARAM; } status = CUDADRV_FUNC(cuMemImportFromShareableHandle( mc_handle, &share_data->data.fabric, CU_MEM_HANDLE_TYPE_FABRIC)); if (status != UCC_OK) { tl_error( UCC_TL_TEAM_LIB(team), "failed to import fabric handle from rank 0. status (%d) %s", status, ucc_status_string(status)); return status; } return UCC_OK; } ucc_status_t ucc_tl_cuda_nvls_init( struct ucc_tl_cuda_team *team, struct ucc_base_context *tl_context) { ucc_tl_cuda_context_t *ctx = ucc_derived_of( tl_context, ucc_tl_cuda_context_t); ucc_tl_cuda_lib_t *lib = ucc_derived_of(tl_context->lib, ucc_tl_cuda_lib_t); ucc_tl_cuda_nvls_t *nvls = &team->nvls; const size_t symmetric_size = lib->cfg.max_concurrent * (lib->cfg.nvls_symmetric_size + NVLS_CONTROL_SIZE); size_t mc_size = 0; CUdeviceptr uc_va = 0; CUdeviceptr mc_va = 0; ucc_status_t status = UCC_OK; CUmemGenericAllocationHandle mc_handle = 0; CUmemGenericAllocationHandle mem_handle = 0; CUmulticastObjectProp mc_prop = {}; CUmemAllocationProp prop = {}; CUmemAccessDesc accessDesc = {}; CUmemAllocationHandleType handle_types; switch (team->state) { case UCC_TL_CUDA_NVLS_STATE_INIT: // Initialize nvls struct to ensure safe cleanup on error memset(nvls, 0, sizeof(*nvls)); // Get current device from context nvls->device = ctx->device; nvls->is_multinode = !ucc_team_map_is_single_node( team->super.super.params.team, team->super.super.params.map); status = ucc_tl_cuda_nvls_check_support( lib, nvls->device, nvls->is_multinode); if (status != UCC_OK) { return status; } if (UCC_TL_TEAM_SIZE(team) > UCC_TL_CUDA_MAX_NVLS_PEERS) { tl_warn(lib, "NVLS: team size %u exceeds maximum supported peers %d; " "disabling NVLS for this team", UCC_TL_TEAM_SIZE(team), UCC_TL_CUDA_MAX_NVLS_PEERS); return UCC_ERR_NOT_SUPPORTED; } if (nvls->is_multinode) { ucc_team_t *ucc_team = UCC_TL_CORE_TEAM(team); ucc_rank_t min_ppn, max_ppn; if (!ucc_topo_is_single_nvlink_domain(ucc_team->topo)) { tl_warn(lib, "NVLS: multinode team does not share a single NVLink " "fabric domain (mismatched clique IDs, different NVL " "partitions, or fabric info unavailable); " "run with UCC_LOG_LEVEL=DEBUG for details; " "disabling multinode NVLS"); return UCC_ERR_NOT_SUPPORTED; } min_ppn = ucc_topo_min_ppn(ucc_team->topo); max_ppn = ucc_topo_max_ppn(ucc_team->topo); if (min_ppn != max_ppn) { tl_warn(lib, "NVLS: non-uniform ppn across nodes " "(min_ppn=%u max_ppn=%u); disabling multinode NVLS", min_ppn, max_ppn); return UCC_ERR_NOT_SUPPORTED; } } handle_types = nvls->is_multinode ? CU_MEM_HANDLE_TYPE_FABRIC : CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR; // Initialize multicast properties mc_prop.numDevices = UCC_TL_TEAM_SIZE(team); mc_prop.size = symmetric_size; mc_prop.handleTypes = handle_types; mc_prop.flags = 0; tl_debug( UCC_TL_TEAM_LIB(team), "RANK %d: numDevices: %d, size: %zu, handleTypes: %lld, " "flags: %lld\n", UCC_TL_TEAM_RANK(team), mc_prop.numDevices, mc_prop.size, mc_prop.handleTypes, mc_prop.flags); // Get granularity requirements status = ucc_tl_cuda_nvls_get_granularity( &mc_prop, &nvls->minGran, &nvls->gran); if (status != UCC_OK) { return status; } if (UCC_TL_TEAM_RANK(team) == 0) { tl_debug( UCC_TL_TEAM_LIB(team), "NVLS multicast granularity: gran = %lu, minGran = %lu\n", nvls->gran, nvls->minGran); } // Calculate aligned size mc_size = ucc_align_up(symmetric_size, nvls->gran); mc_prop.size = mc_size; nvls->mc_size = mc_size; // Store for later use // Allocate buffer for gathering data from all ranks nvls->share_data = (ucc_tl_cuda_nvls_handle_t *)ucc_malloc( sizeof(ucc_tl_cuda_nvls_handle_t) * UCC_TL_TEAM_SIZE(team), "nvls_share_data"); if (!nvls->share_data) { return UCC_ERR_NO_MEMORY; } // Create multicast object on rank 0 if (UCC_TL_TEAM_RANK(team) == 0) { // Initialize local handle type nvls->local_handle.type = nvls->is_multinode ? UCC_TL_CUDA_NVLS_HANDLE_TYPE_FABRIC : UCC_TL_CUDA_NVLS_HANDLE_TYPE_POSIX; if (nvls->is_multinode) { status = ucc_tl_cuda_nvls_create_multicast_object_fabric( &mc_prop, &mc_handle, &nvls->local_handle.data.fabric); } else { status = ucc_tl_cuda_nvls_create_multicast_object_posix( &mc_prop, &mc_handle, &nvls->local_handle.data.posix.handle); } if (status != UCC_OK) { tl_error( UCC_TL_TEAM_LIB(team), "failed to create multicast object. status (%d) %s", status, ucc_status_string(status)); /* Keep going to unblock peers waiting in the allgather; * propagate the error via the status field of the shared * handle so non-root ranks detect the failure explicitly. */ nvls->status_supported = UCC_ERR_NOT_SUPPORTED; } else { nvls->status_supported = UCC_OK; } /* Embed the error status into the handle that will be allgathered * so every non-root rank can detect rank-0 failure directly. */ nvls->local_handle.status = nvls->status_supported; // Store PID for POSIX handles if (!nvls->is_multinode) { nvls->local_handle.data.posix.pid = getpid(); // Allow peer processes to use pidfd_getfd on this process // A more aggressive solution would be to modify Yama ptrace policy by // running `echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope` or to run // the docker container with `--cap-add=SYS_PTRACE` and `--sysctl // kernel.yama.ptrace_scope=0`. if (prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY) != 0) { tl_warn( UCC_TL_TEAM_LIB(team), "failed to set PR_SET_PTRACER: %s (errno=%d). " "This may cause pidfd_getfd to fail on peer processes. " "Consider adjusting Yama ptrace_scope settings: `echo " "0 | sudo tee /proc/sys/kernel/yama/ptrace_scope` or " "to run " "the docker container with `--cap-add=SYS_PTRACE` and " "`--sysctl kernel.yama.ptrace_scope=0`", strerror(errno), errno); } } nvls->mc_handle = mc_handle; } team->state = UCC_TL_CUDA_NVLS_STATE_SHARE_HANDLES; // fall through case UCC_TL_CUDA_NVLS_STATE_SHARE_HANDLES: // Share handles across ranks using unified function status = ucc_tl_cuda_nvls_share_handles(team, &nvls->local_handle); if (status == UCC_INPROGRESS) { return status; } if (status != UCC_OK) { goto cleanup; } team->state = UCC_TL_CUDA_NVLS_STATE_IMPORT_HANDLE; // fall through case UCC_TL_CUDA_NVLS_STATE_IMPORT_HANDLE: /* Non-root ranks check the status field broadcast by rank 0 before * attempting to import a potentially garbage handle. */ if (UCC_TL_TEAM_RANK(team) != 0) { if (nvls->share_data[0].status != UCC_OK) { tl_warn(UCC_TL_TEAM_LIB(team), "NVLS: rank 0 failed to create multicast object " "(status=%d); disabling NVLS for this team", nvls->share_data[0].status); status = nvls->share_data[0].status; nvls->status_supported = status; goto cleanup; } if (nvls->is_multinode) { status = ucc_tl_cuda_nvls_import_handle_fabric( team, &nvls->share_data[0], &mc_handle); } else { status = ucc_tl_cuda_nvls_import_handle_posix( team, &nvls->share_data[0], &mc_handle); } if (status != UCC_OK) { goto cleanup; } nvls->mc_handle = mc_handle; } if (nvls->status_supported != UCC_OK) { // Propagate the supported status to the caller status = nvls->status_supported; goto cleanup; } team->state = UCC_TL_CUDA_NVLS_STATE_ADD_DEVICE; // fall through case UCC_TL_CUDA_NVLS_STATE_ADD_DEVICE: { // Allocate physical memory prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; prop.location.id = nvls->device; prop.requestedHandleTypes = nvls->is_multinode ? CU_MEM_HANDLE_TYPE_FABRIC : CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR; mc_size = nvls->mc_size; status = CUDADRV_FUNC(cuMemCreate(&mem_handle, mc_size, &prop, 0)); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to create memory allocation for multicast"); goto cleanup; } // Set up memory access descriptor accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; accessDesc.location.id = nvls->device; accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; // Reserve and map unicast virtual address space status = CUDADRV_FUNC( cuMemAddressReserve(&uc_va, mc_size, nvls->minGran, 0U, 0)); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to reserve virtual address space"); goto cleanup; } status = CUDADRV_FUNC(cuMemMap(uc_va, mc_size, 0, mem_handle, 0)); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to map memory allocation"); goto cleanup; } status = CUDADRV_FUNC(cuMemSetAccess(uc_va, mc_size, &accessDesc, 1)); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to set memory access"); goto cleanup; } /* Allocate and zero coll_ids */ nvls->coll_ids = (size_t *)ucc_malloc( lib->cfg.max_concurrent * sizeof(size_t), "coll_ids"); if (!nvls->coll_ids) { status = UCC_ERR_NO_MEMORY; goto cleanup; } memset(nvls->coll_ids, 0, lib->cfg.max_concurrent * sizeof(size_t)); /* ALL ranks zero their control region via UC VA BEFORE cuMulticastAddDevice. * cuMulticastBindAddr is a collective barrier; when it returns every rank * has finished its memset, so arrival_counter starts at 0 everywhere. */ tl_debug(UCC_TL_TEAM_LIB(team), "NVLS init: rank %d initialising control regions " "uc_va=%p symm_size=%zu ctrl_size=%d slots=%u", UCC_TL_TEAM_RANK(team), (void *)uc_va, lib->cfg.nvls_symmetric_size, NVLS_CONTROL_SIZE, lib->cfg.max_concurrent); CUDA_CHECK_GOTO(cudaMemset2D( (void *)(uc_va + lib->cfg.nvls_symmetric_size), lib->cfg.nvls_symmetric_size + NVLS_CONTROL_SIZE, 0, NVLS_CONTROL_SIZE, lib->cfg.max_concurrent), cleanup, status); /* Add device to multicast object */ status = CUDADRV_FUNC( cuMulticastAddDevice(nvls->mc_handle, nvls->device)); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to add device to multicast"); goto cleanup; } tl_debug(UCC_TL_TEAM_LIB(team), "RANK %d: added device %d to multicast", UCC_TL_TEAM_RANK(team), nvls->device); /* Bind memory to multicast object; blocks until all ranks call * cuMulticastAddDevice, acting as a collective barrier. */ status = CUDADRV_FUNC(cuMulticastBindAddr( nvls->mc_handle, 0 /*mcOffset*/, uc_va, mc_size, 0)); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to bind memory to multicast"); goto cleanup; } // Reserve and map multicast virtual address space status = CUDADRV_FUNC( cuMemAddressReserve(&mc_va, mc_size, nvls->minGran, 0U, 0)); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to reserve multicast virtual address space"); goto cleanup; } status = CUDADRV_FUNC(cuMemMap(mc_va, mc_size, 0, nvls->mc_handle, 0)); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to map multicast memory"); goto cleanup; } status = CUDADRV_FUNC(cuMemSetAccess(mc_va, mc_size, &accessDesc, 1)); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to set multicast memory access"); goto cleanup; } tl_debug(UCC_TL_TEAM_LIB(team), "Rank: %d symmetric memory is set: %p [%ld bytes]", UCC_TL_TEAM_RANK(team), (void *)mc_va, mc_size); // Store the handles for cleanup in team destroy nvls->mc_va = mc_va; nvls->uc_va = uc_va; nvls->mc_memhandle = mem_handle; nvls->mc_offset = 0; team->state = UCC_TL_CUDA_NVLS_STATE_BARRIER; /* fall through */ } case UCC_TL_CUDA_NVLS_STATE_BARRIER: { /* Final OOB barrier — ensures all ranks have completed the NVLS * memory setup (including any async fabric operations) before any * collective can use the team. */ if (nvls->barrier_data == NULL) { nvls->barrier_data = (char *)ucc_malloc( UCC_TL_TEAM_SIZE(team), "nvls_barrier"); if (!nvls->barrier_data) { status = UCC_ERR_NO_MEMORY; goto cleanup; } nvls->barrier_data[UCC_TL_TEAM_RANK(team)] = 1; } if (team->oob_req == NULL) { status = team->oob.allgather( &nvls->barrier_data[UCC_TL_TEAM_RANK(team)], nvls->barrier_data, 1, team->oob.coll_info, &team->oob_req); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to initiate NVLS barrier"); ucc_free(nvls->barrier_data); nvls->barrier_data = NULL; goto cleanup; } } status = team->oob.req_test(team->oob_req); if (status > 0) { return UCC_INPROGRESS; } if (status < 0) { tl_error(UCC_TL_TEAM_LIB(team), "NVLS barrier failed"); team->oob.req_free(team->oob_req); team->oob_req = NULL; ucc_free(nvls->barrier_data); nvls->barrier_data = NULL; goto cleanup; } team->oob.req_free(team->oob_req); team->oob_req = NULL; ucc_free(nvls->barrier_data); nvls->barrier_data = NULL; tl_debug(UCC_TL_TEAM_LIB(team), "NVLS init: rank %d OOB barrier complete — team ready", UCC_TL_TEAM_RANK(team)); break; } default: break; } return UCC_OK; cleanup: // Cleanup on error - free state-stored temporary allocations if (nvls->share_data) { ucc_free(nvls->share_data); nvls->share_data = NULL; } if (nvls->barrier_data) { ucc_free(nvls->barrier_data); nvls->barrier_data = NULL; } // Clean up CUDA resources - check local variables for partial allocations // Unmap and free multicast VA if it was reserved/mapped if (mc_va != 0) { if (CUDADRV_FUNC(cuMemUnmap(mc_va, mc_size)) != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to unmap mc_va during cleanup"); } if (CUDADRV_FUNC(cuMemAddressFree(mc_va, mc_size)) != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to free mc_va during cleanup"); } nvls->mc_va = 0; } // Unmap and free unicast VA if it was reserved/mapped if (uc_va != 0) { if (CUDADRV_FUNC(cuMemUnmap(uc_va, mc_size)) != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to unmap uc_va during cleanup"); } if (CUDADRV_FUNC(cuMemAddressFree(uc_va, mc_size)) != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to free uc_va during cleanup"); } nvls->uc_va = 0; } // Release memory handle if it was created if (mem_handle != 0) { if (CUDADRV_FUNC(cuMemRelease(mem_handle)) != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to release mem_handle during cleanup"); } nvls->mc_memhandle = 0; } // Release multicast handle if it was created or imported if (nvls->mc_handle != 0) { if (CUDADRV_FUNC(cuMemRelease(nvls->mc_handle)) != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(team), "failed to release mc_handle during cleanup"); } nvls->mc_handle = 0; } // Free coll_ids if allocated if (nvls->coll_ids) { ucc_free(nvls->coll_ids); nvls->coll_ids = NULL; } return status; } ucc_status_t ucc_tl_cuda_nvls_destroy(ucc_tl_cuda_team_t *team) { int device = team->nvls.device; // Rank 0: unbind the multicast object if (UCC_TL_TEAM_RANK(team) == 0 && team->nvls.mc_handle) { CUDADRV_FUNC(cuMulticastUnbind( team->nvls.mc_handle, device, team->nvls.mc_offset, team->nvls.mc_size)); } // ALL ranks: release their multicast handle (created or imported) if (team->nvls.mc_handle) { CUDADRV_FUNC(cuMemRelease(team->nvls.mc_handle)); team->nvls.mc_handle = 0; } // ALL ranks: clean up their mapped memory if (team->nvls.mc_va) { CUDADRV_FUNC(cuMemUnmap(team->nvls.mc_va, team->nvls.mc_size)); CUDADRV_FUNC(cuMemAddressFree(team->nvls.mc_va, team->nvls.mc_size)); } if (team->nvls.uc_va) { CUDADRV_FUNC(cuMemUnmap(team->nvls.uc_va, team->nvls.mc_size)); CUDADRV_FUNC(cuMemAddressFree(team->nvls.uc_va, team->nvls.mc_size)); } // ALL ranks: release their memory handle if (team->nvls.mc_memhandle) { CUDADRV_FUNC(cuMemRelease(team->nvls.mc_memhandle)); } // ALL ranks: free coll_ids if (team->nvls.coll_ids) { ucc_free(team->nvls.coll_ids); team->nvls.coll_ids = NULL; } if (team->nvls.share_data) { ucc_free(team->nvls.share_data); team->nvls.share_data = NULL; } if (team->nvls.barrier_data) { ucc_free(team->nvls.barrier_data); team->nvls.barrier_data = NULL; } return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/reduce_scatterv/0000775000175000017500000000000015211535620022147 5ustar alastairalastairucc-1.8.0/src/components/tl/cuda/reduce_scatterv/reduce_scatterv.h0000664000175000017500000000644715211535620025515 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef REDUCE_SCATTERV_H_ #define REDUCE_SCATTERV_H_ #include "tl_cuda.h" #include "tl_cuda_coll.h" enum { UCC_TL_CUDA_REDUCE_SCATTERV_ALG_AUTO, UCC_TL_CUDA_REDUCE_SCATTERV_ALG_RING, UCC_TL_CUDA_REDUCE_SCATTERV_ALG_LINEAR, #ifdef HAVE_NVLS UCC_TL_CUDA_REDUCE_SCATTERV_ALG_NVLS, #endif /* HAVE_NVLS */ UCC_TL_CUDA_REDUCE_SCATTERV_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_cuda_reduce_scatterv_algs[UCC_TL_CUDA_REDUCE_SCATTERV_ALG_LAST + 1]; #define UCC_TL_CUDA_REDUCE_SCATTERV_DEFAULT_ALG_SELECT_STR "reduce_scatterv:cuda:@0" ucc_status_t ucc_tl_cuda_reduce_scatterv_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p); size_t ucc_tl_cuda_reduce_scatterv_get_count(const ucc_tl_cuda_task_t *task, ucc_rank_t block); size_t ucc_tl_cuda_reduce_scatterv_get_offset(const ucc_tl_cuda_task_t *task, ucc_rank_t block); /* Reduce scatterv ring */ ucc_status_t ucc_tl_cuda_reduce_scatterv_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p); ucc_status_t ucc_tl_cuda_reduce_scatterv_ring_start(ucc_coll_task_t *task); void ucc_tl_cuda_reduce_scatterv_ring_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_reduce_scatterv_ring_finalize(ucc_coll_task_t *task); /* Reduce scatterv linear */ ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p); ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_start(ucc_coll_task_t *task); void ucc_tl_cuda_reduce_scatterv_linear_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_finalize(ucc_coll_task_t *task); #ifdef HAVE_NVLS /* Reduce scatterv NVLS */ typedef size_t (*nvls_get_count_fn_t)(const ucc_tl_cuda_task_t *, ucc_rank_t); ucc_status_t ucc_tl_cuda_reduce_scatterv_nvls_triggered_post( ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_cuda_reduce_scatterv_nvls_init_common( ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p, nvls_get_count_fn_t get_offset, nvls_get_count_fn_t get_count); ucc_status_t ucc_tl_cuda_reduce_scatterv_nvls_init( ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p); ucc_status_t ucc_tl_cuda_reduce_scatterv_nvls_start(ucc_coll_task_t *task); void ucc_tl_cuda_reduce_scatterv_nvls_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_cuda_reduce_scatterv_nvls_finalize(ucc_coll_task_t *task); #endif /* HAVE_NVLS */ static inline int ucc_tl_cuda_reduce_scatterv_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_CUDA_REDUCE_SCATTERV_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_cuda_reduce_scatterv_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/cuda/reduce_scatterv/reduce_scatterv_ring.c0000664000175000017500000003225515211535620026523 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatterv.h" #include "components/ec/ucc_ec.h" #include "tl_cuda_cache.h" #include "tl_cuda_ring.h" #include "utils/arch/cpu.h" #include "utils/arch/cuda_def.h" #define BLOCK_ALIGN 64 ucc_status_t ucc_tl_cuda_reduce_scatterv_ring_finalize(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); tl_trace(UCC_TASK_LIB(task), "finalizing task %p", task); ucc_tl_cuda_task_put(task); return UCC_OK; } ucc_status_t ucc_tl_cuda_reduce_scatterv_ring_setup_start(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); int chunk; for (chunk = 0; chunk < 1; chunk++) { set_rank_step(task, trank, 0, chunk); } ucc_memory_cpu_store_fence(); return ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), task->bar); } ucc_status_t ucc_tl_cuda_reduce_scatterv_ring_setup_test(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); return ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); } static inline int ucc_tl_cuda_check_peers_ready(ucc_tl_cuda_task_t *task, int chunk, int step) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); int nrings = task->reduce_scatterv_ring.num_rings; int ring, peer_step; ucc_rank_t peer; for (ring = 0; ring < nrings; ring++) { peer = get_send_to(team, trank, tsize, ring); peer_step = get_rank_step(task, peer, chunk); if (peer_step < step) { return 0; } peer = get_recv_from(team, trank, tsize, ring); peer_step = get_rank_step(task, peer, chunk); if (peer_step < step) { return 0; } } return 1; } ucc_status_t ucc_tl_cuda_reduce_scatterv_ring_progress_ring(ucc_tl_cuda_task_t * task, int chunk) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); int tsize = UCC_TL_TEAM_SIZE(team); int nsteps = tsize; int nrings = task->reduce_scatterv_ring.num_rings; int nfrags = task->reduce_scatterv_ring.num_frags; ucc_datatype_t dt = task->reduce_scatterv_ring.dt; size_t dt_size = ucc_dt_size(dt); void *sbuf = task->reduce_scatterv_ring.sbuf; void *dbuf = task->reduce_scatterv_ring.rbuf; size_t ssize = get_scratch_size(team, nrings, 1, dt_size); ucc_rank_t recvfrom; ucc_ee_executor_t *exec; ucc_ee_executor_task_args_t eargs; ucc_ee_executor_task_t *etask; void *rsrc1, *rsrc2, *rdst; ucc_status_t st; int step, ring, frag, frag_step, k; size_t ring_frag_count, frag_count, block_count, block, ring_frag_offset, remote_offset, local_offset, frag_offset, block_offset, ring_scratch_offset; if (UCC_IS_INPLACE(*args)) { sbuf = dbuf; } st = ucc_coll_task_get_executor(&task->super, &exec); if (ucc_unlikely(st != UCC_OK)) { return st; } step = get_rank_step(task, trank, chunk); while (step < (nsteps * nfrags)) { etask = task->reduce_scatterv_ring.exec_task[chunk]; if (etask != NULL) { st = ucc_ee_executor_task_test(etask); if (st != UCC_OK) { if (ucc_likely(st > 0)) { st = UCC_INPROGRESS; } return st; } ucc_ee_executor_task_finalize(etask); task->reduce_scatterv_ring.exec_task[chunk] = NULL; step++; set_rank_step(task, trank, step, chunk); continue; } if (!ucc_tl_cuda_check_peers_ready(task, chunk, step)) { return UCC_INPROGRESS; } frag = step / tsize; frag_step = step % tsize; if (step % 2) { remote_offset = 0; local_offset = ssize / 2; } else { remote_offset = ssize / 2; local_offset = 0; } k = 0; ring_scratch_offset = ssize / 2 / nrings; for (ring = 0; ring < nrings; ring++) { recvfrom = get_recv_from(team, trank, tsize, ring); block = get_recv_block(team, trank, tsize, frag_step, ring); block_count = task->reduce_scatterv_ring.get_count(task, block); frag_count = ucc_buffer_block_count_aligned( block_count, nfrags, frag, BLOCK_ALIGN); ring_frag_count = ucc_buffer_block_count_aligned( frag_count, nrings, ring, BLOCK_ALIGN); if (ring_frag_count == 0) { continue; } block_offset = task->reduce_scatterv_ring.get_offset(task, block); frag_offset = ucc_buffer_block_offset_aligned( block_count, nfrags, frag, BLOCK_ALIGN); ring_frag_offset = ucc_buffer_block_offset_aligned( frag_count, nrings, ring, BLOCK_ALIGN); if (frag_step == 0) { /* copy data from local source buffer to local scratch buffer */ eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY_MULTI; eargs.copy_multi.src[k] = PTR_OFFSET(sbuf, (block_offset + frag_offset + ring_frag_offset) * dt_size); eargs.copy_multi.dst[k] = PTR_OFFSET(TASK_SCRATCH(task, trank), local_offset + ring_scratch_offset * ring); eargs.copy_multi.counts[k] = ring_frag_count * dt_size; eargs.copy_multi.num_vectors = k + 1; } else { /* start reduction for data in remote scratch and local source buffer */ rsrc1 = PTR_OFFSET(sbuf, (block_offset + frag_offset + ring_frag_offset) * dt_size); rsrc2 = PTR_OFFSET(TASK_SCRATCH(task, recvfrom), remote_offset + ring_scratch_offset * ring); if (frag_step == tsize - 1) { /* save reduction result to local destination buffer on last step */ if (UCC_IS_INPLACE(*args)) { rdst = PTR_OFFSET(dbuf, (block_offset + frag_offset + ring_frag_offset) * dt_size); } else { rdst = PTR_OFFSET(dbuf, (frag_offset + ring_frag_offset) * dt_size); } } else { /* save reduction result to local scratch buffer */ rdst = PTR_OFFSET(TASK_SCRATCH(task, trank), local_offset + ring_scratch_offset * ring); } eargs.task_type = UCC_EE_EXECUTOR_TASK_REDUCE_MULTI_DST; eargs.reduce_multi_dst.dst[k] = rdst; eargs.reduce_multi_dst.src1[k] = rsrc1; eargs.reduce_multi_dst.src2[k] = rsrc2; eargs.reduce_multi_dst.counts[k] = ring_frag_count; eargs.reduce_multi_dst.dt = dt; eargs.reduce_multi_dst.op = args->op; eargs.reduce_multi_dst.n_bufs = k + 1; } k++; } if (k == 0) { /* have no work to do at current step, go to next step */ step++; set_rank_step(task, trank, step, chunk); continue; } st = ucc_ee_executor_task_post(exec, &eargs, &task->reduce_scatterv_ring.exec_task[chunk]); if (ucc_unlikely(st != UCC_OK)) { return st; } } return UCC_OK; } void ucc_tl_cuda_reduce_scatterv_ring_progress(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_status_t st; int chunk, num_done; task->super.status = UCC_INPROGRESS; switch (task->reduce_scatterv_ring.stage) { case RING_STAGE_SYNC: if (ucc_tl_cuda_get_sync(task) != UCC_OK) { task->super.status = UCC_INPROGRESS; return; } st = ucc_tl_cuda_reduce_scatterv_ring_setup_start(task); if (st != UCC_OK) { task->super.status = st; return; } task->reduce_scatterv_ring.stage = RING_STAGE_SETUP; /* fall through */ case RING_STAGE_SETUP: st = ucc_tl_cuda_reduce_scatterv_ring_setup_test(task); if (st != UCC_OK) { task->super.status = st; return; } task->reduce_scatterv_ring.stage = RING_STAGE_RING; /* fall through */ case RING_STAGE_RING: num_done = 0; for (chunk = 0; chunk < 1; chunk++) { st = ucc_tl_cuda_reduce_scatterv_ring_progress_ring(task, chunk); if (ucc_unlikely(st < 0)) { task->super.status = st; return; } else if (st == UCC_OK) { num_done++; } } if (num_done != 1) { return; } st = ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), task->bar); if (ucc_unlikely(st != UCC_OK)) { task->super.status = st; return; } task->reduce_scatterv_ring.stage = RING_STAGE_BARRIER; break; default: ucc_assert(task->reduce_scatterv_ring.stage == RING_STAGE_BARRIER); break; } task->super.status = ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); if (task->super.status == UCC_OK) { ucc_tl_cuda_put_sync(task); UCC_TL_CUDA_PROFILE_REQUEST_EVENT(coll_task, "cuda_rsv_ring_done", 0); } } ucc_status_t ucc_tl_cuda_reduce_scatterv_ring_start(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_tl_cuda_lib_t *lib = UCC_TL_CUDA_TEAM_LIB(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_datatype_t dt = task->reduce_scatterv_ring.dt; size_t dt_size = ucc_dt_size(dt); size_t send_size, frag_size, ssize; int nrings; ucc_rank_t i; UCC_TL_CUDA_PROFILE_REQUEST_EVENT(coll_task, "cuda_rsv_ring_start", 0); send_size = task->reduce_scatterv_ring.get_count(task, 0); for (i = 1; i < tsize; i++) { send_size = ucc_max(send_size, task->reduce_scatterv_ring.get_count(task, i)); } if (send_size == 0) { task->super.status = UCC_OK; return ucc_task_complete(&task->super); } nrings = get_num_rings(team, send_size * dt_size, lib->cfg.reduce_scatter_ring_max_rings); task->reduce_scatterv_ring.sbuf = args->src.info.buffer; task->reduce_scatterv_ring.num_rings = nrings; if (args->coll_type == UCC_COLL_TYPE_REDUCE_SCATTERV) { task->reduce_scatterv_ring.rbuf = args->dst.info_v.buffer; } else { task->reduce_scatterv_ring.rbuf = args->dst.info.buffer; } ssize = get_scratch_size(team, nrings, 1, dt_size); frag_size = ucc_min(ssize / dt_size / 2, send_size); task->reduce_scatterv_ring.num_frags = ucc_div_round_up(send_size, frag_size); task->reduce_scatterv_ring.stage = RING_STAGE_SYNC; task->reduce_scatterv_ring.exec_task[0] = NULL; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_cuda_reduce_scatterv_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_task_t *task; ucc_status_t status; if (coll_args->args.op == UCC_OP_AVG) { return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_cuda_reduce_scatterv_ring_start; task->super.progress = ucc_tl_cuda_reduce_scatterv_ring_progress; task->super.finalize = ucc_tl_cuda_reduce_scatterv_ring_finalize; task->reduce_scatterv_ring.get_count = ucc_tl_cuda_reduce_scatterv_get_count; task->reduce_scatterv_ring.get_offset = ucc_tl_cuda_reduce_scatterv_get_offset; task->reduce_scatterv_ring.dt = coll_args->args.dst.info_v.datatype; task->bar = TASK_BAR(task); *task_p = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/cuda/reduce_scatterv/reduce_scatterv.c0000664000175000017500000000423715211535620025503 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatterv.h" #include "components/mc/ucc_mc.h" ucc_base_coll_alg_info_t ucc_tl_cuda_reduce_scatterv_algs[UCC_TL_CUDA_REDUCE_SCATTERV_ALG_LAST + 1] = { [UCC_TL_CUDA_REDUCE_SCATTERV_ALG_AUTO] = {.id = UCC_TL_CUDA_REDUCE_SCATTERV_ALG_AUTO, .name = "auto", .desc = "choose reduce scatterv algorithm based on CUDA topology"}, [UCC_TL_CUDA_REDUCE_SCATTERV_ALG_RING] = {.id = UCC_TL_CUDA_REDUCE_SCATTERV_ALG_RING, .name = "ring", .desc = "multiring reduce scatterv algorithm"}, [UCC_TL_CUDA_REDUCE_SCATTERV_ALG_LINEAR] = {.id = UCC_TL_CUDA_REDUCE_SCATTERV_ALG_LINEAR, .name = "linear", .desc = "linear reduce scatterv algorithm"}, [UCC_TL_CUDA_REDUCE_SCATTERV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; size_t ucc_tl_cuda_reduce_scatterv_get_count(const ucc_tl_cuda_task_t *task, ucc_rank_t rank) { const ucc_coll_args_t *args = &TASK_ARGS(task); return ucc_coll_args_get_count(args, args->dst.info_v.counts, rank); } size_t ucc_tl_cuda_reduce_scatterv_get_offset(const ucc_tl_cuda_task_t *task, ucc_rank_t rank) { size_t offset = 0; ucc_rank_t i; for (i = 0; i < rank; i++) { offset += ucc_tl_cuda_reduce_scatterv_get_count(task, i); } return offset; } ucc_status_t ucc_tl_cuda_reduce_scatterv_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); if (ucc_tl_cuda_team_topo_is_fully_connected(team->topo)) { return ucc_tl_cuda_reduce_scatterv_linear_init(coll_args, tl_team, task_p); } else { return ucc_tl_cuda_reduce_scatterv_ring_init(coll_args, tl_team, task_p); } } ucc-1.8.0/src/components/tl/cuda/reduce_scatterv/reduce_scatterv_nvls.c0000664000175000017500000002374215211535620026547 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatterv/reduce_scatterv.h" #include #include "core/ucc_ee.h" #include "utils/arch/cuda_def.h" #include "tl_cuda_nvls.h" #include "kernels/reduce_scatter_kernel.h" #include "components/ec/ucc_ec.h" #include "components/ec/cuda/ec_cuda_resources.h" ucc_status_t ucc_tl_cuda_reduce_scatterv_nvls_start(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_datatype_t dt = task->reduce_scatterv_nvls.dt; ucc_ee_h ee = coll_task->ee; cudaStream_t stream = (ee) ? (cudaStream_t)ee->ee_context : team->stream; ucc_ec_cuda_event_t *ec_event = (ucc_ec_cuda_event_t *)task ->reduce_scatterv_nvls.evt_completion; cudaEvent_t evt = ec_event->event; uint32_t sm_count = UCC_TL_CUDA_TEAM_LIB(team)->cfg.nvls_sm_count; uint32_t threads = UCC_TL_CUDA_TEAM_LIB(team)->cfg.nvls_threads; CUdeviceptr mc_va = task->reduce_scatterv_nvls.mc_va; CUdeviceptr uc_va = task->reduce_scatterv_nvls.uc_va; ucc_status_t status; size_t src_size_bytes; void *sbuf; void *rbuf; if (args->coll_type == UCC_COLL_TYPE_REDUCE_SCATTER) { if (UCC_IS_INPLACE(*args)) { sbuf = args->dst.info.buffer; rbuf = PTR_OFFSET( sbuf, task->reduce_scatterv_nvls.offset * sizeof(uint32_t)); src_size_bytes = args->dst.info.count * UCC_TL_TEAM_SIZE(team) * ucc_dt_size(dt); } else { sbuf = args->src.info.buffer; rbuf = args->dst.info.buffer; src_size_bytes = args->src.info.count * ucc_dt_size(dt); } } else { /* RSV: reduce_scatterv */ if (UCC_IS_INPLACE(*args)) { sbuf = args->dst.info_v.buffer; rbuf = PTR_OFFSET( sbuf, task->reduce_scatterv_nvls.offset * sizeof(uint32_t)); src_size_bytes = ucc_coll_args_get_total_count( args, args->dst.info_v.counts, UCC_TL_TEAM_SIZE(team)) * ucc_dt_size(dt); } else { sbuf = args->src.info.buffer; rbuf = args->dst.info_v.buffer; src_size_bytes = args->src.info.count * ucc_dt_size(dt); } } /* copy src buffer to symmetric memory first */ status = CUDA_FUNC(cudaMemcpyAsync( (void *)uc_va, sbuf, src_size_bytes, cudaMemcpyDeviceToDevice, stream)); if (ucc_unlikely(status != UCC_OK)) { return status; } status = post_reduce_scatter_kernel( stream, sm_count, threads, (CUdeviceptr)rbuf, mc_va, TASK_NVLS_CONTROL_MC(task), TASK_NVLS_CONTROL_UC(task), task->reduce_scatterv_nvls.coll_id, task->reduce_scatterv_nvls.offset, task->reduce_scatterv_nvls.count, dt, UCC_TL_TEAM_SIZE(team)); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to post reduce scatter kernel"); return status; } status = CUDA_FUNC(cudaEventRecord(evt, stream)); if (ucc_unlikely(status != UCC_OK)) { return status; } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } void ucc_tl_cuda_reduce_scatterv_nvls_progress(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_ec_cuda_event_t *ec_event = (ucc_ec_cuda_event_t *)task ->reduce_scatterv_nvls.evt_completion; cudaEvent_t evt = ec_event->event; task->super.status = cuda_error_to_ucc_status(cudaEventQuery(evt)); } ucc_status_t ucc_tl_cuda_reduce_scatterv_nvls_triggered_post( ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *coll_task) // NOLINT { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_status_t status; ucc_ev_t post_event; ucc_assert(ee->ee_type == UCC_EE_CUDA_STREAM); coll_task->ee = ee; tl_trace(UCC_TASK_LIB(task), "triggered post. task:%p", coll_task); status = coll_task->post(coll_task); if (ucc_likely(status == UCC_OK)) { post_event.ev_type = UCC_EVENT_COLLECTIVE_POST; post_event.ev_context_size = 0; post_event.ev_context = NULL; post_event.req = &coll_task->super; status = ucc_ee_set_event_internal( coll_task->ee, &post_event, &coll_task->ee->event_out_queue); if (ucc_unlikely(status != UCC_OK)) { tl_error( UCC_TASK_LIB(task), "failed to set EE event: %s", ucc_status_string(status)); return status; } } return status; } ucc_status_t ucc_tl_cuda_reduce_scatterv_nvls_finalize(ucc_coll_task_t *task) { ucc_tl_cuda_task_t *tl_task = ucc_derived_of(task, ucc_tl_cuda_task_t); tl_trace( UCC_TASK_LIB(tl_task), "task: %p reduce_scatterv_nvls_finalize", task); ucc_ec_destroy_event( tl_task->reduce_scatterv_nvls.evt_completion, UCC_EE_CUDA_STREAM); ucc_tl_cuda_task_put(tl_task); return UCC_OK; } ucc_status_t ucc_tl_cuda_reduce_scatterv_nvls_init_common( ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p, nvls_get_count_fn_t get_offset, nvls_get_count_fn_t get_count) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_datatype_t dt = (coll_args->args.coll_type == UCC_COLL_TYPE_REDUCE_SCATTER) ? coll_args->args.dst.info.datatype : coll_args->args.dst.info_v.datatype; ucc_tl_cuda_task_t *task; ucc_status_t status; size_t offset_elements; size_t count_elements; size_t offset; size_t count; /* Validate op and datatype before allocating task */ if (coll_args->args.op != UCC_OP_SUM) { tl_debug( UCC_TL_TEAM_LIB(team), "NVLS reduce scatter(v) supported only with SUM operation"); return UCC_ERR_NOT_SUPPORTED; } if (dt != UCC_DT_FLOAT32 && dt != UCC_DT_BFLOAT16 && dt != UCC_DT_INT32 && dt != UCC_DT_UINT32 && dt != UCC_DT_INT64 && dt != UCC_DT_UINT64) { tl_debug( UCC_TL_TEAM_LIB(team), "NVLS reduce scatter(v) supported only with " "float32/bfloat16/int32/uint32/int64/uint64"); return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } offset_elements = get_offset(task, trank); count_elements = get_count(task, trank); /* Convert from datatype elements to uint32_t indices for the kernel. * For float32/int32/uint32: 1 element = 1 uint32_t * For int64/uint64: 1 element = 2 uint32_t * For bfloat16: 2 elements = 1 uint32_t */ if (dt == UCC_DT_FLOAT32 || dt == UCC_DT_INT32 || dt == UCC_DT_UINT32) { offset = offset_elements; count = count_elements; } else if (dt == UCC_DT_INT64 || dt == UCC_DT_UINT64) { offset = offset_elements * 2; count = count_elements * 2; } else { /* UCC_DT_BFLOAT16 */ if (offset_elements % 2 != 0 || count_elements % 2 != 0) { tl_debug( UCC_TL_TEAM_LIB(team), "BF16 offset and count must be even, got offset=%zu " "count=%zu", offset_elements, count_elements); goto err_task_put; } offset = offset_elements / 2; count = count_elements / 2; } /* NVLS requires 16-byte alignment (4 uint32_t elements) */ if (ucc_unlikely(offset % 4 != 0)) { tl_debug( UCC_TL_TEAM_LIB(team), "NVLS requires 16-byte alignment for offset, got offset=%zu", offset); goto err_task_put; } if (ucc_unlikely(count % 4 != 0)) { tl_debug( UCC_TL_TEAM_LIB(team), "NVLS requires 16-byte alignment for count, got count=%zu", count); goto err_task_put; } status = ucc_ec_create_event( &task->reduce_scatterv_nvls.evt_completion, UCC_EE_CUDA_STREAM); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TL_TEAM_LIB(team), "failed to create CUDA event"); goto err_task_put; } task->reduce_scatterv_nvls.dt = dt; task->reduce_scatterv_nvls.offset = offset; task->reduce_scatterv_nvls.count = count; task->reduce_scatterv_nvls.mc_va = (CUdeviceptr)TASK_SYMMETRIC_MC(task); task->reduce_scatterv_nvls.uc_va = (CUdeviceptr)TASK_SYMMETRIC_UC(task); task->reduce_scatterv_nvls.coll_id = team->nvls.coll_ids[task->coll_id]++; task->super.post = ucc_tl_cuda_reduce_scatterv_nvls_start; task->super .triggered_post = ucc_tl_cuda_reduce_scatterv_nvls_triggered_post; task->super.progress = ucc_tl_cuda_reduce_scatterv_nvls_progress; task->super.finalize = ucc_tl_cuda_reduce_scatterv_nvls_finalize; *task_p = &task->super; return UCC_OK; err_task_put: ucc_tl_cuda_task_put(task); return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_tl_cuda_reduce_scatterv_nvls_init( ucc_base_coll_args_t *coll_args, ucc_base_team_t *tl_team, ucc_coll_task_t **task_p) { return ucc_tl_cuda_reduce_scatterv_nvls_init_common( coll_args, tl_team, task_p, ucc_tl_cuda_reduce_scatterv_get_offset, ucc_tl_cuda_reduce_scatterv_get_count); } ucc-1.8.0/src/components/tl/cuda/reduce_scatterv/reduce_scatterv_linear.c0000664000175000017500000004116215211535620027033 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatterv.h" #include "components/ec/ucc_ec.h" #include "tl_cuda_cache.h" #include "utils/arch/cpu.h" #include "utils/arch/cuda_def.h" /* * fragmented buffered copy linear reduce scatterv algorithm * * Description: * scratch buffer is split into 2 parts to guarantee data consistency e.g. * when ranks in the ring are running at different steps of the algorithm. * 2 parts is enough since max difference between ranks steps is 1 * * Definitions: * blockI - full send buffer at Rank I * fragI_J - fragment of send buffer at Rank I and step J * NS - number of steps * NF - number of fragments * N - team size * * Setup: * max_frag_size = ucc_min(ucc_max(block0, block1, ..., block N -1 ), * scratch_size / 2 / N) * NF = ucc_max(block0, block1, ..., block N-1) / max_frag_size * NS = 1 + NF * * Algorithm * for rank R * step 0: copy fragR_0 to remote scratch buffers for all ranks * * step 1: reduce frag1_0, frag2_0, ..., fragN_0 from local scratch buffer * to local dst buffer * copy fragR_1 from local src buffer to remote scratch buffers * for all ranks * ... * * step NS-2: reduce frag1_(NS-3), frag2_(NS-3), ..., fragN_(NS-3) from local * scratch buffer to local dst buffer * copy fragR_(NS-2) from local src buffer to remote scratch buffers * for all ranks * * step NS-1: reduce frag1_(NS-2), frag2_(NS-2), ..., fragN_(NS-2) from local * scratch buffer to local dst buffer */ enum { STAGE_SYNC, /*< Wait for free SYNC segment */ STAGE_SETUP, /*< Wait for memhandle setup to finish */ STAGE_COPIES, /*< Linear algorithm is running */ STAGE_BARRIER, /*< Linear algorithm is done, waiting for * other ranks to finish */ }; ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_finalize(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); tl_trace(UCC_TASK_LIB(task), "finalizing task %p", task); ucc_tl_cuda_task_put(task); return UCC_OK; } /* * get maximum size of scratch memory evenly divisible by * (2 * team_size * datatype_size) */ static inline size_t get_scratch_size(ucc_tl_cuda_team_t *team, ucc_datatype_t dt) { size_t dt_size = ucc_dt_size(dt); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); size_t divisor = 2 * dt_size * tsize; return (UCC_TL_CUDA_TEAM_LIB(team)->cfg.scratch_size / divisor) * divisor; } /* get stride between consecutive rank slots in scratch memory */ static inline size_t get_scratch_stride(ucc_tl_cuda_team_t *team, ucc_datatype_t dt) { size_t step_ssize = get_scratch_size(team, dt) / 2; ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); return step_ssize / tsize; } /* get offset of rank slot inside scratch */ static inline size_t get_scratch_offset(ucc_tl_cuda_team_t *team, ucc_datatype_t dt, ucc_rank_t rank) { size_t step_ssize = get_scratch_size(team, dt) / 2; ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); return rank * step_ssize / tsize; } ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_setup_start(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_status_t status; set_rank_step(task, trank, 0, 0); ucc_memory_cpu_store_fence(); status = ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), task->bar); if (ucc_unlikely(status != UCC_OK)) { goto exit_err; } return UCC_OK; exit_err: return status; } ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_setup_test(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); return ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); } ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_copy(ucc_tl_cuda_task_t *task, ucc_ee_executor_t *exec, void *sbuf, int step, size_t remote_offset, ucc_ee_executor_task_t **etask) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_datatype_t dt = task->reduce_scatterv_linear.dt; size_t dt_size = ucc_dt_size(dt); int nfrags = task->reduce_scatterv_linear.num_frags; size_t scratch_offset, scratch_stride, send_size, frag_size, frag_offset, rank_offset; ucc_ee_executor_task_args_t eargs; ucc_rank_t i, nv, peer; scratch_offset = get_scratch_offset(team, dt, trank); scratch_stride = get_scratch_stride(team, dt); eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY_MULTI; eargs.flags = 0; for (i = 0, nv = 0; i < tsize; i++) { peer = (trank + i) % UCC_TL_TEAM_SIZE(team); if (peer == trank) { continue; } send_size = task->reduce_scatterv_linear.get_count(task, peer); frag_size = ucc_buffer_block_count(send_size, nfrags, step); frag_offset = ucc_buffer_block_offset(send_size, nfrags, step); rank_offset = task->reduce_scatterv_linear.get_offset(task, peer); if (frag_size == 0) { continue; } eargs.copy_multi.src[nv] = PTR_OFFSET(sbuf, (rank_offset + frag_offset) * dt_size); eargs.copy_multi.counts[nv] = frag_size * dt_size; if (trank < peer) { eargs.copy_multi.dst[nv] = PTR_OFFSET(TASK_SCRATCH(task, peer), remote_offset + scratch_offset); } else { eargs.copy_multi.dst[nv] = PTR_OFFSET(TASK_SCRATCH(task, peer), remote_offset + scratch_offset - scratch_stride); } nv++; } if (nv == 0) { *etask = NULL; return UCC_OK; } eargs.copy_multi.num_vectors = nv; return ucc_ee_executor_task_post(exec, &eargs, etask); } ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_reduce(ucc_tl_cuda_task_t *task, ucc_ee_executor_t *exec, void *sbuf, void *rbuf, int step, size_t local_offset, ucc_ee_executor_task_t **etask) { ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_datatype_t dt = task->reduce_scatterv_linear.dt; size_t dt_size = ucc_dt_size(dt); int nfrags = task->reduce_scatterv_linear.num_frags; ucc_coll_args_t *args = &TASK_ARGS(task); size_t send_size, frag_size, frag_offset, rank_offset; ucc_ee_executor_task_args_t eargs; send_size = task->reduce_scatterv_linear.get_count(task, trank); frag_size = ucc_buffer_block_count(send_size, nfrags, step); frag_offset = ucc_buffer_block_offset(send_size, nfrags, step); rank_offset = task->reduce_scatterv_linear.get_offset(task, trank); if (frag_size == 0) { *etask = NULL; return UCC_OK; } eargs.task_type = UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED; eargs.flags = 0; eargs.reduce_strided.src1 = PTR_OFFSET(sbuf, (rank_offset + frag_offset) * dt_size); eargs.reduce_strided.src2 = PTR_OFFSET(TASK_SCRATCH(task, trank), local_offset); eargs.reduce_strided.stride = get_scratch_stride(team, dt); eargs.reduce_strided.count = frag_size; eargs.reduce_strided.dst = PTR_OFFSET(rbuf, frag_offset * dt_size); eargs.reduce_strided.op = args->op; eargs.reduce_strided.n_src2 = tsize - 1; eargs.reduce_strided.dt = dt; eargs.flags = 0; return ucc_ee_executor_task_post(exec, &eargs, etask); } ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_progress_frag(ucc_tl_cuda_task_t *task) { ucc_tl_cuda_team_t * team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_coll_args_t * args = &TASK_ARGS(task); ucc_datatype_t dt = task->reduce_scatterv_linear.dt; size_t ssize = get_scratch_size(team, dt); int nfrags = task->reduce_scatterv_linear.num_frags; int num_steps = nfrags + 1; ucc_ee_executor_task_t *etask; ucc_status_t st; ucc_ee_executor_t *exec; int step, i; void *sbuf, *rbuf; size_t local_offset, remote_offset, rank_offset; st = ucc_coll_task_get_executor(&task->super, &exec); if (ucc_unlikely(st != UCC_OK)) { return st; } step = get_rank_step(task, trank, 0); while (step < num_steps) { if ((task->reduce_scatterv_linear.exec_task[0] != NULL) || (task->reduce_scatterv_linear.exec_task[1] != NULL)) { for (i = 0; i < 2; i++) { etask = task->reduce_scatterv_linear.exec_task[i]; if (etask != NULL) { st = ucc_ee_executor_task_test(etask); if (st == UCC_OK) { ucc_ee_executor_task_finalize(etask); task->reduce_scatterv_linear.exec_task[i] = NULL; } else { if (ucc_likely(st > 0)) { return UCC_INPROGRESS; } return st; } } } step++; set_rank_step(task, trank, step, 0); continue; } for (i = 0; i < tsize; i++) { if (get_rank_step(task, i, 0) < step) { return UCC_INPROGRESS; } } if (step % 2) { remote_offset = ssize / 2; local_offset = 0; } else { remote_offset = 0; local_offset = ssize / 2; } if (UCC_IS_INPLACE(*args)) { rank_offset = task->reduce_scatterv_linear.get_offset(task, trank); sbuf = task->reduce_scatterv_linear.rbuf; rbuf = PTR_OFFSET(sbuf, rank_offset * ucc_dt_size(dt)); } else { sbuf = task->reduce_scatterv_linear.sbuf; rbuf = task->reduce_scatterv_linear.rbuf; } if (step == 0) { st = ucc_tl_cuda_reduce_scatterv_linear_copy(task, exec, sbuf, step, remote_offset, &task->reduce_scatterv_linear.exec_task[0]); } else if (step == (num_steps - 1)) { st = ucc_tl_cuda_reduce_scatterv_linear_reduce(task, exec, sbuf, rbuf, step - 1, local_offset, &task->reduce_scatterv_linear.exec_task[1]); } else { st = ucc_tl_cuda_reduce_scatterv_linear_copy(task, exec, sbuf, step, remote_offset, &task->reduce_scatterv_linear.exec_task[0]); if (ucc_unlikely(st != UCC_OK)) { return st; } st = ucc_tl_cuda_reduce_scatterv_linear_reduce(task, exec, sbuf, rbuf, step - 1, local_offset, &task->reduce_scatterv_linear.exec_task[1]); } if (ucc_unlikely(st != UCC_OK)) { return st; } if ((task->reduce_scatterv_linear.exec_task[0] == NULL) && (task->reduce_scatterv_linear.exec_task[1] == NULL)) { /* have no work to do at current step, go to next step */ step++; set_rank_step(task, trank, step, 0); continue; } } return UCC_OK; } void ucc_tl_cuda_reduce_scatterv_linear_progress(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_status_t st; task->super.status = UCC_INPROGRESS; switch (task->reduce_scatterv_linear.stage) { case STAGE_SYNC: if (ucc_tl_cuda_get_sync(task) != UCC_OK) { task->super.status = UCC_INPROGRESS; return; } st = ucc_tl_cuda_reduce_scatterv_linear_setup_start(task); if (st != UCC_OK) { task->super.status = st; return; } task->reduce_scatterv_linear.stage = STAGE_SETUP; case STAGE_SETUP: st = ucc_tl_cuda_reduce_scatterv_linear_setup_test(task); if (st != UCC_OK) { task->super.status = st; return; } task->reduce_scatterv_linear.stage = STAGE_COPIES; case STAGE_COPIES: st = ucc_tl_cuda_reduce_scatterv_linear_progress_frag(task); if (st != UCC_OK) { task->super.status = st; return; } st = ucc_tl_cuda_shm_barrier_start(UCC_TL_TEAM_RANK(team), task->bar); if (ucc_unlikely(st != UCC_OK)) { task->super.status = st; return; } task->reduce_scatterv_linear.stage = STAGE_BARRIER; default: ucc_assert(task->reduce_scatterv_linear.stage == STAGE_BARRIER); break; } task->super.status = ucc_tl_cuda_shm_barrier_test(UCC_TL_TEAM_RANK(team), task->bar); if (task->super.status == UCC_OK) { ucc_tl_cuda_put_sync(task); } } ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_start(ucc_coll_task_t *coll_task) { ucc_tl_cuda_task_t *task = ucc_derived_of(coll_task, ucc_tl_cuda_task_t); ucc_tl_cuda_team_t *team = TASK_TEAM(task); ucc_coll_args_t * args = &TASK_ARGS(task); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_datatype_t dt = task->reduce_scatterv_linear.dt; ucc_rank_t i; size_t send_size, frag_size, ssize; /* need to set rbuf in collective start since frag_setup of pipeline * schedule can update pointer */ if (args->coll_type == UCC_COLL_TYPE_REDUCE_SCATTER) { task->reduce_scatterv_linear.rbuf = args->dst.info.buffer; } else { task->reduce_scatterv_linear.rbuf = args->dst.info_v.buffer; } task->reduce_scatterv_linear.stage = STAGE_SYNC; task->reduce_scatterv_linear.sbuf = args->src.info.buffer; send_size = task->reduce_scatterv_linear.get_count(task, 0); for (i = 1; i < tsize; i++) { send_size = ucc_max(send_size, task->reduce_scatterv_linear.get_count(task, i)); } if (send_size == 0) { task->super.status = UCC_OK; return ucc_task_complete(&task->super); } ssize = get_scratch_size(team, dt); frag_size = ucc_min(ssize / 2 / ucc_dt_size(dt) / tsize, send_size); task->reduce_scatterv_linear.num_frags = ucc_div_round_up(send_size, frag_size); memset(task->reduce_scatterv_linear.exec_task, 0, 2 * sizeof(ucc_ee_executor_task_t*)); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_cuda_reduce_scatterv_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * tl_team, ucc_coll_task_t ** task_p) { ucc_tl_cuda_team_t *team = ucc_derived_of(tl_team, ucc_tl_cuda_team_t); ucc_tl_cuda_task_t *task; ucc_status_t status; if (coll_args->args.op == UCC_OP_AVG) { return UCC_ERR_NOT_SUPPORTED; } if (ucc_unlikely(!ucc_tl_cuda_team_topo_is_fully_connected(team->topo) || UCC_TL_TEAM_SIZE(team) - 1 > UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS)) { return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_cuda_task_init(coll_args, team, &task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->reduce_scatterv_linear.get_count = ucc_tl_cuda_reduce_scatterv_get_count; task->reduce_scatterv_linear.get_offset = ucc_tl_cuda_reduce_scatterv_get_offset; task->reduce_scatterv_linear.dt = coll_args->args.dst.info_v.datatype; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_cuda_reduce_scatterv_linear_start; task->super.progress = ucc_tl_cuda_reduce_scatterv_linear_progress; task->super.finalize = ucc_tl_cuda_reduce_scatterv_linear_finalize; task->bar = TASK_BAR(task); *task_p = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/0000775000175000017500000000000015211535620016640 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/reduce/0000775000175000017500000000000015211535620020107 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/reduce/reduce.h0000664000175000017500000000532515211535620021534 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef REDUCE_H_ #define REDUCE_H_ #include "tl_ucp_coll.h" enum { UCC_TL_UCP_REDUCE_ALG_KNOMIAL, UCC_TL_UCP_REDUCE_ALG_DBT, UCC_TL_UCP_REDUCE_ALG_SRG, UCC_TL_UCP_REDUCE_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_reduce_algs[UCC_TL_UCP_REDUCE_ALG_LAST + 1]; #define UCC_TL_UCP_REDUCE_DEFAULT_ALG_SELECT_STR \ "reduce:0-32K:@0#reduce:32K-inf:@2" /* A set of convenience macros used to implement sw based progress of the reduce algorithm that uses kn pattern */ enum { UCC_REDUCE_KN_PHASE_INIT, UCC_REDUCE_KN_PHASE_PROGRESS, /* checks progress */ UCC_REDUCE_KN_PHASE_MULTI /* reduce multi after recv from children in current step */ }; #define UCC_REDUCE_KN_CHECK_PHASE(_p) \ case _p: \ goto _p; #define UCC_REDUCE_KN_GOTO_PHASE(_phase) \ do { \ switch (_phase) { \ UCC_REDUCE_KN_CHECK_PHASE(UCC_REDUCE_KN_PHASE_MULTI); \ UCC_REDUCE_KN_CHECK_PHASE(UCC_REDUCE_KN_PHASE_PROGRESS); \ UCC_REDUCE_KN_CHECK_PHASE(UCC_REDUCE_KN_PHASE_INIT); \ }; \ } while (0) static inline int ucc_tl_ucp_reduce_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_UCP_REDUCE_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_ucp_reduce_algs[i].name)) { break; } } return i; } ucc_status_t ucc_tl_ucp_reduce_init(ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_reduce_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_reduce_knomial_start(ucc_coll_task_t *task); void ucc_tl_ucp_reduce_knomial_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_reduce_knomial_finalize(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_reduce_dbt_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_reduce_srg_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); #endif ucc-1.8.0/src/components/tl/ucp/reduce/reduce_dbt.c0000664000175000017500000003316315211535620022361 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "reduce.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "utils/ucc_dt_reduce.h" enum { RECV, REDUCE, TEST, TEST_ROOT, }; #define UCC_REDUCE_DBT_CHECK_STATE(_p) \ case _p: \ goto _p; #define UCC_REDUCE_DBT_GOTO_STATE(_state) \ do { \ switch (_state) { \ UCC_REDUCE_DBT_CHECK_STATE(REDUCE); \ UCC_REDUCE_DBT_CHECK_STATE(TEST); \ UCC_REDUCE_DBT_CHECK_STATE(TEST_ROOT); \ }; \ } while (0) static void recv_completion_common(void *request, ucs_status_t status, const ucp_tag_recv_info_t *info, /* NOLINT */ void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in recv completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } ucc_atomic_add32(&task->tagged.recv_completed, 1); if (request) { ucp_request_free(request); } } static void recv_completion_1(void *request, ucs_status_t status, const ucp_tag_recv_info_t *info, /* NOLINT */ void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; task->reduce_dbt.trees[0].recv++; recv_completion_common(request, status, info, user_data); } static void recv_completion_2(void *request, ucs_status_t status, const ucp_tag_recv_info_t *info, /* NOLINT */ void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; task->reduce_dbt.trees[1].recv++; recv_completion_common(request, status, info, user_data); } static inline void single_tree_reduce(ucc_tl_ucp_task_t *task, void *sbuf, void *rbuf, int n_children, size_t count, size_t data_size, ucc_datatype_t dt, ucc_coll_args_t *args, int is_avg) { ucc_status_t status; status = ucc_dt_reduce_strided( sbuf,rbuf, rbuf, n_children, count, data_size, dt, args, is_avg ? UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA : 0, AVG_ALPHA(task), task->reduce_dbt.executor, &task->reduce_dbt.etask); if (ucc_unlikely(UCC_OK != status)) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); task->super.status = status; return; } EXEC_TASK_WAIT(task->reduce_dbt.etask); } void ucc_tl_ucp_reduce_dbt_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_dbt_single_tree_t *trees = task->reduce_dbt.trees ; ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t coll_root = (ucc_rank_t)args->root; int is_root = rank == coll_root; ucp_tag_recv_nbx_callback_t cb[2] = {recv_completion_1, recv_completion_2}; void *sbuf[2], *rbuf[2]; uint32_t i, j, k; ucc_memory_type_t mtype; ucc_datatype_t dt; size_t count, data_size, data_size_t1; size_t counts[2]; int avg_pre_op, avg_post_op; if (is_root) { mtype = args->dst.info.mem_type; dt = args->dst.info.datatype; count = args->dst.info.count; } else { mtype = args->src.info.mem_type; dt = args->src.info.datatype; count = args->src.info.count; } counts[0] = (count % 2) ? count / 2 + 1 : count / 2; counts[1] = count / 2; data_size = count * ucc_dt_size(dt); data_size_t1 = counts[0] * ucc_dt_size(dt); avg_pre_op = ((args->op == UCC_OP_AVG) && UCC_TL_UCP_TEAM_LIB(team)->cfg.reduce_avg_pre_op); avg_post_op = ((args->op == UCC_OP_AVG) && !UCC_TL_UCP_TEAM_LIB(team)->cfg.reduce_avg_pre_op); rbuf[0] = task->reduce_dbt.scratch; rbuf[1] = PTR_OFFSET(rbuf[0], data_size_t1 * 2);; sbuf[0] = avg_pre_op ? PTR_OFFSET(rbuf[0], data_size * 2) : args->src.info.buffer;; sbuf[1] = PTR_OFFSET(sbuf[0], data_size_t1); UCC_REDUCE_DBT_GOTO_STATE(task->reduce_dbt.state); for (i = 0; i < 2; i++) { j = 0; for (k = 0; k < 2; k++) { if (trees[i].children[k] != UCC_RANK_INVALID) { UCPCHECK_GOTO(ucc_tl_ucp_recv_cb( PTR_OFFSET(rbuf[i], counts[i] * ucc_dt_size(dt) * j), counts[i] * ucc_dt_size(dt), mtype, trees[i].children[k], team, task, cb[i], (void *)task), task, out); j++; } } } task->reduce_dbt.state = REDUCE; REDUCE: /* test_recv is needed to progress ucp_worker */ ucc_tl_ucp_test_recv(task); for (i = 0; i < 2; i++) { if (trees[i].recv == trees[i].n_children && !task->reduce_dbt.reduction_comp[i]) { if (trees[i].n_children > 0) { single_tree_reduce(task, sbuf[i], rbuf[i], trees[i].n_children, counts[i], counts[i] * ucc_dt_size(dt), dt, args, avg_post_op && trees[i].root == rank); } task->reduce_dbt.reduction_comp[i] = 1; } } for (i = 0; i < 2; i++) { if (rank != trees[i].root && task->reduce_dbt.reduction_comp[i] && !task->reduce_dbt.send_comp[i]) { UCPCHECK_GOTO(ucc_tl_ucp_send_nb((trees[i].n_children > 0) ? rbuf[i] : sbuf[i], counts[i] * ucc_dt_size(dt), mtype, trees[i].parent, team, task), task, out); task->reduce_dbt.send_comp[i] = 1; } } if (!task->reduce_dbt.reduction_comp[0] || !task->reduce_dbt.reduction_comp[1]) { return; } TEST: if (UCC_INPROGRESS == ucc_tl_ucp_test_send(task)) { task->reduce_dbt.state = TEST; return; } /* tree roots send to coll root*/ for (i = 0; i < 2; i++) { if (rank == trees[i].root && !is_root) { UCPCHECK_GOTO(ucc_tl_ucp_send_nb(rbuf[i], counts[i] * ucc_dt_size(dt), mtype, coll_root, team, task), task, out); } } task->reduce_dbt.reduction_comp[0] = trees[0].recv; task->reduce_dbt.reduction_comp[1] = trees[1].recv; for (i = 0; i < 2; i++) { if (is_root && rank != trees[i].root) { UCPCHECK_GOTO(ucc_tl_ucp_recv_cb(PTR_OFFSET(args->dst.info.buffer, i * counts[0] * ucc_dt_size(dt)), counts[i] * ucc_dt_size(dt), mtype, trees[i].root, team, task, cb[i], (void *)task), task, out); task->reduce_dbt.reduction_comp[i]++; } } TEST_ROOT: /* test_recv is needed to progress ucp_worker */ ucc_tl_ucp_test_recv(task); if (UCC_INPROGRESS == ucc_tl_ucp_test_send(task) || task->reduce_dbt.reduction_comp[0] != trees[0].recv || task->reduce_dbt.reduction_comp[1] != trees[1].recv) { task->reduce_dbt.state = TEST_ROOT; return; } for (i = 0; i < 2; i++) { if (is_root && rank == trees[i].root) { UCPCHECK_GOTO(ucc_mc_memcpy(PTR_OFFSET(args->dst.info.buffer, i * counts[(i + 1) % 2] * ucc_dt_size(dt)), rbuf[i], counts[i] * ucc_dt_size(dt), mtype, mtype), task, out); } } task->super.status = UCC_OK; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_reduce_dbt_done", 0); out: return; } ucc_status_t ucc_tl_ucp_reduce_dbt_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t team_size = UCC_TL_TEAM_SIZE(team); int avg_pre_op = UCC_TL_UCP_TEAM_LIB(TASK_TEAM(task))->cfg.reduce_avg_pre_op; ucc_datatype_t dt; size_t count, data_size; ucc_status_t status; task->reduce_dbt.trees[0].recv = 0; task->reduce_dbt.trees[1].recv = 0; task->reduce_dbt.reduction_comp[0] = 0; task->reduce_dbt.reduction_comp[1] = 0; task->reduce_dbt.send_comp[0] = 0; task->reduce_dbt.send_comp[1] = 0; ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if (args->root == rank) { count = args->dst.info.count; dt = args->dst.info.datatype; } else { count = args->src.info.count; dt = args->src.info.datatype; } data_size = count * ucc_dt_size(dt); status = ucc_coll_task_get_executor(&task->super, &task->reduce_dbt.executor); if (ucc_unlikely(status != UCC_OK)) { return status; } if (UCC_IS_INPLACE(*args) && (rank == args->root)) { args->src.info.buffer = args->dst.info.buffer; } if (avg_pre_op && args->op == UCC_OP_AVG) { /* In case of avg_pre_op, each process must divide itself by team_size */ status = ucc_dt_reduce(args->src.info.buffer, args->src.info.buffer, PTR_OFFSET(task->reduce_dbt.scratch, data_size * 2), count, dt, args, UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA, 1.0 / (double)(team_size * 2), task->reduce_dbt.executor, &task->reduce_dbt.etask); if (ucc_unlikely(UCC_OK != status)) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); return status; } EXEC_TASK_WAIT(task->reduce_dbt.etask, status); } task->reduce_dbt.state = RECV; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_reduce_dbt_start", 0); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_reduce_dbt_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); if (task->reduce_dbt.scratch_mc_header) { ucc_mc_free(task->reduce_dbt.scratch_mc_header); } return ucc_tl_ucp_coll_finalize(coll_task); } ucc_status_t ucc_tl_ucp_reduce_dbt_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team; ucc_tl_ucp_task_t *task; ucc_rank_t rank, size; ucc_memory_type_t mtype; ucc_datatype_t dt; size_t count; size_t data_size; ucc_status_t status; task = ucc_tl_ucp_init_task(coll_args, team); task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_ucp_reduce_dbt_start; task->super.progress = ucc_tl_ucp_reduce_dbt_progress; task->super.finalize = ucc_tl_ucp_reduce_dbt_finalize; tl_team = TASK_TEAM(task); rank = UCC_TL_TEAM_RANK(tl_team); size = UCC_TL_TEAM_SIZE(tl_team); ucc_dbt_build_trees(rank, size, &task->reduce_dbt.trees[0], &task->reduce_dbt.trees[1]); if (coll_args->args.root == rank) { count = coll_args->args.dst.info.count; dt = coll_args->args.dst.info.datatype; mtype = coll_args->args.dst.info.mem_type; } else { count = coll_args->args.src.info.count; dt = coll_args->args.src.info.datatype; mtype = coll_args->args.src.info.mem_type; } data_size = count * ucc_dt_size(dt); task->reduce_dbt.scratch_mc_header = NULL; status = ucc_mc_alloc(&task->reduce_dbt.scratch_mc_header, 3 * data_size, mtype); if (ucc_unlikely(status != UCC_OK)) { return status; } task->reduce_dbt.scratch = task->reduce_dbt.scratch_mc_header->addr; *task_h = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/reduce/reduce_knomial.c0000664000175000017500000002043415211535620023237 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "reduce.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "utils/ucc_math.h" #include "utils/ucc_dt_reduce.h" #define SAVE_STATE(_phase) \ do { \ task->reduce_kn.phase = _phase; \ } while (0) void ucc_tl_ucp_reduce_knomial_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); int avg_pre_op = UCC_TL_UCP_TEAM_LIB(team)->cfg.reduce_avg_pre_op; ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_rank_t root = (ucc_rank_t)args->root; uint32_t radix = task->reduce_kn.radix; ucc_rank_t vrank = (rank - root + size) % size; void *rbuf = (rank == root) ? args->dst.info.buffer : task->reduce_kn.scratch; ucc_memory_type_t mtype; ucc_datatype_t dt; size_t count, data_size; void *received_vectors, *scratch_offset; ucc_rank_t vpeer, peer, vroot_at_level, root_at_level, pos; uint32_t i; ucc_status_t status; int is_avg; if (root == rank) { count = args->dst.info.count; data_size = count * ucc_dt_size(args->dst.info.datatype); mtype = args->dst.info.mem_type; dt = args->dst.info.datatype; } else { count = args->src.info.count; data_size = count * ucc_dt_size(args->src.info.datatype); mtype = args->src.info.mem_type; dt = args->src.info.datatype; } received_vectors = PTR_OFFSET(task->reduce_kn.scratch, data_size); UCC_REDUCE_KN_PHASE_PROGRESS: if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } UCC_REDUCE_KN_GOTO_PHASE(task->reduce_kn.phase); UCC_REDUCE_KN_PHASE_INIT: while (task->reduce_kn.dist <= task->reduce_kn.max_dist) { if (vrank % task->reduce_kn.dist == 0) { pos = (vrank / task->reduce_kn.dist) % radix; if (pos == 0) { scratch_offset = received_vectors; task->reduce_kn.children_per_cycle = 0; for (i = 1; i < radix; i++) { vpeer = vrank + i * task->reduce_kn.dist; if (vpeer >= size) { break; } else { task->reduce_kn.children_per_cycle += 1; peer = (vpeer + root) % size; UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(scratch_offset, data_size, mtype, peer, team, task), task, out); scratch_offset = PTR_OFFSET(scratch_offset, data_size); } } SAVE_STATE(UCC_REDUCE_KN_PHASE_MULTI); goto UCC_REDUCE_KN_PHASE_PROGRESS; UCC_REDUCE_KN_PHASE_MULTI: if (task->reduce_kn.children_per_cycle && count > 0) { is_avg = args->op == UCC_OP_AVG && (avg_pre_op ? (task->reduce_kn.dist == 1) : (task->reduce_kn.dist == task->reduce_kn.max_dist)); status = ucc_dt_reduce_strided( (task->reduce_kn.dist == 1) ? args->src.info.buffer : rbuf, received_vectors, rbuf, task->reduce_kn.children_per_cycle, count, data_size, dt, args, is_avg ? UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA : 0, AVG_ALPHA(task), task->reduce_kn.executor, &task->reduce_kn.etask); if (ucc_unlikely(UCC_OK != status)) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); task->super.status = status; return; } EXEC_TASK_WAIT(task->reduce_kn.etask); } } else { vroot_at_level = vrank - pos * task->reduce_kn.dist; root_at_level = (vroot_at_level + root) % size; UCPCHECK_GOTO(ucc_tl_ucp_send_nb(task->reduce_kn.scratch, data_size, mtype, root_at_level, team, task), task, out); } } task->reduce_kn.dist *= radix; SAVE_STATE(UCC_REDUCE_KN_PHASE_INIT); goto UCC_REDUCE_KN_PHASE_PROGRESS; } ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_reduce_kn_done", 0); out: return; } ucc_status_t ucc_tl_ucp_reduce_knomial_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); uint32_t radix = task->reduce_kn.radix; ucc_rank_t root = (ucc_rank_t)args->root; ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_rank_t vrank = (rank - root + size) % size; int isleaf = (vrank % radix != 0 || vrank == size - 1); int avg_pre_op = UCC_TL_UCP_TEAM_LIB(team)->cfg.reduce_avg_pre_op; int self_avg = (args->op == UCC_OP_AVG && avg_pre_op && vrank % radix == 0); size_t count; ucc_datatype_t dt; ucc_status_t status; if (root == rank) { count = args->dst.info.count; dt = args->dst.info.datatype; } else { count = args->src.info.count; dt = args->src.info.datatype; } UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_reduce_kn_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if (UCC_IS_INPLACE(*args) && (rank == root)) { args->src.info.buffer = args->dst.info.buffer; } if (isleaf && !self_avg) { task->reduce_kn.scratch = args->src.info.buffer; } if (args->coll_type != UCC_COLL_TYPE_FANIN) { status = ucc_coll_task_get_executor(&task->super, &task->reduce_kn.executor); if (ucc_unlikely(status != UCC_OK)) { return status; } } if (isleaf && self_avg) { /* In case of avg_pre_op, single leaf process which does not take part in first iteration reduction must divide itself by team_size */ status = ucc_dt_reduce(args->src.info.buffer, args->src.info.buffer, task->reduce_kn.scratch, count, dt, args, UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA, 1.0 / (double)(UCC_TL_TEAM_SIZE(TASK_TEAM(task)) * 2), task->reduce_kn.executor, &task->reduce_kn.etask); if (ucc_unlikely(UCC_OK != status)) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); task->super.status = status; return status; } EXEC_TASK_WAIT(task->reduce_kn.etask, status); } task->reduce_kn.dist = 1; task->reduce_kn.phase = UCC_REDUCE_KN_PHASE_INIT; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_reduce_knomial_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); if (task->reduce_kn.scratch_mc_header) { ucc_mc_free(task->reduce_kn.scratch_mc_header); } return ucc_tl_ucp_coll_finalize(coll_task); } ucc-1.8.0/src/components/tl/ucp/reduce/reduce_srg_knomial.c0000664000175000017500000003640015211535620024112 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "reduce.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "coll_patterns/sra_knomial.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "components/mc/ucc_mc.h" #include "../reduce_scatter/reduce_scatter.h" #include "../gather/gather.h" #include "../allgather/allgather.h" /* SRG - scatter-reduce-gather knomial algorithm 1. The algorithm performs collective reduce operation as a sequence of K-nomial Reduce-Scatter followed by K-nomial (with the same radix K) gather. 2. In essence this is an extension of the Bi-nomial SRA algorithm algorithm proposed by Rabenseifner2004 (https://doi.org/10.1007/978-3-540-24685-5_1). The extension adds the support for arbitrary radix. 3. The algorithm targets Large message sizes (ie. optimized for max bandwidth). 4. If number of ranks in the team can not form a full radix subtree (this means the team size is not a power of the radix) then there will be "extra" ranks which don't participate in the main exchange loop. They will send the data to their "proxy" ranks in the beginning and then wait for the response with the final data. 5. The knomial reduce-scatter and gather primitives can be used separately. However, if they are used together as part of SRG reduce one has to provide the same radix for both routines. 6. After the completion of reduce-scatter phase the local result (at non EXTRA ranks) will be located in the dst buffer at an offset that can be computed by the routine from coll_patterns/sra_knomial.h: ucc_sra_kn_get_offset. */ #define GET_DT(_args, _trank) \ (UCC_IS_ROOT(*(_args), _trank)) \ ? (_args)->dst.info.datatype \ : (_args)->src.info.datatype #define GET_MT(_args, _trank) \ (UCC_IS_ROOT(*(_args), _trank)) \ ? (_args)->dst.info.mem_type \ : (_args)->src.info.mem_type #define GET_COUNT(_args, _trank) \ (UCC_IS_ROOT(*(_args), _trank)) \ ? (_args)->dst.info.count \ : (_args)->src.info.count /** * Get the buffers for the reduce-scatter and gather tasks from reduce args */ static void get_rs_ag_buffers(ucc_tl_ucp_schedule_t *rsg_schedule, ucc_coll_args_t *reduce_args, ucc_rank_t trank, void **rs_rbuf, void **rs_sbuf, void **g_rbuf, void **g_sbuf) { if (UCC_IS_ROOT(*reduce_args, trank)) { if (UCC_IS_INPLACE(*reduce_args)) { *rs_sbuf = reduce_args->dst.info.buffer; *rs_rbuf = rsg_schedule->scratch_mc_header->addr; *g_sbuf = rsg_schedule->scratch_mc_header->addr; *g_rbuf = reduce_args->dst.info.buffer; } else { *rs_sbuf = reduce_args->src.info.buffer; *rs_rbuf = reduce_args->dst.info.buffer; // inplace gather case *g_sbuf = reduce_args->dst.info.buffer; *g_rbuf = reduce_args->dst.info.buffer; } } else { *rs_sbuf = reduce_args->src.info.buffer; *rs_rbuf = rsg_schedule->scratch_mc_header->addr; *g_sbuf = rsg_schedule->scratch_mc_header->addr; /* non-root gather has no receive buffer but we use rbuf to pass * scratch space needed for gather knomial algorithm */ *g_rbuf = rsg_schedule->scratch_mc_header->addr; } } static ucc_status_t ucc_tl_ucp_reduce_srg_knomial_frag_start(ucc_coll_task_t *task) { return ucc_schedule_start(task); } static ucc_status_t ucc_tl_ucp_reduce_srg_knomial_frag_finalize(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_status_t status; status = ucc_schedule_finalize(task); ucc_tl_ucp_put_schedule(schedule); return status; } static ucc_status_t ucc_tl_ucp_reduce_srg_knomial_frag_setup(ucc_schedule_pipelined_t *schedule_p, ucc_schedule_t *frag, int frag_num) { ucc_tl_ucp_schedule_t *rsg_schedule = ucc_derived_of(schedule_p, ucc_tl_ucp_schedule_t); int n_frags = schedule_p->super.n_tasks; ucc_coll_args_t *args = &schedule_p->super.super.bargs.args; ucc_rank_t trank = UCC_TL_TEAM_RANK(TASK_TEAM(&schedule_p->super)); size_t dt_size = ucc_dt_size(GET_DT(args, trank)); size_t count = GET_COUNT(args, trank); size_t frag_count; size_t offset; ucc_coll_args_t *targs; void *rs_rbuf, *rs_sbuf, *g_rbuf, *g_sbuf; frag_count = ucc_buffer_block_count(count, n_frags, frag_num); offset = ucc_buffer_block_offset(count, n_frags, frag_num); get_rs_ag_buffers(rsg_schedule, args, trank, &rs_rbuf, &rs_sbuf, &g_rbuf, &g_sbuf); targs = &frag->tasks[0]->bargs.args; /* REDUCE_SCATTER */ targs->src.info.buffer = PTR_OFFSET(rs_sbuf, offset * dt_size); if (UCC_IS_ROOT(*args, trank) && !UCC_IS_INPLACE(*args)) { targs->dst.info.buffer = PTR_OFFSET(rs_rbuf, offset * dt_size); } targs->src.info.count = frag_count; targs->dst.info.count = frag_count; targs = &frag->tasks[1]->bargs.args; /* GATHER */ if (UCC_IS_ROOT(*args, trank) && !UCC_IS_INPLACE(*args)) { targs->src.info.buffer = PTR_OFFSET(g_sbuf, offset * dt_size); } targs->src.info.count = frag_count; if (UCC_IS_ROOT(*args, trank)) { targs->dst.info.buffer = PTR_OFFSET(g_rbuf, offset * dt_size); } targs->dst.info.count = frag_count; return UCC_OK; } static ucc_status_t ucc_tl_ucp_reduce_srg_knomial_frag_init(ucc_base_coll_args_t *coll_args, ucc_schedule_pipelined_t *sp, ucc_base_team_t *team, ucc_schedule_t **frag_p) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t trank = UCC_TL_TEAM_RANK(tl_team); ucc_tl_ucp_schedule_t *rsg_schedule = ucc_derived_of(sp, ucc_tl_ucp_schedule_t); ucc_datatype_t dt = GET_DT(&coll_args->args, trank); size_t dt_size = ucc_dt_size(dt); ucc_memory_type_t mt = GET_MT(&coll_args->args, trank); size_t count = GET_COUNT(&coll_args->args, trank); ucc_base_coll_args_t args = *coll_args; ucc_mrange_uint_t *p = &tl_team->cfg.reduce_srg_kn_radix; int n_frags = sp->super.n_tasks; ucc_kn_radix_t radix, cfg_radix; ucc_schedule_t *schedule; ucc_coll_task_t *g_task, *rs_task; ucc_status_t status; ptrdiff_t scratch_offset; void *rs_rbuf, *rs_sbuf, *g_rbuf, *g_sbuf; scratch_offset = ucc_buffer_block_count(count, n_frags, 0); status = ucc_tl_ucp_get_schedule(tl_team, coll_args, (ucc_tl_ucp_schedule_t **)&schedule); if (ucc_unlikely(UCC_OK != status)) { return status; } cfg_radix = ucc_tl_ucp_get_radix_from_range(tl_team, count * dt_size, mt, p, 4); radix = ucc_knomial_pattern_get_min_radix(cfg_radix, UCC_TL_TEAM_SIZE(tl_team), count); get_rs_ag_buffers(rsg_schedule, &coll_args->args, trank, &rs_rbuf, &rs_sbuf, &g_rbuf, &g_sbuf); /* 1st step of reduce: knomial reduce_scatter. Actual data pointer is set in the setup function */ args.args.flags &= ~UCC_COLL_ARGS_FLAG_IN_PLACE; args.args.src.info.buffer = rs_sbuf; args.args.src.info.count = count; args.args.src.info.datatype = dt; args.args.src.info.mem_type = mt; if (!UCC_IS_ROOT(coll_args->args, trank) || UCC_IS_INPLACE(coll_args->args)) { args.args.dst.info.buffer = PTR_OFFSET(rs_rbuf, rsg_schedule->reduce_srg_kn.frag_offset * dt_size); } args.args.dst.info.count = count; args.args.dst.info.datatype = dt; args.args.dst.info.mem_type = mt; UCC_CHECK_GOTO(ucc_tl_ucp_reduce_scatter_knomial_init_r(&args, team, &rs_task, radix), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, rs_task), out, status); /* reduce scatter task starts when schedule is started */ UCC_CHECK_GOTO(ucc_task_subscribe_dep(&schedule->super, rs_task, UCC_EVENT_SCHEDULE_STARTED), out, status); /* 2nd step of reduce: knomial gather */ if (!UCC_IS_ROOT(coll_args->args, trank) || UCC_IS_INPLACE(coll_args->args)) { args.args.src.info.buffer = PTR_OFFSET(g_sbuf, rsg_schedule->reduce_srg_kn.frag_offset * dt_size); } args.args.src.info.count = count; args.args.src.info.datatype = dt; args.args.src.info.mem_type = mt; if (UCC_IS_ROOT(coll_args->args, trank)) { args.args.dst.info.buffer = PTR_OFFSET(g_rbuf, rsg_schedule->reduce_srg_kn.frag_offset * dt_size); } args.args.dst.info.count = count; args.args.dst.info.datatype = dt; args.args.dst.info.mem_type = mt; if (UCC_IS_ROOT(coll_args->args, trank) && !UCC_IS_INPLACE(coll_args->args)) { args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; } UCC_CHECK_GOTO(ucc_tl_ucp_gather_knomial_init_r(&args, team, &g_task, radix), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, g_task), out, status); /* gather task starts when reduce scatter task is completed */ UCC_CHECK_GOTO(ucc_task_subscribe_dep(rs_task, g_task, UCC_EVENT_COMPLETED), out, status); rsg_schedule->reduce_srg_kn.frag_offset += scratch_offset; schedule->super.finalize = ucc_tl_ucp_reduce_srg_knomial_frag_finalize; schedule->super.post = ucc_tl_ucp_reduce_srg_knomial_frag_start; *frag_p = schedule; return UCC_OK; out: return status; } static ucc_status_t ucc_tl_ucp_reduce_srg_knomial_finalize(ucc_coll_task_t *task) { ucc_tl_ucp_schedule_t *schedule = ucc_derived_of(task, ucc_tl_ucp_schedule_t); ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(schedule, "ucp_reduce_srg_kn_done", 0); if (schedule->scratch_mc_header) { ucc_mc_free(schedule->scratch_mc_header); } status = ucc_schedule_pipelined_finalize(task); ucc_tl_ucp_put_schedule(&schedule->super.super); return status; } ucc_status_t ucc_tl_ucp_reduce_srg_knomial_start(ucc_coll_task_t *task) { UCC_TL_UCP_PROFILE_REQUEST_EVENT(task, "ucp_reduce_srg_kn_start", 0); return ucc_schedule_pipelined_post(task); } static void ucc_tl_ucp_reduce_srg_knomial_get_pipeline_params(ucc_tl_ucp_team_t *team, ucc_memory_type_t mtype, ucc_pipeline_params_t *pp) { ucc_tl_ucp_lib_config_t *cfg = &team->cfg; ucc_mc_attr_t mc_attr; if (!ucc_pipeline_params_is_auto(&cfg->reduce_srg_kn_pipeline)) { *pp = cfg->reduce_srg_kn_pipeline; return; } if (mtype == UCC_MEMORY_TYPE_CUDA) { mc_attr.field_mask = UCC_MC_ATTR_FIELD_FAST_ALLOC_SIZE; ucc_mc_get_attr(&mc_attr, UCC_MEMORY_TYPE_CUDA); pp->threshold = mc_attr.fast_alloc_size; pp->n_frags = 2; pp->order = UCC_PIPELINE_PARALLEL; pp->pdepth = 2; pp->frag_size = mc_attr.fast_alloc_size / pp->pdepth; } else { pp->threshold = SIZE_MAX; pp->n_frags = 0; pp->pdepth = 1; pp->order = UCC_PIPELINE_PARALLEL; pp->frag_size = 0; } } ucc_status_t ucc_tl_ucp_reduce_srg_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_coll_args_t *args = &coll_args->args; ucc_rank_t trank = UCC_TL_TEAM_RANK(tl_team); ucc_memory_type_t mt = GET_MT(args, trank); size_t count = GET_COUNT(args, trank); size_t dt_size = ucc_dt_size(GET_DT(args, trank)); int n_frags, pipeline_depth; ucc_tl_ucp_schedule_t *schedule; ucc_status_t st; ucc_base_coll_args_t bargs; size_t max_frag_count; ucc_pipeline_params_t pipeline_params; st = ucc_tl_ucp_get_schedule(tl_team, coll_args, &schedule); if (ucc_unlikely(UCC_OK != st)) { goto err_out; } schedule->scratch_mc_header = NULL; schedule->reduce_srg_kn.frag_offset = 0; bargs = *coll_args; max_frag_count = (bargs.mask & UCC_BASE_CARGS_MAX_FRAG_COUNT) ? bargs.max_frag_count: count; ucc_tl_ucp_reduce_srg_knomial_get_pipeline_params(tl_team, mt, &pipeline_params); ucc_pipeline_nfrags_pdepth(&pipeline_params, max_frag_count * dt_size, &n_frags, &pipeline_depth); bargs.max_frag_count = ucc_buffer_block_count(max_frag_count, n_frags, 0); if (n_frags > 1) { bargs.mask |= UCC_BASE_CARGS_MAX_FRAG_COUNT; } if (!UCC_IS_ROOT(*args, trank) || UCC_IS_INPLACE(*args)) { st = ucc_mc_alloc(&schedule->scratch_mc_header, bargs.max_frag_count * dt_size * pipeline_depth, mt); if (ucc_unlikely(UCC_OK != st)) { tl_error(team->context->lib, "failed to alloc scratch memory"); goto err_free_schedule; } } st = ucc_schedule_pipelined_init(&bargs, team, ucc_tl_ucp_reduce_srg_knomial_frag_init, ucc_tl_ucp_reduce_srg_knomial_frag_setup, pipeline_depth, n_frags, pipeline_params.order, &schedule->super); if (ucc_unlikely(UCC_OK != st)) { tl_error(team->context->lib, "failed to init pipelined schedule"); goto err_free_scratch; } schedule->super.super.super.finalize = ucc_tl_ucp_reduce_srg_knomial_finalize; schedule->super.super.super.post = ucc_tl_ucp_reduce_srg_knomial_start; *task_h = &schedule->super.super.super; return UCC_OK; err_free_scratch: ucc_mc_free(schedule->scratch_mc_header); err_free_schedule: ucc_tl_ucp_put_schedule(&schedule->super.super); err_out: return st; } ucc-1.8.0/src/components/tl/ucp/reduce/reduce.c0000664000175000017500000000712415211535620021526 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce.h" #include "components/mc/ucc_mc.h" ucc_base_coll_alg_info_t ucc_tl_ucp_reduce_algs[UCC_TL_UCP_REDUCE_ALG_LAST + 1] = { [UCC_TL_UCP_REDUCE_ALG_KNOMIAL] = {.id = UCC_TL_UCP_REDUCE_ALG_KNOMIAL, .name = "knomial", .desc = "reduce over knomial tree with arbitrary radix " "(optimized for latency)"}, [UCC_TL_UCP_REDUCE_ALG_DBT] = {.id = UCC_TL_UCP_REDUCE_ALG_DBT, .name = "dbt", .desc = "reduce over double binary tree where a leaf in one tree " "will be intermediate in other (optimized for BW)"}, [UCC_TL_UCP_REDUCE_ALG_SRG] = {.id = UCC_TL_UCP_REDUCE_ALG_SRG, .name = "srg", .desc = "recursive knomial scatter-reduce followed by knomial " "gather"}, [UCC_TL_UCP_REDUCE_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_reduce_init(ucc_tl_ucp_task_t *task) { ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t myrank = UCC_TL_TEAM_RANK(team); ucc_rank_t team_size = UCC_TL_TEAM_SIZE(team); ucc_rank_t root = args->root; ucc_rank_t vrank = (myrank - root + team_size) % team_size; ucc_status_t status = UCC_OK; ucc_memory_type_t mtype; ucc_datatype_t dt; size_t count, data_size; int isleaf; int self_avg; if (root == myrank) { count = args->dst.info.count; dt = args->dst.info.datatype; mtype = args->dst.info.mem_type; } else { count = args->src.info.count; dt = args->src.info.datatype; mtype = args->src.info.mem_type; } data_size = count * ucc_dt_size(dt); task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_ucp_reduce_knomial_start; task->super.progress = ucc_tl_ucp_reduce_knomial_progress; task->super.finalize = ucc_tl_ucp_reduce_knomial_finalize; task->reduce_kn.radix = ucc_min(UCC_TL_UCP_TEAM_LIB(team)->cfg.reduce_kn_radix, team_size); CALC_KN_TREE_DIST(team_size, task->reduce_kn.radix, task->reduce_kn.max_dist); isleaf = (vrank % task->reduce_kn.radix != 0 || vrank == team_size - 1); self_avg = (vrank % task->reduce_kn.radix == 0 && args->op == UCC_OP_AVG && UCC_TL_UCP_TEAM_LIB(team)->cfg.reduce_avg_pre_op); task->reduce_kn.scratch_mc_header = NULL; if (!isleaf || self_avg) { /* scratch of size radix to fit up to radix - 1 received vectors from its children at each step, and an additional 1 for previous step reduce multi result */ status = ucc_mc_alloc(&task->reduce_kn.scratch_mc_header, task->reduce_kn.radix * data_size, mtype); task->reduce_kn.scratch = task->reduce_kn.scratch_mc_header->addr; } return status; } ucc_status_t ucc_tl_ucp_reduce_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_task_t *task; ucc_status_t status; task = ucc_tl_ucp_init_task(coll_args, team); status = ucc_tl_ucp_reduce_init(task); *task_h = &task->super; return status; } ucc-1.8.0/src/components/tl/ucp/tl_ucp_dpu_offload.h0000664000175000017500000000320515211535620022641 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_UCP_DPU_OFFLOAD_H_ #define UCC_TL_UCP_DPU_OFFLOAD_H_ #include "tl_ucp.h" #include "schedule/ucc_schedule_pipelined.h" #include "components/mc/base/ucc_mc_base.h" #include "components/ec/ucc_ec.h" #include "tl_ucp_tag.h" #define ALLREDUCE_PACKED_KEY_MAX_LEN 1024 typedef struct ucc_tl_ucp_allreduce_sw_global_work_buf_info { void *packed_src_memh; void *packed_dst_memh; } ucc_tl_ucp_allreduce_sw_global_work_buf_info_t; struct ucc_tl_ucp_allreduce_sw_export_buf { ucp_context_h ucp_context; ucp_mem_h memh; void *packed_memh; void *packed_key; size_t packed_key_len; }; typedef struct ucc_tl_ucp_allreduce_sw_host_allgather { void *src_buf; void *dst_buf; char packed_src_key[ALLREDUCE_PACKED_KEY_MAX_LEN]; char packed_dst_key[ALLREDUCE_PACKED_KEY_MAX_LEN]; } ucc_tl_ucp_allreduce_sw_host_allgather_t; typedef struct ucc_tl_ucp_dpu_offload_buf_info { ucp_rkey_h *src_rkeys; //unpacked ucp_rkey_h *dst_rkeys; //unpacked void **sbufs; void **rbufs; struct ucc_tl_ucp_allreduce_sw_export_buf *src_ebuf; struct ucc_tl_ucp_allreduce_sw_export_buf *dst_ebuf; } ucc_tl_ucp_dpu_offload_buf_info_t; ucc_status_t ucc_tl_ucp_allreduce_sliding_window_register( ucp_context_h ucp_context, ucc_tl_ucp_team_t *tl_team, struct ucc_tl_ucp_allreduce_sw_export_buf *ebuf, void *packed_memh); #endif ucc-1.8.0/src/components/tl/ucp/tl_ucp_dpu_offload.c0000664000175000017500000000271615211535620022642 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp_dpu_offload.h" ucc_status_t ucc_tl_ucp_allreduce_sliding_window_register( ucp_context_h ucp_context, ucc_tl_ucp_team_t *tl_team, struct ucc_tl_ucp_allreduce_sw_export_buf *ebuf, void *packed_memh) { ucp_mem_map_params_t params = {0}; ucs_status_t ucs_status, unmap_status; ebuf->ucp_context = ucp_context; params.field_mask = UCP_MEM_MAP_PARAM_FIELD_EXPORTED_MEMH_BUFFER; params.exported_memh_buffer = packed_memh; ucs_status = ucp_mem_map(ucp_context, ¶ms, &ebuf->memh); if (UCS_OK != ucs_status) { tl_error(UCC_TL_TEAM_LIB(tl_team), "import using ucp_mem_map() returned error: %s", ucs_status_string(ucs_status)); return ucs_status_to_ucc_status(ucs_status); } ucs_status = ucp_rkey_pack(ucp_context, ebuf->memh, &ebuf->packed_key, &ebuf->packed_key_len); if (UCS_OK != ucs_status) { unmap_status = ucp_mem_unmap(ucp_context, ebuf->memh); tl_error(UCC_TL_TEAM_LIB(tl_team), "ucp_rkey_pack() returned error: %s%s", ucs_status_string(ucs_status), unmap_status == UCS_OK ? "" : ". While handling this error, unmapping the memh had an error"); return ucs_status_to_ucc_status(ucs_status); } return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/fanin/0000775000175000017500000000000015211535620017733 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/fanin/fanin.h0000664000175000017500000000070115211535620021175 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef FANIN_H_ #define FANIN_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" enum { UCC_TL_UCP_FANIN_ALG_KNOMIAL, UCC_TL_UCP_FANIN_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_fanin_algs[UCC_TL_UCP_FANIN_ALG_LAST + 1]; ucc_status_t ucc_tl_ucp_fanin_init(ucc_tl_ucp_task_t *task); #endif ucc-1.8.0/src/components/tl/ucp/fanin/fanin.c0000664000175000017500000000301515211535620021171 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "fanin.h" #include "../reduce/reduce.h" ucc_base_coll_alg_info_t ucc_tl_ucp_fanin_algs[UCC_TL_UCP_FANIN_ALG_LAST + 1] = { [UCC_TL_UCP_FANIN_ALG_KNOMIAL] = {.id = UCC_TL_UCP_FANIN_ALG_KNOMIAL, .name = "knomial", .desc = "fanin over knomial tree with arbitrary radix"}, [UCC_TL_UCP_FANIN_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_fanin_init(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t team_size = UCC_TL_TEAM_SIZE(team); ucc_status_t status = UCC_OK; TASK_ARGS(task).src.info.buffer = NULL; TASK_ARGS(task).src.info.count = 0; TASK_ARGS(task).src.info.mem_type = UCC_MEMORY_TYPE_UNKNOWN; TASK_ARGS(task).src.info.datatype = UCC_DT_INT8; TASK_ARGS(task).dst.info.buffer = NULL; TASK_ARGS(task).dst.info.count = 0; TASK_ARGS(task).dst.info.mem_type = UCC_MEMORY_TYPE_UNKNOWN; TASK_ARGS(task).dst.info.datatype = UCC_DT_INT8; task->super.post = ucc_tl_ucp_reduce_knomial_start; task->super.progress = ucc_tl_ucp_reduce_knomial_progress; task->reduce_kn.radix = ucc_min(UCC_TL_UCP_TEAM_LIB(team)->cfg.fanin_kn_radix, team_size); CALC_KN_TREE_DIST(team_size, task->reduce_kn.radix, task->reduce_kn.max_dist); return status; } ucc-1.8.0/src/components/tl/ucp/tl_ucp_lib.c0000664000175000017500000001147115211535620021124 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "utils/ucc_malloc.h" /* NOLINTNEXTLINE params is not used */ UCC_CLASS_INIT_FUNC(ucc_tl_ucp_lib_t, const ucc_base_lib_params_t *params, const ucc_base_config_t *config) { const ucc_tl_ucp_lib_config_t *tl_ucp_config = ucc_derived_of(config, ucc_tl_ucp_lib_config_t); int n_plugins = ucc_tl_ucp.super.coll_plugins.n_components; ucc_tl_coll_plugin_iface_t *tlcp; int i; ucc_status_t status; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_lib_t, &ucc_tl_ucp.super, &tl_ucp_config->super); status = ucc_config_clone_table(tl_ucp_config, &self->cfg, ucc_tl_ucp_lib_config_table); if (UCC_OK != status) { return status; } if (tl_ucp_config->kn_radix > 0) { self->cfg.barrier_kn_radix = tl_ucp_config->kn_radix; self->cfg.reduce_scatter_kn_radix = tl_ucp_config->kn_radix; self->cfg.bcast_kn_radix = tl_ucp_config->kn_radix; self->cfg.reduce_kn_radix = tl_ucp_config->kn_radix; self->cfg.scatter_kn_radix = tl_ucp_config->kn_radix; self->cfg.gather_kn_radix = tl_ucp_config->kn_radix; } self->cfg.alltoallv_hybrid_radix = 2; self->tlcp_configs = NULL; if (n_plugins) { self->tlcp_configs = ucc_malloc(sizeof(void*)*n_plugins, "tlcp_configs"); if (!self->tlcp_configs) { tl_error(&self->super, "failed to allocate %zd bytes for tlcp_configs", sizeof(void*)*n_plugins); status = UCC_ERR_NO_MEMORY; goto err; } for (i = 0; i < n_plugins; i++) { tlcp = ucc_derived_of(ucc_tl_ucp.super.coll_plugins.components[i], ucc_tl_coll_plugin_iface_t); tlcp->id = i; self->tlcp_configs[i] = ucc_malloc(tlcp->config.size, "tlcp_cfg"); if (!self->tlcp_configs[i]) { tl_error(&self->super, "failed to allocate %zd bytes for tlcp_cfg", tlcp->config.size); status = UCC_ERR_NO_MEMORY; goto err_cfg; } status = ucc_config_parser_fill_opts(self->tlcp_configs[i], &tlcp->config, params->full_prefix, 0); if (status != UCC_OK) { tl_error(&self->super, "failed to read tlcp config"); goto err_cfg; } } } tl_debug(&self->super, "initialized lib object: %p", self); return UCC_OK; err_cfg: for (i--; i >= 0; i--) { ucc_free(self->tlcp_configs[i]); } err: return status; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_ucp_lib_t) { ucc_config_parser_release_opts(&self->cfg, ucc_tl_ucp_lib_config_table); tl_debug(&self->super, "finalizing lib object: %p", self); } UCC_CLASS_DEFINE(ucc_tl_ucp_lib_t, ucc_tl_lib_t); ucc_status_t ucc_tl_ucp_get_lib_attr(const ucc_base_lib_t *lib, /* NOLINT */ ucc_base_lib_attr_t *base_attr) { ucc_tl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_tl_lib_attr_t); ucs_status_t status; ucp_lib_attr_t params; memset(¶ms, 0, sizeof(ucp_lib_attr_t)); params.field_mask = UCP_LIB_ATTR_FIELD_MAX_THREAD_LEVEL; status = ucp_lib_query(¶ms); if (status != UCS_OK) { ucc_error("failed to query UCP lib attributes"); return ucs_status_to_ucc_status(status); } switch (params.max_thread_level) { case UCS_THREAD_MODE_SINGLE: attr->super.attr.thread_mode = UCC_THREAD_SINGLE; break; case UCS_THREAD_MODE_SERIALIZED: attr->super.attr.thread_mode = UCC_THREAD_SINGLE; break; case UCS_THREAD_MODE_MULTI: attr->super.attr.thread_mode = UCC_THREAD_MULTIPLE; break; default: ucc_error("Unsupported UCS thread mode"); return UCC_ERR_NO_RESOURCE; } attr->super.attr.coll_types = UCC_TL_UCP_SUPPORTED_COLLS; attr->super.flags = UCC_BASE_LIB_FLAG_TEAM_ID_REQUIRED; if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MIN_TEAM_SIZE) { attr->super.min_team_size = lib->min_team_size; } if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MAX_TEAM_SIZE) { attr->super.max_team_size = UCC_RANK_MAX; } return UCC_OK; } ucc_status_t ucc_tl_ucp_get_lib_properties(ucc_base_lib_properties_t *prop) { prop->default_team_size = 2; prop->min_team_size = 2; prop->max_team_size = UCC_RANK_MAX; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/allgatherv/0000775000175000017500000000000015211535620020771 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/allgatherv/allgatherv.h0000664000175000017500000000276415211535620023304 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLGATHERV_H_ #define ALLGATHERV_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" enum { UCC_TL_UCP_ALLGATHERV_ALG_RING, UCC_TL_UCP_ALLGATHERV_ALG_KNOMIAL, UCC_TL_UCP_ALLGATHERV_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_allgatherv_algs[UCC_TL_UCP_ALLGATHERV_ALG_LAST + 1]; #define UCC_TL_UCP_ALLGATHERV_DEFAULT_ALG_SELECT_STR \ "allgatherv:0-4k:@knomial#allgatherv:4k-inf:@ring" char *ucc_tl_ucp_allgatherv_score_str_get(ucc_tl_ucp_team_t *team); static inline int ucc_tl_ucp_allgatherv_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_UCP_ALLGATHERV_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_ucp_allgatherv_algs[i].name)) { break; } } return i; } ucc_status_t ucc_tl_ucp_allgatherv_ring_init_common(ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_allgatherv_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_allgatherv_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_allgatherv_init(ucc_tl_ucp_task_t *task); #endif ucc-1.8.0/src/components/tl/ucp/allgatherv/allgatherv_knomial.c0000664000175000017500000000141115211535620024775 0ustar alastairalastair/** * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allgatherv/allgatherv.h" #include "allgather/allgather.h" ucc_status_t ucc_tl_ucp_allgatherv_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(tl_team); if (!ucc_coll_args_is_disp_contig(&coll_args->args, tsize)) { return ucc_tl_ucp_allgatherv_ring_init(coll_args, team, task_h); } return ucc_tl_ucp_allgather_knomial_init(coll_args, team, task_h); } ucc-1.8.0/src/components/tl/ucp/allgatherv/allgatherv.c0000664000175000017500000000211715211535620023267 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "allgatherv.h" #include "utils/ucc_coll_utils.h" ucc_base_coll_alg_info_t ucc_tl_ucp_allgatherv_algs[UCC_TL_UCP_ALLGATHERV_ALG_LAST + 1] = { [UCC_TL_UCP_ALLGATHERV_ALG_RING] = {.id = UCC_TL_UCP_ALLGATHERV_ALG_RING, .name = "ring", .desc = "O(N) Ring"}, [UCC_TL_UCP_ALLGATHERV_ALG_KNOMIAL] = {.id = UCC_TL_UCP_ALLGATHERV_ALG_KNOMIAL, .name = "knomial", .desc = "recursive k-ing with arbitrary radix"}, [UCC_TL_UCP_ALLGATHERV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_allgatherv_init(ucc_tl_ucp_task_t *task) { if (!ucc_coll_args_is_predefined_dt(&TASK_ARGS(task), UCC_RANK_INVALID)) { tl_error(UCC_TASK_LIB(task), "user defined datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } return ucc_tl_ucp_allgatherv_ring_init_common(task); } ucc-1.8.0/src/components/tl/ucp/allgatherv/allgatherv_ring.c0000664000175000017500000001533015211535620024307 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "allgatherv.h" #include "core/ucc_progress_queue.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "tl_ucp_sendrecv.h" #include "coll_patterns/ring.h" #define MAX_RINGS 8 void ucc_tl_ucp_allgatherv_ring_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_ring_pattern_t *ring = team->cuda_ring; ucc_rank_t ring_id; ucc_rank_t nrings; ucc_rank_t rrank; ptrdiff_t rbuf = (ptrdiff_t)args->dst.info_v.buffer; ucc_memory_type_t rmem = args->dst.info_v.mem_type; size_t rdt_size = ucc_dt_size(args->dst.info_v.datatype); ucc_rank_t send_idx, recv_idx, sendto, recvfrom, step, tsize; size_t data_size, data_displ; size_t block_count, ring_count, ring_offset, base_displ; if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } nrings = ucc_min(MAX_RINGS, ring->num_rings); tsize = ucc_ring_pattern_size(ring, 0); while (task->tagged.send_posted < 1 + nrings * (tsize - 1)) { ucc_assert(task->tagged.send_posted > 0); ucc_assert(task->tagged.recv_posted > 0); ucc_assert(task->tagged.send_posted == task->tagged.recv_posted); step = (ucc_rank_t)((task->tagged.send_posted - 1) / nrings); for (ring_id = 0; ring_id < nrings; ring_id++) { rrank = ucc_ring_pattern_rank(ring, ring_id); sendto = ucc_ring_pattern_get_send_peer(ring, ring_id, rrank); recvfrom = ucc_ring_pattern_get_recv_peer(ring, ring_id, rrank); send_idx = ucc_ring_pattern_get_send_block(ring, ring_id, rrank, step); block_count = ucc_coll_args_get_count( args, args->dst.info_v.counts, send_idx); ring_offset = ucc_buffer_block_offset(block_count, nrings, ring_id); ring_count = ucc_buffer_block_count(block_count, nrings, ring_id); base_displ = ucc_coll_args_get_displacement( args, args->dst.info_v.displacements, send_idx); data_displ = (base_displ + ring_offset) * rdt_size; data_size = ring_count * rdt_size; UCPCHECK_GOTO(ucc_tl_ucp_send_nb((void *)(rbuf + data_displ), data_size, rmem, sendto, team, task), task, out); recv_idx = ucc_ring_pattern_get_recv_block(ring, ring_id, rrank, step); block_count = ucc_coll_args_get_count( args, args->dst.info_v.counts, recv_idx); ring_offset = ucc_buffer_block_offset(block_count, nrings, ring_id); ring_count = ucc_buffer_block_count(block_count, nrings, ring_id); base_displ = ucc_coll_args_get_displacement( args, args->dst.info_v.displacements, recv_idx); data_displ = (base_displ + ring_offset) * rdt_size; data_size = ring_count * rdt_size; UCPCHECK_GOTO(ucc_tl_ucp_recv_nb((void *)(rbuf + data_displ), data_size, rmem, recvfrom, team, task), task, out); } if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } } ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; out: return; } ucc_status_t ucc_tl_ucp_allgatherv_ring_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); void * sbuf = args->src.info.buffer; void * rbuf = args->dst.info_v.buffer; ucc_memory_type_t smem = args->src.info.mem_type; ucc_memory_type_t rmem = args->dst.info_v.mem_type; ucc_rank_t grank = UCC_TL_TEAM_RANK(team); size_t data_size, data_displ, rdt_size; ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if (!UCC_IS_INPLACE(*args)) { /* TODO replace local sendrecv with memcpy? */ rdt_size = ucc_dt_size(args->dst.info_v.datatype); data_displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, grank) * rdt_size; data_size = ucc_coll_args_get_count(args, args->dst.info_v.counts, grank) * rdt_size; UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(PTR_OFFSET(rbuf, data_displ), data_size, rmem, grank, team, task), task, error); UCPCHECK_GOTO(ucc_tl_ucp_send_nb(sbuf, data_size, smem, grank, team, task), task, error); } else { /* to simplify progress fucnction and make it identical for in-place and non in-place */ task->tagged.send_posted = task->tagged.send_completed = 1; task->tagged.recv_posted = task->tagged.recv_completed = 1; } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); error: return task->super.status; } ucc_status_t ucc_tl_ucp_allgatherv_ring_init_common(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_sbgp_t *sbgp; if (!ucc_coll_args_is_predefined_dt(&TASK_ARGS(task), UCC_RANK_INVALID)) { tl_error(UCC_TASK_LIB(task), "user defined datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (team->cfg.use_reordering) { sbgp = ucc_topo_get_sbgp(team->topo, UCC_SBGP_FULL_HOST_ORDERED); task->subset.myrank = sbgp->group_rank; task->subset.map = sbgp->map; } task->super.post = ucc_tl_ucp_allgatherv_ring_start; task->super.progress = ucc_tl_ucp_allgatherv_ring_progress; return UCC_OK; } ucc_status_t ucc_tl_ucp_allgatherv_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_task_t *task; ucc_status_t status; task = ucc_tl_ucp_init_task(coll_args, team); status = ucc_tl_ucp_allgatherv_ring_init_common(task); if (status != UCC_OK) { ucc_tl_ucp_put_task(task); return status; } *task_h = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/gatherv/0000775000175000017500000000000015211535620020300 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/gatherv/gatherv_linear.c0000664000175000017500000001155715211535620023447 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "gatherv.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" static inline ucc_rank_t get_peer(ucc_rank_t rank, ucc_rank_t size, ucc_rank_t step) { return (rank + step) % size; } void ucc_tl_ucp_gatherv_linear_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); void* rbuf = args->dst.info_v.buffer; ucc_memory_type_t rmem = args->dst.info_v.mem_type; ucc_rank_t grank = UCC_TL_TEAM_RANK(team); ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); int polls = 0; ucc_rank_t peer, posts, nreqs; size_t dt_size, data_size, data_displ; if (UCC_IS_ROOT(*args, grank)) { posts = UCC_TL_UCP_TEAM_LIB(team)->cfg.gatherv_linear_num_posts; nreqs = (posts > gsize || posts == 0) ? gsize : posts; dt_size = ucc_dt_size(TASK_ARGS(task).dst.info_v.datatype); while (polls++ < task->n_polls) { ucp_worker_progress(UCC_TL_UCP_TEAM_CTX(team)->worker.ucp_worker); while ((task->tagged.recv_posted < gsize) && ((task->tagged.recv_posted - task->tagged.recv_completed) < nreqs)) { peer = get_peer(grank, gsize, task->tagged.recv_posted); data_size = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer) * dt_size; data_displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, peer) * dt_size; UCPCHECK_GOTO(ucc_tl_ucp_recv_nz(PTR_OFFSET(rbuf, data_displ), data_size, rmem, peer, team, task), task, out); polls = 0; } } if (task->tagged.recv_posted < gsize) { return; } } task->super.status = ucc_tl_ucp_test(task); out: if (task->super.status != UCC_INPROGRESS) { UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_gatherv_linear_done", 0); } } ucc_status_t ucc_tl_ucp_gatherv_linear_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t grank = UCC_TL_TEAM_RANK(team); ucc_memory_type_t smem = args->src.info.mem_type; ucc_memory_type_t rmem = args->dst.info_v.mem_type; void * sbuf = args->src.info.buffer; void *rbuf; size_t dt_size, data_displ, data_size; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_gatherv_linear_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if (UCC_IS_ROOT(*args, grank)) { if (!UCC_IS_INPLACE(*args)) { dt_size = ucc_dt_size(args->dst.info_v.datatype); data_size = ucc_coll_args_get_count(args, args->dst.info_v.counts, grank) * dt_size; data_displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, grank) * dt_size; rbuf = PTR_OFFSET(args->dst.info_v.buffer, data_displ); UCPCHECK_GOTO(ucc_tl_ucp_recv_nz(rbuf, data_size, rmem, grank, team, task), task, error); UCPCHECK_GOTO(ucc_tl_ucp_send_nz(sbuf, data_size, smem, grank, team, task), task, error); } else { /* to simplify progress fucnction and make it identical for in-place and non in-place */ task->tagged.send_posted = task->tagged.send_completed = 1; task->tagged.recv_posted = task->tagged.recv_completed = 1; } } else { dt_size = ucc_dt_size(args->src.info.datatype); data_size = args->src.info.count * dt_size; UCPCHECK_GOTO(ucc_tl_ucp_send_nz(sbuf, data_size, smem, args->root, team, task), task, error); } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); error: return task->super.status; } ucc_status_t ucc_tl_ucp_gatherv_linear_init(ucc_tl_ucp_task_t *task) { task->super.post = ucc_tl_ucp_gatherv_linear_start; task->super.progress = ucc_tl_ucp_gatherv_linear_progress; task->n_polls = ucc_max(1, task->n_polls); return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/gatherv/gatherv.h0000664000175000017500000000072015211535620022110 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef GATHERV_H_ #define GATHERV_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" enum { UCC_TL_UCP_GATHERV_ALG_LINEAR, UCC_TL_UCP_GATHERV_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_gatherv_algs[UCC_TL_UCP_GATHERV_ALG_LAST + 1]; ucc_status_t ucc_tl_ucp_gatherv_init(ucc_tl_ucp_task_t *task); #endif ucc-1.8.0/src/components/tl/ucp/gatherv/gatherv.c0000664000175000017500000000217615211535620022112 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "gatherv.h" #include "utils/ucc_coll_utils.h" ucc_status_t ucc_tl_ucp_gatherv_linear_start(ucc_coll_task_t *task); void ucc_tl_ucp_gatherv_linear_progress(ucc_coll_task_t *task); ucc_base_coll_alg_info_t ucc_tl_ucp_gatherv_algs[UCC_TL_UCP_GATHERV_ALG_LAST + 1] = { [UCC_TL_UCP_GATHERV_ALG_LINEAR] = {.id = UCC_TL_UCP_GATHERV_ALG_LINEAR, .name = "linear", .desc = "linear gatherv algorithm"}, [UCC_TL_UCP_GATHERV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_gatherv_linear_init(ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_gatherv_init(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); if (!ucc_coll_args_is_predefined_dt(args, trank)) { return UCC_ERR_NOT_SUPPORTED; } return ucc_tl_ucp_gatherv_linear_init(task); } ucc-1.8.0/src/components/tl/ucp/tl_ucp_ep.c0000664000175000017500000001040515211535620020756 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "tl_ucp_ep.h" //NOLINTNEXTLINE static void ucc_tl_ucp_err_handler(void *arg, ucp_ep_h ep, ucs_status_t status) { /* In case we don't have OOB barrier, errors are expected. * This cb will suppress UCX from raising errors*/ ; } static inline ucc_status_t ucc_tl_ucp_connect_ep(ucc_tl_ucp_context_t *ctx, int is_service, ucp_ep_h *ep, void *ucp_address) { ucp_worker_h worker = (is_service) ? ctx->service_worker.ucp_worker : ctx->worker.ucp_worker; ucp_ep_params_t ep_params; ucs_status_t status; if (*ep) { /* Already connected */ return UCC_OK; } ep_params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS; ep_params.address = (ucp_address_t *)ucp_address; if (!UCC_TL_CTX_HAS_OOB(ctx)) { ep_params.err_mode = UCP_ERR_HANDLING_MODE_PEER; ep_params.err_handler.cb = ucc_tl_ucp_err_handler; ep_params.err_handler.arg = NULL; ep_params.field_mask |= UCP_EP_PARAM_FIELD_ERR_HANDLING_MODE | UCP_EP_PARAM_FIELD_ERR_HANDLER; } status = ucp_ep_create(worker, &ep_params, ep); if (ucc_unlikely(UCS_OK != status)) { tl_error(ctx->super.super.lib, "ucp returned connect error: %s", ucs_status_string(status)); return ucs_status_to_ucc_status(status); } return UCC_OK; } ucc_status_t ucc_tl_ucp_connect_team_ep(ucc_tl_ucp_team_t *team, ucc_rank_t core_rank, ucp_ep_h *ep) { ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(team); int use_service_worker = USE_SERVICE_WORKER(team); void *addr; addr = ucc_get_team_ep_addr(UCC_TL_CORE_CTX(team), UCC_TL_CORE_TEAM(team), core_rank, ucc_tl_ucp.super.super.id); addr = use_service_worker ? TL_UCP_EP_ADDR_WORKER_SERVICE(addr) : TL_UCP_EP_ADDR_WORKER(addr); return ucc_tl_ucp_connect_ep(ctx, use_service_worker, ep, addr); } /* Finds next non-NULL ep in the storage and returns that handle for closure. In case of "hash" storage it pops the item, in case of "array" sets it to NULL */ static inline ucp_ep_h get_next_ep_to_close(ucc_tl_ucp_worker_t * worker, ucc_tl_ucp_context_t *ctx, int *i) { ucp_ep_h ep = NULL; ucc_rank_t size; if (worker->eps) { size = (ucc_rank_t)ctx->super.super.ucc_context->params.oob.n_oob_eps; while (NULL == ep && (*i) < size) { ep = worker->eps[*i]; worker->eps[*i] = NULL; (*i)++; } } else { ep = tl_ucp_hash_pop(worker->ep_hash); } return ep; } void ucc_tl_ucp_close_eps(ucc_tl_ucp_worker_t * worker, ucc_tl_ucp_context_t *ctx) { int i = 0; ucp_ep_h ep; ucs_status_t status; ucs_status_ptr_t close_req; ucp_request_param_t param; param.op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS; param.flags = 0; // 0 means FLUSH ep = get_next_ep_to_close(worker, ctx, &i); while (ep) { close_req = ucp_ep_close_nbx(ep, ¶m); if (UCS_PTR_IS_PTR(close_req)) { do { ucp_worker_progress(ctx->worker.ucp_worker); if (ctx->cfg.service_worker != 0) { ucp_worker_progress(ctx->service_worker.ucp_worker); } status = ucp_request_check_status(close_req); } while (status == UCS_INPROGRESS); ucp_request_free(close_req); } else { status = UCS_PTR_STATUS(close_req); } ucc_assert(status <= UCS_OK); if (status != UCS_OK) { tl_error(ctx->super.super.lib, "error during ucp ep close, ep %p, status %s", ep, ucs_status_string(status)); } ep = get_next_ep_to_close(worker, ctx, &i); } } ucc-1.8.0/src/components/tl/ucp/coll_plugins/0000775000175000017500000000000015211535620021332 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/coll_plugins/example/0000775000175000017500000000000015211535620022765 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/coll_plugins/example/configure.m40000664000175000017500000000070315211535620025210 0ustar alastairalastair# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # $COPYRIGHT$ # Additional copyrights may follow CHECK_TLCP_REQUIRED("ucp_example") AS_IF([test "$CHECKED_TLCP_REQUIRED" = "y"], [ tlcp_modules="${tlcp_modules}:ucp_example" tlcp_ucp_example_enabled=y ], []) AM_CONDITIONAL([TLCP_UCP_EXAMPLE_ENABLED], [test "$tlcp_ucp_example_enabled" = "y"]) AC_CONFIG_FILES([src/components/tl/ucp/coll_plugins/example/Makefile]) ucc-1.8.0/src/components/tl/ucp/coll_plugins/example/Makefile.am0000664000175000017500000000126215211535620025022 0ustar alastairalastair# # Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # if TLCP_UCP_EXAMPLE_ENABLED sources = example.c module_LTLIBRARIES = libucc_tlcp_ucp_example.la libucc_tlcp_ucp_example_la_SOURCES = $(sources) libucc_tlcp_ucp_example_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(UCX_CPPFLAGS) libucc_tlcp_ucp_example_la_CFLAGS = $(BASE_CFLAGS) libucc_tlcp_ucp_example_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(UCX_LDFLAGS) libucc_tlcp_ucp_example_la_LIBADD = $(UCX_LIBADD) $(UCC_TOP_BUILDDIR)/src/libucc.la \ $(UCC_TOP_BUILDDIR)/src/components/tl/ucp/libucc_tl_ucp.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/tl/ucp/coll_plugins/example/example.c0000664000175000017500000001104415211535620024564 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "components/tl/ucp/tl_ucp.h" #include "components/tl/ucp/tl_ucp_coll.h" #include "core/ucc_progress_queue.h" #include "components/tl/ucp/tl_ucp_sendrecv.h" #include "coll_patterns/recursive_knomial.h" #include "coll_score/ucc_coll_score.h" #include "utils/ucc_math.h" ucc_tl_coll_plugin_iface_t ucc_tlcp_ucp_example; typedef struct ucc_tlcp_ucp_example_config { char *score_str; } ucc_tlcp_ucp_example_config_t; #define CONFIG(_lib) ((ucc_tlcp_ucp_example_config_t*)((_lib)->tlcp_configs[ucc_tlcp_ucp_example.id])) static ucc_config_field_t ucc_tlcp_ucp_example_table[] = { {"TLCP_EXAMPLE_TUNE", "", "Collective score modifier", ucc_offsetof(ucc_tlcp_ucp_example_config_t, score_str), UCC_CONFIG_TYPE_STRING}, {NULL}}; static ucs_config_global_list_entry_t ucc_tlcp_ucp_example_cfg_entry = { .name = "TLCP_EXAMPLE", .prefix = "TL_UCP_", .table = ucc_tlcp_ucp_example_table, .size = sizeof(ucc_tlcp_ucp_example_config_t) }; UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_tlcp_ucp_example_cfg_entry, &ucc_config_global_list); #define UCC_TLCP_UCP_EXAMPLE_SCORE 100 void ucc_tlcp_ucp_example_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); tl_info(TASK_LIB(task), "completing tl_ucp_example coll task"); ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; } ucc_status_t ucc_tlcp_ucp_example_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); tl_info(TASK_LIB(task), "starting tl_ucp_example coll task"); task->super.status = UCC_INPROGRESS; ucc_progress_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); return UCC_OK; } ucc_status_t ucc_tlcp_ucp_example_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task = ucc_tl_ucp_get_task(tl_team); ucc_coll_task_init(&task->super, coll_args, team); task->tagged.tag = tl_team->seq_num; tl_team->seq_num = (tl_team->seq_num + 1) % UCC_TL_UCP_MAX_COLL_TAG; task->super.finalize = ucc_tl_ucp_coll_finalize; task->super.post = ucc_tlcp_ucp_example_start; task->super.progress = ucc_tlcp_ucp_example_progress; *task_h = &task->super; return UCC_OK; } ucc_status_t ucc_tlcp_ucp_example_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p) { ucc_tl_ucp_team_t *team = ucc_derived_of(tl_team, ucc_tl_ucp_team_t); ucc_tl_ucp_lib_t *lib = UCC_TL_UCP_TEAM_LIB(team); const char *score_str; ucc_coll_score_t *score; ucc_status_t status; /* There can be a different logic for different coll_type/mem_type. Right now just init everything the same way. */ status = ucc_coll_score_alloc(&score); if (UCC_OK != status) { tl_error(lib, "failed to alloc score"); return status; } status = ucc_coll_score_add_range(score, UCC_COLL_TYPE_ALLTOALL, UCC_MEMORY_TYPE_HOST, 0, 4096, UCC_TLCP_UCP_EXAMPLE_SCORE, ucc_tlcp_ucp_example_coll_init, tl_team); if (UCC_OK != status) { tl_error(lib, "failed to add range"); return status; } score_str = CONFIG(lib)->score_str; if (strlen(score_str) > 0) { status = ucc_coll_score_update_from_str(score_str, score, UCC_TL_TEAM_SIZE(team), ucc_tlcp_ucp_example_coll_init, &team->super.super, UCC_TLCP_UCP_EXAMPLE_SCORE, NULL); if (status == UCC_ERR_INVALID_PARAM) { /* User provided incorrect input - try to proceed */ status = UCC_OK; } } *score_p = score; return status; } ucc_tl_coll_plugin_iface_t ucc_tlcp_ucp_example = { .super.name = "tl_ucp_example", .super.score = UCC_TLCP_UCP_EXAMPLE_SCORE, .config.table = ucc_tlcp_ucp_example_table, .config.size = sizeof(ucc_tlcp_ucp_example_config_t), .get_scores = ucc_tlcp_ucp_example_get_scores }; ucc-1.8.0/src/components/tl/ucp/tl_ucp_reduce.h0000664000175000017500000000037115211535620021627 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_UCP_REDUCE_H_ #define UCC_TL_UCP_REDUCE_H_ #include "tl_ucp_coll.h" #include "utils/ucc_dt_reduce.h" #endif ucc-1.8.0/src/components/tl/ucp/gather/0000775000175000017500000000000015211535620020112 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/gather/gather_knomial.c0000664000175000017500000003632715211535620023255 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "gather.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "utils/ucc_math.h" #include "coll_patterns/sra_knomial.h" #define SAVE_STATE(_phase) \ do { \ task->gather_kn.phase = _phase; \ } while (0) static inline uint32_t calc_buffer_size(ucc_rank_t vrank, uint32_t radix, ucc_rank_t tsize) { uint32_t radix_valuation; if (vrank == 0) { return tsize; } radix_valuation = calc_valuation(vrank, radix); return (uint32_t)ucc_min(pow(radix, radix_valuation), tsize - vrank); } /* gather knomial is used as regular gather collective and as part of reduce SRG */ void ucc_tl_ucp_gather_knomial_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t root = (ucc_rank_t)args->root; ucc_kn_radix_t radix = task->gather_kn.radix; ucc_rank_t vrank = VRANK(rank, root, tsize); ucc_memory_type_t mtype = args->src.info.mem_type; ucc_status_t status = UCC_OK; ucc_knomial_pattern_t *p = &task->gather_kn.p; size_t dt_size = ucc_dt_size(args->src.info.datatype); size_t data_size = args->src.info.count * dt_size; ucc_coll_type_t ct = args->coll_type; size_t msg_size, peer_seg_count; void *scratch_offset; ucc_rank_t vpeer, peer, vroot_at_level, root_at_level, num_blocks; ucc_kn_radix_t loop_step; ptrdiff_t peer_seg_offset; if (task->gather_kn.p.node_type == KN_NODE_EXTRA) { ucc_assert(ct == UCC_COLL_TYPE_REDUCE); task->super.status = UCC_OK; return; } UCC_GATHER_KN_GOTO_PHASE(task->gather_kn.phase); UCC_GATHER_KN_PHASE_INIT: while (!ucc_knomial_pattern_loop_done(p)) { if (task->tagged.send_posted > 0) { goto UCC_GATHER_KN_PHASE_PROGRESS; } scratch_offset = task->gather_kn.scratch; vroot_at_level = ucc_knomial_pattern_get_base_rank(p, vrank); if (vroot_at_level == vrank) { for (loop_step = 1; loop_step < radix; loop_step++) { vpeer = ucc_knomial_pattern_get_loop_peer(p, vrank, loop_step); if (vpeer == UCC_KN_PEER_NULL) { continue; } ucc_kn_g_pattern_peer_seg(vpeer, p, &peer_seg_count, &peer_seg_offset); peer = INV_VRANK(vpeer, root, tsize); if (vrank != 0) { msg_size = peer_seg_count * dt_size; if (args->coll_type != UCC_COLL_TYPE_GATHER) { scratch_offset = PTR_OFFSET(task->gather_kn.scratch, peer_seg_offset * dt_size); } else { scratch_offset = PTR_OFFSET(scratch_offset, data_size * task->gather_kn.dist); } UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(scratch_offset, msg_size, mtype, peer, team, task), task, out); } else { /* the root is a particular case because it must aggregate the data sorted by ranks */ scratch_offset = PTR_OFFSET(task->gather_kn.scratch, data_size * peer); num_blocks = ucc_min(task->gather_kn.dist, tsize - vpeer); /* check if received data correspond to contiguous ranks */ if ((ct == UCC_COLL_TYPE_REDUCE) || (num_blocks <= tsize - peer)) { msg_size = peer_seg_count * dt_size; if (args->coll_type != UCC_COLL_TYPE_GATHER) { scratch_offset = PTR_OFFSET(task->gather_kn.scratch, peer_seg_offset * dt_size); } UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(scratch_offset, msg_size, mtype, peer, team, task), task, out); } else { /* in this case, data must be split in two at the destination buffer */ msg_size = data_size * (tsize - peer); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(scratch_offset, msg_size, mtype, peer, team, task), task, out); msg_size = data_size * (num_blocks - (tsize - peer)); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(task->gather_kn.scratch, msg_size, mtype, peer, team, task), task, out); } } } if ((ct != UCC_COLL_TYPE_REDUCE) && ucc_knomial_pattern_loop_first_iteration(p)) { msg_size = data_size; if (rank != root) { status = ucc_mc_memcpy(task->gather_kn.scratch, args->src.info.buffer, msg_size, args->src.info.mem_type, mtype); } else if (!UCC_IS_INPLACE(*args)) { status = ucc_mc_memcpy( PTR_OFFSET(task->gather_kn.scratch, data_size * rank), args->src.info.buffer, msg_size, args->src.info.mem_type, mtype); } if (ucc_unlikely(UCC_OK != status)) { task->super.status = status; return; } } else { if (rank == root && ucc_knomial_pattern_loop_first_iteration(p) && !UCC_IS_INPLACE(*args)) { ucc_kn_g_pattern_peer_seg(vrank, p, &peer_seg_count, &peer_seg_offset); status = ucc_mc_memcpy( PTR_OFFSET(task->gather_kn.scratch, peer_seg_offset * dt_size), PTR_OFFSET(args->src.info.buffer, peer_seg_offset * dt_size), peer_seg_count * dt_size, args->src.info.mem_type, mtype); if (ucc_unlikely(UCC_OK != status)) { task->super.status = status; return; } } } } else { root_at_level = INV_VRANK(vroot_at_level, root, tsize); num_blocks = ucc_min(task->gather_kn.dist, tsize - vrank); if ((ct == UCC_COLL_TYPE_REDUCE) || (root_at_level != root) || (num_blocks <= tsize - rank)) { ucc_kn_g_pattern_peer_seg(vrank, p, &peer_seg_count, &peer_seg_offset); msg_size = peer_seg_count * dt_size; if (args->coll_type == UCC_COLL_TYPE_GATHER) { scratch_offset = task->gather_kn.scratch; } else { scratch_offset = PTR_OFFSET(task->gather_kn.scratch, peer_seg_offset * dt_size); } UCPCHECK_GOTO(ucc_tl_ucp_send_nb(scratch_offset, msg_size, mtype, root_at_level, team, task), task, out); } else { // need to split in this case due to root and tree topology msg_size = data_size * (tsize - rank); UCPCHECK_GOTO(ucc_tl_ucp_send_nb(task->gather_kn.scratch, msg_size, mtype, root_at_level, team, task), task, out); msg_size = data_size * (num_blocks - (tsize - rank)); UCPCHECK_GOTO( ucc_tl_ucp_send_nb(PTR_OFFSET(task->gather_kn.scratch, data_size * (tsize - rank)), msg_size, mtype, root_at_level, team, task), task, out); } } UCC_GATHER_KN_PHASE_PROGRESS: if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_GATHER_KN_PHASE_PROGRESS); return; } task->gather_kn.dist *= radix; ucc_kn_g_pattern_next_iter(p); } ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_gather_kn_done", 0); out: return; } ucc_status_t ucc_tl_ucp_gather_knomial_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t root = (ucc_rank_t)args->root; ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); if (root == trank && UCC_IS_INPLACE(*args)) { args->src.info = args->dst.info; args->src.info.count = args->dst.info.count / size; } if (args->coll_type == UCC_COLL_TYPE_GATHER) { ucc_kn_g_pattern_init(size, VRANK(trank, root, size), task->gather_kn.radix, args->src.info.count * size, &task->gather_kn.p); } else { /* reduce srg */ ucc_assert(args->coll_type == UCC_COLL_TYPE_REDUCE); task->gather_kn.scratch = args->dst.info.buffer; ucc_kn_gx_pattern_init(size, VRANK(trank, root, size), task->gather_kn.radix, args->dst.info.count, &task->gather_kn.p); } UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_gather_kn_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); task->gather_kn.dist = 1; task->gather_kn.phase = UCC_GATHER_KN_PHASE_INIT; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_gather_knomial_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); if (task->gather_kn.scratch_mc_header) { ucc_mc_free(task->gather_kn.scratch_mc_header); } return ucc_tl_ucp_coll_finalize(coll_task); } ucc_status_t ucc_tl_ucp_gather_knomial_init_common(ucc_tl_ucp_task_t *task, ucc_kn_radix_t radix) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t root = args->root; ucc_rank_t vrank = VRANK(trank, root, tsize); ucc_status_t status = UCC_OK; ucc_memory_type_t mtype; ucc_datatype_t dt; size_t count, data_size; uint32_t buffer_size; int is_leaf; if (UCC_IS_ROOT(*args, trank)) { count = args->dst.info.count; dt = args->dst.info.datatype; mtype = args->dst.info.mem_type; } else { count = args->src.info.count; dt = args->src.info.datatype; mtype = args->src.info.mem_type; } data_size = count * ucc_dt_size(dt); task->super.post = ucc_tl_ucp_gather_knomial_start; task->super.progress = ucc_tl_ucp_gather_knomial_progress; task->super.finalize = ucc_tl_ucp_gather_knomial_finalize; task->gather_kn.radix = radix; CALC_KN_TREE_DIST(tsize, task->gather_kn.radix, task->gather_kn.max_dist); task->gather_kn.scratch_mc_header = NULL; if (args->coll_type == UCC_COLL_TYPE_REDUCE) { task->gather_kn.scratch = args->dst.info.buffer; } else { ucc_assert(args->coll_type == UCC_COLL_TYPE_GATHER); is_leaf = ((vrank % radix != 0) || (vrank == tsize - 1)); if (vrank == 0) { task->gather_kn.scratch = args->dst.info.buffer; } else if (is_leaf) { task->gather_kn.scratch = args->src.info.buffer; } else { buffer_size = calc_buffer_size(vrank, task->gather_kn.radix, tsize); status = ucc_mc_alloc(&task->gather_kn.scratch_mc_header, buffer_size * data_size, mtype); task->gather_kn.scratch = task->gather_kn.scratch_mc_header->addr; } } return status; } ucc_status_t ucc_tl_ucp_gather_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(tl_team); ucc_tl_ucp_task_t *task; ucc_status_t status; ucc_kn_radix_t radix; task = ucc_tl_ucp_init_task(coll_args, team); if (ucc_unlikely(!task)) { return UCC_ERR_NO_MEMORY; } radix = ucc_min(UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.gather_kn_radix, tsize); status = ucc_tl_ucp_gather_knomial_init_common(task, radix); if (ucc_unlikely(status != UCC_OK)) { ucc_tl_ucp_put_task(task); return status; } *task_h = &task->super; return UCC_OK; } ucc_status_t ucc_tl_ucp_gather_knomial_init_r(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, ucc_kn_radix_t radix) { ucc_tl_ucp_task_t *task; ucc_status_t status; task = ucc_tl_ucp_init_task(coll_args, team); if (ucc_unlikely(!task)) { return UCC_ERR_NO_MEMORY; } status = ucc_tl_ucp_gather_knomial_init_common(task, radix); if (ucc_unlikely(status != UCC_OK)) { ucc_tl_ucp_put_task(task); return status; } *task_h = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/gather/gather.h0000664000175000017500000000401315211535620021533 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef GATHER_H_ #define GATHER_H_ #include "tl_ucp_coll.h" #include "components/mc/ucc_mc.h" enum { UCC_TL_UCP_GATHER_ALG_KNOMIAL, UCC_TL_UCP_GATHER_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_gather_algs[UCC_TL_UCP_GATHER_ALG_LAST + 1]; /* A set of convenience macros used to implement sw based progress of the gather algorithm that uses kn pattern */ enum { UCC_GATHER_KN_PHASE_INIT, UCC_GATHER_KN_PHASE_PROGRESS, /* checks progress */ }; #define UCC_GATHER_KN_CHECK_PHASE(_p) \ case _p: \ goto _p; #define UCC_GATHER_KN_GOTO_PHASE(_phase) \ do { \ switch (_phase) { \ UCC_GATHER_KN_CHECK_PHASE(UCC_GATHER_KN_PHASE_PROGRESS); \ UCC_GATHER_KN_CHECK_PHASE(UCC_GATHER_KN_PHASE_INIT); \ }; \ } while (0) ucc_status_t ucc_tl_ucp_gather_init(ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_gather_knomial_start(ucc_coll_task_t *task); void ucc_tl_ucp_gather_knomial_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_gather_knomial_finalize(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_gather_knomial_init_common(ucc_tl_ucp_task_t *task, ucc_kn_radix_t radix); /* Internal interface with custom radix */ ucc_status_t ucc_tl_ucp_gather_knomial_init_r(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, ucc_kn_radix_t radix); #endif ucc-1.8.0/src/components/tl/ucp/gather/gather.c0000664000175000017500000000164215211535620021533 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "gather.h" ucc_base_coll_alg_info_t ucc_tl_ucp_gather_algs[UCC_TL_UCP_GATHER_ALG_LAST + 1] = { [UCC_TL_UCP_GATHER_ALG_KNOMIAL] = {.id = UCC_TL_UCP_GATHER_ALG_KNOMIAL, .name = "knomial", .desc = "gather over knomial tree with arbitrary radix " "(optimized for latency)"}, [UCC_TL_UCP_GATHER_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_gather_init(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_kn_radix_t radix; radix = ucc_min(UCC_TL_UCP_TEAM_LIB(team)->cfg.gather_kn_radix, size); return ucc_tl_ucp_gather_knomial_init_common(task, radix); } ucc-1.8.0/src/components/tl/ucp/configure.m40000664000175000017500000000125215211535620021063 0ustar alastairalastair# # Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # tl_ucp_enabled=n CHECK_TLS_REQUIRED(["ucp"]) AS_IF([test "$CHECKED_TL_REQUIRED" = "y"], [ if test $ucx_happy = "yes"; then tl_modules="${tl_modules}:ucp" tl_ucp_enabled=y CHECK_NEED_TL_PROFILING(["tl_ucp"]) AS_IF([test "$TL_PROFILING_REQUIRED" = "y"], [ AC_DEFINE([HAVE_PROFILING_TL_UCP], [1], [Enable profiling for TL UCP]) prof_modules="${prof_modules}:tl_ucp" ], []) fi ], []) AM_CONDITIONAL([TL_UCP_ENABLED], [test "$tl_ucp_enabled" = "y"]) AC_CONFIG_FILES([src/components/tl/ucp/Makefile]) ucc-1.8.0/src/components/tl/ucp/tl_ucp_coll.c0000664000175000017500000002617715211535620021320 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "tl_ucp_coll.h" #include "components/mc/ucc_mc.h" #include "core/ucc_team.h" #include "barrier/barrier.h" #include "alltoall/alltoall.h" #include "alltoallv/alltoallv.h" #include "allreduce/allreduce.h" #include "allgather/allgather.h" #include "allgatherv/allgatherv.h" #include "reduce_scatter/reduce_scatter.h" #include "reduce_scatterv/reduce_scatterv.h" #include "bcast/bcast.h" #include "reduce/reduce.h" #include "gather/gather.h" #include "gatherv/gatherv.h" #include "fanin/fanin.h" #include "fanout/fanout.h" #include "scatterv/scatterv.h" const ucc_tl_ucp_default_alg_desc_t ucc_tl_ucp_default_alg_descs[UCC_TL_UCP_N_DEFAULT_ALG_SELECT_STR] = { { .select_str = NULL, .str_get_fn = ucc_tl_ucp_allgather_score_str_get }, { .select_str = UCC_TL_UCP_ALLGATHERV_DEFAULT_ALG_SELECT_STR, .str_get_fn = NULL }, { .select_str = NULL, .str_get_fn = ucc_tl_ucp_alltoall_score_str_get }, { .select_str = UCC_TL_UCP_ALLREDUCE_DEFAULT_ALG_SELECT_STR, .str_get_fn = NULL }, { .select_str = UCC_TL_UCP_BCAST_DEFAULT_ALG_SELECT_STR, .str_get_fn = NULL }, { .select_str = UCC_TL_UCP_REDUCE_DEFAULT_ALG_SELECT_STR, .str_get_fn = NULL }, { .select_str = UCC_TL_UCP_REDUCE_SCATTER_DEFAULT_ALG_SELECT_STR, .str_get_fn = NULL }, { .select_str = UCC_TL_UCP_REDUCE_SCATTERV_DEFAULT_ALG_SELECT_STR, .str_get_fn = NULL }, { .select_str = UCC_TL_UCP_ALLTOALLV_DEFAULT_ALG_SELECT_STR, .str_get_fn = NULL } }; ucc_status_t ucc_tl_ucp_team_default_score_str_alloc(ucc_tl_ucp_team_t *team, char *default_select_str[UCC_TL_UCP_N_DEFAULT_ALG_SELECT_STR]) { ucc_status_t st = UCC_OK; int i; for (i = 0; i < UCC_TL_UCP_N_DEFAULT_ALG_SELECT_STR; i++) { if (ucc_tl_ucp_default_alg_descs[i].select_str) { default_select_str[i] = strdup(ucc_tl_ucp_default_alg_descs[i].select_str); } else { default_select_str[i] = ucc_tl_ucp_default_alg_descs[i].str_get_fn(team); } if (!default_select_str[i]) { st = UCC_ERR_NO_MEMORY; goto exit; } } exit: return st; } void ucc_tl_ucp_team_default_score_str_free( char *default_select_str[UCC_TL_UCP_N_DEFAULT_ALG_SELECT_STR]) { int i; for (i = 0; i < UCC_TL_UCP_N_DEFAULT_ALG_SELECT_STR; i++) { ucc_free(default_select_str[i]); } } ucc_status_t ucc_tl_ucp_coll_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); tl_trace(UCC_TASK_LIB(task), "finalizing task %p", task); ucc_tl_ucp_put_task(task); return UCC_OK; } ucc_status_t ucc_tl_ucp_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_task_t *task = ucc_tl_ucp_init_task(coll_args, team); ucc_status_t status; switch (coll_args->args.coll_type) { case UCC_COLL_TYPE_BARRIER: status = ucc_tl_ucp_barrier_init(task); break; case UCC_COLL_TYPE_ALLTOALL: status = ucc_tl_ucp_alltoall_init(task); break; case UCC_COLL_TYPE_ALLTOALLV: status = ucc_tl_ucp_alltoallv_init(task); break; case UCC_COLL_TYPE_ALLREDUCE: status = ucc_tl_ucp_allreduce_init(task); break; case UCC_COLL_TYPE_ALLGATHER: status = ucc_tl_ucp_allgather_init(task); break; case UCC_COLL_TYPE_ALLGATHERV: status = ucc_tl_ucp_allgatherv_init(task); break; case UCC_COLL_TYPE_BCAST: status = ucc_tl_ucp_bcast_init(task); break; case UCC_COLL_TYPE_REDUCE: status = ucc_tl_ucp_reduce_init(task); break; case UCC_COLL_TYPE_GATHER: status = ucc_tl_ucp_gather_init(task); break; case UCC_COLL_TYPE_FANIN: status = ucc_tl_ucp_fanin_init(task); break; case UCC_COLL_TYPE_FANOUT: status = ucc_tl_ucp_fanout_init(task); break; case UCC_COLL_TYPE_SCATTERV: status = ucc_tl_ucp_scatterv_init(task); break; case UCC_COLL_TYPE_GATHERV: status = ucc_tl_ucp_gatherv_init(task); break; default: status = UCC_ERR_NOT_SUPPORTED; } if (ucc_unlikely(status != UCC_OK)) { ucc_tl_ucp_put_task(task); return status; } tl_trace(team->context->lib, "init coll req %p", task); *task_h = &task->super; return status; } static inline int alg_id_from_str(ucc_coll_type_t coll_type, const char *str) { switch (coll_type) { case UCC_COLL_TYPE_ALLGATHER: return ucc_tl_ucp_allgather_alg_from_str(str); case UCC_COLL_TYPE_ALLGATHERV: return ucc_tl_ucp_allgatherv_alg_from_str(str); case UCC_COLL_TYPE_ALLREDUCE: return ucc_tl_ucp_allreduce_alg_from_str(str); case UCC_COLL_TYPE_ALLTOALL: return ucc_tl_ucp_alltoall_alg_from_str(str); case UCC_COLL_TYPE_ALLTOALLV: return ucc_tl_ucp_alltoallv_alg_from_str(str); case UCC_COLL_TYPE_BCAST: return ucc_tl_ucp_bcast_alg_from_str(str); case UCC_COLL_TYPE_REDUCE: return ucc_tl_ucp_reduce_alg_from_str(str); case UCC_COLL_TYPE_REDUCE_SCATTER: return ucc_tl_ucp_reduce_scatter_alg_from_str(str); case UCC_COLL_TYPE_REDUCE_SCATTERV: return ucc_tl_ucp_reduce_scatterv_alg_from_str(str); default: break; } return -1; } ucc_status_t ucc_tl_ucp_alg_id_to_init(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, //NOLINT ucc_base_coll_init_fn_t *init) { ucc_status_t status = UCC_OK; if (alg_id_str) { alg_id = alg_id_from_str(coll_type, alg_id_str); } switch (coll_type) { case UCC_COLL_TYPE_ALLGATHER: switch (alg_id) { case UCC_TL_UCP_ALLGATHER_ALG_KNOMIAL: *init = ucc_tl_ucp_allgather_knomial_init; break; case UCC_TL_UCP_ALLGATHER_ALG_RING: *init = ucc_tl_ucp_allgather_ring_init; break; case UCC_TL_UCP_ALLGATHER_ALG_NEIGHBOR: *init = ucc_tl_ucp_allgather_neighbor_init; break; case UCC_TL_UCP_ALLGATHER_ALG_BRUCK: *init = ucc_tl_ucp_allgather_bruck_init; break; case UCC_TL_UCP_ALLGATHER_ALG_SPARBIT: *init = ucc_tl_ucp_allgather_sparbit_init; break; case UCC_TL_UCP_ALLGATHER_ALG_LINEAR: *init = ucc_tl_ucp_allgather_linear_init; break; case UCC_TL_UCP_ALLGATHER_ALG_LINEAR_BATCHED: *init = ucc_tl_ucp_allgather_linear_batched_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_ALLGATHERV: switch (alg_id) { case UCC_TL_UCP_ALLGATHERV_ALG_KNOMIAL: *init = ucc_tl_ucp_allgatherv_knomial_init; break; case UCC_TL_UCP_ALLGATHERV_ALG_RING: *init = ucc_tl_ucp_allgatherv_ring_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_ALLREDUCE: switch (alg_id) { case UCC_TL_UCP_ALLREDUCE_ALG_KNOMIAL: *init = ucc_tl_ucp_allreduce_knomial_init; break; case UCC_TL_UCP_ALLREDUCE_ALG_SRA_KNOMIAL: *init = ucc_tl_ucp_allreduce_sra_knomial_init; break; case UCC_TL_UCP_ALLREDUCE_ALG_DBT: *init = ucc_tl_ucp_allreduce_dbt_init; break; case UCC_TL_UCP_ALLREDUCE_ALG_SLIDING_WINDOW: *init = ucc_tl_ucp_allreduce_sliding_window_init; break; case UCC_TL_UCP_ALLREDUCE_ALG_RING: *init = ucc_tl_ucp_allreduce_ring_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_BCAST: switch (alg_id) { case UCC_TL_UCP_BCAST_ALG_KNOMIAL: *init = ucc_tl_ucp_bcast_knomial_init; break; case UCC_TL_UCP_BCAST_ALG_SAG_KNOMIAL: *init = ucc_tl_ucp_bcast_sag_knomial_init; break; case UCC_TL_UCP_BCAST_ALG_DBT: *init = ucc_tl_ucp_bcast_dbt_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_ALLTOALL: switch (alg_id) { case UCC_TL_UCP_ALLTOALL_ALG_PAIRWISE: *init = ucc_tl_ucp_alltoall_pairwise_init; break; case UCC_TL_UCP_ALLTOALL_ALG_BRUCK: *init = ucc_tl_ucp_alltoall_bruck_init; break; case UCC_TL_UCP_ALLTOALL_ALG_ONESIDED: *init = ucc_tl_ucp_alltoall_onesided_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_ALLTOALLV: switch (alg_id) { case UCC_TL_UCP_ALLTOALLV_ALG_PAIRWISE: *init = ucc_tl_ucp_alltoallv_pairwise_init; break; case UCC_TL_UCP_ALLTOALLV_ALG_HYBRID: *init = ucc_tl_ucp_alltoallv_hybrid_init; break; case UCC_TL_UCP_ALLTOALLV_ALG_ONESIDED: *init = ucc_tl_ucp_alltoallv_onesided_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_REDUCE: switch (alg_id) { case UCC_TL_UCP_REDUCE_ALG_KNOMIAL: *init = ucc_tl_ucp_reduce_knomial_init; break; case UCC_TL_UCP_REDUCE_ALG_DBT: *init = ucc_tl_ucp_reduce_dbt_init; break; case UCC_TL_UCP_REDUCE_ALG_SRG: *init = ucc_tl_ucp_reduce_srg_knomial_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_REDUCE_SCATTER: switch (alg_id) { case UCC_TL_UCP_REDUCE_SCATTER_ALG_RING: *init = ucc_tl_ucp_reduce_scatter_ring_init; break; case UCC_TL_UCP_REDUCE_SCATTER_ALG_KNOMIAL: *init = ucc_tl_ucp_reduce_scatter_knomial_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; case UCC_COLL_TYPE_REDUCE_SCATTERV: switch (alg_id) { case UCC_TL_UCP_REDUCE_SCATTERV_ALG_RING: *init = ucc_tl_ucp_reduce_scatterv_ring_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; default: status = UCC_ERR_NOT_SUPPORTED; break; } return status; } ucc-1.8.0/src/components/tl/ucp/tl_ucp_context.c0000664000175000017500000011261415211535620022043 0ustar alastairalastair/** * Copyright (c) 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "tl_ucp_tag.h" #include "tl_ucp_coll.h" #include "tl_ucp_ep.h" #include "utils/ucc_math.h" #include "utils/ucc_string.h" #include "utils/arch/cpu.h" #include "schedule/ucc_schedule_pipelined.h" #include "tl_ucp_copy.h" #include #include "tl_ucp_sendrecv.h" #define UCP_CHECK(function, msg, go, ctx) \ status = function; \ if (UCS_OK != status) { \ tl_error(ctx->super.super.lib, msg ", %s", ucs_status_string(status)); \ ucc_status = ucs_status_to_ucc_status(status); \ goto go; \ } #define CHECK(test, msg, go, return_status, ctx) \ if (test) { \ tl_error(ctx->super.super.lib, msg); \ ucc_status = return_status; \ goto go; \ } #define MAX_NET_DEVICES_STR_LEN 512 unsigned ucc_tl_ucp_service_worker_progress(void *progress_arg) { ucc_tl_ucp_context_t *ctx = (ucc_tl_ucp_context_t *)progress_arg; int throttling_count = ucc_atomic_fadd32(&ctx->service_worker_throttling_count, 1); if (throttling_count == ctx->cfg.service_throttling_thresh) { ctx->service_worker_throttling_count = 0; return ucp_worker_progress(ctx->service_worker.ucp_worker); } return 0; } static inline ucc_status_t ucc_tl_ucp_eps_ephash_init(const ucc_base_context_params_t *params, ucc_tl_ucp_context_t * ctx, tl_ucp_ep_hash_t **ep_hash, ucp_ep_h **eps) { if (params->context->params.mask & UCC_CONTEXT_PARAM_FIELD_OOB) { /* Global ctx mode, we will have ctx_map so can use array for eps */ *eps = ucc_calloc(params->context->params.oob.n_oob_eps, sizeof(ucp_ep_h), "ucp_eps"); if (!(*eps)) { tl_error(ctx->super.super.lib, "failed to allocate %zd bytes for ucp_eps", params->context->params.oob.n_oob_eps * sizeof(ucp_ep_h)); return UCC_ERR_NO_MEMORY; } } else { *eps = NULL; *ep_hash = kh_init(tl_ucp_ep_hash); } return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_context_service_init(const char *prefix, ucp_params_t ucp_params, ucp_worker_params_t worker_params, const ucc_base_context_params_t *params, ucc_tl_ucp_context_t * ctx) { ucp_config_t *ucp_config = NULL; ucc_status_t ucc_status; ucp_context_h ucp_context_service; ucp_worker_h ucp_worker_service; ucs_status_t status; char * service_prefix; ucc_status = ucc_str_concat(prefix, "_SERVICE", &service_prefix); if (UCC_OK != ucc_status) { tl_error(ctx->super.super.lib, "failed to concat service prefix str"); return ucc_status; } UCP_CHECK(ucp_config_read(service_prefix, NULL, &ucp_config), "failed to read ucp configuration", err_cfg_read, ctx); ucc_free(service_prefix); service_prefix = NULL; UCP_CHECK(ucp_init(&ucp_params, ucp_config, &ucp_context_service), "failed to init ucp context for service worker", err_cfg, ctx); ucp_config_release(ucp_config); ucp_config = NULL; UCP_CHECK(ucp_worker_create(ucp_context_service, &worker_params, &ucp_worker_service), "failed to create ucp service worker", err_worker_create, ctx); ctx->service_worker.ucp_context = ucp_context_service; ctx->service_worker.ucp_worker = ucp_worker_service; ctx->service_worker.worker_address = NULL; CHECK(UCC_OK != ucc_tl_ucp_eps_ephash_init(params, ctx, &ctx->service_worker.ep_hash, &ctx->service_worker.eps), "failed to allocate memory for endpoint storage for service worker", err_thread_mode, UCC_ERR_NO_MESSAGE, ctx); ctx->service_worker_throttling_count = 0; CHECK(UCC_OK != ucc_context_progress_register( params->context, (ucc_context_progress_fn_t)ucc_tl_ucp_service_worker_progress, ctx), "failed to register progress function for service worker", err_thread_mode, UCC_ERR_NO_MESSAGE, ctx); return UCC_OK; err_thread_mode: ucp_worker_destroy(ucp_worker_service); err_worker_create: ucp_cleanup(ucp_context_service); err_cfg: if (ucp_config) { ucp_config_release(ucp_config); } err_cfg_read: if (service_prefix) { ucc_free(service_prefix); } return ucc_status; } UCC_CLASS_INIT_FUNC(ucc_tl_ucp_context_t, const ucc_base_context_params_t *params, const ucc_base_config_t *config) { ucc_tl_ucp_context_config_t *tl_ucp_config = ucc_derived_of(config, ucc_tl_ucp_context_config_t); ucp_config_t *ucp_config = NULL; ucc_tl_ucp_lib_t *lib; ucc_status_t ucc_status = UCC_OK; ucp_context_attr_t context_attr; ucp_worker_params_t worker_params; ucp_worker_attr_t worker_attr; ucp_params_t ucp_params; ucp_context_h ucp_context; ucp_worker_h ucp_worker; ucs_status_t status; char * prefix; char net_devices_str[MAX_NET_DEVICES_STR_LEN]; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_context_t, &tl_ucp_config->super, params->context); memcpy(&self->cfg, tl_ucp_config, sizeof(*tl_ucp_config)); self->thread_mode = params->thread_mode; lib = ucc_derived_of(self->super.super.lib, ucc_tl_ucp_lib_t); prefix = strdup(params->prefix); if (!prefix) { tl_error(self->super.super.lib, "failed to duplicate prefix str"); return UCC_ERR_NO_MEMORY; } prefix[strlen(prefix) - 1] = '\0'; UCP_CHECK(ucp_config_read(prefix, NULL, &ucp_config), "failed to read ucp configuration", err_cfg_read, self); if (!tl_ucp_config->memtype_copy_enable) { UCP_CHECK(ucp_config_modify(ucp_config, "MEMTYPE_COPY_ENABLE", "no"), "failed to set memtype copy enable option for UCX", err_cfg, self); if (tl_ucp_config->local_copy_type == UCC_TL_UCP_LOCAL_COPY_TYPE_MC) { tl_info(self->super.super.lib, "memtype copy is disabled in UCX, " "using local copy type MC might lead to deadlocks in CUDA " "applications"); } else if (tl_ucp_config->local_copy_type == UCC_TL_UCP_LOCAL_COPY_TYPE_EC) { tl_info(self->super.super.lib, "memtype copy is disabled in UCX, " "using local copy type EC might lead to deadlocks in CUDA " "applications when CUDA kernel depends on collective " "communication and stream is not provided to the collective"); } } if (!ucc_config_names_array_is_all(&self->super.super.ucc_context->net_devices)) { ucc_status = ucc_config_names_array_to_string( &self->super.super.ucc_context->net_devices, net_devices_str, MAX_NET_DEVICES_STR_LEN); if (UCC_OK != ucc_status) { tl_error(self->super.super.lib, "failed to convert net devices to string"); goto err_cfg; } UCP_CHECK(ucp_config_modify(ucp_config, "NET_DEVICES", net_devices_str), "failed to set net devices", err_cfg, self); } ucp_params.field_mask = UCP_PARAM_FIELD_FEATURES | UCP_PARAM_FIELD_TAG_SENDER_MASK | UCP_PARAM_FIELD_NAME; ucp_params.features = UCP_FEATURE_TAG | UCP_FEATURE_AM | UCP_FEATURE_RMA | UCP_FEATURE_AMO64; if (self->cfg.exported_memory_handle) { ucp_params.features |= UCP_FEATURE_EXPORTED_MEMH; } ucp_params.tag_sender_mask = UCC_TL_UCP_TAG_SENDER_MASK; ucp_params.name = "UCC_UCP_CONTEXT"; if (params->estimated_num_ppn > 0) { ucp_params.field_mask |= UCP_PARAM_FIELD_ESTIMATED_NUM_PPN; ucp_params.estimated_num_ppn = params->estimated_num_ppn; } if (params->estimated_num_eps > 0) { ucp_params.field_mask |= UCP_PARAM_FIELD_ESTIMATED_NUM_EPS; ucp_params.estimated_num_eps = params->estimated_num_eps; } #ifdef HAVE_UCX_NODE_LOCAL_ID if (params->node_local_id != UCC_ULUNITS_AUTO) { ucp_params.field_mask |= UCP_PARAM_FIELD_NODE_LOCAL_ID; ucp_params.node_local_id = params->node_local_id; } #endif UCP_CHECK(ucp_init(&ucp_params, ucp_config, &ucp_context), "failed to init ucp context", err_cfg, self); ucp_config_release(ucp_config); ucp_config = NULL; context_attr.field_mask = UCP_ATTR_FIELD_MEMORY_TYPES; UCP_CHECK(ucp_context_query(ucp_context, &context_attr), "failed to query supported memory types", err_worker_create, self); self->ucp_memory_types = context_attr.memory_types; worker_params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE; switch (params->thread_mode) { case UCC_THREAD_SINGLE: case UCC_THREAD_FUNNELED: worker_params.thread_mode = UCS_THREAD_MODE_SINGLE; self->sendrecv_cbs.ucc_tl_ucp_send_nb = ucc_tl_ucp_send_nb_st; self->sendrecv_cbs.ucc_tl_ucp_recv_nb = ucc_tl_ucp_recv_nb_st; self->sendrecv_cbs.ucc_tl_ucp_send_nz = ucc_tl_ucp_send_nz_st; self->sendrecv_cbs.ucc_tl_ucp_recv_nz = ucc_tl_ucp_recv_nz_st; self->sendrecv_cbs.send_cb = ucc_tl_ucp_send_completion_cb_st; self->sendrecv_cbs.recv_cb = ucc_tl_ucp_recv_completion_cb_st; self->sendrecv_cbs.p2p_counter_inc = ucc_tl_ucp_send_recv_counter_inc_st; break; case UCC_THREAD_MULTIPLE: worker_params.thread_mode = UCS_THREAD_MODE_MULTI; self->sendrecv_cbs.ucc_tl_ucp_send_nb = ucc_tl_ucp_send_nb_mt; self->sendrecv_cbs.ucc_tl_ucp_recv_nb = ucc_tl_ucp_recv_nb_mt; self->sendrecv_cbs.ucc_tl_ucp_send_nz = ucc_tl_ucp_send_nz_mt; self->sendrecv_cbs.ucc_tl_ucp_recv_nz = ucc_tl_ucp_recv_nz_mt; self->sendrecv_cbs.send_cb = ucc_tl_ucp_send_completion_cb_mt; self->sendrecv_cbs.recv_cb = ucc_tl_ucp_recv_completion_cb_mt; self->sendrecv_cbs.p2p_counter_inc = ucc_tl_ucp_send_recv_counter_inc_mt; break; default: /* unreachable */ ucc_assert(0); break; } UCP_CHECK(ucp_worker_create(ucp_context, &worker_params, &ucp_worker), "failed to create ucp worker", err_worker_create, self); if (params->thread_mode == UCC_THREAD_MULTIPLE) { worker_attr.field_mask = UCP_WORKER_ATTR_FIELD_THREAD_MODE; ucp_worker_query(ucp_worker, &worker_attr); if (worker_attr.thread_mode != UCS_THREAD_MODE_MULTI) { tl_error(self->super.super.lib, "thread mode multiple is not supported by ucp worker"); ucc_status = UCC_ERR_NOT_SUPPORTED; goto err_thread_mode; } } self->worker.ucp_context = ucp_context; self->worker.ucp_worker = ucp_worker; self->worker.worker_address = NULL; self->topo_required = (((lib->cfg.use_topo == UCC_TRY || lib->cfg.use_topo == UCC_AUTO) && (self->super.super.ucc_context->params.mask & UCC_CONTEXT_PARAM_FIELD_OOB)) || lib->cfg.use_topo == UCC_YES) ? 1 : 0; ucc_status = ucc_mpool_init( &self->req_mp, 0, ucc_max(sizeof(ucc_tl_ucp_task_t), sizeof(ucc_tl_ucp_schedule_t)), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_coll_task_mpool_ops, params->thread_mode, "tl_ucp_req_mp"); if (UCC_OK != ucc_status) { tl_error(self->super.super.lib, "failed to initialize tl_ucp_req mpool"); goto err_thread_mode; } CHECK(UCC_OK != ucc_context_progress_register( params->context, (ucc_context_progress_fn_t)ucp_worker_progress, self->worker.ucp_worker), "failed to register progress function", err_thread_mode, UCC_ERR_NO_MESSAGE, self); self->remote_info = NULL; self->n_rinfo_segs = 0; self->rkeys = NULL; if (params->params.mask & UCC_CONTEXT_PARAM_FIELD_MEM_PARAMS && params->params.mask & UCC_CONTEXT_PARAM_FIELD_OOB) { ucc_status = ucc_tl_ucp_ctx_remote_populate( self, params->params.mem_params, params->params.oob); if (UCC_OK != ucc_status) { tl_error(self->super.super.lib, "failed to gather RMA information"); goto err_thread_mode; } } CHECK(UCC_OK != ucc_tl_ucp_eps_ephash_init( params, self, &self->worker.ep_hash, &self->worker.eps), "failed to allocate memory for endpoint storage", err_thread_mode, UCC_ERR_NO_MESSAGE, self); if (self->cfg.service_worker) { CHECK(UCC_OK != ucc_tl_ucp_context_service_init( prefix, ucp_params, worker_params, params, self), "failed to init service worker", err_cfg, UCC_ERR_NO_MESSAGE, self); } ucc_free(prefix); prefix = NULL; switch (self->cfg.local_copy_type) { case UCC_TL_UCP_LOCAL_COPY_TYPE_MC: self->copy.post = ucc_tl_ucp_mc_copy_post; self->copy.test = ucc_tl_ucp_mc_copy_test; self->copy.finalize = ucc_tl_ucp_mc_copy_finalize; tl_debug(self->super.super.lib, "using MC for local copy"); break; case UCC_TL_UCP_LOCAL_COPY_TYPE_EC: case UCC_TL_UCP_LOCAL_COPY_TYPE_AUTO: self->cfg.local_copy_type = UCC_TL_UCP_LOCAL_COPY_TYPE_EC; self->copy.post = ucc_tl_ucp_ec_copy_post; self->copy.test = ucc_tl_ucp_ec_copy_test; self->copy.finalize = ucc_tl_ucp_ec_copy_finalize; tl_debug(self->super.super.lib, "using EC for local copy"); break; case UCC_TL_UCP_LOCAL_COPY_TYPE_UCP: self->copy.post = ucc_tl_ucp_ucp_copy_post; self->copy.test = ucc_tl_ucp_ucp_copy_test; self->copy.finalize = ucc_tl_ucp_ucp_copy_finalize; tl_debug(self->super.super.lib, "using UCP for local copy"); break; default: self->copy.post = ucc_tl_ucp_ec_copy_post; self->copy.test = ucc_tl_ucp_ec_copy_test; self->copy.finalize = ucc_tl_ucp_ec_copy_finalize; tl_error(self->super.super.lib, "not valid copy type: %d, using EC copy instead", self->cfg.local_copy_type); }; tl_debug(self->super.super.lib, "initialized tl context: %p", self); return UCC_OK; err_thread_mode: ucp_worker_destroy(ucp_worker); err_worker_create: ucp_cleanup(ucp_context); err_cfg: if (ucp_config) { ucp_config_release(ucp_config); } err_cfg_read: if (prefix) { ucc_free(prefix); } return ucc_status; } static void ucc_tl_ucp_context_barrier(ucc_tl_ucp_context_t *ctx, ucc_context_oob_coll_t *oob) { char *rbuf; ucc_status_t status; char sbuf; void *req; if (ucc_unlikely(oob->n_oob_eps < 2)) { return; } rbuf = ucc_malloc(sizeof(char) * oob->n_oob_eps, "tmp_barrier"); if (!rbuf) { tl_error(ctx->super.super.lib, "failed to allocate %zd bytes for tmp barrier array", sizeof(char) * oob->n_oob_eps); return; } if (UCC_OK == oob->allgather(&sbuf, rbuf, sizeof(char), oob->coll_info, &req)) { ucc_assert(req != NULL); while (UCC_OK != (status = oob->req_test(req))) { ucp_worker_progress(ctx->worker.ucp_worker); if (ctx->cfg.service_worker != 0) { ucp_worker_progress(ctx->service_worker.ucp_worker); } if (status < 0) { tl_error(ctx->super.super.lib, "failed to test oob req"); break; } } oob->req_free(req); } ucc_free(rbuf); } ucc_status_t ucc_tl_ucp_rinfo_destroy(ucc_tl_ucp_context_t *ctx) { ucc_rank_t size = UCC_TL_CTX_OOB(ctx).n_oob_eps; int i, j; for (i = 0; i < size; i++) { for (j = 0; j < ctx->n_rinfo_segs; j++) { if (UCC_TL_UCP_REMOTE_RKEY(ctx, i, j)) { ucp_rkey_destroy(UCC_TL_UCP_REMOTE_RKEY(ctx, i, j)); } } } for (i = 0; i < ctx->n_rinfo_segs; i++) { if (ctx->remote_info[i].mem_h) { ucp_mem_unmap(ctx->worker.ucp_context, ctx->remote_info[i].mem_h); } if (ctx->remote_info[i].packed_key) { ucp_rkey_buffer_release(ctx->remote_info[i].packed_key); } } ucc_free(ctx->remote_info); ucc_free(ctx->rkeys); ctx->remote_info = NULL; ctx->rkeys = NULL; return UCC_OK; } static inline void ucc_tl_ucp_eps_cleanup(ucc_tl_ucp_worker_t * worker, ucc_tl_ucp_context_t *ctx) { ucc_tl_ucp_close_eps(worker, ctx); if (worker->eps) { ucc_free(worker->eps); } else { kh_destroy(tl_ucp_ep_hash, worker->ep_hash); } } static inline void ucc_tl_ucp_worker_cleanup(ucc_tl_ucp_worker_t worker) { if (worker.worker_address) { ucp_worker_release_address(worker.ucp_worker, worker.worker_address); } ucp_worker_destroy(worker.ucp_worker); ucp_cleanup(worker.ucp_context); } UCC_CLASS_CLEANUP_FUNC(ucc_tl_ucp_context_t) { tl_debug(self->super.super.lib, "finalizing tl context: %p", self); if (self->remote_info) { ucc_tl_ucp_rinfo_destroy(self); } ucc_context_progress_deregister( self->super.super.ucc_context, (ucc_context_progress_fn_t)ucp_worker_progress, self->worker.ucp_worker); if (self->cfg.service_worker != 0) { ucc_context_progress_deregister( self->super.super.ucc_context, (ucc_context_progress_fn_t)ucc_tl_ucp_service_worker_progress, self); } ucc_mpool_cleanup(&self->req_mp, 1); ucc_tl_ucp_eps_cleanup(&self->worker, self); if (self->cfg.service_worker != 0) { ucc_tl_ucp_eps_cleanup(&self->service_worker, self); } if (UCC_TL_CTX_HAS_OOB(self)) { ucc_tl_ucp_context_barrier(self, &UCC_TL_CTX_OOB(self)); } ucc_tl_ucp_worker_cleanup(self->worker); if (self->cfg.service_worker != 0) { ucc_tl_ucp_worker_cleanup(self->service_worker); } } UCC_CLASS_DEFINE(ucc_tl_ucp_context_t, ucc_tl_context_t); ucc_status_t ucc_tl_ucp_populate_rcache(void *addr, size_t length, ucs_memory_type_t mem_type, ucc_tl_ucp_context_t *ctx) { ucp_mem_map_params_t mmap_params; ucp_mem_h mh; ucs_status_t status; mmap_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH | UCP_MEM_MAP_PARAM_FIELD_MEMORY_TYPE; mmap_params.address = addr; mmap_params.length = length; mmap_params.memory_type = mem_type; /* do map and umap to populate the cache */ status = ucp_mem_map(ctx->worker.ucp_context, &mmap_params, &mh); if (ucc_unlikely(status != UCS_OK)) { return ucs_status_to_ucc_status(status); } status = ucp_mem_unmap(ctx->worker.ucp_context, mh); if (ucc_unlikely(status != UCS_OK)) { return ucs_status_to_ucc_status(status); } return UCC_OK; } ucc_status_t ucc_tl_ucp_ctx_remote_populate(ucc_tl_ucp_context_t * ctx, ucc_mem_map_params_t map, ucc_context_oob_coll_t oob) { uint32_t size = oob.n_oob_eps; uint64_t nsegs = map.n_segments; ucp_mem_map_params_t mmap_params; ucp_mem_h mh; ucs_status_t status; ucc_status_t ucc_status; int i; if (size < 2) { tl_error( ctx->super.super.lib, "oob.n_oob_eps set to incorrect value for remote exchange (%d)", size); return UCC_ERR_INVALID_PARAM; } if (nsegs > MAX_NR_SEGMENTS) { tl_error(ctx->super.super.lib, "cannot map more than %d segments", MAX_NR_SEGMENTS); return UCC_ERR_INVALID_PARAM; } ctx->rkeys = (ucp_rkey_h *)ucc_calloc(sizeof(ucp_rkey_h), nsegs * size, "ucp_ctx_rkeys"); if (NULL == ctx->rkeys) { tl_error(ctx->super.super.lib, "failed to allocated %zu bytes", sizeof(ucp_rkey_h) * nsegs * size); return UCC_ERR_NO_MEMORY; } ctx->remote_info = (ucc_tl_ucp_remote_info_t *)ucc_calloc( nsegs, sizeof(ucc_tl_ucp_remote_info_t), "ucp_remote_info"); if (NULL == ctx->remote_info) { tl_error(ctx->super.super.lib, "failed to allocated %zu bytes", sizeof(ucc_tl_ucp_remote_info_t) * nsegs); ucc_status = UCC_ERR_NO_MEMORY; goto fail_alloc_remote_segs; } for (i = 0; i < nsegs; i++) { mmap_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH; mmap_params.address = map.segments[i].address; mmap_params.length = map.segments[i].len; status = ucp_mem_map(ctx->worker.ucp_context, &mmap_params, &mh); if (UCS_OK != status) { tl_error(ctx->super.super.lib, "ucp_mem_map failed with error code: %d", status); ucc_status = ucs_status_to_ucc_status(status); goto fail_mem_map; } ctx->remote_info[i].mem_h = (void *)mh; status = ucp_rkey_pack(ctx->worker.ucp_context, mh, &ctx->remote_info[i].packed_key, &ctx->remote_info[i].packed_key_len); if (UCS_OK != status) { tl_error(ctx->super.super.lib, "failed to pack UCP key with error code: %d", status); ucc_status = ucs_status_to_ucc_status(status); goto fail_mem_map; } ctx->remote_info[i].va_base = map.segments[i].address; ctx->remote_info[i].len = map.segments[i].len; } ctx->n_rinfo_segs = nsegs; return UCC_OK; fail_mem_map: for (i = 0; i < nsegs; i++) { if (ctx->remote_info[i].mem_h) { ucp_mem_unmap(ctx->worker.ucp_context, ctx->remote_info[i].mem_h); } if (ctx->remote_info[i].packed_key) { ucp_rkey_buffer_release(ctx->remote_info[i].packed_key); } } fail_alloc_remote_segs: ucc_free(ctx->remote_info); ucc_free(ctx->rkeys); return ucc_status; } static void ucc_tl_ucp_ctx_remote_pack_data(ucc_tl_ucp_context_t *ctx, void *pack) { uint64_t nsegs = ctx->n_rinfo_segs; uint64_t offset = 0; size_t section_offset = sizeof(uint64_t) * nsegs; void *keys; uint64_t *rvas; uint64_t *lens; uint64_t *key_sizes; int i; /* pack into one data object in following order: */ /* rva, len, pack sizes, packed keys */ rvas = pack; lens = PTR_OFFSET(pack, section_offset); key_sizes = PTR_OFFSET(pack, (section_offset * 2)); keys = PTR_OFFSET(pack, (section_offset * 3)); for (i = 0; i < nsegs; i++) { rvas[i] = (uint64_t)ctx->remote_info[i].va_base; lens[i] = ctx->remote_info[i].len; key_sizes[i] = ctx->remote_info[i].packed_key_len; memcpy(PTR_OFFSET(keys, offset), ctx->remote_info[i].packed_key, ctx->remote_info[i].packed_key_len); offset += ctx->remote_info[i].packed_key_len; } } ucc_status_t ucc_tl_ucp_mem_map_memhbuf(ucc_tl_ucp_context_t *ctx, void *pack_buffer, ucp_mem_h *mh) { ucp_mem_map_params_t mmap_params; ucs_status_t status; void *packed_memh; *mh = NULL; /* unpack here */ packed_memh = UCC_TL_UCP_MEMH_TL_PACKED_MEMH(pack_buffer); mmap_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_EXPORTED_MEMH_BUFFER; mmap_params.exported_memh_buffer = packed_memh; status = ucp_mem_map(ctx->worker.ucp_context, &mmap_params, mh); if (UCS_OK != status) { tl_error(ctx->super.super.lib, "ucp_mem_map failed with error code: %s", ucs_status_string(status)); return ucs_status_to_ucc_status(status); } return UCC_OK; } ucc_status_t ucc_tl_ucp_mem_map_export(ucc_tl_ucp_context_t *ctx, void *address, size_t len, ucc_mem_map_mode_t mode, ucc_tl_ucp_memh_data_t *m_data) { ucc_status_t ucc_status = UCC_OK; ucp_mem_h mh; ucp_mem_map_params_t mmap_params; ucs_status_t status; if (mode == UCC_MEM_MAP_MODE_EXPORT) { mmap_params.address = address; mmap_params.length = len; mmap_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH; status = ucp_mem_map(ctx->worker.ucp_context, &mmap_params, &mh); if (UCS_OK != status) { tl_error(ctx->super.super.lib, "ucp_mem_map failed with error code: %d", status); ucc_status = ucs_status_to_ucc_status(status); return ucc_status; } m_data->rinfo.mem_h = mh; m_data->rinfo.va_base = address; m_data->rinfo.len = len; } return ucc_status; } ucc_status_t ucc_tl_ucp_mem_map_offload_import(ucc_tl_ucp_context_t *ctx, void *address, size_t len, ucc_tl_ucp_memh_data_t *m_data, ucc_mem_map_memh_t *l_memh, ucc_mem_map_tl_t *tl_h) { int i = 0; size_t offset = 0; ucp_mem_h mh; ucc_status_t ucc_status; char *name; size_t *packed_size; for (; i < l_memh->num_tls; i++) { name = (char *)PTR_OFFSET(l_memh->pack_buffer, offset); packed_size = (size_t *)PTR_OFFSET(l_memh->pack_buffer, offset + UCC_MEM_MAP_TL_NAME_LEN); if (strncmp(tl_h->tl_name, name, UCC_MEM_MAP_TL_NAME_LEN - 1) == 0) { break; } /* this is not the index, skip this section of buffer if exists */ offset += *packed_size; } if (i == l_memh->num_tls) { tl_error(ctx->super.super.lib, "unable to find TL UCP in memory handle"); return UCC_ERR_NOT_FOUND; } ucc_status = ucc_tl_ucp_mem_map_memhbuf( ctx, PTR_OFFSET(l_memh->pack_buffer, offset), &mh); if (ucc_status != UCC_OK) { tl_error(ctx->super.super.lib, "ucp_mem_map failed to map memh"); return ucc_status; } m_data->rinfo.mem_h = mh; /* used for xgvmi */ m_data->rinfo.va_base = address; m_data->rinfo.len = len; return UCC_OK; } ucc_status_t ucc_tl_ucp_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h) { ucc_tl_ucp_context_t *ctx = ucc_derived_of(context, ucc_tl_ucp_context_t); ucc_status_t ucc_status = UCC_OK; ucc_tl_ucp_memh_data_t *m_data; m_data = ucc_calloc(1, sizeof(ucc_tl_ucp_memh_data_t), "tl data"); if (!m_data) { tl_error(ctx->super.super.lib, "failed to allocate tl data"); return UCC_ERR_NO_MEMORY; } tl_h->tl_data = m_data; // copy name information for look ups later strncpy(tl_h->tl_name, "ucp", UCC_MEM_MAP_TL_NAME_LEN - 1); /* nothing to do for UCC_MEM_MAP_MODE_EXPORT_OFFLOAD and UCC_MEM_MAP_MODE_IMPORT */ if (mode == UCC_MEM_MAP_MODE_EXPORT) { ucc_status = ucc_tl_ucp_mem_map_export(ctx, memh->address, memh->len, mode, m_data); if (UCC_OK != ucc_status) { tl_error(ctx->super.super.lib, "failed to export memory handles"); ucc_free(m_data); return ucc_status; } } else if (mode == UCC_MEM_MAP_MODE_IMPORT_OFFLOAD) { ucc_status = ucc_tl_ucp_mem_map_offload_import(ctx, memh->address, memh->len, m_data, memh, tl_h); if (UCC_OK != ucc_status) { tl_error(ctx->super.super.lib, "failed to import memory handle"); ucc_free(m_data); return ucc_status; } } return ucc_status; } ucc_status_t ucc_tl_ucp_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *memh) { ucc_tl_ucp_context_t *ctx = ucc_derived_of(context, ucc_tl_ucp_context_t); ucc_tl_ucp_memh_data_t *data; ucs_status_t status; if (!memh) { return UCC_OK; } data = (ucc_tl_ucp_memh_data_t *)memh->tl_data; if (mode == UCC_MEM_MAP_MODE_EXPORT || mode == UCC_MEM_MAP_MODE_EXPORT_OFFLOAD) { status = ucp_mem_unmap(ctx->worker.ucp_context, data->rinfo.mem_h); if (status != UCS_OK) { tl_error(ctx->super.super.lib, "ucp_mem_unmap failed with error code %d", status); return ucs_status_to_ucc_status(status); } /* Free the UCP-allocated buffers that are not freed by the core context */ if (data->packed_memh) { ucp_memh_buffer_release(data->packed_memh, NULL); data->packed_memh = NULL; } if (data->rinfo.packed_key) { ucp_rkey_buffer_release(data->rinfo.packed_key); data->rinfo.packed_key = NULL; } } else if (mode == UCC_MEM_MAP_MODE_IMPORT || mode == UCC_MEM_MAP_MODE_IMPORT_OFFLOAD) { // need to free rkeys (data->rkey) , packed memh (data->packed_memh) if (data->packed_memh) { ucp_memh_buffer_release(data->packed_memh, NULL); } if (data->rinfo.packed_key) { ucp_rkey_buffer_release(data->rinfo.packed_key); } if (data->rkey) { ucp_rkey_destroy(data->rkey); } } else { ucc_error("Unknown mem map mode entered: %d", mode); return UCC_ERR_INVALID_PARAM; } /* Free the TL data structure */ if (data) { ucc_free(data); memh->tl_data = NULL; } return UCC_OK; } ucc_status_t ucc_tl_ucp_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **pack_buffer) { ucc_tl_ucp_context_t *ctx = ucc_derived_of(context, ucc_tl_ucp_context_t); ucc_tl_ucp_memh_data_t *data = tl_h->tl_data; ucc_status_t ucc_status = UCC_OK; void *packed_buffer; size_t *key_size; size_t *memh_size; ucp_memh_pack_params_t pack_params; ucs_status_t status; if (!data || !data->rinfo.mem_h) { tl_error(ctx->super.super.lib, "unable to pack memory handle without TL handles"); return UCC_ERR_INVALID_PARAM; } if (ctx->cfg.exported_memory_handle && mode == UCC_MEM_MAP_MODE_EXPORT) { pack_params.field_mask = UCP_MEMH_PACK_PARAM_FIELD_FLAGS; pack_params.flags = UCP_MEMH_PACK_FLAG_EXPORT; status = ucp_memh_pack(data->rinfo.mem_h, &pack_params, &data->packed_memh, &data->packed_memh_len); if (status != UCS_OK) { // we don't support memory pack, or it failed tl_error(ctx->super.super.lib, "ucp_memh_pack() returned error %s", ucs_status_string(status)); data->packed_memh = NULL; data->packed_memh_len = 0; return ucs_status_to_ucc_status(status); } } status = ucp_rkey_pack(ctx->worker.ucp_context, data->rinfo.mem_h, &data->rinfo.packed_key, &data->rinfo.packed_key_len); if (status != UCS_OK) { tl_error(ctx->super.super.lib, "unable to pack rkey with error %s", ucs_status_string(status)); ucc_status = ucs_status_to_ucc_status(status); goto failed_rkey_pack; } /* * data order * * packed_key_size | packed_memh_size | packed_key | packed_memh */ packed_buffer = ucc_malloc(sizeof(size_t) * 2 + data->packed_memh_len + data->rinfo.packed_key_len, "packed buffer"); if (!packed_buffer) { tl_error(ctx->super.super.lib, "failed to allocate a packed buffer of size %lu", data->packed_memh_len + data->rinfo.packed_key_len + 2 * sizeof(size_t)); ucc_status = UCC_ERR_NO_MEMORY; goto failed_alloc_buffer; } key_size = packed_buffer; *key_size = data->rinfo.packed_key_len; memh_size = PTR_OFFSET(packed_buffer, sizeof(size_t)); *memh_size = data->packed_memh_len; memcpy(PTR_OFFSET(packed_buffer, UCC_TL_UCP_MEMH_TL_HEADER_SIZE), data->rinfo.packed_key, *key_size); memcpy(PTR_OFFSET(packed_buffer, UCC_TL_UCP_MEMH_TL_HEADER_SIZE + data->rinfo.packed_key_len), data->packed_memh, *memh_size); tl_h->packed_size = UCC_TL_UCP_MEMH_TL_HEADER_SIZE + data->packed_memh_len + data->rinfo.packed_key_len; *pack_buffer = packed_buffer; return ucc_status; failed_alloc_buffer: ucp_rkey_buffer_release(data->rinfo.packed_key); failed_rkey_pack: if (data->packed_memh) { ucp_memh_buffer_release(data->packed_memh, NULL); } return ucc_status; } ucc_status_t ucc_tl_ucp_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *attr) { ucc_tl_ucp_context_t *ctx = ucc_derived_of(context, ucc_tl_ucp_context_t); uint64_t * offset = (uint64_t *)attr->attr.ctx_addr; ucs_status_t ucs_status; size_t packed_length; int i; ucc_base_ctx_attr_clear(attr); if (attr->attr.mask & (UCC_CONTEXT_ATTR_FIELD_CTX_ADDR_LEN | UCC_CONTEXT_ATTR_FIELD_CTX_ADDR)) { if (NULL == ctx->worker.worker_address) { ucs_status = ucp_worker_get_address(ctx->worker.ucp_worker, &ctx->worker.worker_address, &ctx->worker.ucp_addrlen); if (UCS_OK != ucs_status) { tl_error(ctx->super.super.lib, "failed to get ucp worker address"); return ucs_status_to_ucc_status(ucs_status); } if (ctx->cfg.service_worker != 0 && (NULL == ctx->service_worker.worker_address)) { ucs_status = ucp_worker_get_address(ctx->service_worker.ucp_worker, &ctx->service_worker.worker_address, &ctx->service_worker.ucp_addrlen); if (UCS_OK != ucs_status) { tl_error( ctx->super.super.lib, "failed to get ucp special service worker address"); return ucs_status_to_ucc_status(ucs_status); } } } } if (attr->attr.mask & UCC_CONTEXT_ATTR_FIELD_CTX_ADDR_LEN) { packed_length = TL_UCP_EP_ADDRLEN_SIZE + ctx->worker.ucp_addrlen; if (ctx->cfg.service_worker != 0) { packed_length += TL_UCP_EP_ADDRLEN_SIZE + ctx->service_worker.ucp_addrlen; } if (NULL != ctx->remote_info) { packed_length += ctx->n_rinfo_segs * (sizeof(size_t) * 3); for (i = 0; i < ctx->n_rinfo_segs; i++) { packed_length += ctx->remote_info[i].packed_key_len; } } attr->attr.ctx_addr_len = packed_length; } if (attr->attr.mask & UCC_CONTEXT_ATTR_FIELD_CTX_ADDR) { *offset = ctx->worker.ucp_addrlen; offset = TL_UCP_EP_ADDR_WORKER(offset); memcpy(offset, ctx->worker.worker_address, ctx->worker.ucp_addrlen); offset = PTR_OFFSET(offset, ctx->worker.ucp_addrlen); if (ctx->cfg.service_worker != 0) { *offset = ctx->service_worker.ucp_addrlen; offset = TL_UCP_EP_ADDR_WORKER(offset); memcpy(offset, ctx->service_worker.worker_address, ctx->service_worker.ucp_addrlen); offset = PTR_OFFSET(offset, ctx->service_worker.ucp_addrlen); } if (NULL != ctx->remote_info) { ucc_tl_ucp_ctx_remote_pack_data(ctx, offset); } } if (attr->attr.mask & UCC_CONTEXT_ATTR_FIELD_WORK_BUFFER_SIZE) { attr->attr.global_work_buffer_size = ONESIDED_SYNC_SIZE + ONESIDED_REDUCE_SIZE; } attr->topo_required = ctx->topo_required; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/tl_ucp_copy.c0000664000175000017500000001556515211535620021340 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "tl_ucp_coll.h" #include "tl_ucp_tag.h" #include "tl_ucp_ep.h" #include "tl_ucp_sendrecv.h" #include "tl_ucp_copy.h" ucc_status_t ucc_tl_ucp_mc_copy_post(void *dst, ucc_memory_type_t dst_mtype, ucp_mem_h dst_memh, //NOLINT void *src, ucc_memory_type_t src_mtype, ucp_mem_h src_memh, //NOLINT size_t size, ucc_tl_ucp_task_t *coll_task, //NOLINT ucc_tl_ucp_copy_task_t **copy_task) //NOLINT { return ucc_mc_memcpy(dst, src, size, dst_mtype, src_mtype); } ucc_status_t ucc_tl_ucp_mc_copy_test(ucc_tl_ucp_context_t *ctx, //NOLINT ucc_tl_ucp_copy_task_t *copy_task) //NOLINT { /* mc copy is blocking, test always returns UCC_OK */ return UCC_OK; } ucc_status_t ucc_tl_ucp_mc_copy_finalize(ucc_tl_ucp_copy_task_t *copy_task) //NOLINT { return UCC_OK; } ucc_status_t ucc_tl_ucp_ec_copy_post(void *dst, ucc_memory_type_t dst_mtype, //NOLINT ucp_mem_h dst_memh, //NOLINT void *src, ucc_memory_type_t src_mtype, //NOLINT ucp_mem_h src_memh, //NOLINT size_t size, ucc_tl_ucp_task_t *coll_task, ucc_tl_ucp_copy_task_t **copy_task) { ucc_ee_executor_task_args_t eargs = {0}; ucc_ee_executor_task_t **eee_task = (ucc_ee_executor_task_t **)copy_task; ucc_status_t status; ucc_ee_executor_t *exec; status = ucc_coll_task_get_executor(&coll_task->super, &exec); if (ucc_unlikely(status != UCC_OK)) { return status; } eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY; eargs.copy.dst = dst; eargs.copy.src = src; eargs.copy.len = size; return ucc_ee_executor_task_post(exec, &eargs, eee_task); } ucc_status_t ucc_tl_ucp_ec_copy_test(ucc_tl_ucp_context_t *ctx, //NOLINT ucc_tl_ucp_copy_task_t *copy_task) { ucc_ee_executor_task_t *eee_task = (ucc_ee_executor_task_t *)copy_task; return ucc_ee_executor_task_test(eee_task); } ucc_status_t ucc_tl_ucp_ec_copy_finalize(ucc_tl_ucp_copy_task_t *copy_task) { ucc_ee_executor_task_t *eee_task = (ucc_ee_executor_task_t *)copy_task; return ucc_ee_executor_task_finalize(eee_task); } void ucc_tl_ucp_copy_send_completion_cb(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in copy send completion %s", ucs_status_string(status)); } ucp_request_free(request); } void ucc_tl_ucp_copy_recv_completion_cb(void *request, ucs_status_t status, //NOLINT const ucp_tag_recv_info_t *info, /* NOLINT */ void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in copy recv completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } /* no request free, it will be called by user */ } ucc_status_t ucc_tl_ucp_ucp_copy_post(void *dst, ucc_memory_type_t dst_mtype, ucp_mem_h dst_memh, void *src, ucc_memory_type_t src_mtype, ucp_mem_h src_memh, size_t size, ucc_tl_ucp_task_t *task, ucc_tl_ucp_copy_task_t **copy_task) { ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t dest_group_rank = trank; ucp_request_param_t req_param; ucc_status_t status; ucp_ep_h ep; ucp_tag_t ucp_tag, ucp_tag_mask; ucs_status_ptr_t ucp_status; UCC_TL_UCP_MAKE_RECV_TAG(ucp_tag, ucp_tag_mask, (args->mask & UCC_COLL_ARGS_FIELD_TAG), task->tagged.tag, trank, team->super.super.params.id, team->super.super.params.scope_id, team->super.super.params.scope); req_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_DATATYPE | UCP_OP_ATTR_FIELD_USER_DATA | UCP_OP_ATTR_FIELD_MEMORY_TYPE; req_param.datatype = ucp_dt_make_contig(size); req_param.memory_type = ucc_memtype_to_ucs[dst_mtype]; req_param.cb.recv = ucc_tl_ucp_copy_recv_completion_cb; req_param.user_data = task; if (dst_memh) { req_param.op_attr_mask |= UCP_OP_ATTR_FIELD_MEMH; req_param.memh = dst_memh; } ucp_status = ucp_tag_recv_nbx(team->worker->ucp_worker, dst, 1, ucp_tag, ucp_tag_mask, &req_param); UCC_TL_UCP_CHECK_REQ_STATUS(); (*copy_task)= ucp_status; status = ucc_tl_ucp_get_ep(team, trank, &ep); if (ucc_unlikely(UCC_OK != status)) { return status; } ucp_tag = UCC_TL_UCP_MAKE_SEND_TAG((args->mask & UCC_COLL_ARGS_FIELD_TAG), task->tagged.tag, trank, team->super.super.params.id, team->super.super.params.scope_id, team->super.super.params.scope); req_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_DATATYPE | UCP_OP_ATTR_FIELD_USER_DATA | UCP_OP_ATTR_FIELD_MEMORY_TYPE; req_param.datatype = ucp_dt_make_contig(size); req_param.memory_type = ucc_memtype_to_ucs[src_mtype]; req_param.cb.send = ucc_tl_ucp_copy_send_completion_cb; req_param.user_data = task; if (src_memh) { req_param.op_attr_mask |= UCP_OP_ATTR_FIELD_MEMH; req_param.memh = src_memh; } ucp_status = ucp_tag_send_nbx(ep, src, 1, ucp_tag, &req_param); UCC_TL_UCP_CHECK_REQ_STATUS(); return UCC_OK; } ucc_status_t ucc_tl_ucp_ucp_copy_test(ucc_tl_ucp_context_t *ctx, ucc_tl_ucp_copy_task_t *copy_task) { ucs_status_ptr_t req_status = (ucs_status_ptr_t)copy_task; ucp_worker_progress(ctx->worker.ucp_worker); return ucs_status_to_ucc_status(ucp_request_check_status(req_status)); } ucc_status_t ucc_tl_ucp_ucp_copy_finalize(ucc_tl_ucp_copy_task_t *copy_task) { ucs_status_ptr_t req_status = (ucs_status_ptr_t)copy_task; ucp_request_free(req_status); return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/alltoallv/0000775000175000017500000000000015211535620020632 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/alltoallv/alltoallv.c0000664000175000017500000000347315211535620022777 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "alltoallv.h" ucc_base_coll_alg_info_t ucc_tl_ucp_alltoallv_algs[UCC_TL_UCP_ALLTOALLV_ALG_LAST + 1] = { [UCC_TL_UCP_ALLTOALLV_ALG_PAIRWISE] = {.id = UCC_TL_UCP_ALLTOALLV_ALG_PAIRWISE, .name = "pairwise", .desc = "O(N) pairwise exchange with adjustable number " "of outstanding sends/recvs"}, [UCC_TL_UCP_ALLTOALLV_ALG_HYBRID] = {.id = UCC_TL_UCP_ALLTOALLV_ALG_HYBRID, .name = "hybrid", .desc = "hybrid a2av alg "}, [UCC_TL_UCP_ALLTOALLV_ALG_ONESIDED] = {.id = UCC_TL_UCP_ALLTOALLV_ALG_ONESIDED, .name = "onesided", .desc = "O(N) onesided alltoallv"}, [UCC_TL_UCP_ALLTOALLV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_alltoallv_init(ucc_tl_ucp_task_t *task) { ucc_status_t status; ALLTOALLV_TASK_CHECK(TASK_ARGS(task), TASK_TEAM(task)); status = ucc_tl_ucp_alltoallv_pairwise_init_common(task); out: return status; } ucc_status_t ucc_tl_ucp_alltoallv_pairwise_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task; ucc_status_t status; ALLTOALLV_TASK_CHECK(coll_args->args, tl_team); task = ucc_tl_ucp_init_task(coll_args, team); *task_h = &task->super; status = ucc_tl_ucp_alltoallv_pairwise_init_common(task); out: return status; } ucc-1.8.0/src/components/tl/ucp/alltoallv/alltoallv_onesided.c0000664000175000017500000001120215211535620024636 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "alltoallv.h" #include "core/ucc_progress_queue.h" #include "utils/ucc_math.h" #include "tl_ucp_sendrecv.h" ucc_status_t ucc_tl_ucp_alltoallv_onesided_start(ucc_coll_task_t *ctask) { ucc_tl_ucp_task_t *task = ucc_derived_of(ctask, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ptrdiff_t src = (ptrdiff_t)TASK_ARGS(task).src.info_v.buffer; ptrdiff_t dest = (ptrdiff_t)TASK_ARGS(task).dst.info_v.buffer; ucc_memory_type_t mtype = TASK_ARGS(task).src.info_v.mem_type; ucc_rank_t grank = UCC_TL_TEAM_RANK(team); ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); long *pSync = TASK_ARGS(task).global_work_buffer; ucc_aint_t *s_disp = TASK_ARGS(task).src.info_v.displacements; ucc_aint_t *d_disp = TASK_ARGS(task).dst.info_v.displacements; size_t sdt_size = ucc_dt_size(TASK_ARGS(task).src.info_v.datatype); size_t rdt_size = ucc_dt_size(TASK_ARGS(task).dst.info_v.datatype); ucc_mem_map_mem_h src_memh = TASK_ARGS(task).src_memh.local_memh; ucc_mem_map_mem_h *dst_memh = TASK_ARGS(task).dst_memh.global_memh; ucc_rank_t peer; size_t sd_disp, dd_disp, data_size; ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); /* perform a put to each member peer using the peer's index in the * destination displacement. */ for (peer = (grank + 1) % gsize; task->onesided.put_posted < gsize; peer = (peer + 1) % gsize) { sd_disp = ucc_coll_args_get_displacement(&TASK_ARGS(task), s_disp, peer) * sdt_size; dd_disp = ucc_coll_args_get_displacement(&TASK_ARGS(task), d_disp, peer) * rdt_size; data_size = ucc_coll_args_get_count( &TASK_ARGS(task), TASK_ARGS(task).src.info_v.counts, peer) * sdt_size; UCPCHECK_GOTO(ucc_tl_ucp_put_nb(PTR_OFFSET(src, sd_disp), PTR_OFFSET(dest, dd_disp), data_size, mtype, peer, src_memh, dst_memh, team, task), task, out); UCPCHECK_GOTO(ucc_tl_ucp_atomic_inc(pSync, peer, dst_memh, team), task, out); } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); out: return task->super.status; } void ucc_tl_ucp_alltoallv_onesided_progress(ucc_coll_task_t *ctask) { ucc_tl_ucp_task_t *task = ucc_derived_of(ctask, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); long *pSync = TASK_ARGS(task).global_work_buffer; if (ucc_tl_ucp_test_onesided(task, gsize) == UCC_INPROGRESS) { return; } pSync[0] = 0; task->super.status = UCC_OK; } ucc_status_t ucc_tl_ucp_alltoallv_onesided_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task; ucc_status_t status; ALLTOALLV_TASK_CHECK(coll_args->args, tl_team); if (!(coll_args->args.mask & UCC_COLL_ARGS_FIELD_GLOBAL_WORK_BUFFER)) { tl_error(UCC_TL_TEAM_LIB(tl_team), "global work buffer not provided nor associated with team"); status = UCC_ERR_NOT_SUPPORTED; goto out; } if (coll_args->args.mask & UCC_COLL_ARGS_FIELD_FLAGS) { if (!(coll_args->args.flags & UCC_COLL_ARGS_FLAG_MEM_MAPPED_BUFFERS)) { tl_error(UCC_TL_TEAM_LIB(tl_team), "non memory mapped buffers are not supported"); status = UCC_ERR_NOT_SUPPORTED; goto out; } } if (!(coll_args->args.mask & UCC_COLL_ARGS_FIELD_MEM_MAP_SRC_MEMH)) { coll_args->args.src_memh.global_memh = NULL; } if (!(coll_args->args.mask & UCC_COLL_ARGS_FIELD_MEM_MAP_DST_MEMH)) { coll_args->args.dst_memh.global_memh = NULL; } task = ucc_tl_ucp_init_task(coll_args, team); *task_h = &task->super; task->super.post = ucc_tl_ucp_alltoallv_onesided_start; task->super.progress = ucc_tl_ucp_alltoallv_onesided_progress; status = UCC_OK; out: return status; } ucc-1.8.0/src/components/tl/ucp/alltoallv/alltoallv_pairwise.c0000664000175000017500000001402115211535620024671 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "alltoallv.h" #include "core/ucc_progress_queue.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "tl_ucp_sendrecv.h" /* TODO: add as parameters */ #define NP_THRESH 32 static inline ucc_rank_t get_recv_peer(ucc_rank_t rank, ucc_rank_t size, ucc_rank_t step) { return (rank + step) % size; } static inline ucc_rank_t get_send_peer(ucc_rank_t rank, ucc_rank_t size, ucc_rank_t step) { return (rank - step + size) % size; } static ucc_rank_t get_num_posts(const ucc_tl_ucp_team_t *team) { unsigned long posts = UCC_TL_UCP_TEAM_LIB(team)->cfg.alltoallv_pairwise_num_posts; ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); if (posts == UCC_ULUNITS_AUTO) { if (UCC_TL_TEAM_SIZE(team) <= NP_THRESH) { /* use linear algorithm */ posts = 0; } else { /* use pairwise algorithm */ posts = 1; } } posts = (posts > tsize || posts == 0) ? tsize: posts; return posts; } static void ucc_tl_ucp_alltoallv_pairwise_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ptrdiff_t sbuf = (ptrdiff_t)TASK_ARGS(task).src.info_v.buffer; ptrdiff_t rbuf = (ptrdiff_t)TASK_ARGS(task).dst.info_v.buffer; ucc_memory_type_t smem = TASK_ARGS(task).src.info_v.mem_type; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info_v.mem_type; ucc_rank_t grank = UCC_TL_TEAM_RANK(team); ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); int polls = 0; ucc_rank_t peer, nreqs; size_t rdt_size, sdt_size, data_size, data_displ; nreqs = get_num_posts(team); rdt_size = ucc_dt_size(TASK_ARGS(task).dst.info_v.datatype); sdt_size = ucc_dt_size(TASK_ARGS(task).src.info_v.datatype); while ((task->tagged.send_posted < gsize || task->tagged.recv_posted < gsize) && (polls++ < task->n_polls)) { ucp_worker_progress(UCC_TL_UCP_TEAM_CTX(team)->worker.ucp_worker); while ((task->tagged.recv_posted < gsize) && ((task->tagged.recv_posted - task->tagged.recv_completed) < nreqs)) { peer = get_recv_peer(grank, gsize, task->tagged.recv_posted); data_size = ucc_coll_args_get_count( &TASK_ARGS(task), TASK_ARGS(task).dst.info_v.counts, peer) * rdt_size; data_displ = ucc_coll_args_get_displacement( &TASK_ARGS(task), TASK_ARGS(task).dst.info_v.displacements, peer) * rdt_size; UCPCHECK_GOTO(ucc_tl_ucp_recv_nz((void *)(rbuf + data_displ), data_size, rmem, peer, team, task), task, out); polls = 0; } while ((task->tagged.send_posted < gsize) && ((task->tagged.send_posted - task->tagged.send_completed) < nreqs)) { peer = get_send_peer(grank, gsize, task->tagged.send_posted); data_size = ucc_coll_args_get_count( &TASK_ARGS(task), TASK_ARGS(task).src.info_v.counts, peer) * sdt_size; data_displ = ucc_coll_args_get_displacement( &TASK_ARGS(task), TASK_ARGS(task).src.info_v.displacements, peer) * sdt_size; UCPCHECK_GOTO(ucc_tl_ucp_send_nz((void *)(sbuf + data_displ), data_size, smem, peer, team, task), task, out); polls = 0; } } if ((task->tagged.send_posted < gsize) || (task->tagged.recv_posted < gsize)) { return; } task->super.status = ucc_tl_ucp_test(task); out: if (task->super.status != UCC_INPROGRESS) { UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_alltoallv_pairwise_done", 0); } } static ucc_status_t ucc_tl_ucp_alltoallv_pairwise_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_alltoallv_pairwise_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_alltoallv_pairwise_init_common(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_coll_args_t *args = &TASK_ARGS(task); task->super.post = ucc_tl_ucp_alltoallv_pairwise_start; task->super.progress = ucc_tl_ucp_alltoallv_pairwise_progress; task->n_polls = ucc_max(1, task->n_polls); if (UCC_TL_UCP_TEAM_CTX(team)->cfg.pre_reg_mem) { if (args->flags & UCC_COLL_ARGS_FLAG_CONTIG_SRC_BUFFER) { ucc_tl_ucp_pre_register_mem( team, args->src.info_v.buffer, (ucc_coll_args_get_total_count(args, args->src.info_v.counts, size) * ucc_dt_size(args->src.info_v.datatype)), args->src.info_v.mem_type); } if (args->flags & UCC_COLL_ARGS_FLAG_CONTIG_DST_BUFFER) { ucc_tl_ucp_pre_register_mem( team, args->dst.info_v.buffer, (ucc_coll_args_get_total_count(args, args->dst.info_v.counts, size) * ucc_dt_size(args->dst.info_v.datatype)), args->dst.info_v.mem_type); } } return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/alltoallv/alltoallv_hybrid.c0000664000175000017500000012332215211535620024334 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "alltoallv.h" #include "core/ucc_progress_queue.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "tl_ucp_sendrecv.h" #include "components/mc/ucc_mc.h" #include "coll_patterns/bruck_alltoall.h" #include "utils/ucc_atomic.h" /* scratch structure buff_size - alltoallv_hybrid_buff_size ====================================================================================================================================================================================================================== |sdispls |scounts |calc |seg |lb |ptrs |buf | |tsize * sizeof(int)|tsize * sizeof(int)|2 * tsize * sizeof(int)|tsize * sizeof(char)|sizeof(ucc_tl_ucp_alltoallv_hybrid_buf_meta_t) * (radix -1)|ucc_div_round_up(tsize, 2) * sizeof(void*)|(radix - 1) * buff_size | ====================================================================================================================================================================================================================== */ #define NUM_BINS 5 #define MAX_BRUCK 1250 #define COUNT_DIRECT UINT_MAX #define DENSE_PACK_FORMAT UINT_MAX enum { UCC_ALLTOALLV_HYBRID_PHASE_START, UCC_ALLTOALLV_HYBRID_PHASE_SENT, UCC_ALLTOALLV_HYBRID_PHASE_RECV }; enum { ALLTOALLV_HYBRID_SEG_SEND_DIRECT = 0, ALLTOALLV_HYBRID_SEG_RECV_DIRECT = 1, ALLTOALLV_HYBRID_SEG_DIGIT = 2 }; typedef struct ucc_tl_ucp_alltoallv_hybrid_merge_bin { ucc_tl_ucp_task_t *task; int start; int len; } ucc_tl_ucp_alltoallv_hybrid_merge_bin_t; typedef struct ucc_tl_ucp_alltoallv_hybrid_buf_meta { ucc_tl_ucp_alltoallv_hybrid_merge_bin_t bins[NUM_BINS]; int cur_bin; int offset; } ucc_tl_ucp_alltoallv_hybrid_buf_meta_t; #define TASK_SCRATCH(_task) ({ \ (_task)->alltoallv_hybrid.scratch_mc_header->addr; \ }) #define TASK_TMP(_task, _tsize) ({ \ PTR_OFFSET(TASK_SCRATCH(_task), 2*_tsize*sizeof(int)); \ }) #define TASK_SEG(_task, _tsize) ({ \ PTR_OFFSET(TASK_SCRATCH(_task), 4 * _tsize*sizeof(int)); \ }) #define TASK_LB(_task, _tsize) ({ \ PTR_OFFSET(TASK_SEG(_task, _tsize), _tsize*sizeof(char)); \ }) #define TASK_PTRS(_task, _tsize) ({ \ uint32_t _radix = (_task)->alltoallv_hybrid.radix; \ PTR_OFFSET(TASK_LB(_task, _tsize), sizeof(ucc_tl_ucp_alltoallv_hybrid_buf_meta_t)*(_radix - 1)); \ }) #define TASK_BUF(_task, _i, _tsize) ({ \ size_t _buff_size = UCC_TL_UCP_TEAM_LIB(TASK_TEAM(_task))->cfg.alltoallv_hybrid_buff_size; \ PTR_OFFSET(TASK_PTRS(_task, _tsize), ucc_div_round_up(_tsize, 2)*sizeof(void*) + \ (_i) * _buff_size); \ }) #define ALIGN(x) ucc_ceil(x, 4) #define IS_DIRECT_SEND(_seg) ((_seg) & UCC_BIT(ALLTOALLV_HYBRID_SEG_SEND_DIRECT)) #define SET_DIRECT_SEND(_seg) ((_seg) |= UCC_BIT(ALLTOALLV_HYBRID_SEG_SEND_DIRECT)) #define IS_DIRECT_RECV(_seg) ((_seg) & UCC_BIT(ALLTOALLV_HYBRID_SEG_RECV_DIRECT)) #define SET_DIRECT_RECV(_seg) ((_seg) |= UCC_BIT(ALLTOALLV_HYBRID_SEG_RECV_DIRECT)) #define GET_BRUCK_DIGIT(_seg) ((_seg) >> ALLTOALLV_HYBRID_SEG_DIGIT) #define SET_BRUCK_DIGIT(_seg, _digit) \ ((_seg) = (((_digit) << ALLTOALLV_HYBRID_SEG_DIGIT) + ((_seg) & UCC_MASK(ALLTOALLV_HYBRID_SEG_DIGIT)))) static inline ucc_rank_t get_pairwise_send_peer(ucc_rank_t trank, ucc_rank_t tsize, ucc_rank_t step) { return (trank + step) % tsize; } static inline ucc_rank_t get_pairwise_recv_peer(ucc_rank_t trank, ucc_rank_t tsize, ucc_rank_t step) { return (trank - step + tsize) % tsize; } static void send_completion(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_alltoallv_hybrid_merge_bin_t *bin = (ucc_tl_ucp_alltoallv_hybrid_merge_bin_t*)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(bin->task), "failure in alltoallv_hybird completion %s", ucs_status_string(status)); bin->task->super.status = ucs_status_to_ucc_status(status); } if (request) { ucp_request_free(request); } ucc_atomic_add32(&bin->task->tagged.send_completed, 1); bin->len = 0; } static ucc_status_t ucc_tl_ucp_alltoallv_hybrid_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); if (task->alltoallv_hybrid.scratch_mc_header) { ucc_mc_free(task->alltoallv_hybrid.scratch_mc_header); } return ucc_tl_ucp_coll_finalize(coll_task); } static inline void radix_setup(ucc_tl_ucp_task_t *task, int id, ucc_rank_t tsize, size_t merge_buf_size, ucc_tl_ucp_alltoallv_hybrid_buf_meta_t** meta, void** merge_buf, void** tmp_buf) { ucc_tl_ucp_alltoallv_hybrid_buf_meta_t *bm = TASK_LB(task, tsize); (*meta) = &bm[id]; (*merge_buf) = TASK_BUF(task, id, tsize); (*tmp_buf) = PTR_OFFSET(*merge_buf, merge_buf_size); } /* get the number of blocks of data to be sent for the given step and edge in the algorithm. */ static inline int get_send_block_count(ucc_rank_t tsize, int radix, int node_edge_id, int radix_pow) { int k = tsize / radix_pow; int send_count = (k / radix) * radix_pow; if ((k % radix) > node_edge_id) { send_count += radix_pow; } else if ((k % radix) == node_edge_id) { send_count += (tsize % radix_pow); } return send_count; } static inline int calculate_head_size(int snd_count, size_t dt_size) { int head_num_elements = 0; if (dt_size <= sizeof(unsigned int)) { head_num_elements = ((snd_count + 1) * sizeof(unsigned int)) / dt_size; } else { /* The size of dtype is greater than unsigned int */ if (((snd_count + 1) * sizeof(unsigned int)) % dt_size == 0) { /* The size of head can fit */ head_num_elements = ((snd_count + 1) * sizeof(unsigned int)) / dt_size; } else { /* The size of head can not fit*/ head_num_elements = ((snd_count + 1) * sizeof(unsigned int)) / dt_size + 1; } } return head_num_elements; } static int fit_in_send_buffer(int num, ucc_tl_ucp_alltoallv_hybrid_buf_meta_t* meta, int size_req, int mem_size) { int *cur_bin = &meta->cur_bin; int start_ok = 1; int bin_rdy = -1; int i, k, pos, valid_pos; ucc_assert(size_req <= mem_size); for (i = 0; i < num; i++) { if (meta->bins[i].len == 0) { bin_rdy = i; break; } } *cur_bin = bin_rdy; if (bin_rdy == -1) { /* no free bins, exit */ return -1; } for (i = 0; i < num; i++) { if ((meta->bins[i].len > 0) && (meta->bins[i].start < size_req)) { start_ok = 0; break; } } if (start_ok == 1) { /* we have enough empty space at the beggining of buffer */ return 0; } /* at this point at least one bin is not empty */ for (k = 0; k < num; k++) { /* complexity O(num_bins^2) */ if (meta->bins[k].len > 0) { /* bin is not empty, check if there is enough space after it*/ valid_pos = 1; pos = meta->bins[k].start + meta->bins[k].len; if (pos + size_req < mem_size) { /* there is enough space till the end of the buffer, * need to check collisions with other bins */ for (i = 0; i < num; i++) { if ((i == k) || (meta->bins[i].len == 0)) { /* skip current bin and empty bins */ continue; } if (meta->bins[i].start < (pos + size_req) && pos < meta->bins[i].start + meta->bins[i].len) { /* two regions overlap => collision */ valid_pos = 0; break; } } if (valid_pos) { /* found a slot, no collisions with other bins */ return pos; } } } } /* no available slots */ return -1; } static inline ucc_status_t get_send_buffer(void *p_tmp_send_region, ucc_tl_ucp_alltoallv_hybrid_buf_meta_t* meta, int num_bins, int required_buf_size, ucc_tl_ucp_task_t *task, void **buf) { int merge_buf_size = task->alltoallv_hybrid.merge_buf_size; int polls = 0; int send_buf_offset; /* get send buffer pointer */ send_buf_offset = fit_in_send_buffer(num_bins, meta, required_buf_size, merge_buf_size); if (send_buf_offset < 0) { do { ucp_worker_progress(TASK_CTX(task)->worker.ucp_worker); send_buf_offset = fit_in_send_buffer(num_bins, meta, required_buf_size, merge_buf_size); } while ((send_buf_offset < 0) && (polls++ < task->n_polls)); if (send_buf_offset < 0) { return UCC_INPROGRESS; } } meta->bins[meta->cur_bin].start = send_buf_offset; *buf = PTR_OFFSET(p_tmp_send_region, send_buf_offset); return UCC_OK; } /* Pack data for sending * packed_send_buf (output) contains the send buffer that is ready to be sent * returns length of the buffer to be sent */ static size_t pack_send_data(ucc_tl_ucp_task_t *task, int step, int send_count, int node_edge_id, size_t dt_size, int send_limit, int step_header_size, void *packed_send_buf) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); int *BytesForPacking = TASK_TMP(task, tsize); char *seg_st = TASK_SEG(task, tsize); int tmp_send_region_size = task->alltoallv_hybrid.merge_buf_size; uint32_t radix = task->alltoallv_hybrid.radix; void **src_iovec = TASK_PTRS(task, tsize); void *user_sbuf = TASK_ARGS(task).src.info_v.buffer; int sparse_cnt = 0; int n = send_count; int logi = 1; void *op_metadata = task->alltoallv_hybrid.scratch_mc_header->addr; ucc_rank_t index; int send_len, snd_offset, i, k; unsigned int temp_count; char loc; size_t offset; for (i = 0; i < n; i++) { BytesForPacking[i] = send_limit; } while (logi < n) { logi = logi * radix; } index = get_bruck_step_finish(tsize - 1, radix, node_edge_id, step); while (n > 0) { ucc_assert((index / step) % radix == node_edge_id); n--; if (logi > n) { logi = logi / radix; } temp_count = ((unsigned int *)op_metadata)[index + tsize]; if (temp_count == COUNT_DIRECT) { /* data will be directly sent */ send_len = 0; } else { loc = GET_BRUCK_DIGIT(seg_st[index]); snd_offset = ((unsigned int *)op_metadata)[index] * dt_size; send_len = dt_size * temp_count; if (loc) { /* data is in scratch buffer */ src_iovec[n] = PTR_OFFSET(TASK_BUF(task, (loc - 1), tsize), tmp_send_region_size + snd_offset); } else { /* data is in user buffer */ if ((send_len <= BytesForPacking[n]) && (send_len < MAX_BRUCK || n < step)) { src_iovec[n] = PTR_OFFSET(user_sbuf, snd_offset); } else { /* direct exchange */ temp_count = COUNT_DIRECT; task->alltoallv_hybrid.num2send++; /* mark the send as direct */ SET_DIRECT_SEND(seg_st[index]); send_len = 0; } } } ((unsigned int *)packed_send_buf)[n + 1] = temp_count; if (send_len > 0) { sparse_cnt++; } if (logi > 0) { BytesForPacking[n % logi] += BytesForPacking[n] - send_len; } index = GET_PREV_BRUCK_NUM(index, radix, step); } /* if there is data for less than half the destinations, send meta data * in sparse format. */ if ((sparse_cnt * 2) < send_count) { /* sparse format: store nonzero messages and source rank index */ ((unsigned int *)packed_send_buf)[0] = sparse_cnt; /* compute space requirement for sparse data exchange header */ offset = calculate_head_size(sparse_cnt * 2, dt_size) * dt_size; i = k = 0; while (k < sparse_cnt) { unsigned int cur_len = ((unsigned int *)packed_send_buf)[i + 1]; if (cur_len != 0 && cur_len != COUNT_DIRECT) { BytesForPacking[2 * k] = i; BytesForPacking[2 * k + 1] = cur_len; ++k; } ++i; } for (i = 0; i < sparse_cnt; i++){ ((unsigned int *)packed_send_buf)[2 * i + 1] = BytesForPacking[2 * i]; ((unsigned int *)packed_send_buf)[2 * i + 2] = BytesForPacking[2 * i + 1]; } /* pack the send buffer */ for (k = 0; k < sparse_cnt; ++k) { i = ((unsigned int *)packed_send_buf)[2 * k + 1]; temp_count = ((unsigned int *)packed_send_buf)[2 * k + 2]; memcpy(PTR_OFFSET(packed_send_buf, offset), src_iovec[i], temp_count * dt_size); offset = offset + temp_count * dt_size; } } else { /* dense format: store all message information even if message size is 0 */ ((unsigned int *)packed_send_buf)[0] = DENSE_PACK_FORMAT; offset = step_header_size; for (i = 0; i < send_count; i++) { temp_count = ((unsigned int *)packed_send_buf)[i + 1]; if (temp_count != COUNT_DIRECT) { memcpy(PTR_OFFSET(packed_send_buf, offset), src_iovec[i], temp_count * dt_size); offset = offset + temp_count * dt_size; } } } return offset; } static inline ucc_status_t send_data(void *buf, int send_size, ucc_rank_t dst, ucc_tl_ucp_alltoallv_hybrid_buf_meta_t *meta, ucc_tl_ucp_task_t *task) { ucc_status_t status; ucc_assert(meta->bins[meta->cur_bin].len != 0); status = ucc_tl_ucp_send_cb(buf, send_size, UCC_MEMORY_TYPE_HOST, dst, TASK_TEAM(task), task, send_completion, (void*)&meta->bins[meta->cur_bin]); task->alltoallv_hybrid.phase = UCC_ALLTOALLV_HYBRID_PHASE_SENT; return status; } static int receive_buffer_recycler(ucc_rank_t tsize, unsigned int* rcv_start, int* rcv_len, char* seg_st, void* buf, size_t dt_size, int* tmp_buf, int step, void* rbuf, int* rdisps, int my_group_index, int radix, int node_edge_id) { int cur = 0; int mstep = step / radix; int i, k, in_buf, offset, idx; for (i = 0; i < tsize; ++i) { if (GET_BRUCK_DIGIT(seg_st[i]) == node_edge_id) { tmp_buf[i + tsize] = 1; ++cur; } else { tmp_buf[i + tsize] = 0; } } in_buf = cur; --cur; while (cur >= 0) { for (i = tsize - 1; i >= 0; --i) { if (tmp_buf[i + tsize] && ((i / mstep) % radix == node_edge_id)) { tmp_buf[cur] = i; tmp_buf[i + tsize] = 0; --cur; } } mstep = mstep / radix; } k = 0; offset = 0; while (k < in_buf) { cur = tmp_buf[k]; if ((rcv_start[cur] == COUNT_DIRECT) || (rcv_len[cur] == 0)) { ++k; continue; } if (cur < step) { /* final destination of data */ idx = (my_group_index - cur + tsize) % tsize; memcpy(PTR_OFFSET(rbuf, rdisps[idx] * dt_size), PTR_OFFSET(buf, rcv_start[cur] * dt_size), rcv_len[cur] * dt_size); rcv_start[cur] = COUNT_DIRECT; rcv_len[cur] = 0; seg_st[cur] = seg_st[cur] % 4; } else { if (offset < rcv_start[cur]) { memmove(PTR_OFFSET(buf, offset * dt_size), PTR_OFFSET(buf, rcv_start[cur] * dt_size), rcv_len[cur] * dt_size); rcv_start[cur] = offset; offset += rcv_len[cur]; } else { ucc_assert(offset == rcv_start[cur]); offset += rcv_len[cur]; } } ++k; } return offset; } static ucc_status_t post_recv(ucc_rank_t recvfrom, ucc_rank_t tsize, size_t dt_size, int node_edge_id, int step, ucc_rank_t trank, void *op_metadata, int tmp_buf_size, char *seg_st, ucc_tl_ucp_alltoallv_hybrid_buf_meta_t *meta, int step_buf_size, int *BytesForPacking, void *p_tmp_recv_region, ucc_tl_ucp_task_t *task) { int *rdisps = (int*)TASK_ARGS(task).dst.info_v.displacements; void *user_rbuf = TASK_ARGS(task).dst.info_v.buffer; uint32_t radix = task->alltoallv_hybrid.radix; ucc_status_t status = UCC_OK; int new_offset; void* dst_buf; if (task->alltoallv_hybrid.phase != UCC_ALLTOALLV_HYBRID_PHASE_SENT) { return UCC_OK; } /* check if we have space for maximum receive. If not, recycle */ if (meta->offset * dt_size + step_buf_size > tmp_buf_size) { new_offset = receive_buffer_recycler(tsize, (unsigned int *)op_metadata, (int *)op_metadata + tsize, seg_st, p_tmp_recv_region, dt_size, BytesForPacking, step, user_rbuf, rdisps, trank, radix, node_edge_id); meta->offset = new_offset; } ucc_assert(meta->offset * dt_size + step_buf_size <= tmp_buf_size); dst_buf = PTR_OFFSET(p_tmp_recv_region, meta->offset * dt_size); status = ucc_tl_ucp_recv_nb(dst_buf, step_buf_size, UCC_MEMORY_TYPE_HOST, recvfrom, TASK_TEAM(task), task); if (ucc_unlikely(status != UCC_OK)) { return status; } task->alltoallv_hybrid.phase = UCC_ALLTOALLV_HYBRID_PHASE_START; task->alltoallv_hybrid.cur_radix++; if (task->alltoallv_hybrid.cur_radix == radix) { /* setup the control variables for the next phase of processing */ task->alltoallv_hybrid.phase = UCC_ALLTOALLV_HYBRID_PHASE_RECV; task->alltoallv_hybrid.cur_radix = 1; } return UCC_OK; } /* complete all receives in current step */ static ucc_status_t complete_current_step_receives(ucc_rank_t tsize, int step, size_t dt_size, ucc_rank_t trank, void *op_metadata, char *seg_st, ucc_tl_ucp_task_t *task) { size_t tmp_send_region_size = task->alltoallv_hybrid.merge_buf_size; uint32_t radix = task->alltoallv_hybrid.radix; int *rcounts = (int*)TASK_ARGS(task).dst.info_v.counts; int polls = 0; ucc_tl_ucp_alltoallv_hybrid_buf_meta_t *meta; unsigned int rcv_sparse, next_p, cur_p; void *p_tmp_recv_region, *p_tmp_send_region, *dst_buf, *temp_offset; int i, k, n, node_edge_id, recv_count, cur_buf_length, recv_size; while ((task->tagged.recv_posted != task->tagged.recv_completed) && (polls++ < task->n_polls)) { ucp_worker_progress(TASK_CTX(task)->worker.ucp_worker); } if (task->tagged.recv_posted != task->tagged.recv_completed) { return UCC_INPROGRESS; } while (task->alltoallv_hybrid.cur_radix < radix) { node_edge_id = task->alltoallv_hybrid.cur_radix; radix_setup(task, node_edge_id - 1, tsize, tmp_send_region_size, &meta, &p_tmp_send_region, &p_tmp_recv_region); n = get_send_block_count(tsize, radix, node_edge_id, step); if (!n) { /* nothing to be done here */ task->alltoallv_hybrid.cur_radix++; continue; } /* fill in the information to be be used in subsequenst aggrgation * steps to extract data received for packing. */ dst_buf = PTR_OFFSET(p_tmp_recv_region, meta->offset * dt_size); rcv_sparse = ((unsigned int *)dst_buf)[0]; i = get_bruck_step_start(step, node_edge_id); if (rcv_sparse == DENSE_PACK_FORMAT) { recv_count = 0; recv_size = calculate_head_size(n, dt_size); temp_offset = PTR_OFFSET(dst_buf, recv_size * dt_size); /* this is where we parse the received bruck-like packet and * set the pointers to point on the important data segments. */ while (i < tsize) { cur_buf_length = ((unsigned int *)dst_buf)[1 + recv_count]; if (cur_buf_length != COUNT_DIRECT) { ((unsigned int *)op_metadata)[i] = ((char *)temp_offset-(char *)p_tmp_recv_region) / dt_size; /* mark data received destined to rank i, and its length */ SET_BRUCK_DIGIT(seg_st[i], node_edge_id); ((int *)op_metadata)[i + tsize] = cur_buf_length; recv_size += cur_buf_length; temp_offset = PTR_OFFSET(temp_offset, cur_buf_length * dt_size); } else { /* data will be sent pairwise */ ((int *)op_metadata)[i] = (int)COUNT_DIRECT; ((int *)op_metadata)[i + tsize] = (int)COUNT_DIRECT; if (i < (step * radix)) { int pairwise_src = (trank - i + tsize) % tsize; if (rcounts[pairwise_src] > 0) { task->alltoallv_hybrid.num2recv++; SET_DIRECT_RECV(seg_st[i]); } } } ++recv_count; i = GET_NEXT_BRUCK_NUM(i, radix, step); } } else { recv_size = calculate_head_size(2*rcv_sparse, dt_size); temp_offset= PTR_OFFSET(dst_buf, recv_size*dt_size); k = 0; if (rcv_sparse > 0) { next_p = ((unsigned int *)dst_buf)[2 * k + 1]; } else { next_p = tsize; } cur_p = 0; while (i < tsize) { if (cur_p == next_p) { cur_buf_length = ((unsigned int *)dst_buf)[2 * k + 2]; ((int *)op_metadata)[i] = (((char *)temp_offset - (char *)p_tmp_recv_region) / dt_size); ((int *)op_metadata)[i + tsize] = cur_buf_length; recv_size+= cur_buf_length; temp_offset = PTR_OFFSET(temp_offset, cur_buf_length * dt_size); SET_BRUCK_DIGIT(seg_st[i], node_edge_id); ++k; if (k < rcv_sparse) { next_p= ((unsigned int *)dst_buf)[2*k + 1]; } else { next_p = tsize; } } else { ((int *)op_metadata)[i] = (int)COUNT_DIRECT; ((int *)op_metadata)[i + tsize] = (int)COUNT_DIRECT; if (i < (step * radix)) { int pairwise_src = (trank - i + tsize) % tsize; if (rcounts[pairwise_src] > 0) { task->alltoallv_hybrid.num2recv++; SET_DIRECT_RECV(seg_st[i]); } } } ++cur_p; i = GET_NEXT_BRUCK_NUM(i, radix, step); } ucc_assert(next_p == tsize); } meta->offset += recv_size; task->alltoallv_hybrid.cur_radix++; } return UCC_OK; } static inline void hybrid_reverse_rotation(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); char *seg_st = TASK_SEG(task, tsize); size_t merge_buf_size = task->alltoallv_hybrid.merge_buf_size; void *user_sbuf = TASK_ARGS(task).src.info_v.buffer; void *user_rbuf = TASK_ARGS(task).dst.info_v.buffer; int *rdisps = (int*)TASK_ARGS(task).dst.info_v.displacements; void *metainfo = task->alltoallv_hybrid.scratch_mc_header->addr; size_t dt_size; int i, idx, cur_buf_index, cur_buf_size; char loc; void *lb; dt_size = ucc_dt_size(TASK_ARGS(task).dst.info_v.datatype); for (i = 0; i < tsize; i++) { cur_buf_index = ((int *)metainfo)[i]; cur_buf_size = ((int *)metainfo)[i + tsize]; if (cur_buf_index != COUNT_DIRECT ) { loc = GET_BRUCK_DIGIT(seg_st[i]); idx = (trank - i + tsize) % tsize; if (loc == 0) { /* This block of data is in user send buffer */ memcpy(PTR_OFFSET(user_rbuf, rdisps[idx] * dt_size), PTR_OFFSET(user_sbuf, cur_buf_index * dt_size), cur_buf_size * dt_size); } else { /* This block of data is in scratch buffer */ lb = TASK_BUF(task, loc - 1, tsize); memcpy(PTR_OFFSET(user_rbuf, rdisps[idx] * dt_size), PTR_OFFSET(lb, merge_buf_size + cur_buf_index * dt_size), cur_buf_size * dt_size); } } } } static ucc_status_t pairwise_manager(ucc_rank_t trank, ucc_rank_t tsize, size_t dt_size, ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); char *seg_st = TASK_SEG(task, tsize); void *user_sbuf = TASK_ARGS(task).src.info_v.buffer; void *user_rbuf = TASK_ARGS(task).dst.info_v.buffer; int *s_disps = (int*)TASK_ARGS(task).src.info_v.displacements; int *r_disps = (int*)TASK_ARGS(task).dst.info_v.displacements; int *scounts = (int*)TASK_ARGS(task).src.info_v.counts; int *rcounts = (int*)TASK_ARGS(task).dst.info_v.counts; ucc_rank_t *cur = &task->alltoallv_hybrid.cur_out; int chunk_num_limit = UCC_TL_UCP_TEAM_LIB(team)->cfg.alltoallv_hybrid_pairwise_num_posts; int chunk_byte_limit = UCC_TL_UCP_TEAM_LIB(team)->cfg.alltoallv_hybrid_chunk_byte_limit; ucc_status_t status; void* mem_dst; int pairwise_dest, msg_size; if ((task->alltoallv_hybrid.num_in < chunk_num_limit) && (task->alltoallv_hybrid.traffic_in <= chunk_byte_limit) && (task->alltoallv_hybrid.traffic_out <= chunk_byte_limit)) { if ((task->alltoallv_hybrid.num2send == 0) && (task->alltoallv_hybrid.num2recv == 0)) { return UCC_OK; } while (!(IS_DIRECT_SEND(seg_st[*cur]) || IS_DIRECT_RECV(seg_st[*cur]))) { ++(*cur); ucc_assert(*cur < tsize); } if (IS_DIRECT_SEND(seg_st[*cur])) { pairwise_dest = get_pairwise_send_peer(trank, tsize, *cur); mem_dst = PTR_OFFSET(user_sbuf, s_disps[pairwise_dest] * dt_size); msg_size = scounts[pairwise_dest] * dt_size; task->alltoallv_hybrid.traffic_out += msg_size; if ((task->alltoallv_hybrid.num_in > 0) && (task->alltoallv_hybrid.traffic_out > chunk_byte_limit)) { /* too much outgoing trafic, exit */ return UCC_OK; } status = ucc_tl_ucp_send_nb(mem_dst, msg_size, UCC_MEMORY_TYPE_HOST, pairwise_dest, team, task); if (ucc_unlikely(UCC_OK != status)) { return status; } seg_st[(*cur)] = seg_st[(*cur)] - 1; task->alltoallv_hybrid.num2send--; } if (IS_DIRECT_RECV(seg_st[*cur])) { pairwise_dest = get_pairwise_recv_peer(trank, tsize, *cur); mem_dst = PTR_OFFSET(user_rbuf, r_disps[pairwise_dest] * dt_size); msg_size = rcounts[pairwise_dest] * dt_size; status = ucc_tl_ucp_recv_nb(mem_dst, msg_size, UCC_MEMORY_TYPE_HOST, pairwise_dest, team, task); if (ucc_unlikely(UCC_OK != status)) { return status; } task->alltoallv_hybrid.traffic_in += msg_size; seg_st[(*cur)] = seg_st[(*cur)] - 2; task->alltoallv_hybrid.num2recv--; } task->alltoallv_hybrid.num_in++; ++(*cur); } else { if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return UCC_INPROGRESS; } task->alltoallv_hybrid.num_in = 0; task->alltoallv_hybrid.traffic_in = 0; task->alltoallv_hybrid.traffic_out = 0; } return UCC_OK; } static void ucc_tl_ucp_alltoallv_hybrid_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); ucc_rank_t grank = UCC_TL_TEAM_RANK(team); uint32_t radix = task->alltoallv_hybrid.radix; char *seg_st = TASK_SEG(task, gsize); void *op_metadata = task->alltoallv_hybrid.scratch_mc_header->addr; size_t merge_buf_size = task->alltoallv_hybrid.merge_buf_size; size_t send_limit = task->alltoallv_hybrid.byte_send_limit; int *BytesForPacking = TASK_TMP(task, gsize); size_t buff_size = UCC_TL_UCP_TEAM_LIB(team)->cfg.alltoallv_hybrid_buff_size; size_t tmp_buf_size = buff_size - (merge_buf_size + sizeof(ucc_tl_ucp_alltoallv_hybrid_buf_meta_t)); size_t dt_size = ucc_dt_size(coll_task->bargs.args.dst.info_v.datatype); ucc_rank_t sendto, recvfrom; int node_edge_id, step, i, istep, step_header_size, step_buf_size; size_t snd_count, send_size; void *p_tmp_recv_region, *p_tmp_send_region, *buf; ucc_tl_ucp_alltoallv_hybrid_buf_meta_t *meta; ucc_status_t status; istep = 1; for (i = 1; i < task->alltoallv_hybrid.iteration; i++) { istep *= radix; } /* Loop over algorithm stage - number of stages is ceil(log_k(N)), * where k is algorithm radix and N is communicator size */ for (step = istep; step < gsize; step *= radix) { while ((task->alltoallv_hybrid.phase == UCC_ALLTOALLV_HYBRID_PHASE_START) || (task->alltoallv_hybrid.phase == UCC_ALLTOALLV_HYBRID_PHASE_SENT)) { /* initiate sends * set the current substage index. node_edge_id=1, 2, ..., k-1 */ node_edge_id = task->alltoallv_hybrid.cur_radix; radix_setup(task, node_edge_id - 1, gsize, merge_buf_size, &meta, &p_tmp_send_region, &p_tmp_recv_region); /* figure out number of desinaiton ranks the will be included in the current send */ snd_count = get_send_block_count(gsize, radix, node_edge_id, step); if (!snd_count) { task->alltoallv_hybrid.cur_radix++; if (task->alltoallv_hybrid.cur_radix == radix) { task->alltoallv_hybrid.phase = UCC_ALLTOALLV_HYBRID_PHASE_RECV; task->alltoallv_hybrid.cur_radix = 1; } continue; } /* peers to communicate with in the rth exchange of the current stage */ sendto = get_bruck_send_peer(grank, gsize, step, node_edge_id); recvfrom = get_bruck_recv_peer(grank, gsize, step, node_edge_id); /* Send aggregated data */ step_header_size = calculate_head_size(snd_count, dt_size) * dt_size; step_buf_size = (send_limit * snd_count) + step_header_size; if (task->alltoallv_hybrid.phase == UCC_ALLTOALLV_HYBRID_PHASE_START) { /* Initialize the space data destined to process i, * where i is the index in the array. Buddy buffer algorithm may * increase this value. */ status = get_send_buffer(p_tmp_send_region, meta, NUM_BINS, step_buf_size, task, &buf); if (UCC_OK != status) { task->super.status = status; goto out; } /* pack the data */ send_size = pack_send_data(task, step, snd_count, node_edge_id, dt_size, send_limit, step_header_size, buf); ucc_assert(step_buf_size >= send_size); meta->bins[meta->cur_bin].len = send_size; status = send_data(buf, send_size, sendto, meta, task); if (ucc_unlikely(UCC_OK != status)) { task->super.status = status; goto out; } } status = post_recv(recvfrom, gsize, dt_size, node_edge_id, step, grank, op_metadata, tmp_buf_size, seg_st, meta, step_buf_size, BytesForPacking, p_tmp_recv_region, task); if (ucc_unlikely(UCC_OK != status)) { task->super.status = status; goto out; } } status = complete_current_step_receives(gsize, step, dt_size, grank, op_metadata, seg_st, task); if (UCC_OK != status) { task->super.status = status; goto out; } task->alltoallv_hybrid.phase = UCC_ALLTOALLV_HYBRID_PHASE_START; task->alltoallv_hybrid.cur_radix = 1; task->alltoallv_hybrid.iteration++; } /* The brucks iterations are done. Now we send and recv all the * pairwise we didn't already send and receive */ while ((task->alltoallv_hybrid.num2recv > 0) || (task->alltoallv_hybrid.num2send > 0)) { status = pairwise_manager(grank, gsize, dt_size, task); if (UCC_OK != status) { task->super.status = status; goto out; } } if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } hybrid_reverse_rotation(task); task->super.status = UCC_OK; out: if (task->super.status != UCC_INPROGRESS) { UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_alltoallv_hybrid_done", 0); } } static inline void meta_init(ucc_tl_ucp_alltoallv_hybrid_buf_meta_t* meta, ucc_tl_ucp_task_t *task) { int i; meta->cur_bin = 0; meta->offset = 0; for (i = 0; i < NUM_BINS; i++) { meta->bins[i].len = 0; meta->bins[i].task = task; } } static inline void copy_brucks_rotation(void *scratch, void *scounts, void *sdisps, ucc_rank_t trank, ucc_rank_t tsize) { size_t size = sizeof(int); /* currently support for 32 bit counts */ memcpy(scratch, PTR_OFFSET(sdisps, trank * size), (tsize - trank) * size); memcpy(PTR_OFFSET(scratch, tsize * size), PTR_OFFSET(scounts, trank * size), (tsize - trank) * size); /* Copy the rest part */ if (trank != 0) { memcpy(PTR_OFFSET(scratch, (tsize - trank) * size), sdisps, trank * size); memcpy(PTR_OFFSET(scratch, (tsize - trank + tsize) * size), scounts, trank * size); } } static ucc_status_t ucc_tl_ucp_alltoallv_hybrid_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); uint32_t radix = task->alltoallv_hybrid.radix; ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_alltoallv_hybrid_buf_meta_t *lbm; int i; lbm = TASK_LB(task, tsize); UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_alltoallv_hybrid_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); copy_brucks_rotation(task->alltoallv_hybrid.scratch_mc_header->addr, args->src.info_v.counts, args->src.info_v.displacements, trank, tsize); task->alltoallv_hybrid.num_in = 0; task->alltoallv_hybrid.cur_out = 1; task->alltoallv_hybrid.cur_radix = 1; task->alltoallv_hybrid.traffic_in = 0; task->alltoallv_hybrid.traffic_out = 0; task->alltoallv_hybrid.num2send = 0; task->alltoallv_hybrid.num2recv = 0; task->alltoallv_hybrid.phase = UCC_ALLTOALLV_HYBRID_PHASE_START; task->alltoallv_hybrid.iteration = 1; memset(TASK_SEG(task, tsize), 0, tsize * sizeof(char)); for (i = 0; i < radix - 1; ++i) { meta_init(&lbm[i], task); } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_alltoallv_hybrid_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(tl_team); uint32_t radix = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.alltoallv_hybrid_radix; size_t buff_size = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.alltoallv_hybrid_buff_size; uint32_t snd_size = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.alltoallv_hybrid_num_scratch_sends; uint32_t rcv_size = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.alltoallv_hybrid_num_scratch_recvs; ucc_tl_ucp_task_t *task; size_t scratch_size, calc_limit, max_snd_count, dt_size; ucc_status_t status; if (UCC_COLL_ARGS_DISPL64(&coll_args->args) || UCC_COLL_ARGS_COUNT64(&coll_args->args) || coll_args->args.src.info_v.mem_type != UCC_MEMORY_TYPE_HOST || coll_args->args.dst.info_v.mem_type != UCC_MEMORY_TYPE_HOST) { return UCC_ERR_NOT_SUPPORTED; } task = ucc_tl_ucp_init_task(coll_args, team); if (ucc_unlikely(!task)) { return UCC_ERR_NO_MEMORY; } task->super.post = ucc_tl_ucp_alltoallv_hybrid_start; task->super.progress = ucc_tl_ucp_alltoallv_hybrid_progress; task->super.finalize = ucc_tl_ucp_alltoallv_hybrid_finalize; dt_size = ucc_dt_size(coll_args->args.dst.info_v.datatype); task->alltoallv_hybrid.radix = radix; scratch_size = 2 * tsize * sizeof(int) /* rotated scounts and sdispls */ + 2 * tsize * sizeof(int) /* BytesForPacking */ + tsize * sizeof(char*) /* seg_st */ + (radix - 1) * sizeof(ucc_tl_ucp_alltoallv_hybrid_buf_meta_t) + ucc_div_round_up(tsize, 2) * sizeof(void*) + (radix - 1) * buff_size; status = ucc_mc_alloc(&task->alltoallv_hybrid.scratch_mc_header, scratch_size, UCC_MEMORY_TYPE_HOST); if (ucc_unlikely(UCC_OK != status)) { ucc_tl_ucp_put_task(task); return status; } /* TODO: fix for radix > 2 */ max_snd_count = ucc_ceil(tsize, radix) / radix; calc_limit = ((buff_size - 256) / (snd_size + rcv_size) - ucc_ceil(sizeof(int) * (max_snd_count + 1), dt_size)) / max_snd_count; calc_limit -= (calc_limit % 4); ucc_assert(calc_limit > 0); task->alltoallv_hybrid.byte_send_limit = calc_limit; task->alltoallv_hybrid.merge_buf_size = ALIGN((calc_limit*max_snd_count + ucc_ceil(sizeof(int) * (max_snd_count + 1), dt_size)) * snd_size); *task_h = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/alltoallv/alltoallv.h0000664000175000017500000000631615211535620023003 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLTOALLV_H_ #define ALLTOALLV_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" enum { UCC_TL_UCP_ALLTOALLV_ALG_PAIRWISE, UCC_TL_UCP_ALLTOALLV_ALG_HYBRID, UCC_TL_UCP_ALLTOALLV_ALG_ONESIDED, UCC_TL_UCP_ALLTOALLV_ALG_LAST }; #define UCC_TL_UCP_ALLTOALLV_DEFAULT_ALG_SELECT_STR \ "alltoallv:host:[64-inf]:@hybrid" extern ucc_base_coll_alg_info_t ucc_tl_ucp_alltoallv_algs[UCC_TL_UCP_ALLTOALLV_ALG_LAST + 1]; ucc_status_t ucc_tl_ucp_alltoallv_init(ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_alltoallv_pairwise_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_alltoallv_hybrid_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_alltoallv_onesided_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_alltoallv_pairwise_init_common(ucc_tl_ucp_task_t *task); #define ALLTOALLV_CHECK_INPLACE(_args, _team) \ do { \ if (UCC_IS_INPLACE(_args)) { \ tl_error(UCC_TL_TEAM_LIB(_team), \ "inplace alltoallv is not supported"); \ status = UCC_ERR_NOT_SUPPORTED; \ goto out; \ } \ } while (0) #define ALLTOALLV_CHECK_USERDEFINED_DT(_args, _team) \ do { \ if (!ucc_coll_args_is_predefined_dt(&(_args), UCC_RANK_INVALID)) { \ tl_error(UCC_TL_TEAM_LIB(_team), \ "user defined datatype is not supported"); \ status = UCC_ERR_NOT_SUPPORTED; \ goto out; \ } \ } while (0) #define ALLTOALLV_TASK_CHECK(_args, _team) \ ALLTOALLV_CHECK_INPLACE((_args), (_team)); \ ALLTOALLV_CHECK_USERDEFINED_DT((_args), (_team)); static inline int ucc_tl_ucp_alltoallv_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_UCP_ALLTOALLV_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_ucp_alltoallv_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/ucp/tl_ucp_service_coll.c0000664000175000017500000002035115211535620023024 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "tl_ucp_coll.h" #include "tl_ucp_tag.h" #include "allreduce/allreduce.h" #include "allgather/allgather.h" #include "bcast/bcast.h" //NOLINTNEXTLINE subset unused static ucc_rank_t ucc_tl_ucp_service_ring_get_send_block(ucc_subset_t *subset, ucc_rank_t trank, ucc_rank_t tsize, int step) { return (trank - step + tsize) % tsize; } //NOLINTNEXTLINE subset unused static ucc_rank_t ucc_tl_ucp_service_ring_get_recv_block(ucc_subset_t *subset, ucc_rank_t trank, ucc_rank_t tsize, int step) { return (trank - step - 1 + tsize) % tsize; } static ucc_status_t ucc_tl_ucp_service_coll_start_executor(ucc_coll_task_t *task) { ucc_ee_executor_params_t eparams; ucc_status_t status; eparams.mask = UCC_EE_EXECUTOR_PARAM_FIELD_TYPE; eparams.ee_type = UCC_EE_CPU_THREAD; status = ucc_ee_executor_init(&eparams, &task->executor); if (status != UCC_OK) { return status; } status = ucc_ee_executor_start(task->executor, NULL); if (status != UCC_OK) { ucc_ee_executor_finalize(task->executor); return status; } task->flags |= UCC_COLL_TASK_FLAG_EXECUTOR_STOP; return UCC_OK; } static ucc_status_t ucc_tl_ucp_service_coll_stop_executor(ucc_coll_task_t *task) { ucc_status_t status, gl_status; gl_status = UCC_OK; status = ucc_ee_executor_stop(task->executor); if (status != UCC_OK) { gl_status = status; } status = ucc_ee_executor_finalize(task->executor); if (status != UCC_OK) { gl_status = status; } return gl_status; } ucc_status_t ucc_tl_ucp_service_allreduce(ucc_base_team_t *team, void *sbuf, void *rbuf, ucc_datatype_t dt, size_t count, ucc_reduction_op_t op, ucc_subset_t subset, ucc_coll_task_t **task_p) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task = ucc_tl_ucp_get_task(tl_team); int in_place = (sbuf == rbuf); ucc_status_t status; ucc_base_coll_args_t bargs = { .args = { .mask = in_place ? UCC_COLL_ARGS_FIELD_FLAGS : 0, .coll_type = UCC_COLL_TYPE_ALLREDUCE, .op = op, .flags = in_place ? UCC_COLL_ARGS_FLAG_IN_PLACE : 0, .src.info = { .buffer = sbuf, .count = count, .datatype = dt, .mem_type = UCC_MEMORY_TYPE_HOST }, .dst.info = { .buffer = rbuf, .count = count, .datatype = dt, .mem_type = UCC_MEMORY_TYPE_HOST } } }; status = ucc_coll_task_init(&task->super, &bargs, team); if (status != UCC_OK) { goto free_task; } task->flags = UCC_TL_UCP_TASK_FLAG_SUBSET; task->subset = subset; task->tagged.tag = UCC_TL_UCP_SERVICE_TAG; task->n_polls = UCC_TL_UCP_TEAM_CTX(tl_team)->cfg.oob_npolls; task->super.progress = ucc_tl_ucp_allreduce_knomial_progress; task->super.finalize = ucc_tl_ucp_allreduce_knomial_finalize; status = ucc_tl_ucp_allreduce_knomial_init_common(task); if (status != UCC_OK) { goto free_task; } status = ucc_tl_ucp_service_coll_start_executor(&task->super); if (status != UCC_OK) { goto free_task; } status = ucc_tl_ucp_allreduce_knomial_start(&task->super); if (status != UCC_OK) { goto finalize_coll; } *task_p = &task->super; return status; finalize_coll: ucc_tl_ucp_allreduce_knomial_finalize(&task->super); ucc_tl_ucp_service_coll_stop_executor(&task->super); free_task: ucc_tl_ucp_put_task(task); return status; } ucc_status_t ucc_tl_ucp_service_allgather(ucc_base_team_t *team, void *sbuf, void *rbuf, size_t msgsize, ucc_subset_t subset, ucc_coll_task_t **task_p) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task = ucc_tl_ucp_get_task(tl_team); uint32_t npolls = UCC_TL_UCP_TEAM_CTX(tl_team)->cfg.oob_npolls; int in_place = sbuf == PTR_OFFSET(rbuf, msgsize * subset.myrank); ucc_base_coll_args_t bargs = { .args = { .coll_type = UCC_COLL_TYPE_ALLGATHER, .mask = UCC_COLL_ARGS_FIELD_FLAGS, .flags = in_place ? UCC_COLL_ARGS_FLAG_IN_PLACE : 0, .src.info = {.buffer = sbuf, .count = msgsize, .datatype = UCC_DT_UINT8, .mem_type = UCC_MEMORY_TYPE_HOST}, .dst.info = {.buffer = rbuf, .count = msgsize * subset.map.ep_num, .datatype = UCC_DT_UINT8, .mem_type = UCC_MEMORY_TYPE_HOST} } }; ucc_status_t status; status = ucc_coll_task_init(&task->super, &bargs, team); if (status != UCC_OK) { goto free_task; } task->allgather_ring.get_send_block = ucc_tl_ucp_service_ring_get_send_block; task->allgather_ring.get_recv_block = ucc_tl_ucp_service_ring_get_recv_block; task->flags = UCC_TL_UCP_TASK_FLAG_SUBSET; task->subset = subset; task->tagged.tag = UCC_TL_UCP_SERVICE_TAG; task->n_polls = npolls; task->super.progress = ucc_tl_ucp_allgather_ring_progress; task->super.finalize = ucc_tl_ucp_coll_finalize; status = ucc_tl_ucp_allgather_ring_start(&task->super); if (status != UCC_OK) { goto finalize_coll; } *task_p = &task->super; return status; finalize_coll: ucc_tl_ucp_coll_finalize(*task_p); free_task: ucc_tl_ucp_put_task(task); return status; } ucc_status_t ucc_tl_ucp_service_bcast(ucc_base_team_t *team, void *buf, size_t msgsize, ucc_rank_t root, ucc_subset_t subset, ucc_coll_task_t **task_p) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task = ucc_tl_ucp_get_task(tl_team); ucc_base_coll_args_t bargs = { .args = { .coll_type = UCC_COLL_TYPE_BCAST, .src.info = { .buffer = buf, .count = msgsize, .datatype = UCC_DT_INT8, .mem_type = UCC_MEMORY_TYPE_HOST }, .root = root } }; ucc_status_t status; status = ucc_coll_task_init(&task->super, &bargs, team); if (status != UCC_OK) { goto free_task; } task->flags = UCC_TL_UCP_TASK_FLAG_SUBSET; task->subset = subset; task->tagged.tag = UCC_TL_UCP_SERVICE_TAG; task->n_polls = UCC_TL_UCP_TEAM_CTX(tl_team)->cfg.oob_npolls; task->super.progress = ucc_tl_ucp_bcast_knomial_progress; task->super.finalize = ucc_tl_ucp_coll_finalize; task->bcast_kn.radix = 2; status = ucc_tl_ucp_bcast_knomial_start(&task->super); if (status != UCC_OK) { goto finalize_coll; } *task_p = &task->super; return status; finalize_coll: ucc_tl_ucp_coll_finalize(*task_p); free_task: ucc_tl_ucp_put_task(task); return status; } void ucc_tl_ucp_service_update_id(ucc_base_team_t *team, uint16_t id) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); tl_team->super.super.params.id = id; } ucc-1.8.0/src/components/tl/ucp/barrier/0000775000175000017500000000000015211535620020266 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/barrier/barrier.c0000664000175000017500000000160715211535620022064 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "barrier.h" ucc_status_t ucc_tl_ucp_barrier_knomial_start(ucc_coll_task_t *task); void ucc_tl_ucp_barrier_knomial_progress(ucc_coll_task_t *task); ucc_base_coll_alg_info_t ucc_tl_ucp_barrier_algs[UCC_TL_UCP_BARRIER_ALG_LAST + 1] = { [UCC_TL_UCP_BARRIER_ALG_KNOMIAL] = {.id = UCC_TL_UCP_BARRIER_ALG_KNOMIAL, .name = "knomial", .desc = "recursive knomial with arbitrary radix"}, [UCC_TL_UCP_BARRIER_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_barrier_init(ucc_tl_ucp_task_t *task) { task->super.post = ucc_tl_ucp_barrier_knomial_start; task->super.progress = ucc_tl_ucp_barrier_knomial_progress; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/barrier/barrier.h0000664000175000017500000000071715211535620022072 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef BARRIER_H_ #define BARRIER_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" enum { UCC_TL_UCP_BARRIER_ALG_KNOMIAL, UCC_TL_UCP_BARRIER_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_barrier_algs[UCC_TL_UCP_BARRIER_ALG_LAST + 1]; ucc_status_t ucc_tl_ucp_barrier_init(ucc_tl_ucp_task_t *task); #endif ucc-1.8.0/src/components/tl/ucp/barrier/barrier_knomial.c0000664000175000017500000001036315211535620023575 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "barrier.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "coll_patterns/recursive_knomial.h" #include "utils/ucc_math.h" #define SAVE_STATE(_phase) \ do { \ task->barrier.phase = _phase; \ } while (0) void ucc_tl_ucp_barrier_knomial_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_kn_radix_t radix = task->barrier.p.radix; uint8_t node_type = task->barrier.p.node_type; ucc_knomial_pattern_t *p = &task->barrier.p; ucc_memory_type_t mtype = UCC_MEMORY_TYPE_UNKNOWN; ucc_rank_t peer; ucc_kn_radix_t loop_step; UCC_KN_GOTO_PHASE(task->barrier.phase); if (KN_NODE_EXTRA == node_type) { peer = ucc_knomial_pattern_get_proxy(p, rank); UCPCHECK_GOTO(ucc_tl_ucp_send_nb(NULL, 0, mtype, peer, team, task), task, out); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(NULL, 0, mtype, peer, team, task), task, out); } if (KN_NODE_PROXY == node_type) { peer = ucc_knomial_pattern_get_extra(p, rank); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(NULL, 0, mtype, peer, team, task), task, out); } UCC_KN_PHASE_EXTRA: if (KN_NODE_PROXY == node_type || KN_NODE_EXTRA == node_type) { if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_EXTRA); return; } if (KN_NODE_EXTRA == node_type) { goto completion; } } while(!ucc_knomial_pattern_loop_done(p)) { for (loop_step = 1; loop_step < radix; loop_step++) { peer = ucc_knomial_pattern_get_loop_peer(p, rank, loop_step); if (peer == UCC_KN_PEER_NULL) continue; UCPCHECK_GOTO(ucc_tl_ucp_send_nb(NULL, 0, mtype, peer, team, task), task, out); } for (loop_step = 1; loop_step < radix; loop_step++) { peer = ucc_knomial_pattern_get_loop_peer(p, rank, loop_step); if (peer == UCC_KN_PEER_NULL) continue; UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(NULL, 0, mtype, peer, team, task), task, out); } UCC_KN_PHASE_LOOP: if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_LOOP); return; } ucc_knomial_pattern_next_iteration(p); } if (KN_NODE_PROXY == node_type) { peer = ucc_knomial_pattern_get_extra(p, rank); UCPCHECK_GOTO(ucc_tl_ucp_send_nb(NULL, 0, mtype, peer, team, task), task, out); goto UCC_KN_PHASE_PROXY; } else { goto completion; } UCC_KN_PHASE_PROXY: if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_PROXY); return; } completion: ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_barrier_kn_done", 0); out: return; } ucc_status_t ucc_tl_ucp_barrier_knomial_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_barrier_kn_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); task->barrier.phase = UCC_KN_PHASE_INIT; ucc_knomial_pattern_init(size, rank, ucc_min(UCC_TL_UCP_TEAM_LIB(team)-> cfg.barrier_kn_radix, size), &task->barrier.p); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc-1.8.0/src/components/tl/ucp/alltoall/0000775000175000017500000000000015211535620020444 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/alltoall/alltoall.h0000664000175000017500000000626515211535620022432 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLTOALL_H_ #define ALLTOALL_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" enum { UCC_TL_UCP_ALLTOALL_ALG_PAIRWISE, UCC_TL_UCP_ALLTOALL_ALG_BRUCK, UCC_TL_UCP_ALLTOALL_ALG_ONESIDED, UCC_TL_UCP_ALLTOALL_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_alltoall_algs[UCC_TL_UCP_ALLTOALL_ALG_LAST + 1]; #define UCC_TL_UCP_ALLTOALL_DEFAULT_ALG_SELECT_STR_PATTERN \ "alltoall:host:0-%d:@bruck" char* ucc_tl_ucp_alltoall_score_str_get(ucc_tl_ucp_team_t *team); ucc_status_t ucc_tl_ucp_alltoall_init(ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_alltoall_pairwise_init_common(ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_alltoall_pairwise_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_alltoall_bruck_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_alltoall_onesided_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); #define ALLTOALL_CHECK_INPLACE(_args, _team) \ do { \ if (UCC_IS_INPLACE(_args)) { \ tl_error(UCC_TL_TEAM_LIB(_team), \ "inplace alltoall is not supported"); \ status = UCC_ERR_NOT_SUPPORTED; \ goto out; \ } \ } while (0) #define ALLTOALL_CHECK_USERDEFINED_DT(_args, _team ) \ do { \ if (!ucc_coll_args_is_predefined_dt(&(_args), UCC_RANK_INVALID)) { \ tl_error(UCC_TL_TEAM_LIB(_team), \ "user defined datatype is not supported"); \ status = UCC_ERR_NOT_SUPPORTED; \ goto out; \ } \ } while (0) #define ALLTOALL_TASK_CHECK(_args, _team) \ ALLTOALL_CHECK_INPLACE((_args), (_team)); \ ALLTOALL_CHECK_USERDEFINED_DT((_args), (_team)); static inline int ucc_tl_ucp_alltoall_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_UCP_ALLTOALL_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_ucp_alltoall_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/ucp/alltoall/alltoall_onesided.c0000664000175000017500000003040515211535620024270 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "alltoall.h" #include "core/ucc_progress_queue.h" #include "utils/ucc_math.h" #include "tl_ucp_sendrecv.h" #define CONGESTION_THRESHOLD 8 /* Common helper function to check completion and handle polling */ static inline int alltoall_onesided_handle_completion( ucc_tl_ucp_task_t *task, uint32_t *posted, uint32_t *completed, uint32_t nreqs, int64_t npolls) { int64_t polls = 0; if ((*posted - *completed) >= nreqs) { while (polls < npolls) { ucp_worker_progress(TASK_CTX(task)->worker.ucp_worker); ++polls; if ((*posted - *completed) < nreqs) { break; } } if (polls >= npolls) { return 0; /* Return 0 to indicate should return */ } } return 1; /* Return 1 to indicate should continue */ } /* Common helper function to wait for all operations to complete */ static inline void alltoall_onesided_wait_completion(ucc_tl_ucp_task_t *task, int64_t npolls) { int64_t polls = 0; if (!UCC_TL_UCP_TASK_ONESIDED_P2P_COMPLETE(task)) { while (polls++ < npolls) { ucp_worker_progress(TASK_CTX(task)->worker.ucp_worker); if (UCC_TL_UCP_TASK_ONESIDED_P2P_COMPLETE(task)) { task->super.status = UCC_OK; return; } } return; } task->super.status = UCC_OK; } ucc_status_t ucc_tl_ucp_alltoall_onesided_sched_start(ucc_coll_task_t *ctask) { return ucc_schedule_start(ctask); } ucc_status_t ucc_tl_ucp_alltoall_onesided_sched_finalize(ucc_coll_task_t *ctask) { ucc_schedule_t *schedule = ucc_derived_of(ctask, ucc_schedule_t); ucc_status_t status; status = ucc_schedule_finalize(ctask); ucc_tl_ucp_put_schedule(schedule); return status; } ucc_status_t ucc_tl_ucp_alltoall_onesided_finalize(ucc_coll_task_t *coll_task) { ucc_status_t status; status = ucc_tl_ucp_coll_finalize(coll_task); if (ucc_unlikely(UCC_OK != status)) { tl_error(UCC_TASK_LIB(coll_task), "failed to finalize collective"); } return status; } void ucc_tl_ucp_alltoall_onesided_get_progress(ucc_coll_task_t *ctask) { ucc_tl_ucp_task_t *task = ucc_derived_of(ctask, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ptrdiff_t src = (ptrdiff_t)TASK_ARGS(task).src.info.buffer; ptrdiff_t dest = (ptrdiff_t)TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t mtype = TASK_ARGS(task).dst.info.mem_type; ucc_rank_t grank = UCC_TL_TEAM_RANK(team); ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); uint32_t ntokens = task->alltoall_onesided.tokens; int64_t npolls = task->alltoall_onesided.npolls; /* To resolve remote virtual addresses, the dst_memh is the one that must * have the rkey information. For this algorithm, we need to swap the * src and dst handles to operate correctly */ ucc_mem_map_mem_h *dst_memh = TASK_ARGS(task).src_memh.global_memh; uint32_t *posted = &task->onesided.get_posted; uint32_t *completed = &task->onesided.get_completed; ucc_rank_t peer = (grank + *posted + 1) % gsize; ucc_mem_map_mem_h src_memh; size_t nelems; nelems = TASK_ARGS(task).src.info.count; nelems = (nelems / gsize) * ucc_dt_size(TASK_ARGS(task).src.info.datatype); src_memh = (TASK_ARGS(task).flags & UCC_COLL_ARGS_FLAG_DST_MEMH_GLOBAL) ? TASK_ARGS(task).dst_memh.global_memh[grank] : TASK_ARGS(task).dst_memh.local_memh; for (; *posted < gsize; peer = (peer + 1) % gsize) { UCPCHECK_GOTO(ucc_tl_ucp_get_nb(PTR_OFFSET(dest, peer * nelems), PTR_OFFSET(src, grank * nelems), nelems, mtype, peer, src_memh, dst_memh, team, task), task, out); if (!alltoall_onesided_handle_completion(task, posted, completed, ntokens, npolls)) { return; } } alltoall_onesided_wait_completion(task, npolls); out: return; } void ucc_tl_ucp_alltoall_onesided_put_progress(ucc_coll_task_t *ctask) { ucc_tl_ucp_task_t *task = ucc_derived_of(ctask, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ptrdiff_t src = (ptrdiff_t)TASK_ARGS(task).src.info.buffer; ptrdiff_t dest = (ptrdiff_t)TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t mtype = TASK_ARGS(task).src.info.mem_type; ucc_rank_t grank = UCC_TL_TEAM_RANK(team); ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); uint32_t ntokens = task->alltoall_onesided.tokens; int64_t npolls = task->alltoall_onesided.npolls; ucc_mem_map_mem_h *dst_memh = TASK_ARGS(task).dst_memh.global_memh; uint32_t *posted = &task->onesided.put_posted; uint32_t *completed = &task->onesided.put_completed; ucc_rank_t peer = (grank + *posted + 1) % gsize; ucc_mem_map_mem_h src_memh; size_t nelems; nelems = TASK_ARGS(task).src.info.count; nelems = (nelems / gsize) * ucc_dt_size(TASK_ARGS(task).src.info.datatype); src_memh = (TASK_ARGS(task).flags & UCC_COLL_ARGS_FLAG_SRC_MEMH_GLOBAL) ? TASK_ARGS(task).src_memh.global_memh[grank] : TASK_ARGS(task).src_memh.local_memh; for (; *posted < gsize; peer = (peer + 1) % gsize) { UCPCHECK_GOTO( ucc_tl_ucp_put_nb(PTR_OFFSET(src, peer * nelems), PTR_OFFSET(dest, grank * nelems), nelems, mtype, peer, src_memh, dst_memh, team, task), task, out); UCPCHECK_GOTO(ucc_tl_ucp_ep_flush(peer, team, task), task, out); if (!alltoall_onesided_handle_completion(task, posted, completed, ntokens, npolls)) { return; } } alltoall_onesided_wait_completion(task, npolls); out: return; } ucc_status_t ucc_tl_ucp_alltoall_onesided_start(ucc_coll_task_t *ctask) { ucc_tl_ucp_task_t *task = ucc_derived_of(ctask, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_alltoall_onesided_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_schedule_t *schedule = NULL; ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_base_coll_args_t barrier_coll_args = { .team = team->params.team, .args.coll_type = UCC_COLL_TYPE_BARRIER, }; size_t perc_bw = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.alltoall_onesided_percent_bw; ucc_tl_ucp_alltoall_onesided_alg_t alg = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.alltoall_onesided_alg; ucc_tl_ucp_schedule_t *tl_schedule = NULL; ucc_rank_t group_size = 1; ucc_coll_task_t *barrier_task; ucc_coll_task_t *a2a_task; ucc_tl_ucp_task_t *task; ucc_status_t status; size_t nelems; double rate; size_t ratio; ucp_ep_h ep; ucp_ep_evaluate_perf_param_t param; ucp_ep_evaluate_perf_attr_t attr; int64_t npolls; ucc_sbgp_t *sbgp; ALLTOALL_TASK_CHECK(coll_args->args, tl_team); if (!(coll_args->args.mask & UCC_COLL_ARGS_FIELD_FLAGS) || (coll_args->args.mask & UCC_COLL_ARGS_FIELD_FLAGS && (!(coll_args->args.flags & UCC_COLL_ARGS_FLAG_MEM_MAPPED_BUFFERS)))) { tl_error(UCC_TL_TEAM_LIB(tl_team), "non memory mapped buffers are not supported"); status = UCC_ERR_NOT_SUPPORTED; return status; } if (!(coll_args->args.mask & UCC_COLL_ARGS_FIELD_MEM_MAP_SRC_MEMH)) { coll_args->args.src_memh.global_memh = NULL; } else { if (!(coll_args->args.flags & UCC_COLL_ARGS_FLAG_SRC_MEMH_GLOBAL)) { tl_error(UCC_TL_TEAM_LIB(tl_team), "onesided alltoall requires global memory handles for src buffers"); status = UCC_ERR_INVALID_PARAM; return status; } } if (!(coll_args->args.mask & UCC_COLL_ARGS_FIELD_MEM_MAP_DST_MEMH)) { coll_args->args.dst_memh.global_memh = NULL; } else { if (!(coll_args->args.flags & UCC_COLL_ARGS_FLAG_DST_MEMH_GLOBAL)) { tl_error(UCC_TL_TEAM_LIB(tl_team), "onesided alltoall requires global memory handles for dst buffers"); status = UCC_ERR_INVALID_PARAM; return status; } } status = ucc_tl_ucp_get_schedule(tl_team, coll_args, (ucc_tl_ucp_schedule_t **)&tl_schedule); if (ucc_unlikely(UCC_OK != status)) { return status; } schedule = &tl_schedule->super.super; ucc_schedule_init(schedule, coll_args, team); schedule->super.post = ucc_tl_ucp_alltoall_onesided_sched_start; schedule->super.progress = NULL; schedule->super.finalize = ucc_tl_ucp_alltoall_onesided_sched_finalize; sbgp = ucc_topo_get_sbgp(tl_team->topo, UCC_SBGP_NODE); if (sbgp->status == UCC_SBGP_NOT_EXISTS) { /* 1 PPN, use put */ if (alg == UCC_TL_UCP_ALLTOALL_ONESIDED_AUTO) { alg = UCC_TL_UCP_ALLTOALL_ONESIDED_PUT; } } else { group_size = sbgp->group_size; } task = ucc_tl_ucp_init_task(coll_args, team); task->super.finalize = ucc_tl_ucp_alltoall_onesided_finalize; a2a_task = &task->super; status = ucc_tl_ucp_coll_init(&barrier_coll_args, team, &barrier_task); if (status != UCC_OK) { goto out; } if (perc_bw > 100) { perc_bw = 100; } else if (perc_bw == 0) { perc_bw = 1; } nelems = TASK_ARGS(task).src.info.count; nelems = nelems / UCC_TL_TEAM_SIZE(tl_team); param.field_mask = UCP_EP_PERF_PARAM_FIELD_MESSAGE_SIZE; attr.field_mask = UCP_EP_PERF_ATTR_FIELD_ESTIMATED_TIME; param.message_size = nelems * ucc_dt_size(TASK_ARGS(task).src.info.datatype);; ucc_tl_ucp_get_ep( tl_team, (UCC_TL_TEAM_RANK(tl_team) + 1) % UCC_TL_TEAM_SIZE(tl_team), &ep); ucp_ep_evaluate_perf(ep, ¶m, &attr); rate = (1 / attr.estimated_time) * (double)(perc_bw / 100.0); ratio = (nelems > 0) ? nelems * group_size : 1; task->alltoall_onesided.tokens = rate / ratio; if (task->alltoall_onesided.tokens < 1) { task->alltoall_onesided.tokens = 1; } task->super.post = ucc_tl_ucp_alltoall_onesided_start; npolls = task->n_polls; if (alg == UCC_TL_UCP_ALLTOALL_ONESIDED_GET || (alg == UCC_TL_UCP_ALLTOALL_ONESIDED_AUTO && group_size >= CONGESTION_THRESHOLD)) { npolls = nelems * ucc_dt_size(TASK_ARGS(task).src.info.datatype); if (npolls < task->n_polls) { npolls = task->n_polls; } task->super.progress = ucc_tl_ucp_alltoall_onesided_get_progress; } else { task->super.progress = ucc_tl_ucp_alltoall_onesided_put_progress; } task->alltoall_onesided.npolls = npolls; ucc_schedule_add_task(schedule, a2a_task); ucc_task_subscribe_dep(&schedule->super, a2a_task, UCC_EVENT_SCHEDULE_STARTED); ucc_schedule_add_task(schedule, barrier_task); ucc_task_subscribe_dep(a2a_task, barrier_task, UCC_EVENT_COMPLETED); *task_h = &schedule->super; return status; out: if (tl_schedule) { ucc_tl_ucp_put_schedule(&tl_schedule->super.super); } return status; } ucc-1.8.0/src/components/tl/ucp/alltoall/alltoall_pairwise.c0000664000175000017500000001167115211535620024325 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "alltoall.h" #include "core/ucc_progress_queue.h" #include "utils/ucc_math.h" #include "tl_ucp_sendrecv.h" /* TODO: add as parameters */ #define MSG_MEDIUM 66000 #define NP_THRESH 32 static inline ucc_rank_t get_recv_peer(ucc_rank_t rank, ucc_rank_t size, ucc_rank_t step) { return (rank + step) % size; } static inline ucc_rank_t get_send_peer(ucc_rank_t rank, ucc_rank_t size, ucc_rank_t step) { return (rank - step + size) % size; } static ucc_rank_t get_num_posts(const ucc_tl_ucp_team_t *team, const ucc_coll_args_t *args) { unsigned long posts = UCC_TL_UCP_TEAM_LIB(team)->cfg.alltoall_pairwise_num_posts; ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); size_t data_size; data_size = (size_t)args->src.info.count * ucc_dt_size(args->src.info.datatype); if (posts == UCC_ULUNITS_AUTO) { if ((data_size > MSG_MEDIUM) && (tsize > NP_THRESH)) { /* use pairwise algorithm */ posts = 1; } else { /* use linear algorithm */ posts = 0; } } posts = (posts > tsize || posts == 0) ? tsize: posts; return posts; } void ucc_tl_ucp_alltoall_pairwise_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ptrdiff_t sbuf = (ptrdiff_t)TASK_ARGS(task).src.info.buffer; ptrdiff_t rbuf = (ptrdiff_t)TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t smem = TASK_ARGS(task).src.info.mem_type; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_rank_t grank = UCC_TL_TEAM_RANK(team); ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); int polls = 0; ucc_rank_t peer, nreqs; size_t data_size; nreqs = get_num_posts(team, &TASK_ARGS(task)); data_size = (size_t)(TASK_ARGS(task).src.info.count / gsize) * ucc_dt_size(TASK_ARGS(task).src.info.datatype); while ((task->tagged.send_posted < gsize || task->tagged.recv_posted < gsize) && (polls++ < task->n_polls)) { ucp_worker_progress(UCC_TL_UCP_TEAM_CTX(team)->worker.ucp_worker); while ((task->tagged.recv_posted < gsize) && ((task->tagged.recv_posted - task->tagged.recv_completed) < nreqs)) { peer = get_recv_peer(grank, gsize, task->tagged.recv_posted); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb((void *)(rbuf + peer * data_size), data_size, rmem, peer, team, task), task, out); polls = 0; } while ((task->tagged.send_posted < gsize) && ((task->tagged.send_posted - task->tagged.send_completed) < nreqs)) { peer = get_send_peer(grank, gsize, task->tagged.send_posted); UCPCHECK_GOTO(ucc_tl_ucp_send_nb((void *)(sbuf + peer * data_size), data_size, smem, peer, team, task), task, out); polls = 0; } } if ((task->tagged.send_posted < gsize) || (task->tagged.recv_posted < gsize)) { return; } task->super.status = ucc_tl_ucp_test(task); out: if (task->super.status != UCC_INPROGRESS) { UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_alltoall_pairwise_done", 0); } } ucc_status_t ucc_tl_ucp_alltoall_pairwise_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_alltoall_pairwise_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_alltoall_pairwise_init_common(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); size_t data_size; task->super.post = ucc_tl_ucp_alltoall_pairwise_start; task->super.progress = ucc_tl_ucp_alltoall_pairwise_progress; task->n_polls = ucc_max(1, task->n_polls); if (UCC_TL_UCP_TEAM_CTX(team)->cfg.pre_reg_mem) { data_size = (size_t)args->src.info.count * ucc_dt_size(args->src.info.datatype); ucc_tl_ucp_pre_register_mem(team, args->src.info.buffer, data_size, args->src.info.mem_type); ucc_tl_ucp_pre_register_mem(team, args->dst.info.buffer, data_size, args->dst.info.mem_type); } return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/alltoall/alltoall.c0000664000175000017500000000500215211535620022411 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "alltoall.h" #define ALLTOALL_MAX_PATTERN_SIZE (sizeof(UCC_TL_UCP_ALLTOALL_DEFAULT_ALG_SELECT_STR_PATTERN) + 32) #define ALLTOALL_DEFAULT_ALG_SWITCH 129 ucc_status_t ucc_tl_ucp_alltoall_pairwise_start(ucc_coll_task_t *task); void ucc_tl_ucp_alltoall_pairwise_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_alltoall_onesided_start(ucc_coll_task_t *task); void ucc_tl_ucp_alltoall_onesided_progress(ucc_coll_task_t *task); char* ucc_tl_ucp_alltoall_score_str_get(ucc_tl_ucp_team_t *team) { int max_size = ALLTOALL_MAX_PATTERN_SIZE; char *str; str = ucc_malloc(max_size * sizeof(char)); ucc_snprintf_safe(str, max_size, UCC_TL_UCP_ALLTOALL_DEFAULT_ALG_SELECT_STR_PATTERN, ALLTOALL_DEFAULT_ALG_SWITCH * UCC_TL_TEAM_SIZE(team)); return str; } ucc_base_coll_alg_info_t ucc_tl_ucp_alltoall_algs[UCC_TL_UCP_ALLTOALL_ALG_LAST + 1] = { [UCC_TL_UCP_ALLTOALL_ALG_PAIRWISE] = {.id = UCC_TL_UCP_ALLTOALL_ALG_PAIRWISE, .name = "pairwise", .desc = "pairwise two-sided implementation"}, [UCC_TL_UCP_ALLTOALL_ALG_BRUCK] = {.id = UCC_TL_UCP_ALLTOALL_ALG_BRUCK, .name = "bruck", .desc = "Bruck alltoall"}, [UCC_TL_UCP_ALLTOALL_ALG_ONESIDED] = {.id = UCC_TL_UCP_ALLTOALL_ALG_ONESIDED, .name = "onesided", .desc = "naive, linear one-sided implementation"}, [UCC_TL_UCP_ALLTOALL_ALG_LAST] = {.id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_alltoall_init(ucc_tl_ucp_task_t *task) { ucc_status_t status; ALLTOALL_TASK_CHECK(TASK_ARGS(task), TASK_TEAM(task)); status = ucc_tl_ucp_alltoall_pairwise_init_common(task); out: return status; } ucc_status_t ucc_tl_ucp_alltoall_pairwise_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task; ucc_status_t status; ALLTOALL_TASK_CHECK(coll_args->args, tl_team); task = ucc_tl_ucp_init_task(coll_args, team); *task_h = &task->super; status = ucc_tl_ucp_alltoall_pairwise_init_common(task); out: return status; } ucc-1.8.0/src/components/tl/ucp/alltoall/alltoall_bruck.c0000664000175000017500000002672615211535620023617 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "alltoall.h" #include "tl_ucp_sendrecv.h" #include "components/mc/ucc_mc.h" #include "coll_patterns/bruck_alltoall.h" #define RADIX 2 #define SAVE_STATE(_phase) \ do { \ task->alltoall_bruck.phase = _phase; \ } while (0) enum { PHASE_MERGE, PHASE_SENDRECV, PHASE_BCOPY }; static inline int msb_pos_for_level(unsigned int nthbit, ucc_rank_t number) { int msb_set = -1; unsigned int i; for (i = 0; i < nthbit - 1; i++) { if (1 & number >> i) { msb_set = i; } } return msb_set; } static inline int find_seg_index(ucc_rank_t seg_index, int level, int nsegs_per_rblock) { int block, blockseg; if (0 == seg_index) { return -1; } block = msb_pos_for_level(level, seg_index); if (block < 0) { return -1; } /* remove block bit from seg_index */ blockseg = ((seg_index >> (block + 1)) << block) | (seg_index & UCC_MASK(block)); return block * nsegs_per_rblock + blockseg; } ucc_status_t ucc_tl_ucp_alltoall_bruck_backward_rotation(void *dst, void *src, ucc_rank_t trank, ucc_rank_t tsize, size_t seg_size) { ucc_status_t st; ucc_rank_t index, level, nsegs_per_rblock; size_t snd_offset; int send_buffer_index; level = lognum(tsize); nsegs_per_rblock = tsize / 2; for (index = 1; index < tsize; index++) { send_buffer_index = find_seg_index(index, level + 1, nsegs_per_rblock); ucc_assert(send_buffer_index >= 0); snd_offset = send_buffer_index * seg_size; st = ucc_mc_memcpy(PTR_OFFSET(dst, seg_size * ((trank - index + tsize) % tsize)), PTR_OFFSET(src, snd_offset), seg_size, UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_HOST); if (ucc_unlikely(st != UCC_OK)) { return st; } } return UCC_OK; } ucc_status_t ucc_tl_ucp_alltoall_bruck_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_status_t st, global_st; global_st = ucc_mc_free(task->alltoall_bruck.scratch_mc_header); if (ucc_unlikely(global_st != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to free scratch buffer"); } st = ucc_tl_ucp_coll_finalize(&task->super); if (ucc_unlikely(st != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed finalize collective"); global_st = st; } return global_st; } void ucc_tl_ucp_alltoall_bruck_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); ucc_coll_args_t *args = &TASK_ARGS(task); void *scratch = task->alltoall_bruck.scratch_mc_header->addr; void *mergebuf = task->alltoall_bruck.dst; const ucc_rank_t nrecv_segs = tsize / 2; const size_t seg_size = ucc_dt_size(args->src.info.datatype) * args->src.info.count / tsize; ucc_memory_type_t smtype = args->src.info.mem_type; ucc_memory_type_t dmtype = args->dst.info.mem_type; ucc_rank_t sendto, recvfrom, step, index; void *data; ucc_rank_t level, snd_count; int send_buffer_index; ucc_status_t status; ucc_ee_executor_t *exec; ucc_ee_executor_task_args_t eargs; EXEC_TASK_TEST(task->alltoall_bruck.phase, "failed to copy data from user buffer to scratch", task->alltoall_bruck.etask); switch (task->alltoall_bruck.phase) { case PHASE_SENDRECV: goto ALLTOALL_BRUCK_PHASE_SENDRECV; case PHASE_BCOPY: task->super.status = UCC_OK; goto out; } step = 1 << (task->alltoall_bruck.iteration - 1); while (step < tsize) { level = task->alltoall_bruck.iteration - 1; sendto = get_bruck_send_peer(trank, tsize, step, 1); recvfrom = get_bruck_recv_peer(trank, tsize, step, 1); snd_count = 0; for (index = get_bruck_step_start(step, 1); index <= get_bruck_step_finish(tsize - 1, RADIX, 1, step); index = GET_NEXT_BRUCK_NUM(index, RADIX, step)) { send_buffer_index = find_seg_index(index, level + 1, nrecv_segs); if (send_buffer_index == -1) { data = PTR_OFFSET(task->alltoall_bruck.src, ((index + trank) % tsize) * seg_size); } else { data = PTR_OFFSET(scratch, send_buffer_index * seg_size); } status = ucc_mc_memcpy(PTR_OFFSET(mergebuf, seg_size * snd_count), data, seg_size, UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_HOST); if (ucc_unlikely(UCC_OK != status)) { task->super.status = status; return; } snd_count++; } data = PTR_OFFSET(scratch, level * nrecv_segs * seg_size); UCPCHECK_GOTO(ucc_tl_ucp_send_nb(mergebuf, snd_count * seg_size, UCC_MEMORY_TYPE_HOST, sendto, team, task), task, out); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(data, snd_count * seg_size, UCC_MEMORY_TYPE_HOST, recvfrom, team, task), task, out); ALLTOALL_BRUCK_PHASE_SENDRECV: if (ucc_tl_ucp_test(task) == UCC_INPROGRESS) { SAVE_STATE(PHASE_SENDRECV); return; } task->alltoall_bruck.iteration++; step = 1 << (task->alltoall_bruck.iteration - 1); } status = ucc_mc_memcpy(PTR_OFFSET(task->alltoall_bruck.dst, trank * seg_size), PTR_OFFSET(task->alltoall_bruck.src, trank * seg_size), seg_size, UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_HOST); if (ucc_unlikely(status != UCC_OK)) { task->super.status = status; return; } status = ucc_tl_ucp_alltoall_bruck_backward_rotation(mergebuf, scratch, trank, tsize, seg_size); if (ucc_unlikely(status != UCC_OK)) { task->super.status = status; return; } if (smtype != UCC_MEMORY_TYPE_HOST || dmtype != UCC_MEMORY_TYPE_HOST) { task->alltoall_bruck.phase = PHASE_BCOPY; status = ucc_coll_task_get_executor(&task->super, &exec); if (ucc_unlikely(status != UCC_OK)) { task->super.status = status; return; } eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY; eargs.copy.src = mergebuf; eargs.copy.dst = args->dst.info.buffer; eargs.copy.len = seg_size * tsize; eargs.flags = 0; status = ucc_ee_executor_task_post(exec, &eargs, &task->alltoall_bruck.etask); if (ucc_unlikely(status != UCC_OK)) { task->super.status = status; return; } EXEC_TASK_TEST(PHASE_BCOPY, "failed to copy data to user buffer", task->alltoall_bruck.etask); } task->super.status = UCC_OK; out: return; } ucc_status_t ucc_tl_ucp_alltoall_bruck_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); size_t size = ucc_dt_size(args->src.info.datatype) * args->src.info.count; ucc_ee_executor_t *exec; ucc_ee_executor_task_args_t eargs; ucc_status_t status; ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); task->alltoall_bruck.iteration = 1; task->alltoall_bruck.phase = PHASE_MERGE; task->alltoall_bruck.etask = NULL; if ((args->src.info.mem_type != UCC_MEMORY_TYPE_HOST) || (args->dst.info.mem_type != UCC_MEMORY_TYPE_HOST)) { status = ucc_coll_task_get_executor(&task->super, &exec); if (ucc_unlikely(status != UCC_OK)) { return status; } eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY; eargs.copy.src = args->src.info.buffer; eargs.copy.dst = task->alltoall_bruck.src; eargs.copy.len = size; eargs.flags = 0; status = ucc_ee_executor_task_post(exec, &eargs, &task->alltoall_bruck.etask); if (ucc_unlikely(status != UCC_OK)) { return status; } } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_alltoall_bruck_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(tl_team); ucc_coll_args_t *args = &coll_args->args; size_t ssize = ucc_dt_size(args->src.info.datatype) * args->src.info.count; size_t seg_size = ssize / tsize; int is_bcopy = 0; size_t scratch_size; ucc_tl_ucp_task_t *task; ucc_status_t status; ALLTOALL_TASK_CHECK(coll_args->args, tl_team); task = ucc_tl_ucp_init_task(coll_args, team); task->super.post = ucc_tl_ucp_alltoall_bruck_start; task->super.progress = ucc_tl_ucp_alltoall_bruck_progress; task->super.finalize = ucc_tl_ucp_alltoall_bruck_finalize; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; scratch_size = lognum(tsize) * ucc_div_round_up(tsize, 2) * seg_size; if ((coll_args->args.src.info.mem_type != UCC_MEMORY_TYPE_HOST) || (coll_args->args.dst.info.mem_type != UCC_MEMORY_TYPE_HOST)) { is_bcopy = 1; scratch_size += 2 * ssize; } status = ucc_mc_alloc(&task->alltoall_bruck.scratch_mc_header, scratch_size, UCC_MEMORY_TYPE_HOST); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to allocate scratch buffer"); ucc_tl_ucp_coll_finalize(&task->super); return status; } if (is_bcopy) { task->alltoall_bruck.src = PTR_OFFSET(task->alltoall_bruck.scratch_mc_header->addr, lognum(tsize) * ucc_div_round_up(tsize, 2) * seg_size); task->alltoall_bruck.dst = PTR_OFFSET(task->alltoall_bruck.src, ssize); } else { task->alltoall_bruck.src = args->src.info.buffer; task->alltoall_bruck.dst = args->dst.info.buffer; } *task_h = &task->super; return UCC_OK; out: return status; } ucc-1.8.0/src/components/tl/ucp/allreduce/0000775000175000017500000000000015211535620020600 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/allreduce/allreduce_sliding_window.h0000664000175000017500000000500315211535620026007 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLREDUCE_SW_H_ #define ALLREDUCE_SW_H_ #include "tl_ucp_coll.h" #include "tl_ucp_dpu_offload.h" typedef enum ucc_tl_ucp_allreduce_sw_buf_state { FREE, RECVING, REDUCING, REDUCED, SENDING, IDLE, } ucc_tl_ucp_allreduce_sw_buf_state_t; typedef struct ucc_tl_ucp_allreduce_sw_buf { void *buf; ucc_tl_ucp_allreduce_sw_buf_state_t state; ucs_status_ptr_t ucp_req; size_t count; size_t bytes; } ucc_tl_ucp_allreduce_sw_buf_t; typedef struct ucc_tl_ucp_allreduce_sw_pipeline { ucc_tl_ucp_allreduce_sw_buf_t accbuf; ucc_tl_ucp_allreduce_sw_buf_t *getbuf; ucs_status_ptr_t *put_requests; size_t buffer_size; size_t num_buffers; size_t avail_buffs; size_t my_count; size_t my_offset; size_t count_received; size_t count_reduced; size_t count_serviced; size_t get_idx; size_t red_idx; ucc_rank_t src_rank; ucc_rank_t dst_rank; int done_get; int done_red; int done_put; int posted_put; } ucc_tl_ucp_allreduce_sw_pipeline_t; void ucc_tl_ucp_allreduce_sliding_window_free_task(ucc_coll_task_t *coll_task); void ucc_tl_ucp_allreduce_sliding_window_free_pipe(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_ucp_allreduce_sliding_window_alloc_pipe(ucc_base_team_t *team, ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_allreduce_sliding_window_task_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_allreduce_sliding_window_allgather_info_finalize( ucc_tl_ucp_task_t *sw_task); ucc_status_t ucc_tl_ucp_allreduce_sliding_window_rdma_task_post(ucc_coll_task_t *coll_task); void ucc_tl_ucp_allreduce_sliding_window_rdma_progress(ucc_coll_task_t *task); #endif ucc-1.8.0/src/components/tl/ucp/allreduce/allreduce_knomial.c0000664000175000017500000002641215211535620024423 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allreduce.h" #include "core/ucc_progress_queue.h" #include "utils/ucc_dt_reduce.h" #include "tl_ucp_sendrecv.h" #include "coll_patterns/recursive_knomial.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "components/ec/ucc_ec.h" #define SAVE_STATE(_phase) \ do { \ task->allreduce_kn.phase = _phase; \ } while (0) void ucc_tl_ucp_allreduce_knomial_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); int avg_pre_op = team->cfg.reduce_avg_pre_op; ucc_kn_radix_t radix = task->allreduce_kn.p.radix; uint8_t node_type = task->allreduce_kn.p.node_type; ucc_knomial_pattern_t *p = &task->allreduce_kn.p; void *scratch = task->allreduce_kn.scratch; void *sbuf = args->src.info.buffer; void *rbuf = args->dst.info.buffer; ucc_memory_type_t mem_type = args->dst.info.mem_type; size_t count = args->dst.info.count; ucc_datatype_t dt = args->dst.info.datatype; size_t data_size = count * ucc_dt_size(dt); ucc_rank_t rank = task->subset.myrank; void *send_buf; ptrdiff_t recv_offset; ucc_rank_t peer; ucc_status_t status; ucc_kn_radix_t loop_step; int is_avg; ucc_kn_radix_t index; if (UCC_IS_INPLACE(*args)) { sbuf = rbuf; } UCC_KN_REDUCE_GOTO_PHASE(task->allreduce_kn.phase); if (KN_NODE_EXTRA == node_type) { peer = ucc_ep_map_eval(task->subset.map, ucc_knomial_pattern_get_proxy(p, rank)); UCPCHECK_GOTO( ucc_tl_ucp_send_nb(sbuf, data_size, mem_type, peer, team, task), task, out); UCPCHECK_GOTO( ucc_tl_ucp_recv_nb(rbuf, data_size, mem_type, peer, team, task), task, out); } if (KN_NODE_PROXY == node_type) { peer = ucc_ep_map_eval(task->subset.map, ucc_knomial_pattern_get_extra(p, rank)); UCPCHECK_GOTO( ucc_tl_ucp_recv_nb(scratch, data_size, mem_type, peer, team, task), task, out); } UCC_KN_PHASE_EXTRA: if (KN_NODE_PROXY == node_type || KN_NODE_EXTRA == node_type) { if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_EXTRA); return; } if (KN_NODE_EXTRA == node_type) { goto completion; } else { status = ucc_dt_reduce(sbuf, scratch, rbuf, count, dt, args, 0, 0, task->allreduce_kn.executor, &task->allreduce_kn.etask); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); task->super.status = status; return; } UCC_KN_PHASE_EXTRA_REDUCE: EXEC_TASK_TEST(UCC_KN_PHASE_EXTRA_REDUCE, "failed to perform dt reduction", task->allreduce_kn.etask); } } while(!ucc_knomial_pattern_loop_done(p)) { for (loop_step = 1; loop_step < radix; loop_step++) { peer = ucc_knomial_pattern_get_loop_peer(p, rank, loop_step); if (peer == UCC_KN_PEER_NULL) continue; peer = ucc_ep_map_eval(task->subset.map, peer); if ((ucc_knomial_pattern_loop_first_iteration(p)) && (KN_NODE_PROXY != node_type) && !UCC_IS_INPLACE(*args)) { send_buf = sbuf; } else { send_buf = rbuf; } UCPCHECK_GOTO( ucc_tl_ucp_send_nb(send_buf, data_size, mem_type, peer, team, task), task, out); } recv_offset = 0; for (loop_step = radix - 1 ; loop_step > 0; loop_step--) { peer = ucc_knomial_pattern_get_loop_peer(p, rank, loop_step); if (peer == UCC_KN_PEER_NULL) continue; index = ucc_knomial_pattern_get_loop_index(p, peer); peer = ucc_ep_map_eval(task->subset.map, peer); task->allreduce_kn.reduce_bufs[index] = PTR_OFFSET(scratch, recv_offset); UCPCHECK_GOTO( ucc_tl_ucp_recv_nb(PTR_OFFSET(scratch, recv_offset), data_size, mem_type, peer, team, task), task, out); recv_offset += data_size; } UCC_KN_PHASE_LOOP: if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_LOOP); return; } if (task->tagged.send_posted > p->iteration * (radix - 1)) { if ((ucc_knomial_pattern_loop_first_iteration(p)) && (KN_NODE_PROXY != node_type) && !UCC_IS_INPLACE(*args)) { send_buf = sbuf; } else { send_buf = rbuf; } index = ucc_knomial_pattern_get_loop_index(p, rank); task->allreduce_kn.reduce_bufs[index] = send_buf; is_avg = args->op == UCC_OP_AVG && (avg_pre_op ? ucc_knomial_pattern_loop_first_iteration(p) : ucc_knomial_pattern_loop_last_iteration(p)); status = ucc_dt_reduce_multi( task->allreduce_kn.reduce_bufs, rbuf, task->tagged.send_posted - p->iteration * (radix - 1) + 1, count, dt, args, is_avg ? UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA : 0, AVG_ALPHA(task), task->allreduce_kn.executor, &task->allreduce_kn.etask); if (ucc_unlikely(UCC_OK != status)) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); task->super.status = status; return; } UCC_KN_PHASE_REDUCE: EXEC_TASK_TEST(UCC_KN_PHASE_REDUCE, "failed to perform dt reduction", task->allreduce_kn.etask); } ucc_knomial_pattern_next_iteration(p); } if (KN_NODE_PROXY == node_type) { peer = ucc_ep_map_eval(task->subset.map, ucc_knomial_pattern_get_extra(p, rank)); UCPCHECK_GOTO( ucc_tl_ucp_send_nb(rbuf, data_size, mem_type, peer, team, task), task, out); goto UCC_KN_PHASE_PROXY; } else { goto completion; } UCC_KN_PHASE_PROXY: if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_PROXY); return; } completion: ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allreduce_kn_done", 0); UCC_KN_PHASE_COMPLETE: /* unused label */ out: return; } ucc_status_t ucc_tl_ucp_allreduce_knomial_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t size = (ucc_rank_t)task->subset.map.ep_num; ucc_rank_t rank = task->subset.myrank; ucc_memory_type_t mem_type = TASK_ARGS(task).dst.info.mem_type; size_t count = TASK_ARGS(task).dst.info.count; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; size_t data_size = count * ucc_dt_size(dt); ucc_mrange_uint_t *p = &team->cfg.allreduce_kn_radix; ucc_kn_radix_t cfg_radix, radix; ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allreduce_kn_start", 0); task->allreduce_kn.phase = UCC_KN_PHASE_INIT; ucc_assert(UCC_IS_INPLACE(TASK_ARGS(task)) || (TASK_ARGS(task).src.info.mem_type == mem_type)); cfg_radix = ucc_tl_ucp_get_radix_from_range(team, data_size, mem_type, p, UCC_UUNITS_AUTO_RADIX); radix = ucc_min(cfg_radix, size); /* max radix is limited by the number of buffers in the executor */ radix = ucc_min(radix, UCC_EE_EXECUTOR_NUM_BUFS); ucc_knomial_pattern_init(size, rank, radix, &task->allreduce_kn.p); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); status = ucc_coll_task_get_executor(&task->super, &task->allreduce_kn.executor); if (ucc_unlikely(status != UCC_OK)) { return status; } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_allreduce_knomial_init_common(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_memory_type_t mem_type = TASK_ARGS(task).dst.info.mem_type; size_t count = TASK_ARGS(task).dst.info.count; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; size_t data_size = count * ucc_dt_size(dt); ucc_mrange_uint_t *p = &team->cfg.allreduce_kn_radix; ucc_sbgp_t *sbgp; ucc_rank_t size; ucc_kn_radix_t radix, cfg_radix; ucc_status_t status; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_ucp_allreduce_knomial_start; task->super.progress = ucc_tl_ucp_allreduce_knomial_progress; task->super.finalize = ucc_tl_ucp_allreduce_knomial_finalize; if (!(task->flags & UCC_TL_UCP_TASK_FLAG_SUBSET) && team->cfg.use_reordering) { sbgp = ucc_topo_get_sbgp(team->topo, UCC_SBGP_FULL_HOST_ORDERED); task->subset.myrank = sbgp->group_rank; task->subset.map = sbgp->map; } size = (ucc_rank_t)task->subset.map.ep_num; cfg_radix = ucc_tl_ucp_get_radix_from_range(team, data_size, mem_type, p, UCC_UUNITS_AUTO_RADIX); radix = ucc_min(cfg_radix, size); status = ucc_mc_alloc(&task->allreduce_kn.scratch_mc_header, (radix - 1) * data_size, TASK_ARGS(task).dst.info.mem_type); task->allreduce_kn.scratch = task->allreduce_kn.scratch_mc_header->addr; if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to allocate scratch buffer"); return status; } return UCC_OK; } ucc_status_t ucc_tl_ucp_allreduce_knomial_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_status_t st, global_st; global_st = ucc_mc_free(task->allreduce_kn.scratch_mc_header); if (ucc_unlikely(global_st != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to free scratch buffer"); } st = ucc_tl_ucp_coll_finalize(&task->super); if (ucc_unlikely(st != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed finalize collective"); global_st = st; } return global_st; } ucc-1.8.0/src/components/tl/ucp/allreduce/allreduce.h0000664000175000017500000000725315211535620022720 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLREDUCE_H_ #define ALLREDUCE_H_ #include "tl_ucp_coll.h" enum { UCC_TL_UCP_ALLREDUCE_ALG_KNOMIAL, UCC_TL_UCP_ALLREDUCE_ALG_SRA_KNOMIAL, UCC_TL_UCP_ALLREDUCE_ALG_SLIDING_WINDOW, UCC_TL_UCP_ALLREDUCE_ALG_DBT, UCC_TL_UCP_ALLREDUCE_ALG_RING, UCC_TL_UCP_ALLREDUCE_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_allreduce_algs[UCC_TL_UCP_ALLREDUCE_ALG_LAST + 1]; ucc_status_t ucc_tl_ucp_allreduce_init(ucc_tl_ucp_task_t *task); #define UCC_TL_UCP_ALLREDUCE_DEFAULT_ALG_SELECT_STR \ "allreduce:0-4k:@0#allreduce:4k-inf:@1" #define CHECK_SAME_MEMTYPE(_args, _team) \ do { \ if (!UCC_IS_INPLACE(_args) && \ (_args.src.info.mem_type != _args.dst.info.mem_type)) { \ tl_error(UCC_TL_TEAM_LIB(_team), \ "asymmetric src/dst memory types are not supported yet"); \ status = UCC_ERR_NOT_SUPPORTED; \ goto out; \ } \ } while (0) #define ALLREDUCE_TASK_CHECK(_args, _team) \ CHECK_SAME_MEMTYPE((_args), (_team)); ucc_status_t ucc_tl_ucp_allreduce_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_allreduce_sliding_window_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_allreduce_knomial_init_common(ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_allreduce_knomial_start(ucc_coll_task_t *task); void ucc_tl_ucp_allreduce_knomial_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_allreduce_sliding_window_start(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_ucp_allreduce_sliding_window_finalize(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_allreduce_knomial_finalize(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_allreduce_sra_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_allreduce_sra_knomial_start(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_allreduce_sra_knomial_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_allreduce_dbt_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_allreduce_dbt_start(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_allreduce_dbt_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_allreduce_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); static inline int ucc_tl_ucp_allreduce_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_UCP_ALLREDUCE_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_ucp_allreduce_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/ucp/allreduce/allreduce_sra_knomial.c0000664000175000017500000002356515211535620025276 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "allreduce.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "coll_patterns/sra_knomial.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "components/mc/ucc_mc.h" #include "../reduce_scatter/reduce_scatter.h" #include "../allgather/allgather.h" /* SRA - scatter-reduce-allgather knomial algorithm 1. The algorithm performs collective allreduce operation as a sequence of K-nomial Reduce-Scatter followed by K-nomial (with the same radix K) allgather. 2. In essence this is an extension of the Bi-nomial SRA algorithm algorithm proposed by Rabenseifner2004 (https://doi.org/10.1007/978-3-540-24685-5_1). The extension adds the support for arbitrary radix. 3. The algorithm targets Large message sizes (ie. optimized for max bandwidth). 4. If number of ranks in the team can not form a full radix subtree (for radix=2 this means the team size is not power of 2) then there will be "extra" ranks which don't participate in the main exchange loop. They will send the data to their "proxy" ranks in the beginning and then wait for the response with the final data. 5. The knomial reduce-scatter and allgather primitives can be used separately. However, if they are used together as part of SRA allreduce one has to provide the same radix for both routines. 6. If the allreduce is INPLACE or if a rank serves as a PROXY then the algorithm requires allocation of a scratch buffer of the size equal to input buffer. 7. After the completion of reduce-scatter phase the local result (at non EXTRA ranks) will be located in dst buffer at offset the can be commputed by the routine from coll_patterns/sra_knomial.h: ucc_sra_kn_get_offset. */ static ucc_status_t ucc_tl_ucp_allreduce_sra_knomial_frag_start(ucc_coll_task_t *task) { return ucc_schedule_start(task); } static ucc_status_t ucc_tl_ucp_allreduce_sra_knomial_frag_finalize(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_status_t status; status = ucc_schedule_finalize(task); ucc_tl_ucp_put_schedule(schedule); return status; } static ucc_status_t ucc_tl_ucp_allreduce_sra_knomial_frag_setup(ucc_schedule_pipelined_t *schedule_p, ucc_schedule_t *frag, int frag_num) { ucc_coll_args_t *args = &schedule_p->super.super.bargs.args; ucc_datatype_t dt = args->dst.info.datatype; size_t dt_size = ucc_dt_size(dt); int n_frags = schedule_p->super.n_tasks; size_t frag_count = ucc_buffer_block_count(args->dst.info.count, n_frags, frag_num); size_t offset = ucc_buffer_block_offset(args->dst.info.count, n_frags, frag_num); ucc_coll_args_t *targs; targs = &frag->tasks[0]->bargs.args; /* REDUCE_SCATTER */ targs->src.info.buffer = PTR_OFFSET(args->src.info.buffer, offset * dt_size); targs->src.info.count = frag_count; targs->dst.info.buffer = PTR_OFFSET(args->dst.info.buffer, offset * dt_size); targs->dst.info.count = frag_count; targs = &frag->tasks[1]->bargs.args; /* ALLGATHER */ targs->src.info.buffer = NULL; targs->src.info.count = 0; targs->dst.info.buffer = PTR_OFFSET(args->dst.info.buffer, offset * dt_size); targs->dst.info.count = frag_count; return UCC_OK; } static ucc_status_t ucc_tl_ucp_allreduce_sra_knomial_frag_init(ucc_base_coll_args_t *coll_args, ucc_schedule_pipelined_t *sp, //NOLINT ucc_base_team_t *team, ucc_schedule_t **frag_p) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_datatype_t dtype = coll_args->args.dst.info.datatype; ucc_memory_type_t mem_type = coll_args->args.dst.info.mem_type; ucc_base_coll_args_t args = *coll_args; ucc_mrange_uint_t *p = &tl_team->cfg.allreduce_sra_kn_radix; ucc_schedule_t *schedule; ucc_coll_task_t *task, *rs_task; ucc_status_t status; ucc_kn_radix_t radix; size_t count; status = ucc_tl_ucp_get_schedule(tl_team, coll_args, (ucc_tl_ucp_schedule_t **)&schedule); if (ucc_unlikely(UCC_OK != status)) { return status; } if (coll_args->mask & UCC_BASE_CARGS_MAX_FRAG_COUNT) { count = coll_args->max_frag_count; } else { count = coll_args->args.dst.info.count; } radix = ucc_tl_ucp_get_knomial_radix(tl_team, count, dtype, mem_type, p, 1); /* 1st step of allreduce: knomial reduce_scatter */ UCC_CHECK_GOTO( ucc_tl_ucp_reduce_scatter_knomial_init_r(&args, team, &task, radix), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, task), out, status); UCC_CHECK_GOTO(ucc_task_subscribe_dep(&schedule->super, task, UCC_EVENT_SCHEDULE_STARTED), out, status); rs_task = task; /* 2nd step of allreduce: knomial allgather. 2nd task subscribes to completion event of reduce_scatter task. */ args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; UCC_CHECK_GOTO( ucc_tl_ucp_allgather_knomial_init_r(&args, team, &task, radix), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, task), out, status); UCC_CHECK_GOTO(ucc_task_subscribe_dep(rs_task, task, UCC_EVENT_COMPLETED), out, status); schedule->super.finalize = ucc_tl_ucp_allreduce_sra_knomial_frag_finalize; schedule->super.post = ucc_tl_ucp_allreduce_sra_knomial_frag_start; *frag_p = schedule; return UCC_OK; out: return status; } static ucc_status_t ucc_tl_ucp_allreduce_sra_knomial_finalize(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(schedule, "ucp_allreduce_sra_kn_done", 0); status = ucc_schedule_pipelined_finalize(task); ucc_tl_ucp_put_schedule(schedule); return status; } ucc_status_t ucc_tl_ucp_allreduce_sra_knomial_start(ucc_coll_task_t *task) { UCC_TL_UCP_PROFILE_REQUEST_EVENT(task, "ucp_allreduce_sra_kn_start", 0); return ucc_schedule_pipelined_post(task); } static void ucc_tl_ucp_allreduce_sra_knomial_get_pipeline_params(ucc_tl_ucp_team_t *team, ucc_coll_args_t *args, ucc_pipeline_params_t *pp) { ucc_tl_ucp_lib_config_t *cfg = &team->cfg; if (!ucc_pipeline_params_is_auto(&cfg->allreduce_sra_kn_pipeline)) { *pp = cfg->allreduce_sra_kn_pipeline; return; } if ((args->src.info.mem_type == UCC_MEMORY_TYPE_CUDA) && (UCC_IS_INPLACE(*args))) { ucc_mc_attr_t mc_attr; mc_attr.field_mask = UCC_MC_ATTR_FIELD_FAST_ALLOC_SIZE; ucc_mc_get_attr(&mc_attr, UCC_MEMORY_TYPE_CUDA); pp->threshold = mc_attr.fast_alloc_size; pp->n_frags = 2; pp->frag_size = mc_attr.fast_alloc_size; pp->order = UCC_PIPELINE_PARALLEL; pp->pdepth = 2; } else { pp->threshold = SIZE_MAX; pp->n_frags = 0; pp->frag_size = 0; pp->pdepth = 1; pp->order = UCC_PIPELINE_PARALLEL; } } ucc_status_t ucc_tl_ucp_allreduce_sra_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_coll_args_t *args = &coll_args->args; size_t dt_size = ucc_dt_size(args->dst.info.datatype); int n_frags, pipeline_depth; ucc_schedule_pipelined_t *schedule_p; ucc_status_t st; ucc_base_coll_args_t bargs; size_t max_frag_count; ucc_pipeline_params_t pipeline_params; st = ucc_tl_ucp_get_schedule(tl_team, coll_args, (ucc_tl_ucp_schedule_t **)&schedule_p); if (ucc_unlikely(UCC_OK != st)) { return st; } bargs = *coll_args; max_frag_count = (bargs.mask & UCC_BASE_CARGS_MAX_FRAG_COUNT) ? bargs.max_frag_count: args->dst.info.count; ucc_tl_ucp_allreduce_sra_knomial_get_pipeline_params(tl_team, args, &pipeline_params); ucc_pipeline_nfrags_pdepth(&pipeline_params, max_frag_count * dt_size, &n_frags, &pipeline_depth); if (n_frags > 1) { bargs.mask |= UCC_BASE_CARGS_MAX_FRAG_COUNT; bargs.max_frag_count = ucc_buffer_block_count(max_frag_count, n_frags, 0); } st = ucc_schedule_pipelined_init(&bargs, team, ucc_tl_ucp_allreduce_sra_knomial_frag_init, ucc_tl_ucp_allreduce_sra_knomial_frag_setup, pipeline_depth, n_frags, pipeline_params.order, schedule_p); if (ucc_unlikely(UCC_OK != st)) { tl_error(team->context->lib, "failed to init pipelined schedule"); ucc_tl_ucp_put_schedule(&schedule_p->super); return st; } schedule_p->super.super.finalize = ucc_tl_ucp_allreduce_sra_knomial_finalize; schedule_p->super.super.post = ucc_tl_ucp_allreduce_sra_knomial_start; *task_h = &schedule_p->super.super; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/allreduce/allreduce_sliding_window_setup.c0000664000175000017500000002526515211535620027236 0ustar alastairalastair/** * Copyright(c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allreduce.h" #include "allreduce_sliding_window.h" #include "../allgather/allgather.h" #include "utils/ucc_dt_reduce.h" #include "tl_ucp_ep.h" ucc_status_t ucc_tl_ucp_allreduce_sliding_window_alloc_pipe(ucc_base_team_t *team, ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t team_size = UCC_TL_TEAM_SIZE(tl_team); ucc_tl_ucp_lib_config_t *cfg = &UCC_TL_UCP_TEAM_LIB(tl_team)->cfg; const size_t buf_size = cfg->allreduce_sliding_window_buf_size; uint32_t put_window_size = cfg->allreduce_sliding_window_put_window_size; uint32_t num_get_bufs = cfg->allreduce_sliding_window_num_get_bufs; int i, j; ucc_tl_ucp_allreduce_sw_pipeline *pipe; pipe = ucc_malloc(sizeof(ucc_tl_ucp_allreduce_sw_pipeline)); if (pipe == NULL) { goto err; } ucc_assert(team_size > 0); // Bypass clang linter if (put_window_size == 0 || put_window_size > team_size) { put_window_size = team_size; } if (num_get_bufs == 0) { num_get_bufs = team_size; } pipe->accbuf.buf = ucc_malloc(buf_size); if (pipe->accbuf.buf == NULL) { goto free_pipe; } pipe->getbuf = (ucc_tl_ucp_allreduce_sw_buf_t *)ucc_malloc( num_get_bufs * sizeof(ucc_tl_ucp_allreduce_sw_buf_t)); if (pipe->getbuf == NULL) { goto free_acc; } for (i = 0; i < num_get_bufs; i++) { pipe->getbuf[i].buf = ucc_malloc(buf_size); if (pipe->getbuf[i].buf == NULL) { goto free_getbuf; } } pipe->buffer_size = buf_size; pipe->num_buffers = num_get_bufs; pipe->put_requests = (ucs_status_ptr_t *)ucc_malloc( put_window_size * sizeof(ucs_status_ptr_t)); if (pipe->put_requests == NULL) { goto free_getbuf; } task->allreduce_sliding_window.pipe = pipe; task->allreduce_sliding_window.put_requests = task->allreduce_sliding_window.pipe->put_requests; return UCC_OK; free_getbuf: for (j = 0; j < i; j++) { ucc_free(pipe->getbuf[j].buf); } ucc_free(pipe->getbuf); free_acc: ucc_free(pipe->accbuf.buf); free_pipe: ucc_free(pipe); err: tl_error(UCC_TL_TEAM_LIB(tl_team), "error allocating sliding window pipe"); return UCC_ERR_NO_RESOURCE; } ucc_status_t ucc_tl_ucp_allreduce_sliding_window_task_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_tl_ucp_task_t *task) { void *src_buf = coll_args->args.src.info.buffer; void *dst_buf = coll_args->args.dst.info.buffer; ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t team_size = UCC_TL_TEAM_SIZE(tl_team); int inplace = UCC_IS_INPLACE(coll_args->args); ucc_tl_ucp_allreduce_sw_global_work_buf_info_t *gwbi_p = NULL; size_t allgather_size = sizeof(ucc_tl_ucp_allreduce_sw_host_allgather_t); ucc_tl_ucp_allreduce_sw_host_allgather_t *allgather_data; ucc_rank_t i; void *buffer; void *ptr; size_t bufs_sz, allgather_data_sz, rbufs_sz, dst_rkeys_sz, dst_ebuf_sz, sbufs_sz, src_rkeys_sz, src_ebuf_sz; ucc_assert(team_size > 0); bufs_sz = sizeof(ucc_tl_ucp_dpu_offload_buf_info_t); allgather_data_sz = allgather_size * (team_size + 1); rbufs_sz = sizeof(void *) * team_size; dst_rkeys_sz = sizeof(ucp_rkey_h) * team_size; dst_ebuf_sz = sizeof(struct ucc_tl_ucp_allreduce_sw_export_buf); if (!inplace) { sbufs_sz = sizeof(void *) * team_size; src_rkeys_sz = sizeof(ucp_rkey_h) * team_size; src_ebuf_sz = sizeof(struct ucc_tl_ucp_allreduce_sw_export_buf); } else { sbufs_sz = 0; src_rkeys_sz = 0; src_ebuf_sz = 0; } buffer = ucc_malloc(bufs_sz + allgather_data_sz + rbufs_sz + dst_rkeys_sz + dst_ebuf_sz + sbufs_sz + src_rkeys_sz + src_ebuf_sz); if (buffer == NULL) { tl_error(UCC_TL_TEAM_LIB(tl_team), "error while allocating task"); return UCC_ERR_NO_RESOURCE; } ptr = buffer; task->allreduce_sliding_window.bufs = ptr; ptr = allgather_data = PTR_OFFSET(ptr, bufs_sz); task->allreduce_sliding_window.allgather_data = allgather_data; gwbi_p = coll_args->args.global_work_buffer; task->super.bargs.args.global_work_buffer = gwbi_p; task->allreduce_sliding_window.reduce_task = NULL; ptr = task->allreduce_sliding_window.bufs->rbufs = PTR_OFFSET(ptr, allgather_data_sz); ptr = task->allreduce_sliding_window.bufs->dst_rkeys = PTR_OFFSET(ptr, rbufs_sz); for (i = 0; i < team_size; i++) { task->allreduce_sliding_window.bufs->dst_rkeys[i] = NULL; } ptr = task->allreduce_sliding_window.bufs->dst_ebuf = PTR_OFFSET(ptr, dst_rkeys_sz); task->allreduce_sliding_window.bufs->dst_ebuf->memh = NULL; allgather_data->dst_buf = dst_buf; task->allreduce_sliding_window.allgather_data = allgather_data; task->allreduce_sliding_window.allgather_task = NULL; if (!inplace) { allgather_data->src_buf = src_buf; ptr = task->allreduce_sliding_window.bufs->sbufs = PTR_OFFSET(ptr, dst_ebuf_sz); ptr = task->allreduce_sliding_window.bufs->src_rkeys = PTR_OFFSET(ptr, sbufs_sz); for (i = 0; i < team_size; i++) { task->allreduce_sliding_window.bufs->src_rkeys[i] = NULL; } task->allreduce_sliding_window.bufs->src_ebuf = PTR_OFFSET(ptr, src_rkeys_sz); task->allreduce_sliding_window.bufs->src_ebuf->memh = NULL; } else { task->allreduce_sliding_window.bufs->src_ebuf = NULL; } return UCC_OK; } ucc_status_t ucc_tl_ucp_allreduce_sliding_window_allgather_info_finalize( ucc_tl_ucp_task_t *sw_task) { ucs_status_t ucs_status = UCS_OK; ucc_base_team_t *base_team = sw_task->super.team; ucc_tl_ucp_team_t *tl_team = ucc_derived_of(base_team, ucc_tl_ucp_team_t); ucc_rank_t team_size = UCC_TL_TEAM_SIZE(tl_team); void *recvbuf = sw_task->allreduce_sliding_window. allgather_task->bargs.args.dst.info.buffer; ucc_tl_ucp_allreduce_sw_host_allgather_t *all_host_allgather = recvbuf; ucc_status_t status = UCC_OK; int inplace = UCC_IS_INPLACE(TASK_ARGS(sw_task)); ucc_rank_t i; ucp_ep_h ep; ucp_rkey_h src_unpacked, dst_unpacked; ucc_assert(team_size > 0); for (i = 0; i < team_size; i++) { status = ucc_tl_ucp_get_ep(tl_team, i, &ep); if (ucc_unlikely(UCC_OK != status)) { return status; } ucs_status = ucp_ep_rkey_unpack( ep, all_host_allgather[i].packed_dst_key, &dst_unpacked); if (UCS_OK != ucs_status) { tl_error(UCC_TL_TEAM_LIB(tl_team), "dst rkey unpack failed"); return ucs_status_to_ucc_status(ucs_status); } sw_task->allreduce_sliding_window.bufs->rbufs[i] = all_host_allgather[i].dst_buf; sw_task->allreduce_sliding_window.bufs->dst_rkeys[i] = dst_unpacked; if (!inplace) { ucs_status = ucp_ep_rkey_unpack( ep, all_host_allgather[i].packed_src_key, &src_unpacked); if (UCS_OK != ucs_status) { tl_error(UCC_TL_TEAM_LIB(tl_team), "src rkey unpack failed"); return ucs_status_to_ucc_status(ucs_status); } sw_task->allreduce_sliding_window.bufs->sbufs[i] = all_host_allgather[i].src_buf; sw_task->allreduce_sliding_window.bufs->src_rkeys[i] = src_unpacked; } else { sw_task->allreduce_sliding_window.bufs->sbufs = sw_task->allreduce_sliding_window.bufs->rbufs; sw_task->allreduce_sliding_window.bufs->src_rkeys = sw_task->allreduce_sliding_window.bufs->dst_rkeys; } } return status; } void ucc_tl_ucp_allreduce_sliding_window_free_task(ucc_coll_task_t *coll_task) { ucc_base_team_t *team = coll_task->team; ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); int inplace = UCC_IS_INPLACE(coll_task->bargs.args); ucc_tl_ucp_context_t *tl_ctx = UCC_TL_UCP_TEAM_CTX(tl_team); if (task->allreduce_sliding_window.bufs) { if (!inplace) { if (task->allreduce_sliding_window.bufs->src_ebuf->memh != NULL) { ucp_mem_unmap(tl_ctx->worker.ucp_context, task->allreduce_sliding_window.bufs->src_ebuf->memh); task->allreduce_sliding_window.bufs->src_ebuf->memh = NULL; } } if (task->allreduce_sliding_window.bufs->dst_ebuf->memh != NULL) { ucp_mem_unmap(tl_ctx->worker.ucp_context, task->allreduce_sliding_window.bufs->dst_ebuf->memh); } ucc_free(task->allreduce_sliding_window.bufs); } } void ucc_tl_ucp_allreduce_sliding_window_free_pipe(ucc_coll_task_t *coll_task) { ucc_base_team_t *team = coll_task->team; ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t team_size = UCC_TL_TEAM_SIZE(tl_team); ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_allreduce_sw_pipeline *pipe = task->allreduce_sliding_window.pipe; int num_get_bufs = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.allreduce_sliding_window_num_get_bufs; int i; if (num_get_bufs == 0) { num_get_bufs = team_size; } if (pipe) { ucc_free(pipe->accbuf.buf); for (i = 0; i < num_get_bufs; i++) { ucc_free(pipe->getbuf[i].buf); } ucc_free(pipe->getbuf); ucc_free(pipe->put_requests); ucc_free(pipe); } } ucc-1.8.0/src/components/tl/ucp/allreduce/allreduce.c0000664000175000017500000000501115211535620022701 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "allreduce.h" #include "utils/ucc_coll_utils.h" ucc_base_coll_alg_info_t ucc_tl_ucp_allreduce_algs[UCC_TL_UCP_ALLREDUCE_ALG_LAST + 1] = { [UCC_TL_UCP_ALLREDUCE_ALG_KNOMIAL] = {.id = UCC_TL_UCP_ALLREDUCE_ALG_KNOMIAL, .name = "knomial", .desc = "recursive knomial with arbitrary radix (optimized for latency)"}, [UCC_TL_UCP_ALLREDUCE_ALG_SRA_KNOMIAL] = {.id = UCC_TL_UCP_ALLREDUCE_ALG_SRA_KNOMIAL, .name = "sra_knomial", .desc = "recursive knomial scatter-reduce followed by knomial " "allgather (optimized for BW)"}, [UCC_TL_UCP_ALLREDUCE_ALG_DBT] = {.id = UCC_TL_UCP_ALLREDUCE_ALG_DBT, .name = "dbt", .desc = "allreduce over double binary tree where a leaf in one tree " "will be intermediate in other (optimized for BW)"}, [UCC_TL_UCP_ALLREDUCE_ALG_SLIDING_WINDOW] = {.id = UCC_TL_UCP_ALLREDUCE_ALG_SLIDING_WINDOW, .name = "sliding_window", .desc = "sliding window allreduce (optimized for running on DPU)"}, [UCC_TL_UCP_ALLREDUCE_ALG_RING] = {.id = UCC_TL_UCP_ALLREDUCE_ALG_RING, .name = "ring", .desc = "ring-based allreduce using reduce-scatter + allgather"}, [UCC_TL_UCP_ALLREDUCE_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_allreduce_init(ucc_tl_ucp_task_t *task) { ucc_status_t status; ALLREDUCE_TASK_CHECK(TASK_ARGS(task), TASK_TEAM(task)); status = ucc_tl_ucp_allreduce_knomial_init_common(task); out: return status; } ucc_status_t ucc_tl_ucp_allreduce_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task; ucc_status_t status; ALLREDUCE_TASK_CHECK(coll_args->args, tl_team); task = ucc_tl_ucp_init_task(coll_args, team); *task_h = &task->super; status = ucc_tl_ucp_allreduce_knomial_init_common(task); if (ucc_unlikely(status != UCC_OK)) { ucc_tl_ucp_put_task(task); } out: return status; } ucc-1.8.0/src/components/tl/ucp/allreduce/allreduce_sliding_window.c0000664000175000017500000006316315211535620026015 0ustar alastairalastair/** * Copyright(c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "allreduce.h" #include "allreduce_sliding_window.h" #include "../allgather/allgather.h" #include "../barrier/barrier.h" #include "utils/ucc_dt_reduce.h" #include "tl_ucp_ep.h" static inline void ucc_tl_ucp_allreduce_sliding_window_reset_buf(ucc_tl_ucp_allreduce_sw_buf_t *buf) { buf->state = FREE; buf->count = 0; buf->bytes = 0; buf->ucp_req = NULL; } static inline void ucc_tl_ucp_allreduce_sliding_window_reset_pipeline( ucc_tl_ucp_allreduce_sw_pipeline_t *pipe, ucc_rank_t rank, ucc_rank_t put_window_size) { int i; pipe->avail_buffs = pipe->num_buffers; pipe->src_rank = pipe->dst_rank = rank; pipe->get_idx = pipe->red_idx = 0; pipe->done_get = pipe->done_red = 0; pipe->done_put = pipe->posted_put = 0; pipe->count_reduced = pipe->count_serviced = 0; pipe->my_count = pipe->my_offset = 0; pipe->count_received = 0; ucc_tl_ucp_allreduce_sliding_window_reset_buf(&pipe->accbuf); for (i = 0; i < pipe->num_buffers; i++) { ucc_tl_ucp_allreduce_sliding_window_reset_buf(&pipe->getbuf[i]); } memset(pipe->put_requests, 0, put_window_size * sizeof(ucs_status_ptr_t)); } ucc_status_t ucc_tl_ucp_allreduce_sliding_window_start(ucc_coll_task_t *coll_task) { ucc_base_coll_args_t *coll_args = &coll_task->bargs; ucc_schedule_t *schedule = ucc_derived_of(coll_task, ucc_schedule_t); ucc_base_team_t *base_team = schedule->super.team; ucc_tl_ucp_team_t *team = ucc_derived_of(base_team, ucc_tl_ucp_team_t); ucc_tl_ucp_context_t *tl_ctx = UCC_TL_UCP_TEAM_CTX(team); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); uint32_t count_total = coll_task->bargs.args.dst.info.count; ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_datatype_t dtype = coll_args->args.dst.info.datatype; size_t dt_size = ucc_dt_size(dtype); int inplace = UCC_IS_INPLACE(coll_args->args); ucc_status_t status = UCC_OK; ucc_rank_t put_window_size = UCC_TL_UCP_TEAM_LIB(team) ->cfg.allreduce_sliding_window_put_window_size; ucc_tl_ucp_allreduce_sw_global_work_buf_info_t *gwbi_p = coll_args->args.global_work_buffer; ucc_tl_ucp_task_t *rdma_task = ucc_derived_of(schedule->tasks[0], ucc_tl_ucp_task_t); ucc_tl_ucp_allreduce_sw_pipeline_t *pipe; ucc_tl_ucp_allreduce_sw_host_allgather_t *allgather_data; pipe = rdma_task->allreduce_sliding_window.pipe; allgather_data = rdma_task->allreduce_sliding_window.allgather_data; // Register the src buf if (!inplace) { status = ucc_tl_ucp_allreduce_sliding_window_register( tl_ctx->worker.ucp_context, team, rdma_task->allreduce_sliding_window.bufs->src_ebuf, gwbi_p->packed_src_memh); if (status != UCC_OK) { tl_error(UCC_TASK_LIB(rdma_task), "failed to register src memh: %s", ucc_status_string(status)); goto out; } ucc_assert( rdma_task->allreduce_sliding_window.bufs->src_ebuf->packed_key_len <= ALLREDUCE_PACKED_KEY_MAX_LEN); memcpy(allgather_data->packed_src_key, rdma_task->allreduce_sliding_window.bufs->src_ebuf->packed_key, rdma_task->allreduce_sliding_window.bufs->src_ebuf->packed_key_len); } // Register the dst buf status = ucc_tl_ucp_allreduce_sliding_window_register( tl_ctx->worker.ucp_context, team, rdma_task->allreduce_sliding_window.bufs->dst_ebuf, gwbi_p->packed_dst_memh); if (status != UCC_OK) { tl_error(UCC_TASK_LIB(rdma_task), "failed to register dst memh: %s", ucc_status_string(status)); goto out; } ucc_assert( rdma_task->allreduce_sliding_window.bufs->dst_ebuf->packed_key_len <= ALLREDUCE_PACKED_KEY_MAX_LEN); memcpy(allgather_data->packed_dst_key, rdma_task->allreduce_sliding_window.bufs->dst_ebuf->packed_key, rdma_task->allreduce_sliding_window.bufs->dst_ebuf->packed_key_len); if (put_window_size == 0 || put_window_size > size) { put_window_size = size; } ucc_tl_ucp_allreduce_sliding_window_reset_pipeline( pipe, rank, put_window_size); pipe->my_count = ucc_buffer_block_count(count_total, size, rank); pipe->my_offset = ucc_buffer_block_offset(count_total * dt_size, size, rank); rdma_task->allreduce_sliding_window.reduce_task = NULL; UCC_CHECK_GOTO(ucc_tl_ucp_allgather_ring_start( rdma_task->allreduce_sliding_window.allgather_task), out, status); return ucc_schedule_start(coll_task); out: tl_error(UCC_TASK_LIB(rdma_task), "failed to start allreduce sliding window: %s", ucc_status_string(status)); return status; } ucc_status_t ucc_tl_ucp_allreduce_sliding_window_finalize(ucc_coll_task_t *coll_task) { ucc_schedule_t *schedule = ucc_derived_of(coll_task, ucc_schedule_t); ucc_status_t status; status = ucc_schedule_finalize(coll_task); ucc_tl_ucp_put_schedule(schedule); return status; } ucc_status_t ucc_tl_ucp_allreduce_sliding_window_rdma_task_post( ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } static inline void ucc_tl_ucp_allreduce_sliding_window_free_rkeys( ucc_coll_task_t *coll_task) { ucc_base_team_t *team = coll_task->team; ucc_rank_t team_size = (ucc_rank_t)team->params.size; ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); int inplace = UCC_IS_INPLACE(coll_task->bargs.args); ucc_rank_t i; for (i = 0; i < team_size; i++) { if (!inplace && task->allreduce_sliding_window.bufs->src_rkeys[i] != NULL) { ucp_rkey_destroy(task->allreduce_sliding_window.bufs->src_rkeys[i]); } if (task->allreduce_sliding_window.bufs->dst_rkeys[i] != NULL) { ucp_rkey_destroy(task->allreduce_sliding_window.bufs->dst_rkeys[i]); } } } static ucc_status_t ucc_tl_ucp_allreduce_sliding_window_rdma_task_finalize( ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_status_t st = UCC_OK; ucc_tl_ucp_allreduce_sliding_window_free_rkeys(coll_task); ucc_tl_ucp_allreduce_sliding_window_free_task(coll_task); ucc_tl_ucp_allreduce_sliding_window_free_pipe(coll_task); st = ucc_tl_ucp_coll_finalize(coll_task); if (ucc_unlikely(st != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to finalize collective"); } return st; } static inline void ucc_tl_ucp_allreduce_sliding_window_reduction( ucc_coll_task_t *coll_task, ucc_tl_ucp_allreduce_sw_buf_t *accbuf, ucc_tl_ucp_allreduce_sw_buf_t *getbuf) { ucc_status_t status = UCC_OK; ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; ucc_ee_executor_t *exec; status = ucc_coll_task_get_executor(&task->super, &exec); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to get executor"); } status = ucc_dt_reduce(accbuf->buf, getbuf->buf, accbuf->buf, accbuf->count, dt, args, 0, 0, exec, &task->allreduce_sliding_window.reduce_task); if (ucc_unlikely(UCC_OK != status)) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); task->super.status = status; return; } } static inline void ucc_tl_ucp_allreduce_sliding_window_test_reduction(ucc_tl_ucp_task_t *task) { ucc_status_t status; #define SAVE_STATE(_phase) EXEC_TASK_TEST(NULL, "failed to perform dt reduction", task->allreduce_sliding_window.reduce_task); // If it didn't complete, we would have returned by now. So, clear the flag task->allreduce_sliding_window.reduce_task = NULL; } static inline ucc_status_t ucc_tl_ucp_allreduce_sliding_window_req_test(ucs_status_ptr_t request, ucc_tl_ucp_task_t *task) { if (request == NULL) { return UCC_OK; } else if (UCS_PTR_IS_ERR(request)) { tl_error(UCC_TASK_LIB(task), "unable to complete UCX request=%p: %d", request, UCS_PTR_STATUS(request)); return ucs_status_to_ucc_status(UCS_PTR_STATUS(request)); } else { return ucs_status_to_ucc_status(ucp_request_check_status(request)); } } static inline void ucc_tl_ucp_allreduce_sliding_window_key_exchange_progress( ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_task_t *allgather_task = task->allreduce_sliding_window.allgather_task; ucc_status_t status = allgather_task->super.status; if (status < 0) { goto err; } if (UCC_INPROGRESS == status) { ucc_tl_ucp_allgather_ring_progress(allgather_task); return; } ucc_assert(status == UCC_OK); // copy from allgather recvbuf into sliding_window task UCC_CHECK_GOTO( ucc_tl_ucp_allreduce_sliding_window_allgather_info_finalize(task), err, status); out: ucc_tl_ucp_coll_finalize(allgather_task); task->allreduce_sliding_window.allgather_task = NULL; return; err: ucc_tl_ucp_allreduce_sliding_window_free_task(coll_task); ucc_tl_ucp_allreduce_sliding_window_free_pipe(coll_task); tl_error(coll_task->team->context->lib, "key exchange failure: %s", ucc_status_string(status)); goto out; } static inline void ucc_tl_ucp_allreduce_sliding_window_mark_redbuf_free( ucc_tl_ucp_allreduce_sw_pipeline_t *pipe, ucc_tl_ucp_allreduce_sw_buf_t *accbuf, ucc_tl_ucp_allreduce_sw_buf_t *redbuf, ucc_rank_t host_team_size) { redbuf->state = FREE; pipe->avail_buffs++; pipe->red_idx++; pipe->done_red++; if (pipe->done_red == host_team_size - 1) { accbuf->state = REDUCED; pipe->count_reduced += accbuf->count; } } void ucc_tl_ucp_allreduce_sliding_window_rdma_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_rank_t size = (ucc_rank_t)task->subset.map.ep_num; ucc_datatype_t dtype = TASK_ARGS(task).dst.info.datatype; size_t dt_size = ucc_dt_size(dtype); ucc_rank_t host_team_size = size; ucc_base_team_t *base_team = coll_task->team; ucc_tl_ucp_team_t *tl_team = ucc_derived_of(base_team, ucc_tl_ucp_team_t); ucc_tl_ucp_allreduce_sw_pipeline_t *pipe = task->allreduce_sliding_window.pipe; ucc_tl_ucp_context_t *tl_ctx = UCC_TL_UCP_TEAM_CTX(tl_team); ucc_tl_ucp_allreduce_sw_buf_t *accbuf = &pipe->accbuf; ucp_request_param_t req_param = {0}; int i = 0; ucc_coll_task_t *allgather_task = task->allreduce_sliding_window.allgather_task; ucc_ee_executor_task_t **reduce_task = &task->allreduce_sliding_window.reduce_task; ucc_rank_t put_window_size = UCC_TL_UCP_TEAM_LIB(tl_team)-> cfg.allreduce_sliding_window_put_window_size; ucc_tl_ucp_allreduce_sw_buf_t *redbuf; ucc_tl_ucp_allreduce_sw_buf_t *getbuf; size_t remaining_elems; ucc_rank_t get_idx; size_t count; size_t get_offset; size_t data_size; ucc_rank_t src_rank; ucc_rank_t dst_rank; void *src_addr; void *dst_addr; ucs_status_ptr_t request; size_t red_idx; size_t put_offset; int window; int put_idx; ucp_ep_h ep; ucc_status_t status; ucc_assert(host_team_size > 0); if (allgather_task != NULL) { ucc_tl_ucp_allreduce_sliding_window_key_exchange_progress(coll_task); return; } if (*reduce_task != NULL) { // We've previously started a reduction on the accbuf that hasn't yet // completed. ucc_tl_ucp_allreduce_sliding_window_test_reduction(task); if (*reduce_task != NULL) { return; } red_idx = pipe->red_idx % pipe->num_buffers; redbuf = &pipe->getbuf[red_idx]; ucc_tl_ucp_allreduce_sliding_window_mark_redbuf_free( pipe, accbuf, redbuf, host_team_size); } if (pipe->count_serviced < pipe->my_count) { if ((pipe->count_received < pipe->my_count) && (pipe->done_get < host_team_size) && (pipe->avail_buffs > 0) && (accbuf->state != REDUCED && accbuf->state != SENDING)) { remaining_elems = pipe->my_count - pipe->count_received; get_idx = pipe->get_idx % pipe->num_buffers; count = ucc_min(pipe->buffer_size / dt_size, remaining_elems); get_offset = pipe->count_received * dt_size + pipe->my_offset; data_size = count * dt_size; src_rank = pipe->src_rank; getbuf = accbuf->state == FREE ? accbuf : &pipe->getbuf[get_idx]; src_addr = PTR_OFFSET( task->allreduce_sliding_window.bufs->sbufs[src_rank], get_offset); dst_addr = getbuf->buf; ucc_assert(getbuf->state == FREE); getbuf->state = RECVING; getbuf->count = count; getbuf->bytes = data_size; ucc_tl_ucp_get_ep(tl_team, src_rank, &ep); getbuf->ucp_req = ucp_get_nbx( ep, dst_addr, data_size, (uint64_t)src_addr, task->allreduce_sliding_window.bufs->src_rkeys[src_rank], &req_param); pipe->src_rank = (src_rank + 1) % host_team_size; if (getbuf != accbuf) { pipe->avail_buffs--; pipe->get_idx++; } pipe->done_get++; if (pipe->done_get == host_team_size) { pipe->count_received += count; } } if (accbuf->state == RECVING) { request = accbuf->ucp_req; status = ucc_tl_ucp_allreduce_sliding_window_req_test(request, task); if (status == UCC_OK) { if (request) ucp_request_free(request); accbuf->state = REDUCING; accbuf->ucp_req = NULL; } else if (status < 0) { tl_error(UCC_TL_TEAM_LIB(tl_team), "accbuf request failed: %s", ucc_status_string(status)); } } red_idx = pipe->red_idx % pipe->num_buffers; redbuf = &pipe->getbuf[red_idx]; if (accbuf->state == REDUCING && redbuf->state == RECVING) { request = redbuf->ucp_req; status = ucc_tl_ucp_allreduce_sliding_window_req_test(request, task); if (status == UCC_OK) { if (request) ucp_request_free(request); redbuf->state = REDUCING; redbuf->ucp_req = NULL; ucc_tl_ucp_allreduce_sliding_window_reduction(coll_task, accbuf, redbuf); ucc_tl_ucp_allreduce_sliding_window_test_reduction(task); if (*reduce_task != NULL) { return; } ucc_tl_ucp_allreduce_sliding_window_mark_redbuf_free( pipe, accbuf, redbuf, host_team_size); } else if (status < 0) { tl_error(UCC_TL_TEAM_LIB(tl_team), "redbuf request failed: %s", ucc_status_string(status)); } } if ((pipe->count_serviced < pipe->count_reduced) && (accbuf->state == REDUCED)) { data_size = accbuf->bytes; put_offset = pipe->count_serviced * dt_size + pipe->my_offset; if (put_window_size == 0 || put_window_size > host_team_size) { put_window_size = host_team_size; } window = ucc_min(put_window_size, host_team_size - pipe->posted_put); for (i = 0; i < window; i++) { dst_rank = pipe->dst_rank; src_addr = accbuf->buf; dst_addr = PTR_OFFSET( task->allreduce_sliding_window.bufs->rbufs[dst_rank], put_offset); put_idx = pipe->posted_put % put_window_size; if (task->allreduce_sliding_window.put_requests[put_idx] != NULL) { // We've already posted a put at this index that didn't yet // complete, left this function and came back. Skip to check // whether this request finished instead of overwriting it // with another put break; } ucp_worker_fence(tl_ctx->worker.ucp_worker); ucc_tl_ucp_get_ep(tl_team, dst_rank, &ep); task->allreduce_sliding_window.put_requests[put_idx] = ucp_put_nbx( ep, src_addr, data_size, (uint64_t)dst_addr, task->allreduce_sliding_window.bufs->dst_rkeys[dst_rank], &req_param); pipe->posted_put++; pipe->dst_rank = (dst_rank + 1) % host_team_size; } for (i = pipe->done_put; i < pipe->posted_put; i++) { put_idx = i % put_window_size; request = task->allreduce_sliding_window.put_requests[put_idx]; // These are fenced, so if the first fails, the proceding will // too if (ucc_tl_ucp_allreduce_sliding_window_req_test( request, task) != UCC_OK) break; if (request != NULL) ucp_request_free(request); task->allreduce_sliding_window.put_requests[put_idx] = NULL; pipe->done_put++; } if (pipe->done_put == host_team_size) { ucc_assert(pipe->avail_buffs == pipe->num_buffers); ucc_assert(pipe->done_get == host_team_size); ucc_assert(pipe->done_red == host_team_size - 1); pipe->count_serviced += accbuf->count; ucc_tl_ucp_allreduce_sliding_window_reset_buf(accbuf); pipe->done_get = 0; pipe->done_red = pipe->done_put = pipe->posted_put = 0; } } ucp_worker_progress(tl_ctx->worker.ucp_worker); } if (pipe->count_serviced == pipe->my_count) { task->super.status = UCC_OK; } } ucc_status_t ucc_tl_ucp_allreduce_sliding_window_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_schedule_t *schedule = NULL; ucc_status_t status = UCC_OK; ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); size_t allgather_size = sizeof(ucc_tl_ucp_allreduce_sw_host_allgather_t); ucc_rank_t size = UCC_TL_TEAM_SIZE(tl_team); ucc_base_coll_args_t bargs = { .mask = 0, .args = { .coll_type = UCC_COLL_TYPE_ALLGATHER, .mask = 0, .src.info = {.buffer = NULL, .count = allgather_size, .datatype = UCC_DT_UINT8, .mem_type = UCC_MEMORY_TYPE_HOST}, .dst.info = {.buffer = NULL, .count = allgather_size * size, .datatype = UCC_DT_UINT8, .mem_type = UCC_MEMORY_TYPE_HOST} } }; ucc_base_coll_args_t barrier_coll_args = { .team = team->params.team, .args.coll_type = UCC_COLL_TYPE_BARRIER, }; ucc_tl_ucp_allreduce_sw_host_allgather_t *allgather_data; ucc_tl_ucp_task_t *rdma_task; ucc_coll_task_t *barrier_task; if (!(coll_args->args.mask & UCC_COLL_ARGS_FIELD_GLOBAL_WORK_BUFFER) || !(coll_args->args.mask & UCC_COLL_ARGS_FIELD_FLAGS) || !(coll_args->args.flags & UCC_COLL_ARGS_FLAG_MEM_MAPPED_BUFFERS)) { tl_debug(UCC_TL_TEAM_LIB(tl_team), "sliding window allreduce requires " "exported memh to be passed via " "global work buffer"); return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_ucp_get_schedule(tl_team, coll_args, (ucc_tl_ucp_schedule_t **)&schedule); if (ucc_unlikely(UCC_OK != status)) { return status; } *task_h = &schedule->super; schedule->super.post = ucc_tl_ucp_allreduce_sliding_window_start; schedule->super.progress = NULL; schedule->super.finalize = ucc_tl_ucp_allreduce_sliding_window_finalize; schedule->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; rdma_task = ucc_tl_ucp_init_task(coll_args, team); if (ucc_unlikely(!rdma_task)) { tl_error(UCC_TL_TEAM_LIB(tl_team), "Couldnt allocate task"); return UCC_ERR_NO_MEMORY; } if (ucc_tl_ucp_allreduce_sliding_window_alloc_pipe(team, rdma_task) != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(tl_team), "failed to alloc pipe: %s", ucc_status_string(status)); goto free_rdma_task; } status = ucc_tl_ucp_allreduce_sliding_window_task_init(coll_args, team, rdma_task); if (status != UCC_OK) { tl_error(UCC_TL_TEAM_LIB(tl_team), "failed to init task: %s", ucc_status_string(status)); goto out; } allgather_data = rdma_task->allreduce_sliding_window.allgather_data; bargs.args.src.info.buffer = allgather_data; bargs.args.dst.info.buffer = PTR_OFFSET(allgather_data, allgather_size); rdma_task->super.post = ucc_tl_ucp_allreduce_sliding_window_rdma_task_post; rdma_task->super.progress = ucc_tl_ucp_allreduce_sliding_window_rdma_progress; rdma_task->super.finalize = ucc_tl_ucp_allreduce_sliding_window_rdma_task_finalize; UCC_CHECK_GOTO(ucc_tl_ucp_allgather_ring_init(&bargs, team, &rdma_task->allreduce_sliding_window.allgather_task), free_rdma_pipe, status); status = ucc_tl_ucp_coll_init(&barrier_coll_args, team, &barrier_task); if (status < 0) { tl_error(UCC_TL_TEAM_LIB(tl_team), "failure during sliding window barrier init: %s", ucc_status_string(status)); goto free_allgather_task; } UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, &rdma_task->super), out, status); UCC_CHECK_GOTO(ucc_event_manager_subscribe(&schedule->super, UCC_EVENT_SCHEDULE_STARTED, &rdma_task->super, ucc_task_start_handler), free_barrier_task, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, barrier_task), out, status); UCC_CHECK_GOTO(ucc_event_manager_subscribe(&rdma_task->super, UCC_EVENT_COMPLETED, barrier_task, ucc_task_start_handler), free_barrier_task, status); return status; free_barrier_task: ucc_tl_ucp_coll_finalize(barrier_task); free_allgather_task: ucc_tl_ucp_coll_finalize(rdma_task->allreduce_sliding_window.allgather_task); free_rdma_pipe: ucc_tl_ucp_allreduce_sliding_window_free_pipe(&rdma_task->super); free_rdma_task: ucc_tl_ucp_allreduce_sliding_window_free_task(&rdma_task->super); out: ucc_tl_ucp_put_schedule(schedule); return status; } ucc-1.8.0/src/components/tl/ucp/allreduce/allreduce_ring.c0000664000175000017500000001616415211535620023733 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "allreduce.h" #include "tl_ucp.h" #include "tl_ucp_coll.h" #include "core/ucc_progress_queue.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "../reduce_scatter/reduce_scatter.h" #include "../allgather/allgather.h" /* * Ring Allreduce - Implements allreduce as a composition of: * 1. Reduce-scatter ring: Each rank reduces and scatters data * 2. Allgather ring: Collects the scattered results to all ranks * * This approach uses UCC schedules to chain the two operations, * providing a clean, composable implementation of ring allreduce. */ static ucc_status_t ucc_tl_ucp_allreduce_ring_start(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_coll_args_t *args = &schedule->super.bargs.args; size_t count; size_t data_size; ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(task, "ucp_allreduce_ring_start", 0); /* Handle in-place vs out-of-place cases */ if (UCC_IS_INPLACE(*args)) { /* Case 1: Allreduce is in-place (src == dst) * Data is already in dst buffer, no copy needed. * Reduce-scatter and allgather will both operate in-place on dst. */ } else { /* Case 2: Allreduce is NOT in-place (src != dst) * Copy src to dst before starting reduce-scatter. * * Both sub-collectives run in-place on dst so each rank's reduced * segment lands at rank * (count/tsize). The source data must * already reside in dst before the pipeline starts. * * Performance trade-off: this adds one full-buffer memory pass per * out-of-place call. On large counts or high-latency memory types * (e.g. CUDA managed memory) the cost is non-trivial, but it avoids * threading src through both sub-collectives' ring logic while still * producing a correct result. */ count = args->dst.info.count; data_size = count * ucc_dt_size(args->dst.info.datatype); status = ucc_mc_memcpy(args->dst.info.buffer, args->src.info.buffer, data_size, args->dst.info.mem_type, args->src.info.mem_type); if (ucc_unlikely(status != UCC_OK)) { return status; } } return ucc_schedule_start(task); } static ucc_status_t ucc_tl_ucp_allreduce_ring_finalize(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(task, "ucp_allreduce_ring_done", 0); status = ucc_schedule_finalize(task); ucc_tl_ucp_put_schedule(schedule); return status; } ucc_status_t ucc_tl_ucp_allreduce_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(tl_team); size_t count = coll_args->args.dst.info.count; ucc_schedule_t *schedule; ucc_coll_task_t *rs_task; ucc_coll_task_t *ag_task; ucc_base_coll_args_t rs_args; ucc_base_coll_args_t ag_args; ucc_status_t status; /* Check for predefined datatypes */ if (!ucc_coll_args_is_predefined_dt(&coll_args->args, UCC_RANK_INVALID)) { tl_error(team->context->lib, "user defined datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.reduce_avg_pre_op && coll_args->args.op == UCC_OP_AVG) { return UCC_ERR_NOT_SUPPORTED; } /* Check that count is divisible by team size for ring algorithm */ if (count % tsize != 0) { tl_debug(team->context->lib, "ring requires count (%zu) divisible by team size (%u)", count, tsize); return UCC_ERR_NOT_SUPPORTED; } /* Check for same memory types (required by allreduce) */ ALLREDUCE_TASK_CHECK(coll_args->args, tl_team); /* Allocate schedule */ status = ucc_tl_ucp_get_schedule(tl_team, coll_args, (ucc_tl_ucp_schedule_t **)&schedule); if (ucc_unlikely(UCC_OK != status)) { goto out; } /* * Step 1: Reduce-Scatter Ring */ rs_args = *coll_args; rs_args.args.coll_type = UCC_COLL_TYPE_REDUCE_SCATTER; rs_args.args.dst.info.count = count; rs_args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; rs_args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; status = ucc_tl_ucp_reduce_scatter_ring_init(&rs_args, team, &rs_task); if (ucc_unlikely(UCC_OK != status)) { tl_error(team->context->lib, "failed to init reduce_scatter_ring task"); goto err_schedule; } /* * Step 2: Allgather Ring */ ag_args = *coll_args; ag_args.args.coll_type = UCC_COLL_TYPE_ALLGATHER; ag_args.args.src.info.buffer = NULL; ag_args.args.src.info.count = 0; ag_args.args.dst.info.buffer = coll_args->args.dst.info.buffer; ag_args.args.dst.info.count = count; ag_args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; ag_args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; status = ucc_tl_ucp_allgather_ring_init(&ag_args, team, &ag_task); if (ucc_unlikely(UCC_OK != status)) { tl_error(team->context->lib, "failed to init allgather_ring task"); goto err_rs; } /* ucc_schedule_add_task always links the task into schedule->tasks[] * before returning, even on error. Jump targets must therefore reflect * the state *after* the call, not only on success. */ UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, rs_task), err_ag_rs_owned, status); UCC_CHECK_GOTO( ucc_event_manager_subscribe( &schedule->super, UCC_EVENT_SCHEDULE_STARTED, rs_task, ucc_task_start_handler), err_ag_rs_owned, status); /* Same rule for ag_task: once add_task is called it is in the schedule. */ UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, ag_task), err_schedule, status); UCC_CHECK_GOTO( ucc_event_manager_subscribe( rs_task, UCC_EVENT_COMPLETED, ag_task, ucc_task_start_handler), err_schedule, status); schedule->super.post = ucc_tl_ucp_allreduce_ring_start; schedule->super.finalize = ucc_tl_ucp_allreduce_ring_finalize; *task_h = &schedule->super; return UCC_OK; err_ag_rs_owned: /* rs_task is in the schedule; ucc_schedule_finalize will clean it up */ ag_task->finalize(ag_task); goto err_schedule; err_rs: rs_task->finalize(rs_task); err_schedule: ucc_schedule_finalize(&schedule->super); ucc_tl_ucp_put_schedule(schedule); out: return status; } ucc-1.8.0/src/components/tl/ucp/allreduce/allreduce_dbt.c0000664000175000017500000000663315211535620023545 0ustar alastairalastair/** * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "allreduce.h" #include "../reduce/reduce.h" #include "../bcast/bcast.h" ucc_status_t ucc_tl_ucp_allreduce_dbt_start(ucc_coll_task_t *coll_task) { ucc_schedule_t *schedule = ucc_derived_of(coll_task, ucc_schedule_t); ucc_coll_args_t *args = &schedule->super.bargs.args; ucc_coll_task_t *reduce_task, *bcast_task; reduce_task = schedule->tasks[0]; reduce_task->bargs.args.src.info.buffer = args->src.info.buffer; reduce_task->bargs.args.dst.info.buffer = args->dst.info.buffer; reduce_task->bargs.args.src.info.count = args->src.info.count; reduce_task->bargs.args.dst.info.count = args->dst.info.count; bcast_task = schedule->tasks[1]; bcast_task->bargs.args.src.info.buffer = args->dst.info.buffer; bcast_task->bargs.args.src.info.count = args->dst.info.count; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allreduce_dbt_start", 0); return ucc_schedule_start(coll_task); } ucc_status_t ucc_tl_ucp_allreduce_dbt_finalize(ucc_coll_task_t *coll_task) { ucc_schedule_t *schedule = ucc_derived_of(coll_task, ucc_schedule_t); ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(schedule, "ucp_allreduce_dbt_done", 0); status = ucc_schedule_finalize(coll_task); ucc_tl_ucp_put_schedule(schedule); return status; } ucc_status_t ucc_tl_ucp_allreduce_dbt_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_base_coll_args_t args = *coll_args; ucc_schedule_t *schedule; ucc_coll_task_t *reduce_task, *bcast_task; ucc_status_t status; if (UCC_IS_INPLACE(args.args)) { return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_ucp_get_schedule(tl_team, coll_args, (ucc_tl_ucp_schedule_t **)&schedule); if (ucc_unlikely(UCC_OK != status)) { return status; } args.args.root = 0; UCC_CHECK_GOTO(ucc_tl_ucp_reduce_dbt_init(&args, team, &reduce_task), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, reduce_task), out, status); UCC_CHECK_GOTO(ucc_event_manager_subscribe(&schedule->super, UCC_EVENT_SCHEDULE_STARTED, reduce_task, ucc_task_start_handler), out, status); UCC_CHECK_GOTO(ucc_tl_ucp_bcast_dbt_init(&args, team, &bcast_task), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, bcast_task), out, status); UCC_CHECK_GOTO(ucc_event_manager_subscribe(reduce_task, UCC_EVENT_COMPLETED, bcast_task, ucc_task_start_handler), out, status); schedule->super.post = ucc_tl_ucp_allreduce_dbt_start; schedule->super.progress = NULL; schedule->super.finalize = ucc_tl_ucp_allreduce_dbt_finalize; *task_h = &schedule->super; return UCC_OK; out: ucc_tl_ucp_put_schedule(schedule); return status; } ucc-1.8.0/src/components/tl/ucp/fanout/0000775000175000017500000000000015211535620020134 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/fanout/fanout.h0000664000175000017500000000071015211535620021577 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef FANOUT_H_ #define FANOUT_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" enum { UCC_TL_UCP_FANOUT_ALG_KNOMIAL, UCC_TL_UCP_FANOUT_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_fanout_algs[UCC_TL_UCP_FANOUT_ALG_LAST + 1]; ucc_status_t ucc_tl_ucp_fanout_init(ucc_tl_ucp_task_t *task); #endif ucc-1.8.0/src/components/tl/ucp/fanout/fanout.c0000664000175000017500000000232715211535620021600 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "fanout.h" #include "../bcast/bcast.h" ucc_base_coll_alg_info_t ucc_tl_ucp_fanout_algs[UCC_TL_UCP_FANOUT_ALG_LAST + 1] = { [UCC_TL_UCP_FANOUT_ALG_KNOMIAL] = {.id = UCC_TL_UCP_FANOUT_ALG_KNOMIAL, .name = "knomial", .desc = "fanout over knomial tree with arbitrary radix"}, [UCC_TL_UCP_FANOUT_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_fanout_init(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t team_size = UCC_TL_TEAM_SIZE(team); ucc_status_t status = UCC_OK; TASK_ARGS(task).src.info.buffer = NULL; TASK_ARGS(task).src.info.count = 0; TASK_ARGS(task).src.info.mem_type = UCC_MEMORY_TYPE_UNKNOWN; TASK_ARGS(task).src.info.datatype = UCC_DT_INT8; task->bcast_kn.radix = ucc_min(UCC_TL_UCP_TEAM_LIB(team)->cfg.fanout_kn_radix, team_size); task->super.post = ucc_tl_ucp_bcast_knomial_start; task->super.progress = ucc_tl_ucp_bcast_knomial_progress; return status; } ucc-1.8.0/src/components/tl/ucp/tl_ucp_ep_hash.h0000664000175000017500000000411215211535620021764 0ustar alastairalastair/** * Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_UCP_EP_HASH_H_ #define UCC_TL_UCP_EP_HASH_H_ #include "config.h" #include "core/ucc_context.h" #include "utils/khash.h" #include static inline uint32_t tl_ucp_ctx_id_hash_fn_impl(uint32_t h, uint32_t k) { uint32_t H = h & 0xf8000000; h = h << 5; h = h ^ ( H >> 27 ); h = h ^ k; return h; } /* Collisions are handled in khash implementation */ static inline khint32_t tl_ucp_ctx_id_hash_fn(ucc_context_id_t k) { uint32_t h = 0; ucc_assert(sizeof(k.pi.host_hash) == 8); h = tl_ucp_ctx_id_hash_fn_impl(h, kh_int64_hash_func(k.pi.host_hash)); h = tl_ucp_ctx_id_hash_fn_impl(h, k.pi.pid); h = tl_ucp_ctx_id_hash_fn_impl(h, k.seq_num); return (khint32_t)h; } #define tl_ucp_ctx_id_equal_fn(_a, _b) \ (((_a).pi.host_hash == (_b).pi.host_hash) && \ ((_a).pi.pid == (_b).pi.pid) && ((_a).seq_num == (_b).seq_num)) KHASH_INIT(tl_ucp_ep_hash, ucc_context_id_t, void*, 1, \ tl_ucp_ctx_id_hash_fn, tl_ucp_ctx_id_equal_fn); #define tl_ucp_ep_hash_t khash_t(tl_ucp_ep_hash) static inline void* tl_ucp_hash_get(tl_ucp_ep_hash_t *h, ucc_context_id_t key) { khiter_t k; void *value; k = kh_get(tl_ucp_ep_hash, h , key); if (k == kh_end(h)) { return NULL; } value = kh_value(h, k); return value; } static inline void tl_ucp_hash_put(tl_ucp_ep_hash_t *h, ucc_context_id_t key, void *value) { int ret; khiter_t k; k = kh_put(tl_ucp_ep_hash, h, key, &ret); kh_value(h, k) = value; } static inline void* tl_ucp_hash_pop(tl_ucp_ep_hash_t *h) { void *ep = NULL; khiter_t k; k = kh_begin(h); while (k != kh_end(h)) { if (kh_exist(h, k)) { ep = kh_value(h, k); break; } k++; } if (ep) { kh_del(tl_ucp_ep_hash, h, k); } return ep; } #endif ucc-1.8.0/src/components/tl/ucp/Makefile.am0000664000175000017500000000770615211535620020706 0ustar alastairalastair# # Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # if TL_UCP_ENABLED if BUILD_TL_COLL_PLUGINS SUBDIRS = . include makefile.coll_plugins.am endif allgather = \ allgather/allgather.h \ allgather/allgather.c \ allgather/allgather_ring.c \ allgather/allgather_neighbor.c \ allgather/allgather_bruck.c \ allgather/allgather_sparbit.c \ allgather/allgather_linear.c \ allgather/allgather_knomial.c allgatherv = \ allgatherv/allgatherv.h \ allgatherv/allgatherv.c \ allgatherv/allgatherv_ring.c \ allgatherv/allgatherv_knomial.c alltoall = \ alltoall/alltoall.h \ alltoall/alltoall.c \ alltoall/alltoall_onesided.c \ alltoall/alltoall_pairwise.c \ alltoall/alltoall_bruck.c alltoallv = \ alltoallv/alltoallv.h \ alltoallv/alltoallv.c \ alltoallv/alltoallv_pairwise.c \ alltoallv/alltoallv_hybrid.c \ alltoallv/alltoallv_onesided.c allreduce = \ allreduce/allreduce.h \ allreduce/allreduce.c \ allreduce/allreduce_knomial.c \ allreduce/allreduce_sra_knomial.c \ allreduce/allreduce_sliding_window.h \ allreduce/allreduce_sliding_window.c \ allreduce/allreduce_sliding_window_setup.c \ allreduce/allreduce_dbt.c \ allreduce/allreduce_ring.c barrier = \ barrier/barrier.h \ barrier/barrier.c \ barrier/barrier_knomial.c bcast = \ bcast/bcast.h \ bcast/bcast.c \ bcast/bcast_knomial.c \ bcast/bcast_sag_knomial.c \ bcast/bcast_dbt.c fanin = \ fanin/fanin.h \ fanin/fanin.c fanout = \ fanout/fanout.h \ fanout/fanout.c gather = \ gather/gather.h \ gather/gather.c \ gather/gather_knomial.c gatherv = \ gatherv/gatherv.h \ gatherv/gatherv.c \ gatherv/gatherv_linear.c reduce = \ reduce/reduce.h \ reduce/reduce.c \ reduce/reduce_knomial.c \ reduce/reduce_dbt.c \ reduce/reduce_srg_knomial.c reduce_scatter = \ reduce_scatter/reduce_scatter.h \ reduce_scatter/reduce_scatter_knomial.c \ reduce_scatter/reduce_scatter_ring.c \ reduce_scatter/reduce_scatter.c reduce_scatterv = \ reduce_scatterv/reduce_scatterv.h \ reduce_scatterv/reduce_scatterv_ring.c \ reduce_scatterv/reduce_scatterv.c scatter = \ scatter/scatter.h \ scatter/scatter_knomial.c scatterv = \ scatterv/scatterv.h \ scatterv/scatterv.c \ scatterv/scatterv_linear.c sources = \ tl_ucp.h \ tl_ucp.c \ tl_ucp_lib.c \ tl_ucp_context.c \ tl_ucp_team.c \ tl_ucp_ep.h \ tl_ucp_ep.c \ tl_ucp_coll.c \ tl_ucp_service_coll.c \ tl_ucp_dpu_offload.h \ tl_ucp_dpu_offload.c \ tl_ucp_copy.c \ tl_ucp_sendrecv.c \ $(allgather) \ $(allgatherv) \ $(alltoall) \ $(alltoallv) \ $(allreduce) \ $(barrier) \ $(bcast) \ $(fanin) \ $(fanout) \ $(gather) \ $(gatherv) \ $(reduce) \ $(reduce_scatter) \ $(reduce_scatterv) \ $(scatter) \ $(scatterv) module_LTLIBRARIES = libucc_tl_ucp.la libucc_tl_ucp_la_SOURCES = $(sources) libucc_tl_ucp_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(UCX_CPPFLAGS) libucc_tl_ucp_la_CFLAGS = $(BASE_CFLAGS) libucc_tl_ucp_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(UCX_LDFLAGS) libucc_tl_ucp_la_LIBADD = $(UCX_LIBADD) $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/tl/ucp/tl_ucp_copy.h0000664000175000017500000000720615211535620021336 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "tl_ucp_coll.h" #define COPY_TASK_TEST(_phase, _ctx, _errmsg, _ctask) do { \ if (_ctask != NULL) { \ status = _ctx->copy.test(ctx, _ctask); \ if (status > 0) { \ task->super.status = UCC_INPROGRESS; \ SAVE_STATE(_phase); \ return; \ } \ _ctx->copy.finalize(_ctask); \ _ctask = NULL; \ if (ucc_unlikely(status < 0)) { \ tl_error(UCC_TASK_LIB(task), _errmsg); \ task->super.status = status; \ return; \ } \ } \ } while(0) typedef union ucc_tl_ucp_copy_task { ucc_ee_executor_task_t ee_task; ucs_status_ptr_t ucp_task; } ucc_tl_ucp_copy_task_t; /* copies based on MC */ ucc_status_t ucc_tl_ucp_mc_copy_post(void *dst, ucc_memory_type_t dst_mtype, ucp_mem_h dst_memh, void *src, ucc_memory_type_t src_mtype, ucp_mem_h src_memh, size_t size, ucc_tl_ucp_task_t *coll_task, ucc_tl_ucp_copy_task_t **copy_task); ucc_status_t ucc_tl_ucp_mc_copy_test(ucc_tl_ucp_context_t *ctx, ucc_tl_ucp_copy_task_t *copy_task); ucc_status_t ucc_tl_ucp_mc_copy_finalize(ucc_tl_ucp_copy_task_t *copy_task); /* copies based on EC */ ucc_status_t ucc_tl_ucp_ec_copy_post(void *dst, ucc_memory_type_t dst_mtype, ucp_mem_h dst_memh, void *src, ucc_memory_type_t src_mtype, ucp_mem_h src_memh, size_t size, ucc_tl_ucp_task_t *coll_task, ucc_tl_ucp_copy_task_t **copy_task); ucc_status_t ucc_tl_ucp_ec_copy_test(ucc_tl_ucp_context_t *ctx, ucc_tl_ucp_copy_task_t *copy_task); ucc_status_t ucc_tl_ucp_ec_copy_finalize(ucc_tl_ucp_copy_task_t *copy_task); /* copies based on UCX */ ucc_status_t ucc_tl_ucp_ucp_copy_post(void *dst, ucc_memory_type_t dst_mtype, ucp_mem_h dst_memh, void *src, ucc_memory_type_t src_mtype, ucp_mem_h src_memh, size_t size, ucc_tl_ucp_task_t *coll_task, ucc_tl_ucp_copy_task_t **copy_task); ucc_status_t ucc_tl_ucp_ucp_copy_test(ucc_tl_ucp_context_t *ctx, ucc_tl_ucp_copy_task_t *copy_task); ucc_status_t ucc_tl_ucp_ucp_copy_finalize(ucc_tl_ucp_copy_task_t *copy_task); ucc-1.8.0/src/components/tl/ucp/bcast/0000775000175000017500000000000015211535620017734 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/bcast/bcast_dbt.c0000664000175000017500000002536315211535620022036 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "bcast.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" enum { RECV, SEND_T1, SEND_T2, TEST, }; #define UCC_BCAST_DBT_CHECK_STATE(_p) \ case _p: \ goto _p; #define UCC_BCAST_DBT_GOTO_STATE(_state) \ do { \ switch (_state) { \ UCC_BCAST_DBT_CHECK_STATE(SEND_T1); \ UCC_BCAST_DBT_CHECK_STATE(SEND_T2); \ UCC_BCAST_DBT_CHECK_STATE(TEST); \ }; \ } while (0) static void recv_completion_common(void *request, ucs_status_t status, const ucp_tag_recv_info_t *info, /* NOLINT */ void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in recv completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } ucc_atomic_add32(&task->tagged.recv_completed, 1); if (request) { ucp_request_free(request); } } static void recv_completion_1(void *request, ucs_status_t status, const ucp_tag_recv_info_t *info, /* NOLINT */ void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; task->bcast_dbt.t1.recv++; recv_completion_common(request, status, info, user_data); } static void recv_completion_2(void *request, ucs_status_t status, const ucp_tag_recv_info_t *info, /* NOLINT */ void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; task->bcast_dbt.t2.recv++; recv_completion_common(request, status, info, user_data); } void ucc_tl_ucp_bcast_dbt_progress(ucc_coll_task_t *coll_task) { uint32_t i; ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t rank = task->subset.myrank; ucc_dbt_single_tree_t t1 = task->bcast_dbt.t1; ucc_dbt_single_tree_t t2 = task->bcast_dbt.t2; void *buffer = args->src.info.buffer; ucc_memory_type_t mtype = args->src.info.mem_type; ucc_datatype_t dt = args->src.info.datatype; size_t count = args->src.info.count; size_t count_t1 = (count % 2) ? count / 2 + 1 : count / 2; size_t data_size_t1 = count_t1 * ucc_dt_size(dt); size_t data_size_t2 = count / 2 * ucc_dt_size(dt); ucc_rank_t coll_root = (ucc_rank_t)args->root; ucp_tag_recv_nbx_callback_t cb[2] = {recv_completion_1, recv_completion_2}; if (UCC_COLL_ARGS_ACTIVE_SET(&(TASK_ARGS(task)))) { coll_root = ucc_ep_map_local_rank(task->subset.map, coll_root); } UCC_BCAST_DBT_GOTO_STATE(task->bcast_dbt.state); if (rank != t1.root && rank != coll_root) { UCPCHECK_GOTO(ucc_tl_ucp_recv_cb(buffer, data_size_t1, mtype, ucc_ep_map_eval(task->subset.map, t1.parent), team, task, cb[0], (void *)task), task, out); } if (rank != t2.root && rank != coll_root) { UCPCHECK_GOTO(ucc_tl_ucp_recv_cb(PTR_OFFSET(buffer, data_size_t1), data_size_t2, mtype, ucc_ep_map_eval(task->subset.map, t2.parent), team, task, cb[1], (void *)task), task, out); } task->bcast_dbt.state = SEND_T1; SEND_T1: /* test_recv is needed to progress ucp_worker */ ucc_tl_ucp_test_recv(task); if ((coll_root == rank) || (task->bcast_dbt.t1.recv > 0)) { for (i = 0; i < 2; i++) { if ((t1.children[i] != UCC_RANK_INVALID) && (t1.children[i] != coll_root)) { UCPCHECK_GOTO(ucc_tl_ucp_send_nb(buffer, data_size_t1, mtype, ucc_ep_map_eval( task->subset.map, t1.children[i]), team, task), task, out); } } } else { goto out; } task->bcast_dbt.state = SEND_T2; SEND_T2: /* test_recv is needed to progress ucp_worker */ ucc_tl_ucp_test_recv(task); if ((coll_root == rank) || (task->bcast_dbt.t2.recv > 0)) { for (i = 0; i < 2; i++) { if ((t2.children[i] != UCC_RANK_INVALID) && (t2.children[i] != coll_root)) { UCPCHECK_GOTO(ucc_tl_ucp_send_nb(PTR_OFFSET(buffer, data_size_t1), data_size_t2, mtype, ucc_ep_map_eval( task->subset.map, t2.children[i]), team, task), task, out); } } } else { goto out; } TEST: if (UCC_INPROGRESS == ucc_tl_ucp_test_send(task)) { task->bcast_dbt.state = TEST; return; } task->super.status = UCC_OK; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_bcast_dbt_done", 0); out: return; } ucc_status_t ucc_tl_ucp_bcast_dbt_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_status_t status = UCC_OK; ucc_rank_t rank = task->subset.myrank; void *buffer = args->src.info.buffer; ucc_memory_type_t mtype = args->src.info.mem_type; ucc_datatype_t dt = args->src.info.datatype; size_t count = args->src.info.count; size_t count_t1 = (count % 2) ? count / 2 + 1 : count / 2; size_t data_size_t1 = count_t1 * ucc_dt_size(dt); size_t data_size_t2 = count / 2 * ucc_dt_size(dt); ucc_rank_t coll_root = ucc_ep_map_local_rank( task->subset.map, (ucc_rank_t)args->root); ucc_rank_t t1_root = task->bcast_dbt.t1.root; ucc_rank_t t2_root = task->bcast_dbt.t2.root; ucp_tag_recv_nbx_callback_t cb[2] = {recv_completion_1, recv_completion_2}; task->bcast_dbt.t1.recv = 0; task->bcast_dbt.t2.recv = 0; ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if (rank == coll_root && coll_root != t1_root) { status = ucc_tl_ucp_send_nb(buffer, data_size_t1, mtype, ucc_ep_map_eval(task->subset.map, t1_root), team, task); if (UCC_OK != status) { return status; } } if (rank == coll_root && coll_root != t2_root) { status = ucc_tl_ucp_send_nb(PTR_OFFSET(buffer, data_size_t1), data_size_t2, mtype, ucc_ep_map_eval(task->subset.map, t2_root), team, task); if (UCC_OK != status) { return status; } } if (rank != coll_root && rank == t1_root) { status = ucc_tl_ucp_recv_cb(buffer, data_size_t1, mtype, ucc_ep_map_eval(task->subset.map, coll_root), team, task, cb[0], (void *)task); if (UCC_OK != status) { return status; } } if (rank != coll_root && rank == t2_root) { status = ucc_tl_ucp_recv_cb(PTR_OFFSET(buffer, data_size_t1), data_size_t2, mtype, ucc_ep_map_eval(task->subset.map, coll_root), team, task, cb[1], (void *)task); if (UCC_OK != status) { return status; } } task->bcast_dbt.state = RECV; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_bcast_dbt_start", 0); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_bcast_dbt_finalize(ucc_coll_task_t *coll_task) { return ucc_tl_ucp_coll_finalize(coll_task); } ucc_status_t ucc_tl_ucp_bcast_dbt_init( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_task_t *task; ucc_rank_t rank, size; task = ucc_tl_ucp_init_task(coll_args, team); task->super.post = ucc_tl_ucp_bcast_dbt_start; task->super.progress = ucc_tl_ucp_bcast_dbt_progress; task->super.finalize = ucc_tl_ucp_bcast_dbt_finalize; task->n_polls = ucc_max(1, task->n_polls); rank = task->subset.myrank; size = (ucc_rank_t)task->subset.map.ep_num; ucc_dbt_build_trees(rank, size, &task->bcast_dbt.t1, &task->bcast_dbt.t2); *task_h = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/bcast/bcast.c0000664000175000017500000000370115211535620021175 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "bcast.h" ucc_base_coll_alg_info_t ucc_tl_ucp_bcast_algs[UCC_TL_UCP_BCAST_ALG_LAST + 1] = { [UCC_TL_UCP_BCAST_ALG_KNOMIAL] = {.id = UCC_TL_UCP_BCAST_ALG_KNOMIAL, .name = "knomial", .desc = "bcast over knomial tree with arbitrary radix " "(optimized for latency)"}, [UCC_TL_UCP_BCAST_ALG_SAG_KNOMIAL] = {.id = UCC_TL_UCP_BCAST_ALG_SAG_KNOMIAL, .name = "sag_knomial", .desc = "recursive knomial scatter followed by knomial " "allgather (optimized for BW)"}, [UCC_TL_UCP_BCAST_ALG_DBT] = {.id = UCC_TL_UCP_BCAST_ALG_DBT, .name = "dbt", .desc = "bcast over double binary tree where a leaf in one tree " "will be intermediate in other (optimized for BW)"}, [UCC_TL_UCP_BCAST_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_bcast_init(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t team_size = (ucc_rank_t)task->subset.map.ep_num; task->bcast_kn.radix = ucc_min(UCC_TL_UCP_TEAM_LIB(team)->cfg.bcast_kn_radix, team_size); task->super.post = ucc_tl_ucp_bcast_knomial_start; task->super.progress = ucc_tl_ucp_bcast_knomial_progress; return UCC_OK; } ucc_status_t ucc_tl_ucp_bcast_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_task_t *task; ucc_status_t status; task = ucc_tl_ucp_init_task(coll_args, team); status = ucc_tl_ucp_bcast_init(task); *task_h = &task->super; return status; } ucc-1.8.0/src/components/tl/ucp/bcast/bcast_knomial.c0000664000175000017500000000700415211535620022707 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "bcast.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "utils/ucc_math.h" void ucc_tl_ucp_bcast_knomial_progress(ucc_coll_task_t *coll_task) { uint32_t i; ucc_rank_t vrank; ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t rank = task->subset.myrank; ucc_rank_t size = (ucc_rank_t)task->subset.map.ep_num; uint32_t radix = task->bcast_kn.radix; ucc_rank_t root = (ucc_rank_t)TASK_ARGS(task).root; ucc_rank_t dist = task->bcast_kn.dist; void *buffer = TASK_ARGS(task).src.info.buffer; ucc_memory_type_t mtype = TASK_ARGS(task).src.info.mem_type; size_t data_size = TASK_ARGS(task).src.info.count * ucc_dt_size(TASK_ARGS(task).src.info.datatype); ucc_rank_t vpeer, peer, vroot_at_level, root_at_level, pos; if (UCC_COLL_ARGS_ACTIVE_SET(&(TASK_ARGS(task)))) { root = ucc_ep_map_local_rank(task->subset.map, root); } vrank = (rank - root + size) % size; if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } while (dist >= 1) { if (vrank % dist == 0) { pos = (vrank / dist) % radix; if (pos == 0) { for (i = radix - 1; i >= 1; i--) { vpeer = vrank + i * dist; if (vpeer < size) { peer = ucc_ep_map_eval(task->subset.map, (vpeer + root) % size); UCPCHECK_GOTO(ucc_tl_ucp_send_nb(buffer, data_size, mtype, peer, team, task), task, out); } } } else { vroot_at_level = vrank - pos * dist; root_at_level = (vroot_at_level + root) % size; peer = ucc_ep_map_eval(task->subset.map, root_at_level); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(buffer, data_size, mtype, peer, team, task), task, out); } } dist /= radix; if (UCC_INPROGRESS == ucc_tl_ucp_test_recv(task)) { task->bcast_kn.dist = dist; return; } } if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { task->bcast_kn.dist = dist; return; } ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_bcast_kn_done", 0); out: return; } ucc_status_t ucc_tl_ucp_bcast_knomial_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t size = (ucc_rank_t)task->subset.map.ep_num; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_bcast_kn_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); CALC_KN_TREE_DIST(size, task->bcast_kn.radix, task->bcast_kn.dist); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc-1.8.0/src/components/tl/ucp/bcast/bcast.h0000664000175000017500000000303115211535620021176 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef BCAST_H_ #define BCAST_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" enum { UCC_TL_UCP_BCAST_ALG_KNOMIAL, UCC_TL_UCP_BCAST_ALG_SAG_KNOMIAL, UCC_TL_UCP_BCAST_ALG_DBT, UCC_TL_UCP_BCAST_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_bcast_algs[UCC_TL_UCP_BCAST_ALG_LAST + 1]; /* SAG bcast supports team size 2, but Knomial is always better in this case */ #define UCC_TL_UCP_BCAST_DEFAULT_ALG_SELECT_STR \ "bcast:0-inf:[2-2]:@0#bcast:0-32k:[3-inf]:@0#bcast:32k-inf:[3-inf]:@1" static inline int ucc_tl_ucp_bcast_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_UCP_BCAST_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_ucp_bcast_algs[i].name)) { break; } } return i; } ucc_status_t ucc_tl_ucp_bcast_init(ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_bcast_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); void ucc_tl_ucp_bcast_knomial_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_bcast_knomial_start(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_bcast_sag_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_bcast_dbt_init( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); #endif ucc-1.8.0/src/components/tl/ucp/bcast/bcast_sag_knomial.c0000664000175000017500000001445415211535620023550 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "bcast.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "coll_patterns/sra_knomial.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "components/mc/ucc_mc.h" #include "../scatter/scatter.h" #include "../allgather/allgather.h" /* SAG - scatter-allgather knomial algorithm 1. The algorithm performs collective bcast operation for large messages as a sequence of K-nomial Scatter followed by K-nomial (with the same radix K) allgather. 2. In essence this is an extension of the Bi-nomial SRA algorithm algorithm proposed by Rabenseifner2004 (https://doi.org/10.1007/978-3-540-24685-5_1). The extension adds the support for arbitrary radix. 3. The algorithm targets Large message sizes (ie. optimized for max bandwidth). 4. If number of ranks in the team can not form a full radix subtree (for radix=2 this means the team size is not power of 2) then there will be "extra" ranks which don't participate in the main exchange loop. They will wait to receive the final data from their "proxy" ranks at the end of exchange loop of all other ranks. 5. The knomial scatter and allgather primitives can be used separately. However, if they are used together as part of SAG bcast one has to provide the same radix for both routines. 6. After the completion of scatter phase the local result (at non EXTRA ranks) will be located in dst buffer at offset the can be commputed by the routine from coll_patterns/sra_knomial.h: ucc_sra_kn_get_offset. */ ucc_status_t ucc_tl_ucp_bcast_sag_knomial_start(ucc_coll_task_t *coll_task) { ucc_schedule_t *schedule = ucc_derived_of(coll_task, ucc_schedule_t); ucc_coll_args_t *args = &schedule->super.bargs.args; ucc_coll_task_t *ag_task, *scatter_task; scatter_task = schedule->tasks[0]; scatter_task->bargs.args.src.info.buffer = args->src.info.buffer; scatter_task->bargs.args.dst.info.buffer = args->src.info.buffer; scatter_task->bargs.args.src.info.count = args->src.info.count; scatter_task->bargs.args.dst.info.count = args->src.info.count; ag_task = schedule->tasks[1]; ag_task->bargs.args.dst.info.buffer = args->src.info.buffer; ag_task->bargs.args.dst.info.count = args->src.info.count; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_bcast_sag_kn_start", 0); return ucc_schedule_start(coll_task); } ucc_status_t ucc_tl_ucp_bcast_sag_knomial_finalize(ucc_coll_task_t *coll_task) { ucc_schedule_t *schedule = ucc_derived_of(coll_task, ucc_schedule_t); ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(schedule, "ucp_bcast_sag_kn_done", 0); status = ucc_schedule_finalize(coll_task); ucc_tl_ucp_put_schedule(schedule); return status; } ucc_status_t ucc_tl_ucp_bcast_sag_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); size_t count = coll_args->args.src.info.count; ucc_datatype_t dtype = coll_args->args.src.info.datatype; ucc_memory_type_t mem_type = coll_args->args.src.info.mem_type; ucc_base_coll_args_t args = *coll_args; ucc_mrange_uint_t *p = &tl_team->cfg.bcast_sag_kn_radix; ucc_schedule_t *schedule; ucc_coll_task_t *task, *rs_task; ucc_status_t status; ucc_kn_radix_t radix, cfg_radix, opt_radix; if (UCC_COLL_ARGS_ACTIVE_SET(&coll_args->args)) { /* ActiveSets currently are only supported with KN alg */ return ucc_tl_ucp_bcast_knomial_init(coll_args, team, task_h); } opt_radix = (mem_type == UCC_MEMORY_TYPE_HOST) ? tl_team->opt_radix_host : tl_team->opt_radix; cfg_radix = ucc_tl_ucp_get_radix_from_range(tl_team, count * ucc_dt_size(dtype), mem_type, p, opt_radix); radix = ucc_knomial_pattern_get_min_radix(cfg_radix, UCC_TL_TEAM_SIZE(tl_team), count); status = ucc_tl_ucp_get_schedule(tl_team, coll_args, (ucc_tl_ucp_schedule_t **)&schedule); if (ucc_unlikely(UCC_OK != status)) { return status; } /* 1st step of bcast: knomial scatter */ args.args.dst.info.buffer = args.args.src.info.buffer; args.args.dst.info.mem_type = args.args.src.info.mem_type; args.args.dst.info.datatype = args.args.src.info.datatype; args.args.dst.info.count = args.args.src.info.count; UCC_CHECK_GOTO(ucc_tl_ucp_scatter_knomial_init_r(&args, team, &task, radix), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, task), out, status); UCC_CHECK_GOTO(ucc_event_manager_subscribe(&schedule->super, UCC_EVENT_SCHEDULE_STARTED, task, ucc_task_start_handler), out, status); rs_task = task; /* 2nd step of bcast: knomial allgather. 2nd task subscribes to completion event of scatter task. */ args.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; args.args.flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; UCC_CHECK_GOTO( ucc_tl_ucp_allgather_knomial_init_r(&args, team, &task, radix), out, status); UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, task), out, status); UCC_CHECK_GOTO(ucc_event_manager_subscribe(rs_task, UCC_EVENT_COMPLETED, task, ucc_task_start_handler), out, status); schedule->super.post = ucc_tl_ucp_bcast_sag_knomial_start; schedule->super.progress = NULL; schedule->super.finalize = ucc_tl_ucp_bcast_sag_knomial_finalize; *task_h = &schedule->super; return UCC_OK; out: ucc_tl_ucp_put_schedule(schedule); return status; } ucc-1.8.0/src/components/tl/ucp/scatter/0000775000175000017500000000000015211535620020305 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/scatter/scatter_knomial.c0000664000175000017500000002411015211535620023626 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "tl_ucp_coll.h" #include "tl_ucp_sendrecv.h" #include "core/ucc_progress_queue.h" #include "components/mc/ucc_mc.h" #include "coll_patterns/sra_knomial.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #define SAVE_STATE(_phase) \ do { \ task->scatter_kn.phase = _phase; \ } while (0) enum { UCC_SCATTER_KN_PHASE_INIT, UCC_SCATTER_KN_PHASE_LOOP, /* main loop of recursive k-ing */ }; void ucc_tl_ucp_scatter_knomial_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_kn_radix_t radix = task->scatter_kn.p.radix; ucc_knomial_pattern_t *p = &task->scatter_kn.p; void *rbuf = args->dst.info.buffer; ucc_memory_type_t mem_type = args->src.info.mem_type; size_t count = args->src.info.count; ucc_datatype_t dt = args->src.info.datatype; size_t dt_size = ucc_dt_size(dt); ucc_rank_t root = (ucc_rank_t)args->root; ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_rank_t rank = VRANK(UCC_TL_TEAM_RANK(team), root, size); ucc_rank_t team_size = size - p->n_extra; void *sbuf; ucc_rank_t peer, vroot, vpeer, peer_recv_dist; ucc_rank_t step_radix, peer_seg_index, local_seg_index; ptrdiff_t peer_seg_offset, offset; ucc_kn_radix_t loop_step; size_t block_count, peer_seg_count, local_seg_count; ucc_status_t status; root = VRANK(root, root, size); sbuf = (rank == root) ? args->src.info.buffer : args->dst.info.buffer; if (task->scatter_kn.phase == UCC_SCATTER_KN_PHASE_LOOP) { goto UCC_SCATTER_KN_PHASE_LOOP; } if (KN_NODE_EXTRA == p->node_type) { goto out; } while (!ucc_knomial_pattern_loop_done(p)) { step_radix = ucc_kn_compute_step_radix(p); block_count = ucc_sra_kn_compute_block_count(count, rank, p); local_seg_index = ucc_kn_compute_seg_index(rank, p->radix_pow, p); /* Each rank's receive (beside's root) must only happen once, and at its correct distance which is previously calclulated and saved in task->scatter_kn.recv_dist. Receive will only occur in the following iteration to that of it's parent's send. */ if ((rank != root) && (task->scatter_kn.recv_dist == p->radix_pow)) { ucc_assert(task->tagged.recv_posted == 0); local_seg_count = ucc_sra_kn_compute_seg_size(block_count, step_radix, local_seg_index); for (loop_step = 1; loop_step < radix; loop_step++) { peer = ucc_knomial_pattern_get_loop_peer(p, rank, loop_step); if (peer == UCC_KN_PEER_NULL) continue; vpeer = ucc_knomial_pattern_loop_rank(p, peer); vroot = ucc_knomial_pattern_loop_rank(p, root); peer_recv_dist = ucc_knomial_calc_recv_dist(team_size, vpeer, radix, vroot); task->scatter_kn.recv_size = local_seg_count * dt_size; if (peer_recv_dist < task->scatter_kn.recv_dist) { UCPCHECK_GOTO(ucc_tl_ucp_recv_nb( PTR_OFFSET(rbuf, task->scatter_kn.recv_offset), local_seg_count * dt_size, mem_type, INV_VRANK(peer, (ucc_rank_t)args->root, size), team, task), task, out); goto UCC_SCATTER_KN_PHASE_LOOP; } } } /* Each non leaf rank will send per iteration to up to radix - 1 "children" who are within it's current distance. Distance is initialized to 1 and each iteration is multiplied by radix. Each rank's send (besides leaf ranks) happens only after it's receive from previous iteration has completed. */ if ((root == rank) || (task->tagged.recv_posted > 0)) { ucc_assert(UCC_TL_UCP_TASK_RECV_COMPLETE(task)); for (loop_step = 1; loop_step < radix; loop_step++) { peer = ucc_knomial_pattern_get_loop_peer(p, rank, loop_step); if (peer == UCC_KN_PEER_NULL) continue; peer_seg_index = ucc_kn_compute_seg_index(peer, p->radix_pow, p); peer_seg_count = ucc_sra_kn_compute_seg_size( block_count, step_radix, peer_seg_index); peer_seg_offset = ucc_sra_kn_compute_seg_offset( block_count, step_radix, peer_seg_index); UCPCHECK_GOTO(ucc_tl_ucp_send_nb(PTR_OFFSET(sbuf, task->scatter_kn.recv_offset + peer_seg_offset * dt_size + task->scatter_kn.send_offset), peer_seg_count * dt_size, mem_type, INV_VRANK(peer, (ucc_rank_t)args->root, size), team, task), task, out); } /*TODO: local_seg_index is always zero since rank that sends is base root? */ offset = ucc_sra_kn_compute_seg_offset( block_count, step_radix, local_seg_index); task->scatter_kn.send_offset += offset * dt_size; } UCC_SCATTER_KN_PHASE_LOOP: if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_SCATTER_KN_PHASE_LOOP); return; } ucc_knomial_pattern_next_iteration(p); } if (task->scatter_kn.recv_offset == 0 && (rank != root)) { ucc_sra_kn_get_offset_and_seglen(count, dt_size, rank, size, radix, &offset, &local_seg_count); status = ucc_mc_memcpy(PTR_OFFSET(rbuf, offset), rbuf, task->scatter_kn.recv_size, mem_type, mem_type); if (ucc_unlikely(status != UCC_OK)) { task->super.status = status; return; } } out: UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_scatter_kn_done", 0); task->super.status = UCC_OK; } ucc_status_t ucc_tl_ucp_scatter_knomial_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_knomial_pattern_t *p = &task->scatter_kn.p; ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t root = args->root; ucc_rank_t vrank, vroot; ucc_on_off_auto_value_t is_zcopy; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_scatter_kn_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); ucc_knomial_pattern_init(size, VRANK(rank, root, size), task->scatter_kn.p.radix, &task->scatter_kn.p); task->scatter_kn.phase = UCC_SCATTER_KN_PHASE_INIT; vroot = ucc_knomial_pattern_loop_rank(p, VRANK(root, root, size)); vrank = ucc_knomial_pattern_loop_rank(p, VRANK(rank, root, size)); task->scatter_kn.recv_dist = ucc_knomial_calc_recv_dist(size - p->n_extra, vrank, p->radix, vroot); task->scatter_kn.recv_offset = 0; is_zcopy = UCC_TL_UCP_TEAM_LIB(team)->cfg.scatter_kn_enable_recv_zcopy; if (((is_zcopy == UCC_CONFIG_AUTO) && (args->src.info.mem_type != UCC_MEMORY_TYPE_HOST)) || (is_zcopy == UCC_CONFIG_ON)) { ucc_sra_kn_get_offset_and_seglen(args->src.info.count, ucc_dt_size(args->src.info.datatype), VRANK(rank, root, size), size, p->radix, &task->scatter_kn.recv_offset, NULL); } task->scatter_kn.send_offset = 0; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_scatter_knomial_finalize(ucc_coll_task_t *coll_task) { return ucc_tl_ucp_coll_finalize(coll_task); } ucc_status_t ucc_tl_ucp_scatter_knomial_init_r( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, ucc_kn_radix_t radix) { ucc_tl_ucp_task_t *task; ucc_assert(coll_args->args.src.info.mem_type == coll_args->args.dst.info.mem_type); task = ucc_tl_ucp_init_task(coll_args, team); task->super.post = ucc_tl_ucp_scatter_knomial_start; task->super.progress = ucc_tl_ucp_scatter_knomial_progress; task->super.finalize = ucc_tl_ucp_scatter_knomial_finalize; task->scatter_kn.p.radix = radix; *task_h = &task->super; return UCC_OK; } ucc_status_t ucc_tl_ucp_scatter_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); size_t count = coll_args->args.src.info.count; ucc_kn_radix_t radix, cfg_radix; cfg_radix = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.scatter_kn_radix; radix = ucc_knomial_pattern_get_min_radix(cfg_radix, UCC_TL_TEAM_SIZE(tl_team), count); return ucc_tl_ucp_scatter_knomial_init_r(coll_args, team, task_h, radix); } ucc-1.8.0/src/components/tl/ucp/scatter/scatter.h0000664000175000017500000000130015211535620022115 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef SCATTER_H_ #define SCATTER_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" /* Base interface signature: uses scatter_kn_radix from config. */ ucc_status_t ucc_tl_ucp_scatter_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); /* Internal interface to KN scatter with custom radix */ ucc_status_t ucc_tl_ucp_scatter_knomial_init_r( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, ucc_kn_radix_t radix); #endif ucc-1.8.0/src/components/tl/ucp/tl_ucp_task.h0000664000175000017500000002571615211535620021334 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #ifndef UCC_TL_UCP_TASK_H_ #define UCC_TL_UCP_TASK_H_ #include "tl_ucp.h" #include "tl_ucp_tag.h" #include "coll_patterns/recursive_knomial.h" #include "coll_patterns/double_binary_tree.h" #include "components/mc/base/ucc_mc_base.h" #define UCC_TL_UCP_TASK_PLUGIN_MAX_DATA 128 #define TASK_TEAM(_task) \ (ucc_derived_of((_task)->super.team, ucc_tl_ucp_team_t)) #define TASK_CTX(_task) \ (ucc_derived_of((_task)->super.team->context, ucc_tl_ucp_context_t)) #define TASK_LIB(_task) \ (ucc_derived_of((_task)->super.team->context->lib, ucc_tl_ucp_lib_t)) #define TASK_ARGS(_task) (_task)->super.bargs.args typedef struct ucc_tl_ucp_allreduce_sw_pipeline ucc_tl_ucp_allreduce_sw_pipeline; typedef struct ucc_tl_ucp_allreduce_sw_host_allgather ucc_tl_ucp_allreduce_sw_host_allgather; typedef struct ucc_tl_ucp_dpu_offload_buf_info ucc_tl_ucp_dpu_offload_buf_info_t; enum ucc_tl_ucp_task_flags { /*indicates whether subset field of tl_ucp_task is set*/ UCC_TL_UCP_TASK_FLAG_SUBSET = UCC_BIT(0), }; typedef struct ucc_tl_ucp_task { ucc_coll_task_t super; uint32_t flags; union { struct { uint32_t send_posted; uint32_t send_completed; uint32_t recv_posted; uint32_t recv_completed; uint32_t tag; } tagged; struct { uint32_t put_posted; uint32_t put_completed; uint32_t get_posted; uint32_t get_completed; } onesided; }; uint32_t n_polls; ucc_subset_t subset; union { struct { int phase; ucc_knomial_pattern_t p; } barrier; struct { int phase; ucc_knomial_pattern_t p; void *scratch; void *reduce_bufs[UCC_EE_EXECUTOR_NUM_BUFS]; ucc_mc_buffer_header_t *scratch_mc_header; ucc_ee_executor_task_t *etask; ucc_ee_executor_t *executor; } allreduce_kn; struct { ucc_tl_ucp_allreduce_sw_pipeline *pipe; ucs_status_ptr_t *put_requests; ucc_tl_ucp_allreduce_sw_host_allgather *allgather_data; ucc_coll_task_t *allgather_task; ucc_ee_executor_task_t *reduce_task; ucc_tl_ucp_dpu_offload_buf_info_t *bufs; } allreduce_sliding_window; struct { int phase; ucc_knomial_pattern_t p; void *scratch; ucc_mc_buffer_header_t *scratch_mc_header; ucc_ee_executor_task_t *etask; ucc_ee_executor_t *executor; size_t max_seg; } reduce_scatter_kn; struct { void *scratch; size_t max_block_count; ucc_ep_map_t inv_map; int n_frags; int frag; char s_scratch_busy[2]; ucc_ee_executor_task_t *etask; ucc_ee_executor_t *executor; } reduce_scatter_ring; struct { void *scratch; size_t max_block_count; ucc_ep_map_t inv_map; int n_frags; int frag; char s_scratch_busy[2]; ucc_ee_executor_task_t *etask; ucc_ee_executor_t *executor; } reduce_scatterv_ring; struct { int phase; ucc_knomial_pattern_t p; ucc_rank_t recv_dist; ptrdiff_t send_offset; ptrdiff_t recv_offset; size_t recv_size; } scatter_kn; struct { int phase; ucc_knomial_pattern_t p; void *sbuf; ucc_tl_ucp_copy_task_t *copy_task; ucc_rank_t recv_dist; ucp_mem_h dst_memh; ucp_mem_h src_memh; } allgather_kn; struct { /* * get send/recv block depends on subset type being used. * For service allgather we need to get context endpoints but keep * subset numbering. * For regular allgather with rank reordering both endpoints * and blocks permutation are necessary. */ ucc_rank_t (*get_send_block)(ucc_subset_t *subset, ucc_rank_t trank, ucc_rank_t tsize, int step); ucc_rank_t (*get_recv_block)(ucc_subset_t *subset, ucc_rank_t trank, ucc_rank_t tsize, int step); } allgather_ring; struct { int nreqs; // number of send/recv requests in progress ucc_tl_ucp_copy_task_t *copy_task; } allgather_linear; struct { ucc_mc_buffer_header_t *scratch_header; size_t scratch_size; } allgather_bruck; struct { uint32_t i; int data_expected; } allgather_sparbit; struct { ucc_rank_t dist; uint32_t radix; } bcast_kn; struct { ucc_dbt_single_tree_t t1; ucc_dbt_single_tree_t t2; int state; } bcast_dbt; struct { ucc_rank_t dist; ucc_rank_t max_dist; int children_per_cycle; uint32_t radix; int phase; void *scratch; ucc_mc_buffer_header_t *scratch_mc_header; ucc_ee_executor_task_t *etask; ucc_ee_executor_t *executor; } reduce_kn; struct { int state; ucc_dbt_single_tree_t trees[2]; int reduction_comp[2]; int send_comp[2]; void *scratch; ucc_mc_buffer_header_t *scratch_mc_header; ucc_ee_executor_task_t *etask; ucc_ee_executor_t *executor; } reduce_dbt; struct { int phase; ucc_knomial_pattern_t p; ucc_rank_t dist; ucc_rank_t max_dist; uint32_t radix; void * scratch; ucc_mc_buffer_header_t *scratch_mc_header; } gather_kn; struct { size_t merge_buf_size; ucc_mc_buffer_header_t *scratch_mc_header; size_t byte_send_limit; int phase; uint32_t radix; uint32_t cur_radix; uint32_t iteration; ucc_rank_t cur_out; size_t traffic_in; size_t traffic_out; ucc_rank_t num_in; ucc_rank_t num2send; ucc_rank_t num2recv; } alltoallv_hybrid; struct { ucc_mc_buffer_header_t *scratch_mc_header; ucc_ee_executor_task_t *etask; void *src; void *dst; ucc_rank_t iteration; int phase; } alltoall_bruck; struct { uint32_t tokens; uint32_t npolls; } alltoall_onesided; char plugin_data[UCC_TL_UCP_TASK_PLUGIN_MAX_DATA]; }; uint32_t flush_posted; uint32_t flush_completed; } ucc_tl_ucp_task_t; static inline void ucc_tl_ucp_task_reset(ucc_tl_ucp_task_t *task, ucc_status_t status) { task->tagged.send_posted = 0; task->tagged.send_completed = 0; task->tagged.recv_posted = 0; task->tagged.recv_completed = 0; task->flush_posted = 0; task->flush_completed = 0; task->super.status = status; } static inline ucc_tl_ucp_task_t *ucc_tl_ucp_get_task(ucc_tl_ucp_team_t *team) { ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(team); ucc_tl_ucp_task_t *task = ucc_mpool_get(&ctx->req_mp);; UCC_TL_UCP_PROFILE_REQUEST_NEW(task, "tl_ucp_task", 0); task->super.flags = 0; task->flags = 0; task->n_polls = ctx->cfg.n_polls; task->super.team = &team->super.super; task->subset.map.type = UCC_EP_MAP_FULL; task->subset.map.ep_num = UCC_TL_TEAM_SIZE(team); task->subset.myrank = UCC_TL_TEAM_RANK(team); ucc_tl_ucp_task_reset(task, UCC_OPERATION_INITIALIZED); return task; } static inline void ucc_tl_ucp_put_task(ucc_tl_ucp_task_t *task) { UCC_TL_UCP_PROFILE_REQUEST_FREE(task); ucc_mpool_put(task); } typedef struct ucc_tl_ucp_schedule { ucc_schedule_pipelined_t super; ucc_mc_buffer_header_t *scratch_mc_header; union { ptrdiff_t frag_offset; } reduce_srg_kn; } ucc_tl_ucp_schedule_t; static inline ucc_status_t ucc_tl_ucp_get_schedule(ucc_tl_ucp_team_t *team, ucc_base_coll_args_t *args, ucc_tl_ucp_schedule_t **schedule) { ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(team); *schedule = ucc_mpool_get(&ctx->req_mp); if (ucc_unlikely(!(*schedule))) { return UCC_ERR_NO_MEMORY; } UCC_TL_UCP_PROFILE_REQUEST_NEW(schedule, "tl_ucp_sched", 0); return ucc_schedule_init(&((*schedule)->super.super), args, &team->super.super); } static inline void ucc_tl_ucp_put_schedule(ucc_schedule_t *schedule) { UCC_TL_UCP_PROFILE_REQUEST_FREE(schedule); ucc_mpool_put(schedule); } #endif ucc-1.8.0/src/components/tl/ucp/scatterv/0000775000175000017500000000000015211535620020473 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/scatterv/scatterv.h0000664000175000017500000000072715211535620022505 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef SCATTERV_H_ #define SCATTERV_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" enum { UCC_TL_UCP_SCATTERV_ALG_LINEAR, UCC_TL_UCP_SCATTERV_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_scatterv_algs[UCC_TL_UCP_SCATTERV_ALG_LAST + 1]; ucc_status_t ucc_tl_ucp_scatterv_init(ucc_tl_ucp_task_t *task); #endif ucc-1.8.0/src/components/tl/ucp/scatterv/scatterv_linear.c0000664000175000017500000001157015211535620024030 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "scatterv.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" static inline ucc_rank_t get_peer(ucc_rank_t rank, ucc_rank_t size, ucc_rank_t step) { return (rank + step) % size; } void ucc_tl_ucp_scatterv_linear_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); void* sbuf = args->src.info_v.buffer; ucc_memory_type_t smem = args->src.info_v.mem_type; ucc_rank_t grank = UCC_TL_TEAM_RANK(team); ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); int polls = 0; ucc_rank_t peer, posts, nreqs; size_t dt_size, data_size, data_displ; if (UCC_IS_ROOT(*args, grank)) { posts = UCC_TL_UCP_TEAM_LIB(team)->cfg.scatterv_linear_num_posts; nreqs = (posts > gsize || posts == 0) ? gsize : posts; dt_size = ucc_dt_size(TASK_ARGS(task).src.info_v.datatype); while (polls++ < task->n_polls) { ucp_worker_progress(UCC_TL_UCP_TEAM_CTX(team)->worker.ucp_worker); while ((task->tagged.send_posted < gsize) && ((task->tagged.send_posted - task->tagged.send_completed) < nreqs)) { peer = get_peer(grank, gsize, task->tagged.send_posted); data_size = ucc_coll_args_get_count(args, args->src.info_v.counts, peer) * dt_size; data_displ = ucc_coll_args_get_displacement(args, args->src.info_v.displacements, peer) * dt_size; UCPCHECK_GOTO(ucc_tl_ucp_send_nz(PTR_OFFSET(sbuf, data_displ), data_size, smem, peer, team, task), task, out); polls = 0; } } if (task->tagged.send_posted < gsize) { return; } } task->super.status = ucc_tl_ucp_test(task); out: if (task->super.status != UCC_INPROGRESS) { UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_scatterv_linear_done", 0); } } ucc_status_t ucc_tl_ucp_scatterv_linear_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t grank = UCC_TL_TEAM_RANK(team); ucc_memory_type_t smem = args->src.info_v.mem_type; ucc_memory_type_t rmem = args->dst.info.mem_type; void *rbuf = args->dst.info.buffer; void * sbuf; size_t dt_size, data_displ, data_size; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_scatterv_linear_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if (UCC_IS_ROOT(*args, grank)) { if (!UCC_IS_INPLACE(*args)) { dt_size = ucc_dt_size(args->src.info_v.datatype); data_size = ucc_coll_args_get_count(args, args->src.info_v.counts, grank) * dt_size; data_displ = ucc_coll_args_get_displacement(args, args->src.info_v.displacements, grank) * dt_size; sbuf = PTR_OFFSET(args->src.info_v.buffer, data_displ); UCPCHECK_GOTO(ucc_tl_ucp_recv_nz(rbuf, data_size, rmem, grank, team, task), task, error); UCPCHECK_GOTO(ucc_tl_ucp_send_nz(sbuf, data_size, smem, grank, team, task), task, error); } else { /* to simplify progress fucnction and make it identical for in-place and non in-place */ task->tagged.send_posted = task->tagged.send_completed = 1; task->tagged.recv_posted = task->tagged.recv_completed = 1; } } else { dt_size = ucc_dt_size(args->dst.info.datatype); data_size = args->dst.info.count * dt_size; UCPCHECK_GOTO(ucc_tl_ucp_recv_nz(rbuf, data_size, rmem, args->root, team, task), task, error); } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); error: return task->super.status; } ucc_status_t ucc_tl_ucp_scatterv_linear_init(ucc_tl_ucp_task_t *task) { task->super.post = ucc_tl_ucp_scatterv_linear_start; task->super.progress = ucc_tl_ucp_scatterv_linear_progress; task->n_polls = ucc_max(1, task->n_polls); return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/scatterv/scatterv.c0000664000175000017500000000221215211535620022467 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "scatterv.h" #include "utils/ucc_coll_utils.h" ucc_status_t ucc_tl_ucp_scatterv_linear_start(ucc_coll_task_t *task); void ucc_tl_ucp_scatterv_linear_progress(ucc_coll_task_t *task); ucc_base_coll_alg_info_t ucc_tl_ucp_scatterv_algs[UCC_TL_UCP_SCATTERV_ALG_LAST + 1] = { [UCC_TL_UCP_SCATTERV_ALG_LINEAR] = {.id = UCC_TL_UCP_SCATTERV_ALG_LINEAR, .name = "linear", .desc = "linear scatterv algorithm"}, [UCC_TL_UCP_SCATTERV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_scatterv_linear_init(ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_scatterv_init(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); if (!ucc_coll_args_is_predefined_dt(args, trank)) { return UCC_ERR_NOT_SUPPORTED; } return ucc_tl_ucp_scatterv_linear_init(task); } ucc-1.8.0/src/components/tl/ucp/tl_ucp_sendrecv.c0000664000175000017500000001452415211535620022171 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp_sendrecv.h" void ucc_tl_ucp_send_recv_counter_inc_st(uint32_t *counter) { ++(*counter); } void ucc_tl_ucp_send_recv_counter_inc_mt(uint32_t *counter) { ucc_atomic_add32(counter, 1); } void ucc_tl_ucp_send_completion_cb_st(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in send completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } ++task->tagged.send_completed; ucp_request_free(request); } void ucc_tl_ucp_send_completion_cb_mt(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in send completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } ucc_atomic_add32(&task->tagged.send_completed, 1); ucp_request_free(request); } void ucc_tl_ucp_put_completion_cb(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in put completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } task->onesided.put_completed++; ucp_request_free(request); } void ucc_tl_ucp_get_completion_cb(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in get completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } task->onesided.get_completed++; ucp_request_free(request); } void ucc_tl_ucp_flush_completion_cb(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in ep flush completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } task->flush_completed++; ucp_request_free(request); } void ucc_tl_ucp_recv_completion_cb_mt(void *request, ucs_status_t status, const ucp_tag_recv_info_t *info, /* NOLINT */ void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in recv completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } ucc_atomic_add32(&task->tagged.recv_completed, 1); ucp_request_free(request); } void ucc_tl_ucp_recv_completion_cb_st(void *request, ucs_status_t status, const ucp_tag_recv_info_t *info, /* NOLINT */ void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in recv completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } ++task->tagged.recv_completed; ucp_request_free(request); } ucc_status_t ucc_tl_ucp_send_nbx(void *buffer, size_t msglen, ucc_rank_t dest_group_rank, const ucp_request_param_t *req_param, ucc_tl_ucp_task_t *task) { const ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_status_t status; ucp_ep_h ep; ucp_tag_t ucp_tag; ucs_status_ptr_t ucp_status; status = ucc_tl_ucp_get_ep(team, dest_group_rank, &ep); if (ucc_unlikely(UCC_OK != status)) { return status; } ucp_tag = UCC_TL_UCP_MAKE_SEND_TAG((args->mask & UCC_COLL_ARGS_FIELD_TAG), task->tagged.tag, UCC_TL_TEAM_RANK(team), team->super.super.params.id, team->super.super.params.scope_id, team->super.super.params.scope); ucp_status = ucp_tag_send_nbx(ep, buffer, msglen, ucp_tag, req_param); task->tagged.send_posted++; if (UCS_OK != ucp_status) { UCC_TL_UCP_CHECK_REQ_STATUS(); } else { UCC_TL_UCP_TEAM_CTX(team)->sendrecv_cbs.p2p_counter_inc( &task->tagged.send_completed); } return UCC_OK; } ucc_status_t ucc_tl_ucp_recv_nbx(void *buffer, size_t msglen, ucc_rank_t dest_group_rank, const ucp_request_param_t *req_param, ucc_tl_ucp_task_t *task) { const ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucp_tag_t ucp_tag, ucp_tag_mask; ucs_status_ptr_t ucp_status; UCC_TL_UCP_MAKE_RECV_TAG(ucp_tag, ucp_tag_mask, (args->mask & UCC_COLL_ARGS_FIELD_TAG), task->tagged.tag, dest_group_rank, team->super.super.params.id, team->super.super.params.scope_id, team->super.super.params.scope); ucp_status = ucp_tag_recv_nbx(team->worker->ucp_worker, buffer, msglen, ucp_tag, ucp_tag_mask, req_param); task->tagged.recv_posted++; if (UCS_OK != ucp_status) { UCC_TL_UCP_CHECK_REQ_STATUS(); } else { UCC_TL_UCP_TEAM_CTX(team)->sendrecv_cbs.p2p_counter_inc( &task->tagged.recv_completed); } return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/reduce_scatter/0000775000175000017500000000000015211535620021634 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/reduce_scatter/reduce_scatter.c0000664000175000017500000000142015211535620024771 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "reduce_scatter.h" #include "utils/ucc_coll_utils.h" ucc_base_coll_alg_info_t ucc_tl_ucp_reduce_scatter_algs[UCC_TL_UCP_REDUCE_SCATTER_ALG_LAST + 1] = { [UCC_TL_UCP_REDUCE_SCATTER_ALG_RING] = {.id = UCC_TL_UCP_REDUCE_SCATTER_ALG_RING, .name = "ring", .desc = "O(N) ring"}, [UCC_TL_UCP_REDUCE_SCATTER_ALG_KNOMIAL] = {.id = UCC_TL_UCP_REDUCE_SCATTER_ALG_KNOMIAL, .name = "knomial", .desc = "recursive k-ing with arbitrary radix"}, [UCC_TL_UCP_REDUCE_SCATTER_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc-1.8.0/src/components/tl/ucp/reduce_scatter/reduce_scatter.h0000664000175000017500000000305715211535620025006 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef REDUCE_SCATTER_H_ #define REDUCE_SCATTER_H_ #include "tl_ucp_coll.h" enum { UCC_TL_UCP_REDUCE_SCATTER_ALG_RING, UCC_TL_UCP_REDUCE_SCATTER_ALG_KNOMIAL, UCC_TL_UCP_REDUCE_SCATTER_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_reduce_scatter_algs[UCC_TL_UCP_REDUCE_SCATTER_ALG_LAST + 1]; #define UCC_TL_UCP_REDUCE_SCATTER_DEFAULT_ALG_SELECT_STR \ "reduce_scatter:@ring" static inline int ucc_tl_ucp_reduce_scatter_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_UCP_REDUCE_SCATTER_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_ucp_reduce_scatter_algs[i].name)) { break; } } return i; } /* Base interface signature: uses reduce_scatter_kn_radix from config. */ ucc_status_t ucc_tl_ucp_reduce_scatter_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); /* Internal interface to KN reduce scatter with custom radix */ ucc_status_t ucc_tl_ucp_reduce_scatter_knomial_init_r( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, ucc_kn_radix_t radix); ucc_status_t ucc_tl_ucp_reduce_scatter_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h); #endif ucc-1.8.0/src/components/tl/ucp/reduce_scatter/reduce_scatter_knomial.c0000664000175000017500000005737315211535620026525 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatter.h" #include "tl_ucp_sendrecv.h" #include "coll_patterns/sra_knomial.h" #include "utils/ucc_math.h" #include "utils/ucc_dt_reduce.h" #define SAVE_STATE(_phase) \ do { \ task->reduce_scatter_kn.phase = _phase; \ } while (0) #define GET_COUNT(_args) \ ({ \ size_t _count = 0; \ switch ((_args)->coll_type) { \ case UCC_COLL_TYPE_ALLREDUCE: \ case UCC_COLL_TYPE_REDUCE: \ _count = (_args)->dst.info.count; \ break; \ case UCC_COLL_TYPE_REDUCE_SCATTER: \ _count = UCC_IS_INPLACE(*(_args)) \ ? (_args)->dst.info.count \ : (_args)->src.info.count; \ break; \ default: \ break; \ } \ _count; \ }) #define GET_DT(_args) \ ((_args)->coll_type == UCC_COLL_TYPE_REDUCE_SCATTERV) \ ? (_args)->dst.info_v.datatype \ : (_args)->dst.info.datatype typedef struct ucc_tl_ucp_rs_work_buf { void *src_data; void *dst_data; void *src_loop; void *dst_loop; void *dst_proxy; void *reduce_proxy; void *reduce_loop; } ucc_tl_ucp_rs_work_buf_t; /* get work buffers for allreduce and reduce */ static inline void get_sbuf_rbuf_ar(ucc_tl_ucp_task_t *task, size_t block_count, ucc_tl_ucp_rs_work_buf_t *wb) { ucc_coll_args_t *args = &TASK_ARGS(task); size_t dt_size = ucc_dt_size(args->dst.info.datatype); void *scratch = task->reduce_scatter_kn.scratch; ucc_knomial_pattern_t *p = &task->reduce_scatter_kn.p; size_t offset, local_seg_count; ptrdiff_t local_seg_offset; void *sbuf, *rbuf; if (ucc_knomial_pattern_loop_first_iteration(p)) { sbuf = ((KN_NODE_PROXY == p->node_type) || UCC_IS_INPLACE(*args)) ? args->dst.info.buffer: args->src.info.buffer; rbuf = scratch; } else { sbuf = scratch; if (!ucc_knomial_pattern_loop_last_iteration(p) || (task->reduce_scatter_kn.scratch_mc_header != NULL)) { rbuf = PTR_OFFSET(sbuf, block_count * dt_size); } else { ucc_sra_kn_get_offset_and_seglen(args->dst.info.count, dt_size, p->rank, p->size, p->radix, &local_seg_offset, &local_seg_count); local_seg_offset = local_seg_offset / dt_size; if ((local_seg_offset <= block_count) || (local_seg_count == 0)) { rbuf = PTR_OFFSET(sbuf, block_count * dt_size); } else { offset = (local_seg_offset - block_count) % local_seg_count; /* check we have enough space to store segments */ ucc_assert(args->dst.info.count - (block_count + offset) >= local_seg_count * (ucc_kn_compute_step_radix(p) - 1)); rbuf = PTR_OFFSET(sbuf, (block_count + offset) * dt_size); } } } if (ucc_knomial_pattern_loop_last_iteration(p)) { ucc_sra_kn_get_offset_and_seglen(args->dst.info.count, dt_size, p->rank, p->size, p->radix, &local_seg_offset, &local_seg_count); wb->reduce_loop = PTR_OFFSET(args->dst.info.buffer, local_seg_offset); } else { wb->reduce_loop = scratch; } wb->src_data = UCC_IS_INPLACE(*args) ? args->dst.info.buffer : args->src.info.buffer; wb->src_loop = sbuf; wb->dst_loop = rbuf; wb->dst_proxy = scratch; wb->reduce_proxy = args->dst.info.buffer; } /* get work buffers for reduce scatter */ static inline void get_sbuf_rbuf_rs(ucc_tl_ucp_task_t *task, ucc_tl_ucp_rs_work_buf_t *wb) { ucc_coll_args_t *args = &TASK_ARGS(task); ucc_knomial_pattern_t *p = &task->reduce_scatter_kn.p; void *scratch = task->reduce_scatter_kn.scratch; size_t dt_size = ucc_dt_size(args->dst.info.datatype); ucc_rank_t trank = task->subset.myrank; ucc_rank_t tsize = task->subset.map.ep_num; ucc_kn_radix_t radix = p->radix; size_t max_seg; void *sbuf, *rbuf, *data_buf; max_seg = task->reduce_scatter_kn.max_seg; if (UCC_IS_INPLACE(*args)) { data_buf = args->dst.info.buffer; wb->dst_data = PTR_OFFSET(args->dst.info.buffer, (args->dst.info.count / tsize) * trank * dt_size); } else { data_buf = args->src.info.buffer; wb->dst_data = args->dst.info.buffer; } if (KN_NODE_PROXY == p->node_type) { if (UCC_IS_INPLACE(*args)) { rbuf = PTR_OFFSET(scratch, max_seg * dt_size); if (ucc_knomial_pattern_loop_first_iteration(p)) { sbuf = args->dst.info.buffer; } else { sbuf = scratch; } wb->dst_proxy = scratch; wb->reduce_proxy = args->dst.info.buffer; } else { sbuf = PTR_OFFSET(scratch, max_seg * (radix - 1) * dt_size); rbuf = scratch; wb->dst_proxy = PTR_OFFSET(scratch, max_seg * (radix - 1) * dt_size); wb->reduce_proxy = wb->dst_proxy; } } else { rbuf = PTR_OFFSET(scratch, max_seg * dt_size); if (ucc_knomial_pattern_loop_first_iteration(p)) { sbuf = data_buf; } else { sbuf = scratch; } } if (ucc_knomial_pattern_loop_last_iteration(p)) { if (KN_NODE_PROXY == p->node_type) { wb->reduce_loop = PTR_OFFSET(scratch, max_seg * (radix - 1) * dt_size); } else { wb->reduce_loop = wb->dst_data; } } else { if (KN_NODE_PROXY == p->node_type) { if (UCC_IS_INPLACE(*args)) { wb->reduce_loop = scratch; } else { wb->reduce_loop = PTR_OFFSET(scratch, max_seg * (radix - 1) * dt_size); } } else { wb->reduce_loop = scratch; } } wb->src_loop = sbuf; wb->dst_loop = rbuf; wb->src_data = data_buf; } static inline void get_rs_work_buf(ucc_tl_ucp_task_t *task, size_t block_count, ucc_tl_ucp_rs_work_buf_t *wb) { ucc_coll_args_t *args = &TASK_ARGS(task); switch (args->coll_type) { case UCC_COLL_TYPE_ALLREDUCE: case UCC_COLL_TYPE_REDUCE: return get_sbuf_rbuf_ar(task, block_count, wb); case UCC_COLL_TYPE_REDUCE_SCATTER: return get_sbuf_rbuf_rs(task, wb); case UCC_COLL_TYPE_REDUCE_SCATTERV: default: ucc_assert(0); return; } } /* return the rank of the peer for the given rank and pattern taken into account the root and map */ static inline ucc_rank_t get_physical_rank(ucc_tl_ucp_task_t *task, ucc_rank_t rank, ucc_rank_t root, ucc_rank_t size) { return INV_VRANK(ucc_ep_map_eval(task->subset.map, rank), root, size); } void ucc_tl_ucp_reduce_scatter_knomial_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_knomial_pattern_t *p = &task->reduce_scatter_kn.p; ucc_kn_radix_t radix = p->radix; uint8_t node_type = p->node_type; ucc_memory_type_t mem_type = args->dst.info.mem_type; size_t count = GET_COUNT(args); ucc_datatype_t dt = GET_DT(args); size_t dt_size = ucc_dt_size(dt); size_t data_size = count * dt_size; ucc_rank_t rank = task->subset.myrank; ucc_rank_t size = task->subset.map.ep_num; ucc_rank_t root = 0; size_t local_seg_count = 0; ucc_tl_ucp_rs_work_buf_t wb = (ucc_tl_ucp_rs_work_buf_t){0}; ptrdiff_t peer_seg_offset, local_seg_offset; ucc_rank_t peer; ucc_status_t status; ucc_kn_radix_t step_radix, loop_step; size_t block_count, peer_seg_count; void *local_data; int is_avg; if (args->coll_type == UCC_COLL_TYPE_REDUCE) { root = args->root; rank = VRANK(rank, root, size); } UCC_KN_REDUCE_GOTO_PHASE(task->reduce_scatter_kn.phase); block_count = ucc_sra_kn_compute_block_count(count, rank, p); get_rs_work_buf(task, block_count, &wb); if (KN_NODE_EXTRA == node_type) { peer = get_physical_rank(task, ucc_knomial_pattern_get_proxy(p, rank), root, size); UCPCHECK_GOTO(ucc_tl_ucp_send_nb(wb.src_data, data_size, mem_type, peer, team, task), task, out); if (p->type != KN_PATTERN_REDUCE_SCATTERX) { UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(wb.dst_data, (count / size) * dt_size, mem_type, peer, team, task), task, out); } } if (KN_NODE_PROXY == node_type) { peer = get_physical_rank(task, ucc_knomial_pattern_get_extra(p, rank), root, size); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(wb.dst_proxy, data_size, mem_type, peer, team, task), task, out); } UCC_KN_PHASE_EXTRA: block_count = ucc_sra_kn_compute_block_count(count, rank, p); get_rs_work_buf(task, block_count, &wb); if ((KN_NODE_PROXY == node_type) || (KN_NODE_EXTRA == node_type)) { if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_EXTRA); return; } if (KN_NODE_EXTRA == node_type) { goto out; } status = ucc_dt_reduce(wb.src_data, wb.dst_proxy, wb.reduce_proxy, count, dt, args, 0, 0, task->reduce_scatter_kn.executor, &task->reduce_scatter_kn.etask); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); task->super.status = status; return; } UCC_KN_PHASE_EXTRA_REDUCE: EXEC_TASK_TEST(UCC_KN_PHASE_EXTRA_REDUCE, "failed to perform dt reduction", task->reduce_scatter_kn.etask); } while (!ucc_knomial_pattern_loop_done(p)) { block_count = ucc_sra_kn_compute_block_count(count, rank, p); local_seg_count = 0; ucc_kn_rs_pattern_peer_seg(rank, p, &local_seg_count, &local_seg_offset); get_rs_work_buf(task, block_count, &wb); for (loop_step = radix - 1; loop_step > 0; loop_step--) { peer = ucc_knomial_pattern_get_loop_peer(p, rank, loop_step); if (peer == UCC_KN_PEER_NULL) { continue; } ucc_kn_rs_pattern_peer_seg(peer, p, &peer_seg_count, &peer_seg_offset); peer = get_physical_rank(task, peer, root, size); UCPCHECK_GOTO( ucc_tl_ucp_send_nb(PTR_OFFSET(wb.src_loop, peer_seg_offset * dt_size), peer_seg_count * dt_size, mem_type, peer, team, task), task, out); UCPCHECK_GOTO( ucc_tl_ucp_recv_nb(wb.dst_loop, local_seg_count * dt_size, mem_type, peer, team, task), task, out); wb.dst_loop = PTR_OFFSET(wb.dst_loop, local_seg_count * dt_size); } UCC_KN_PHASE_LOOP: if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_LOOP); return; } if (task->tagged.send_posted > p->iteration * (radix - 1)) { step_radix = ucc_kn_compute_step_radix(p); local_seg_count = 0; block_count = ucc_sra_kn_compute_block_count(count, rank, p); ucc_kn_rs_pattern_peer_seg(rank, p, &local_seg_count, &local_seg_offset); get_rs_work_buf(task, block_count, &wb); local_data = PTR_OFFSET(wb.src_loop, local_seg_offset * dt_size); is_avg = (args->op == UCC_OP_AVG) && (UCC_TL_UCP_TEAM_LIB(team)->cfg.reduce_avg_pre_op ? ucc_knomial_pattern_loop_first_iteration(p) : ucc_knomial_pattern_loop_last_iteration(p)); status = ucc_dt_reduce_strided(local_data, wb.dst_loop, wb.reduce_loop, step_radix - 1, local_seg_count, local_seg_count * dt_size, dt, args, is_avg ? UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA : 0, AVG_ALPHA(task), task->reduce_scatter_kn.executor, &task->reduce_scatter_kn.etask); if (ucc_unlikely(UCC_OK != status)) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); task->super.status = status; return; } UCC_KN_PHASE_REDUCE: EXEC_TASK_TEST(UCC_KN_PHASE_REDUCE, "failed to perform dt reduction", task->reduce_scatter_kn.etask); } if ((args->coll_type == UCC_COLL_TYPE_REDUCE_SCATTER) && (KN_NODE_PROXY == node_type) && ucc_knomial_pattern_loop_last_iteration(p)) { get_rs_work_buf(task, 0, &wb); peer = ucc_knomial_pattern_get_extra(p, rank); peer_seg_count = 0; peer_seg_offset = 0; ucc_kn_rs_pattern_extra_seg(p, &peer_seg_count, &peer_seg_offset); UCPCHECK_GOTO(ucc_tl_ucp_send_nb( PTR_OFFSET(wb.reduce_loop, peer_seg_offset * dt_size), peer_seg_count * dt_size, mem_type, peer, team, task), task, out); ucc_mc_memcpy(wb.dst_data, wb.reduce_loop, peer_seg_count * dt_size, mem_type, mem_type); } ucc_kn_rs_pattern_next_iter(p); } UCC_KN_PHASE_COMPLETE: UCC_KN_PHASE_PROXY: if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_PROXY); return; } out: UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_reduce_scatter_kn_done", 0); task->super.status = UCC_OK; } ucc_status_t ucc_tl_ucp_reduce_scatter_knomial_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t size = task->subset.map.ep_num; ucc_coll_type_t ct = args->coll_type; ucc_rank_t root = (ct == UCC_COLL_TYPE_REDUCE) ? args->root : 0; ucc_rank_t rank = VRANK(task->subset.myrank, root, size); size_t count = GET_COUNT(args); ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_reduce_scatter_kn_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if ((ct == UCC_COLL_TYPE_ALLREDUCE) || (ct == UCC_COLL_TYPE_REDUCE)) { ucc_kn_rsx_pattern_init(size, rank, task->reduce_scatter_kn.p.radix, count, &task->reduce_scatter_kn.p); } else { ucc_kn_rs_pattern_init(size, rank, task->reduce_scatter_kn.p.radix, count, &task->reduce_scatter_kn.p); } if (!task->reduce_scatter_kn.scratch_mc_header) { task->reduce_scatter_kn.scratch = args->dst.info.buffer; } task->reduce_scatter_kn.phase = UCC_KN_PHASE_INIT; status = ucc_coll_task_get_executor(&task->super, &task->reduce_scatter_kn.executor); if (ucc_unlikely(status != UCC_OK)) { return status; } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_reduce_scatter_knomial_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); if (task->reduce_scatter_kn.scratch_mc_header) { ucc_mc_free(task->reduce_scatter_kn.scratch_mc_header); } return ucc_tl_ucp_coll_finalize(coll_task); } static size_t compute_scratch_size(ucc_tl_ucp_task_t *task) { ucc_coll_args_t *args = &TASK_ARGS(task); ucc_base_coll_args_t *coll_args = &task->super.bargs; size_t count = GET_COUNT(args); size_t dt_size = ucc_dt_size(GET_DT(args)); size_t max_seg = task->reduce_scatter_kn.max_seg; size_t data_size; ucc_kn_radix_t step_radix; size_t max_recv_size; if ((args->coll_type == UCC_COLL_TYPE_ALLREDUCE) || (args->coll_type == UCC_COLL_TYPE_REDUCE)) { if (KN_NODE_EXTRA != task->reduce_scatter_kn.p.node_type) { if (coll_args->mask & UCC_BASE_CARGS_MAX_FRAG_COUNT) { count = coll_args->max_frag_count; } data_size = count * dt_size; step_radix = ucc_kn_compute_step_radix(&task->reduce_scatter_kn.p); max_recv_size = ucc_sra_kn_compute_seg_size(count, step_radix, 0) * step_radix * dt_size; if (UCC_IS_INPLACE(coll_args->args) || (KN_NODE_PROXY == task->reduce_scatter_kn.p.node_type) || max_recv_size > data_size) { return ucc_max(max_recv_size, data_size); } else { return 0; } } } else { step_radix = task->reduce_scatter_kn.p.radix; if (KN_NODE_PROXY == task->reduce_scatter_kn.p.node_type) { if (UCC_IS_INPLACE(*args)) { return max_seg * step_radix * dt_size; } else { return (max_seg * (step_radix - 1) + count) * dt_size; } } else if (KN_NODE_EXTRA != task->reduce_scatter_kn.p.node_type) { return max_seg * step_radix * dt_size; } } return 0; } ucc_status_t ucc_tl_ucp_reduce_scatter_knomial_init_r(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, ucc_kn_radix_t radix) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_memory_type_t mem_type = coll_args->args.dst.info.mem_type; size_t count = GET_COUNT(&coll_args->args); ucc_coll_type_t ct = coll_args->args.coll_type; ucc_sbgp_t *sbgp; ucc_tl_ucp_task_t *task; ucc_status_t status; size_t scratch_size; ucc_rank_t rank, size; ptrdiff_t max_seg_offset; if (ucc_unlikely(!UCC_IS_INPLACE(coll_args->args) && (coll_args->args.src.info.mem_type != coll_args->args.dst.info.mem_type))) { return UCC_ERR_NOT_SUPPORTED; } task = ucc_tl_ucp_init_task(coll_args, team); task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; task->super.post = ucc_tl_ucp_reduce_scatter_knomial_start; task->super.progress = ucc_tl_ucp_reduce_scatter_knomial_progress; task->super.finalize = ucc_tl_ucp_reduce_scatter_knomial_finalize; if (tl_team->cfg.use_reordering && ct == UCC_COLL_TYPE_ALLREDUCE) { sbgp = ucc_topo_get_sbgp(tl_team->topo, UCC_SBGP_FULL_HOST_ORDERED); task->subset.myrank = sbgp->group_rank; task->subset.map = sbgp->map; } rank = task->subset.myrank; size = task->subset.map.ep_num; if (ct == UCC_COLL_TYPE_ALLREDUCE) { ucc_kn_rsx_pattern_init(size, rank, radix, count, &task->reduce_scatter_kn.p); } else if (ct == UCC_COLL_TYPE_REDUCE) { ucc_kn_rsx_pattern_init(size, VRANK(rank, coll_args->args.root, size), radix, count, &task->reduce_scatter_kn.p); } else { ucc_kn_rs_pattern_init(size, rank, radix, count, &task->reduce_scatter_kn.p); } ucc_kn_rs_pattern_peer_seg(0, &task->reduce_scatter_kn.p, &task->reduce_scatter_kn.max_seg, &max_seg_offset); task->reduce_scatter_kn.scratch_mc_header = NULL; scratch_size = compute_scratch_size(task); if (scratch_size != 0) { status = ucc_mc_alloc(&task->reduce_scatter_kn.scratch_mc_header, scratch_size, mem_type); if (ucc_unlikely(UCC_OK != status)) { tl_error(UCC_TASK_LIB(task), "failed to allocate scratch buffer"); ucc_tl_ucp_coll_finalize(&task->super); return status; } task->reduce_scatter_kn.scratch = task->reduce_scatter_kn.scratch_mc_header->addr; } *task_h = &task->super; return UCC_OK; } ucc_status_t ucc_tl_ucp_reduce_scatter_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t size = UCC_TL_TEAM_SIZE(tl_team); size_t count = coll_args->args.dst.info.count; ucc_kn_radix_t radix, cfg_radix; cfg_radix = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.reduce_scatter_kn_radix; radix = ucc_knomial_pattern_get_min_radix(cfg_radix, size, count); return ucc_tl_ucp_reduce_scatter_knomial_init_r(coll_args, team, task_h, radix); } ucc-1.8.0/src/components/tl/ucp/reduce_scatter/reduce_scatter_ring.c0000664000175000017500000004204615211535620026021 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatter.h" #include "tl_ucp_sendrecv.h" #include "core/ucc_progress_queue.h" #include "components/mc/ucc_mc.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "utils/ucc_dt_reduce.h" #include "utils/ucc_atomic.h" #define REVERSED_FRAG 1 static inline void send_completion_common(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in rs ring completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } ucc_atomic_add32(&task->tagged.send_completed, 1); if (request) { ucp_request_free(request); } } static void send_completion_1(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; task->reduce_scatter_ring.s_scratch_busy[0] = 0; send_completion_common(request, status, user_data); } static void send_completion_2(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; task->reduce_scatter_ring.s_scratch_busy[1] = 0; send_completion_common(request, status, user_data); } static inline void ucc_ring_frag_count(ucc_tl_ucp_task_t *task, size_t count, ucc_rank_t block, size_t *frag_count) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); size_t size = UCC_TL_TEAM_SIZE(team); int n_frags, frag; size_t block_count; n_frags = task->reduce_scatter_ring.n_frags; frag = task->reduce_scatter_ring.frag; block_count = ucc_buffer_block_count(count, size, block); *frag_count = ucc_buffer_block_count(block_count, n_frags, frag); } static inline void ucc_ring_frag_block_offset(ucc_tl_ucp_task_t *task, size_t count, ucc_rank_t block, size_t *block_offset, size_t *frag_offset) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); size_t size = UCC_TL_TEAM_SIZE(team); int n_frags, frag; size_t block_count; n_frags = task->reduce_scatter_ring.n_frags; frag = task->reduce_scatter_ring.frag; block_count = ucc_buffer_block_count(count, size, block); *frag_offset = ucc_buffer_block_offset(block_count, n_frags, frag); *block_offset = ucc_buffer_block_offset(count, size, block); } static void ucc_tl_ucp_reduce_scatter_ring_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t size = task->subset.map.ep_num; ucc_rank_t rank = task->subset.myrank; void *sbuf = args->src.info.buffer; ucc_memory_type_t mem_type = args->dst.info.mem_type; size_t count = args->dst.info.count * size; ucc_datatype_t dt = args->dst.info.datatype; size_t dt_size = ucc_dt_size(dt); ucc_rank_t sendto = (rank + 1) % size; ucc_rank_t recvfrom = (rank - 1 + size) % size; ucp_send_nbx_callback_t cb[2] = {send_completion_1, send_completion_2}; ucc_rank_t prevblock, recv_data_from; ucc_status_t status; size_t max_block_size, block_offset, frag_count, frag_offset, final_offset; int step, is_avg, id; void *r_scratch, *s_scratch[2], *reduce_target; volatile char *busy; final_offset = 0; if (UCC_IS_INPLACE(*args)) { sbuf = args->dst.info.buffer; count /= size; final_offset = ucc_buffer_block_offset(count, size, UCC_TL_TEAM_RANK(team)); } sendto = ucc_ep_map_eval(task->reduce_scatter_ring.inv_map, sendto); recvfrom = ucc_ep_map_eval(task->reduce_scatter_ring.inv_map, recvfrom); if (team->cfg.use_reordering) { sendto = ucc_ep_map_eval(task->subset.map, sendto); recvfrom = ucc_ep_map_eval(task->subset.map, recvfrom); } max_block_size = task->reduce_scatter_ring.max_block_count * dt_size; busy = task->reduce_scatter_ring.s_scratch_busy; r_scratch = task->reduce_scatter_ring.scratch; s_scratch[0] = PTR_OFFSET(r_scratch, max_block_size); s_scratch[1] = PTR_OFFSET(s_scratch[0], max_block_size); if (UCC_INPROGRESS == ucc_tl_ucp_test_ring(task)) { return; } while (task->tagged.recv_posted > 0) { /* always have at least 1 send completion, ie 1 free slot */ ucc_assert(!busy[0] || !busy[1]); id = busy[0] ? 1 : 0; reduce_target = s_scratch[id]; step = task->tagged.send_posted; prevblock = (rank - 1 - step + size) % size; prevblock = ucc_ep_map_eval(task->reduce_scatter_ring.inv_map, prevblock); if (team->cfg.use_reordering) { prevblock = ucc_ep_map_eval(task->subset.map, prevblock); } /* reduction */ ucc_assert(task->tagged.recv_posted == task->tagged.recv_completed); ucc_assert(task->tagged.recv_posted < size); ucc_ring_frag_count(task, count, prevblock, &frag_count); ucc_ring_frag_block_offset(task, count, prevblock, &block_offset, &frag_offset); if (task->tagged.recv_completed == size - 1) { reduce_target = PTR_OFFSET(args->dst.info.buffer, (frag_offset + final_offset) * dt_size); } is_avg = (args->op == UCC_OP_AVG) && (task->tagged.recv_completed == (size - 1)); if (UCC_OK != (status = ucc_dt_reduce( r_scratch, PTR_OFFSET(sbuf, (block_offset + frag_offset) * dt_size), reduce_target, frag_count, dt, args, is_avg ? UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA : 0, AVG_ALPHA(task), task->reduce_scatter_ring.executor, &task->reduce_scatter_ring.etask))) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); task->super.status = status; return; } EXEC_TASK_WAIT(task->reduce_scatter_ring.etask); if (task->tagged.recv_completed == size - 1) { task->tagged.recv_posted = task->tagged.recv_completed = 0; break; } ucc_assert(task->tagged.send_posted - task->tagged.send_completed <= 1); ucc_assert(task->tagged.send_posted < size); busy[id] = 1; UCPCHECK_GOTO(ucc_tl_ucp_send_cb(reduce_target, frag_count * dt_size, mem_type, sendto, team, task, cb[id], (void *)task), task, out); recv_data_from = (rank - 2 - step + size) % size; recv_data_from = ucc_ep_map_eval(task->reduce_scatter_ring.inv_map, recv_data_from); if (team->cfg.use_reordering) { recv_data_from = ucc_ep_map_eval(task->subset.map, recv_data_from); } ucc_ring_frag_count(task, count, recv_data_from, &frag_count); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(r_scratch, frag_count * dt_size, mem_type, recvfrom, team, task), task, out); if (UCC_INPROGRESS == ucc_tl_ucp_test_ring(task)) { return; } } if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } task->super.status = UCC_OK; out: return; } static ucc_status_t ucc_tl_ucp_reduce_scatter_ring_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t * args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t size = task->subset.map.ep_num; ucc_rank_t rank = task->subset.myrank; size_t count = args->dst.info.count * size; ucc_datatype_t dt = args->dst.info.datatype; size_t dt_size = ucc_dt_size(dt); ucc_memory_type_t mem_type = args->dst.info.mem_type; void * sbuf = args->src.info.buffer; int step = 0; ucc_rank_t sendto = (rank + 1) % size; ucc_rank_t recvfrom = (rank - 1 + size) % size; ucc_rank_t recv_block = (rank - 2 - step + size) % size; ucc_rank_t send_block = (rank - 1 - step + size) % size; size_t block_offset, frag_count, frag_offset; void *r_scratch; ucc_status_t status; ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if (UCC_IS_INPLACE(*args)) { sbuf = args->dst.info.buffer; count /= size; } status = ucc_coll_task_get_executor(&task->super, &task->reduce_scatter_ring.executor); if (ucc_unlikely(status != UCC_OK)) { return status; } r_scratch = task->reduce_scatter_ring.scratch; sendto = ucc_ep_map_eval(task->reduce_scatter_ring.inv_map, sendto); recvfrom = ucc_ep_map_eval(task->reduce_scatter_ring.inv_map, recvfrom); recv_block = ucc_ep_map_eval(task->reduce_scatter_ring.inv_map, recv_block); send_block = ucc_ep_map_eval(task->reduce_scatter_ring.inv_map, send_block); if (team->cfg.use_reordering) { sendto = ucc_ep_map_eval(task->subset.map, sendto); recvfrom = ucc_ep_map_eval(task->subset.map, recvfrom); recv_block = ucc_ep_map_eval(task->subset.map, recv_block); send_block = ucc_ep_map_eval(task->subset.map, send_block); } ucc_ring_frag_count(task, count, recv_block, &frag_count); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(r_scratch, frag_count * dt_size, mem_type, recvfrom, team, task), task, out); ucc_ring_frag_count(task, count, send_block, &frag_count); ucc_ring_frag_block_offset(task, count, send_block, &block_offset, &frag_offset); UCPCHECK_GOTO(ucc_tl_ucp_send_nb( PTR_OFFSET(sbuf, (block_offset + frag_offset) * dt_size), frag_count * dt_size, mem_type, sendto, team, task), task, out); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); out: return task->super.status; } static ucc_status_t ucc_tl_ucp_reduce_scatter_ring_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); if (task->reduce_scatter_ring.frag == REVERSED_FRAG) { ucc_ep_map_destroy(&task->reduce_scatter_ring.inv_map); } return ucc_tl_ucp_coll_finalize(coll_task); } static ucc_status_t ucc_tl_ucp_reduce_scatter_ring_init_subset( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, ucc_subset_t *subsets, int n_frags, int frag, void *scratch, size_t max_block_count) { ucc_tl_ucp_task_t *task; ucc_tl_ucp_team_t *tl_team; ucc_status_t status; task = ucc_tl_ucp_init_task(coll_args, team); tl_team = TASK_TEAM(task); task->super.post = ucc_tl_ucp_reduce_scatter_ring_start; task->super.progress = ucc_tl_ucp_reduce_scatter_ring_progress; task->super.finalize = ucc_tl_ucp_reduce_scatter_ring_finalize; task->subset.map = subsets[frag].map; task->subset.myrank = subsets[frag].myrank; if (frag == REVERSED_FRAG) { if (tl_team->cfg.use_reordering) { task->subset.map = subsets[0].map; } status = ucc_ep_map_create_inverse(subsets[frag].map, &task->reduce_scatter_ring.inv_map, frag && tl_team->cfg.use_reordering); if (UCC_OK != status) { return status; } } else { task->reduce_scatter_ring.inv_map.type = UCC_EP_MAP_FULL; task->reduce_scatter_ring.inv_map.ep_num = task->subset.map.ep_num; } task->reduce_scatter_ring.n_frags = n_frags; task->reduce_scatter_ring.frag = frag; task->reduce_scatter_ring.scratch = scratch; task->reduce_scatter_ring.max_block_count = max_block_count; task->reduce_scatter_ring.s_scratch_busy[0] = 0; task->reduce_scatter_ring.s_scratch_busy[1] = 0; *task_h = &task->super; return UCC_OK; } static ucc_status_t ucc_tl_ucp_reduce_scatter_ring_sched_post(ucc_coll_task_t *coll_task) { return ucc_schedule_start(coll_task); } static ucc_status_t ucc_tl_ucp_reduce_scatter_ring_sched_finalize(ucc_coll_task_t *task) { ucc_tl_ucp_schedule_t *schedule = ucc_derived_of(task, ucc_tl_ucp_schedule_t); ucc_status_t status; ucc_mc_free(schedule->scratch_mc_header); status = ucc_schedule_finalize(task); ucc_tl_ucp_put_schedule(&schedule->super.super); return status; } ucc_status_t ucc_tl_ucp_reduce_scatter_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t size = UCC_TL_TEAM_SIZE(tl_team); size_t count = coll_args->args.dst.info.count; ucc_datatype_t dt = coll_args->args.dst.info.datatype; size_t dt_size = ucc_dt_size(dt); ucc_memory_type_t mem_type = coll_args->args.dst.info.mem_type; int bidir = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.reduce_scatter_ring_bidirectional; size_t to_alloc_per_set, max_segcount, count_per_set; ucc_tl_ucp_schedule_t *tl_schedule; ucc_schedule_t *schedule; ucc_coll_task_t *ctask; ucc_sbgp_t *sbgp; ucc_status_t status; ucc_subset_t s[2]; int i, n_subsets; if (UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.reduce_avg_pre_op && coll_args->args.op == UCC_OP_AVG) { return UCC_ERR_NOT_SUPPORTED; } if (!UCC_IS_INPLACE(coll_args->args)) { count *= size; } status = ucc_tl_ucp_get_schedule(tl_team, coll_args, &tl_schedule); if (ucc_unlikely(UCC_OK != status)) { return status; } schedule = &tl_schedule->super.super; /* if count == size then we have 1 elem per rank, not enough to split into 2 sets */ n_subsets = (bidir && (count > size)) ? 2 : 1; if (tl_team->cfg.use_reordering) { sbgp = ucc_topo_get_sbgp(tl_team->topo, UCC_SBGP_FULL_HOST_ORDERED); s[0].myrank = sbgp->group_rank; s[0].map = sbgp->map; } else { s[0].myrank = UCC_TL_TEAM_RANK(tl_team); s[0].map.type = UCC_EP_MAP_FULL; s[0].map.ep_num = UCC_TL_TEAM_SIZE(tl_team); } s[1].map = ucc_ep_map_create_reverse(UCC_TL_TEAM_SIZE(tl_team)); s[1].myrank = ucc_ep_map_eval(s[1].map, s[0].myrank); count_per_set = (count + n_subsets - 1) / n_subsets; max_segcount = ucc_buffer_block_count(count_per_set, size, 0); /* in flight we can have 2 sends from 2 differnt blocks and 1 recv: need 3 * max_segcount of scratch per set */ to_alloc_per_set = max_segcount * 3; UCC_CHECK_GOTO(ucc_mc_alloc(&tl_schedule->scratch_mc_header, to_alloc_per_set * dt_size * n_subsets, mem_type), out, status); for (i = 0; i < n_subsets; i++) { UCC_CHECK_GOTO(ucc_tl_ucp_reduce_scatter_ring_init_subset( coll_args, team, &ctask, s, n_subsets, i, PTR_OFFSET(tl_schedule->scratch_mc_header->addr, to_alloc_per_set * i * dt_size), max_segcount), out_free, status); ctask->n_deps = 1; UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, ctask), out_free, status); UCC_CHECK_GOTO(ucc_event_manager_subscribe( &schedule->super, UCC_EVENT_SCHEDULE_STARTED, ctask, ucc_task_start_handler), out_free, status); } schedule->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; schedule->super.post = ucc_tl_ucp_reduce_scatter_ring_sched_post; schedule->super.finalize = ucc_tl_ucp_reduce_scatter_ring_sched_finalize; *task_h = &schedule->super; return UCC_OK; out_free: ucc_mc_free(tl_schedule->scratch_mc_header); out: ucc_tl_ucp_put_schedule(schedule); return status; } ucc-1.8.0/src/components/tl/ucp/allgather/0000775000175000017500000000000015211535620020603 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/allgather/allgather.c0000664000175000017500000001023315211535620022711 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "allgather.h" #include "../../../utils/ucc_string.h" #include "../../../utils/ucc_log.h" #include "../../../utils/ucc_coll_utils.h" #define ALLGATHER_MAX_PATTERN_SIZE \ (sizeof(UCC_TL_UCP_ALLGATHER_DEFAULT_ALG_SELECT_STR_1PPN) * 2) ucc_base_coll_alg_info_t ucc_tl_ucp_allgather_algs[UCC_TL_UCP_ALLGATHER_ALG_LAST + 1] = { [UCC_TL_UCP_ALLGATHER_ALG_KNOMIAL] = {.id = UCC_TL_UCP_ALLGATHER_ALG_KNOMIAL, .name = "knomial", .desc = "recursive k-ing with arbitrary radix"}, [UCC_TL_UCP_ALLGATHER_ALG_RING] = {.id = UCC_TL_UCP_ALLGATHER_ALG_RING, .name = "ring", .desc = "O(N) Ring"}, [UCC_TL_UCP_ALLGATHER_ALG_NEIGHBOR] = {.id = UCC_TL_UCP_ALLGATHER_ALG_NEIGHBOR, .name = "neighbor", .desc = "O(N) Neighbor Exchange N/2 steps"}, [UCC_TL_UCP_ALLGATHER_ALG_BRUCK] = {.id = UCC_TL_UCP_ALLGATHER_ALG_BRUCK, .name = "bruck", .desc = "O(log(N)) Variation of Bruck algorithm"}, [UCC_TL_UCP_ALLGATHER_ALG_SPARBIT] = {.id = UCC_TL_UCP_ALLGATHER_ALG_SPARBIT, .name = "sparbit", .desc = "O(log(N)) SPARBIT algorithm"}, [UCC_TL_UCP_ALLGATHER_ALG_LINEAR] = {.id = UCC_TL_UCP_ALLGATHER_ALG_LINEAR, .name = "linear", .desc = "O(N - 1) Linear algorithm, one-shot"}, [UCC_TL_UCP_ALLGATHER_ALG_LINEAR_BATCHED] = {.id = UCC_TL_UCP_ALLGATHER_ALG_LINEAR_BATCHED, .name = "batched", .desc = "O(N - 1) Linear algorithm, K-send/receive in flight"}, [UCC_TL_UCP_ALLGATHER_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc_status_t ucc_tl_ucp_allgather_init(ucc_tl_ucp_task_t *task) { return ucc_tl_ucp_allgather_ring_init_common(task); } char *ucc_tl_ucp_allgather_score_str_get(ucc_tl_ucp_team_t *team) { int max_size = ALLGATHER_MAX_PATTERN_SIZE; int algo_num = UCC_TL_TEAM_SIZE(team) % 2 ? UCC_TL_UCP_ALLGATHER_ALG_RING : UCC_TL_UCP_ALLGATHER_ALG_NEIGHBOR; char * str = ucc_malloc(max_size * sizeof(char)); ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(team); uint64_t cuda_types = ctx->ucp_memory_types & (UCC_BIT(UCC_MEMORY_TYPE_CUDA) | UCC_BIT(UCC_MEMORY_TYPE_CUDA_MANAGED)); uint64_t non_cuda_types = ctx->ucp_memory_types & (~cuda_types); ucc_sbgp_t *sbgp; char * non_cuda_str; char * cuda_str; if (team->cfg.use_reordering) { sbgp = ucc_topo_get_sbgp(team->topo, UCC_SBGP_FULL_HOST_ORDERED); if (!ucc_ep_map_is_identity(&sbgp->map)) { algo_num = UCC_TL_UCP_ALLGATHER_ALG_RING; } } if (team->topo && ucc_topo_is_single_ppn(team->topo)) { if (cuda_types) { cuda_str = ucc_malloc(max_size * sizeof(char)); ucc_mtype_map_to_str(cuda_types, ",", cuda_str, max_size); if (non_cuda_types) { non_cuda_str = ucc_malloc(max_size * sizeof(char)); ucc_mtype_map_to_str(non_cuda_types, ",", non_cuda_str, max_size); ucc_snprintf_safe( str, max_size, UCC_TL_UCP_ALLGATHER_DEFAULT_ALG_SELECT_STR_1PPN, cuda_str, non_cuda_str, algo_num); ucc_free(cuda_str); ucc_free(non_cuda_str); return str; } ucc_snprintf_safe( str, max_size, UCC_TL_UCP_ALLGATHER_DEFAULT_ALG_SELECT_STR_1PPN_CUDA, cuda_str, algo_num); ucc_free(cuda_str); return str; } } ucc_snprintf_safe(str, max_size, UCC_TL_UCP_ALLGATHER_DEFAULT_ALG_SELECT_STR, algo_num); return str; } ucc-1.8.0/src/components/tl/ucp/allgather/allgather_neighbor.c0000664000175000017500000001433115211535620024571 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "allgather.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "components/mc/ucc_mc.h" static ucc_rank_t get_recv_from_rank(ucc_rank_t rank, ucc_rank_t size, int i) { const int i_parity = i % 2; int offset_at_step[2]; ucc_rank_t recv_data_from; if (rank % 2) { recv_data_from = (rank - 1 + size) % size; offset_at_step[0] = (-2); offset_at_step[1] = (+2); } else { recv_data_from = rank; offset_at_step[0] = (+2); offset_at_step[1] = (-2); } return (recv_data_from + offset_at_step[i_parity] * ucc_div_round_up(i, 2) + size) % size; } ucc_status_t ucc_tl_ucp_allgather_neighbor_start(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_ucp_allgather_neighbor_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_status_t status = UCC_OK; ucc_tl_ucp_task_t *task; ucc_tl_ucp_team_t *ucp_team; task = ucc_tl_ucp_init_task(coll_args, team); ucp_team = TASK_TEAM(task); if (!ucc_coll_args_is_predefined_dt(&TASK_ARGS(task), UCC_RANK_INVALID)) { tl_error(UCC_TASK_LIB(task), "user defined datatype is not supported"); status = UCC_ERR_NOT_SUPPORTED; goto out; } if (UCC_TL_TEAM_SIZE(ucp_team) % 2) { tl_debug(UCC_TASK_LIB(task), "odd team size is not supported, switching to ring"); status = ucc_tl_ucp_allgather_ring_init_common(task); } else { task->super.post = ucc_tl_ucp_allgather_neighbor_start; task->super.progress = ucc_tl_ucp_allgather_neighbor_progress; } out: if (status != UCC_OK) { ucc_tl_ucp_put_task(task); return status; } *task_h = &task->super; return status; } /* Original implementation: https://github.com/open-mpi/ompi/blob/main/ompi/mca/coll/base/coll_base_allgather.c */ void ucc_tl_ucp_allgather_neighbor_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); void *rbuf = TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; size_t count = TASK_ARGS(task).dst.info.count; size_t data_size = (count / tsize) * ucc_dt_size(dt); ucc_rank_t neighbors[2], i; int i_parity, even_rank; void *tmprecv, *tmpsend; if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } even_rank = !(trank % 2); if (even_rank) { neighbors[0] = (trank + 1) % tsize; neighbors[1] = (trank - 1 + tsize) % tsize; } else { neighbors[0] = (trank - 1 + tsize) % tsize; neighbors[1] = (trank + 1) % tsize; } while (task->tagged.send_posted < (tsize / 2)) { i = task->tagged.send_posted; i_parity = i % 2; tmprecv = PTR_OFFSET(rbuf, get_recv_from_rank(trank, tsize, i) * data_size); tmpsend = PTR_OFFSET(rbuf, get_recv_from_rank(trank, tsize, i - 1) * data_size); /* Sendreceive */ UCPCHECK_GOTO(ucc_tl_ucp_send_nb(tmpsend, 2 * data_size, rmem, neighbors[i_parity], team, task), task, out); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(tmprecv, 2 * data_size, rmem, neighbors[i_parity], team, task), task, out); if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } } ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; out: UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_neighbor_done", 0); } ucc_status_t ucc_tl_ucp_allgather_neighbor_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); size_t count = TASK_ARGS(task).dst.info.count; void *sbuf = TASK_ARGS(task).src.info.buffer; void *rbuf = TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t smem = TASK_ARGS(task).src.info.mem_type; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); size_t data_size = (count / tsize) * ucc_dt_size(dt); ucc_status_t status; ucc_rank_t neighbor; void *tmprecv, *tmpsend; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_neighbor_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if (!UCC_IS_INPLACE(TASK_ARGS(task))) { status = ucc_mc_memcpy(PTR_OFFSET(rbuf, data_size * trank), sbuf, data_size, rmem, smem); if (ucc_unlikely(UCC_OK != status)) { return status; } } if (trank % 2) { neighbor = (trank - 1 + tsize) % tsize; } else { neighbor = (trank + 1) % tsize; } tmprecv = PTR_OFFSET(rbuf, neighbor * data_size); tmpsend = PTR_OFFSET(rbuf, trank * data_size); /* Sendreceive */ UCPCHECK_GOTO( ucc_tl_ucp_send_nb(tmpsend, data_size, rmem, neighbor, team, task), task, out); UCPCHECK_GOTO( ucc_tl_ucp_recv_nb(tmprecv, data_size, rmem, neighbor, team, task), task, out); out: return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc-1.8.0/src/components/tl/ucp/allgather/allgather_sparbit.c0000664000175000017500000001357315211535620024447 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "allgather.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "components/mc/ucc_mc.h" static void ucc_tl_ucp_allgather_sparbit_progress(ucc_coll_task_t *task); static ucc_status_t ucc_tl_ucp_allgather_sparbit_start(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_allgather_sparbit_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_task_t *task = ucc_tl_ucp_init_task(coll_args, team); if (!ucc_coll_args_is_predefined_dt(&TASK_ARGS(task), UCC_RANK_INVALID)) { tl_error(UCC_TASK_LIB(task), "user defined datatype is not supported"); ucc_tl_ucp_put_task(task); return UCC_ERR_NOT_SUPPORTED; } tl_trace(UCC_TASK_LIB(task), "ucc_tl_ucp_allgather_sparbit_init"); task->super.post = ucc_tl_ucp_allgather_sparbit_start; task->super.progress = ucc_tl_ucp_allgather_sparbit_progress; *task_h = &task->super; return UCC_OK; } /* Inspired by implementation: https://github.com/open-mpi/ompi/blob/main/ompi/mca/coll/base/coll_base_allgather.c */ void ucc_tl_ucp_allgather_sparbit_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); void *rbuf = TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; size_t count = TASK_ARGS(task).dst.info.count; size_t data_size = (count / tsize) * ucc_dt_size(dt); uint32_t i = task->allgather_sparbit.i; // restore iteration number uint32_t tsize_log = ucc_ilog2_ceil(tsize); ucc_rank_t recvfrom, sendto, distance; uint32_t last_ignore, ignore_steps, data_expected, transfer_count; uint32_t exclusion; void *tmprecv, *tmpsend; // here we can't make any progress while transfers from previous step are running, emulation of wait all in async manner if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } last_ignore = __builtin_ctz(tsize); // count trailing zeros ignore_steps = (~((uint32_t)tsize >> last_ignore) | 1) << last_ignore; while (i < tsize_log) { data_expected = task->allgather_sparbit.data_expected; distance = 1 << (tsize_log - 1); distance >>= i; // restore distance in case of continuation depending on step recvfrom = (trank + tsize - distance) % tsize; sendto = (trank + distance) % tsize; exclusion = (distance & ignore_steps) == distance; for (transfer_count = 0; transfer_count < data_expected - exclusion; transfer_count++) { tmprecv = PTR_OFFSET( rbuf, (trank - (2 * transfer_count + 1) * distance + tsize) % tsize * data_size); tmpsend = PTR_OFFSET( rbuf, (trank - 2 * transfer_count * distance + tsize) % tsize * data_size); /* Sendreceive */ UCPCHECK_GOTO(ucc_tl_ucp_send_nb(tmpsend, data_size, rmem, sendto, team, task), task, out); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(tmprecv, data_size, rmem, recvfrom, team, task), task, out); } task->allgather_sparbit.data_expected = (data_expected << 1) - exclusion; task->allgather_sparbit.i++; // check if we could make one more step right now or we should yeld task if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } i = task->allgather_sparbit.i; } ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; out: UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_sparbit_done", 0); } ucc_status_t ucc_tl_ucp_allgather_sparbit_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); size_t count = TASK_ARGS(task).dst.info.count; void *sbuf = TASK_ARGS(task).src.info.buffer; void *rbuf = TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t smem = TASK_ARGS(task).src.info.mem_type; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); size_t data_size = (count / tsize) * ucc_dt_size(dt); ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_sparbit_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); task->allgather_sparbit.i = 0; // setup iteration task->allgather_sparbit.data_expected = 1; if (!UCC_IS_INPLACE(TASK_ARGS(task))) { status = ucc_mc_memcpy(PTR_OFFSET(rbuf, data_size * trank), sbuf, data_size, rmem, smem); if (ucc_unlikely(UCC_OK != status)) { return status; } } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc-1.8.0/src/components/tl/ucp/allgather/allgather.h0000664000175000017500000000760615211535620022730 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLGATHER_H_ #define ALLGATHER_H_ #include "../tl_ucp.h" #include "../tl_ucp_coll.h" enum { UCC_TL_UCP_ALLGATHER_ALG_KNOMIAL, UCC_TL_UCP_ALLGATHER_ALG_RING, UCC_TL_UCP_ALLGATHER_ALG_NEIGHBOR, UCC_TL_UCP_ALLGATHER_ALG_BRUCK, UCC_TL_UCP_ALLGATHER_ALG_SPARBIT, UCC_TL_UCP_ALLGATHER_ALG_LINEAR, UCC_TL_UCP_ALLGATHER_ALG_LINEAR_BATCHED, UCC_TL_UCP_ALLGATHER_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_allgather_algs[UCC_TL_UCP_ALLGATHER_ALG_LAST + 1]; #define UCC_TL_UCP_ALLGATHER_DEFAULT_ALG_SELECT_STR \ "allgather:0-4k:@0#allgather:4k-inf:@%d" #define UCC_TL_UCP_ALLGATHER_DEFAULT_ALG_SELECT_STR_1PPN \ "allgather:0-4k:@0#allgather:4k-inf:%s:@0#allgather:4k-inf:%s:@%d" #define UCC_TL_UCP_ALLGATHER_DEFAULT_ALG_SELECT_STR_1PPN_CUDA \ "allgather:0-4k:@0#allgather:4k-inf:%s:@0#allgather:4k-inf:@%d" char *ucc_tl_ucp_allgather_score_str_get(ucc_tl_ucp_team_t *team); static inline int ucc_tl_ucp_allgather_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_UCP_ALLGATHER_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_ucp_allgather_algs[i].name)) { break; } } return i; } ucc_status_t ucc_tl_ucp_allgather_init(ucc_tl_ucp_task_t *task); /* Ring */ ucc_status_t ucc_tl_ucp_allgather_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_ucp_allgather_ring_init_common(ucc_tl_ucp_task_t *task); void ucc_tl_ucp_allgather_ring_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_allgather_ring_start(ucc_coll_task_t *task); /* Neighbor Exchange */ ucc_status_t ucc_tl_ucp_allgather_neighbor_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); void ucc_tl_ucp_allgather_neighbor_progress(ucc_coll_task_t *task); /* Bruck */ ucc_status_t ucc_tl_ucp_allgather_bruck_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); void ucc_tl_ucp_allgather_bruck_progress(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_allgather_bruck_finalize(ucc_coll_task_t *coll_task); /* Sparbit */ ucc_status_t ucc_tl_ucp_allgather_sparbit_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); /* Linear One-Shot version of allgather */ ucc_status_t ucc_tl_ucp_allgather_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); void ucc_tl_ucp_allgather_linear_progress(ucc_coll_task_t *task); /* Linear Batched K-send/receive in flight */ ucc_status_t ucc_tl_ucp_allgather_linear_batched_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); /* Uses allgather_kn_radix from config */ ucc_status_t ucc_tl_ucp_allgather_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); /* Internal interface with custom radix */ ucc_status_t ucc_tl_ucp_allgather_knomial_init_r( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, ucc_kn_radix_t radix); #endif ucc-1.8.0/src/components/tl/ucp/allgather/allgather_ring.c0000664000175000017500000001377115211535620023742 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "allgather.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "components/mc/ucc_mc.h" static ucc_rank_t ucc_tl_ucp_allgather_ring_get_send_block(ucc_subset_t *subset, ucc_rank_t trank, ucc_rank_t tsize, int step) { return ucc_ep_map_eval(subset->map, (trank - step + tsize) % tsize); } static ucc_rank_t ucc_tl_ucp_allgather_ring_get_recv_block(ucc_subset_t *subset, ucc_rank_t trank, ucc_rank_t tsize, int step) { return ucc_ep_map_eval(subset->map, (trank - step - 1 + tsize) % tsize); } void ucc_tl_ucp_allgather_ring_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t trank = task->subset.myrank; ucc_rank_t tsize = (ucc_rank_t)task->subset.map.ep_num; void *rbuf = TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; size_t count = TASK_ARGS(task).dst.info.count; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; size_t data_size = (count / tsize) * ucc_dt_size(dt); ucc_rank_t sendto, recvfrom, sblock, rblock; int step; void *buf; if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } sendto = ucc_ep_map_eval(task->subset.map, (trank + 1) % tsize); recvfrom = ucc_ep_map_eval(task->subset.map, (trank - 1 + tsize) % tsize); while (task->tagged.send_posted < tsize - 1) { step = task->tagged.send_posted; sblock = task->allgather_ring.get_send_block(&task->subset, trank, tsize, step); rblock = task->allgather_ring.get_recv_block(&task->subset, trank, tsize, step); buf = PTR_OFFSET(rbuf, sblock * data_size); UCPCHECK_GOTO( ucc_tl_ucp_send_nb(buf, data_size, rmem, sendto, team, task), task, out); buf = PTR_OFFSET(rbuf, rblock * data_size); UCPCHECK_GOTO( ucc_tl_ucp_recv_nb(buf, data_size, rmem, recvfrom, team, task), task, out); if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } } ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; out: UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_ring_done", 0); } ucc_status_t ucc_tl_ucp_allgather_ring_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); size_t count = TASK_ARGS(task).dst.info.count; void *sbuf = TASK_ARGS(task).src.info.buffer; void *rbuf = TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t smem = TASK_ARGS(task).src.info.mem_type; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; ucc_rank_t trank = task->subset.myrank; ucc_rank_t tsize = (ucc_rank_t)task->subset.map.ep_num; size_t data_size = (count / tsize) * ucc_dt_size(dt); ucc_status_t status; ucc_rank_t block; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_ring_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if (!UCC_IS_INPLACE(TASK_ARGS(task))) { block = task->allgather_ring.get_send_block(&task->subset, trank, tsize, 0); status = ucc_mc_memcpy(PTR_OFFSET(rbuf, data_size * block), sbuf, data_size, rmem, smem); if (ucc_unlikely(UCC_OK != status)) { return status; } } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_allgather_ring_init_common(ucc_tl_ucp_task_t *task) { ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_sbgp_t *sbgp; if (!ucc_coll_args_is_predefined_dt(&TASK_ARGS(task), UCC_RANK_INVALID)) { tl_error(UCC_TASK_LIB(task), "user defined datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (!(task->flags & UCC_TL_UCP_TASK_FLAG_SUBSET)) { if (team->cfg.use_reordering) { sbgp = ucc_topo_get_sbgp(team->topo, UCC_SBGP_FULL_HOST_ORDERED); task->subset.myrank = sbgp->group_rank; task->subset.map = sbgp->map; } } task->allgather_ring.get_send_block = ucc_tl_ucp_allgather_ring_get_send_block; task->allgather_ring.get_recv_block = ucc_tl_ucp_allgather_ring_get_recv_block; task->super.post = ucc_tl_ucp_allgather_ring_start; task->super.progress = ucc_tl_ucp_allgather_ring_progress; return UCC_OK; } ucc_status_t ucc_tl_ucp_allgather_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h) { ucc_tl_ucp_task_t *task; ucc_status_t status; task = ucc_tl_ucp_init_task(coll_args, team); status = ucc_tl_ucp_allgather_ring_init_common(task); if (status != UCC_OK) { ucc_tl_ucp_put_task(task); return status; } *task_h = &task->super; return UCC_OK; } ucc-1.8.0/src/components/tl/ucp/allgather/allgather_bruck.c0000664000175000017500000002466515211535620024115 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "allgather.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "components/mc/ucc_mc.h" ucc_status_t ucc_tl_ucp_allgather_bruck_start(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_ucp_allgather_bruck_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task = ucc_tl_ucp_init_task(coll_args, team); ucc_status_t status = UCC_OK; ucc_rank_t trank = UCC_TL_TEAM_RANK(tl_team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(tl_team); ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; size_t count = TASK_ARGS(task).dst.info.count; size_t data_size = (count / tsize) * ucc_dt_size(dt); size_t scratch_size = (tsize - trank) * data_size; if (!ucc_coll_args_is_predefined_dt(&TASK_ARGS(task), UCC_RANK_INVALID)) { tl_error(UCC_TASK_LIB(task), "user defined datatype is not supported"); status = UCC_ERR_NOT_SUPPORTED; goto out; } tl_trace(UCC_TASK_LIB(task), "ucc_tl_ucp_allgather_bruck_init"); task->super.post = ucc_tl_ucp_allgather_bruck_start; task->super.progress = ucc_tl_ucp_allgather_bruck_progress; task->super.finalize = ucc_tl_ucp_allgather_bruck_finalize; /* allocate scratch buffer only on non root rank */ if (trank != 0) { if (UCC_MEMORY_TYPE_HOST != rmem) { scratch_size = tsize * data_size; } status = ucc_mc_alloc(&task->allgather_bruck.scratch_header, scratch_size, UCC_MEMORY_TYPE_HOST); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to allocate scratch buffer"); ucc_tl_ucp_coll_finalize(&task->super); goto out; } task->allgather_bruck.scratch_size = scratch_size; } else { task->allgather_bruck.scratch_header = NULL; task->allgather_bruck.scratch_size = 0; } out: if (status != UCC_OK) { ucc_tl_ucp_put_task(task); return status; } *task_h = &task->super; return status; } ucc_status_t ucc_tl_ucp_allgather_bruck_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_status_t global_status = UCC_OK; ucc_status_t status; tl_trace(UCC_TASK_LIB(task), "ucc_tl_ucp_allgather_bruck_finalize"); if (task->allgather_bruck.scratch_header != NULL) { /* deallocate scratch buffer */ global_status = ucc_mc_free(task->allgather_bruck.scratch_header); if (ucc_unlikely(global_status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to free scratch buffer memory"); } task->allgather_bruck.scratch_size = 0; } status = ucc_tl_ucp_coll_finalize(&task->super); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to finalize allgather bruck collective"); global_status = status; } return global_status; } /* Inspired by implementation: https://github.com/open-mpi/ompi/blob/main/ompi/mca/coll/base/coll_base_allgather.c */ void ucc_tl_ucp_allgather_bruck_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); void *rbuf = TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; size_t count = TASK_ARGS(task).dst.info.count; ucc_mc_buffer_header_t *scratch_header = task->allgather_bruck.scratch_header; size_t scratch_size = task->allgather_bruck.scratch_size; size_t data_size = (count / tsize) * ucc_dt_size(dt); ucc_rank_t recvfrom, sendto; ucc_status_t status; size_t blockcount, distance; void *tmprecv, *tmpsend; if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } /* On each step doubles distance */ distance = 1 << task->tagged.recv_posted; tmpsend = rbuf; while (distance < tsize) { recvfrom = (trank + distance) % tsize; sendto = (trank + tsize - distance) % tsize; tmprecv = PTR_OFFSET(tmpsend, distance * data_size); if (distance <= tsize >> 1) { blockcount = distance; } else { /* send-recv all reminder */ blockcount = tsize - distance; } /* Sendreceive */ UCPCHECK_GOTO(ucc_tl_ucp_send_nb(tmpsend, blockcount * data_size, rmem, sendto, team, task), task, out); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(tmprecv, blockcount * data_size, rmem, recvfrom, team, task), task, out); if (UCC_INPROGRESS == ucc_tl_ucp_test_recv(task)) { return; } distance = 1 << task->tagged.recv_posted; } if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } /* post processing step */ if (trank != 0) { if (UCC_MEMORY_TYPE_HOST == rmem) { // copy blocks [0 .. (size - rank - 1)] from rbuf to shift buffer status = ucc_mc_memcpy(scratch_header->addr, rbuf, scratch_size, UCC_MEMORY_TYPE_HOST, rmem); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to copy data to scratch buffer"); task->super.status = status; return; } // move blocks [(size - rank) .. size] from rbuf to beginning of rbuf // TODO: rewrite to cycle to get rid of overlap memmove(rbuf, PTR_OFFSET(rbuf, scratch_size), trank * data_size); // copy blocks from shift buffer starting at block [rank] in rbuf. status = ucc_mc_memcpy(PTR_OFFSET(rbuf, trank * data_size), scratch_header->addr, scratch_size, rmem, UCC_MEMORY_TYPE_HOST); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to copy data from scratch to rbuff buffer"); task->super.status = status; return; } } else { /* In case of non host memory we perform two copy to host buffer and then back to device, 3 memcopy in total */ /* TODO: replace with generic kernel to do bruck post step in sinle launch on device */ status = ucc_mc_memcpy( PTR_OFFSET(scratch_header->addr, trank * data_size), rbuf, (tsize - trank) * data_size, UCC_MEMORY_TYPE_HOST, rmem); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to copy first data part to scratch buffer"); task->super.status = status; return; } status = ucc_mc_memcpy(scratch_header->addr, PTR_OFFSET(rbuf, (tsize - trank) * data_size), trank * data_size, UCC_MEMORY_TYPE_HOST, rmem); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to copy second data part to scratch buffer"); task->super.status = status; return; } status = ucc_mc_memcpy(rbuf, scratch_header->addr, tsize * data_size, rmem, UCC_MEMORY_TYPE_HOST); if (ucc_unlikely(status != UCC_OK)) { tl_error(UCC_TASK_LIB(task), "failed to copy from scratch buffer to dst"); task->super.status = status; return; } } } ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; out: UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_bruck_done", 0); } ucc_status_t ucc_tl_ucp_allgather_bruck_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); size_t count = TASK_ARGS(task).dst.info.count; void *sbuf = TASK_ARGS(task).src.info.buffer; void *rbuf = TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t smem = TASK_ARGS(task).src.info.mem_type; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); size_t data_size = (count / tsize) * ucc_dt_size(dt); ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_bruck_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); /* initial step: copy data on non root ranks to the beginning of buffer */ if (!UCC_IS_INPLACE(TASK_ARGS(task))) { // not inplace: copy chunk from source buff to beginning of receive status = ucc_mc_memcpy(rbuf, sbuf, data_size, rmem, smem); if (ucc_unlikely(UCC_OK != status)) { return status; } } else if (trank != 0) { // inplace: copy chunk to the begin status = ucc_mc_memcpy(rbuf, PTR_OFFSET(rbuf, data_size * trank), data_size, rmem, rmem); if (ucc_unlikely(UCC_OK != status)) { return status; } } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc-1.8.0/src/components/tl/ucp/allgather/allgather_knomial.c0000664000175000017500000004244415211535620024434 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "tl_ucp_coll.h" #include "tl_ucp_sendrecv.h" #include "tl_ucp_copy.h" #include "core/ucc_progress_queue.h" #include "coll_patterns/sra_knomial.h" #include "tl_ucp_task.h" #include "ucc/api/ucc.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include #define SAVE_STATE(_phase) \ do { \ task->allgather_kn.phase = _phase; \ } while (0) #define GET_LOCAL_COUNT(_args, _size, _rank) \ ((_args)->coll_type == UCC_COLL_TYPE_ALLGATHERV) \ ? ucc_coll_args_get_count((_args), (_args)->dst.info_v.counts, \ (_rank)) \ : (_args)->dst.info.count / (_size) #define GET_TOTAL_COUNT(_args, _size) \ ((_args)->coll_type == UCC_COLL_TYPE_ALLGATHERV) \ ? ucc_coll_args_get_total_count((_args), (_args)->dst.info_v.counts, \ (_size)) \ : (_args)->dst.info.count #define GET_DT(_args) \ ((_args)->coll_type == UCC_COLL_TYPE_ALLGATHERV) \ ? (_args)->dst.info_v.datatype \ : (_args)->dst.info.datatype #define GET_DST(_args) \ ((_args)->coll_type == UCC_COLL_TYPE_ALLGATHERV) \ ? (_args)->dst.info_v.buffer \ : (_args)->dst.info.buffer #define GET_MT(_args) \ ((_args)->coll_type == UCC_COLL_TYPE_ALLGATHERV) \ ? (_args)->dst.info_v.mem_type \ : (_args)->dst.info.mem_type /* Bcast will first call scatter and then allgather. * In case of non-full tree with "extra" ranks, scatter will give each rank * a new virtual rank number - "vrank". * As such allgather must keep to this ranking to be aligned with scatter. */ void ucc_tl_ucp_allgather_knomial_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(team); ucc_kn_radix_t radix = task->allgather_kn.p.radix; uint8_t node_type = task->allgather_kn.p.node_type; ucc_knomial_pattern_t *p = &task->allgather_kn.p; void *rbuf = GET_DST(args); ucc_memory_type_t mem_type = GET_MT(args); size_t dt_size = ucc_dt_size(GET_DT(args)); ucc_rank_t size = task->subset.map.ep_num; size_t data_size = GET_TOTAL_COUNT(args, size); ucc_rank_t broot = args->coll_type == UCC_COLL_TYPE_BCAST ? args->root : 0; ucc_rank_t rank = VRANK(task->subset.myrank, broot, size); size_t local = GET_LOCAL_COUNT(args, size, rank); void *sbuf; ptrdiff_t peer_seg_offset, local_seg_offset; ucc_rank_t peer, peer_dist; ucc_kn_radix_t loop_step; size_t peer_seg_count, local_seg_count; ucc_status_t status; size_t extra_count; ucp_request_param_t send_param, recv_param; send_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_MEMORY_TYPE | UCP_OP_ATTR_FIELD_USER_DATA; send_param.memory_type = ucc_memtype_to_ucs[mem_type]; send_param.cb.send = TASK_CTX(task)->sendrecv_cbs.send_cb; send_param.user_data = task; recv_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_MEMORY_TYPE | UCP_OP_ATTR_FIELD_USER_DATA; recv_param.memory_type = ucc_memtype_to_ucs[mem_type]; recv_param.cb.recv = TASK_CTX(task)->sendrecv_cbs.recv_cb; recv_param.user_data = task; if (task->allgather_kn.dst_memh) { recv_param.op_attr_mask |= UCP_OP_ATTR_FIELD_MEMH; recv_param.memh = task->allgather_kn.dst_memh; send_param.op_attr_mask |= UCP_OP_ATTR_FIELD_MEMH; send_param.memh = task->allgather_kn.dst_memh; } COPY_TASK_TEST(UCC_KN_PHASE_INIT, ctx, "failed during copy task test", task->allgather_kn.copy_task); UCC_KN_GOTO_PHASE(task->allgather_kn.phase); if (KN_NODE_EXTRA == node_type) { peer = ucc_knomial_pattern_get_proxy(p, rank); if (p->type != KN_PATTERN_ALLGATHERX) { UCPCHECK_GOTO(ucc_tl_ucp_send_nbx(task->allgather_kn.sbuf, local * dt_size, ucc_ep_map_eval(task->subset.map, INV_VRANK(peer, broot, size)), &send_param, task), task, out); } UCPCHECK_GOTO(ucc_tl_ucp_recv_nbx(rbuf, data_size * dt_size, ucc_ep_map_eval(task->subset.map, INV_VRANK(peer, broot, size)), &recv_param, task), task, out); } if ((p->type != KN_PATTERN_ALLGATHERX) && (node_type == KN_NODE_PROXY)) { peer = ucc_knomial_pattern_get_extra(p, rank); extra_count = coll_task->bargs.args.coll_type == UCC_COLL_TYPE_ALLGATHER ? local : ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); peer = ucc_ep_map_eval(task->subset.map, peer); UCPCHECK_GOTO(ucc_tl_ucp_recv_nbx(PTR_OFFSET(task->allgather_kn.sbuf, local * dt_size), extra_count * dt_size, peer, &recv_param, task), task, out); } UCC_KN_PHASE_EXTRA: if ((KN_NODE_EXTRA == node_type) || (KN_NODE_PROXY == node_type)) { if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_EXTRA); return; } if (KN_NODE_EXTRA == node_type) { goto out; } } while (!ucc_knomial_pattern_loop_done(p)) { ucc_kn_ag_pattern_peer_seg(rank, p, &local_seg_count, &local_seg_offset); sbuf = PTR_OFFSET(rbuf, local_seg_offset * dt_size); for (loop_step = radix - 1; loop_step > 0; loop_step--) { peer = ucc_knomial_pattern_get_loop_peer(p, rank, loop_step); if (peer == UCC_KN_PEER_NULL) continue; if (coll_task->bargs.args.coll_type == UCC_COLL_TYPE_BCAST) { peer_dist = ucc_knomial_calc_recv_dist(size - p->n_extra, ucc_knomial_pattern_loop_rank(p, peer), p->radix, 0); if (peer_dist < task->allgather_kn.recv_dist) { continue; } } UCPCHECK_GOTO(ucc_tl_ucp_send_nbx(sbuf, local_seg_count * dt_size, ucc_ep_map_eval(task->subset.map, INV_VRANK(peer, broot, size)), &send_param, task), task, out); } for (loop_step = 1; loop_step < radix; loop_step++) { peer = ucc_knomial_pattern_get_loop_peer(p, rank, loop_step); if (peer == UCC_KN_PEER_NULL) continue; ucc_kn_ag_pattern_peer_seg(peer, p, &peer_seg_count, &peer_seg_offset); if (coll_task->bargs.args.coll_type == UCC_COLL_TYPE_BCAST) { peer_dist = ucc_knomial_calc_recv_dist(size - p->n_extra, ucc_knomial_pattern_loop_rank(p, peer), p->radix, 0); if (peer_dist > task->allgather_kn.recv_dist) { continue; } } UCPCHECK_GOTO( ucc_tl_ucp_recv_nbx(PTR_OFFSET(rbuf, peer_seg_offset * dt_size), peer_seg_count * dt_size, ucc_ep_map_eval(task->subset.map, INV_VRANK(peer, broot, size)), &recv_param, task), task, out); } UCC_KN_PHASE_LOOP: if (UCC_INPROGRESS == ucc_tl_ucp_test_recv(task)) { SAVE_STATE(UCC_KN_PHASE_LOOP); return; } ucc_kn_ag_pattern_next_iter(p); } if (KN_NODE_PROXY == node_type) { peer = ucc_knomial_pattern_get_extra(p, rank); UCPCHECK_GOTO(ucc_tl_ucp_send_nbx(rbuf, data_size * dt_size, ucc_ep_map_eval(task->subset.map, INV_VRANK(peer, broot, size)), &send_param, task), task, out); } UCC_KN_PHASE_PROXY: if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { SAVE_STATE(UCC_KN_PHASE_PROXY); return; } out: ucc_assert(UCC_TL_UCP_TASK_P2P_COMPLETE(task)); task->super.status = UCC_OK; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_kn_done", 0); } ucc_status_t ucc_tl_ucp_allgather_knomial_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(team); ucc_coll_type_t ct = args->coll_type; ucc_rank_t size = task->subset.map.ep_num; ucc_kn_radix_t radix = task->allgather_kn.p.radix; ucc_knomial_pattern_t *p = &task->allgather_kn.p; ucc_rank_t rank = VRANK(task->subset.myrank, ct == UCC_COLL_TYPE_BCAST ? args->root : 0, size); ucc_status_t status; ptrdiff_t offset; void *rbuf; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_kn_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); task->allgather_kn.copy_task = NULL; task->allgather_kn.phase = UCC_KN_PHASE_INIT; if (ct == UCC_COLL_TYPE_ALLGATHER) { ucc_kn_ag_pattern_init(size, rank, radix, args->dst.info.count, &task->allgather_kn.p); offset = ucc_buffer_block_offset(args->dst.info.count, size, rank) * ucc_dt_size(args->dst.info.datatype); rbuf = args->dst.info.buffer; if (!UCC_IS_INPLACE(*args)) { status = ctx->copy.post(PTR_OFFSET(args->dst.info.buffer, offset), args->dst.info.mem_type, task->allgather_kn.dst_memh, args->src.info.buffer, args->src.info.mem_type, task->allgather_kn.src_memh, args->src.info.count * ucc_dt_size(args->src.info.datatype), task, &task->allgather_kn.copy_task); if (ucc_unlikely(status != UCC_OK)) { task->super.status = status; return status; } } } else if (ct == UCC_COLL_TYPE_ALLGATHERV) { ucc_kn_agv_pattern_init(size, rank, radix, args->dst.info_v.counts, UCC_COLL_ARGS_COUNT64(args), &task->allgather_kn.p); offset = ucc_buffer_vector_block_offset(args->dst.info_v.counts, UCC_COLL_ARGS_COUNT64(args), rank) * ucc_dt_size(args->dst.info_v.datatype); rbuf = args->dst.info_v.buffer; if (!UCC_IS_INPLACE(*args)) { status = ctx->copy.post(PTR_OFFSET(args->dst.info_v.buffer, offset), args->dst.info_v.mem_type, task->allgather_kn.dst_memh, args->src.info.buffer, args->src.info.mem_type, task->allgather_kn.src_memh, args->src.info.count * ucc_dt_size(args->src.info.datatype), task, &task->allgather_kn.copy_task); if (ucc_unlikely(status != UCC_OK)) { task->super.status = status; return status; } } } else { ucc_kn_agx_pattern_init(size, rank, radix, args->dst.info.count, &task->allgather_kn.p); offset = ucc_sra_kn_get_offset(args->dst.info.count, ucc_dt_size(args->dst.info.datatype), rank, size, radix); rbuf = args->dst.info.buffer; task->allgather_kn.recv_dist = ucc_knomial_calc_recv_dist( size - p->n_extra, ucc_knomial_pattern_loop_rank(p, rank), p->radix, 0); } task->allgather_kn.sbuf = PTR_OFFSET(rbuf, offset); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_ucp_allgather_knomial_init_r( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, ucc_kn_radix_t radix) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(tl_team); ucc_tl_ucp_task_t *task; ucc_sbgp_t *sbgp; ucc_status_t status; task = ucc_tl_ucp_init_task(coll_args, team); if (tl_team->cfg.use_reordering && coll_args->args.coll_type == UCC_COLL_TYPE_ALLREDUCE) { sbgp = ucc_topo_get_sbgp(tl_team->topo, UCC_SBGP_FULL_HOST_ORDERED); task->subset.myrank = sbgp->group_rank; task->subset.map = sbgp->map; } task->allgather_kn.p.radix = radix; if (!UCC_IS_INPLACE(coll_args->args)) { if (ctx->cfg.local_copy_type == UCC_TL_UCP_LOCAL_COPY_TYPE_EC) { task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; } } task->allgather_kn.dst_memh = NULL; task->allgather_kn.src_memh = NULL; if (coll_args->args.mask & UCC_COLL_ARGS_FIELD_MEM_MAP_DST_MEMH) { status = ucc_tl_ucp_get_memh(tl_team, coll_args->args.dst_memh.local_memh, (void**)&task->allgather_kn.dst_memh); if (ucc_unlikely(UCC_OK != status)) { tl_warn(TASK_LIB(task), "failed to get dst buffer memh"); } } if (!UCC_IS_INPLACE(coll_args->args) && (coll_args->args.mask & UCC_COLL_ARGS_FIELD_MEM_MAP_SRC_MEMH)) { status = ucc_tl_ucp_get_memh(tl_team, coll_args->args.src_memh.local_memh, (void**)&task->allgather_kn.src_memh); if (ucc_unlikely(UCC_OK != status)) { tl_warn(TASK_LIB(task), "failed to get src buffer memh"); } } task->super.post = ucc_tl_ucp_allgather_knomial_start; task->super.progress = ucc_tl_ucp_allgather_knomial_progress; *task_h = &task->super; return UCC_OK; } ucc_status_t ucc_tl_ucp_allgather_knomial_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_mrange_uint_t *p = &tl_team->cfg.allgather_kn_radix; ucc_rank_t tsize = UCC_TL_TEAM_SIZE(tl_team); ucc_memory_type_t mtype = GET_MT(&coll_args->args); size_t count = GET_TOTAL_COUNT(&coll_args->args, tsize); ucc_datatype_t dtype = GET_DT(&coll_args->args); ucc_kn_radix_t radix; radix = ucc_tl_ucp_get_knomial_radix(tl_team, count, dtype, mtype, p, 0); return ucc_tl_ucp_allgather_knomial_init_r(coll_args, team, task_h, radix); } ucc-1.8.0/src/components/tl/ucp/allgather/allgather_linear.c0000664000175000017500000001745315211535620024256 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "tl_ucp.h" #include "allgather.h" #include "core/ucc_progress_queue.h" #include "tl_ucp_sendrecv.h" #include "components/mc/ucc_mc.h" ucc_status_t ucc_tl_ucp_allgather_linear_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(team); size_t count = TASK_ARGS(task).dst.info.count; void *sbuf = TASK_ARGS(task).src.info.buffer; void *rbuf = TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t smem = TASK_ARGS(task).src.info.mem_type; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); size_t data_size = (count / tsize) * ucc_dt_size(dt); ucc_status_t status; UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_linear_start", 0); ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); task->allgather_linear.copy_task = NULL; /* Copy local data to the receive buffer if not in-place */ if (!UCC_IS_INPLACE(TASK_ARGS(task))) { status = ctx->copy.post(PTR_OFFSET(rbuf, data_size * trank), rmem, NULL, sbuf, smem, NULL, data_size, task, &task->allgather_linear.copy_task); if (ucc_unlikely(UCC_OK != status)) { return status; } } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } /* Get the number of requests in flight to be used for the allgather batched algorithm * If the number of requests is not specified, use the number of team size - 1 * If number of request is bigger than the team size - 1, use the team size - 1 */ static unsigned long get_num_reqs(const ucc_tl_ucp_team_t *team) { unsigned long reqs = UCC_TL_UCP_TEAM_LIB(team)->cfg.allgather_batched_num_posts; ucc_rank_t max_req = UCC_TL_TEAM_SIZE(team) - 1; reqs = (reqs > max_req || reqs == UCC_ULUNITS_AUTO) ? max_req : reqs; return reqs; } ucc_status_t ucc_tl_ucp_allgather_batched_init( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, unsigned long nreqs) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task = ucc_tl_ucp_init_task(coll_args, team); if (!ucc_coll_args_is_predefined_dt(&TASK_ARGS(task), UCC_RANK_INVALID)) { tl_error(UCC_TASK_LIB(task), "user defined datatype is not supported"); ucc_tl_ucp_put_task(task); return UCC_ERR_NOT_SUPPORTED; } if (!UCC_IS_INPLACE(coll_args->args)) { if (UCC_TL_UCP_TEAM_CTX(tl_team)->cfg.local_copy_type == UCC_TL_UCP_LOCAL_COPY_TYPE_EC) { task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; } } task->super.post = ucc_tl_ucp_allgather_linear_start; task->super.progress = ucc_tl_ucp_allgather_linear_progress; task->allgather_linear.nreqs = nreqs == 0 ? UCC_TL_TEAM_SIZE(tl_team) - 1 : nreqs; *task_h = &task->super; return UCC_OK; } /* Linear Batched K-send/receive in flight */ ucc_status_t ucc_tl_ucp_allgather_linear_batched_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { return ucc_tl_ucp_allgather_batched_init( coll_args, team, task_h, get_num_reqs(ucc_derived_of(team, ucc_tl_ucp_team_t))); } /* Linear One-Shot version of allgather */ ucc_status_t ucc_tl_ucp_allgather_linear_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { // 0 means one-shot, team size - 1 request will be used return ucc_tl_ucp_allgather_batched_init(coll_args, team, task_h, 0); } void ucc_tl_ucp_allgather_linear_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(team); ucc_rank_t trank = UCC_TL_TEAM_RANK(team); ucc_rank_t tsize = UCC_TL_TEAM_SIZE(team); void *rbuf = TASK_ARGS(task).dst.info.buffer; ucc_memory_type_t rmem = TASK_ARGS(task).dst.info.mem_type; ucc_datatype_t dt = TASK_ARGS(task).dst.info.datatype; size_t count = TASK_ARGS(task).dst.info.count; size_t data_size = (count / tsize) * ucc_dt_size(dt); int nreqs = task->allgather_linear.nreqs; int polls = 0; void *tmpsend = UCC_IS_INPLACE(TASK_ARGS(task)) ? PTR_OFFSET(rbuf, trank * data_size) : TASK_ARGS(task).src.info.buffer; ucc_memory_type_t smem = UCC_IS_INPLACE(TASK_ARGS(task)) ? rmem : TASK_ARGS(task).src.info.mem_type; void *tmprecv; ucc_rank_t peer; ucc_status_t status; while ((task->tagged.send_posted < tsize - 1 || task->tagged.recv_posted < tsize - 1) && (polls++ < task->n_polls)) { /* Progress UCP worker */ ucp_worker_progress(UCC_TL_UCP_TEAM_CTX(team)->worker.ucp_worker); /* Try to send data to clockwise peer */ while ((task->tagged.send_posted < tsize - 1) && ((task->tagged.send_posted - task->tagged.send_completed) < nreqs)) { peer = (trank + 1 + task->tagged.send_posted) % tsize; /* Send my data to peer */ UCPCHECK_GOTO( ucc_tl_ucp_send_nb(tmpsend, data_size, smem, peer, team, task), task, err); polls = 0; } /* Receive peer's data from counter-clockwise peer to avoid deadlock*/ while ((task->tagged.recv_posted < tsize - 1) && ((task->tagged.recv_posted - task->tagged.recv_completed) < nreqs)) { peer = (tsize + trank - 1 - task->tagged.recv_posted) % tsize; tmprecv = PTR_OFFSET(rbuf, peer * data_size); UCPCHECK_GOTO( ucc_tl_ucp_recv_nb(tmprecv, data_size, rmem, peer, team, task), task, err); polls = 0; } } if (task->tagged.send_posted < tsize - 1 || task->tagged.recv_posted < tsize - 1) { return; } task->super.status = ucc_tl_ucp_test(task); if (task->super.status != UCC_OK) { return; } /* Need to check copy task if it is not in-place */ if (task->allgather_linear.copy_task != NULL) { status = ctx->copy.test(ctx, task->allgather_linear.copy_task); if (status > 0) { task->super.status = UCC_INPROGRESS; return; } task->super.status = status; ctx->copy.finalize(task->allgather_linear.copy_task); task->allgather_linear.copy_task = NULL; } UCC_TL_UCP_PROFILE_REQUEST_EVENT(coll_task, "ucp_allgather_linear_done", 0); return; err: ucc_error("allgather linear progress failed with status %d: %s", task->super.status, ucc_status_string(task->super.status)); } ucc-1.8.0/src/components/tl/ucp/tl_ucp_team.c0000664000175000017500000003021015211535620021274 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "tl_ucp_ep.h" #include "tl_ucp_coll.h" #include "tl_ucp_sendrecv.h" #include "utils/ucc_parser.h" #include "coll_score/ucc_coll_score.h" #include "coll_patterns/ring.h" static inline ucc_status_t ucc_tl_ucp_get_topo(ucc_tl_ucp_team_t *team) { ucc_subset_t subset; ucc_status_t status; if (UCC_TL_IS_SERVICE_TEAM(team)) { return UCC_OK; } status = ucc_ep_map_create_nested(&UCC_TL_CORE_TEAM(team)->ctx_map, &UCC_TL_TEAM_MAP(team), &team->ctx_map); if (UCC_OK != status) { tl_error(UCC_TL_TEAM_LIB(team), "failed to create ctx map"); return status; } subset.map = team->ctx_map; subset.myrank = UCC_TL_TEAM_RANK(team); status = ucc_topo_init(subset, UCC_TL_CORE_CTX(team)->topo, &team->topo); if (UCC_OK != status) { tl_error(UCC_TL_TEAM_LIB(team), "failed to init team topo"); goto err_topo_init; } return UCC_OK; err_topo_init: ucc_ep_map_destroy_nested(&team->ctx_map); return status; } UCC_CLASS_INIT_FUNC(ucc_tl_ucp_team_t, ucc_base_context_t *tl_context, const ucc_base_team_params_t *params) { ucc_tl_ucp_context_t *ctx = ucc_derived_of(tl_context, ucc_tl_ucp_context_t); ucc_kn_radix_t max_radix, min_radix; ucc_rank_t tsize; ucc_status_t status; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_team_t, &ctx->super, params); /* TODO: init based on ctx settings and on params: need to check if all the necessary ranks mappings are provided */ self->preconnect_task = NULL; self->seq_num = 0; self->status = UCC_INPROGRESS; self->tuning_str = ""; self->topo = NULL; self->opt_radix = UCC_UUNITS_AUTO_RADIX; self->opt_radix_host = UCC_UUNITS_AUTO_RADIX; self->cuda_ring = NULL; status = ucc_config_clone_table(&UCC_TL_UCP_TEAM_LIB(self)->cfg, &self->cfg, ucc_tl_ucp_lib_config_table); if (UCC_OK != status) { return status; } if (ctx->topo_required) { status = ucc_tl_ucp_get_topo(self); if (UCC_OK != status) { return status; } } if (ucc_global_config.file_cfg && !UCC_TL_IS_SERVICE_TEAM(self) && ctx->topo_required && tl_context->lib->use_tuning) { status = ucc_add_team_sections(&self->cfg, ucc_tl_ucp_lib_config_table, self->topo, &self->tuning_str, "UCC_TL_UCP_TUNE", UCC_TL_CORE_CTX(self)->lib->full_prefix, ucc_tl_ucp.super.tl_lib_config.prefix); if (status != UCC_OK) { ucc_debug("section not found"); } } if (!self->topo && self->cfg.use_reordering) { tl_debug(tl_context->lib, "topo is not available, disabling ranks reordering"); self->cfg.use_reordering = 0; } if (self->topo && !UCC_TL_IS_SERVICE_TEAM(self)) { tsize = UCC_TL_TEAM_SIZE(self); min_radix = ucc_min(tsize, 3); max_radix = tsize; self->opt_radix = ucc_kn_get_opt_radix(tsize, min_radix, max_radix); if (ucc_topo_is_single_ppn(self->topo)) { self->opt_radix_host = self->opt_radix; } else { if (self->topo->topo->sock_bound) { min_radix = 2; max_radix = ucc_min( tsize, ucc_topo_min_socket_size(self->topo)); self->opt_radix_host = ucc_kn_get_opt_radix( tsize, min_radix, max_radix); } } tl_debug( tl_context->lib, "opt knomial radix: general %d host %d", self->opt_radix, self->opt_radix_host); self->cuda_ring = ucc_calloc(1, sizeof(*self->cuda_ring), "cuda_ring"); if (!self->cuda_ring) { tl_error(UCC_TL_TEAM_LIB(self), "failed to allocate cuda ring"); return UCC_ERR_NO_MEMORY; } status = ucc_ring_pattern_init_topo( self->topo, UCC_MEMORY_TYPE_CUDA, 8, self->cuda_ring); if (UCC_OK != status) { ucc_free(self->cuda_ring); self->cuda_ring = NULL; tl_debug(UCC_TL_TEAM_LIB(self), "failed to init cuda ring"); } else { ucc_ring_pattern_set_rank(self->cuda_ring, UCC_TL_TEAM_RANK(self)); if (UCC_TL_TEAM_RANK(self) == 0) { ucc_ring_pattern_print(self->cuda_ring); } } } tl_debug(tl_context->lib, "posted tl team: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_ucp_team_t) { ucc_config_parser_release_opts(&self->cfg, ucc_tl_ucp_lib_config_table); tl_debug(self->super.super.context->lib, "finalizing tl team: %p", self); } UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_ucp_team_t, ucc_base_team_t); UCC_CLASS_DEFINE(ucc_tl_ucp_team_t, ucc_tl_team_t); ucc_status_t ucc_tl_ucp_team_destroy(ucc_base_team_t *tl_team) { ucc_tl_ucp_team_t *team = ucc_derived_of(tl_team, ucc_tl_ucp_team_t); if (team->cuda_ring) { ucc_ring_pattern_destroy(team->cuda_ring); ucc_free(team->cuda_ring); team->cuda_ring = NULL; } if (team->topo) { ucc_ep_map_destroy_nested(&team->ctx_map); ucc_topo_cleanup(team->topo); } UCC_CLASS_DELETE_FUNC_NAME(ucc_tl_ucp_team_t)(tl_team); return UCC_OK; } static ucc_status_t ucc_tl_ucp_team_preconnect(ucc_tl_ucp_team_t *team) { ucc_rank_t src, dst, size, rank; ucc_status_t status; int i; size = UCC_TL_TEAM_SIZE(team); rank = UCC_TL_TEAM_RANK(team); if (!team->preconnect_task) { team->preconnect_task = ucc_tl_ucp_get_task(team); team->preconnect_task->tagged.tag = 0; team->preconnect_task->super.bargs.args.mask = 0; } if (UCC_INPROGRESS == ucc_tl_ucp_test(team->preconnect_task)) { ucp_worker_progress(team->worker->ucp_worker); return UCC_INPROGRESS; } for (i = team->preconnect_task->tagged.send_posted; i < size; i++) { src = (rank - i + size) % size; dst = (rank + i) % size; status = ucc_tl_ucp_send_nb(NULL, 0, UCC_MEMORY_TYPE_UNKNOWN, src, team, team->preconnect_task); if (UCC_OK != status) { return status; } status = ucc_tl_ucp_recv_nb(NULL, 0, UCC_MEMORY_TYPE_UNKNOWN, dst, team, team->preconnect_task); if (UCC_OK != status) { return status; } if (UCC_INPROGRESS == ucc_tl_ucp_test(team->preconnect_task)) { return UCC_INPROGRESS; } } tl_debug(UCC_TL_TEAM_LIB(team), "preconnected tl team: %p, num_eps %d", team, size); ucc_tl_ucp_put_task(team->preconnect_task); team->preconnect_task = NULL; return UCC_OK; } ucc_status_t ucc_tl_ucp_team_create_test(ucc_base_team_t *tl_team) { ucc_tl_ucp_team_t * team = ucc_derived_of(tl_team, ucc_tl_ucp_team_t); ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(team); int i; ucc_status_t status; if (USE_SERVICE_WORKER(team)) { team->worker = &ctx->service_worker; } else { team->worker = &ctx->worker; } if (team->status == UCC_OK) { return UCC_OK; } if (UCC_TL_TEAM_SIZE(team) <= ctx->cfg.preconnect) { status = ucc_tl_ucp_team_preconnect(team); if (UCC_INPROGRESS == status) { return UCC_INPROGRESS; } else if (UCC_OK != status) { goto err_preconnect; } } if (ctx->remote_info) { for (i = 0; i < ctx->n_rinfo_segs; i++) { team->va_base[i] = ctx->remote_info[i].va_base; team->base_length[i] = ctx->remote_info[i].len; } } tl_debug(tl_team->context->lib, "initialized tl team: %p", team); team->status = UCC_OK; return UCC_OK; err_preconnect: return status; } ucc_status_t ucc_tl_ucp_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p) { ucc_tl_ucp_team_t *team = ucc_derived_of(tl_team, ucc_tl_ucp_team_t); ucc_component_framework_t *plugins = &ucc_tl_ucp.super.coll_plugins; ucc_tl_ucp_context_t *tl_ctx = UCC_TL_UCP_TEAM_CTX(team); ucc_base_context_t *ctx = UCC_TL_TEAM_CTX(team); int mt_n = 0; ucc_memory_type_t mem_types[UCC_MEMORY_TYPE_LAST]; ucc_coll_score_t *score, *tlcp_score; ucc_tl_coll_plugin_iface_t *tlcp; ucc_status_t status; unsigned i; char *ucc_tl_ucp_default_alg_select_str [UCC_TL_UCP_N_DEFAULT_ALG_SELECT_STR]; ucc_coll_score_team_info_t team_info; for (i = 0; i < UCC_MEMORY_TYPE_LAST; i++) { if (tl_ctx->ucp_memory_types & UCC_BIT(ucc_memtype_to_ucs[i])) { tl_debug(tl_team->context->lib, "enable support for memory type %s", ucc_memory_type_names[i]); mem_types[mt_n++] = (ucc_memory_type_t)i; } } team_info.alg_fn = ucc_tl_ucp_alg_id_to_init; team_info.default_score = UCC_TL_UCP_DEFAULT_SCORE; team_info.init = ucc_tl_ucp_coll_init; team_info.num_mem_types = mt_n; team_info.supported_mem_types = mem_types; /* all memory types supported*/ team_info.supported_colls = UCC_TL_UCP_SUPPORTED_COLLS; team_info.size = UCC_TL_TEAM_SIZE(team); /* There can be a different logic for different coll_type/mem_type. Right now just init everything the same way. */ status = ucc_coll_score_build_default(tl_team, UCC_TL_UCP_DEFAULT_SCORE, ucc_tl_ucp_coll_init, UCC_TL_UCP_SUPPORTED_COLLS, mem_types, mt_n, &score); if (UCC_OK != status) { return status; } status = ucc_tl_ucp_team_default_score_str_alloc(team, ucc_tl_ucp_default_alg_select_str); if (UCC_OK != status) { return status; } for (i = 0; i < UCC_TL_UCP_N_DEFAULT_ALG_SELECT_STR; i++) { status = ucc_coll_score_update_from_str( ucc_tl_ucp_default_alg_select_str[i], &team_info, &team->super.super, score); if (UCC_OK != status) { tl_error(tl_team->context->lib, "failed to apply default coll select setting: %s", ucc_tl_ucp_default_alg_select_str[i]); goto err; } } if (strlen(ctx->score_str) > 0) { status = ucc_coll_score_update_from_str(ctx->score_str, &team_info, &team->super.super, score); } else if (strlen(team->tuning_str) > 0) { status = ucc_coll_score_update_from_str(team->tuning_str, &team_info, &team->super.super, score); } /* If INVALID_PARAM - User provided incorrect input - try to proceed */ if ((status < 0) && (status != UCC_ERR_INVALID_PARAM) && (status != UCC_ERR_NOT_SUPPORTED)) { goto err; } for (i = 0; i < plugins->n_components; i++) { tlcp = ucc_derived_of(plugins->components[i], ucc_tl_coll_plugin_iface_t); status = tlcp->get_scores(tl_team, &tlcp_score); if (UCC_OK != status) { goto err; } status = ucc_coll_score_merge_in(&score, tlcp_score); if (UCC_OK != status) { goto err; } } ucc_tl_ucp_team_default_score_str_free(ucc_tl_ucp_default_alg_select_str); *score_p = score; return UCC_OK; err: ucc_tl_ucp_team_default_score_str_free(ucc_tl_ucp_default_alg_select_str); ucc_coll_score_free(score); return status; } ucc-1.8.0/src/components/tl/ucp/tl_ucp_ep.h0000664000175000017500000000710415211535620020765 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #ifndef UCC_TL_UCP_EP_H_ #define UCC_TL_UCP_EP_H_ #include "ucc/api/ucc.h" #include #include "tl_ucp.h" #include "core/ucc_team.h" /* TL/UCP endpoint address layout: (ucp_addrlen may vary per proc) [worker->ucp_addrlen][ucp_worker_address][onesided_info] 8 bytes ucp_addrlen bytes If a special service worker is set through UCC_TL_UCP_SERVICE_TLS: [worker->ucp_addrlen][ucp_worker_address][service_worker->ucp_addrlen][ucp_service_worker_address][onesided_info] 8 bytes ucp_addrlen bytes 8 bytes service.ucp_addrlen bytes */ #define TL_UCP_EP_ADDRLEN_SIZE 8 #define TL_UCP_EP_ADDR_WORKER_LEN(_addr) (*((uint64_t*)(_addr))) #define TL_UCP_EP_ADDR_WORKER(_addr) PTR_OFFSET((_addr), TL_UCP_EP_ADDRLEN_SIZE) #define TL_UCP_EP_OFFSET_WORKER_INFO(_addr) \ PTR_OFFSET((_addr), \ TL_UCP_EP_ADDRLEN_SIZE + TL_UCP_EP_ADDR_WORKER_LEN(_addr)) #define TL_UCP_EP_ADDR_WORKER_SERVICE(_addr) \ TL_UCP_EP_ADDR_WORKER(TL_UCP_EP_OFFSET_WORKER_INFO(_addr)) #define TL_UCP_EP_ADDR_ONESIDED_INFO(_addr, _ctx) \ _ctx->cfg.service_worker \ ? TL_UCP_EP_OFFSET_WORKER_INFO(TL_UCP_EP_OFFSET_WORKER_INFO(_addr)) \ : TL_UCP_EP_OFFSET_WORKER_INFO(_addr) typedef struct ucc_tl_ucp_context ucc_tl_ucp_context_t; typedef struct ucc_tl_ucp_team ucc_tl_ucp_team_t; ucc_status_t ucc_tl_ucp_connect_team_ep(ucc_tl_ucp_team_t *team, ucc_rank_t team_rank, ucp_ep_h *ep); void ucc_tl_ucp_close_eps(ucc_tl_ucp_worker_t * worker, ucc_tl_ucp_context_t *ctx); static inline ucc_context_addr_header_t * ucc_tl_ucp_get_team_ep_header(ucc_tl_ucp_team_t *team, ucc_rank_t core_rank) { return ucc_get_team_ep_header(UCC_TL_CORE_CTX(team), UCC_TL_CORE_TEAM(team), core_rank); } static inline ucc_status_t ucc_tl_ucp_get_ep(ucc_tl_ucp_team_t *team, ucc_rank_t rank, ucp_ep_h *ep) { ucc_context_addr_header_t *h = NULL; ucc_rank_t ctx_rank = 0; ucc_status_t status; ucc_rank_t core_rank; core_rank = ucc_ep_map_eval(UCC_TL_TEAM_MAP(team), rank); if (team->worker->eps) { ucc_team_t *core_team = UCC_TL_CORE_TEAM(team); /* Core super.super.team ptr is NULL for service_team which has scope == UCC_CL_LAST + 1*/ ucc_assert((NULL != core_team) || UCC_TL_IS_SERVICE_TEAM(team)); ctx_rank = core_team ? ucc_get_ctx_rank(core_team, core_rank) : core_rank; *ep = team->worker->eps[ctx_rank]; } else { h = ucc_tl_ucp_get_team_ep_header(team, core_rank); *ep = tl_ucp_hash_get(team->worker->ep_hash, h->ctx_id); } if (NULL == (*ep)) { /* Not connected yet */ status = ucc_tl_ucp_connect_team_ep(team, core_rank, ep); if (ucc_unlikely(UCC_OK != status)) { tl_error(UCC_TL_TEAM_LIB(team), "failed to connect team ep"); *ep = NULL; return status; } if (!h) { team->worker->eps[ctx_rank] = *ep; } else { tl_ucp_hash_put(team->worker->ep_hash, h->ctx_id, *ep); } } return UCC_OK; } #endif ucc-1.8.0/src/components/tl/ucp/tl_ucp.h0000664000175000017500000003437415211535620020312 0ustar alastairalastair/** * Copyright (c) 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_UCP_H_ #define UCC_TL_UCP_H_ #include "components/tl/ucc_tl.h" #include "components/tl/ucc_tl_log.h" #include "core/ucc_ee.h" #include "utils/ucc_mpool.h" #include "tl_ucp_ep_hash.h" #include "schedule/ucc_schedule_pipelined.h" #include #include #include "core/ucc_service_coll.h" #include "coll_patterns/ring.h" #ifndef UCC_TL_UCP_DEFAULT_SCORE #define UCC_TL_UCP_DEFAULT_SCORE 10 #endif #ifdef HAVE_PROFILING_TL_UCP #include "utils/profile/ucc_profile.h" #else #include "utils/profile/ucc_profile_off.h" #endif #define UCC_TL_UCP_PROFILE_FUNC UCC_PROFILE_FUNC #define UCC_TL_UCP_PROFILE_FUNC_VOID UCC_PROFILE_FUNC_VOID #define UCC_TL_UCP_PROFILE_REQUEST_NEW UCC_PROFILE_REQUEST_NEW #define UCC_TL_UCP_PROFILE_REQUEST_EVENT UCC_PROFILE_REQUEST_EVENT #define UCC_TL_UCP_PROFILE_REQUEST_FREE UCC_PROFILE_REQUEST_FREE #define MAX_NR_SEGMENTS 32 #define ONESIDED_SYNC_SIZE 1 #define ONESIDED_REDUCE_SIZE 4 typedef struct ucc_tl_ucp_iface { ucc_tl_iface_t super; } ucc_tl_ucp_iface_t; /* Extern iface should follow the pattern: ucc_tl_ */ extern ucc_tl_ucp_iface_t ucc_tl_ucp; typedef enum ucc_tl_ucp_alltoall_onesided_alg_type { UCC_TL_UCP_ALLTOALL_ONESIDED_PUT, UCC_TL_UCP_ALLTOALL_ONESIDED_GET, UCC_TL_UCP_ALLTOALL_ONESIDED_AUTO, UCC_TL_UCP_ALLTOALL_ONESIDED_LAST } ucc_tl_ucp_alltoall_onesided_alg_t; typedef struct ucc_tl_ucp_lib_config { ucc_tl_lib_config_t super; uint32_t kn_radix; uint32_t fanin_kn_radix; uint32_t fanout_kn_radix; uint32_t barrier_kn_radix; size_t allreduce_sliding_window_buf_size; uint32_t allreduce_sliding_window_put_window_size; uint32_t allreduce_sliding_window_num_get_bufs; ucc_mrange_uint_t allreduce_kn_radix; ucc_mrange_uint_t allreduce_sra_kn_radix; uint32_t reduce_scatter_kn_radix; ucc_mrange_uint_t allgather_kn_radix; uint32_t bcast_kn_radix; ucc_mrange_uint_t bcast_sag_kn_radix; uint32_t reduce_kn_radix; ucc_pipeline_params_t reduce_srg_kn_pipeline; ucc_mrange_uint_t reduce_srg_kn_radix; uint32_t gather_kn_radix; uint32_t gatherv_linear_num_posts; uint32_t scatter_kn_radix; ucc_on_off_auto_value_t scatter_kn_enable_recv_zcopy; uint32_t scatterv_linear_num_posts; unsigned long alltoall_pairwise_num_posts; unsigned long alltoallv_pairwise_num_posts; unsigned long allgather_batched_num_posts; ucc_pipeline_params_t allreduce_sra_kn_pipeline; int reduce_avg_pre_op; int reduce_scatter_ring_bidirectional; int reduce_scatterv_ring_bidirectional; uint32_t alltoallv_hybrid_radix; size_t alltoallv_hybrid_buff_size; size_t alltoallv_hybrid_chunk_byte_limit; uint32_t alltoallv_hybrid_num_scratch_sends; uint32_t alltoallv_hybrid_num_scratch_recvs; uint32_t alltoallv_hybrid_pairwise_num_posts; ucc_ternary_auto_value_t use_topo; int use_reordering; uint32_t alltoall_onesided_percent_bw; ucc_tl_ucp_alltoall_onesided_alg_t alltoall_onesided_alg; } ucc_tl_ucp_lib_config_t; typedef enum ucc_tl_ucp_local_copy_type { UCC_TL_UCP_LOCAL_COPY_TYPE_UCP, UCC_TL_UCP_LOCAL_COPY_TYPE_MC, UCC_TL_UCP_LOCAL_COPY_TYPE_EC, UCC_TL_UCP_LOCAL_COPY_TYPE_AUTO, UCC_TL_UCP_LOCAL_COPY_TYPE_LAST } ucc_tl_ucp_local_copy_type_t; typedef struct ucc_tl_ucp_context_config { ucc_tl_context_config_t super; uint32_t preconnect; uint32_t n_polls; uint32_t oob_npolls; uint32_t pre_reg_mem; uint32_t service_worker; uint32_t service_throttling_thresh; ucc_tl_ucp_local_copy_type_t local_copy_type; int memtype_copy_enable; uint32_t exported_memory_handle; } ucc_tl_ucp_context_config_t; typedef ucc_tl_ucp_lib_config_t ucc_tl_ucp_team_config_t; typedef struct ucc_tl_ucp_lib { ucc_tl_lib_t super; ucc_tl_ucp_lib_config_t cfg; void **tlcp_configs; } ucc_tl_ucp_lib_t; UCC_CLASS_DECLARE(ucc_tl_ucp_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_ucp_remote_info { void * va_base; size_t len; void * mem_h; void * packed_key; size_t packed_key_len; } ucc_tl_ucp_remote_info_t; typedef struct ucc_tl_ucp_memh_data { ucc_tl_ucp_remote_info_t rinfo; void *packed_memh; size_t packed_memh_len; ucp_rkey_h rkey; } ucc_tl_ucp_memh_data_t; typedef struct ucc_tl_ucp_worker { ucp_context_h ucp_context; ucp_worker_h ucp_worker; size_t ucp_addrlen; ucp_address_t * worker_address; tl_ucp_ep_hash_t *ep_hash; ucp_ep_h * eps; } ucc_tl_ucp_worker_t; typedef struct ucc_tl_ucp_task ucc_tl_ucp_task_t; typedef struct ucc_tl_ucp_context ucc_tl_ucp_context_t; typedef union ucc_tl_ucp_copy_task ucc_tl_ucp_copy_task_t; typedef ucc_status_t (*ucc_tl_ucp_copy_post_fn_t)(void *dst, ucc_memory_type_t dst_mtype, ucp_mem_h dst_memh, void *src, ucc_memory_type_t src_mtype, ucp_mem_h src_memh, size_t size, ucc_tl_ucp_task_t *coll_task, ucc_tl_ucp_copy_task_t **copy_task); typedef ucc_status_t (*ucc_tl_ucp_copy_test_fn_t)(ucc_tl_ucp_context_t *ctx, ucc_tl_ucp_copy_task_t *copy_task); typedef ucc_status_t (*ucc_tl_ucp_copy_finalize_fn_t)(ucc_tl_ucp_copy_task_t *copy_task); typedef struct ucc_tl_ucp_team { ucc_tl_team_t super; ucc_status_t status; uint32_t seq_num; ucc_tl_ucp_task_t *preconnect_task; void * va_base[MAX_NR_SEGMENTS]; size_t base_length[MAX_NR_SEGMENTS]; ucc_tl_ucp_worker_t * worker; ucc_tl_ucp_team_config_t cfg; const char * tuning_str; ucc_topo_t *topo; ucc_ep_map_t ctx_map; ucc_rank_t opt_radix; /* generic opt radix */ ucc_rank_t opt_radix_host; /* host specific opt radix */ ucc_ring_pattern_t *cuda_ring; } ucc_tl_ucp_team_t; UCC_CLASS_DECLARE(ucc_tl_ucp_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); typedef ucc_status_t (*ucc_tl_ucp_send_nb_fn_t)(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task); typedef ucc_status_t (*ucc_tl_ucp_recv_nb_fn_t)(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task); typedef ucc_status_t (*ucc_tl_ucp_recv_nz_fn_t)(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task); typedef ucc_status_t (*ucc_tl_ucp_send_nz_fn_t)(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task); typedef void (*ucc_tl_ucp_send_recv_counter_inc_fn_t)(uint32_t *counter); typedef struct ucc_tl_ucp_context { ucc_tl_context_t super; ucc_tl_ucp_context_config_t cfg; ucc_thread_mode_t thread_mode; ucc_tl_ucp_worker_t worker; ucc_tl_ucp_worker_t service_worker; struct { ucc_tl_ucp_send_nb_fn_t ucc_tl_ucp_send_nb; ucc_tl_ucp_recv_nb_fn_t ucc_tl_ucp_recv_nb; ucc_tl_ucp_send_nz_fn_t ucc_tl_ucp_send_nz; ucc_tl_ucp_recv_nz_fn_t ucc_tl_ucp_recv_nz; ucp_send_nbx_callback_t send_cb; ucp_tag_recv_nbx_callback_t recv_cb; ucc_tl_ucp_send_recv_counter_inc_fn_t p2p_counter_inc; } sendrecv_cbs; uint32_t service_worker_throttling_count; ucc_mpool_t req_mp; ucc_tl_ucp_remote_info_t * remote_info; ucp_rkey_h * rkeys; uint64_t n_rinfo_segs; uint64_t ucp_memory_types; int topo_required; struct { ucc_tl_ucp_copy_post_fn_t post; ucc_tl_ucp_copy_test_fn_t test; ucc_tl_ucp_copy_finalize_fn_t finalize; } copy; } ucc_tl_ucp_context_t; UCC_CLASS_DECLARE(ucc_tl_ucp_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); extern ucc_config_field_t ucc_tl_ucp_lib_config_table[]; #define UCC_TL_UCP_SUPPORTED_COLLS \ (UCC_COLL_TYPE_ALLGATHER | \ UCC_COLL_TYPE_ALLGATHERV | \ UCC_COLL_TYPE_ALLREDUCE | \ UCC_COLL_TYPE_ALLTOALL | \ UCC_COLL_TYPE_ALLTOALLV | \ UCC_COLL_TYPE_BARRIER | \ UCC_COLL_TYPE_BCAST | \ UCC_COLL_TYPE_FANIN | \ UCC_COLL_TYPE_FANOUT | \ UCC_COLL_TYPE_GATHER | \ UCC_COLL_TYPE_GATHERV | \ UCC_COLL_TYPE_REDUCE | \ UCC_COLL_TYPE_REDUCE_SCATTER | \ UCC_COLL_TYPE_REDUCE_SCATTERV | \ UCC_COLL_TYPE_SCATTERV) #define UCC_TL_UCP_TEAM_LIB(_team) \ (ucc_derived_of((_team)->super.super.context->lib, ucc_tl_ucp_lib_t)) #define UCC_TL_UCP_TEAM_CTX(_team) \ (ucc_derived_of((_team)->super.super.context, ucc_tl_ucp_context_t)) #define USE_SERVICE_WORKER(_team) \ (UCC_TL_IS_SERVICE_TEAM(_team) && UCC_TL_UCP_TEAM_CTX(_team)->cfg.service_worker) #define UCC_TL_UCP_TASK_TEAM(_task) \ (ucc_derived_of((_task)->super.team, ucc_tl_ucp_team_t)) #define UCC_TL_CTX_HAS_OOB(_ctx) \ ((_ctx)->super.super.ucc_context->params.mask & UCC_CONTEXT_PARAM_FIELD_OOB) #define UCC_TL_CTX_OOB(_ctx) \ ((_ctx)->super.super.ucc_context->params.oob) #define UCC_TL_UCP_REMOTE_RKEY(_ctx, _rank, _seg) \ ((_ctx)->rkeys[_rank * _ctx->n_rinfo_segs + _seg]) /* * For context, the data order of the MEMH Headers / Packed Headers * * MEMH headers for each TL: * TL NAME (8 bytes) | packed size | packed TL * * Packed TL headers: * packed_key_size | packed_memh_size | packed_key | packed_memh */ #define UCC_TL_UCP_MEMH_TL_HEADERS 2 #define UCC_TL_UCP_MEMH_TL_PACKED_HEADERS 2 #define UCC_TL_UCP_MEMH_TL_HEADER_SIZE \ (sizeof(size_t) * UCC_TL_UCP_MEMH_TL_HEADERS) #define UCC_TL_UCP_MEMH_TL_PACKED_HEADER_SIZE \ (sizeof(size_t) * \ (UCC_TL_UCP_MEMH_TL_HEADERS + UCC_TL_UCP_MEMH_TL_PACKED_HEADERS)) #define UCC_TL_UCP_MEMH_TL_KEY_SIZE(buffer) \ (*(size_t *)(PTR_OFFSET(buffer, UCC_TL_UCP_MEMH_TL_HEADER_SIZE))) #define UCC_TL_UCP_MEMH_TL_PACKED_MEMH(buffer) \ (PTR_OFFSET(buffer, UCC_TL_UCP_MEMH_TL_PACKED_HEADER_SIZE + \ UCC_TL_UCP_MEMH_TL_KEY_SIZE(buffer))) extern ucs_memory_type_t ucc_memtype_to_ucs[UCC_MEMORY_TYPE_LAST+1]; void ucc_tl_ucp_pre_register_mem(ucc_tl_ucp_team_t *team, void *addr, size_t length, ucc_memory_type_t mem_type); ucc_status_t ucc_tl_ucp_ctx_remote_populate(ucc_tl_ucp_context_t *ctx, ucc_mem_map_params_t map, ucc_team_oob_coll_t oob); #endif ucc-1.8.0/src/components/tl/ucp/tl_ucp_sendrecv.h0000664000175000017500000007251615211535620022203 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "config.h" #ifndef UCC_TL_UCP_SENDRECV_H_ #define UCC_TL_UCP_SENDRECV_H_ #include "tl_ucp_tag.h" #include "tl_ucp_ep.h" #include "utils/ucc_compiler_def.h" #include "tl_ucp_task.h" #define UCC_TL_UCP_MAKE_TAG(_user_tag, _tag, _rank, _id, _scope_id, _scope) \ ((((uint64_t) (_user_tag)) << UCC_TL_UCP_USER_TAG_BITS_OFFSET) | \ (((uint64_t) (_tag)) << UCC_TL_UCP_TAG_BITS_OFFSET) | \ (((uint64_t) (_rank)) << UCC_TL_UCP_SENDER_BITS_OFFSET) | \ (((uint64_t) (_scope)) << UCC_TL_UCP_SCOPE_BITS_OFFSET) | \ (((uint64_t) (_scope_id)) << UCC_TL_UCP_SCOPE_ID_BITS_OFFSET) | \ (((uint64_t) (_id)) << UCC_TL_UCP_ID_BITS_OFFSET)) #define UCC_TL_UCP_MAKE_SEND_TAG(_user_tag, _tag, _rank, _id, _scope_id, _scope) \ UCC_TL_UCP_MAKE_TAG(_user_tag, _tag, _rank, _id, _scope_id, _scope) #define UCC_TL_UCP_MAKE_RECV_TAG(_ucp_tag, _ucp_tag_mask, _user_tag, _tag, \ _src, _id, _scope_id, _scope) \ do { \ ucc_assert((_tag) <= UCC_TL_UCP_MAX_TAG); \ ucc_assert((_src) <= UCC_TL_UCP_MAX_SENDER); \ ucc_assert((_id) <= UCC_TL_UCP_MAX_ID); \ (_ucp_tag_mask) = (uint64_t)(-1); \ (_ucp_tag) = \ UCC_TL_UCP_MAKE_TAG((_user_tag), (_tag), (_src), (_id), \ (_scope_id), (_scope)); \ } while (0) #define UCC_TL_UCP_CHECK_REQ_STATUS() \ do { \ if (ucc_unlikely(UCS_PTR_IS_ERR(ucp_status))) { \ tl_error(UCC_TL_TEAM_LIB(team), \ "tag %u; dest %d; team_id %u; errmsg %s", \ task->tagged.tag, dest_group_rank, \ team->super.super.params.id, \ ucs_status_string(UCS_PTR_STATUS(ucp_status))); \ return ucs_status_to_ucc_status(UCS_PTR_STATUS(ucp_status)); \ } \ } while (0) void ucc_tl_ucp_send_completion_cb_st(void *request, ucs_status_t status, void *user_data); void ucc_tl_ucp_send_completion_cb_mt(void *request, ucs_status_t status, void *user_data); void ucc_tl_ucp_put_completion_cb(void *request, ucs_status_t status, void *user_data); void ucc_tl_ucp_get_completion_cb(void *request, ucs_status_t status, void *user_data); void ucc_tl_ucp_flush_completion_cb(void *request, ucs_status_t status, void *user_data); void ucc_tl_ucp_recv_completion_cb_st(void *request, ucs_status_t status, const ucp_tag_recv_info_t *info, void *user_data); void ucc_tl_ucp_recv_completion_cb_mt(void *request, ucs_status_t status, const ucp_tag_recv_info_t *info, void *user_data); void ucc_tl_ucp_send_recv_counter_inc_st(uint32_t *counter); void ucc_tl_ucp_send_recv_counter_inc_mt(uint32_t *counter); static inline ucs_status_ptr_t ucc_tl_ucp_send_common(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task, ucp_send_nbx_callback_t cb, void *user_data) { ucc_coll_args_t *args = &TASK_ARGS(task); ucp_request_param_t req_param; ucc_status_t status; ucp_ep_h ep; ucp_tag_t ucp_tag; status = ucc_tl_ucp_get_ep(team, dest_group_rank, &ep); if (ucc_unlikely(UCC_OK != status)) { return UCS_STATUS_PTR(UCS_ERR_NO_MESSAGE); } ucp_tag = UCC_TL_UCP_MAKE_SEND_TAG((args->mask & UCC_COLL_ARGS_FIELD_TAG), task->tagged.tag, UCC_TL_TEAM_RANK(team), team->super.super.params.id, team->super.super.params.scope_id, team->super.super.params.scope); req_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_DATATYPE | UCP_OP_ATTR_FIELD_USER_DATA | UCP_OP_ATTR_FIELD_MEMORY_TYPE; req_param.datatype = ucp_dt_make_contig(msglen); req_param.cb.send = cb; req_param.memory_type = ucc_memtype_to_ucs[mtype]; req_param.user_data = user_data; task->tagged.send_posted++; return ucp_tag_send_nbx(ep, buffer, 1, ucp_tag, &req_param); } ucc_status_t ucc_tl_ucp_send_nbx(void *buffer, size_t msglen, ucc_rank_t dest_group_rank, const ucp_request_param_t *req_param, ucc_tl_ucp_task_t *task); ucc_status_t ucc_tl_ucp_recv_nbx(void *buffer, size_t msglen, ucc_rank_t dest_group_rank, const ucp_request_param_t *req_param, ucc_tl_ucp_task_t *task); static inline ucc_status_t ucc_tl_ucp_send_nb_st(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { ucs_status_ptr_t ucp_status; ucp_status = ucc_tl_ucp_send_common( buffer, msglen, mtype, dest_group_rank, team, task, ucc_tl_ucp_send_completion_cb_st, (void *)task); if (UCS_OK != ucp_status) { UCC_TL_UCP_CHECK_REQ_STATUS(); } else { ++task->tagged.send_completed; } return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_send_nb_mt(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { ucs_status_ptr_t ucp_status; ucp_status = ucc_tl_ucp_send_common( buffer, msglen, mtype, dest_group_rank, team, task, ucc_tl_ucp_send_completion_cb_mt, (void *)task); if (UCS_OK != ucp_status) { UCC_TL_UCP_CHECK_REQ_STATUS(); } else { ucc_atomic_add32(&task->tagged.send_completed, 1); } return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_send_nb(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { return UCC_TL_UCP_TEAM_CTX(team)->sendrecv_cbs.ucc_tl_ucp_send_nb( buffer, msglen, mtype, dest_group_rank, team, task); } static inline ucc_status_t ucc_tl_ucp_send_cb(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task, ucp_send_nbx_callback_t cb, void *user_data) { ucs_status_ptr_t ucp_status; ucp_status = ucc_tl_ucp_send_common(buffer, msglen, mtype, dest_group_rank, team, task, cb, user_data); if (UCS_OK != ucp_status) { UCC_TL_UCP_CHECK_REQ_STATUS(); } else { cb(NULL, UCS_OK, user_data); } return UCC_OK; } static inline ucs_status_ptr_t ucc_tl_ucp_recv_common(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task, ucp_tag_recv_nbx_callback_t cb, void *user_data) { ucc_coll_args_t *args = &TASK_ARGS(task); ucp_request_param_t req_param; ucp_tag_t ucp_tag, ucp_tag_mask; // coverity[result_independent_of_operands:FALSE] UCC_TL_UCP_MAKE_RECV_TAG(ucp_tag, ucp_tag_mask, (args->mask & UCC_COLL_ARGS_FIELD_TAG), task->tagged.tag, dest_group_rank, team->super.super.params.id, team->super.super.params.scope_id, team->super.super.params.scope); req_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_DATATYPE | UCP_OP_ATTR_FIELD_USER_DATA | UCP_OP_ATTR_FIELD_MEMORY_TYPE; req_param.datatype = ucp_dt_make_contig(msglen); req_param.cb.recv = cb; req_param.memory_type = ucc_memtype_to_ucs[mtype]; req_param.user_data = user_data; task->tagged.recv_posted++; return ucp_tag_recv_nbx(team->worker->ucp_worker, buffer, 1, ucp_tag, ucp_tag_mask, &req_param); } static inline ucc_status_t ucc_tl_ucp_recv_nb_mt(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { ucs_status_ptr_t ucp_status; ucp_status = ucc_tl_ucp_recv_common( buffer, msglen, mtype, dest_group_rank, team, task, ucc_tl_ucp_recv_completion_cb_mt, (void *)task); if (UCS_OK != ucp_status) { UCC_TL_UCP_CHECK_REQ_STATUS(); } else { ucc_atomic_add32(&task->tagged.recv_completed, 1); } return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_recv_nb_st(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { ucs_status_ptr_t ucp_status; ucp_status = ucc_tl_ucp_recv_common( buffer, msglen, mtype, dest_group_rank, team, task, ucc_tl_ucp_recv_completion_cb_st, (void *)task); if (UCS_OK != ucp_status) { UCC_TL_UCP_CHECK_REQ_STATUS(); } else { ++task->tagged.recv_completed; } return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_recv_nb(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { return UCC_TL_UCP_TEAM_CTX(team)->sendrecv_cbs.ucc_tl_ucp_recv_nb( buffer, msglen, mtype, dest_group_rank, team, task); } static inline ucc_status_t ucc_tl_ucp_recv_cb(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task, ucp_tag_recv_nbx_callback_t cb, void *user_data) { ucs_status_ptr_t ucp_status; ucp_status = ucc_tl_ucp_recv_common(buffer, msglen, mtype, dest_group_rank, team, task, cb, user_data); if (UCS_OK != ucp_status) { UCC_TL_UCP_CHECK_REQ_STATUS(); } else { cb(NULL, UCS_OK, NULL, user_data); } return UCC_OK; } /* Non-Zero recv: if msglen == 0 then it is a no-op */ static inline ucc_status_t ucc_tl_ucp_recv_nz_mt(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { if (msglen == 0) { task->tagged.recv_posted++; ucc_atomic_add32(&task->tagged.recv_completed, 1); return UCC_OK; } return ucc_tl_ucp_recv_nb_mt(buffer, msglen, mtype, dest_group_rank, team, task); } /* Non-Zero recv: if msglen == 0 then it is a no-op */ static inline ucc_status_t ucc_tl_ucp_recv_nz_st(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { if (msglen == 0) { task->tagged.recv_posted++; task->tagged.recv_completed++; return UCC_OK; } return ucc_tl_ucp_recv_nb_st(buffer, msglen, mtype, dest_group_rank, team, task); } /* Non-Zero send: if msglen == 0 then it is a no-op */ static inline ucc_status_t ucc_tl_ucp_send_nz_mt(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { if (msglen == 0) { task->tagged.send_posted++; ucc_atomic_add32(&task->tagged.send_completed, 1); return UCC_OK; } return ucc_tl_ucp_send_nb_mt(buffer, msglen, mtype, dest_group_rank, team, task); } /* Non-Zero send: if msglen == 0 then it is a no-op */ static inline ucc_status_t ucc_tl_ucp_send_nz_st(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { if (msglen == 0) { task->tagged.send_posted++; task->tagged.send_completed++; return UCC_OK; } return ucc_tl_ucp_send_nb_st(buffer, msglen, mtype, dest_group_rank, team, task); } static inline ucc_status_t ucc_tl_ucp_send_nz(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { return UCC_TL_UCP_TEAM_CTX(team)->sendrecv_cbs.ucc_tl_ucp_send_nz( buffer, msglen, mtype, dest_group_rank, team, task); } static inline ucc_status_t ucc_tl_ucp_recv_nz(void *buffer, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { return UCC_TL_UCP_TEAM_CTX(team)->sendrecv_cbs.ucc_tl_ucp_recv_nz( buffer, msglen, mtype, dest_group_rank, team, task); } static inline ucc_status_t find_tl_index(ucc_mem_map_mem_h map_memh, int *tl_index) { ucc_mem_map_memh_t *memh = (ucc_mem_map_memh_t *)map_memh; int i = 0; for (; i < memh->num_tls; i++) { if (strncmp(memh->tl_h[i].tl_name, "ucp", 3) == 0) { *tl_index = i; return UCC_OK; } } return UCC_ERR_NOT_FOUND; } static inline ucc_status_t ucc_tl_ucp_get_memh(ucc_tl_ucp_team_t *team, ucc_mem_map_mem_h map_memh, void **ucp_memh) { ucc_mem_map_memh_t *memh = map_memh; int tl_index = 0; ucc_tl_ucp_memh_data_t *tl_data; ucc_status_t status; status = find_tl_index(memh, &tl_index); if (status == UCC_ERR_NOT_FOUND) { tl_error(UCC_TL_TEAM_LIB(team), "attempt to perform one-sided operation with malformed mem " "map handle"); return status; } tl_data = (ucc_tl_ucp_memh_data_t *)memh->tl_h[tl_index].tl_data; *ucp_memh = tl_data->rinfo.mem_h; return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_check_memh(ucp_ep_h *ep, ucc_rank_t me, ucc_rank_t peer, void *va, uint64_t *rva, ucp_rkey_h *rkey, int tl_index, ucc_mem_map_mem_h *dst_map_memh) { ucc_mem_map_memh_t **dst_memh = (ucc_mem_map_memh_t **)dst_map_memh; ucc_tl_ucp_memh_data_t *dst_tl_data = NULL; uint64_t base; uint64_t end; ucs_status_t ucs_status; int i; size_t offset; base = (uint64_t)dst_memh[me]->address; end = base + dst_memh[me]->len; if (!((uint64_t)va >= base && (uint64_t)va < end)) { return UCC_ERR_NOT_FOUND; } *rva = (uint64_t)PTR_OFFSET(dst_memh[peer]->address, ((uint64_t)va - (uint64_t)dst_memh[me]->address)); dst_tl_data = (ucc_tl_ucp_memh_data_t *)dst_memh[peer]->tl_h[tl_index].tl_data; if (NULL == dst_tl_data->rkey) { offset = 0; /* find pack location for tl */ for (i = 0; i < tl_index; i++) { char *name = PTR_OFFSET(dst_memh[peer]->pack_buffer, offset); size_t *packed_size = PTR_OFFSET(dst_memh[peer]->pack_buffer, offset + UCC_MEM_MAP_TL_NAME_LEN); if (strncmp(name, "ucp", 3) == 0) { break; } offset += UCC_MEM_MAP_TL_NAME_LEN + sizeof(size_t) + *packed_size; } ucs_status = ucp_ep_rkey_unpack( *ep, PTR_OFFSET(dst_memh[peer]->pack_buffer, offset + sizeof(size_t) * (UCC_TL_UCP_MEMH_TL_HEADERS + UCC_TL_UCP_MEMH_TL_PACKED_HEADERS)), &dst_tl_data->rkey); if (UCS_OK != ucs_status) { return ucs_status_to_ucc_status(ucs_status); } } *rkey = dst_tl_data->rkey; return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_resolve_p2p_by_va(ucc_tl_ucp_team_t *team, void *va, ucp_ep_h *ep, ucc_rank_t peer, uint64_t *rva, ucp_rkey_h *rkey, int *segment, ucc_mem_map_mem_h *dst_memh) { ucc_tl_ucp_context_t *ctx = UCC_TL_UCP_TEAM_CTX(team); ucc_rank_t grank = UCC_TL_TEAM_RANK(team); ptrdiff_t key_offset = 0; const size_t section_offset = sizeof(uint64_t) * ctx->n_rinfo_segs; int tl_index = 0; ucc_rank_t core_rank; uint64_t *rvas; uint64_t *key_sizes; void *keys; void *offset; ptrdiff_t base_offset; ucc_status_t status; *segment = -1; core_rank = ucc_ep_map_eval(UCC_TL_TEAM_MAP(team), peer); ucc_assert(UCC_TL_CORE_TEAM(team) != NULL); peer = ucc_get_ctx_rank(UCC_TL_CORE_TEAM(team), core_rank); offset = ucc_get_team_ep_addr(UCC_TL_CORE_CTX(team), UCC_TL_CORE_TEAM(team), core_rank, ucc_tl_ucp.super.super.id); base_offset = (ptrdiff_t)(TL_UCP_EP_ADDR_ONESIDED_INFO(offset, ctx)); rvas = (uint64_t *)base_offset; key_sizes = PTR_OFFSET(base_offset, (section_offset * 2)); keys = PTR_OFFSET(base_offset, (section_offset * 3)); for (int i = 0; i < ctx->n_rinfo_segs; i++) { uint64_t base = (uint64_t)team->va_base[i]; uint64_t end = base + team->base_length[i]; if ((uint64_t)va >= base && (uint64_t)va < end) { *segment = i; break; } key_offset += key_sizes[i]; } if (ucc_unlikely(0 > *segment)) { if (dst_memh) { /* check if segment is in src/dst memh */ status = find_tl_index(dst_memh[grank], &tl_index); if (status == UCC_ERR_NOT_FOUND) { tl_error(UCC_TL_TEAM_LIB(team), "attempt to perform one-sided operation with malformed mem map handle"); return status; } status = ucc_tl_ucp_check_memh(ep, grank, peer, va, rva, rkey, tl_index, dst_memh); if (status == UCC_OK) { return UCC_OK; } } /* general error if nothing was found */ tl_error(UCC_TL_TEAM_LIB(team), "attempt to perform one-sided operation on non-registered memory %p", va); return UCC_ERR_NOT_FOUND; } if (ucc_unlikely(NULL == UCC_TL_UCP_REMOTE_RKEY(ctx, peer, *segment))) { ucs_status_t ucs_status = ucp_ep_rkey_unpack(*ep, PTR_OFFSET(keys, key_offset), &UCC_TL_UCP_REMOTE_RKEY(ctx, peer, *segment)); if (UCS_OK != ucs_status) { return ucs_status_to_ucc_status(ucs_status); } } *rkey = UCC_TL_UCP_REMOTE_RKEY(ctx, peer, *segment); *rva = rvas[*segment] + ((uint64_t)va - (uint64_t)team->va_base[*segment]); return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_flush(ucc_tl_ucp_team_t *team) { ucp_request_param_t req_param = {0}; ucs_status_ptr_t req; req = ucp_worker_flush_nbx(team->worker->ucp_worker, &req_param); if (UCS_OK != req) { if (UCS_PTR_IS_ERR(req)) { return ucs_status_to_ucc_status(UCS_PTR_STATUS(req)); } ucp_request_free(req); } return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_ep_flush(ucc_rank_t dest_group_rank, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { ucp_request_param_t req_param = {0}; ucc_status_t status; ucs_status_ptr_t req; ucp_ep_h ep; status = ucc_tl_ucp_get_ep(team, dest_group_rank, &ep); if (ucc_unlikely(UCC_OK != status)) { return status; } req_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_USER_DATA; req_param.cb.send = ucc_tl_ucp_flush_completion_cb; req_param.user_data = (void *)task; req = ucp_ep_flush_nbx(ep, &req_param); task->flush_posted++; if (UCS_OK != req) { if (UCS_PTR_IS_ERR(req)) { return ucs_status_to_ucc_status(UCS_PTR_STATUS(req)); } } else { task->flush_completed++; } return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_put_nb(void *buffer, void *target, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_mem_map_mem_h src_memh, ucc_mem_map_mem_h *dest_memh, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { ucp_request_param_t req_param = {0}; int segment = 0; ucp_rkey_h rkey = NULL; uint64_t rva = 0; void *ucp_memh = NULL; ucs_status_ptr_t ucp_status; ucc_status_t status; ucp_ep_h ep; status = ucc_tl_ucp_get_ep(team, dest_group_rank, &ep); if (ucc_unlikely(UCC_OK != status)) { return status; } if (src_memh) { status = ucc_tl_ucp_get_memh(team, src_memh, &ucp_memh); if (ucc_unlikely(UCC_OK != status)) { return status; } } status = ucc_tl_ucp_resolve_p2p_by_va(team, target, &ep, dest_group_rank, &rva, &rkey, &segment, dest_memh); if (ucc_unlikely(UCC_OK != status)) { return status; } req_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_USER_DATA | UCP_OP_ATTR_FIELD_MEMORY_TYPE; req_param.cb.send = ucc_tl_ucp_put_completion_cb; req_param.user_data = (void *)task; req_param.memory_type = ucc_memtype_to_ucs[mtype]; if (ucp_memh) { req_param.op_attr_mask |= UCP_OP_ATTR_FIELD_MEMH; req_param.memh = ucp_memh; } ucp_status = ucp_put_nbx(ep, buffer, msglen, rva, rkey, &req_param); task->onesided.put_posted++; if (UCS_OK != ucp_status) { if (UCS_PTR_IS_ERR(ucp_status)) { return ucs_status_to_ucc_status(UCS_PTR_STATUS(ucp_status)); } } else { task->onesided.put_completed++; } return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_get_nb(void *buffer, void *target, size_t msglen, ucc_memory_type_t mtype, ucc_rank_t dest_group_rank, ucc_mem_map_mem_h src_memh, ucc_mem_map_mem_h *dest_memh, ucc_tl_ucp_team_t *team, ucc_tl_ucp_task_t *task) { ucp_request_param_t req_param = {0}; int segment = 0; ucp_rkey_h rkey = NULL; uint64_t rva = 0; void *ucp_memh = NULL; ucs_status_ptr_t ucp_status; ucc_status_t status; ucp_ep_h ep; status = ucc_tl_ucp_get_ep(team, dest_group_rank, &ep); if (ucc_unlikely(UCC_OK != status)) { return status; } if (src_memh) { status = ucc_tl_ucp_get_memh(team, src_memh, &ucp_memh); if (ucc_unlikely(UCC_OK != status)) { return status; } } status = ucc_tl_ucp_resolve_p2p_by_va(team, target, &ep, dest_group_rank, &rva, &rkey, &segment, dest_memh); if (ucc_unlikely(UCC_OK != status)) { return status; } req_param.op_attr_mask = UCP_OP_ATTR_FIELD_CALLBACK | UCP_OP_ATTR_FIELD_USER_DATA | UCP_OP_ATTR_FIELD_MEMORY_TYPE; req_param.cb.send = ucc_tl_ucp_get_completion_cb; req_param.user_data = (void *)task; req_param.memory_type = ucc_memtype_to_ucs[mtype]; if (ucp_memh) { req_param.op_attr_mask |= UCP_OP_ATTR_FIELD_MEMH; req_param.memh = ucp_memh; } ucp_status = ucp_get_nbx(ep, buffer, msglen, rva, rkey, &req_param); task->onesided.get_posted++; if (UCS_OK != ucp_status) { if (UCS_PTR_IS_ERR(ucp_status)) { return ucs_status_to_ucc_status(UCS_PTR_STATUS(ucp_status)); } } else { task->onesided.get_completed++; } return UCC_OK; } static inline ucc_status_t ucc_tl_ucp_atomic_inc(void * target, ucc_rank_t dest_group_rank, ucc_mem_map_mem_h *dest_memh, ucc_tl_ucp_team_t *team) { ucp_request_param_t req_param = {0}; int segment = 0; uint64_t one = 1; ucp_rkey_h rkey = NULL; uint64_t rva = 0; ucs_status_ptr_t ucp_status; ucc_status_t status; ucp_ep_h ep; status = ucc_tl_ucp_get_ep(team, dest_group_rank, &ep); if (ucc_unlikely(UCC_OK != status)) { return status; } status = ucc_tl_ucp_resolve_p2p_by_va(team, target, &ep, dest_group_rank, &rva, &rkey, &segment, dest_memh); if (ucc_unlikely(UCC_OK != status)) { return status; } req_param.op_attr_mask = UCP_OP_ATTR_FIELD_DATATYPE; req_param.datatype = ucp_dt_make_contig(sizeof(uint64_t)); ucp_status = ucp_atomic_op_nbx(ep, UCP_ATOMIC_OP_ADD, &one, 1, rva, rkey, &req_param); if (UCS_OK != ucp_status) { if (UCS_PTR_IS_ERR(ucp_status)) { return ucs_status_to_ucc_status(UCS_PTR_STATUS(ucp_status)); } ucp_request_free(ucp_status); } return UCC_OK; } /* Accept only UCC_OK */ #define UCPCHECK_GOTO(_cmd, _task, _label) \ do { \ ucc_status_t _status = (_cmd); \ if (UCC_OK != _status) { \ _task->super.status = _status; \ goto _label; \ } \ } while (0) #endif ucc-1.8.0/src/components/tl/ucp/tl_ucp_coll.h0000664000175000017500000002666315211535620021325 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #ifndef UCC_TL_UCP_COLL_H_ #define UCC_TL_UCP_COLL_H_ #include "tl_ucp.h" #include "tl_ucp_task.h" #include "coll_patterns/recursive_knomial.h" #define UCC_UUNITS_AUTO_RADIX 4 #define UCC_TL_UCP_N_DEFAULT_ALG_SELECT_STR 9 ucc_status_t ucc_tl_ucp_team_default_score_str_alloc(ucc_tl_ucp_team_t *team, char *default_select_str[UCC_TL_UCP_N_DEFAULT_ALG_SELECT_STR]); void ucc_tl_ucp_team_default_score_str_free( char *default_select_str[UCC_TL_UCP_N_DEFAULT_ALG_SELECT_STR]); #define CALC_KN_TREE_DIST(_size, _radix, _dist) \ do { \ _dist = 1; \ while (_dist * _radix < _size) { \ _dist *= _radix; \ } \ } while (0) #define VRANK(_rank, _root, _team_size) \ (((_rank) - (_root) + (_team_size)) % (_team_size)) #define INV_VRANK(_rank, _root, _team_size) \ (((_rank) + (_root)) % (_team_size)) #define EXEC_TASK_TEST(_phase, _errmsg, _etask) do { \ if (_etask != NULL) { \ status = ucc_ee_executor_task_test(_etask); \ if (status > 0) { \ task->super.status = UCC_INPROGRESS; \ SAVE_STATE(_phase); \ return; \ } \ ucc_ee_executor_task_finalize(_etask); \ _etask = NULL; \ if (ucc_unlikely(status < 0)) { \ tl_error(UCC_TASK_LIB(task), _errmsg); \ task->super.status = status; \ return; \ } \ } \ } while(0) #define EXEC_TASK_WAIT(_etask, ...) \ do { \ if (_etask != NULL) { \ do { \ status = ucc_ee_executor_task_test(_etask); \ } while (status > 0); \ if (status < 0) { \ tl_error(UCC_TASK_LIB(task), "failure in ee task ee task"); \ task->super.status = status; \ return __VA_ARGS__; \ } \ ucc_ee_executor_task_finalize(_etask); \ if (ucc_unlikely(status < 0)) { \ tl_error(UCC_TASK_LIB(task), "failed to finalize ee task"); \ task->super.status = status; \ return __VA_ARGS__; \ } \ } \ } while (0) typedef char* (*ucc_tl_ucp_score_str_get_fn_t)(ucc_tl_ucp_team_t *team); typedef struct ucc_tl_ucp_default_alg_desc { char *select_str; ucc_tl_ucp_score_str_get_fn_t str_get_fn; } ucc_tl_ucp_default_alg_desc_t; #define AVG_ALPHA(_task) (1.0 / (double)UCC_TL_TEAM_SIZE(TASK_TEAM(_task))) ucc_status_t ucc_tl_ucp_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h); ucc_status_t ucc_tl_ucp_coll_finalize(ucc_coll_task_t *coll_task); static inline ucc_tl_ucp_task_t *ucc_tl_ucp_init_task(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_tl_ucp_task_t *task = ucc_tl_ucp_get_task(tl_team); if (ucc_unlikely(!task)) { return NULL; } ucc_coll_task_init(&task->super, coll_args, team); if (UCC_COLL_ARGS_ACTIVE_SET(&coll_args->args)) { task->tagged.tag = (coll_args->args.mask & UCC_COLL_ARGS_FIELD_TAG) ? coll_args->args.tag : UCC_TL_UCP_ACTIVE_SET_TAG; task->flags |= UCC_TL_UCP_TASK_FLAG_SUBSET; task->subset.map = ucc_active_set_to_ep_map(&coll_args->args); task->subset.myrank = ucc_ep_map_local_rank(task->subset.map, UCC_TL_TEAM_RANK(tl_team)); ucc_assert(coll_args->args.coll_type == UCC_COLL_TYPE_BCAST); } else { if (coll_args->args.mask & UCC_COLL_ARGS_FIELD_TAG) { task->tagged.tag = coll_args->args.tag; } else { tl_team->seq_num = (tl_team->seq_num + 1) % UCC_TL_UCP_MAX_COLL_TAG; task->tagged.tag = tl_team->seq_num; } } task->super.finalize = ucc_tl_ucp_coll_finalize; return task; } #define UCC_TL_UCP_TASK_P2P_COMPLETE(_task) \ (((_task)->tagged.send_posted == (_task)->tagged.send_completed) && \ ((_task)->tagged.recv_posted == (_task)->tagged.recv_completed)) static inline ucc_status_t ucc_tl_ucp_test(ucc_tl_ucp_task_t *task) { int polls = 0; if (UCC_TL_UCP_TASK_P2P_COMPLETE(task)) { return UCC_OK; } while (polls++ < task->n_polls) { if (UCC_TL_UCP_TASK_P2P_COMPLETE(task)) { return UCC_OK; } ucp_worker_progress(UCC_TL_UCP_TASK_TEAM(task)->worker->ucp_worker); } return UCC_INPROGRESS; } #define UCC_TL_UCP_TASK_RECV_COMPLETE(_task) \ (((_task)->tagged.recv_posted == (_task)->tagged.recv_completed)) #define UCC_TL_UCP_TASK_SEND_COMPLETE(_task) \ (((_task)->tagged.send_posted == (_task)->tagged.send_completed)) static inline ucc_status_t ucc_tl_ucp_test_recv(ucc_tl_ucp_task_t *task) { int polls = 0; if (UCC_TL_UCP_TASK_RECV_COMPLETE(task)) { return UCC_OK; } while (polls++ < task->n_polls) { if (UCC_TL_UCP_TASK_RECV_COMPLETE(task)) { return UCC_OK; } ucp_worker_progress(UCC_TL_UCP_TASK_TEAM(task)->worker->ucp_worker); } return UCC_INPROGRESS; } static inline ucc_status_t ucc_tl_ucp_test_send(ucc_tl_ucp_task_t *task) { int polls = 0; if (UCC_TL_UCP_TASK_SEND_COMPLETE(task)) { return UCC_OK; } while (polls++ < task->n_polls) { if (UCC_TL_UCP_TASK_SEND_COMPLETE(task)) { return UCC_OK; } ucp_worker_progress(UCC_TL_UCP_TASK_TEAM(task)->worker->ucp_worker); } return UCC_INPROGRESS; } #define UCC_TL_UCP_TASK_RING_P2P_COMPLETE(_task) \ ((((_task)->tagged.send_posted - (_task)->tagged.send_completed) <= 1) && \ ((_task)->tagged.recv_posted == (_task)->tagged.recv_completed)) static inline ucc_status_t ucc_tl_ucp_test_ring(ucc_tl_ucp_task_t *task) { int polls = 0; if (UCC_TL_UCP_TASK_RING_P2P_COMPLETE(task)) { return UCC_OK; } while (polls++ < task->n_polls) { if (UCC_TL_UCP_TASK_RING_P2P_COMPLETE(task)) { return UCC_OK; } ucp_worker_progress(TASK_CTX(task)->worker.ucp_worker); } return UCC_INPROGRESS; } #define UCC_TL_UCP_TASK_ONESIDED_P2P_COMPLETE(_task) \ (((_task)->onesided.put_posted == (_task)->onesided.put_completed) && \ ((_task)->onesided.get_posted == (_task)->onesided.get_completed) && \ ((_task)->flush_posted == (_task)->flush_completed)) #define UCC_TL_UCP_TASK_ONESIDED_SYNC_COMPLETE(_task, _end) \ (*((long *)(TASK_ARGS(_task).global_work_buffer)) == _end) static inline ucc_status_t ucc_tl_ucp_test_onesided(ucc_tl_ucp_task_t *task, int sync_end) { int polls = 0; if (UCC_TL_UCP_TASK_ONESIDED_P2P_COMPLETE(task) && UCC_TL_UCP_TASK_ONESIDED_SYNC_COMPLETE(task, sync_end)) { return UCC_OK; } while (polls++ < task->n_polls) { if (UCC_TL_UCP_TASK_ONESIDED_P2P_COMPLETE(task) && UCC_TL_UCP_TASK_ONESIDED_SYNC_COMPLETE(task, sync_end)) { return UCC_OK; } ucp_worker_progress(UCC_TL_UCP_TASK_TEAM(task)->worker->ucp_worker); } return UCC_INPROGRESS; } ucc_status_t ucc_tl_ucp_alg_id_to_init(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, ucc_base_coll_init_fn_t *init); static inline unsigned ucc_tl_ucp_get_radix_from_range(ucc_tl_ucp_team_t *team, size_t msgsize, ucc_memory_type_t mem_type, ucc_mrange_uint_t *p, ucc_rank_t default_value) { unsigned radix; radix = ucc_mrange_uint_get(p, msgsize, mem_type); if (UCC_UUNITS_AUTO == radix) { return default_value; } return radix; } /* * Get the radix for knomial patterns. * If need_scratch is true, the radix is the minimum radix that can be used to fit into scratch buffer. * Otherwise, the radix is the minimum radix that can be used to fit into team size. */ static inline unsigned ucc_tl_ucp_get_knomial_radix(ucc_tl_ucp_team_t *team, size_t count, ucc_datatype_t dtype, ucc_memory_type_t mem_type, ucc_mrange_uint_t *p, int need_scratch) { size_t msgsize = count * ucc_dt_size(dtype); unsigned opt_radix, cfg_radix, radix; opt_radix = (mem_type == UCC_MEMORY_TYPE_HOST) ? team->opt_radix_host : team->opt_radix; cfg_radix = ucc_tl_ucp_get_radix_from_range(team, msgsize, mem_type, p, opt_radix); if (need_scratch) { radix = ucc_knomial_pattern_get_min_radix(cfg_radix, UCC_TL_TEAM_SIZE(team), count); } else { radix = ucc_min(cfg_radix, UCC_TL_TEAM_SIZE(team)); } return radix; } #endif ucc-1.8.0/src/components/tl/ucp/tl_ucp_tag.h0000664000175000017500000000524415211535620021137 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #ifndef UCC_TL_UCP_TAG_H_ #define UCC_TL_UCP_TAG_H_ #include "utils/ucc_compiler_def.h" /* * UCP tag structure: * * 01 | 2 | 34567 01234567 01 | 234 | 567 | 01234567 01234567 01234567 | 01234567 01234567 * | | | | | | * RESERV(2) | user tag (1) | message tag (15) | SCOPE(3) | SCOPE_ID(3) | source rank (24) | team id (16) */ #define UCC_TL_UCP_RESERVED_BITS 2 #define UCC_TL_UCP_SCOPE_BITS 3 #define UCC_TL_UCP_SCOPE_ID_BITS 3 #define UCC_TL_UCP_USER_TAG_BITS 1 #define UCC_TL_UCP_TAG_BITS 15 #define UCC_TL_UCP_SENDER_BITS 24 #define UCC_TL_UCP_ID_BITS 16 #define UCC_TL_UCP_RESERVED_BITS_OFFSET \ (UCC_TL_UCP_ID_BITS + UCC_TL_UCP_SENDER_BITS + UCC_TL_UCP_SCOPE_ID_BITS + \ UCC_TL_UCP_SCOPE_BITS + UCC_TL_UCP_TAG_BITS + UCC_TL_UCP_USER_TAG_BITS) #define UCC_TL_UCP_USER_TAG_BITS_OFFSET \ (UCC_TL_UCP_ID_BITS + UCC_TL_UCP_SENDER_BITS + UCC_TL_UCP_SCOPE_ID_BITS + \ UCC_TL_UCP_SCOPE_BITS + UCC_TL_UCP_TAG_BITS) #define UCC_TL_UCP_TAG_BITS_OFFSET \ (UCC_TL_UCP_ID_BITS + UCC_TL_UCP_SENDER_BITS + UCC_TL_UCP_SCOPE_ID_BITS + \ UCC_TL_UCP_SCOPE_BITS) #define UCC_TL_UCP_SCOPE_BITS_OFFSET \ (UCC_TL_UCP_ID_BITS + UCC_TL_UCP_SENDER_BITS + UCC_TL_UCP_SCOPE_ID_BITS) #define UCC_TL_UCP_SCOPE_ID_BITS_OFFSET (UCC_TL_UCP_ID_BITS + UCC_TL_UCP_SENDER_BITS) #define UCC_TL_UCP_SENDER_BITS_OFFSET (UCC_TL_UCP_ID_BITS) #define UCC_TL_UCP_ID_BITS_OFFSET 0 #define UCC_TL_UCP_MAX_TAG UCC_MASK(UCC_TL_UCP_TAG_BITS) #define UCC_TL_UCP_RESERVED_TAGS 8 #define UCC_TL_UCP_MAX_COLL_TAG (UCC_TL_UCP_MAX_TAG - UCC_TL_UCP_RESERVED_TAGS) #define UCC_TL_UCP_SERVICE_TAG (UCC_TL_UCP_MAX_COLL_TAG + 1) #define UCC_TL_UCP_ACTIVE_SET_TAG (UCC_TL_UCP_MAX_COLL_TAG + 2) #define UCC_TL_UCP_MAX_SENDER UCC_MASK(UCC_TL_UCP_SENDER_BITS) #define UCC_TL_UCP_MAX_ID UCC_MASK(UCC_TL_UCP_ID_BITS) #define UCC_TL_UCP_TAG_SENDER_MASK \ UCC_MASK(UCC_TL_UCP_ID_BITS + UCC_TL_UCP_SENDER_BITS + \ UCC_TL_UCP_SCOPE_ID_BITS + UCC_TL_UCP_SCOPE_BITS) #define UCC_TL_UCP_GET_SENDER(_tag) ((uint32_t)(((_tag) >> UCC_TL_UCP_SENDER_BITS_OFFSET) & \ UCC_MASK(UCC_TL_UCP_SENDER_BITS))) #endif ucc-1.8.0/src/components/tl/ucp/reduce_scatterv/0000775000175000017500000000000015211535620022022 5ustar alastairalastairucc-1.8.0/src/components/tl/ucp/reduce_scatterv/reduce_scatterv.h0000664000175000017500000000177515211535620025367 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef REDUCE_SCATTERV_H_ #define REDUCE_SCATTERV_H_ #include "tl_ucp_coll.h" enum { UCC_TL_UCP_REDUCE_SCATTERV_ALG_RING, UCC_TL_UCP_REDUCE_SCATTERV_ALG_LAST }; extern ucc_base_coll_alg_info_t ucc_tl_ucp_reduce_scatterv_algs[UCC_TL_UCP_REDUCE_SCATTERV_ALG_LAST + 1]; #define UCC_TL_UCP_REDUCE_SCATTERV_DEFAULT_ALG_SELECT_STR \ "reduce_scatterv:@ring" static inline int ucc_tl_ucp_reduce_scatterv_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_UCP_REDUCE_SCATTERV_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_ucp_reduce_scatterv_algs[i].name)) { break; } } return i; } ucc_status_t ucc_tl_ucp_reduce_scatterv_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h); #endif ucc-1.8.0/src/components/tl/ucp/reduce_scatterv/reduce_scatterv_ring.c0000664000175000017500000004261615211535620026400 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "reduce_scatterv.h" #include "tl_ucp_sendrecv.h" #include "core/ucc_progress_queue.h" #include "components/mc/ucc_mc.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "utils/ucc_dt_reduce.h" #define REVERSED_FRAG 1 static inline void send_completion_common(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; if (ucc_unlikely(UCS_OK != status)) { tl_error(UCC_TASK_LIB(task), "failure in rs ring completion %s", ucs_status_string(status)); task->super.status = ucs_status_to_ucc_status(status); } ucc_atomic_add32(&task->tagged.send_completed, 1); if (request) { ucp_request_free(request); } } static void send_completion_1(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; task->reduce_scatterv_ring.s_scratch_busy[0] = 0; send_completion_common(request, status, user_data); } static void send_completion_2(void *request, ucs_status_t status, void *user_data) { ucc_tl_ucp_task_t *task = (ucc_tl_ucp_task_t *)user_data; task->reduce_scatterv_ring.s_scratch_busy[1] = 0; send_completion_common(request, status, user_data); } static inline void ucc_ring_frag_count(ucc_tl_ucp_task_t *task, ucc_rank_t block, size_t *frag_count) { int n_frags, frag; size_t block_count; n_frags = task->reduce_scatterv_ring.n_frags; frag = task->reduce_scatterv_ring.frag; block_count = ucc_coll_args_get_count( &TASK_ARGS(task), TASK_ARGS(task).dst.info_v.counts, block); *frag_count = ucc_buffer_block_count(block_count, n_frags, frag); } static inline size_t get_block_offset(ucc_coll_args_t *args, ucc_rank_t block) { size_t offset = 0; int i; for (i = 0; i < block; i++) { offset += ucc_coll_args_get_count(args, args->dst.info_v.counts, i); } return offset; } static inline void ucc_ring_frag_block_offset(ucc_tl_ucp_task_t *task, ucc_rank_t block, size_t * block_offset, size_t * frag_offset) { int n_frags, frag; size_t block_count; n_frags = task->reduce_scatterv_ring.n_frags; frag = task->reduce_scatterv_ring.frag; block_count = ucc_coll_args_get_count( &TASK_ARGS(task), TASK_ARGS(task).dst.info_v.counts, block); *frag_offset = ucc_buffer_block_offset(block_count, n_frags, frag); *block_offset = get_block_offset(&TASK_ARGS(task), block); } static void ucc_tl_ucp_reduce_scatterv_ring_progress(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t * task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t * args = &TASK_ARGS(task); ucc_tl_ucp_team_t * team = TASK_TEAM(task); ucc_rank_t size = task->subset.map.ep_num; ucc_rank_t rank = task->subset.myrank; void * sbuf = args->src.info.buffer; ucc_memory_type_t mem_type = args->dst.info_v.mem_type; ucc_datatype_t dt = args->dst.info_v.datatype; size_t dt_size = ucc_dt_size(dt); ucc_rank_t sendto = (rank + 1) % size; ucc_rank_t recvfrom = (rank - 1 + size) % size; ucp_send_nbx_callback_t cb[2] = {send_completion_1, send_completion_2}; ucc_rank_t prevblock, recv_data_from; ucc_status_t status; size_t max_block_size, block_offset, frag_count, frag_offset, final_offset; int step, is_avg, id; void * r_scratch, *s_scratch[2], *reduce_target; volatile char *busy; final_offset = 0; if (UCC_IS_INPLACE(*args)) { sbuf = args->dst.info_v.buffer; final_offset = get_block_offset(args, UCC_TL_TEAM_RANK(team)); } sendto = ucc_ep_map_eval(task->reduce_scatterv_ring.inv_map, sendto); recvfrom = ucc_ep_map_eval(task->reduce_scatterv_ring.inv_map, recvfrom); if (team->cfg.use_reordering) { sendto = ucc_ep_map_eval(task->subset.map, sendto); recvfrom = ucc_ep_map_eval(task->subset.map, recvfrom); } max_block_size = task->reduce_scatterv_ring.max_block_count * dt_size; busy = task->reduce_scatterv_ring.s_scratch_busy; r_scratch = task->reduce_scatterv_ring.scratch; s_scratch[0] = PTR_OFFSET(r_scratch, max_block_size); s_scratch[1] = PTR_OFFSET(s_scratch[0], max_block_size); if (UCC_INPROGRESS == ucc_tl_ucp_test_ring(task)) { return; } while (task->tagged.recv_posted > 0) { /* always have at least 1 send completion, ie 1 free slot */ ucc_assert(!busy[0] || !busy[1]); id = busy[0] ? 1 : 0; reduce_target = s_scratch[id]; step = task->tagged.send_posted; prevblock = (rank - 1 - step + size) % size; prevblock = ucc_ep_map_eval(task->reduce_scatterv_ring.inv_map, prevblock); /* reduction */ ucc_assert(task->tagged.recv_posted == task->tagged.recv_completed); ucc_assert(task->tagged.recv_posted < size); if (team->cfg.use_reordering) { prevblock = ucc_ep_map_eval(task->subset.map, prevblock); } ucc_ring_frag_count(task, prevblock, &frag_count); ucc_ring_frag_block_offset(task, prevblock, &block_offset, &frag_offset); if (task->tagged.recv_completed == size - 1) { reduce_target = PTR_OFFSET(args->dst.info_v.buffer, (frag_offset + final_offset) * dt_size); } is_avg = (args->op == UCC_OP_AVG) && (task->tagged.recv_completed == (size - 1)); if (UCC_OK != (status = ucc_dt_reduce( r_scratch, PTR_OFFSET(sbuf, (block_offset + frag_offset) * dt_size), reduce_target, frag_count, dt, args, is_avg ? UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA : 0, AVG_ALPHA(task), task->reduce_scatterv_ring.executor, &task->reduce_scatterv_ring.etask))) { tl_error(UCC_TASK_LIB(task), "failed to perform dt reduction"); task->super.status = status; return; } EXEC_TASK_WAIT(task->reduce_scatterv_ring.etask); if (task->tagged.recv_completed == size - 1) { task->tagged.recv_posted = task->tagged.recv_completed = 0; break; } ucc_assert(task->tagged.send_posted - task->tagged.send_completed <= 1); ucc_assert(task->tagged.send_posted < size); busy[id] = 1; UCPCHECK_GOTO(ucc_tl_ucp_send_cb(reduce_target, frag_count * dt_size, mem_type, sendto, team, task, cb[id], (void *)task), task, out); recv_data_from = (rank - 2 - step + size) % size; recv_data_from = ucc_ep_map_eval(task->reduce_scatterv_ring.inv_map, recv_data_from); if (team->cfg.use_reordering) { recv_data_from = ucc_ep_map_eval(task->subset.map, recv_data_from); } ucc_ring_frag_count(task, recv_data_from, &frag_count); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(r_scratch, frag_count * dt_size, mem_type, recvfrom, team, task), task, out); if (UCC_INPROGRESS == ucc_tl_ucp_test_ring(task)) { return; } } if (UCC_INPROGRESS == ucc_tl_ucp_test(task)) { return; } task->super.status = UCC_OK; out: return; } static ucc_status_t ucc_tl_ucp_reduce_scatterv_ring_start(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_ucp_team_t *team = TASK_TEAM(task); ucc_rank_t size = task->subset.map.ep_num; ucc_rank_t rank = task->subset.myrank; ucc_datatype_t dt = args->dst.info_v.datatype; size_t dt_size = ucc_dt_size(dt); ucc_memory_type_t mem_type = args->dst.info_v.mem_type; void *sbuf = args->src.info.buffer; int step = 0; ucc_rank_t sendto = (rank + 1) % size; ucc_rank_t recvfrom = (rank - 1 + size) % size; ucc_rank_t recv_block = (rank - 2 - step + size) % size; ucc_rank_t send_block = (rank - 1 - step + size) % size; size_t block_offset, frag_count, frag_offset; void *r_scratch; ucc_status_t status; ucc_tl_ucp_task_reset(task, UCC_INPROGRESS); if (UCC_IS_INPLACE(*args)) { sbuf = args->dst.info_v.buffer; } status = ucc_coll_task_get_executor(&task->super, &task->reduce_scatterv_ring.executor); if (ucc_unlikely(status != UCC_OK)) { return status; } r_scratch = task->reduce_scatterv_ring.scratch; sendto = ucc_ep_map_eval(task->reduce_scatterv_ring.inv_map, sendto); recvfrom = ucc_ep_map_eval(task->reduce_scatterv_ring.inv_map, recvfrom); recv_block = ucc_ep_map_eval(task->reduce_scatterv_ring.inv_map, recv_block); send_block = ucc_ep_map_eval(task->reduce_scatterv_ring.inv_map, send_block); if (team->cfg.use_reordering) { sendto = ucc_ep_map_eval(task->subset.map, sendto); recvfrom = ucc_ep_map_eval(task->subset.map, recvfrom); recv_block = ucc_ep_map_eval(task->subset.map, recv_block); send_block = ucc_ep_map_eval(task->subset.map, send_block); } ucc_ring_frag_count(task, recv_block, &frag_count); UCPCHECK_GOTO(ucc_tl_ucp_recv_nb(r_scratch, frag_count * dt_size, mem_type, recvfrom, team, task), task, out); ucc_ring_frag_count(task, send_block, &frag_count); ucc_ring_frag_block_offset(task, send_block, &block_offset, &frag_offset); UCPCHECK_GOTO(ucc_tl_ucp_send_nb( PTR_OFFSET(sbuf, (block_offset + frag_offset) * dt_size), frag_count * dt_size, mem_type, sendto, team, task), task, out); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); out: return task->super.status; } static ucc_status_t ucc_tl_ucp_reduce_scatterv_ring_finalize(ucc_coll_task_t *coll_task) { ucc_tl_ucp_task_t *task = ucc_derived_of(coll_task, ucc_tl_ucp_task_t); if (task->reduce_scatterv_ring.frag == REVERSED_FRAG) { ucc_ep_map_destroy(&task->reduce_scatterv_ring.inv_map); } return ucc_tl_ucp_coll_finalize(coll_task); } static ucc_status_t ucc_tl_ucp_reduce_scatterv_ring_init_subset( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h, ucc_subset_t *subsets, int n_frags, int frag, void *scratch, size_t max_block_count) { ucc_tl_ucp_task_t *task; ucc_tl_ucp_team_t *tl_team; ucc_status_t status; task = ucc_tl_ucp_init_task(coll_args, team); tl_team = TASK_TEAM(task); task->super.post = ucc_tl_ucp_reduce_scatterv_ring_start; task->super.progress = ucc_tl_ucp_reduce_scatterv_ring_progress; task->super.finalize = ucc_tl_ucp_reduce_scatterv_ring_finalize; task->subset.map = subsets[frag].map; task->subset.myrank = subsets[frag].myrank; if (frag == REVERSED_FRAG) { if (tl_team->cfg.use_reordering) { task->subset.map = subsets[0].map; } status = ucc_ep_map_create_inverse(subsets[frag].map, &task->reduce_scatterv_ring.inv_map, frag && tl_team->cfg.use_reordering); if (UCC_OK != status) { return status; } } else { task->reduce_scatterv_ring.inv_map.type = UCC_EP_MAP_FULL; task->reduce_scatterv_ring.inv_map.ep_num = task->subset.map.ep_num; } task->reduce_scatterv_ring.n_frags = n_frags; task->reduce_scatterv_ring.frag = frag; task->reduce_scatterv_ring.scratch = scratch; task->reduce_scatterv_ring.max_block_count = max_block_count; task->reduce_scatterv_ring.s_scratch_busy[0] = 0; task->reduce_scatterv_ring.s_scratch_busy[1] = 0; *task_h = &task->super; return UCC_OK; } static ucc_status_t ucc_tl_ucp_reduce_scatterv_ring_sched_post(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); int i; for (i = 0; i < schedule->n_tasks; i++) { schedule->tasks[i]->bargs.args.src = task->bargs.args.src; schedule->tasks[i]->bargs.args.dst = task->bargs.args.dst; } return ucc_schedule_start(task); } static ucc_status_t ucc_tl_ucp_reduce_scatterv_ring_sched_finalize(ucc_coll_task_t *task) { ucc_tl_ucp_schedule_t *schedule = ucc_derived_of(task, ucc_tl_ucp_schedule_t); ucc_status_t status; ucc_mc_free(schedule->scratch_mc_header); status = ucc_schedule_finalize(task); ucc_tl_ucp_put_schedule(&schedule->super.super); return status; } ucc_status_t ucc_tl_ucp_reduce_scatterv_ring_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h) { ucc_tl_ucp_team_t *tl_team = ucc_derived_of(team, ucc_tl_ucp_team_t); ucc_rank_t size = UCC_TL_TEAM_SIZE(tl_team); ucc_datatype_t dt = coll_args->args.dst.info_v.datatype; size_t dt_size = ucc_dt_size(dt); ucc_memory_type_t mem_type = coll_args->args.dst.info_v.mem_type; int bidir = UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.reduce_scatterv_ring_bidirectional; size_t to_alloc_per_set, max_segcount, count_per_set, count; ucc_tl_ucp_schedule_t *tl_schedule; ucc_schedule_t *schedule; ucc_coll_task_t *ctask; ucc_sbgp_t *sbgp; ucc_status_t status; ucc_subset_t s[2]; int i, n_subsets; if (UCC_TL_UCP_TEAM_LIB(tl_team)->cfg.reduce_avg_pre_op && coll_args->args.op == UCC_OP_AVG) { return UCC_ERR_NOT_SUPPORTED; } if (UCC_IS_INPLACE(coll_args->args)) { count = ucc_coll_args_get_total_count( &coll_args->args, coll_args->args.dst.info_v.counts, size); } else { count = coll_args->args.src.info.count; } status = ucc_tl_ucp_get_schedule(tl_team, coll_args, &tl_schedule); if (ucc_unlikely(UCC_OK != status)) { return status; } schedule = &tl_schedule->super.super; /* if count == size then we have 1 elem per rank, not enough to split into 2 sets */ n_subsets = (bidir && (count > size)) ? 2 : 1; if (tl_team->cfg.use_reordering) { sbgp = ucc_topo_get_sbgp(tl_team->topo, UCC_SBGP_FULL_HOST_ORDERED); s[0].myrank = sbgp->group_rank; s[0].map = sbgp->map; } else { s[0].myrank = UCC_TL_TEAM_RANK(tl_team); s[0].map.type = UCC_EP_MAP_FULL; s[0].map.ep_num = UCC_TL_TEAM_SIZE(tl_team); } s[1].map = ucc_ep_map_create_reverse(UCC_TL_TEAM_SIZE(tl_team)); s[1].myrank = ucc_ep_map_eval(s[1].map, s[0].myrank); if (coll_args->mask & UCC_BASE_CARGS_MAX_FRAG_COUNT) { max_segcount = coll_args->max_frag_count; } else { max_segcount = ucc_coll_args_get_max_count( &coll_args->args, coll_args->args.dst.info_v.counts, size); } /* in flight we can have 2 sends from 2 differnt blocks and 1 recv: need 3 * max_segcount of scratch per set */ count_per_set = (max_segcount + n_subsets - 1) / n_subsets; to_alloc_per_set = count_per_set * 3; UCC_CHECK_GOTO(ucc_mc_alloc(&tl_schedule->scratch_mc_header, to_alloc_per_set * dt_size * n_subsets, mem_type), out, status); for (i = 0; i < n_subsets; i++) { UCC_CHECK_GOTO(ucc_tl_ucp_reduce_scatterv_ring_init_subset( coll_args, team, &ctask, s, n_subsets, i, PTR_OFFSET(tl_schedule->scratch_mc_header->addr, to_alloc_per_set * i * dt_size), count_per_set), out_free, status); ctask->n_deps = 1; UCC_CHECK_GOTO(ucc_schedule_add_task(schedule, ctask), out_free, status); UCC_CHECK_GOTO(ucc_event_manager_subscribe( &schedule->super, UCC_EVENT_SCHEDULE_STARTED, ctask, ucc_task_start_handler), out_free, status); } schedule->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; schedule->super.post = ucc_tl_ucp_reduce_scatterv_ring_sched_post; schedule->super.finalize = ucc_tl_ucp_reduce_scatterv_ring_sched_finalize; *task_h = &schedule->super; return UCC_OK; out_free: ucc_mc_free(tl_schedule->scratch_mc_header); out: ucc_tl_ucp_put_schedule(schedule); return status; } ucc-1.8.0/src/components/tl/ucp/reduce_scatterv/reduce_scatterv.c0000664000175000017500000000110415211535620025344 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "reduce_scatterv.h" #include "utils/ucc_coll_utils.h" ucc_base_coll_alg_info_t ucc_tl_ucp_reduce_scatterv_algs[UCC_TL_UCP_REDUCE_SCATTERV_ALG_LAST + 1] = { [UCC_TL_UCP_REDUCE_SCATTERV_ALG_RING] = {.id = UCC_TL_UCP_REDUCE_SCATTERV_ALG_RING, .name = "ring", .desc = "O(N) ring"}, [UCC_TL_UCP_REDUCE_SCATTERV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; ucc-1.8.0/src/components/tl/ucp/tl_ucp.c0000664000175000017500000004544015211535620020301 0ustar alastairalastair/** * Copyright (c) 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_ucp.h" #include "utils/ucc_malloc.h" #include "components/mc/ucc_mc.h" #include "components/mc/base/ucc_mc_base.h" #include "allreduce/allreduce.h" #include "bcast/bcast.h" #include "barrier/barrier.h" #include "alltoall/alltoall.h" #include "alltoallv/alltoallv.h" #include "allgather/allgather.h" #include "allgatherv/allgatherv.h" #include "reduce_scatter/reduce_scatter.h" #include "reduce_scatterv/reduce_scatterv.h" #include "reduce/reduce.h" #include "gather/gather.h" #include "gatherv/gatherv.h" #include "fanout/fanout.h" #include "fanin/fanin.h" #include "scatterv/scatterv.h" const char *ucc_tl_ucp_alltoall_onesided_names[] = { [UCC_TL_UCP_ALLTOALL_ONESIDED_PUT] = "put", [UCC_TL_UCP_ALLTOALL_ONESIDED_GET] = "get", [UCC_TL_UCP_ALLTOALL_ONESIDED_AUTO] = "auto", [UCC_TL_UCP_ALLTOALL_ONESIDED_LAST] = NULL }; ucc_status_t ucc_tl_ucp_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr); ucc_status_t ucc_tl_ucp_get_lib_properties(ucc_base_lib_properties_t *prop); ucc_status_t ucc_tl_ucp_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *base_attr); ucc_status_t ucc_tl_ucp_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_ucp_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **pack_buffer); ucc_status_t ucc_tl_ucp_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *memh); ucc_config_field_t ucc_tl_ucp_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_ucp_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_lib_config_table)}, {"ALLGATHER_BATCHED_NUM_POSTS", "auto", "Maximum number of outstanding send and receive messages in allgather " "batched algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, allgather_batched_num_posts), UCC_CONFIG_TYPE_ULUNITS}, {"ALLTOALL_PAIRWISE_NUM_POSTS", "auto", "Maximum number of outstanding send and receive messages in alltoall " "pairwise algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, alltoall_pairwise_num_posts), UCC_CONFIG_TYPE_ULUNITS}, {"ALLTOALLV_PAIRWISE_NUM_POSTS", "auto", "Maximum number of outstanding send and receive messages in alltoallv " "pairwise algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, alltoallv_pairwise_num_posts), UCC_CONFIG_TYPE_ULUNITS}, /* TODO: add radix to config once it's fully supported by the algorithm {"ALLTOALLV_HYBRID_RADIX", "2", "Radix of the Hybrid Alltoallv algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, alltoallv_hybrid_radix), UCC_CONFIG_TYPE_UINT}, */ {"ALLTOALLV_HYBRID_NUM_SCRATCH_SENDS", "1", "Number of send operations issued from scratch buffer per radix step", ucc_offsetof(ucc_tl_ucp_lib_config_t, alltoallv_hybrid_num_scratch_sends), UCC_CONFIG_TYPE_UINT}, {"ALLTOALLV_HYBRID_NUM_SCRATCH_RECVS", "3", "Number of recv operations issued from scratch buffer per radix step", ucc_offsetof(ucc_tl_ucp_lib_config_t, alltoallv_hybrid_num_scratch_recvs), UCC_CONFIG_TYPE_UINT}, {"ALLTOALLV_HYBRID_PAIRWISE_NUM_POSTS", "3", "The maximum number of pairwise messages to send before waiting for " "completion", ucc_offsetof(ucc_tl_ucp_lib_config_t, alltoallv_hybrid_pairwise_num_posts), UCC_CONFIG_TYPE_UINT}, {"ALLTOALLV_HYBRID_BUFF_SIZE", "256k", "Total size of scratch buffer, used for sends and receives", ucc_offsetof(ucc_tl_ucp_lib_config_t, alltoallv_hybrid_buff_size), UCC_CONFIG_TYPE_MEMUNITS}, {"ALLTOALLV_HYBRID_CHUNK_BYTE_LIMIT", "12k", "Max size of data send in pairwise step of hybrid alltoallv algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, alltoallv_hybrid_chunk_byte_limit), UCC_CONFIG_TYPE_MEMUNITS}, {"ALLTOALL_ONESIDED_PERCENT_BW", "100", "Percentage (1-100) of NIC bandwidth to use for congestion avoidance", ucc_offsetof(ucc_tl_ucp_lib_config_t, alltoall_onesided_percent_bw), UCC_CONFIG_TYPE_UINT}, {"ALLTOALL_ONESIDED_ALG", "auto", "Algorithm to be used with onesided Alltoall", ucc_offsetof(ucc_tl_ucp_lib_config_t, alltoall_onesided_alg), UCC_CONFIG_TYPE_ENUM(ucc_tl_ucp_alltoall_onesided_names)}, {"KN_RADIX", "0", "Radix of all algorithms based on knomial pattern. When set to a " "positive value it is used as a convenience parameter to set all " "other KN_RADIX values", ucc_offsetof(ucc_tl_ucp_lib_config_t, kn_radix), UCC_CONFIG_TYPE_UINT}, {"BARRIER_KN_RADIX", "8", "Radix of the recursive-knomial barrier algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, barrier_kn_radix), UCC_CONFIG_TYPE_UINT}, {"FANIN_KN_RADIX", "4", "Radix of the knomial tree fanin algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, fanin_kn_radix), UCC_CONFIG_TYPE_UINT}, {"FANOUT_KN_RADIX", "4", "Radix of the knomial tree fanout algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, fanout_kn_radix), UCC_CONFIG_TYPE_UINT}, {"ALLREDUCE_KN_RADIX", "auto", "Radix of the recursive-knomial allreduce algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, allreduce_kn_radix), UCC_CONFIG_TYPE_UINT_RANGED}, {"ALLREDUCE_SLIDING_WIN_BUF_SIZE", "65536", "Buffer size of the sliding window allreduce algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, allreduce_sliding_window_buf_size), UCC_CONFIG_TYPE_MEMUNITS}, {"ALLREDUCE_SLIDING_WIN_PUT_WINDOW_SIZE", "0", "Max concurrent puts in SW Allreduce. 0 means set to team size", ucc_offsetof(ucc_tl_ucp_lib_config_t, allreduce_sliding_window_put_window_size), UCC_CONFIG_TYPE_UINT}, {"ALLREDUCE_SLIDING_WIN_NUM_GET_BUFS", "0", "Number of get buffers for sliding window AR. 0 means set to team size", ucc_offsetof(ucc_tl_ucp_lib_config_t, allreduce_sliding_window_num_get_bufs), UCC_CONFIG_TYPE_UINT}, {"ALLREDUCE_SRA_KN_RADIX", "auto", "Radix of the scatter-reduce-allgather (SRA) knomial allreduce algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, allreduce_sra_kn_radix), UCC_CONFIG_TYPE_UINT_RANGED}, {"ALLREDUCE_SRA_KN_PIPELINE", "auto", "Pipelining settings for SRA Knomial allreduce algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, allreduce_sra_kn_pipeline), UCC_CONFIG_TYPE_PIPELINE_PARAMS}, {"REDUCE_SCATTER_KN_RADIX", "4", "Radix of the knomial reduce-scatter algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, reduce_scatter_kn_radix), UCC_CONFIG_TYPE_UINT}, {"ALLGATHER_KN_RADIX", "auto", "Radix of the knomial allgather algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, allgather_kn_radix), UCC_CONFIG_TYPE_UINT_RANGED}, {"BCAST_KN_RADIX", "4", "Radix of the recursive-knomial bcast algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, bcast_kn_radix), UCC_CONFIG_TYPE_UINT}, {"BCAST_SAG_KN_RADIX", "auto", "Radix of the scatter-allgather (SAG) knomial bcast algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, bcast_sag_kn_radix), UCC_CONFIG_TYPE_UINT_RANGED}, {"REDUCE_KN_RADIX", "4", "Radix of the knomial tree reduce algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, reduce_kn_radix), UCC_CONFIG_TYPE_UINT}, {"REDUCE_SRG_KN_PIPELINE", "auto", "Pipelining settings for SRG Knomial reduce algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, reduce_srg_kn_pipeline), UCC_CONFIG_TYPE_PIPELINE_PARAMS}, {"REDUCE_SRG_KN_RADIX", "auto", "Radix of the scatter-reduce-gather (SRG) knomial reduce algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, reduce_srg_kn_radix), UCC_CONFIG_TYPE_UINT_RANGED}, {"GATHER_KN_RADIX", "4", "Radix of the knomial tree gather algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, gather_kn_radix), UCC_CONFIG_TYPE_UINT}, {"GATHERV_LINEAR_NUM_POSTS", "0", "Maximum number of outstanding send and receive messages in gatherv " "linear algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, gatherv_linear_num_posts), UCC_CONFIG_TYPE_UINT}, {"SCATTER_KN_RADIX", "4", "Radix of the knomial scatter algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, scatter_kn_radix), UCC_CONFIG_TYPE_UINT}, {"SCATTER_KN_ENABLE_RECV_ZCOPY", "auto", "Receive scatter data to user buffer with correct offset using zcopy", ucs_offsetof(ucc_tl_ucp_lib_config_t, scatter_kn_enable_recv_zcopy), UCC_CONFIG_TYPE_ON_OFF_AUTO}, {"SCATTERV_LINEAR_NUM_POSTS", "16", "Maximum number of outstanding send and receive messages in scatterv " "linear algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, scatterv_linear_num_posts), UCC_CONFIG_TYPE_UINT}, {"REDUCE_AVG_PRE_OP", "1", "Reduce will perform division by team_size in early stages of the " "algorithm,\n" "else - in result", ucc_offsetof(ucc_tl_ucp_lib_config_t, reduce_avg_pre_op), UCC_CONFIG_TYPE_BOOL}, {"REDUCE_SCATTER_RING_BIDIRECTIONAL", "y", "Launch 2 inverted rings concurrently during ReduceScatter Ring algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, reduce_scatter_ring_bidirectional), UCC_CONFIG_TYPE_BOOL}, {"REDUCE_SCATTERV_RING_BIDIRECTIONAL", "y", "Launch 2 inverted rings concurrently during ReduceScatterv Ring " "algorithm", ucc_offsetof(ucc_tl_ucp_lib_config_t, reduce_scatterv_ring_bidirectional), UCC_CONFIG_TYPE_BOOL}, {"USE_TOPO", "try", "Allow usage of tl ucp topo", ucc_offsetof(ucc_tl_ucp_lib_config_t, use_topo), UCC_CONFIG_TYPE_TERNARY}, {"RANKS_REORDERING", "y", "Use topology information in TL UCP to reorder ranks. Requires topo info", ucc_offsetof(ucc_tl_ucp_lib_config_t, use_reordering), UCC_CONFIG_TYPE_BOOL}, {NULL}}; const char* ucc_tl_ucp_local_copy_names[] = { [UCC_TL_UCP_LOCAL_COPY_TYPE_UCP] = "ucp", [UCC_TL_UCP_LOCAL_COPY_TYPE_MC] = "mc", [UCC_TL_UCP_LOCAL_COPY_TYPE_EC] = "ec", [UCC_TL_UCP_LOCAL_COPY_TYPE_AUTO] = "auto", [UCC_TL_UCP_LOCAL_COPY_TYPE_LAST] = NULL }; static ucs_config_field_t ucc_tl_ucp_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_ucp_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_context_config_table)}, {"PRECONNECT", "0", "Threshold that defines the number of ranks in the UCC team/context " "below which the team/context endpoints will be preconnected during " "corresponding team/context create call", ucc_offsetof(ucc_tl_ucp_context_config_t, preconnect), UCC_CONFIG_TYPE_UINT}, {"NPOLLS", "10", "Number of ucp progress polling cycles for p2p requests testing", ucc_offsetof(ucc_tl_ucp_context_config_t, n_polls), UCC_CONFIG_TYPE_UINT}, {"OOB_NPOLLS", "20", "Number of polling cycles for oob allgather and service coll request", ucc_offsetof(ucc_tl_ucp_context_config_t, oob_npolls), UCC_CONFIG_TYPE_UINT}, {"PRE_REG_MEM", "0", "Pre Register collective memory region with UCX", ucc_offsetof(ucc_tl_ucp_context_config_t, pre_reg_mem), UCC_CONFIG_TYPE_UINT}, {"SERVICE_WORKER", "n", "If set to 0, uses the same worker for collectives and " "service. If not, creates a special worker for service collectives " "for which UCX_TL and UCX_NET_DEVICES are configured by the variables " "UCC_TL_UCP_SERVICE_TLS and UCC_TL_UCP_SERVICE_NET_DEVICES respectively", ucc_offsetof(ucc_tl_ucp_context_config_t, service_worker), UCC_CONFIG_TYPE_BOOL}, {"SERVICE_THROTTLING_THRESH", "100", "Number of call to ucc_context_progress function between two consecutive " "calls to service worker progress function", ucc_offsetof(ucc_tl_ucp_context_config_t, service_throttling_thresh), UCC_CONFIG_TYPE_UINT}, {"LOCAL_COPY_TYPE", "auto", "Determines what component is responsible for doing local copy " "during collective execution", ucc_offsetof(ucc_tl_ucp_context_config_t, local_copy_type), UCC_CONFIG_TYPE_ENUM(ucc_tl_ucp_local_copy_names)}, {"MEMTYPE_COPY_ENABLE", "y", "Allows memory type copies. This option influences protocol selection in UCX. " "See https://github.com/openucx/ucx/pull/10490 for more details.", ucc_offsetof(ucc_tl_ucp_context_config_t, memtype_copy_enable), UCC_CONFIG_TYPE_BOOL}, {"EXPORTED_MEMORY_HANDLE", "n", "If set to yes, initialize UCP context with the exported memory handle " "feature, which is useful for offload devices such as a DPU. Otherwise " "disable the use of this feature.", ucc_offsetof(ucc_tl_ucp_context_config_t, exported_memory_handle), UCC_CONFIG_TYPE_BOOL}, {NULL}}; UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_ucp_lib_t, ucc_base_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_ucp_lib_t, ucc_base_lib_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_ucp_context_t, ucc_base_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_ucp_context_t, ucc_base_context_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_ucp_team_t, ucc_base_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_tl_ucp_team_create_test(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_ucp_team_destroy(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_ucp_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); ucc_status_t ucc_tl_ucp_populate_rcache(void *addr, size_t length, ucs_memory_type_t mem_type, ucc_tl_ucp_context_t *ctx); ucc_status_t ucc_tl_ucp_service_allreduce(ucc_base_team_t *team, void *sbuf, void *rbuf, ucc_datatype_t dt, size_t count, ucc_reduction_op_t op, ucc_subset_t subset, ucc_coll_task_t **task); ucc_status_t ucc_tl_ucp_service_allgather(ucc_base_team_t *team, void *sbuf, void *rbuf, size_t msgsize, ucc_subset_t subset, ucc_coll_task_t **task_p); ucc_status_t ucc_tl_ucp_service_bcast(ucc_base_team_t *team, void *buf, size_t msgsize, ucc_rank_t root, ucc_subset_t subset, ucc_coll_task_t **task_p); ucc_status_t ucc_tl_ucp_service_test(ucc_coll_task_t *task); ucc_status_t ucc_tl_ucp_service_cleanup(ucc_coll_task_t *task); void ucc_tl_ucp_service_update_id(ucc_base_team_t *team, uint16_t id); ucc_status_t ucc_tl_ucp_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score); UCC_TL_IFACE_DECLARE(ucp, UCP); ucs_memory_type_t ucc_memtype_to_ucs[UCC_MEMORY_TYPE_LAST + 1] = { [UCC_MEMORY_TYPE_HOST] = UCS_MEMORY_TYPE_HOST, [UCC_MEMORY_TYPE_CUDA] = UCS_MEMORY_TYPE_CUDA, [UCC_MEMORY_TYPE_CUDA_MANAGED] = UCS_MEMORY_TYPE_CUDA_MANAGED, [UCC_MEMORY_TYPE_ROCM] = UCS_MEMORY_TYPE_ROCM, [UCC_MEMORY_TYPE_ROCM_MANAGED] = UCS_MEMORY_TYPE_ROCM_MANAGED, [UCC_MEMORY_TYPE_UNKNOWN] = UCS_MEMORY_TYPE_UNKNOWN }; UCC_TL_UCP_PROFILE_FUNC_VOID(ucc_tl_ucp_pre_register_mem, (team, addr, length, mem_type), ucc_tl_ucp_team_t *team, void *addr, size_t length, ucc_memory_type_t mem_type) { void *base_address = addr; size_t alloc_length = length; ucc_mem_attr_t mem_attr; ucc_status_t status; if ((addr == NULL) || (length == 0)) { return; } mem_attr.field_mask = UCC_MEM_ATTR_FIELD_BASE_ADDRESS | UCC_MEM_ATTR_FIELD_ALLOC_LENGTH; mem_attr.alloc_length = length; status = ucc_mc_get_mem_attr(addr, &mem_attr); if (ucc_likely(status == UCC_OK)) { base_address = mem_attr.base_address; alloc_length = mem_attr.alloc_length; } else { tl_warn(UCC_TL_TEAM_LIB(team), "failed to query base addr and len"); } status = ucc_tl_ucp_populate_rcache(base_address, alloc_length, ucc_memtype_to_ucs[mem_type], UCC_TL_UCP_TEAM_CTX(team)); if (ucc_unlikely(status != UCC_OK)) { tl_warn(UCC_TL_TEAM_LIB(team), "ucc_tl_ucp_mem_map failed"); } } __attribute__((constructor)) static void tl_ucp_iface_init(void) { ucc_tl_ucp.super.scoll.allgather = ucc_tl_ucp_service_allgather; ucc_tl_ucp.super.scoll.allreduce = ucc_tl_ucp_service_allreduce; ucc_tl_ucp.super.scoll.bcast = ucc_tl_ucp_service_bcast; ucc_tl_ucp.super.scoll.update_id = ucc_tl_ucp_service_update_id; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLGATHER)] = ucc_tl_ucp_allgather_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLGATHERV)] = ucc_tl_ucp_allgatherv_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLREDUCE)] = ucc_tl_ucp_allreduce_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLTOALL)] = ucc_tl_ucp_alltoall_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLTOALLV)] = ucc_tl_ucp_alltoallv_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_BARRIER)] = ucc_tl_ucp_barrier_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_BCAST)] = ucc_tl_ucp_bcast_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_FANIN)] = ucc_tl_ucp_fanin_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_FANOUT)] = ucc_tl_ucp_fanout_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_GATHER)] = ucc_tl_ucp_gather_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_GATHERV)] = ucc_tl_ucp_gatherv_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_REDUCE)] = ucc_tl_ucp_reduce_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_REDUCE_SCATTER)] = ucc_tl_ucp_reduce_scatter_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_REDUCE_SCATTERV)] = ucc_tl_ucp_reduce_scatterv_algs; ucc_tl_ucp.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_SCATTERV)] = ucc_tl_ucp_scatterv_algs; /* no need to check return value, plugins can be absent */ (void)ucc_components_load("tlcp_ucp", &ucc_tl_ucp.super.coll_plugins); } ucc-1.8.0/src/components/tl/sharp/0000775000175000017500000000000015211535620017166 5ustar alastairalastairucc-1.8.0/src/components/tl/sharp/tl_sharp_team.c0000664000175000017500000002731515211535620022164 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_sharp_coll.h" #include "components/mc/ucc_mc.h" #include "core/ucc_ee.h" #include "coll_score/ucc_coll_score.h" #include "core/ucc_team.h" #include UCC_CLASS_INIT_FUNC(ucc_tl_sharp_team_t, ucc_base_context_t *tl_context, const ucc_base_team_params_t *params) { ucc_tl_sharp_context_t *ctx = ucc_derived_of(tl_context, ucc_tl_sharp_context_t); struct sharp_coll_context *sharp_ctx = ctx->sharp_context; struct sharp_coll_comm_init_spec comm_spec; int ret; ucc_status_t status; ucc_subset_t set; if (!(params->params.mask & UCC_TEAM_PARAM_FIELD_OOB)) { tl_debug(ctx->super.super.lib, "team OOB required for sharp team"); return UCC_ERR_INVALID_PARAM; } UCC_CLASS_CALL_SUPER_INIT(ucc_tl_team_t, &ctx->super, params); self->sharp_context = NULL; self->rcache = NULL; self->oob_ctx.ctx = UCC_TL_TEAM_CTX(self); set.myrank = UCC_TL_TEAM_RANK(self); set.map = UCC_TL_TEAM_MAP(self); if (UCC_TL_SHARP_TEAM_LIB(self)->cfg.use_internal_oob) { status = ucc_ep_map_create_nested(&UCC_TL_CORE_TEAM(self)->ctx_map, &UCC_TL_TEAM_MAP(self), &self->oob_ctx.subset.map); if (status != UCC_OK) { return status; } self->oob_ctx.subset.myrank = UCC_TL_TEAM_RANK(self); } else { self->oob_ctx.oob = &UCC_TL_TEAM_OOB(self); } status = ucc_topo_init(set, ctx->super.super.ucc_context->topo, &self->topo); if (UCC_OK != status) { tl_error(ctx->super.super.lib, "failed to init team topo"); if (UCC_TL_SHARP_TEAM_LIB(self)->cfg.use_internal_oob) { ucc_ep_map_destroy_nested(&self->oob_ctx.subset.map); } return status; } if (ucc_topo_max_ppn(self->topo) > ctx->cfg.team_max_ppn) { tl_debug(ctx->super.super.lib, "sharp team not supported with ppn > 1"); status = UCC_ERR_NOT_SUPPORTED; goto cleanup; } if (sharp_ctx == NULL) { status = ucc_tl_sharp_context_init(ctx, &self->sharp_context, &self->oob_ctx, self->topo); if (status != UCC_OK) { goto cleanup; } if (ctx->cfg.use_rcache) { status = ucc_tl_sharp_rcache_create(self->sharp_context, &self->rcache); if (status != UCC_OK) { tl_error(ctx->super.super.lib, "failed to create rcache"); goto cleanup; } } status = ucc_context_progress_register( tl_context->ucc_context, (ucc_context_progress_fn_t)sharp_coll_progress, self->sharp_context); if (status != UCC_OK) { tl_error(ctx->super.super.lib, "failed to register progress function"); goto cleanup; } sharp_ctx = self->sharp_context; } else { self->sharp_context = sharp_ctx; self->rcache = ctx->rcache; } #if SHARP_API > SHARP_VERSION(3, 0) if ((ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(UCC_DT_INT8)] == SHARP_DTYPE_UNKNOWN) || (ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(SHARP_DTYPE_UINT8)] == SHARP_DTYPE_UNKNOWN) || (ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(SHARP_DTYPE_BFLOAT16)] == SHARP_DTYPE_UNKNOWN)) { if (ctx->sharp_caps.support_mask.dtypes & UCC_BIT(SHARP_DTYPE_INT8)) { tl_debug(ctx->super.super.lib, "enabling support for UCC_DT_INT8"); ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(UCC_DT_INT8)] = SHARP_DTYPE_INT8; } else { tl_debug(ctx->super.super.lib, "disabling support for UCC_DT_INT8"); ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(UCC_DT_INT8)] = SHARP_DTYPE_NULL; } if (ctx->sharp_caps.support_mask.dtypes & UCC_BIT(SHARP_DTYPE_UINT8)) { tl_debug(ctx->super.super.lib, "enabling support for UCC_DT_UINT8"); ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(UCC_DT_UINT8)] = SHARP_DTYPE_UINT8; } else { tl_debug(ctx->super.super.lib, "disabling support for UCC_DT_UINT8"); ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(UCC_DT_UINT8)] = SHARP_DTYPE_NULL; } if (ctx->sharp_caps.support_mask.dtypes & UCC_BIT(SHARP_DTYPE_BFLOAT16)) { tl_debug(ctx->super.super.lib, "enabling support for UCC_DT_BFLOAT16"); ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(UCC_DT_BFLOAT16)] = SHARP_DTYPE_BFLOAT16; } else { tl_debug(ctx->super.super.lib, "disabling support for UCC_DT_BFLOAT16"); ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(UCC_DT_BFLOAT16)] = SHARP_DTYPE_NULL; } } #endif comm_spec.rank = UCC_TL_TEAM_RANK(self); comm_spec.size = UCC_TL_TEAM_SIZE(self); comm_spec.group_world_ranks = NULL; comm_spec.oob_ctx = &self->oob_ctx; ret = sharp_coll_comm_init(sharp_ctx, &comm_spec, &self->sharp_comm); if (ret < 0) { tl_debug(ctx->super.super.lib, "sharp group create failed:%s(%d)", sharp_coll_strerror(ret), ret); status = UCC_ERR_NO_RESOURCE; goto cleanup; } tl_debug(self->super.super.context->lib, "initialized tl team: %p size:%d", self, UCC_TL_TEAM_SIZE(self)); return UCC_OK; cleanup: if (ctx->cfg.context_per_team) { if (self->rcache) { ucc_rcache_destroy(self->rcache); } if (self->sharp_context) { ucc_context_progress_deregister( tl_context->ucc_context, (ucc_context_progress_fn_t)sharp_coll_progress, self->sharp_context); sharp_coll_finalize(self->sharp_context); } } ucc_topo_cleanup(self->topo); if (UCC_TL_SHARP_TEAM_LIB(self)->cfg.use_internal_oob) { ucc_ep_map_destroy_nested(&self->oob_ctx.subset.map); } return status; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_sharp_team_t) { ucc_tl_sharp_context_t *ctx = ucc_derived_of(UCC_TL_TEAM_CTX(self), ucc_tl_sharp_context_t); tl_debug(self->super.super.context->lib, "finalizing tl team: %p", self); sharp_coll_comm_destroy(self->sharp_comm); ucc_topo_cleanup(self->topo); if (ctx->cfg.context_per_team) { if (UCC_TL_SHARP_TEAM_LIB(self)->cfg.use_internal_oob) { if (self->rcache != NULL) { ucc_rcache_destroy(self->rcache); } if (self->sharp_context != NULL) { ucc_context_progress_deregister( self->super.super.context->ucc_context, (ucc_context_progress_fn_t)sharp_coll_progress, self->sharp_context); sharp_coll_finalize(self->sharp_context); } } } if (UCC_TL_SHARP_TEAM_LIB(self)->cfg.use_internal_oob) { ucc_ep_map_destroy_nested(&self->oob_ctx.subset.map); } } UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_sharp_team_t, ucc_base_team_t); UCC_CLASS_DEFINE(ucc_tl_sharp_team_t, ucc_tl_team_t); ucc_status_t ucc_tl_sharp_team_destroy(ucc_base_team_t *tl_team) { UCC_CLASS_DELETE_FUNC_NAME(ucc_tl_sharp_team_t)(tl_team); return UCC_OK; } /* sharp team create is blocking, return UCC_OK always */ ucc_status_t ucc_tl_sharp_team_create_test(ucc_base_team_t *tl_team) //NOLINT { return UCC_OK; } static ucc_status_t ucc_tl_sharp_coll_finalize(ucc_coll_task_t *coll_task) { ucc_tl_sharp_task_t *task = ucc_derived_of(coll_task, ucc_tl_sharp_task_t); tl_debug(UCC_TASK_LIB(task), "finalizing coll task %p", task); UCC_TL_SHARP_PROFILE_REQUEST_FREE(task); ucc_mpool_put(task); return UCC_OK; } ucc_status_t ucc_tl_sharp_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_sharp_context_t *sharp_ctx = ucc_derived_of(team->context, ucc_tl_sharp_context_t); ucc_tl_sharp_task_t *task; ucc_status_t status; task = ucc_mpool_get(&sharp_ctx->req_mp); ucc_coll_task_init(&task->super, coll_args, team); UCC_TL_SHARP_PROFILE_REQUEST_NEW(task, "tl_sharp_task", 0); task->req_handle = NULL; task->super.finalize = ucc_tl_sharp_coll_finalize; switch (coll_args->args.coll_type) { case UCC_COLL_TYPE_ALLREDUCE: status = ucc_tl_sharp_allreduce_init(task); break; case UCC_COLL_TYPE_BARRIER: status = ucc_tl_sharp_barrier_init(task); break; case UCC_COLL_TYPE_BCAST: status = ucc_tl_sharp_bcast_init(task); break; #if HAVE_DECL_SHARP_COLL_DO_REDUCE_SCATTER case UCC_COLL_TYPE_REDUCE_SCATTER: status = ucc_tl_sharp_reduce_scatter_init(task); break; #endif #if HAVE_DECL_SHARP_COLL_DO_ALLGATHER case UCC_COLL_TYPE_ALLGATHER: status = ucc_tl_sharp_allgather_init(task); break; #endif default: tl_debug(UCC_TASK_LIB(task), "collective %d is not supported by sharp tl", coll_args->args.coll_type); status = UCC_ERR_NOT_SUPPORTED; } if (status != UCC_OK) { goto free_task; } tl_debug(UCC_TASK_LIB(task), "init coll task %p", task); *task_h = &task->super; return status; free_task: ucc_mpool_put(task); return status; } ucc_status_t ucc_tl_sharp_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p) { ucc_tl_sharp_team_t *team = ucc_derived_of(tl_team, ucc_tl_sharp_team_t); ucc_base_context_t *ctx = UCC_TL_TEAM_CTX(team); ucc_coll_score_t *score; ucc_status_t status; ucc_coll_score_team_info_t team_info; int i; team_info.alg_fn = NULL; team_info.default_score = UCC_TL_SHARP_DEFAULT_SCORE; team_info.init = ucc_tl_sharp_coll_init; team_info.num_mem_types = 0; team_info.supported_mem_types = NULL; /* all memory types supported*/ team_info.supported_colls = UCC_TL_SHARP_SUPPORTED_COLLS; team_info.size = UCC_TL_TEAM_SIZE(team); /* There can be a different logic for different coll_type/mem_type. Right now just init everything the same way. */ status = ucc_coll_score_build_default(tl_team, UCC_TL_SHARP_DEFAULT_SCORE, ucc_tl_sharp_coll_init, UCC_TL_SHARP_SUPPORTED_COLLS, NULL, 0, &score); if (UCC_OK != status) { return status; } for (i = 0; i < UCC_TL_SHARP_N_DEFAULT_ALG_SELECT_STR; i++) { status = ucc_coll_score_update_from_str( ucc_tl_sharp_default_alg_select_str[i], &team_info, &team->super.super, score); if (UCC_OK != status) { tl_error(tl_team->context->lib, "failed to apply default coll select setting: %s", ucc_tl_sharp_default_alg_select_str[i]); goto err; } } if (strlen(ctx->score_str) > 0) { status = ucc_coll_score_update_from_str(ctx->score_str, &team_info, &team->super.super, score); /* If INVALID_PARAM - User provided incorrect input - try to proceed */ if ((status < 0) && (status != UCC_ERR_INVALID_PARAM) && (status != UCC_ERR_NOT_SUPPORTED)) { goto err; } } *score_p = score; return UCC_OK; err: ucc_coll_score_free(score); return status; } ucc-1.8.0/src/components/tl/sharp/tl_sharp_coll.c0000664000175000017500000005771215211535620022173 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_sharp_coll.h" #include "components/mc/ucc_mc.h" #include "core/ucc_ee.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include #include /* Disable allgather and reduce_scatter for message sizes 0-16k since SHARP SAT is only enabled by default for messages larger than 16k. * TODO: update with precise threshold once SHARP query can return SAT threshold. */ #define UCC_TL_SHARP_ALLGATHER_DEFAULT_ALG_SELECT_STR \ "allgather:0-16k:0" #define UCC_TL_SHARP_REDUCE_SCATTER_DEFAULT_ALG_SELECT_STR \ "reduce_scatter:0-16k:0" const char *ucc_tl_sharp_default_alg_select_str[UCC_TL_SHARP_N_DEFAULT_ALG_SELECT_STR] = { UCC_TL_SHARP_ALLGATHER_DEFAULT_ALG_SELECT_STR, UCC_TL_SHARP_REDUCE_SCATTER_DEFAULT_ALG_SELECT_STR}; enum sharp_datatype ucc_to_sharp_dtype[] = { [UCC_DT_PREDEFINED_ID(UCC_DT_INT16)] = SHARP_DTYPE_SHORT, [UCC_DT_PREDEFINED_ID(UCC_DT_INT32)] = SHARP_DTYPE_INT, [UCC_DT_PREDEFINED_ID(UCC_DT_INT64)] = SHARP_DTYPE_LONG, [UCC_DT_PREDEFINED_ID(UCC_DT_INT128)] = SHARP_DTYPE_NULL, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT16)] = SHARP_DTYPE_UNSIGNED_SHORT, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT32)] = SHARP_DTYPE_UNSIGNED, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT64)] = SHARP_DTYPE_UNSIGNED_LONG, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT128)] = SHARP_DTYPE_NULL, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT16)] = SHARP_DTYPE_FLOAT_SHORT, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT32)] = SHARP_DTYPE_FLOAT, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT64)] = SHARP_DTYPE_DOUBLE, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT128)] = SHARP_DTYPE_NULL, #if SHARP_API > SHARP_VERSION(3, 0) [UCC_DT_PREDEFINED_ID(UCC_DT_INT8)] = (enum sharp_datatype)SHARP_DTYPE_UNKNOWN, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT8)] = (enum sharp_datatype)SHARP_DTYPE_UNKNOWN, [UCC_DT_PREDEFINED_ID(UCC_DT_BFLOAT16)] = (enum sharp_datatype)SHARP_DTYPE_UNKNOWN, #else [UCC_DT_PREDEFINED_ID(UCC_DT_INT8)] = SHARP_DTYPE_NULL, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT8)] = SHARP_DTYPE_NULL, [UCC_DT_PREDEFINED_ID(UCC_DT_BFLOAT16)] = SHARP_DTYPE_NULL, #endif [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT32_COMPLEX)] = SHARP_DTYPE_NULL, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT64_COMPLEX)] = SHARP_DTYPE_NULL, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT128_COMPLEX)] = SHARP_DTYPE_NULL, }; enum sharp_reduce_op ucc_to_sharp_reduce_op[] = { [UCC_OP_SUM] = SHARP_OP_SUM, [UCC_OP_PROD] = SHARP_OP_NULL, [UCC_OP_MAX] = SHARP_OP_MAX, [UCC_OP_MIN] = SHARP_OP_MIN, [UCC_OP_LAND] = SHARP_OP_LAND, [UCC_OP_LOR] = SHARP_OP_LOR, [UCC_OP_LXOR] = SHARP_OP_LXOR, [UCC_OP_BAND] = SHARP_OP_BAND, [UCC_OP_BOR] = SHARP_OP_BOR, [UCC_OP_BXOR] = SHARP_OP_BXOR, [UCC_OP_MAXLOC] = SHARP_OP_MAXLOC, [UCC_OP_MINLOC] = SHARP_OP_MINLOC, [UCC_OP_AVG] = SHARP_OP_NULL, }; enum sharp_data_memory_type ucc_to_sharp_memtype[] = { [UCC_MEMORY_TYPE_HOST] = SHARP_MEM_TYPE_HOST, [UCC_MEMORY_TYPE_CUDA] = SHARP_MEM_TYPE_CUDA, [UCC_MEMORY_TYPE_CUDA_MANAGED] = SHARP_MEM_TYPE_LAST, [UCC_MEMORY_TYPE_ROCM] = SHARP_MEM_TYPE_LAST, [UCC_MEMORY_TYPE_ROCM_MANAGED] = SHARP_MEM_TYPE_LAST, [UCC_MEMORY_TYPE_LAST] = SHARP_MEM_TYPE_LAST, }; static inline ucc_status_t ucc_tl_sharp_status_to_ucc(int status) { switch (status) { case SHARP_COLL_SUCCESS: return UCC_OK; case SHARP_COLL_ENOMEM: return UCC_ERR_NO_MEMORY; case SHARP_COLL_ENOT_SUPP: return UCC_ERR_NOT_SUPPORTED; case SHARP_COLL_EINVAL: return UCC_ERR_INVALID_PARAM; case SHARP_COLL_ENO_RESOURCE: return UCC_ERR_NO_RESOURCE; default: break; } return UCC_ERR_NO_MESSAGE; } static ucc_tl_sharp_reg_t ucc_tl_sharp_reg_null = { .mr = NULL }; static ucc_status_t ucc_tl_sharp_mem_register(ucc_tl_sharp_context_t *ctx, ucc_tl_sharp_team_t *team, void *addr, size_t length, ucc_tl_sharp_reg_t **reg) { ucc_rcache_region_t *rregion; ucc_tl_sharp_rcache_region_t *region; ucc_status_t status; ucc_tl_sharp_reg_t *r; ucc_rcache_t *rcache; struct sharp_coll_context *sharp_ctx; if (length < ctx->cfg.reg_threshold) { *reg = &ucc_tl_sharp_reg_null; return UCC_OK; } sharp_ctx = team->sharp_context; rcache = team->rcache; if (rcache) { status = ucc_rcache_get(rcache, (void *)addr, length, NULL, &rregion); if (status != UCC_OK) { tl_error(ctx->super.super.lib, "ucc_rcache_get failed"); return UCC_ERR_INVALID_PARAM; } region = ucc_derived_of(rregion, ucc_tl_sharp_rcache_region_t); *reg = ®ion->reg; } else { r = ucc_malloc(sizeof(ucc_tl_sharp_reg_t), "sharp reg"); if (!r) { tl_error(ctx->super.super.lib, "failed to allocate reg data"); return UCC_ERR_NO_MEMORY; } sharp_coll_reg_mr(sharp_ctx, addr, length, &r->mr); *reg = r; } return UCC_OK; } static ucc_status_t ucc_tl_sharp_mem_deregister(ucc_tl_sharp_team_t *team, ucc_tl_sharp_reg_t *reg) { ucc_tl_sharp_rcache_region_t *region; ucc_rcache_t *rcache; struct sharp_coll_context *sharp_ctx; if (reg == &ucc_tl_sharp_reg_null) { return UCC_OK; } sharp_ctx = team->sharp_context; rcache = team->rcache; if (rcache) { region = ucc_container_of(reg, ucc_tl_sharp_rcache_region_t, reg); ucc_rcache_region_put(rcache, ®ion->super); } else { sharp_coll_dereg_mr(sharp_ctx, reg->mr); ucc_free(reg); } return UCC_OK; } void ucc_tl_sharp_collective_progress(ucc_coll_task_t *coll_task) { ucc_tl_sharp_task_t *task = ucc_derived_of(coll_task, ucc_tl_sharp_task_t); int completed; ucc_status_t st1, st2; if (task->req_handle != NULL) { completed = sharp_coll_req_test(task->req_handle); if (completed) { st1 = UCC_OK; st2 = UCC_OK; switch(TASK_ARGS(task).coll_type){ case UCC_COLL_TYPE_ALLREDUCE: if (!UCC_IS_INPLACE(TASK_ARGS(task))) { st1 = ucc_tl_sharp_mem_deregister(TASK_TEAM(task), task->allreduce.s_mem_h); } st2 = ucc_tl_sharp_mem_deregister(TASK_TEAM(task), task->allreduce.r_mem_h); break; case UCC_COLL_TYPE_BCAST: st1 = ucc_tl_sharp_mem_deregister(TASK_TEAM(task), task->bcast.mem_h); break; case UCC_COLL_TYPE_ALLGATHER: if (!UCC_IS_INPLACE(TASK_ARGS(task))) { st1 = ucc_tl_sharp_mem_deregister(TASK_TEAM(task), task->allgather.s_mem_h); } st2 = ucc_tl_sharp_mem_deregister(TASK_TEAM(task), task->allgather.r_mem_h); break; case UCC_COLL_TYPE_REDUCE_SCATTER: if (!UCC_IS_INPLACE(TASK_ARGS(task))) { st1 = ucc_tl_sharp_mem_deregister(TASK_TEAM(task), task->reduce_scatter.s_mem_h); } st2 = ucc_tl_sharp_mem_deregister(TASK_TEAM(task), task->reduce_scatter.r_mem_h); break; default: break; } if (ucc_unlikely(st1 != UCC_OK || st2 != UCC_OK)) { tl_warn(UCC_TASK_LIB(task), "ucc_tl_sharp_mem_deregister failed"); } sharp_coll_req_free(task->req_handle); coll_task->status = UCC_OK; UCC_TL_SHARP_PROFILE_REQUEST_EVENT(coll_task, "sharp_collective_done", 0); } } } ucc_status_t ucc_tl_sharp_barrier_start(ucc_coll_task_t *coll_task) { ucc_tl_sharp_task_t *task = ucc_derived_of(coll_task, ucc_tl_sharp_task_t); ucc_tl_sharp_team_t *team = TASK_TEAM(task); int ret; UCC_TL_SHARP_PROFILE_REQUEST_EVENT(coll_task, "sharp_barrier_start", 0); ret = sharp_coll_do_barrier_nb(team->sharp_comm, &task->req_handle); if (ucc_unlikely(ret != SHARP_COLL_SUCCESS)) { tl_error(UCC_TASK_LIB(task), "sharp_coll_do_barrier_nb failed:%s", sharp_coll_strerror(ret)); coll_task->status = ucc_tl_sharp_status_to_ucc(ret); return ucc_task_complete(coll_task); } coll_task->status = UCC_INPROGRESS; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_sharp_allreduce_start(ucc_coll_task_t *coll_task) { ucc_tl_sharp_task_t *task = ucc_derived_of(coll_task, ucc_tl_sharp_task_t); ucc_tl_sharp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); size_t count = args->dst.info.count; ucc_datatype_t dt = args->dst.info.datatype; struct sharp_coll_reduce_spec reduce_spec; enum sharp_datatype sharp_type; enum sharp_reduce_op op_type; size_t data_size; int ret; UCC_TL_SHARP_PROFILE_REQUEST_EVENT(coll_task, "sharp_allreduce_start", 0); sharp_type = ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(dt)]; op_type = ucc_to_sharp_reduce_op[args->op]; data_size = ucc_dt_size(dt) * count; if (!UCC_IS_INPLACE(*args)) { ucc_tl_sharp_mem_register(TASK_CTX(task), team, args->src.info.buffer,data_size, &task->allreduce.s_mem_h); } ucc_tl_sharp_mem_register(TASK_CTX(task), team, args->dst.info.buffer, data_size, &task->allreduce.r_mem_h); if (!UCC_IS_INPLACE(*args)) { reduce_spec.sbuf_desc.buffer.ptr = args->src.info.buffer; reduce_spec.sbuf_desc.buffer.mem_handle = task->allreduce.s_mem_h->mr; reduce_spec.sbuf_desc.mem_type = ucc_to_sharp_memtype[args->src.info.mem_type]; } else { reduce_spec.sbuf_desc.buffer.ptr = args->dst.info.buffer; reduce_spec.sbuf_desc.buffer.mem_handle = task->allreduce.r_mem_h->mr; reduce_spec.sbuf_desc.mem_type = ucc_to_sharp_memtype[args->dst.info.mem_type]; } reduce_spec.sbuf_desc.buffer.length = data_size; reduce_spec.sbuf_desc.type = SHARP_DATA_BUFFER; reduce_spec.rbuf_desc.buffer.ptr = args->dst.info.buffer; reduce_spec.rbuf_desc.buffer.length = data_size; reduce_spec.rbuf_desc.buffer.mem_handle = task->allreduce.r_mem_h->mr; reduce_spec.rbuf_desc.type = SHARP_DATA_BUFFER; reduce_spec.rbuf_desc.mem_type = ucc_to_sharp_memtype[args->dst.info.mem_type]; reduce_spec.aggr_mode = SHARP_AGGREGATION_NONE; reduce_spec.length = count; reduce_spec.dtype = sharp_type; reduce_spec.op = op_type; ret = sharp_coll_do_allreduce_nb(team->sharp_comm, &reduce_spec, &task->req_handle); if (ucc_unlikely(ret != SHARP_COLL_SUCCESS)) { tl_error(UCC_TASK_LIB(task), "sharp_coll_do_allreduce_nb failed:%s", sharp_coll_strerror(ret)); coll_task->status = ucc_tl_sharp_status_to_ucc(ret); return ucc_task_complete(coll_task); } coll_task->status = UCC_INPROGRESS; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_sharp_bcast_start(ucc_coll_task_t *coll_task) { ucc_tl_sharp_task_t *task = ucc_derived_of(coll_task, ucc_tl_sharp_task_t); ucc_tl_sharp_team_t *team = TASK_TEAM(task); ucc_coll_args_t * args = &TASK_ARGS(task); ucc_datatype_t dt = args->src.info.datatype; size_t count = args->src.info.count; ucc_rank_t root = args->root; size_t data_size; struct sharp_coll_bcast_spec bcast_spec; int ret; UCC_TL_SHARP_PROFILE_REQUEST_EVENT(coll_task, "sharp_bcast_start", 0); data_size = ucc_dt_size(dt) * count; ucc_tl_sharp_mem_register(TASK_CTX(task), team, args->src.info.buffer, data_size, &task->bcast.mem_h); bcast_spec.size = data_size; bcast_spec.root = root; bcast_spec.buf_desc.type = SHARP_DATA_BUFFER; bcast_spec.buf_desc.buffer.ptr = args->src.info.buffer; bcast_spec.buf_desc.buffer.length = data_size; bcast_spec.buf_desc.buffer.mem_handle = task->bcast.mem_h->mr; bcast_spec.buf_desc.mem_type = ucc_to_sharp_memtype[args->src.info.mem_type]; ret = sharp_coll_do_bcast_nb(team->sharp_comm, &bcast_spec, &task->req_handle); if (ucc_unlikely(ret != SHARP_COLL_SUCCESS)) { if (ret == SHARP_COLL_ENOT_SUPP) { tl_debug(UCC_TASK_LIB(task), "sharp_coll_do_bcast_nb not supported, msgsize %zd", data_size); } else { tl_error(UCC_TASK_LIB(task), "sharp_coll_do_bcast_nb failed:%s", sharp_coll_strerror(ret)); } coll_task->status = ucc_tl_sharp_status_to_ucc(ret); return ucc_task_complete(coll_task); } coll_task->status = UCC_INPROGRESS; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } #if HAVE_DECL_SHARP_COLL_DO_REDUCE_SCATTER ucc_status_t ucc_tl_sharp_reduce_scatter_start(ucc_coll_task_t *coll_task) { ucc_tl_sharp_task_t *task = ucc_derived_of(coll_task, ucc_tl_sharp_task_t); ucc_tl_sharp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); size_t count = args->dst.info.count; ucc_datatype_t dt = args->dst.info.datatype; struct sharp_coll_reduce_spec reduce_spec; enum sharp_datatype sharp_type; enum sharp_reduce_op op_type; size_t src_data_size, dst_data_size; int ret; UCC_TL_SHARP_PROFILE_REQUEST_EVENT(coll_task, "sharp_reduce_scatter_start", 0); sharp_type = ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(dt)]; op_type = ucc_to_sharp_reduce_op[args->op]; src_data_size = ucc_dt_size(dt) * count * UCC_TL_TEAM_SIZE(team); dst_data_size = ucc_dt_size(dt) * count; if (!UCC_IS_INPLACE(*args)) { ucc_tl_sharp_mem_register(TASK_CTX(task), team, args->src.info.buffer, src_data_size, &task->reduce_scatter.s_mem_h); } ucc_tl_sharp_mem_register(TASK_CTX(task), team, args->dst.info.buffer, dst_data_size, &task->reduce_scatter.r_mem_h); if (!UCC_IS_INPLACE(*args)) { reduce_spec.sbuf_desc.buffer.ptr = args->src.info.buffer; reduce_spec.sbuf_desc.buffer.mem_handle = task->reduce_scatter.s_mem_h->mr; reduce_spec.sbuf_desc.mem_type = ucc_to_sharp_memtype[args->src.info.mem_type]; } else { reduce_spec.sbuf_desc.buffer.ptr = args->dst.info.buffer; reduce_spec.sbuf_desc.buffer.mem_handle = task->reduce_scatter.r_mem_h->mr; reduce_spec.sbuf_desc.mem_type = ucc_to_sharp_memtype[args->dst.info.mem_type]; } reduce_spec.sbuf_desc.buffer.length = src_data_size; reduce_spec.sbuf_desc.type = SHARP_DATA_BUFFER; reduce_spec.rbuf_desc.buffer.ptr = args->dst.info.buffer; reduce_spec.rbuf_desc.buffer.length = dst_data_size; reduce_spec.rbuf_desc.buffer.mem_handle = task->reduce_scatter.r_mem_h->mr; reduce_spec.rbuf_desc.type = SHARP_DATA_BUFFER; reduce_spec.rbuf_desc.mem_type = ucc_to_sharp_memtype[args->dst.info.mem_type]; reduce_spec.aggr_mode = SHARP_AGGREGATION_NONE; reduce_spec.length = count; reduce_spec.dtype = sharp_type; reduce_spec.op = op_type; reduce_spec.offset = 0; ret = sharp_coll_do_reduce_scatter_nb(team->sharp_comm, &reduce_spec, &task->req_handle); if (ret != SHARP_COLL_SUCCESS) { tl_error(UCC_TASK_LIB(task), "sharp_coll_do_reduce_scatter_nb failed:%s", sharp_coll_strerror(ret)); coll_task->status = ucc_tl_sharp_status_to_ucc(ret); return ucc_task_complete(coll_task); } coll_task->status = UCC_INPROGRESS; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_sharp_reduce_scatter_init(ucc_tl_sharp_task_t *task) { ucc_coll_args_t *args = &TASK_ARGS(task); if (!(TASK_CTX(task)->sharp_caps.support_mask.feature_mask & SHARP_FEATURE_SAT)) { return UCC_ERR_NOT_SUPPORTED; } if (!ucc_coll_args_is_predefined_dt(args, UCC_RANK_INVALID)) { return UCC_ERR_NOT_SUPPORTED; } if ((!UCC_IS_INPLACE(*args) && ucc_to_sharp_memtype[args->src.info.mem_type] == SHARP_MEM_TYPE_LAST) || ucc_to_sharp_memtype[args->dst.info.mem_type] == SHARP_MEM_TYPE_LAST || ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(args->dst.info.datatype)] == SHARP_DTYPE_NULL || ucc_to_sharp_reduce_op[args->op] == SHARP_OP_NULL) { return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_sharp_reduce_scatter_start; task->super.progress = ucc_tl_sharp_collective_progress; return UCC_OK; }; #endif ucc_status_t ucc_tl_sharp_allreduce_init(ucc_tl_sharp_task_t *task) { ucc_coll_args_t *args = &TASK_ARGS(task); if (!ucc_coll_args_is_predefined_dt(args, UCC_RANK_INVALID)) { return UCC_ERR_NOT_SUPPORTED; } if ((!UCC_IS_INPLACE(*args) && ucc_to_sharp_memtype[args->src.info.mem_type] == SHARP_MEM_TYPE_LAST) || ucc_to_sharp_memtype[args->dst.info.mem_type] == SHARP_MEM_TYPE_LAST || ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(args->dst.info.datatype)] == SHARP_DTYPE_NULL || ucc_to_sharp_reduce_op[args->op] == SHARP_OP_NULL) { return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_sharp_allreduce_start; task->super.progress = ucc_tl_sharp_collective_progress; return UCC_OK; } ucc_status_t ucc_tl_sharp_bcast_init(ucc_tl_sharp_task_t *task) { ucc_coll_args_t *args = &TASK_ARGS(task); size_t data_size; data_size = ucc_dt_size(args->src.info.datatype) * args->src.info.count; /* check SHARP supports memory type, dataype is contig and data size is even in case of older sharp versions */ if ((ucc_to_sharp_memtype[args->src.info.mem_type] == SHARP_MEM_TYPE_LAST) || !ucc_coll_args_is_predefined_dt(args, UCC_RANK_INVALID) || ((data_size % 2 != 0) && ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(UCC_DT_INT8)] == SHARP_DTYPE_NULL)) { return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_sharp_bcast_start; task->super.progress = ucc_tl_sharp_collective_progress; return UCC_OK; } ucc_status_t ucc_tl_sharp_barrier_init(ucc_tl_sharp_task_t *task) { task->super.post = ucc_tl_sharp_barrier_start; task->super.progress = ucc_tl_sharp_collective_progress; return UCC_OK; }; #if HAVE_DECL_SHARP_COLL_DO_ALLGATHER ucc_status_t ucc_tl_sharp_allgather_start(ucc_coll_task_t *coll_task) { ucc_tl_sharp_task_t *task = ucc_derived_of(coll_task, ucc_tl_sharp_task_t); ucc_tl_sharp_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); size_t count = args->dst.info.count; ucc_datatype_t dt = args->dst.info.datatype; struct sharp_coll_gather_spec gather_spec; size_t src_data_size, dst_data_size; int ret; ucc_status_t status; UCC_TL_SHARP_PROFILE_REQUEST_EVENT(coll_task, "sharp_allgather_start", 0); src_data_size = ucc_dt_size(dt) * count / UCC_TL_TEAM_SIZE(team); dst_data_size = ucc_dt_size(dt) * count; status = ucc_tl_sharp_mem_register(TASK_CTX(task), team, args->dst.info.buffer, dst_data_size, &task->allgather.r_mem_h); if (status != UCC_OK) { tl_error(UCC_TASK_LIB(task), "ucc_tl_sharp_mem_register failed " "for dst buffer: %s", ucc_status_string(status)); return status; } if (!UCC_IS_INPLACE(*args)) { status = ucc_tl_sharp_mem_register(TASK_CTX(task), team, args->src.info.buffer, src_data_size, &task->allgather.s_mem_h); if (status != UCC_OK) { tl_error(UCC_TASK_LIB(task), "ucc_tl_sharp_mem_register failed " "for src buffer: %s", ucc_status_string(status)); return status; } } if (!UCC_IS_INPLACE(*args)) { gather_spec.sbuf_desc.buffer.ptr = args->src.info.buffer; gather_spec.sbuf_desc.buffer.mem_handle = task->allgather.s_mem_h->mr; gather_spec.sbuf_desc.mem_type = ucc_to_sharp_memtype[args->src.info.mem_type]; } else { gather_spec.sbuf_desc.buffer.ptr = PTR_OFFSET(args->dst.info.buffer, UCC_TL_TEAM_RANK(team) * src_data_size); gather_spec.sbuf_desc.buffer.mem_handle = task->allgather.r_mem_h->mr; gather_spec.sbuf_desc.mem_type = ucc_to_sharp_memtype[args->dst.info.mem_type]; } gather_spec.sbuf_desc.buffer.length = src_data_size; gather_spec.sbuf_desc.type = SHARP_DATA_BUFFER; gather_spec.rbuf_desc.buffer.ptr = args->dst.info.buffer; gather_spec.rbuf_desc.buffer.length = dst_data_size; gather_spec.rbuf_desc.buffer.mem_handle = task->allgather.r_mem_h->mr; gather_spec.rbuf_desc.type = SHARP_DATA_BUFFER; gather_spec.rbuf_desc.mem_type = ucc_to_sharp_memtype[args->dst.info.mem_type]; gather_spec.offset = 0; gather_spec.dtype = SHARP_DTYPE_INT8; ret = sharp_coll_do_allgather_nb(team->sharp_comm, &gather_spec, &task->req_handle); if (ret != SHARP_COLL_SUCCESS) { tl_error(UCC_TASK_LIB(task), "sharp_coll_do_allgather_nb failed:%s", sharp_coll_strerror(ret)); coll_task->status = ucc_tl_sharp_status_to_ucc(ret); return ucc_task_complete(coll_task); } coll_task->status = UCC_INPROGRESS; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(team)->pq, &task->super); } ucc_status_t ucc_tl_sharp_allgather_init(ucc_tl_sharp_task_t *task) { ucc_coll_args_t *args = &TASK_ARGS(task); if (!(TASK_CTX(task)->sharp_caps.support_mask.feature_mask & SHARP_FEATURE_SAT)) { return UCC_ERR_NOT_SUPPORTED; } if (!ucc_coll_args_is_predefined_dt(args, UCC_RANK_INVALID)) { return UCC_ERR_NOT_SUPPORTED; } if ((!UCC_IS_INPLACE(*args) && ucc_to_sharp_memtype[args->src.info.mem_type] == SHARP_MEM_TYPE_LAST) || ucc_to_sharp_memtype[args->dst.info.mem_type] == SHARP_MEM_TYPE_LAST || ucc_to_sharp_dtype[UCC_DT_PREDEFINED_ID(args->dst.info.datatype)] == SHARP_DTYPE_NULL) { return UCC_ERR_NOT_SUPPORTED; } task->allgather.s_mem_h = NULL; task->allgather.r_mem_h = NULL; task->super.post = ucc_tl_sharp_allgather_start; task->super.progress = ucc_tl_sharp_collective_progress; return UCC_OK; }; #endif ucc-1.8.0/src/components/tl/sharp/configure.m40000664000175000017500000000137315211535620021415 0ustar alastairalastair# # Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # tl_sharp_enabled=n CHECK_TLS_REQUIRED(["sharp"]) AS_IF([test "$CHECKED_TL_REQUIRED" = "y"], [ CHECK_SHARP AC_MSG_RESULT([SHARP support: $sharp_happy]) if test $sharp_happy = "yes"; then tl_modules="${tl_modules}:sharp" tl_sharp_enabled=y CHECK_NEED_TL_PROFILING(["tl_sharp"]) AS_IF([test "$TL_PROFILING_REQUIRED" = "y"], [ AC_DEFINE([HAVE_PROFILING_TL_SHARP], [1], [Enable profiling for TL SHARP]) prof_modules="${prof_modules}:tl_sharp" ], []) fi ], []) AM_CONDITIONAL([TL_SHARP_ENABLED], [test "$tl_sharp_enabled" = "y"]) AC_CONFIG_FILES([src/components/tl/sharp/Makefile]) ucc-1.8.0/src/components/tl/sharp/tl_sharp_lib.c0000664000175000017500000000420615211535620021776 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include #include "tl_sharp.h" /* NOLINTNEXTLINE params is not used*/ UCC_CLASS_INIT_FUNC(ucc_tl_sharp_lib_t, const ucc_base_lib_params_t *params, const ucc_base_config_t *config) { const ucc_tl_sharp_lib_config_t *tl_sharp_config = ucc_derived_of(config, ucc_tl_sharp_lib_config_t); UCC_CLASS_CALL_SUPER_INIT(ucc_tl_lib_t, &ucc_tl_sharp.super, &tl_sharp_config->super); memcpy(&self->cfg, tl_sharp_config, sizeof(*tl_sharp_config)); tl_debug(&self->super, "initialized lib object: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_sharp_lib_t) { tl_debug(&self->super, "finalizing lib object: %p", self); } UCC_CLASS_DEFINE(ucc_tl_sharp_lib_t, ucc_tl_lib_t); ucc_status_t ucc_tl_sharp_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr) { ucc_tl_sharp_lib_t *sharp_lib = ucc_derived_of(lib, ucc_tl_sharp_lib_t); ucc_tl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_tl_lib_attr_t); attr->super.flags = 0; if (lib != NULL) { if (sharp_lib->cfg.use_internal_oob == UCC_YES) { attr->super.flags |= UCC_BASE_LIB_FLAG_CTX_SERVICE_TEAM_REQUIRED; } } attr->super.attr.thread_mode = UCC_THREAD_MULTIPLE; attr->super.attr.coll_types = UCC_TL_SHARP_SUPPORTED_COLLS; if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MIN_TEAM_SIZE) { if (lib == NULL) { return UCC_ERR_INVALID_PARAM; } attr->super.min_team_size = lib->min_team_size; } if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MAX_TEAM_SIZE) { if (lib == NULL) { return UCC_ERR_INVALID_PARAM; } attr->super.max_team_size = UCC_RANK_MAX; } return UCC_OK; } ucc_status_t ucc_tl_sharp_get_lib_properties(ucc_base_lib_properties_t *prop) { prop->default_team_size = 4; prop->min_team_size = 2; prop->max_team_size = UCC_RANK_MAX; return UCC_OK; } ucc-1.8.0/src/components/tl/sharp/tl_sharp_coll.h0000664000175000017500000000166015211535620022167 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_SHARP_COLL_H_ #define UCC_TL_SHARP_COLL_H_ #include "tl_sharp.h" /* need to query for datatype support at runtime */ #define SHARP_DTYPE_UNKNOWN 0xFFFF #define UCC_TL_SHARP_N_DEFAULT_ALG_SELECT_STR 2 extern const char *ucc_tl_sharp_default_alg_select_str[UCC_TL_SHARP_N_DEFAULT_ALG_SELECT_STR]; extern enum sharp_datatype ucc_to_sharp_dtype[]; ucc_status_t ucc_tl_sharp_allreduce_init(ucc_tl_sharp_task_t *task); ucc_status_t ucc_tl_sharp_barrier_init(ucc_tl_sharp_task_t *task); ucc_status_t ucc_tl_sharp_bcast_init(ucc_tl_sharp_task_t *task); #if HAVE_DECL_SHARP_COLL_DO_REDUCE_SCATTER ucc_status_t ucc_tl_sharp_reduce_scatter_init(ucc_tl_sharp_task_t *task); #endif #if HAVE_DECL_SHARP_COLL_DO_ALLGATHER ucc_status_t ucc_tl_sharp_allgather_init(ucc_tl_sharp_task_t *task); #endif #endif ucc-1.8.0/src/components/tl/sharp/tl_sharp.h0000664000175000017500000001245015211535620021155 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_SHARP_H_ #define UCC_TL_SHARP_H_ #include "components/tl/ucc_tl.h" #include "components/tl/ucc_tl_log.h" #include "utils/ucc_mpool.h" #include "utils/ucc_rcache.h" #include #include #ifndef UCC_TL_SHARP_DEFAULT_SCORE #define UCC_TL_SHARP_DEFAULT_SCORE 30 #endif #ifdef HAVE_PROFILING_TL_SHARP #include "utils/profile/ucc_profile.h" #else #include "utils/profile/ucc_profile_off.h" #endif #define UCC_TL_SHARP_PROFILE_FUNC UCC_PROFILE_FUNC #define UCC_TL_SHARP_PROFILE_FUNC_VOID UCC_PROFILE_FUNC_VOID #define UCC_TL_SHARP_PROFILE_REQUEST_NEW UCC_PROFILE_REQUEST_NEW #define UCC_TL_SHARP_PROFILE_REQUEST_EVENT UCC_PROFILE_REQUEST_EVENT #define UCC_TL_SHARP_PROFILE_REQUEST_FREE UCC_PROFILE_REQUEST_FREE typedef struct ucc_tl_sharp_iface { ucc_tl_iface_t super; } ucc_tl_sharp_iface_t; extern ucc_tl_sharp_iface_t ucc_tl_sharp; typedef struct ucc_tl_sharp_lib_config { ucc_tl_lib_config_t super; int use_internal_oob; } ucc_tl_sharp_lib_config_t; typedef struct ucc_tl_sharp_context_config { ucc_tl_context_config_t super; struct sharp_coll_config cfg; char *dev_list; int use_rcache; size_t reg_threshold; unsigned int rand_seed; unsigned int uprogress_num_polls; int context_per_team; int enable_lazy_group_alloc; int team_max_ppn; int use_multi_channel; } ucc_tl_sharp_context_config_t; typedef struct ucc_tl_sharp_lib { ucc_tl_lib_t super; ucc_tl_sharp_lib_config_t cfg; } ucc_tl_sharp_lib_t; UCC_CLASS_DECLARE(ucc_tl_sharp_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_sharp_oob_ctx { void *ctx; union { ucc_oob_coll_t *oob; ucc_subset_t subset; }; } ucc_tl_sharp_oob_ctx_t; typedef struct ucc_tl_sharp_reg { void *mr; } ucc_tl_sharp_reg_t; typedef struct ucc_tl_sharp_rcache_region { ucc_rcache_region_t super; ucc_tl_sharp_reg_t reg; } ucc_tl_sharp_rcache_region_t; typedef struct ucc_tl_sharp_context { ucc_tl_context_t super; ucc_thread_mode_t tm; struct sharp_coll_context *sharp_context; ucc_tl_sharp_context_config_t cfg; ucc_mpool_t req_mp; ucc_tl_sharp_oob_ctx_t oob_ctx; ucc_rcache_t *rcache; struct sharp_coll_caps sharp_caps; } ucc_tl_sharp_context_t; UCC_CLASS_DECLARE(ucc_tl_sharp_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_sharp_team { ucc_tl_team_t super; struct sharp_coll_context *sharp_context; ucc_rcache_t *rcache; struct sharp_coll_comm *sharp_comm; ucc_tl_sharp_oob_ctx_t oob_ctx; ucc_topo_t *topo; } ucc_tl_sharp_team_t; typedef struct ucc_tl_sharp_task { ucc_coll_task_t super; void *req_handle; union { struct { ucc_tl_sharp_reg_t *s_mem_h; ucc_tl_sharp_reg_t *r_mem_h; } allgather; struct { ucc_tl_sharp_reg_t *s_mem_h; ucc_tl_sharp_reg_t *r_mem_h; } allreduce; struct { ucc_tl_sharp_reg_t *s_mem_h; ucc_tl_sharp_reg_t *r_mem_h; } reduce_scatter; struct { ucc_tl_sharp_reg_t *mem_h; } bcast; }; } ucc_tl_sharp_task_t; ucc_status_t ucc_tl_sharp_context_init(ucc_tl_sharp_context_t *sharp_ctx, struct sharp_coll_context **context, ucc_tl_sharp_oob_ctx_t *oob_ctx, ucc_topo_t *topo); ucc_status_t ucc_tl_sharp_rcache_create(struct sharp_coll_context *contex, ucc_rcache_t **rcache); ucc_status_t sharp_status_to_ucc_status(int status); #define TASK_TEAM(_task) \ (ucc_derived_of((_task)->super.team, ucc_tl_sharp_team_t)) #define TASK_CTX(_task) \ (ucc_derived_of((_task)->super.team->context, ucc_tl_sharp_context_t)) #define TASK_LIB(_task) \ (ucc_derived_of((_task)->super.team->context->lib, ucc_tl_sharp_lib_t)) #define TASK_ARGS(_task) (_task)->super.bargs.args #define UCC_TL_BASIC_SHARP_SUPPORTED_COLLS \ (UCC_COLL_TYPE_ALLREDUCE | UCC_COLL_TYPE_BARRIER | UCC_COLL_TYPE_BCAST) #define UCC_TL_SHARP_SUPPORTED_COLLS \ (UCC_TL_BASIC_SHARP_SUPPORTED_COLLS | \ (HAVE_DECL_SHARP_COLL_DO_REDUCE_SCATTER ? UCC_COLL_TYPE_REDUCE_SCATTER : 0) | \ (HAVE_DECL_SHARP_COLL_DO_ALLGATHER ? UCC_COLL_TYPE_ALLGATHER : 0)) UCC_CLASS_DECLARE(ucc_tl_sharp_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); #define UCC_TL_SHARP_TEAM_LIB(_team) \ (ucc_derived_of((_team)->super.super.context->lib, ucc_tl_sharp_lib_t)) #endif ucc-1.8.0/src/components/tl/sharp/Makefile.am0000664000175000017500000000135015211535620021221 0ustar alastairalastair# # Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # if TL_SHARP_ENABLED sources = \ tl_sharp.h \ tl_sharp.c \ tl_sharp_lib.c \ tl_sharp_context.c \ tl_sharp_team.c \ tl_sharp_coll.h \ tl_sharp_coll.c module_LTLIBRARIES = libucc_tl_sharp.la libucc_tl_sharp_la_SOURCES = $(sources) libucc_tl_sharp_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(SHARP_CPPFLAGS) libucc_tl_sharp_la_CFLAGS = $(BASE_CFLAGS) libucc_tl_sharp_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(SHARP_LDFLAGS) libucc_tl_sharp_la_LIBADD = $(SHARP_LIBADD) $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/tl/sharp/tl_sharp.c0000664000175000017500000001177515211535620021161 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_sharp.h" #include "components/mc/base/ucc_mc_base.h" ucc_status_t ucc_tl_sharp_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr); ucc_status_t ucc_tl_sharp_get_lib_properties(ucc_base_lib_properties_t *prop); ucc_status_t ucc_tl_sharp_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *base_attr); ucc_status_t ucc_tl_sharp_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_sharp_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_sharp_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **pack_buffer); static ucc_config_field_t ucc_tl_sharp_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_sharp_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_lib_config_table)}, {"USE_INTERNAL_OOB", "try", "Use service team to create sharp context", ucc_offsetof(ucc_tl_sharp_lib_config_t, use_internal_oob), UCC_CONFIG_TYPE_TERNARY}, {NULL}}; static ucc_config_field_t ucc_tl_sharp_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_sharp_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_context_config_table)}, {"DEVICES", "", "SHARP device list", ucc_offsetof(ucc_tl_sharp_context_config_t, dev_list), UCC_CONFIG_TYPE_STRING}, {"USE_RCACHE", "y", "Use registration cache for sharp", ucc_offsetof(ucc_tl_sharp_context_config_t, use_rcache), UCC_CONFIG_TYPE_BOOL}, {"REG_THRESH", "0", "Size threshold to register buffers", ucc_offsetof(ucc_tl_sharp_context_config_t, reg_threshold), UCC_CONFIG_TYPE_MEMUNITS}, {"UPROGRESS_NUM_POLLS", "9999", "Number of polls to do before calling user progress", ucc_offsetof(ucc_tl_sharp_context_config_t, uprogress_num_polls), UCC_CONFIG_TYPE_UINT}, {"CONTEXT_PER_TEAM", "n", "Create SHARP context/tree per team", ucc_offsetof(ucc_tl_sharp_context_config_t, context_per_team), UCC_CONFIG_TYPE_BOOL}, #if HAVE_DECL_SHARP_COLL_DISABLE_LAZY_GROUP_RESOURCE_ALLOC {"ENABLE_LAZY_GROUP_ALLOC", "n", "Enable lazy group resource allocation", ucc_offsetof(ucc_tl_sharp_context_config_t, enable_lazy_group_alloc), UCC_CONFIG_TYPE_BOOL}, #endif {"RAND_SEED", "0", "Seed for random sharp job ID. (0 - use default).", ucc_offsetof(ucc_tl_sharp_context_config_t, rand_seed), UCC_CONFIG_TYPE_UINT}, {"TEAM_MAX_PPN", "1", "SHARP team max PPN threshold", ucc_offsetof(ucc_tl_sharp_context_config_t, team_max_ppn), UCC_CONFIG_TYPE_UINT}, {"USE_MULTI_CHANNEL", "0", "Use SHARP Multi-channel feature. Options: 0-disable 1-enable", ucc_offsetof(ucc_tl_sharp_context_config_t, use_multi_channel), UCC_CONFIG_TYPE_BOOL}, {NULL}}; UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_sharp_lib_t, ucc_base_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_sharp_lib_t, ucc_base_lib_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_sharp_context_t, ucc_base_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_sharp_context_t, ucc_base_context_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_sharp_team_t, ucc_base_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_tl_sharp_team_create_test(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_sharp_team_destroy(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_sharp_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); ucc_status_t ucc_tl_sharp_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p); UCC_TL_IFACE_DECLARE(sharp, SHARP); ucc_status_t ucc_tl_sharp_context_create_epilog(ucc_base_context_t *context); ucc_status_t sharp_status_to_ucc_status(int status) { switch (status) { case SHARP_COLL_SUCCESS: return UCC_OK; case SHARP_COLL_ENOMEM: return UCC_ERR_NO_MEMORY; case SHARP_COLL_ENOT_SUPP: return UCC_ERR_NOT_SUPPORTED; case SHARP_COLL_EINVAL: return UCC_ERR_INVALID_PARAM; case SHARP_COLL_ENO_RESOURCE: return UCC_ERR_NO_RESOURCE; default: break; } return UCC_ERR_NO_MESSAGE; } __attribute__((constructor)) static void tl_sharp_iface_init(void) { ucc_tl_sharp.super.context.create_epilog = ucc_tl_sharp_context_create_epilog; } ucc-1.8.0/src/components/tl/sharp/tl_sharp_context.c0000664000175000017500000004626115211535620022723 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include #include "tl_sharp.h" #include "utils/arch/cpu.h" #include "core/ucc_service_coll.h" static int ucc_tl_sharp_oob_barrier(void *arg) { ucc_tl_sharp_oob_ctx_t *oob_ctx = (ucc_tl_sharp_oob_ctx_t *)arg; ucc_tl_sharp_context_t *ctx = oob_ctx->ctx; ucc_oob_coll_t *oob_coll = oob_ctx->oob; ucc_status_t status; char sbuf, *rbuf; void *req; rbuf = ucc_malloc(sizeof(char) * oob_coll->n_oob_eps, "tmp_barrier"); if (!rbuf) { tl_error(ctx->super.super.lib, "failed to allocate %zd bytes for tmp barrier array", sizeof(char) * oob_coll->n_oob_eps); return UCC_ERR_NO_MEMORY; } status = oob_coll->allgather(&sbuf, rbuf, sizeof(char), oob_coll->coll_info, &req); if (UCC_OK == status) { ucc_assert(req != NULL); while (UCC_OK != (status = oob_coll->req_test(req))) { if (status < 0) { tl_error(ctx->super.super.lib, "failed to test oob req"); break; } } oob_coll->req_free(req); } ucc_free(rbuf); return status; } static int ucc_tl_sharp_oob_gather(void *arg, int root, void *sbuf, void *rbuf, int size) { ucc_tl_sharp_oob_ctx_t *oob_ctx = (ucc_tl_sharp_oob_ctx_t *)arg; ucc_tl_sharp_context_t *ctx = oob_ctx->ctx; void *tmp_rbuf = NULL; size_t msg_size = size; ucc_oob_coll_t *oob_coll = oob_ctx->oob; void *req; ucc_status_t status; if (oob_coll->oob_ep != root) { tmp_rbuf = ucc_malloc(msg_size * oob_coll->n_oob_eps, "tmp_gather"); if (!tmp_rbuf) { tl_error(ctx->super.super.lib, "failed to allocate %zd bytes for tmp barrier array", msg_size * oob_coll->n_oob_eps); return UCC_ERR_NO_MEMORY; } rbuf = tmp_rbuf; } status = oob_coll->allgather(sbuf, rbuf, msg_size, oob_coll->coll_info, &req); if (UCC_OK == status) { ucc_assert(req != NULL); while (UCC_OK != (status = oob_coll->req_test(req))) { if (status < 0) { tl_error(ctx->super.super.lib, "failed to test oob req"); break; } } oob_coll->req_free(req); } if (tmp_rbuf) { ucc_free(tmp_rbuf); } return status; } static int ucc_tl_sharp_oob_bcast(void *arg, void *buf, int size, int root) { ucc_tl_sharp_oob_ctx_t *oob_ctx = (ucc_tl_sharp_oob_ctx_t *)arg; ucc_tl_sharp_context_t *ctx = oob_ctx->ctx; size_t msg_size = size; ucc_oob_coll_t *oob_coll = oob_ctx->oob; ucc_status_t status; void *req, *tmp_rbuf; tmp_rbuf = ucc_malloc(msg_size * oob_coll->n_oob_eps, "tmp_bcast"); if (!tmp_rbuf) { tl_error(ctx->super.super.lib, "failed to allocate %zd bytes for tmp barrier array", msg_size * oob_coll->n_oob_eps); return UCC_ERR_NO_MEMORY; } status = oob_coll->allgather(buf, tmp_rbuf, msg_size, oob_coll ->coll_info, &req); if (UCC_OK == status) { ucc_assert(req != NULL); while (UCC_OK != (status = oob_coll ->req_test(req))) { if (status < 0) { tl_error(ctx->super.super.lib, "failed to test oob req"); break; } } oob_coll ->req_free(req); } memcpy(buf, PTR_OFFSET(tmp_rbuf, root * msg_size), msg_size); ucc_free(tmp_rbuf); return status; } static int ucc_tl_sharp_service_barrier(void *arg) { ucc_tl_sharp_oob_ctx_t *oob_ctx = (ucc_tl_sharp_oob_ctx_t*)arg; ucc_tl_sharp_context_t *ctx = (ucc_tl_sharp_context_t*)oob_ctx->ctx; ucc_tl_team_t *steam = ctx->super.super.ucc_context->service_team; ucc_coll_task_t *req; ucc_status_t status; int32_t sbuf, rbuf; status = UCC_TL_TEAM_IFACE(steam)->scoll.allreduce(&steam->super, &sbuf, &rbuf, UCC_DT_INT32, 1, UCC_OP_SUM, oob_ctx->subset, &req); if (status != UCC_OK) { tl_error(ctx->super.super.lib, "tl sharp gather failed\n"); return status; } do { ucc_context_progress(ctx->super.super.ucc_context); status = ucc_collective_test(&req->super); } while (status == UCC_INPROGRESS); ucc_collective_finalize_internal(req); return status; } static int ucc_tl_sharp_service_gather(void *arg, int root, void *sbuf, void *rbuf, int size) { ucc_tl_sharp_oob_ctx_t *oob_ctx = (ucc_tl_sharp_oob_ctx_t*)arg; ucc_tl_sharp_context_t *ctx = (ucc_tl_sharp_context_t*)oob_ctx->ctx; ucc_tl_team_t *steam = ctx->super.super.ucc_context->service_team; size_t msg_size = (size_t)size; ucc_subset_t subset = oob_ctx->subset; ucc_coll_task_t *req; ucc_status_t status; if (subset.myrank != root) { rbuf = ucc_malloc(msg_size * subset.map.ep_num, "tmp_gather"); if (!rbuf) { tl_error(ctx->super.super.lib, "failed to allocate %zd bytes for tmp barrier array", msg_size * subset.map.ep_num); return UCC_ERR_NO_MEMORY; } } status = UCC_TL_TEAM_IFACE(steam)->scoll.allgather(&steam->super, sbuf, rbuf, msg_size, subset, &req); if (status != UCC_OK) { tl_error(ctx->super.super.lib, "tl sharp gather failed\n"); return status; } do { ucc_context_progress(ctx->super.super.ucc_context); status = ucc_collective_test(&req->super); } while (status == UCC_INPROGRESS); ucc_collective_finalize_internal(req); if (subset.myrank != root) { ucc_free(rbuf); } return status; } static int ucc_tl_sharp_service_bcast(void *arg, void *buf, int size, int root) { ucc_tl_sharp_oob_ctx_t *oob_ctx = (ucc_tl_sharp_oob_ctx_t*)arg; ucc_tl_sharp_context_t *ctx = (ucc_tl_sharp_context_t*)oob_ctx->ctx; ucc_tl_team_t *steam = ctx->super.super.ucc_context->service_team; ucc_coll_task_t *req; ucc_status_t status; status = UCC_TL_TEAM_IFACE(steam)->scoll.bcast(&steam->super, buf, size, root, oob_ctx->subset, &req); if (status != UCC_OK) { tl_error(ctx->super.super.lib, "tl sharp bcast failed\n"); return status; } do { ucc_context_progress(ctx->super.super.ucc_context); status = ucc_collective_test(&req->super); } while (status == UCC_INPROGRESS); ucc_collective_finalize_internal(req); return status; } /* rcache, arg, flags unused */ static ucs_status_t ucc_tl_sharp_rcache_mem_reg_cb(void *context, ucc_rcache_t *rcache, //NOLINT void *arg, ucc_rcache_region_t *rregion, //NOLINT uint16_t flags) //NOLINT { ucc_tl_sharp_rcache_region_t *region; void *address; size_t length; int ret; address = (void*)rregion->super.start; length = (size_t)(rregion->super.end - rregion->super.start); region = ucc_derived_of(rregion, ucc_tl_sharp_rcache_region_t); ret = sharp_coll_reg_mr((struct sharp_coll_context *)context, address, length, ®ion->reg.mr); if (ret < 0) { return UCS_ERR_INVALID_PARAM; } else { return UCS_OK; } } /* rcache is unused */ static void ucc_tl_sharp_rcache_mem_dereg_cb(void *context, ucc_rcache_t *rcache, //NOLINT ucc_rcache_region_t *rregion) { ucc_tl_sharp_rcache_region_t *region = ucc_derived_of(rregion, ucc_tl_sharp_rcache_region_t); sharp_coll_dereg_mr((struct sharp_coll_context *)context, region->reg.mr); } /* context, rcache unused */ static void ucc_tl_sharp_rcache_dump_region_cb(void *context, ucs_rcache_t *rcache, //NOLINT ucs_rcache_region_t *rregion, char *buf, size_t max) { ucc_tl_sharp_rcache_region_t *region = ucc_derived_of(rregion, ucc_tl_sharp_rcache_region_t); snprintf(buf, max, "bar ptr:%p", region->reg.mr); } static ucc_rcache_ops_t ucc_tl_sharp_rcache_ops = { .mem_reg = ucc_tl_sharp_rcache_mem_reg_cb, .mem_dereg = ucc_tl_sharp_rcache_mem_dereg_cb, .dump_region = ucc_tl_sharp_rcache_dump_region_cb, #ifdef UCS_HAVE_RCACHE_MERGE_CB .merge = ucc_rcache_merge_cb_empty #endif }; ucc_status_t ucc_tl_sharp_rcache_create(struct sharp_coll_context *context, ucc_rcache_t **rcache) { ucc_rcache_params_t rcache_params; ucc_rcache_set_default_params(&rcache_params); rcache_params.region_struct_size = sizeof(ucc_tl_sharp_rcache_region_t); rcache_params.context = context; rcache_params.ops = &ucc_tl_sharp_rcache_ops; rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED | UCM_EVENT_MEM_TYPE_FREE; return ucc_rcache_create(&rcache_params, "SHARP", rcache); } ucc_status_t ucc_tl_sharp_context_init(ucc_tl_sharp_context_t *sharp_ctx, struct sharp_coll_context **context, ucc_tl_sharp_oob_ctx_t *oob_ctx, ucc_topo_t *topo) { struct sharp_coll_init_spec init_spec = {0}; ucc_tl_sharp_lib_t *lib = ucc_derived_of(sharp_ctx->super.super.lib, ucc_tl_sharp_lib_t); ucc_sbgp_t *sbgp; int local_rank; int ret; sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); if (sbgp->status != UCC_SBGP_ENABLED) { tl_debug(sharp_ctx->super.super.lib, "NODE SBGP is not enabled"); local_rank = 0; } else { local_rank = sbgp->group_rank; } init_spec.progress_func = NULL; init_spec.world_local_rank = local_rank; if (sharp_ctx->cfg.use_multi_channel) { init_spec.group_channel_idx = local_rank; } else { init_spec.group_channel_idx = 0; } init_spec.oob_ctx = oob_ctx; init_spec.config = sharp_coll_default_config; init_spec.config.user_progress_num_polls = sharp_ctx->cfg.uprogress_num_polls; init_spec.config.ib_dev_list = sharp_ctx->cfg.dev_list; #if HAVE_DECL_SHARP_COLL_HIDE_ERRORS if (lib->super.super.log_component.log_level < UCC_LOG_LEVEL_DEBUG) { init_spec.config.flags |= SHARP_COLL_HIDE_ERRORS; } #endif #if HAVE_DECL_SHARP_COLL_DISABLE_LAZY_GROUP_RESOURCE_ALLOC if(!sharp_ctx->cfg.enable_lazy_group_alloc) { init_spec.config.flags |= SHARP_COLL_DISABLE_LAZY_GROUP_RESOURCE_ALLOC; } #endif init_spec.job_id = ((getpid() ^ pthread_self()) ^ rand_r(&sharp_ctx->cfg.rand_seed)); init_spec.enable_thread_support = (sharp_ctx->tm == UCC_THREAD_MULTIPLE) ? 1 : 0; if (lib->cfg.use_internal_oob != UCC_NO && sharp_ctx->super.super.ucc_context->service_team) { tl_debug(sharp_ctx->super.super.lib, "using internal oob. rank:%u size:%lu", oob_ctx->subset.myrank, oob_ctx->subset.map.ep_num); init_spec.world_rank = oob_ctx->subset.myrank; init_spec.world_size = oob_ctx->subset.map.ep_num; init_spec.oob_colls.barrier = ucc_tl_sharp_service_barrier; init_spec.oob_colls.bcast = ucc_tl_sharp_service_bcast; init_spec.oob_colls.gather = ucc_tl_sharp_service_gather; } else { tl_debug(sharp_ctx->super.super.lib, "using user provided oob. rank:%u size:%u", oob_ctx->oob->oob_ep, oob_ctx->oob->n_oob_eps); init_spec.world_rank = oob_ctx->oob->oob_ep; init_spec.world_size = oob_ctx->oob->n_oob_eps; init_spec.oob_colls.barrier = ucc_tl_sharp_oob_barrier; init_spec.oob_colls.bcast = ucc_tl_sharp_oob_bcast; init_spec.oob_colls.gather = ucc_tl_sharp_oob_gather; } //TODO: replace with unique context ID? ret = init_spec.oob_colls.bcast((void *)oob_ctx, &init_spec.job_id, sizeof(uint64_t), 0); if (ret < 0) { tl_error(sharp_ctx->super.super.lib, "failed to broadcast SHARP job_id"); return UCC_ERR_NO_MESSAGE; } ret = sharp_coll_init(&init_spec, context); if (ret < 0 ) { tl_debug(sharp_ctx->super.super.lib, "failed to initialize SHARP " "collectives:%s(%d) job ID:%" PRIu64"\n", sharp_coll_strerror(ret), ret, init_spec.job_id); return UCC_ERR_NO_RESOURCE; } ret = sharp_coll_caps_query(*context, &sharp_ctx->sharp_caps); if (ret < 0) { tl_error(sharp_ctx->super.super.lib, "sharp_coll_caps_query failed: %s(%d)", sharp_coll_strerror(ret), ret); sharp_coll_finalize(*context); return UCC_ERR_NO_RESOURCE; } return UCC_OK; } UCC_CLASS_INIT_FUNC(ucc_tl_sharp_context_t, const ucc_base_context_params_t *params, const ucc_base_config_t *config) { ucc_tl_sharp_context_config_t *tl_sharp_config = ucc_derived_of(config, ucc_tl_sharp_context_config_t); ucc_status_t status; struct timeval tval; if (!(params->params.mask & UCC_CONTEXT_PARAM_FIELD_OOB)) { tl_error(tl_sharp_config->super.tl_lib, "Context OOB is required for SHARP"); status = UCC_ERR_INVALID_PARAM; goto err; } if (params->params.oob.n_oob_eps < 2) { status = UCC_ERR_NOT_SUPPORTED; goto err; } UCC_CLASS_CALL_SUPER_INIT(ucc_tl_context_t, &tl_sharp_config->super, params->context); memcpy(&self->cfg, tl_sharp_config, sizeof(*tl_sharp_config)); if (self->cfg.rand_seed == 0) { gettimeofday(&tval, NULL); self->cfg.rand_seed = (int) tval.tv_usec; } self->sharp_context = NULL; self->rcache = NULL; self->oob_ctx.ctx = self; self->tm = params->thread_mode; status = ucc_mpool_init(&self->req_mp, 0, sizeof(ucc_tl_sharp_task_t), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_coll_task_mpool_ops, params->thread_mode, "tl_sharp_req_mp"); if (status != UCC_OK) { tl_error(self->super.super.lib, "failed to initialize tl_sharp_req mpool"); status = UCC_ERR_NO_MEMORY; goto err; } tl_debug(self->super.super.lib, "initialized tl context: %p", self); return UCC_OK; err: return status; } ucc_status_t ucc_tl_sharp_context_create_epilog(ucc_base_context_t *context) { ucc_tl_sharp_context_t *sharp_ctx = ucc_derived_of(context, ucc_tl_sharp_context_t); ucc_context_t *core_ctx = context->ucc_context; ucc_tl_sharp_lib_t *lib = ucc_derived_of(sharp_ctx->super.super.lib, ucc_tl_sharp_lib_t); ucc_status_t status; ucc_topo_t *topo; ucc_subset_t set; if (sharp_ctx->cfg.context_per_team) { return UCC_OK; } memset(&set.map, 0, sizeof(ucc_ep_map_t)); set.map.type = UCC_EP_MAP_FULL; set.myrank = UCC_TL_CTX_OOB(sharp_ctx).oob_ep; set.map.ep_num = UCC_TL_CTX_OOB(sharp_ctx).n_oob_eps; if (lib->cfg.use_internal_oob != UCC_NO && core_ctx->service_team) { sharp_ctx->oob_ctx.subset = set; } else { sharp_ctx->oob_ctx.oob = &UCC_TL_CTX_OOB(sharp_ctx); } ucc_assert(core_ctx->topo != NULL); status = ucc_topo_init(set, core_ctx->topo, &topo); if (UCC_OK != status) { tl_error(sharp_ctx->super.super.lib, "failed to init topo"); return status; } status = ucc_tl_sharp_context_init(sharp_ctx, &sharp_ctx->sharp_context, &sharp_ctx->oob_ctx, topo); ucc_topo_cleanup(topo); if (status != UCC_OK) { return status; } if (sharp_ctx->cfg.use_rcache) { status = ucc_tl_sharp_rcache_create(sharp_ctx->sharp_context, &sharp_ctx->rcache); if (status != UCC_OK) { tl_error(sharp_ctx->super.super.lib, "failed to create rcache"); goto err_sharp_ctx_init; } } status = ucc_context_progress_register( context->ucc_context, (ucc_context_progress_fn_t)sharp_coll_progress, sharp_ctx->sharp_context); if (status != UCC_OK) { tl_error(context->lib, "failed to register progress function"); goto err_sharp_progress_register; } return UCC_OK; err_sharp_progress_register: if (sharp_ctx->rcache != NULL) { ucc_rcache_destroy(sharp_ctx->rcache); } err_sharp_ctx_init: sharp_coll_finalize(sharp_ctx->sharp_context); return status; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_sharp_context_t) { tl_debug(self->super.super.lib, "finalizing tl context: %p", self); if (self->rcache != NULL) { ucc_rcache_destroy(self->rcache); } if (self->sharp_context != NULL) { ucc_context_progress_deregister( self->super.super.ucc_context, (ucc_context_progress_fn_t)sharp_coll_progress, self->sharp_context); sharp_coll_finalize(self->sharp_context); } ucc_mpool_cleanup(&self->req_mp, 1); } UCC_CLASS_DEFINE(ucc_tl_sharp_context_t, ucc_tl_context_t); ucc_status_t ucc_tl_sharp_get_context_attr(const ucc_base_context_t *context, /* NOLINT */ ucc_base_ctx_attr_t *attr) { ucc_base_ctx_attr_clear(attr); attr->topo_required = 1; return UCC_OK; } ucc_status_t ucc_tl_sharp_mem_map(const ucc_base_context_t *context, int type, /* NOLINT */ void *address, size_t len, void *memh, /* NOLINT */ void *tl_h) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_tl_sharp_mem_unmap(const ucc_base_context_t *context, int type, /* NOLINT */ void *memh) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_tl_sharp_memh_pack(const ucc_base_context_t *context, /* NOLINT */ int type, void *memh, void **pack_buffer) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc-1.8.0/src/components/tl/self/0000775000175000017500000000000015211535620017002 5ustar alastairalastairucc-1.8.0/src/components/tl/self/tl_self_lib.c0000664000175000017500000000331215211535620021423 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "tl_self.h" /* NOLINTNEXTLINE params is not used*/ UCC_CLASS_INIT_FUNC(ucc_tl_self_lib_t, const ucc_base_lib_params_t *params, const ucc_base_config_t *config) { const ucc_tl_self_lib_config_t *tl_config = ucc_derived_of(config, ucc_tl_self_lib_config_t); UCC_CLASS_CALL_SUPER_INIT(ucc_tl_lib_t, &ucc_tl_self.super, &tl_config->super); memcpy(&self->cfg, tl_config, sizeof(*tl_config)); tl_debug(&self->super, "initialized lib object: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_self_lib_t) { tl_debug(&self->super, "finalizing lib object: %p", self); } UCC_CLASS_DEFINE(ucc_tl_self_lib_t, ucc_tl_lib_t); ucc_status_t ucc_tl_self_get_lib_attr(const ucc_base_lib_t *lib, /* NOLINT */ ucc_base_lib_attr_t *base_attr) { ucc_tl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_tl_lib_attr_t); attr->super.flags = 0; attr->super.attr.thread_mode = UCC_THREAD_MULTIPLE; attr->super.attr.coll_types = UCC_TL_SELF_SUPPORTED_COLLS; if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MIN_TEAM_SIZE) { attr->super.min_team_size = 1; } if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MAX_TEAM_SIZE) { attr->super.max_team_size = 1; } return UCC_OK; } ucc_status_t ucc_tl_self_get_lib_properties(ucc_base_lib_properties_t *prop) { prop->default_team_size = 1; prop->min_team_size = 1; prop->max_team_size = 1; return UCC_OK; } ucc-1.8.0/src/components/tl/self/tl_self.c0000664000175000017500000000527715211535620020611 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "tl_self.h" #include "utils/ucc_malloc.h" #include "components/mc/ucc_mc.h" #include "components/mc/base/ucc_mc_base.h" ucc_status_t ucc_tl_self_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr); ucc_status_t ucc_tl_self_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *base_attr); ucc_status_t ucc_tl_self_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_self_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_self_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **pack_buffer); ucc_status_t ucc_tl_self_get_lib_properties(ucc_base_lib_properties_t *prop); static ucc_config_field_t ucc_tl_self_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_self_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_lib_config_table)}, {NULL}}; static ucs_config_field_t ucc_tl_self_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_self_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_context_config_table)}, {NULL}}; UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_self_lib_t, ucc_base_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_self_lib_t, ucc_base_lib_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_self_context_t, ucc_base_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_self_context_t, ucc_base_context_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_self_team_t, ucc_base_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_tl_self_team_create_test(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_self_team_destroy(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_self_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); ucc_status_t ucc_tl_self_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score); UCC_TL_IFACE_DECLARE(self, SELF); ucc-1.8.0/src/components/tl/self/configure.m40000664000175000017500000000123715211535620021230 0ustar alastairalastair# # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Copyright (c) Meta Platforms, Inc. and affiliates. 2022. # tl_self_enabled=n CHECK_TLS_REQUIRED(["self"]) AS_IF([test "$CHECKED_TL_REQUIRED" = "y"], [ tl_modules="${tl_modules}:self" tl_self_enabled=y CHECK_NEED_TL_PROFILING(["tl_self"]) AS_IF([test "$TL_PROFILING_REQUIRED" = "y"], [ AC_DEFINE([HAVE_PROFILING_TL_SELF], [1], [Enable profiling for TL SELF]) prof_modules="${prof_modules}:tl_self" ], []) ], []) AM_CONDITIONAL([TL_SELF_ENABLED], [test "$tl_self_enabled" = "y"]) AC_CONFIG_FILES([src/components/tl/self/Makefile]) ucc-1.8.0/src/components/tl/self/tl_self.h0000664000175000017500000000700115211535620020601 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #ifndef UCC_TL_SELF_H_ #define UCC_TL_SELF_H_ #include #include "components/tl/ucc_tl.h" #include "components/tl/ucc_tl_log.h" #include "core/ucc_ee.h" #include "utils/ucc_mpool.h" #ifndef UCC_TL_SELF_DEFAULT_SCORE #define UCC_TL_SELF_DEFAULT_SCORE 50 #endif #ifdef HAVE_PROFILING_TL_SELF #include "utils/profile/ucc_profile.h" #else #include "utils/profile/ucc_profile_off.h" #endif #define UCC_TL_SELF_PROFILE_FUNC UCC_PROFILE_FUNC #define UCC_TL_SELF_PROFILE_FUNC_VOID UCC_PROFILE_FUNC_VOID #define UCC_TL_SELF_PROFILE_REQUEST_NEW UCC_PROFILE_REQUEST_NEW #define UCC_TL_SELF_PROFILE_REQUEST_EVENT UCC_PROFILE_REQUEST_EVENT #define UCC_TL_SELF_PROFILE_REQUEST_FREE UCC_PROFILE_REQUEST_FREE typedef struct ucc_tl_self_iface { ucc_tl_iface_t super; } ucc_tl_self_iface_t; /* Extern iface should follow the pattern: ucc_tl_ */ extern ucc_tl_self_iface_t ucc_tl_self; typedef struct ucc_tl_self_lib_config { ucc_tl_lib_config_t super; } ucc_tl_self_lib_config_t; typedef struct ucc_tl_self_context_config { ucc_tl_context_config_t super; } ucc_tl_self_context_config_t; typedef struct ucc_tl_self_lib { ucc_tl_lib_t super; ucc_tl_self_lib_config_t cfg; } ucc_tl_self_lib_t; UCC_CLASS_DECLARE(ucc_tl_self_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_self_context { ucc_tl_context_t super; ucc_tl_self_context_config_t cfg; ucc_mpool_t req_mp; } ucc_tl_self_context_t; UCC_CLASS_DECLARE(ucc_tl_self_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_self_task { ucc_coll_task_t super; void *src; void *dst; size_t size; ucc_memory_type_t src_memtype; ucc_memory_type_t dst_memtype; ucc_ee_executor_task_t *etask; } ucc_tl_self_task_t; typedef struct ucc_tl_self_team { ucc_tl_team_t super; ucc_status_t status; } ucc_tl_self_team_t; UCC_CLASS_DECLARE(ucc_tl_self_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); #define UCC_TL_SELF_SUPPORTED_COLLS \ (UCC_COLL_TYPE_ALLTOALL | UCC_COLL_TYPE_ALLTOALLV | \ UCC_COLL_TYPE_ALLGATHER | UCC_COLL_TYPE_ALLGATHERV | \ UCC_COLL_TYPE_ALLREDUCE | UCC_COLL_TYPE_BCAST | UCC_COLL_TYPE_BARRIER | \ UCC_COLL_TYPE_REDUCE | UCC_COLL_TYPE_FANIN | UCC_COLL_TYPE_FANOUT | \ UCC_COLL_TYPE_GATHER | UCC_COLL_TYPE_GATHERV | UCC_COLL_TYPE_SCATTER | \ UCC_COLL_TYPE_SCATTERV | UCC_COLL_TYPE_REDUCE_SCATTER | \ UCC_COLL_TYPE_REDUCE_SCATTERV) #define UCC_TL_SELF_TEAM_LIB(_team) \ (ucc_derived_of((_team)->super.super.context->lib, ucc_tl_self_lib_t)) #define UCC_TL_SELF_TEAM_CTX(_team) \ (ucc_derived_of((_team)->super.super.context, ucc_tl_self_context_t)) ucc_status_t ucc_tl_self_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_self_coll_finalize(ucc_coll_task_t *coll_task); #endif ucc-1.8.0/src/components/tl/self/tl_self_coll.c0000664000175000017500000002265515211535620021621 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "tl_self.h" #include "utils/ucc_coll_utils.h" #include "utils/ucc_malloc.h" static inline ucc_tl_self_task_t * ucc_tl_self_coll_init_task(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team) { ucc_tl_self_team_t *tl_team = ucc_derived_of(team, ucc_tl_self_team_t); ucc_tl_self_context_t *ctx = UCC_TL_SELF_TEAM_CTX(tl_team); ucc_tl_self_task_t *task = ucc_mpool_get(&ctx->req_mp); if (ucc_unlikely(!task)) { return NULL; } ucc_coll_task_init(&task->super, coll_args, team); UCC_TL_SELF_PROFILE_REQUEST_NEW(task, "tl_self_task", 0); task->super.finalize = ucc_tl_self_coll_finalize; task->src = NULL; task->dst = NULL; task->size = 0; task->etask = NULL; return task; } static inline void ucc_tl_self_put_task(ucc_tl_self_task_t *task) { UCC_TL_SELF_PROFILE_REQUEST_FREE(task); ucc_mpool_put(task); } ucc_status_t ucc_tl_self_coll_finalize(ucc_coll_task_t *coll_task) { ucc_tl_self_task_t *task = ucc_derived_of(coll_task, ucc_tl_self_task_t); tl_trace(UCC_TASK_LIB(task), "finalizing task %p", task); ucc_tl_self_put_task(task); return UCC_OK; } void ucc_tl_self_noop_progress(ucc_coll_task_t *task) { task->status = UCC_OK; } void ucc_tl_self_copy_progress(ucc_coll_task_t *coll_task) { ucc_tl_self_task_t *task = ucc_derived_of(coll_task, ucc_tl_self_task_t); ucc_status_t status; if (task->etask != NULL) { status = ucc_ee_executor_task_test(task->etask); if (status == UCC_OPERATION_INITIALIZED || status == UCC_INPROGRESS) { task->super.status = UCC_INPROGRESS; return; } ucc_ee_executor_task_finalize(task->etask); task->etask = NULL; task->super.status = status; } } ucc_status_t ucc_tl_self_copy_start(ucc_coll_task_t *coll_task) { ucc_tl_self_task_t *task = ucc_derived_of(coll_task, ucc_tl_self_task_t); ucc_ee_executor_task_args_t exec_args = {0}; ucc_ee_executor_t *exec; ucc_status_t status; status = ucc_coll_task_get_executor(&task->super, &exec); if (ucc_unlikely(status != UCC_OK)) { return status; } exec_args.task_type = UCC_EE_EXECUTOR_TASK_COPY; exec_args.copy.dst = task->dst; exec_args.copy.src = task->src; exec_args.copy.len = task->size; status = ucc_ee_executor_task_post(exec, &exec_args, &task->etask); if (ucc_unlikely(status != UCC_OK)) { task->super.status = status; return status; } return ucc_progress_queue_enqueue(UCC_TASK_CORE_CTX(coll_task)->pq, coll_task); } ucc_status_t ucc_tl_self_coll_start(ucc_coll_task_t *task) { return ucc_progress_queue_enqueue(UCC_TASK_CORE_CTX(task)->pq, task); } ucc_status_t ucc_tl_self_coll_noop_init(ucc_tl_self_task_t *task) { task->super.post = ucc_tl_self_coll_start; task->super.progress = ucc_tl_self_noop_progress; return UCC_OK; } ucc_status_t ucc_tl_self_coll_copy_init(ucc_tl_self_task_t *task) { ucc_coll_args_t *args = &(task->super.bargs.args); if (!ucc_coll_args_is_predefined_dt(args, task->super.team->params.rank)) { return UCC_ERR_NOT_SUPPORTED; } if (UCC_IS_INPLACE(*args)) { /* no copy is required for in-place */ task->super.post = ucc_tl_self_coll_start; task->super.progress = ucc_tl_self_noop_progress; } else { task->dst = args->dst.info.buffer; task->src = args->src.info.buffer; task->size = args->src.info.count * ucc_dt_size(args->src.info.datatype); task->dst_memtype = args->dst.info.mem_type; task->src_memtype = args->src.info.mem_type; task->super.post = ucc_tl_self_copy_start; task->super.progress = ucc_tl_self_copy_progress; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; } return UCC_OK; } ucc_status_t ucc_tl_self_alltoallv_init(ucc_tl_self_task_t *task) { ucc_coll_args_t *args = &(task->super.bargs.args); if (!ucc_coll_args_is_predefined_dt(args, task->super.team->params.rank)) { return UCC_ERR_NOT_SUPPORTED; } if (UCC_IS_INPLACE(*args)) { /* no copy is required for in-place */ task->super.post = ucc_tl_self_coll_start; task->super.progress = ucc_tl_self_noop_progress; } else { size_t displ = (size_t)ucc_coll_args_get_displacement( args, args->dst.info_v.displacements, 0); task->dst = PTR_OFFSET(args->dst.info_v.buffer, displ); displ = (size_t)ucc_coll_args_get_displacement( args, args->src.info_v.displacements, 0); task->src = PTR_OFFSET(args->src.info_v.buffer, displ); task->size = ucc_coll_args_get_count(args, args->src.info_v.counts, 0) * ucc_dt_size(args->src.info_v.datatype); task->dst_memtype = args->dst.info_v.mem_type; task->src_memtype = args->src.info_v.mem_type; task->super.post = ucc_tl_self_copy_start; task->super.progress = ucc_tl_self_copy_progress; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; } return UCC_OK; } ucc_status_t ucc_tl_self_coll_copyv_init(ucc_tl_self_task_t *task) { ucc_coll_args_t *args = &(task->super.bargs.args); if (!ucc_coll_args_is_predefined_dt(args, task->super.team->params.rank)) { return UCC_ERR_NOT_SUPPORTED; } if (UCC_IS_INPLACE(*args)) { /* no copy is required for in-place */ task->super.post = ucc_tl_self_coll_start; task->super.progress = ucc_tl_self_noop_progress; } else { size_t displ = 0; /* reduce_scatterv may not provide displacements */ if (args->dst.info_v.displacements) { displ = (size_t)ucc_coll_args_get_displacement( args, args->dst.info_v.displacements, 0); } task->dst = PTR_OFFSET(args->dst.info_v.buffer, displ); task->src = args->src.info.buffer; task->size = args->src.info.count * ucc_dt_size(args->src.info.datatype); task->dst_memtype = args->dst.info_v.mem_type; task->src_memtype = args->src.info.mem_type; task->super.post = ucc_tl_self_copy_start; task->super.progress = ucc_tl_self_copy_progress; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; } return UCC_OK; } ucc_status_t ucc_tl_self_scatterv_init(ucc_tl_self_task_t *task) { ucc_coll_args_t *args = &(task->super.bargs.args); if (!ucc_coll_args_is_predefined_dt(args, task->super.team->params.rank)) { return UCC_ERR_NOT_SUPPORTED; } if (UCC_IS_INPLACE(*args)) { /* no copy is required for in-place */ task->super.post = ucc_tl_self_coll_start; task->super.progress = ucc_tl_self_noop_progress; } else { size_t displ = (size_t)ucc_coll_args_get_displacement( args, args->src.info_v.displacements, 0); task->src = PTR_OFFSET(args->src.info_v.buffer, displ); task->dst = args->dst.info.buffer; task->size = args->dst.info.count * ucc_dt_size(args->dst.info.datatype); task->dst_memtype = args->dst.info.mem_type; task->src_memtype = args->src.info_v.mem_type; task->super.post = ucc_tl_self_copy_start; task->super.progress = ucc_tl_self_copy_progress; task->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; } return UCC_OK; } ucc_status_t ucc_tl_self_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_status_t status; ucc_tl_self_task_t *task = ucc_tl_self_coll_init_task(coll_args, team); if (ucc_unlikely(!task)) { return UCC_ERR_NO_MEMORY; } switch (coll_args->args.coll_type) { case UCC_COLL_TYPE_BARRIER: case UCC_COLL_TYPE_BCAST: case UCC_COLL_TYPE_FANIN: case UCC_COLL_TYPE_FANOUT: status = ucc_tl_self_coll_noop_init(task); break; case UCC_COLL_TYPE_REDUCE: case UCC_COLL_TYPE_GATHER: case UCC_COLL_TYPE_ALLTOALL: case UCC_COLL_TYPE_ALLREDUCE: case UCC_COLL_TYPE_ALLGATHER: case UCC_COLL_TYPE_REDUCE_SCATTER: status = ucc_tl_self_coll_copy_init(task); break; case UCC_COLL_TYPE_GATHERV: case UCC_COLL_TYPE_ALLGATHERV: case UCC_COLL_TYPE_REDUCE_SCATTERV: status = ucc_tl_self_coll_copyv_init(task); break; case UCC_COLL_TYPE_ALLTOALLV: status = ucc_tl_self_alltoallv_init(task); break; case UCC_COLL_TYPE_SCATTERV: status = ucc_tl_self_scatterv_init(task); break; default: status = UCC_ERR_NOT_SUPPORTED; } if (ucc_unlikely(status != UCC_OK)) { ucc_tl_self_put_task(task); return status; } tl_trace(team->context->lib, "init coll req %p", task); *task_h = &task->super; return status; } ucc-1.8.0/src/components/tl/self/tl_self_team.c0000664000175000017500000000557715211535620021622 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "coll_score/ucc_coll_score.h" #include "core/ucc_team.h" #include "tl_self.h" UCC_CLASS_INIT_FUNC(ucc_tl_self_team_t, ucc_base_context_t *tl_context, const ucc_base_team_params_t *params) { ucc_tl_self_context_t *ctx = ucc_derived_of(tl_context, ucc_tl_self_context_t); UCC_CLASS_CALL_SUPER_INIT(ucc_tl_team_t, &ctx->super, params); tl_debug(tl_context->lib, "posted tl team: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_self_team_t) { tl_debug(self->super.super.context->lib, "finalizing tl team: %p", self); } UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_self_team_t, ucc_base_team_t); UCC_CLASS_DEFINE(ucc_tl_self_team_t, ucc_tl_team_t); ucc_status_t ucc_tl_self_team_destroy(ucc_base_team_t *tl_team) { UCC_CLASS_DELETE_FUNC_NAME(ucc_tl_self_team_t)(tl_team); return UCC_OK; } ucc_status_t ucc_tl_self_team_create_test(ucc_base_team_t *tl_team) { ucc_tl_self_team_t *team = ucc_derived_of(tl_team, ucc_tl_self_team_t); tl_debug(tl_team->context->lib, "initialized tl team: %p", team); return UCC_OK; } ucc_status_t ucc_tl_self_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p) { ucc_tl_self_team_t *team = ucc_derived_of(tl_team, ucc_tl_self_team_t); ucc_base_context_t *ctx = UCC_TL_TEAM_CTX(team); int mt_n = 0, i; ucc_memory_type_t mem_types[UCC_MEMORY_TYPE_LAST]; ucc_coll_score_t *score; ucc_status_t status; ucc_coll_score_team_info_t team_info; for (i = 0; i < UCC_MEMORY_TYPE_LAST; i++) { mem_types[mt_n++] = (ucc_memory_type_t)i; } team_info.alg_fn = NULL; team_info.default_score = UCC_TL_SELF_DEFAULT_SCORE; team_info.init = ucc_tl_self_coll_init; team_info.num_mem_types = mt_n; team_info.supported_mem_types = mem_types; /* all memory types supported*/ team_info.supported_colls = UCC_TL_SELF_SUPPORTED_COLLS; team_info.size = UCC_TL_TEAM_SIZE(team); status = ucc_coll_score_build_default( tl_team, UCC_TL_SELF_DEFAULT_SCORE, ucc_tl_self_coll_init, UCC_TL_SELF_SUPPORTED_COLLS, mem_types, mt_n, &score); if (UCC_OK != status) { return status; } if (strlen(ctx->score_str) > 0) { status = ucc_coll_score_update_from_str(ctx->score_str, &team_info, &team->super.super, score); if ((status < 0) && (status != UCC_ERR_INVALID_PARAM) && (status != UCC_ERR_NOT_SUPPORTED)) { goto err; } } *score_p = score; return UCC_OK; err: ucc_coll_score_free(score); return status; } ucc-1.8.0/src/components/tl/self/Makefile.am0000664000175000017500000000125315211535620021037 0ustar alastairalastair# # Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Copyright (c) Meta Platforms, Inc. and affiliates. 2022. # if TL_SELF_ENABLED sources = \ tl_self.h \ tl_self.c \ tl_self_coll.c \ tl_self_context.c \ tl_self_lib.c \ tl_self_team.c module_LTLIBRARIES = libucc_tl_self.la libucc_tl_self_la_SOURCES = $(sources) libucc_tl_self_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) libucc_tl_self_la_CFLAGS = $(BASE_CFLAGS) libucc_tl_self_la_LDFLAGS = -version-info $(SOVERSION) --as-needed libucc_tl_self_la_LIBADD = $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/tl/self/tl_self_context.c0000664000175000017500000000442515211535620022347 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "tl_self.h" #include "utils/arch/cpu.h" #include UCC_CLASS_INIT_FUNC(ucc_tl_self_context_t, const ucc_base_context_params_t *params, const ucc_base_config_t *config) { ucc_tl_self_context_config_t *tl_self_config = ucc_derived_of(config, ucc_tl_self_context_config_t); ucc_status_t status; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_context_t, &tl_self_config->super, params->context); memcpy(&self->cfg, tl_self_config, sizeof(*tl_self_config)); status = ucc_mpool_init(&self->req_mp, 0, sizeof(ucc_tl_self_task_t), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_coll_task_mpool_ops, params->thread_mode, "tl_self_req_mp"); if (status != UCC_OK) { tl_error(self->super.super.lib, "failed to initialize tl_self_req mpool"); return status; } return status; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_self_context_t) { tl_debug(self->super.super.lib, "finalizing tl context: %p", self); ucc_mpool_cleanup(&self->req_mp, 1); } UCC_CLASS_DEFINE(ucc_tl_self_context_t, ucc_tl_context_t); ucc_status_t ucc_tl_self_get_context_attr(const ucc_base_context_t *context, /* NOLINT */ ucc_base_ctx_attr_t *attr) /* NOLINT */ { ucc_base_ctx_attr_clear(attr); return UCC_OK; } /* NOLINTBEGIN */ ucc_status_t ucc_tl_self_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h) { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_tl_self_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h) { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_tl_self_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **pack_buffer) { return UCC_ERR_NOT_SUPPORTED; } /* NOLINTEND */ ucc-1.8.0/src/components/tl/rccl/0000775000175000017500000000000015211535620016774 5ustar alastairalastairucc-1.8.0/src/components/tl/rccl/allgatherv/0000775000175000017500000000000015211535620021125 5ustar alastairalastairucc-1.8.0/src/components/tl/rccl/allgatherv/allgatherv.h0000664000175000017500000000357315211535620023437 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef RCCL_ALLGATHERV_H_ #define RCCL_ALLGATHERV_H_ #include "tl_rccl_coll.h" enum { UCC_TL_RCCL_ALLGATHERV_ALG_P2P, UCC_TL_RCCL_ALLGATHERV_ALG_BCOPY, UCC_TL_RCCL_ALLGATHERV_ALG_BCAST, UCC_TL_RCCL_ALLGATHERV_ALG_LAST }; #define UCC_TL_RCCL_ALLGATHERV_DEFAULT_ALG_SELECT_STR \ "allgatherv:rocm:0-16k:@0#allgatherv:rocm:16k-1M:@1#allgatherv:rocm:1M-inf:@2" extern ucc_base_coll_alg_info_t ucc_tl_rccl_allgatherv_algs[UCC_TL_RCCL_ALLGATHERV_ALG_LAST + 1]; ucc_status_t ucc_tl_rccl_allgatherv_p2p_start(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_rccl_allgatherv_p2p_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h); ucc_status_t ucc_tl_rccl_allgatherv_bcopy_start(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_rccl_allgatherv_bcopy_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h); ucc_status_t ucc_tl_rccl_allgatherv_bcast_start(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_rccl_allgatherv_bcast_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h); static inline int ucc_tl_rccl_allgatherv_alg_from_str(const char *str) { int i; for (i = 0; i < UCC_TL_RCCL_ALLGATHERV_ALG_LAST; i++) { if (0 == strcasecmp(str, ucc_tl_rccl_allgatherv_algs[i].name)) { break; } } return i; } #endif ucc-1.8.0/src/components/tl/rccl/allgatherv/allgatherv.c0000664000175000017500000003102515211535620023423 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "tl_rccl.h" #include "allgatherv.h" #include "components/mc/ucc_mc.h" #include "core/ucc_ee.h" ucc_base_coll_alg_info_t ucc_tl_rccl_allgatherv_algs[UCC_TL_RCCL_ALLGATHERV_ALG_LAST + 1] = { [UCC_TL_RCCL_ALLGATHERV_ALG_P2P] = {.id = UCC_TL_RCCL_ALLGATHERV_ALG_P2P, .name = "p2p", .desc = "allgatherv based on rccl point-to-point"}, [UCC_TL_RCCL_ALLGATHERV_ALG_BCOPY] = {.id = UCC_TL_RCCL_ALLGATHERV_ALG_BCOPY, .name = "bcopy", .desc = "allgatherv with buffered copy"}, [UCC_TL_RCCL_ALLGATHERV_ALG_BCAST] = {.id = UCC_TL_RCCL_ALLGATHERV_ALG_BCAST, .name = "bcast", .desc = "allgatherv based on rccl bcast"}, [UCC_TL_RCCL_ALLGATHERV_ALG_LAST] = { .id = 0, .name = NULL, .desc = NULL}}; #define CHECK_INPLACE(_args, _team) \ do { \ if (UCC_IS_INPLACE((_args))) { \ tl_error(UCC_TL_TEAM_LIB((_team)), \ "inplace allgatherv is not supported"); \ status = UCC_ERR_NOT_SUPPORTED; \ goto out; \ } \ } while(0) #define CHECK_USERDEFINED_DT(_args, _team) \ do { \ if (!UCC_DT_IS_PREDEFINED((_args).src.info.datatype) || \ !UCC_DT_IS_PREDEFINED((_args).dst.info_v.datatype)) { \ tl_error(UCC_TL_TEAM_LIB((_team)), \ "user defined datatype is not supported"); \ status = UCC_ERR_NOT_SUPPORTED; \ goto out; \ } \ } while(0) ucc_status_t ucc_tl_rccl_allgatherv_p2p_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; ucc_status_t status = UCC_OK; void *sbuf = args->src.info.buffer; void *rbuf = args->dst.info_v.buffer; size_t sdt_size, rdt_size, count, displ; ucc_rank_t peer; task->super.super.status = UCC_INPROGRESS; sdt_size = ucc_dt_size(args->src.info.datatype); rdt_size = ucc_dt_size(args->dst.info_v.datatype); UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_allgatherv_start", 0); RCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team)); count = args->src.info.count; if (count != 0) { for (peer = 0; peer < size; peer++) { RCCLCHECK_GOTO(ncclSend(sbuf, count * sdt_size, ncclChar, peer, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } } for (peer = 0; peer < size; peer++) { count = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); if (count != 0) { displ = ucc_coll_args_get_displacement( args, args->dst.info_v.displacements, peer); RCCLCHECK_GOTO(ncclRecv(PTR_OFFSET(rbuf, displ * rdt_size), count * rdt_size, ncclChar, peer, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } } RCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team)); status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_allgatherv_p2p_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h) { ucc_tl_rccl_team_t *rccl_team = ucc_derived_of(team, ucc_tl_rccl_team_t); ucc_coll_args_t *args = &coll_args->args; ucc_status_t status = UCC_OK; ucc_tl_rccl_task_t *task; CHECK_INPLACE(*args, rccl_team); CHECK_USERDEFINED_DT(*args, rccl_team); task = ucc_tl_rccl_init_task(coll_args, team); if (!task) { return UCC_ERR_NO_MESSAGE; } task->super.post = ucc_tl_rccl_allgatherv_p2p_start; *task_h = &task->super; out: return status; } ucc_status_t ucc_tl_rccl_allgatherv_bcopy_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; ucc_status_t status = UCC_OK; void *sbuf = args->src.info.buffer; void *rbuf = args->dst.info_v.buffer; void *scratch = task->allgatherv_bcopy.scratch->addr; size_t max_count, rdt_size, sdt_size, displ, scount, rcount; ucc_rank_t peer; task->super.super.status = UCC_INPROGRESS; UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_allgatherv_start", 0); max_count = task->allgatherv_bcopy.max_count; scount = args->src.info.count; rdt_size = ucc_dt_size(args->dst.info_v.datatype); sdt_size = ucc_dt_size(args->src.info.datatype); if (max_count * rdt_size > scount * sdt_size) { HIPCHECK_GOTO(hipMemcpyAsync(PTR_OFFSET(scratch, max_count * rdt_size * size), sbuf, scount * sdt_size, hipMemcpyDeviceToDevice, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); sbuf = PTR_OFFSET(scratch, max_count * rdt_size * size); } RCCLCHECK_GOTO(ncclAllGather(sbuf, scratch, max_count * rdt_size, ncclChar, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); for (peer = 0; peer < size; peer++) { rcount = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, peer); HIPCHECK_GOTO(hipMemcpyAsync(PTR_OFFSET(rbuf, displ * rdt_size), PTR_OFFSET(scratch, peer * max_count * rdt_size), rcount * rdt_size, hipMemcpyDeviceToDevice, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_allgatherv_bcopy_finalize(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_mc_free(task->allgatherv_bcopy.scratch); return ucc_tl_rccl_coll_finalize(coll_task); } ucc_status_t ucc_tl_rccl_allgatherv_bcopy_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h) { ucc_tl_rccl_team_t *rccl_team = ucc_derived_of(team, ucc_tl_rccl_team_t); ucc_coll_args_t *args = &coll_args->args; ucc_status_t status = UCC_OK; ucc_tl_rccl_task_t *task; size_t max_count, sdt_size, rdt_size; ucc_rank_t peer; CHECK_INPLACE(*args, rccl_team); CHECK_USERDEFINED_DT(*args, rccl_team); task = ucc_tl_rccl_init_task(coll_args, team); if (ucc_unlikely(!task)) { return UCC_ERR_NO_MESSAGE; } sdt_size = ucc_dt_size(args->src.info.datatype); rdt_size = ucc_dt_size(args->dst.info_v.datatype); max_count = ucc_coll_args_get_max_count(args, args->dst.info_v.counts, UCC_TL_TEAM_SIZE(rccl_team)); for (peer = 1; peer < team->params.size; peer++) { max_count = ucc_max(ucc_coll_args_get_count(args, args->dst.info_v.counts, peer), max_count); } task->allgatherv_bcopy.max_count = max_count; if (max_count * rdt_size > args->src.info.count * sdt_size) { status = ucc_mc_alloc(&task->allgatherv_bcopy.scratch, (team->params.size + 1) * max_count * ucc_dt_size(args->dst.info_v.datatype), UCC_MEMORY_TYPE_ROCM); } else { status = ucc_mc_alloc(&task->allgatherv_bcopy.scratch, max_count * team->params.size * ucc_dt_size(args->dst.info_v.datatype), UCC_MEMORY_TYPE_ROCM); } if (ucc_unlikely(status != UCC_OK)) { ucc_tl_rccl_free_task(task); return status; } task->super.post = ucc_tl_rccl_allgatherv_bcopy_start; task->super.finalize = ucc_tl_rccl_allgatherv_bcopy_finalize; *task_h = &task->super; out: return status; } ucc_status_t ucc_tl_rccl_allgatherv_bcast_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; ucc_status_t status = UCC_OK; void *sbuf = args->src.info.buffer; ptrdiff_t rbuf = (ptrdiff_t)args->dst.info_v.buffer; size_t rdt_size, count, displ; ucc_rank_t peer; task->super.super.status = UCC_INPROGRESS; rdt_size = ucc_dt_size(args->dst.info_v.datatype); UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_allgatherv_start", 0); RCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team)); for (peer = 0; peer < size; peer++) { count = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, peer); RCCLCHECK_GOTO(ncclBroadcast(sbuf, PTR_OFFSET(rbuf, displ * rdt_size), count * rdt_size, ncclChar, peer, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } RCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team)); status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_allgatherv_bcast_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h) { ucc_tl_rccl_team_t *rccl_team = ucc_derived_of(team, ucc_tl_rccl_team_t); ucc_coll_args_t *args = &coll_args->args; ucc_status_t status = UCC_OK; ucc_tl_rccl_task_t *task; CHECK_INPLACE(*args, rccl_team); CHECK_USERDEFINED_DT(*args, rccl_team); task = ucc_tl_rccl_init_task(coll_args, team); if (ucc_unlikely(!task)) { return UCC_ERR_NO_MESSAGE; } task->super.post = ucc_tl_rccl_allgatherv_bcast_start; *task_h = &task->super; out: return status; } ucc-1.8.0/src/components/tl/rccl/tl_rccl_lib.c0000664000175000017500000000323115211535620021407 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "tl_rccl.h" /* NOLINTNEXTLINE params is not used*/ UCC_CLASS_INIT_FUNC(ucc_tl_rccl_lib_t, const ucc_base_lib_params_t *params, const ucc_base_config_t *config) { const ucc_tl_lib_config_t *tl_config = ucc_derived_of(config, ucc_tl_lib_config_t); UCC_CLASS_CALL_SUPER_INIT(ucc_tl_lib_t, &ucc_tl_rccl.super, tl_config); tl_debug(&self->super, "initialized lib object: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_rccl_lib_t) { tl_debug(&self->super, "finalizing lib object: %p", self); } UCC_CLASS_DEFINE(ucc_tl_rccl_lib_t, ucc_tl_lib_t); ucc_status_t ucc_tl_rccl_get_lib_attr(const ucc_base_lib_t *lib, /* NOLINT */ ucc_base_lib_attr_t *base_attr) { ucc_tl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_tl_lib_attr_t); attr->super.attr.thread_mode = UCC_THREAD_MULTIPLE; attr->super.attr.coll_types = UCC_TL_RCCL_SUPPORTED_COLLS; attr->super.flags = 0; if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MIN_TEAM_SIZE) { attr->super.min_team_size = lib->min_team_size; } if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MAX_TEAM_SIZE) { attr->super.max_team_size = UCC_RANK_MAX; } return UCC_OK; } ucc_status_t ucc_tl_rccl_get_lib_properties(ucc_base_lib_properties_t *prop) { prop->default_team_size = 2; prop->min_team_size = 2; prop->max_team_size = UCC_RANK_MAX; return UCC_OK; } ucc-1.8.0/src/components/tl/rccl/tl_rccl.h0000664000175000017500000001316615211535620020576 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Facebook, Inc. and its affiliates. 2021. * Copyright (C) Advanced Micro Devices, Inc. 2022-2023. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_TL_RCCL_H_ #define UCC_TL_RCCL_H_ #include "components/mc/base/ucc_mc_base.h" #include "components/tl/ucc_tl.h" #include "components/tl/ucc_tl_log.h" #include "utils/ucc_mpool.h" #include #pragma GCC diagnostic push #if defined(__clang__) #pragma GCC diagnostic ignored "-Wunknown-warning-option" #endif #include #pragma GCC diagnostic pop #ifdef RCCL_OLD_HEADERS #include #else #include #endif #ifndef UCC_TL_RCCL_DEFAULT_SCORE #define UCC_TL_RCCL_DEFAULT_SCORE 20 #endif #ifdef HAVE_PROFILING_TL_RCCL #include "utils/profile/ucc_profile.h" #else #include "utils/profile/ucc_profile_off.h" #endif #define UCC_TL_RCCL_PROFILE_FUNC UCC_PROFILE_FUNC #define UCC_TL_RCCL_PROFILE_FUNC_VOID UCC_PROFILE_FUNC_VOID #define UCC_TL_RCCL_PROFILE_REQUEST_NEW UCC_PROFILE_REQUEST_NEW #define UCC_TL_RCCL_PROFILE_REQUEST_EVENT UCC_PROFILE_REQUEST_EVENT #define UCC_TL_RCCL_PROFILE_REQUEST_FREE UCC_PROFILE_REQUEST_FREE typedef struct ucc_tl_rccl_iface { ucc_tl_iface_t super; } ucc_tl_rccl_iface_t; extern ucc_tl_rccl_iface_t ucc_tl_rccl; typedef struct ucc_tl_rccl_lib_config { ucc_tl_lib_config_t super; } ucc_tl_rccl_lib_config_t; typedef enum ucc_tl_rccl_completion_sync_type { UCC_TL_RCCL_COMPLETION_SYNC_TYPE_EVENT, UCC_TL_RCCL_COMPLETION_SYNC_TYPE_MEMOPS, UCC_TL_RCCL_COMPLETION_SYNC_TYPE_AUTO, UCC_TL_RCCL_COMPLETION_SYNC_TYPE_LAST } ucc_tl_rccl_completion_sync_type_t; typedef struct ucc_tl_rccl_context_config { ucc_tl_context_config_t super; ucc_tl_rccl_completion_sync_type_t sync_type; } ucc_tl_rccl_context_config_t; typedef struct ucc_tl_rccl_lib { ucc_tl_lib_t super; } ucc_tl_rccl_lib_t; UCC_CLASS_DECLARE(ucc_tl_rccl_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_rccl_context { ucc_tl_context_t super; ucc_tl_rccl_context_config_t cfg; ucc_mpool_t req_mp; void *scratch_buf; } ucc_tl_rccl_context_t; UCC_CLASS_DECLARE(ucc_tl_rccl_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_rccl_team { ucc_tl_team_t super; ncclUniqueId *unique_id; void *oob_req; ncclComm_t rccl_comm; hipStream_t stream; } ucc_tl_rccl_team_t; typedef struct ucc_tl_rccl_task { ucc_coll_task_t super; ucc_status_t host_status; ucc_status_t *dev_status; void *completed; union { struct { ucc_mc_buffer_header_t *scratch; size_t max_count; } allgatherv_bcopy; }; } ucc_tl_rccl_task_t; #define TASK_TEAM(_task) \ (ucc_derived_of((_task)->super.team, ucc_tl_rccl_team_t)) #define TASK_CTX(_task) \ (ucc_derived_of((_task)->super.team->context, ucc_tl_rccl_context_t)) #define TASK_LIB(_task) \ (ucc_derived_of((_task)->super.team->context->lib, ucc_tl_rccl_lib_t)) #define TASK_ARGS(_task) (_task)->super.bargs.args #define UCC_TL_RCCL_SUPPORTED_COLLS \ (UCC_COLL_TYPE_ALLTOALL | UCC_COLL_TYPE_ALLTOALLV | \ UCC_COLL_TYPE_ALLGATHER | UCC_COLL_TYPE_ALLGATHERV | \ UCC_COLL_TYPE_ALLREDUCE | UCC_COLL_TYPE_BCAST | \ UCC_COLL_TYPE_REDUCE_SCATTER | UCC_COLL_TYPE_REDUCE | \ UCC_COLL_TYPE_BARRIER | UCC_COLL_TYPE_GATHER | \ UCC_COLL_TYPE_GATHERV | UCC_COLL_TYPE_SCATTER | \ UCC_COLL_TYPE_SCATTERV) UCC_CLASS_DECLARE(ucc_tl_rccl_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); #define RCCLCHECK_GOTO(_cmd, _label, _status, _lib) \ do { \ ncclResult_t e = _cmd; \ if (ncclSuccess != e) { \ tl_error(_lib, "RCCL error %d %s", e, ncclGetErrorString(e)); \ _status = UCC_ERR_NO_MESSAGE; \ goto _label; \ } \ } while (0) #define HIPCHECK_GOTO(_cmd, _label, _status, _lib) \ do { \ hipError_t e = _cmd; \ if (hipSuccess != e) { \ tl_error(_lib, "HIP error %d %s", e, hipGetErrorName(e)); \ _status = UCC_ERR_NO_MESSAGE; \ goto _label; \ } \ } while (0) #define UCC_TL_RCCL_TEAM_LIB(_team) \ (ucc_derived_of((_team)->super.super.context->lib, ucc_tl_rccl_lib_t)) #endif ucc-1.8.0/src/components/tl/rccl/tl_rccl_context.c0000664000175000017500000001132315211535620022326 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Facebook, Inc. and its affiliates. 2021. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "tl_rccl.h" #include "components/mc/ucc_mc.h" #include "components/ec/ucc_ec.h" #include "utils/arch/cpu.h" void ucc_tl_rccl_event_collective_progress(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_status_t status; ucc_assert(task->completed != NULL); status = ucc_ec_event_test(task->completed, UCC_EE_ROCM_STREAM); coll_task->status = status; #ifdef HAVE_PROFILING_TL_RCCL if (coll_task->status == UCC_OK) { UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_coll_done", 0); } #endif } void ucc_tl_rccl_driver_collective_progress(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); coll_task->status = task->host_status; #ifdef HAVE_PROFILING_TL_RCCL if (coll_task->status == UCC_OK) { UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_coll_done", 0); } #endif } static void ucc_tl_rccl_req_mpool_obj_init(ucc_mpool_t *mp, void *obj, //NOLINT: mp is unused void *chunk) //NOLINT: chunk is unused { ucc_tl_rccl_task_t *req = (ucc_tl_rccl_task_t*) obj; ucc_coll_task_construct(&req->super); req->super.progress = ucc_tl_rccl_event_collective_progress; } static void ucc_tl_rccl_req_mpool_obj_cleanup(ucc_mpool_t *mp, void *obj) //NOLINT: mp is unused { ucc_coll_task_destruct(obj); } static ucc_mpool_ops_t ucc_tl_rccl_req_mpool_ops = { .chunk_alloc = ucc_mpool_hugetlb_malloc, .chunk_release = ucc_mpool_hugetlb_free, .obj_init = ucc_tl_rccl_req_mpool_obj_init, .obj_cleanup = ucc_tl_rccl_req_mpool_obj_cleanup }; UCC_CLASS_INIT_FUNC(ucc_tl_rccl_context_t, const ucc_base_context_params_t *params, const ucc_base_config_t *config) { ucc_tl_rccl_context_config_t *tl_rccl_config = ucc_derived_of(config, ucc_tl_rccl_context_config_t); ucc_status_t status; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_context_t, &tl_rccl_config->super, params->context); memcpy(&self->cfg, tl_rccl_config, sizeof(*tl_rccl_config)); // RCCL does not currently support memops for synchronization if (self->cfg.sync_type == UCC_TL_RCCL_COMPLETION_SYNC_TYPE_MEMOPS) { tl_error(self->super.super.lib, "memops not supported"); return UCC_ERR_NOT_SUPPORTED; } if (self->cfg.sync_type != UCC_TL_RCCL_COMPLETION_SYNC_TYPE_EVENT) { tl_debug(self->super.super.lib, "fallback to event completion sync"); self->cfg.sync_type = UCC_TL_RCCL_COMPLETION_SYNC_TYPE_EVENT; } ucc_assert(self->cfg.sync_type == UCC_TL_RCCL_COMPLETION_SYNC_TYPE_EVENT); tl_debug(self->super.super.lib, "using event completion sync"); status = ucc_mpool_init(&self->req_mp, 0, sizeof(ucc_tl_rccl_task_t), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_tl_rccl_req_mpool_ops, params->thread_mode, "tl_rccl_req_mp"); if (status != UCC_OK) { tl_error(self->super.super.lib, "failed to initialize tl_rccl_req mpool"); return status; } // scratch buffer for barrier hipError_t hip_st = hipMalloc(&self->scratch_buf, sizeof(float)); if (hip_st != hipSuccess) { return UCC_ERR_NO_MEMORY; } tl_debug(self->super.super.lib, "initialized tl context: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_rccl_context_t) { tl_debug(self->super.super.lib, "finalizing tl context: %p", self); ucc_mpool_cleanup(&self->req_mp, 1); hipFree(self->scratch_buf); self->scratch_buf = NULL; } UCC_CLASS_DEFINE(ucc_tl_rccl_context_t, ucc_tl_context_t); ucc_status_t ucc_tl_rccl_get_context_attr(const ucc_base_context_t *context, /* NOLINT */ ucc_base_ctx_attr_t *attr) { ucc_base_ctx_attr_clear(attr); return UCC_OK; } ucc_status_t ucc_tl_rccl_mem_map(const ucc_base_context_t *context, int type, /* NOLINT */ void *memh, void *tl_h) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_tl_rccl_mem_unmap(const ucc_base_context_t *context, int type, /* NOLINT */ void *memh) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_tl_rccl_memh_pack(const ucc_base_context_t *context, /* NOLINT */ int type, void *memh, void **pack_buffer) /* NOLINT */ { return UCC_ERR_NOT_SUPPORTED; } ucc-1.8.0/src/components/tl/rccl/configure.m40000664000175000017500000000104015211535620021212 0ustar alastairalastair# # Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. # tl_rccl_enabled=n CHECK_TLS_REQUIRED(["rccl"]) AS_IF([test "$CHECKED_TL_REQUIRED" = "y"], [ CHECK_RCCL AC_MSG_RESULT([RCCL support: $rccl_happy]) if test $rccl_happy = "yes"; then tl_modules="${tl_modules}:rccl" tl_rccl_enabled=y fi ], []) AM_CONDITIONAL([TL_RCCL_ENABLED], [test "$tl_rccl_enabled" = "y"]) AC_CONFIG_FILES([src/components/tl/rccl/Makefile]) ucc-1.8.0/src/components/tl/rccl/tl_rccl.c0000664000175000017500000000650615211535620020571 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "tl_rccl.h" #include "components/mc/base/ucc_mc_base.h" #include "allgatherv/allgatherv.h" ucc_status_t ucc_tl_rccl_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t *base_attr); ucc_status_t ucc_tl_rccl_get_lib_properties(ucc_base_lib_properties_t *prop); ucc_status_t ucc_tl_rccl_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t *base_attr); ucc_status_t ucc_tl_rccl_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_rccl_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_rccl_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **pack_buffer); static ucc_config_field_t ucc_tl_rccl_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_rccl_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_lib_config_table)}, {NULL}}; const char* ucc_tl_rccl_completion_sync_names[] = { [UCC_TL_RCCL_COMPLETION_SYNC_TYPE_EVENT] = "event", [UCC_TL_RCCL_COMPLETION_SYNC_TYPE_MEMOPS] = "driver", [UCC_TL_RCCL_COMPLETION_SYNC_TYPE_AUTO] = "auto", [UCC_TL_RCCL_COMPLETION_SYNC_TYPE_LAST] = NULL }; static ucs_config_field_t ucc_tl_rccl_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_rccl_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_context_config_table)}, {"SYNC", "auto", "Determines how UCC tests completion of RCCL collective", ucs_offsetof(ucc_tl_rccl_context_config_t, sync_type), UCS_CONFIG_TYPE_ENUM(ucc_tl_rccl_completion_sync_names) }, {NULL}}; UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_rccl_lib_t, ucc_base_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_rccl_lib_t, ucc_base_lib_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_rccl_context_t, ucc_base_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_rccl_context_t, ucc_base_context_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_rccl_team_t, ucc_base_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_tl_rccl_team_create_test(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_rccl_team_destroy(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_rccl_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task); ucc_status_t ucc_tl_rccl_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p); UCC_TL_IFACE_DECLARE(rccl, RCCL); __attribute__((constructor)) static void tl_rccl_iface_init(void) { ucc_tl_rccl.super.alg_info[ucc_ilog2(UCC_COLL_TYPE_ALLGATHERV)] = ucc_tl_rccl_allgatherv_algs; } ucc-1.8.0/src/components/tl/rccl/tl_rccl_coll.h0000664000175000017500000000417715211535620021611 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Facebook, Inc. and its affiliates. 2021. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_TL_RCCL_COLL_H_ #define UCC_TL_RCCL_COLL_H_ #include "tl_rccl.h" #define UCC_TL_RCCL_N_DEFAULT_ALG_SELECT_STR 1 extern const char *ucc_tl_rccl_default_alg_select_str[UCC_TL_RCCL_N_DEFAULT_ALG_SELECT_STR]; ucc_status_t ucc_tl_rccl_alg_id_to_init(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, ucc_base_coll_init_fn_t *init); ucc_tl_rccl_task_t * ucc_tl_rccl_init_task(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team); void ucc_tl_rccl_free_task(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_triggered_post(ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_rccl_coll_finalize(ucc_coll_task_t *coll_task); ucc_status_t ucc_tl_rccl_collective_sync(ucc_tl_rccl_task_t *task, hipStream_t stream); ucc_status_t ucc_tl_rccl_allgather_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_allgatherv_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_allreduce_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_alltoall_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_alltoallv_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_bcast_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_reduce_scatter_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_reduce_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_barrier_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_gather_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_gatherv_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_scatter_init(ucc_tl_rccl_task_t *task); ucc_status_t ucc_tl_rccl_scatterv_init(ucc_tl_rccl_task_t *task); #endif ucc-1.8.0/src/components/tl/rccl/tl_rccl_team.c0000664000175000017500000002131715211535620021574 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Facebook, Inc. and its affiliates. 2021. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "tl_rccl.h" #include "tl_rccl_coll.h" #include "components/mc/ucc_mc.h" #include "components/ec/ucc_ec.h" #include "coll_score/ucc_coll_score.h" UCC_CLASS_INIT_FUNC(ucc_tl_rccl_team_t, ucc_base_context_t *tl_context, const ucc_base_team_params_t *params) { ucc_tl_rccl_context_t *ctx = ucc_derived_of(tl_context, ucc_tl_rccl_context_t); ucc_status_t status; ucc_rank_t size; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_team_t, &ctx->super, params); size = UCC_TL_TEAM_SIZE(self); self->unique_id = ucc_malloc(sizeof(ncclUniqueId) * (size + 1), "tl_rccl_unique_id"); if (!self->unique_id) { tl_error(ctx->super.super.lib, "failed to allocate %zd bytes for unique_id array", sizeof(ncclUniqueId) * (size + 1)); return UCC_ERR_NO_MEMORY; } if (UCC_TL_TEAM_RANK(self) == 0) { ncclResult_t st; st = ncclGetUniqueId(&self->unique_id[size]); if (st != ncclSuccess) { tl_error(ctx->super.super.lib, "failed to get unique id"); memset(&self->unique_id[size], 0, sizeof(ncclUniqueId)); } } status = UCC_TL_TEAM_OOB(self).allgather( &self->unique_id[size], self->unique_id, sizeof(ncclUniqueId), UCC_TL_TEAM_OOB(self).coll_info, &self->oob_req); if (status != UCC_OK) { tl_error(ctx->super.super.lib, "failed to start oob allgather"); goto free_unique_id; } return UCC_OK; free_unique_id: ucc_free(self->unique_id); return status; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_rccl_team_t) { tl_debug(self->super.super.context->lib, "finalizing tl team: %p", self); if (self->rccl_comm) { ncclCommDestroy(self->rccl_comm); hipStreamDestroy(self->stream); } } UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_rccl_team_t, ucc_base_team_t); UCC_CLASS_DEFINE(ucc_tl_rccl_team_t, ucc_tl_team_t); ucc_status_t ucc_tl_rccl_team_destroy(ucc_base_team_t *tl_team) { UCC_CLASS_DELETE_FUNC_NAME(ucc_tl_rccl_team_t)(tl_team); return UCC_OK; } ucc_status_t ucc_tl_rccl_team_create_test(ucc_base_team_t *tl_team) { ucc_tl_rccl_team_t *team = ucc_derived_of(tl_team, ucc_tl_rccl_team_t); ucc_status_t status; ncclResult_t rccl_status; ncclUniqueId errorid; status = UCC_TL_TEAM_OOB(team).req_test(team->oob_req); if (status == UCC_INPROGRESS) { return UCC_INPROGRESS; } if (status != UCC_OK) { UCC_TL_TEAM_OOB(team).req_free(team->oob_req); tl_error(tl_team->context->lib, "oob req test failed"); goto free_unique_id; } status = UCC_TL_TEAM_OOB(team).req_free(team->oob_req); if (status != UCC_OK) { tl_error(tl_team->context->lib, "oob req free failed"); goto free_unique_id; } /* check unique id is valid */ memset(&errorid, 0, sizeof(errorid)); if (!memcmp(&errorid, team->unique_id, sizeof(errorid))) { tl_error(tl_team->context->lib, "incorrect unique id"); goto free_unique_id; } HIPCHECK_GOTO(hipStreamCreateWithFlags(&team->stream, hipStreamNonBlocking), free_unique_id, status, tl_team->context->lib); rccl_status = ncclCommInitRank(&team->rccl_comm, UCC_TL_TEAM_SIZE(team), team->unique_id[0], UCC_TL_TEAM_RANK(team)); if (rccl_status != ncclSuccess) { tl_debug(tl_team->context->lib, "RCCL error %d %s", rccl_status, ncclGetErrorString(rccl_status)); status = UCC_ERR_NO_MESSAGE; goto free_stream; } ucc_free(team->unique_id); tl_debug(tl_team->context->lib, "initialized tl team: %p", team); return UCC_OK; free_stream: hipStreamDestroy(team->stream); free_unique_id: ucc_free(team->unique_id); return status; } ucc_status_t ucc_tl_rccl_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_rccl_task_t *task; ucc_status_t status; task = ucc_tl_rccl_init_task(coll_args, team); if (ucc_unlikely(!task)) { return UCC_ERR_NO_MESSAGE; } switch (coll_args->args.coll_type) { case UCC_COLL_TYPE_ALLGATHER: status = ucc_tl_rccl_allgather_init(task); break; case UCC_COLL_TYPE_ALLGATHERV: status = ucc_tl_rccl_allgatherv_init(task); break; case UCC_COLL_TYPE_ALLREDUCE: status = ucc_tl_rccl_allreduce_init(task); break; case UCC_COLL_TYPE_ALLTOALL: status = ucc_tl_rccl_alltoall_init(task); break; case UCC_COLL_TYPE_ALLTOALLV: status = ucc_tl_rccl_alltoallv_init(task); break; case UCC_COLL_TYPE_BCAST: status = ucc_tl_rccl_bcast_init(task); break; case UCC_COLL_TYPE_REDUCE_SCATTER: status = ucc_tl_rccl_reduce_scatter_init(task); break; case UCC_COLL_TYPE_REDUCE: status = ucc_tl_rccl_reduce_init(task); break; case UCC_COLL_TYPE_BARRIER: status = ucc_tl_rccl_barrier_init(task); break; case UCC_COLL_TYPE_GATHER: status = ucc_tl_rccl_gather_init(task); break; case UCC_COLL_TYPE_GATHERV: status = ucc_tl_rccl_gatherv_init(task); break; case UCC_COLL_TYPE_SCATTER: status = ucc_tl_rccl_scatter_init(task); break; case UCC_COLL_TYPE_SCATTERV: status = ucc_tl_rccl_scatterv_init(task); break; default: tl_error(UCC_TASK_LIB(task), "collective %d is not supported by rccl tl", coll_args->args.coll_type); status = UCC_ERR_NOT_SUPPORTED; } if (ucc_unlikely(status != UCC_OK)) { goto free_task; } tl_debug(UCC_TASK_LIB(task), "init coll task %p", task); *task_h = &task->super; return status; free_task: ucc_tl_rccl_free_task(task); return status; } ucc_status_t ucc_tl_rccl_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p) { ucc_tl_rccl_team_t *team = ucc_derived_of(tl_team, ucc_tl_rccl_team_t); ucc_base_context_t *ctx = UCC_TL_TEAM_CTX(team); ucc_memory_type_t mt = UCC_MEMORY_TYPE_ROCM; ucc_coll_score_t *score; ucc_status_t status; int i; ucc_coll_score_team_info_t team_info; team_info.alg_fn = ucc_tl_rccl_alg_id_to_init; team_info.default_score = UCC_TL_RCCL_DEFAULT_SCORE; team_info.init = ucc_tl_rccl_coll_init; team_info.num_mem_types = 1; team_info.supported_mem_types = &mt; team_info.supported_colls = UCC_TL_RCCL_SUPPORTED_COLLS; team_info.size = UCC_TL_TEAM_SIZE(team); /* There can be a different logic for different coll_type/mem_type. Right now just init everything the same way. */ status = ucc_coll_score_build_default(tl_team, UCC_TL_RCCL_DEFAULT_SCORE, ucc_tl_rccl_coll_init, UCC_TL_RCCL_SUPPORTED_COLLS, &mt, 1, &score); if (ucc_unlikely(UCC_OK != status)) { return status; } for (i = 0; i < UCC_TL_RCCL_N_DEFAULT_ALG_SELECT_STR; i++) { status = ucc_coll_score_update_from_str( ucc_tl_rccl_default_alg_select_str[i], &team_info, &team->super.super, score); if (ucc_unlikely(UCC_OK != status)) { tl_error(tl_team->context->lib, "failed to apply default coll select setting: %s", ucc_tl_rccl_default_alg_select_str[i]); goto err; } } // add barrier, which might be triggered from host memory type // use lower score status = ucc_coll_score_add_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, 0, UCC_MSG_MAX, 1, ucc_tl_rccl_coll_init, tl_team); if (ucc_unlikely(UCC_OK != status)) { return status; } if (strlen(ctx->score_str) > 0) { status = ucc_coll_score_update_from_str(ctx->score_str, &team_info, &team->super.super, score); /* If INVALID_PARAM - User provided incorrect input - try to proceed */ if ((status < 0) && (status != UCC_ERR_INVALID_PARAM) && (status != UCC_ERR_NOT_SUPPORTED)) { goto err; } } *score_p = score; return UCC_OK; err: ucc_coll_score_free(score); return status; } ucc-1.8.0/src/components/tl/rccl/Makefile.am0000664000175000017500000000167115211535620021035 0ustar alastairalastair# # Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. # if TL_RCCL_ENABLED allgatherv = \ allgatherv/allgatherv.h \ allgatherv/allgatherv.c sources = \ tl_rccl.h \ tl_rccl.c \ tl_rccl_lib.c \ tl_rccl_context.c \ tl_rccl_team.c \ tl_rccl_coll.h \ tl_rccl_coll.c \ $(allgatherv) module_LTLIBRARIES = libucc_tl_rccl.la libucc_tl_rccl_la_SOURCES = $(sources) libucc_tl_rccl_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(ROCM_CPPFLAGS) $(HIP_CPPFLAGS) $(RCCL_CPPFLAGS) libucc_tl_rccl_la_CFLAGS = $(BASE_CFLAGS) libucc_tl_rccl_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(ROCM_LDFLAGS) $(HIP_LDFLAGS) $(RCCL_LDFLAGS) libucc_tl_rccl_la_LIBADD = $(ROCM_LIBS) $(HIP_LIBS) $(RCCL_LIBADD) $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/tl/rccl/tl_rccl_coll.c0000664000175000017500000011354315211535620021602 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Facebook, Inc. and its affiliates. 2021. # Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "tl_rccl_coll.h" #include "components/mc/ucc_mc.h" #include "components/ec/ucc_ec.h" #include "core/ucc_ee.h" #include "utils/ucc_compiler_def.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include "allgatherv/allgatherv.h" #define ncclOpUnsupported (ncclNumOps + 1) #define ncclDataTypeUnsupported (ncclNumTypes + 1) ncclDataType_t ucc_to_rccl_dtype[] = { [UCC_DT_PREDEFINED_ID(UCC_DT_INT8)] = (ncclDataType_t)ncclInt8, [UCC_DT_PREDEFINED_ID(UCC_DT_INT16)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_INT32)] = (ncclDataType_t)ncclInt32, [UCC_DT_PREDEFINED_ID(UCC_DT_INT64)] = (ncclDataType_t)ncclInt64, [UCC_DT_PREDEFINED_ID(UCC_DT_INT128)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT8)] = (ncclDataType_t)ncclUint8, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT16)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT32)] = (ncclDataType_t)ncclUint32, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT64)] = (ncclDataType_t)ncclUint64, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT128)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT16)] = (ncclDataType_t)ncclFloat16, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT32)] = (ncclDataType_t)ncclFloat32, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT64)] = (ncclDataType_t)ncclFloat64, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT128)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT32_COMPLEX)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT64_COMPLEX)] = (ncclDataType_t)ncclDataTypeUnsupported, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT128_COMPLEX)] = (ncclDataType_t)ncclDataTypeUnsupported, #if NCCL_VERSION_CODE >= NCCL_VERSION(2,10,3) [UCC_DT_PREDEFINED_ID(UCC_DT_BFLOAT16)] = (ncclDataType_t)ncclBfloat16, #else [UCC_DT_PREDEFINED_ID(UCC_DT_BFLOAT16)] = (ncclDataType_t)ncclDataTypeUnsupported, #endif }; ncclRedOp_t ucc_to_rccl_reduce_op[] = { [UCC_OP_SUM] = (ncclRedOp_t)ncclSum, [UCC_OP_PROD] = (ncclRedOp_t)ncclProd, [UCC_OP_MAX] = (ncclRedOp_t)ncclMax, [UCC_OP_MIN] = (ncclRedOp_t)ncclMin, #if NCCL_VERSION_CODE < NCCL_VERSION(2,10,3) [UCC_OP_AVG] = (ncclRedOp_t)ncclOpUnsupported, #else [UCC_OP_AVG] = (ncclRedOp_t)ncclAvg, #endif [UCC_OP_LAND] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_LOR] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_LXOR] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_BAND] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_BOR] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_BXOR] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_MAXLOC] = (ncclRedOp_t)ncclOpUnsupported, [UCC_OP_MINLOC] = (ncclRedOp_t)ncclOpUnsupported, }; const char *ucc_tl_rccl_default_alg_select_str[UCC_TL_RCCL_N_DEFAULT_ALG_SELECT_STR] = { UCC_TL_RCCL_ALLGATHERV_DEFAULT_ALG_SELECT_STR}; static inline ucc_status_t ucc_rccl_check_dt_supported(ucc_datatype_t dt1, ucc_datatype_t dt2) { if (ucc_unlikely((dt1 != dt2) || !UCC_DT_IS_PREDEFINED(dt1) || (ucc_to_rccl_dtype[UCC_DT_PREDEFINED_ID(dt1)] == ncclDataTypeUnsupported))) { return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } ucc_tl_rccl_task_t * ucc_tl_rccl_init_task(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team) { ucc_tl_rccl_context_t *rccl_ctx = ucc_derived_of(team->context, ucc_tl_rccl_context_t); ucc_tl_rccl_task_t *task; ucc_status_t status; ucc_coll_progress_fn_t progress_fn; task = ucc_mpool_get(&rccl_ctx->req_mp); if (ucc_unlikely(!task)) { tl_error(team->context->lib,"Failed to allocate task"); return NULL; } progress_fn = task->super.progress; ucc_coll_task_init(&task->super, coll_args, team); UCC_TL_RCCL_PROFILE_REQUEST_NEW(task, "tl_rccl_task", 0); task->super.finalize = ucc_tl_rccl_coll_finalize; task->super.triggered_post = ucc_tl_rccl_triggered_post; task->super.progress = progress_fn; task->completed = NULL; if (rccl_ctx->cfg.sync_type == UCC_TL_RCCL_COMPLETION_SYNC_TYPE_EVENT) { status = ucc_ec_create_event(&task->completed, UCC_EE_ROCM_STREAM); if (ucc_unlikely(status != UCC_OK)) { ucc_mpool_put(task); return NULL; } } return task; } void ucc_tl_rccl_free_task(ucc_tl_rccl_task_t *task) { UCC_TL_RCCL_PROFILE_REQUEST_FREE(task); if (task->completed) { ucc_ec_destroy_event(task->completed, UCC_EE_ROCM_STREAM); } ucc_mpool_put(task); } ucc_status_t ucc_tl_rccl_triggered_post(ucc_ee_h ee, ucc_ev_t *ev, //NOLINT: ev is unused ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_status_t status; ucc_ev_t *post_event; ucc_assert(ee->ee_type == UCC_EE_ROCM_STREAM); coll_task->ee = ee; tl_debug(UCC_TASK_LIB(task), "triggered post. task:%p", coll_task); status = coll_task->post(coll_task); if (ucc_likely(status == UCC_OK)) { /* TODO: mpool */ post_event = ucc_malloc(sizeof(ucc_ev_t), "event"); if (ucc_unlikely(post_event == NULL)) { tl_error(UCC_TASK_LIB(task), "failed to allocate memory for event"); return UCC_ERR_NO_MEMORY; } post_event->ev_type = UCC_EVENT_COLLECTIVE_POST; post_event->ev_context_size = 0; post_event->req = &coll_task->super; ucc_ee_set_event_internal(coll_task->ee, post_event, &coll_task->ee->event_out_queue); } return status; } ucc_status_t ucc_tl_rccl_coll_finalize(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_status_t status = UCC_OK ; tl_debug(UCC_TASK_LIB(task), "finalizing coll task %p", task); ucc_tl_rccl_free_task(task); return status; } ucc_status_t ucc_tl_rccl_collective_sync(ucc_tl_rccl_task_t *task, hipStream_t stream) { ucc_tl_rccl_context_t *ctx = TASK_CTX(task); ucc_status_t status = UCC_OK; task->host_status = task->super.super.status; if (ctx->cfg.sync_type != UCC_TL_RCCL_COMPLETION_SYNC_TYPE_EVENT) { tl_error(UCC_TASK_LIB(task), "RCCL only supports stream synchronization events"); return UCC_ERR_NOT_SUPPORTED; } status = ucc_ec_event_post(stream, task->completed, UCC_EE_ROCM_STREAM); if (ucc_unlikely(status != UCC_OK)) { return status; } return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(TASK_TEAM(task))->pq, &task->super); } ucc_status_t ucc_tl_rccl_alltoall_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; ucc_rank_t gsize = UCC_TL_TEAM_SIZE(team); ucc_status_t status = UCC_OK; ptrdiff_t sbuf = (ptrdiff_t)args->src.info.buffer; ptrdiff_t rbuf = (ptrdiff_t)args->dst.info.buffer; size_t data_size; ucc_rank_t peer; ncclDataType_t dt; task->super.super.status = UCC_INPROGRESS; data_size = (size_t)(args->src.info.count / gsize) * ucc_dt_size(args->src.info.datatype); ucc_assert(args->src.info.count % gsize == 0); if (data_size == 0) { task->super.super.status = UCC_OK; return UCC_OK; } UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_alltoall_start", 0); if (args->src.info.datatype == args->dst.info.datatype) { dt = ucc_to_rccl_dtype[UCC_DT_PREDEFINED_ID(args->dst.info.datatype)]; RCCLCHECK_GOTO(ncclAllToAll((void *)sbuf, (void *)rbuf, (size_t)(args->src.info.count / gsize), dt, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } else { RCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team)); for (peer = 0; peer < gsize; peer++) { RCCLCHECK_GOTO(ncclSend((void *)(sbuf + peer * data_size), data_size, ncclChar, peer, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); RCCLCHECK_GOTO(ncclRecv((void *)(rbuf + peer * data_size), data_size, ncclChar, peer, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } RCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team)); } status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_alltoall_init(ucc_tl_rccl_task_t *task) { ucc_datatype_t dt1 = TASK_ARGS(task).src.info.datatype; ucc_datatype_t dt2 = TASK_ARGS(task).dst.info.datatype; if (UCC_IS_INPLACE(TASK_ARGS(task))) { tl_debug(UCC_TASK_LIB(task), "inplace alltoall is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (UCC_OK != ucc_rccl_check_dt_supported(dt1, dt2)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_rccl_alltoall_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_alltoallv_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; ucc_status_t status = UCC_OK; ptrdiff_t sbuf = (ptrdiff_t)args->src.info_v.buffer; ptrdiff_t rbuf = (ptrdiff_t)args->dst.info_v.buffer; size_t sdt_size, rdt_size, count, displ; ucc_rank_t peer; task->super.super.status = UCC_INPROGRESS; sdt_size = ucc_dt_size(args->src.info_v.datatype); rdt_size = ucc_dt_size(args->dst.info_v.datatype); UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_alltoallv_start", 0); RCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team)); for (peer = 0; peer < UCC_TL_TEAM_SIZE(team); peer++) { count = ucc_coll_args_get_count(args, args->src.info_v.counts, peer); if (count != 0) { displ = ucc_coll_args_get_displacement( args, args->src.info_v.displacements, peer); RCCLCHECK_GOTO(ncclSend((void *)(sbuf + displ * sdt_size), count * sdt_size, ncclChar, peer, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } count = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); if (count != 0) { displ = ucc_coll_args_get_displacement( args, args->dst.info_v.displacements, peer); RCCLCHECK_GOTO(ncclRecv((void *)(rbuf + displ * rdt_size), count * rdt_size, ncclChar, peer, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } } RCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team)); status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_alltoallv_init(ucc_tl_rccl_task_t *task) { ucc_datatype_t dt1 = TASK_ARGS(task).src.info_v.datatype; ucc_datatype_t dt2 = TASK_ARGS(task).dst.info_v.datatype; if (UCC_IS_INPLACE(TASK_ARGS(task))) { tl_debug(UCC_TASK_LIB(task), "inplace alltoallv is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (UCC_OK != ucc_rccl_check_dt_supported(dt1, dt2)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_rccl_alltoallv_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_allreduce_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = UCC_IS_INPLACE(*args) ? args->dst.info.buffer : args->src.info.buffer; ucc_status_t status = UCC_OK; ncclRedOp_t op = ucc_to_rccl_reduce_op[args->op]; size_t count = args->dst.info.count; ncclDataType_t dt; dt = ucc_to_rccl_dtype[UCC_DT_PREDEFINED_ID(args->dst.info.datatype)]; task->super.super.status = UCC_INPROGRESS; UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, args->coll_type == UCC_COLL_TYPE_BARRIER ? "rccl_barrier_start" : "rccl_allreduce_start", 0); RCCLCHECK_GOTO(ncclAllReduce(src, dst, count, dt, op, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_allreduce_init(ucc_tl_rccl_task_t *task) { if (UCC_OK != ucc_rccl_check_dt_supported(TASK_ARGS(task).dst.info.datatype, TASK_ARGS(task).dst.info.datatype)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (ucc_to_rccl_reduce_op[TASK_ARGS(task).op] == ncclOpUnsupported) { tl_debug(UCC_TASK_LIB(task), "reduction operation is not supported"); return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_rccl_allreduce_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_allgather_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; size_t count = args->dst.info.count; ncclDataType_t dt; dt = ucc_to_rccl_dtype[UCC_DT_PREDEFINED_ID(args->dst.info.datatype)]; if (UCC_IS_INPLACE(*args)) { src = (void *)((ptrdiff_t)args->dst.info.buffer + (count / size) * ucc_dt_size(args->dst.info.datatype) * rank); } task->super.super.status = UCC_INPROGRESS; UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_allgather_start", 0); RCCLCHECK_GOTO(ncclAllGather(src, dst, count / size, dt, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_allgather_init(ucc_tl_rccl_task_t *task) { ucc_datatype_t dt1 = UCC_IS_INPLACE(TASK_ARGS(task)) ? TASK_ARGS(task).dst.info.datatype : TASK_ARGS(task).src.info.datatype; ucc_datatype_t dt2 = TASK_ARGS(task).dst.info.datatype; if (UCC_OK != ucc_rccl_check_dt_supported(dt1, dt2)) { tl_debug(UCC_TASK_LIB(task), "dataype is not supported"); return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_rccl_allgather_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_allgatherv_init(ucc_tl_rccl_task_t *task) { ucc_datatype_t dt1 = TASK_ARGS(task).src.info_v.datatype; ucc_datatype_t dt2 = TASK_ARGS(task).dst.info_v.datatype; if (UCC_IS_INPLACE(TASK_ARGS(task))) { tl_debug(UCC_TASK_LIB(task), "inplace allgatherv is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (UCC_OK != ucc_rccl_check_dt_supported(dt1, dt2)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_rccl_allgatherv_p2p_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_bcast_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; size_t count = args->src.info.count; ucc_rank_t root = args->root; ucc_rank_t peer, rank, size; ncclDataType_t dt; ucc_ep_map_t map; dt = ucc_to_rccl_dtype[UCC_DT_PREDEFINED_ID(args->src.info.datatype)]; task->super.super.status = UCC_INPROGRESS; UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_bcast_start", 0); if (UCC_COLL_ARGS_ACTIVE_SET(args)) { map = ucc_active_set_to_ep_map(args); rank = UCC_TL_TEAM_RANK(team); size = (ucc_rank_t)args->active_set.size; if (root == rank) { RCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team)); for (peer = 0; peer < size; peer++) { if (ucc_ep_map_eval(map, peer) == rank) { continue; } RCCLCHECK_GOTO(ncclSend(src, count, dt, ucc_ep_map_eval(map, peer), team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } RCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team)); } else { RCCLCHECK_GOTO(ncclRecv(src, count, dt, root, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } } else { RCCLCHECK_GOTO(ncclBroadcast(src, src, count, dt, root, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_bcast_init(ucc_tl_rccl_task_t *task) { if (UCC_OK != ucc_rccl_check_dt_supported(TASK_ARGS(task).src.info.datatype, TASK_ARGS(task).src.info.datatype)) { /* TODO: can we use rcclChar if datatype is not supported? */ tl_debug(UCC_TASK_LIB(task), "dataype is not supported"); return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_rccl_bcast_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_reduce_scatter_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; ncclRedOp_t op = ucc_to_rccl_reduce_op[args->op]; size_t count = args->dst.info.count; ncclDataType_t dt; dt = ucc_to_rccl_dtype[UCC_DT_PREDEFINED_ID(args->dst.info.datatype)]; task->super.super.status = UCC_INPROGRESS; UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_reduce_scatter_start", 0); if (UCC_IS_INPLACE(*args)) { count /= UCC_TL_TEAM_SIZE(team); src = args->dst.info.buffer; dst = PTR_OFFSET(src, UCC_TL_TEAM_RANK(team) * count * ucc_dt_size(args->dst.info.datatype)); } RCCLCHECK_GOTO(ncclReduceScatter(src, dst, count, dt, op, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_reduce_scatter_init(ucc_tl_rccl_task_t *task) { if (UCC_OK != ucc_rccl_check_dt_supported(TASK_ARGS(task).dst.info.datatype, TASK_ARGS(task).dst.info.datatype)) { tl_debug(UCC_TASK_LIB(task), "dataype is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (ucc_to_rccl_reduce_op[TASK_ARGS(task).op] == ncclOpUnsupported) { tl_debug(UCC_TASK_LIB(task), "reduction operation is not supported"); return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_rccl_reduce_scatter_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_reduce_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info.buffer; ucc_datatype_t ucc_dt = args->src.info.datatype; size_t count = args->src.info.count; ncclRedOp_t op = ucc_to_rccl_reduce_op[args->op]; ucc_status_t status = UCC_OK; ncclDataType_t rccl_dt; UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_reduce_start", 0); if (args->root == UCC_TL_TEAM_RANK(team)) { ucc_dt = TASK_ARGS(task).dst.info.datatype; count = TASK_ARGS(task).dst.info.count; if (UCC_IS_INPLACE(*args)) { src = args->dst.info.buffer; } } rccl_dt = ucc_to_rccl_dtype[UCC_DT_PREDEFINED_ID(ucc_dt)]; task->super.super.status = UCC_INPROGRESS; RCCLCHECK_GOTO(ncclReduce(src, dst, count, rccl_dt, op, args->root, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_reduce_init(ucc_tl_rccl_task_t *task) { ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_datatype_t dt; dt = (TASK_ARGS(task).root == UCC_TL_TEAM_RANK(team)) ? TASK_ARGS(task).dst.info.datatype : TASK_ARGS(task).src.info.datatype; if (UCC_OK != ucc_rccl_check_dt_supported(dt, dt)) { tl_debug(UCC_TASK_LIB(task), "dataype is not supported"); return UCC_ERR_NOT_SUPPORTED; } if (ucc_to_rccl_reduce_op[TASK_ARGS(task).op] == ncclOpUnsupported) { tl_debug(UCC_TASK_LIB(task), "reduction operation is not supported"); return UCC_ERR_NOT_SUPPORTED; } task->super.post = ucc_tl_rccl_reduce_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_barrier_init(ucc_tl_rccl_task_t *task) { /* use 4-byte allreduce to accomplish barrier */ ucc_coll_args_t *args = &TASK_ARGS(task); args->mask |= UCC_COLL_ARGS_FIELD_FLAGS; args->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; args->op = UCC_OP_SUM; args->dst.info.buffer = TASK_CTX(task)->scratch_buf; args->src.info.buffer = args->dst.info.buffer; args->dst.info.datatype = args->src.info.datatype = UCC_DT_FLOAT32; args->dst.info.count = args->src.info.count = 1; task->super.post = ucc_tl_rccl_allreduce_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_gather_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; ucc_datatype_t ucc_dt = args->src.info.datatype; void *dst = args->dst.info.buffer; void *src = args->src.info.buffer; size_t count = args->src.info.count; ucc_status_t status = UCC_OK; ncclDataType_t rccl_dt; UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_gather_start", 0); if (args->root == UCC_TL_TEAM_RANK(team)) { ucc_dt = args->dst.info.datatype; count = args->dst.info.count / UCC_TL_TEAM_SIZE(team); if (UCC_IS_INPLACE(*args)) { src = PTR_OFFSET(dst, UCC_TL_TEAM_RANK(team) * count * ucc_dt_size(args->dst.info.datatype)); } } rccl_dt = ucc_to_rccl_dtype[UCC_DT_PREDEFINED_ID(ucc_dt)]; task->super.super.status = UCC_INPROGRESS; RCCLCHECK_GOTO(ncclGather(src, dst, count, rccl_dt, args->root, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_gather_init(ucc_tl_rccl_task_t *task) { ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_datatype_t dt; dt = (UCC_TL_TEAM_RANK(team) == args->root) ? args->dst.info.datatype : args->src.info.datatype; if (UCC_TL_TEAM_RANK(team) == args->root) { if (ucc_rccl_check_dt_supported (dt, dt)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } } if ((UCC_TL_TEAM_RANK(team) != args->root) || (!UCC_IS_INPLACE(*args))) { if (ucc_rccl_check_dt_supported (dt, dt)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } } task->super.post = ucc_tl_rccl_gather_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_gatherv_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; void *dst = args->dst.info_v.buffer; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; size_t count, displ, dt_size; ucc_rank_t peer; if (rank == args->root) { dt_size = ucc_dt_size(args->dst.info_v.datatype); } else { dt_size = ucc_dt_size(args->src.info.datatype); } UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_gatherv_start", 0); if (rank == args->root) { if (!UCC_IS_INPLACE(*args)) { count = ucc_coll_args_get_count(args, args->dst.info_v.counts, rank); displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, rank); HIPCHECK_GOTO(hipMemcpyAsync(PTR_OFFSET(dst, displ * dt_size), src, count * dt_size, hipMemcpyDeviceToDevice, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } RCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team)); for (peer = 0; peer < size; peer++) { if (peer == args->root) { continue; } count = ucc_coll_args_get_count(args, args->dst.info_v.counts, peer); displ = ucc_coll_args_get_displacement(args, args->dst.info_v.displacements, peer); RCCLCHECK_GOTO(ncclRecv(PTR_OFFSET(dst, displ * dt_size), count * dt_size, ncclChar, peer, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } RCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team)); } else { RCCLCHECK_GOTO(ncclSend(src, args->src.info.count * dt_size, ncclChar, args->root, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } task->super.status = UCC_INPROGRESS; status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_gatherv_init(ucc_tl_rccl_task_t *task) { ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_datatype_t dt; dt = (UCC_TL_TEAM_RANK(team) == args->root) ? args->dst.info_v.datatype : args->src.info.datatype; if (UCC_TL_TEAM_RANK(team) == args->root) { if (ucc_rccl_check_dt_supported (dt, dt)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } } if ((UCC_TL_TEAM_RANK(team) != args->root) || (!UCC_IS_INPLACE(*args))) { if (ucc_rccl_check_dt_supported (dt, dt)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } } task->super.post = ucc_tl_rccl_gatherv_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_scatter_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info.buffer; ucc_status_t status = UCC_OK; ucc_datatype_t ucc_dt = args->dst.info.datatype; size_t count = args->dst.info.count; ncclDataType_t rccl_dt; UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_scatter_start", 0); if (args->root == UCC_TL_TEAM_RANK(team)) { ucc_dt = args->src.info.datatype; count = args->src.info.count / UCC_TL_TEAM_SIZE(team); if (UCC_IS_INPLACE(*args)) { dst = PTR_OFFSET(src, UCC_TL_TEAM_RANK(team) * count * ucc_dt_size(ucc_dt)); } } rccl_dt = ucc_to_rccl_dtype[UCC_DT_PREDEFINED_ID(ucc_dt)]; task->super.status = UCC_INPROGRESS; RCCLCHECK_GOTO(ncclScatter(src, dst, count, rccl_dt, args->root, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_scatter_init(ucc_tl_rccl_task_t *task) { ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_datatype_t dt; dt = (UCC_TL_TEAM_RANK(team) == args->root) ? args->src.info.datatype : args->dst.info.datatype; if (UCC_TL_TEAM_RANK(team) == args->root) { if (ucc_rccl_check_dt_supported(dt, dt)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } } if ((UCC_TL_TEAM_RANK(team) != args->root) || (!UCC_IS_INPLACE(*args))) { if (ucc_rccl_check_dt_supported (dt, dt)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } } task->super.post = ucc_tl_rccl_scatter_start; return UCC_OK; } ucc_status_t ucc_tl_rccl_scatterv_start(ucc_coll_task_t *coll_task) { ucc_tl_rccl_task_t *task = ucc_derived_of(coll_task, ucc_tl_rccl_task_t); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_rank_t rank = UCC_TL_TEAM_RANK(team); ucc_rank_t size = UCC_TL_TEAM_SIZE(team); ucc_ee_h ee = coll_task->ee; hipStream_t stream = (ee) ? (hipStream_t) ee->ee_context : team->stream; void *dst = args->dst.info.buffer; void *src = args->src.info_v.buffer; ucc_status_t status = UCC_OK; size_t count, displ, dt_size; ucc_rank_t peer; if (rank == args->root) { dt_size = ucc_dt_size(args->src.info_v.datatype); } else { dt_size = ucc_dt_size(args->dst.info.datatype); } UCC_TL_RCCL_PROFILE_REQUEST_EVENT(coll_task, "rccl_scatterv_start", 0); if (rank == args->root) { if (!UCC_IS_INPLACE(*args)) { count = ucc_coll_args_get_count(args, args->src.info_v.counts, rank); displ = ucc_coll_args_get_displacement(args, args->src.info_v.displacements, rank); HIPCHECK_GOTO(hipMemcpyAsync(dst, PTR_OFFSET(src, displ * dt_size), count * dt_size, hipMemcpyDeviceToDevice, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } RCCLCHECK_GOTO(ncclGroupStart(), exit_coll, status, UCC_TL_TEAM_LIB(team)); for (peer = 0; peer < size; peer++) { if (peer == args->root) { continue; } count = ucc_coll_args_get_count(args, args->src.info_v.counts, peer); displ = ucc_coll_args_get_displacement(args, args->src.info_v.displacements, peer); RCCLCHECK_GOTO(ncclSend(PTR_OFFSET(src, displ * dt_size), count * dt_size, ncclChar, peer, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } RCCLCHECK_GOTO(ncclGroupEnd(), exit_coll, status, UCC_TL_TEAM_LIB(team)); } else { RCCLCHECK_GOTO(ncclRecv(dst, args->dst.info.count * dt_size, ncclChar, args->root, team->rccl_comm, stream), exit_coll, status, UCC_TL_TEAM_LIB(team)); } task->super.status = UCC_INPROGRESS; status = ucc_tl_rccl_collective_sync(task, stream); exit_coll: return status; } ucc_status_t ucc_tl_rccl_scatterv_init(ucc_tl_rccl_task_t *task) { ucc_tl_rccl_team_t *team = TASK_TEAM(task); ucc_coll_args_t *args = &TASK_ARGS(task); ucc_datatype_t dt; dt = (UCC_TL_TEAM_RANK(team) == args->root) ? args->src.info_v.datatype : args->dst.info.datatype; if (UCC_TL_TEAM_RANK(team) == args->root) { if (ucc_rccl_check_dt_supported(dt, dt)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } } if ((UCC_TL_TEAM_RANK(team) != args->root) || (!UCC_IS_INPLACE(*args))) { if (ucc_rccl_check_dt_supported(dt, dt)) { tl_debug(UCC_TASK_LIB(task), "datatype is not supported"); return UCC_ERR_NOT_SUPPORTED; } } task->super.post = ucc_tl_rccl_scatterv_start; return UCC_OK; } static inline int alg_id_from_str(ucc_coll_type_t coll_type, const char *str) { switch (coll_type) { case UCC_COLL_TYPE_ALLGATHERV: return ucc_tl_rccl_allgatherv_alg_from_str(str); default: break; } return -1; } ucc_status_t ucc_tl_rccl_alg_id_to_init(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, //NOLINT ucc_base_coll_init_fn_t *init) { ucc_status_t status = UCC_OK; if (alg_id_str) { alg_id = alg_id_from_str(coll_type, alg_id_str); } switch (coll_type) { case UCC_COLL_TYPE_ALLGATHERV: switch (alg_id) { case UCC_TL_RCCL_ALLGATHERV_ALG_P2P: *init = ucc_tl_rccl_allgatherv_p2p_init; break; case UCC_TL_RCCL_ALLGATHERV_ALG_BCOPY: *init = ucc_tl_rccl_allgatherv_bcopy_init; break; case UCC_TL_RCCL_ALLGATHERV_ALG_BCAST: *init = ucc_tl_rccl_allgatherv_bcast_init; break; default: status = UCC_ERR_INVALID_PARAM; break; }; break; default: status = UCC_ERR_NOT_SUPPORTED; break; } return status; } ucc-1.8.0/src/components/tl/ucc_tl.c0000664000175000017500000002250415211535620017471 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_tl.h" #include "utils/ucc_log.h" #include "core/ucc_team.h" #include "ucc_tl_log.h" ucc_config_field_t ucc_tl_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_base_lib_config_table)}, {NULL} }; ucc_config_field_t ucc_tl_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_base_ctx_config_table)}, {NULL} }; UCC_CLASS_INIT_FUNC(ucc_tl_lib_t, ucc_tl_iface_t *tl_iface, const ucc_tl_lib_config_t *tl_config) { UCC_CLASS_CALL_BASE_INIT(); ucc_base_lib_properties_t prop; ucc_status_t status; status = tl_iface->lib.get_properties(&prop); if (status != UCC_OK) { return status; } self->iface = tl_iface; self->super.use_tuning = tl_config->super.use_tuning; self->super.log_component = tl_config->super.log_component; self->super.min_team_size = prop.default_team_size; ucc_strncpy_safe(self->super.log_component.name, tl_iface->tl_lib_config.name, sizeof(self->super.log_component.name)); if (tl_config->super.min_team_size != UCC_ULUNITS_AUTO) { if (tl_config->super.min_team_size < prop.min_team_size) { tl_warn(self, "min supported team size is %d, requested %d", prop.min_team_size, (ucc_rank_t)tl_config->super.min_team_size); } else if (tl_config->super.min_team_size > prop.max_team_size) { tl_warn(self, "max supported team size is %d, requested %d", prop.max_team_size, (ucc_rank_t)tl_config->super.min_team_size); } else { self->super.min_team_size = tl_config->super.min_team_size; } } return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_lib_t) { } UCC_CLASS_DEFINE(ucc_tl_lib_t, void); UCC_CLASS_INIT_FUNC(ucc_tl_context_t, const ucc_tl_context_config_t *tl_config, ucc_context_t *ucc_context) { UCC_CLASS_CALL_BASE_INIT(); self->super.lib = &tl_config->tl_lib->super; self->super.ucc_context = ucc_context; self->ref_count = 0; if (0 == strcmp(tl_config->super.score_str, "0")) { return UCC_ERR_LAST; } self->super.score_str = strdup(tl_config->super.score_str); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_context_t) { ucc_free(self->super.score_str); } UCC_CLASS_DEFINE(ucc_tl_context_t, void); ucc_status_t ucc_tl_context_config_read(ucc_tl_lib_t *tl_lib, const ucc_context_config_t *config, ucc_tl_context_config_t **tl_config) { ucc_status_t status; status = ucc_base_config_read(config->lib->full_prefix, &tl_lib->iface->tl_context_config, (ucc_base_config_t **)tl_config); if (UCC_OK == status) { (*tl_config)->tl_lib = tl_lib; } return status; } ucc_status_t ucc_tl_lib_config_read(ucc_tl_iface_t *iface, const char *full_prefix, ucc_tl_lib_config_t **tl_config) { return ucc_base_config_read(full_prefix, &iface->tl_lib_config, (ucc_base_config_t **)tl_config); } ucc_status_t ucc_tl_context_get(ucc_context_t *ctx, const char* name, ucc_tl_context_t **tl_context) { int i; ucc_tl_lib_t *tl_lib; for (i = 0; i < ctx->n_tl_ctx; i++) { tl_lib = ucc_derived_of(ctx->tl_ctx[i]->super.lib, ucc_tl_lib_t); if (0 == strcmp(tl_lib->iface->super.name, name)) { ctx->tl_ctx[i]->ref_count++; *tl_context = ctx->tl_ctx[i]; return UCC_OK; } } return UCC_ERR_NOT_FOUND; } ucc_status_t ucc_tl_context_put(ucc_tl_context_t *tl_context) { tl_context->ref_count--; return UCC_OK; } ucc_status_t ucc_team_multiple_req_alloc(ucc_team_multiple_req_t **req, int n_teams) { ucc_team_multiple_req_t *r; r = ucc_malloc(sizeof(ucc_team_multiple_req_t) + (n_teams - 1) * sizeof(struct ucc_team_team_desc), "team create multiple request"); if (!r) { goto exit_err; } r->last = -1; r->n_teams = n_teams; *req = r; return UCC_OK; exit_err: *req = NULL; return UCC_ERR_NO_MEMORY; } void ucc_team_multiple_req_free(ucc_team_multiple_req_t *req) { ucc_free(req); } static ucc_status_t ucc_tl_is_reachable(const ucc_base_team_params_t *params, unsigned long tl_id) { ucc_team_t *core_team = params->team; ucc_context_t *core_context = core_team->contexts[0]; ucc_addr_storage_t *addr_storage; ucc_context_addr_header_t *addr_header; ucc_rank_t i, rank; int j, use_ctx; ucc_assert(core_team->num_contexts == 1); if (params->size == 1) { return UCC_OK; } if (core_context->addr_storage.storage) { addr_storage = &core_context->addr_storage; use_ctx = 1; } else { addr_storage = &core_team->addr_storage; use_ctx = 0; } if (addr_storage->flags & UCC_ADDR_STORAGE_FLAG_TLS_SYMMETRIC) { return UCC_OK; } for (i = 0; i < params->size; i++) { rank = ucc_ep_map_eval(params->map, i); if (use_ctx) { rank = ucc_ep_map_eval(core_team->ctx_map, rank); } addr_header = UCC_ADDR_STORAGE_RANK_HEADER(addr_storage, rank); for (j = 0; j < addr_header->n_components; j++) { if (addr_header->components[j].id == tl_id) { break; } } if (j == addr_header->n_components) { return UCC_ERR_NOT_FOUND; } } return UCC_OK; } ucc_status_t ucc_tl_team_create_multiple(ucc_team_multiple_req_t *req) { int *id = &req->last; ucc_base_team_t *b_team; ucc_status_t status; ucc_tl_lib_t *lib; if (*id == req->n_teams) { return UCC_OK; } if ((*id == -1) || (req->descs[*id].status != UCC_INPROGRESS)) { /* post next team */ *id += 1; if (*id == req->n_teams) { return UCC_OK; } lib = ucc_derived_of(req->descs[*id].ctx->super.lib, ucc_tl_lib_t); status = ucc_tl_is_reachable(&req->descs[*id].param, lib->iface->super.id); if (UCC_OK != status) { ucc_debug("TL %s is not reachable, skipping\n", lib->iface->super.name); } else { status = UCC_TL_CTX_IFACE(req->descs[*id].ctx) ->team.create_post(&((req->descs[*id].ctx->super)), &req->descs[*id].param, &b_team); } if (UCC_OK != status) { req->descs[*id].status = status; req->descs[*id].team = NULL; } else { req->descs[*id].status = UCC_INPROGRESS; req->descs[*id].team = ucc_derived_of(b_team, ucc_tl_team_t); } return UCC_INPROGRESS; } req->descs[*id].status = UCC_TL_CTX_IFACE(req->descs[*id].ctx) ->team.create_test(&req->descs[*id].team->super); if (req->descs[*id].status < 0) { /* if team create failed in team create test need to cleanup resources */ UCC_TL_CTX_IFACE(req->descs[*id].ctx)->team.destroy( &req->descs[*id].team->super); } return UCC_INPROGRESS; } ucc_status_t ucc_tl_team_destroy_multiple(ucc_team_multiple_req_t *req) { int *id = &req->last; ucc_status_t status; if (*id < 0) (*id)++; while (*id != req->n_teams) { status = UCC_TL_TEAM_IFACE(req->descs[*id].team) ->team.destroy(&req->descs[*id].team->super); req->descs[*id].status = status; if (UCC_INPROGRESS != status) { (*id)++; } else { return UCC_INPROGRESS; } } return UCC_OK; } UCC_CLASS_INIT_FUNC(ucc_tl_team_t, ucc_tl_context_t *tl_context, const ucc_base_team_params_t *params) { UCC_CLASS_CALL_BASE_INIT(); ucc_base_lib_t *lib = tl_context->super.lib; ucc_base_lib_attr_t attr; ucc_tl_iface_t *tl_iface; ucc_status_t status; self->super.context = &tl_context->super; self->super.params = *params; tl_iface = UCC_TL_CTX_IFACE(tl_context); attr.mask = UCC_BASE_LIB_ATTR_FIELD_MIN_TEAM_SIZE | UCC_BASE_LIB_ATTR_FIELD_MAX_TEAM_SIZE; status = tl_iface->lib.get_attr(lib, &attr); if (status != UCC_OK) { return status; } if (attr.min_team_size > params->size) { tl_debug(lib, "team size %d is too small, min supported %d", params->size, attr.min_team_size); return UCC_ERR_NOT_SUPPORTED; } if (attr.max_team_size < params->size) { tl_debug(lib, "team size %d is too big, max supported %d", params->size, attr.max_team_size); return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_team_t) { } UCC_CLASS_DEFINE(ucc_tl_team_t, void); ucc-1.8.0/src/components/tl/mlx5/0000775000175000017500000000000015211535620016736 5ustar alastairalastairucc-1.8.0/src/components/tl/mlx5/tl_mlx5_ib.h0000664000175000017500000000603715211535620021153 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_MLX5_IB_H_ #define UCC_TL_MLX5_IB_H_ #include "tl_mlx5.h" typedef struct mlx5dv_mr_interleaved umr_t; typedef struct ucc_tl_mlx5_qp { struct ibv_qp * qp; struct ibv_qp_ex *qp_ex; } ucc_tl_mlx5_qp_t; typedef struct ucc_tl_mlx5_dci { struct ibv_qp * dci_qp; struct ibv_qp_ex * dc_qpex; struct mlx5dv_qp_ex *dc_mqpex; } ucc_tl_mlx5_dci_t; int ucc_tl_mlx5_get_active_port(struct ibv_context *ctx); int ucc_tl_mlx5_check_port_active(struct ibv_context *ctx, int port_num); ucc_status_t ucc_tl_mlx5_create_ibv_ctx(char ** ib_devname, struct ibv_context **ctx, ucc_base_lib_t * lib); ucc_status_t ucc_tl_mlx5_qp_connect(struct ibv_qp *qp, uint32_t qp_num, uint16_t lid, int port, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib); ucc_status_t ucc_tl_mlx5_init_dct(struct ibv_pd *pd, struct ibv_context *ctx, struct ibv_cq *cq, struct ibv_srq *srq, uint8_t port_num, struct ibv_qp **dct_qp, uint32_t *qpn, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib); ucc_status_t ucc_tl_mlx5_init_dci(ucc_tl_mlx5_dci_t *dci, struct ibv_pd *pd, struct ibv_context *ctx, struct ibv_cq *cq, uint8_t port_num, int tx_depth, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib); ucc_status_t ucc_tl_mlx5_create_rc_qp(struct ibv_context *ctx, struct ibv_pd *pd, struct ibv_cq *cq, int tx_depth, ucc_tl_mlx5_qp_t *qp, uint32_t *qpn, ucc_base_lib_t *lib); ucc_status_t ucc_tl_mlx5_create_ah(struct ibv_pd *pd, uint16_t lid, uint8_t port_num, struct ibv_ah **ah_ptr, ucc_base_lib_t *lib); ucc_status_t ucc_tl_mlx5_create_umr_qp(struct ibv_context *ctx, struct ibv_pd *pd, struct ibv_cq *cq, int ib_port, struct ibv_qp **qp, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib); static inline ucc_status_t tl_mlx5_ah_to_av(struct ibv_ah *ah, struct mlx5_wqe_av *av) { struct mlx5dv_obj obj; struct mlx5dv_ah dv_ah; obj.ah.in = ah; obj.ah.out = &dv_ah; if (ucc_unlikely(mlx5dv_init_obj(&obj, MLX5DV_OBJ_AH))) { return UCC_ERR_NO_MESSAGE; } *av = *(dv_ah.av); return UCC_OK; } #endif ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_lib.c0000664000175000017500000000356615211535620021326 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5.h" #include "utils/ucc_malloc.h" /* NOLINTNEXTLINE params is not used*/ UCC_CLASS_INIT_FUNC(ucc_tl_mlx5_lib_t, const ucc_base_lib_params_t *params, const ucc_base_config_t *config) { const ucc_tl_mlx5_lib_config_t *tl_mlx5_config = ucc_derived_of(config, ucc_tl_mlx5_lib_config_t); UCC_CLASS_CALL_SUPER_INIT(ucc_tl_lib_t, &ucc_tl_mlx5.super, &tl_mlx5_config->super); memcpy(&self->cfg, tl_mlx5_config, sizeof(*tl_mlx5_config)); tl_debug(&self->super, "initialized lib object: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_mlx5_lib_t) { tl_debug(&self->super, "finalizing lib object: %p", self); } UCC_CLASS_DEFINE(ucc_tl_mlx5_lib_t, ucc_tl_lib_t); ucc_status_t ucc_tl_mlx5_get_lib_attr(const ucc_base_lib_t *lib, /* NOLINT */ ucc_base_lib_attr_t * base_attr) { ucc_tl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_tl_lib_attr_t); attr->super.attr.thread_mode = UCC_THREAD_MULTIPLE; //TODO check impacts attr->super.attr.coll_types = UCC_TL_MLX5_SUPPORTED_COLLS; attr->super.flags = UCC_BASE_LIB_FLAG_SERVICE_TEAM_REQUIRED | UCC_BASE_LIB_FLAG_CTX_SERVICE_TEAM_REQUIRED; if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MIN_TEAM_SIZE) { attr->super.min_team_size = lib->min_team_size; } if (base_attr->mask & UCC_BASE_LIB_ATTR_FIELD_MAX_TEAM_SIZE) { attr->super.max_team_size = UCC_RANK_MAX; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_get_lib_properties(ucc_base_lib_properties_t *prop) { prop->default_team_size = 2; prop->min_team_size = 2; prop->max_team_size = UCC_RANK_MAX; return UCC_OK; } ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_team.c0000664000175000017500000003266115211535620021504 0ustar alastairalastair/** * Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_coll.h" #include "tl_mlx5.h" #include "tl_mlx5_coll.h" #include "coll_score/ucc_coll_score.h" #include "alltoall/alltoall.h" #include "core/ucc_team.h" #include #include "mcast/tl_mlx5_mcast.h" #include "mcast/tl_mlx5_mcast_helper.h" static ucc_status_t ucc_tl_mlx5_topo_init(ucc_tl_mlx5_team_t *team) { ucc_subset_t subset; ucc_status_t status; status = ucc_ep_map_create_nested(&UCC_TL_CORE_TEAM(team)->ctx_map, &UCC_TL_TEAM_MAP(team), &team->ctx_map); if (UCC_OK != status) { tl_debug(UCC_TL_TEAM_LIB(team), "failed to create ctx map"); return status; } subset.map = team->ctx_map; subset.myrank = UCC_TL_TEAM_RANK(team); status = ucc_topo_init(subset, UCC_TL_CORE_CTX(team)->topo, &team->topo); if (UCC_OK != status) { tl_debug(UCC_TL_TEAM_LIB(team), "failed to init team topo"); goto err_topo_init; } return UCC_OK; err_topo_init: ucc_ep_map_destroy_nested(&team->ctx_map); return status; } void ucc_tl_mlx5_topo_cleanup(ucc_tl_mlx5_team_t *team) { if (!team->topo) { return; } ucc_ep_map_destroy_nested(&team->ctx_map); ucc_topo_cleanup(team->topo); team->topo = NULL; } UCC_CLASS_INIT_FUNC(ucc_tl_mlx5_team_t, ucc_base_context_t *tl_context, const ucc_base_team_params_t *params) { ucc_tl_mlx5_context_t *ctx = ucc_derived_of(tl_context, ucc_tl_mlx5_context_t); ucc_status_t status = UCC_OK; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_team_t, &ctx->super, params); status = ucc_tl_mlx5_topo_init(self); if (status != UCC_OK) { tl_debug(ctx->super.super.lib, "failed to init team topo"); return status; } self->global_sync_req = NULL; self->a2a = NULL; if (ctx->cfg.enable_alltoall) { status = ucc_tl_mlx5_team_init_alltoall(self); if (UCC_OK != status) { return status; } } self->mcast = NULL; self->local_mcast_team_ready = 0; if (ctx->mcast.mcast_ctx_ready) { status = ucc_tl_mlx5_mcast_team_init(tl_context, &(self->mcast), &(ctx->mcast), params, &(UCC_TL_MLX5_TEAM_LIB(self)->cfg.mcast_conf)); if (UCC_OK != status) { tl_warn(tl_context->lib, "mcast team init failed"); } else { self->local_mcast_team_ready = 1; } } self->mcast_state = TL_MLX5_TEAM_STATE_MCAST_CTX_CHECK; self->a2a_state = TL_MLX5_TEAM_STATE_ALLTOALL_CTX_CHECK; tl_debug(tl_context->lib, "posted tl team: %p", self); return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_mlx5_team_t) { tl_debug(self->super.super.context->lib, "finalizing tl team: %p", self); ucc_tl_mlx5_dm_cleanup(self); if (self->a2a_state != TL_MLX5_TEAM_STATE_ALLTOALL_NOT_AVAILABLE) { ucc_tl_mlx5_alltoall_cleanup(self); } ucc_tl_mlx5_topo_cleanup(self); if (self->mcast_state != TL_MLX5_TEAM_STATE_MCAST_NOT_AVAILABLE) { ucc_tl_mlx5_clean_mcast_comm(self->mcast->mcast_comm); } } UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_mlx5_team_t, ucc_base_team_t); UCC_CLASS_DEFINE(ucc_tl_mlx5_team_t, ucc_tl_team_t); ucc_status_t ucc_tl_mlx5_team_destroy(ucc_base_team_t *tl_team) { UCC_CLASS_DELETE_FUNC_NAME(ucc_tl_mlx5_team_t)(tl_team); return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_alltoall_team_test(ucc_base_team_t *team) { ucc_tl_mlx5_team_t *tl_team = ucc_derived_of(team, ucc_tl_mlx5_team_t); switch (tl_team->a2a_state) { case TL_MLX5_TEAM_STATE_ALLTOALL_INIT: tl_team->a2a_status.local = ucc_tl_mlx5_team_test_alltoall_start(tl_team); tl_team->a2a_state = TL_MLX5_TEAM_STATE_ALLTOALL_POSTED; // coverity[missing_break:FALSE] case TL_MLX5_TEAM_STATE_ALLTOALL_POSTED: // coverity[deref_arg:FALSE] tl_team->a2a_status.local = ucc_tl_mlx5_team_test_alltoall_progress(tl_team); if (UCC_INPROGRESS == tl_team->a2a_status.local) { return UCC_INPROGRESS; } if (UCC_OK != tl_team->a2a_status.local) { tl_debug(UCC_TL_TEAM_LIB(tl_team), "failed to init a2a: %s", ucc_status_string(tl_team->a2a_status.local)); tl_team->a2a_state = TL_MLX5_TEAM_STATE_ALLTOALL_NOT_AVAILABLE; } else { tl_team->a2a_state = TL_MLX5_TEAM_STATE_ALLTOALL_READY; tl_debug(UCC_TL_TEAM_LIB(tl_team), "initialized tl a2a team: %p", tl_team); } case TL_MLX5_TEAM_STATE_ALLTOALL_READY: case TL_MLX5_TEAM_STATE_ALLTOALL_NOT_AVAILABLE: return UCC_OK; default: tl_error(UCC_TL_TEAM_LIB(tl_team), "unknown state during a2a team: %p create", tl_team); return UCC_ERR_NO_RESOURCE; } } ucc_status_t ucc_tl_mlx5_team_create_test(ucc_base_team_t *team) { ucc_tl_mlx5_team_t *tl_team = ucc_derived_of(team, ucc_tl_mlx5_team_t); ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(tl_team); ucc_team_t *core_team = UCC_TL_CORE_TEAM(tl_team); ucc_subset_t subset = {.map = UCC_TL_TEAM_MAP(tl_team), .myrank = UCC_TL_TEAM_RANK(tl_team)}; ucc_status_t a2a_status = UCC_OK; ucc_status_t mcast_status = UCC_OK; ucc_tl_mlx5_mcast_coll_comm_t *comm = NULL; ucc_status_t status; if (tl_team->global_sync_req != NULL) { status = ucc_service_coll_test(tl_team->global_sync_req); if (status < 0) { tl_debug(UCC_TL_TEAM_LIB(tl_team), "failure during service coll exchange: %s", ucc_status_string(status)); return status; } if (UCC_INPROGRESS == status) { return status; } ucc_service_coll_finalize(tl_team->global_sync_req); tl_team->global_sync_req = NULL; if (tl_team->mcast_state == TL_MLX5_TEAM_STATE_MCAST_CTX_CHECK && tl_team->a2a_state == TL_MLX5_TEAM_STATE_ALLTOALL_CTX_CHECK ) { tl_team->a2a_status.global = tl_team->global_status_array[UCC_TL_MLX5_A2A_STATUS_INDEX]; tl_team->a2a_state = TL_MLX5_TEAM_STATE_ALLTOALL_INIT; if (tl_team->global_status_array[UCC_TL_MLX5_MCAST_STATUS_INDEX] != UCC_OK) { /* mcast context is not available for some of the team members so we cannot create * mcast team */ tl_debug(UCC_TL_TEAM_LIB(tl_team), "some of the ranks do not have mcast context available so no mcast team is created"); if (tl_team->local_mcast_team_ready) { comm = tl_team->mcast->mcast_comm; if (ibv_destroy_cq(comm->mcast.rcq)) { tl_warn(UCC_TL_TEAM_LIB(tl_team), "ibv_destroy_cq failed"); } ucc_free(comm->params.oob); ucc_free(comm); ucc_free(tl_team->mcast); } tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_NOT_AVAILABLE; } else { tl_debug(UCC_TL_TEAM_LIB(tl_team), "all team members have mcast ctx ready"); tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_INIT; } return UCC_INPROGRESS; } else { if (tl_team->global_status_array[UCC_TL_MLX5_A2A_STATUS_INDEX] != UCC_OK) { //a2a team not avail for some of nodes so disable it if (tl_team->a2a_state == TL_MLX5_TEAM_STATE_ALLTOALL_READY) { // free the resources ucc_tl_mlx5_alltoall_cleanup(tl_team); } tl_team->a2a_state = TL_MLX5_TEAM_STATE_ALLTOALL_NOT_AVAILABLE; } if (tl_team->global_status_array[UCC_TL_MLX5_MCAST_STATUS_INDEX] != UCC_OK) { //mcast team not avail for some of nodes so disable it if (tl_team->mcast_state == TL_MLX5_TEAM_STATE_MCAST_READY) { // free the resources ucc_tl_mlx5_clean_mcast_comm(tl_team->mcast->mcast_comm); } tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_NOT_AVAILABLE; } tl_debug(team->context->lib, "team %p: MCAST component is %s ALLTOALL component is %s", team, (tl_team->mcast_state == TL_MLX5_TEAM_STATE_MCAST_READY)?"ENABLED":"DISABLED", (tl_team->a2a_state == TL_MLX5_TEAM_STATE_ALLTOALL_READY)?"ENABLED":"DISABLED"); } return UCC_OK; } ucc_assert(tl_team->global_sync_req == NULL); if (tl_team->mcast_state == TL_MLX5_TEAM_STATE_MCAST_CTX_CHECK && tl_team->a2a_state == TL_MLX5_TEAM_STATE_ALLTOALL_CTX_CHECK) { // check if ctx is ready for a2a and mcast tl_team->local_status_array[UCC_TL_MLX5_A2A_STATUS_INDEX] = tl_team->a2a_status.local; tl_team->local_status_array[UCC_TL_MLX5_MCAST_STATUS_INDEX] = (tl_team->local_mcast_team_ready) ? UCC_OK : UCC_ERR_NO_RESOURCE; goto initial_sync_post; } if (ctx->cfg.enable_alltoall) { a2a_status = ucc_tl_mlx5_alltoall_team_test(team); if (a2a_status < 0) { tl_warn(team->context->lib, "ALLTOALL tl team: %p creation failed %d", team, a2a_status); tl_team->a2a_state = TL_MLX5_TEAM_STATE_ALLTOALL_NOT_AVAILABLE; } } else { tl_team->a2a_state = TL_MLX5_TEAM_STATE_ALLTOALL_NOT_AVAILABLE; } if (tl_team->mcast_state != TL_MLX5_TEAM_STATE_MCAST_NOT_AVAILABLE) { mcast_status = ucc_tl_mlx5_mcast_team_test(team); if (mcast_status < 0) { tl_mlx5_mcast_log(ctx->mcast.mcast_enabled, team->context->lib, UCC_LOG_LEVEL_WARN, "MCAST tl team: %p creation failed %d", team, mcast_status); tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_NOT_AVAILABLE; } } if (UCC_INPROGRESS == a2a_status || UCC_INPROGRESS == mcast_status) { return UCC_INPROGRESS; } tl_team->local_status_array[UCC_TL_MLX5_A2A_STATUS_INDEX] = (tl_team->a2a_state == TL_MLX5_TEAM_STATE_ALLTOALL_READY) ? UCC_OK : UCC_ERR_NO_RESOURCE; tl_team->local_status_array[UCC_TL_MLX5_MCAST_STATUS_INDEX] = (tl_team->mcast_state == TL_MLX5_TEAM_STATE_MCAST_READY) ? UCC_OK : UCC_ERR_NO_RESOURCE; tl_debug(UCC_TL_TEAM_LIB(tl_team), "posting global status, local status: ALLTOALL %d MCAST %d", (tl_team->a2a_state == TL_MLX5_TEAM_STATE_ALLTOALL_READY), (tl_team->mcast_state == TL_MLX5_TEAM_STATE_MCAST_READY)); initial_sync_post: status = ucc_service_allreduce( core_team, tl_team->local_status_array, tl_team->global_status_array, UCC_DT_INT32, UCC_TL_MLX5_FEATURES_COUNT, UCC_OP_MIN, subset, &tl_team->global_sync_req); if (status < 0) { tl_debug(UCC_TL_TEAM_LIB(tl_team), "failed to collect global status"); return status; } return UCC_INPROGRESS; } ucc_status_t ucc_tl_mlx5_team_get_scores(ucc_base_team_t *tl_team, ucc_coll_score_t **score_p) { ucc_tl_mlx5_team_t *team = ucc_derived_of(tl_team, ucc_tl_mlx5_team_t); ucc_base_context_t *ctx = UCC_TL_TEAM_CTX(team); ucc_tl_mlx5_context_t *tl_ctx = ucc_derived_of(ctx, ucc_tl_mlx5_context_t); ucc_base_lib_t *lib = UCC_TL_TEAM_LIB(team); ucc_memory_type_t mt[2] = {UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA}; ucc_coll_score_t *score; ucc_status_t status; ucc_coll_score_team_info_t team_info; team_info.alg_fn = NULL; team_info.default_score = UCC_TL_MLX5_DEFAULT_SCORE; team_info.init = ucc_tl_mlx5_coll_init; team_info.num_mem_types = tl_ctx->supported_mem_types & UCC_BIT(UCC_MEMORY_TYPE_CUDA) ? 2 : 1; team_info.supported_mem_types = mt; team_info.supported_colls = (UCC_COLL_TYPE_ALLTOALL * (team->a2a_state == TL_MLX5_TEAM_STATE_ALLTOALL_READY)) | (UCC_COLL_TYPE_BCAST * (team->mcast_state == TL_MLX5_TEAM_STATE_MCAST_READY && tl_ctx->mcast.mcast_bcast_enabled)) | (UCC_COLL_TYPE_ALLGATHER * (team->mcast_state == TL_MLX5_TEAM_STATE_MCAST_READY && tl_ctx->mcast.mcast_allgather_enabled)); team_info.size = UCC_TL_TEAM_SIZE(team); status = ucc_coll_score_build_default( tl_team, UCC_TL_MLX5_DEFAULT_SCORE, ucc_tl_mlx5_coll_init, team_info.supported_colls, mt, 2, &score); if (UCC_OK != status) { tl_debug(lib, "failed to build score map"); return status; } if (strlen(ctx->score_str) > 0) { status = ucc_coll_score_update_from_str(ctx->score_str, &team_info, &team->super.super, score); /* If INVALID_PARAM - User provided incorrect input - try to proceed */ if ((status < 0) && (status != UCC_ERR_INVALID_PARAM) && (status != UCC_ERR_NOT_SUPPORTED)) { goto err; } } *score_p = score; return UCC_OK; err: ucc_coll_score_free(score); *score_p = NULL; return status; } ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_wqe.h0000664000175000017500000000324215211535620021350 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_MLX5_WQE_H_ #define UCC_TL_MLX5_WQE_H_ #include "ucc/api/ucc_status.h" #include #include ucc_status_t ucc_tl_mlx5_post_transpose(struct ibv_qp *qp, uint32_t src_mr_lkey, uint32_t dst_mr_key, uintptr_t src_mkey_addr, uintptr_t dst_addr, uint32_t element_size, uint16_t ncols, uint16_t nrows, int send_flags); ucc_status_t ucc_tl_mlx5_post_umr(struct ibv_qp * qp, struct mlx5dv_mkey *dv_mkey, uint32_t access_flags, uint32_t repeat_count, uint16_t num_entries, struct mlx5dv_mr_interleaved *data, uint32_t ptr_mkey, void *ptr_address); ucc_status_t ucc_tl_mlx5_post_rdma(struct ibv_qp *qp, uint32_t qpn, struct ibv_ah *ah, uintptr_t src_mkey_addr, size_t len, uint32_t src_mr_lkey, uintptr_t dst_addr, uint32_t dst_mr_key, int send_flags, uint64_t wr_id); ucc_status_t ucc_tl_mlx5_post_wait_on_data(struct ibv_qp *qp, uint64_t value, uint32_t lkey, uintptr_t addr, void *task_ptr); #endif ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_dm.c0000664000175000017500000002120715211535620021150 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_dm.h" #include "alltoall/alltoall.h" #define DM_HOST_AUTO_NUM_CHUNKS 8 static void ucc_tl_mlx5_alltoall_atomic_free(ucc_tl_mlx5_team_t *team) { if (!team->a2a || !team->a2a->net.atomic.counters) { return; } ibv_dereg_mr(team->a2a->net.atomic.mr); #if ATOMIC_IN_MEMIC ibv_free_dm(team->a2a->net.atomic.counters); #else ucc_free(team->a2a->net.atomic.counters); #endif team->a2a->net.atomic.counters = NULL; } static ucc_status_t ucc_tl_mlx5_alltoall_atomic_alloc(ucc_tl_mlx5_team_t *team) { ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(team); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; size_t size; size = sizeof(*a2a->net.atomic.counters) * MAX_OUTSTANDING_OPS; #if ATOMIC_IN_MEMIC struct ibv_alloc_dm_attr dm_attr; memset(&dm_attr, 0, sizeof(dm_attr)); dm_attr.length = size; a2a->net.atomic.counters = ibv_alloc_dm(ctx->shared_ctx, &dm_attr); #else a2a->net.atomic.counters = ucc_malloc(size, "atomic"); #endif if (!a2a->net.atomic.counters) { tl_debug(UCC_TL_TEAM_LIB(team), "failed to allocate %zd bytes for atomic counters array", size); return UCC_ERR_NO_MEMORY; } #if ATOMIC_IN_MEMIC a2a->net.atomic.mr = ibv_reg_dm_mr(ctx->shared_pd, a2a->net.atomic.counters, 0, size, IBV_ACCESS_REMOTE_ATOMIC | IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_ZERO_BASED); #else a2a->net.atomic.mr = ibv_reg_mr(ctx->shared_pd, a2a->net.atomic.counters, size, IBV_ACCESS_REMOTE_ATOMIC | IBV_ACCESS_LOCAL_WRITE); #endif if (!a2a->net.atomic.mr) { tl_error(UCC_TL_TEAM_LIB(team), "failed to register atomic couters array"); #if ATOMIC_IN_MEMIC ibv_free_dm(a2a->net.atomic.counters); #else ucc_free(a2a->net.atomic.counters); #endif return UCC_ERR_NO_MESSAGE; } return UCC_OK; } static void ucc_tl_mlx5_dm_chunk_init(ucc_mpool_t *mp, //NOLINT void *obj, void *chunk) //NOLINT { ucc_tl_mlx5_dm_chunk_t *c = (ucc_tl_mlx5_dm_chunk_t *)obj; ucc_tl_mlx5_team_t *team = ucc_container_of(mp, ucc_tl_mlx5_team_t, dm_pool); c->addr = (uintptr_t)PTR_OFFSET( (UCC_TL_MLX5_TEAM_LIB(team)->cfg.dm_host) ? team->dm_ptr : NULL, team->dm_offset); c->posted_sends = 0; c->posted_all = 0; c->completed_sends = 0; team->dm_offset = PTR_OFFSET( team->dm_offset, UCC_TL_MLX5_TEAM_LIB(team)->cfg.dm_buf_size * UCC_TL_MLX5_TEAM_LIB(team)->cfg.block_batch_size); } static ucc_mpool_ops_t ucc_tl_mlx5_dm_ops = { .chunk_alloc = ucc_mpool_hugetlb_malloc, .chunk_release = ucc_mpool_hugetlb_free, .obj_init = ucc_tl_mlx5_dm_chunk_init, .obj_cleanup = NULL}; void ucc_tl_mlx5_dm_pool_cleanup(ucc_tl_mlx5_team_t *team) { if (!team->dm_ptr || !team->a2a) { return; } ucc_mpool_cleanup(&team->dm_pool, 1); ibv_dereg_mr(team->dm_mr); if (UCC_TL_MLX5_TEAM_LIB(team)->cfg.dm_host) { ucc_free(team->dm_ptr); } else { ibv_free_dm(team->dm_ptr); } team->dm_ptr = NULL; } void ucc_tl_mlx5_dm_cleanup(ucc_tl_mlx5_team_t *team) { ucc_tl_mlx5_dm_pool_cleanup(team); ucc_tl_mlx5_alltoall_atomic_free(team); } ucc_status_t ucc_tl_mlx5_dm_alloc_reg(struct ibv_context *ib_ctx, struct ibv_pd *pd, int dm_host, size_t buf_size, size_t *buf_num_p, struct ibv_dm **ptr, struct ibv_mr **mr, ucc_base_lib_t *lib) { struct ibv_dm *dm_ptr = NULL; struct ibv_mr *dm_mr; struct ibv_device_attr_ex attr; struct ibv_alloc_dm_attr dm_attr; int max_chunks_to_alloc, min_chunks_to_alloc, i; if (dm_host) { max_chunks_to_alloc = (*buf_num_p == UCC_ULUNITS_AUTO) ? DM_HOST_AUTO_NUM_CHUNKS : *buf_num_p; dm_attr.length = max_chunks_to_alloc * buf_size; dm_ptr = ucc_malloc(dm_attr.length, "memic_host"); if (!dm_ptr) { tl_debug(lib, " memic_host allocation failed"); return UCC_ERR_NO_MEMORY; } dm_mr = ibv_reg_mr(pd, dm_ptr, dm_attr.length, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); if (!dm_mr) { tl_debug(lib, "failed to reg host memory"); ucc_free(dm_ptr); return UCC_ERR_NO_MESSAGE; } *buf_num_p = max_chunks_to_alloc; } else { attr.comp_mask = 0; if (ibv_query_device_ex(ib_ctx, NULL, &attr)) { tl_debug(lib, "failed to query device (errno=%d)", errno); return UCC_ERR_NO_MESSAGE; } if (!attr.max_dm_size) { tl_debug(lib, "device doesn't support dm allocation"); return UCC_ERR_NO_RESOURCE; } max_chunks_to_alloc = min_chunks_to_alloc = *buf_num_p; if (*buf_num_p == UCC_ULUNITS_AUTO) { max_chunks_to_alloc = attr.max_dm_size / buf_size - 1; //keep reserved memory min_chunks_to_alloc = 1; if (!max_chunks_to_alloc) { tl_debug(lib, "requested buffer size (=%ld) is too large, " "should be set to be strictly less than %ld. " "max allocation size is %ld", buf_size, attr.max_dm_size / 2, attr.max_dm_size); return UCC_ERR_NO_RESOURCE; } } if (attr.max_dm_size < buf_size * min_chunks_to_alloc) { tl_debug(lib, "cannot allocate %i buffer(s) of size %ld, " "max allocation size is %ld", min_chunks_to_alloc, buf_size, attr.max_dm_size); return UCC_ERR_NO_MEMORY; } memset(&dm_attr, 0, sizeof(dm_attr)); for (i = max_chunks_to_alloc; i >= min_chunks_to_alloc; i--) { dm_attr.length = i * buf_size; errno = 0; dm_ptr = ibv_alloc_dm(ib_ctx, &dm_attr); if (dm_ptr) { break; } } if (!dm_ptr) { tl_debug(lib, "dev mem allocation failed, requested %ld, attr.max %zd, " "errno %d", dm_attr.length, attr.max_dm_size, errno); return errno == ENOMEM || errno == ENOSPC ? UCC_ERR_NO_MEMORY : UCC_ERR_NO_MESSAGE; } dm_mr = ibv_reg_dm_mr(pd, dm_ptr, 0, dm_attr.length, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_ZERO_BASED); if (!dm_mr) { tl_debug(lib, "failed to reg memic"); ibv_free_dm(dm_ptr); return UCC_ERR_NO_MESSAGE; } *buf_num_p = i; } *ptr = dm_ptr; *mr = dm_mr; return UCC_OK; } ucc_status_t ucc_tl_mlx5_dm_init(ucc_tl_mlx5_team_t *team) { ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(team); ucc_tl_mlx5_lib_config_t *cfg = &UCC_TL_MLX5_TEAM_LIB(team)->cfg; ucc_status_t status; status = ucc_tl_mlx5_alltoall_atomic_alloc(team); if (UCC_OK != status) { return status; } status = ucc_tl_mlx5_dm_alloc_reg( ctx->shared_ctx, ctx->shared_pd, cfg->dm_host, cfg->dm_buf_size * cfg->block_batch_size, &cfg->dm_buf_num, &team->dm_ptr, &team->dm_mr, UCC_TL_TEAM_LIB(team)); if (status != UCC_OK) { goto err_dm_alloc; } team->dm_offset = 0; // TODO: fix/check the case dm_host=true ucc_assert(!cfg->dm_host); status = ucc_mpool_init( &team->dm_pool, 0, sizeof(ucc_tl_mlx5_dm_chunk_t), 0, UCC_CACHE_LINE_SIZE, 1, cfg->dm_buf_num, &ucc_tl_mlx5_dm_ops, ctx->super.super.ucc_context->thread_mode, "mlx5 dm pool"); if (status != UCC_OK) { tl_debug(UCC_TL_TEAM_LIB(team), "failed to init dm pool"); goto err_mpool_init; } return UCC_OK; err_mpool_init: ibv_dereg_mr(team->dm_mr); if (UCC_TL_MLX5_TEAM_LIB(team)->cfg.dm_host) { ucc_free(team->dm_ptr); } else { ibv_free_dm(team->dm_ptr); } team->dm_ptr = NULL; err_dm_alloc: ucc_tl_mlx5_alltoall_atomic_free(team); return status; } ucc-1.8.0/src/components/tl/mlx5/tl_mlx5.h0000664000175000017500000002015315211535620020474 0ustar alastairalastair/** * Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_MLX5_H_ #define UCC_TL_MLX5_H_ #include "components/tl/ucc_tl.h" #include "components/tl/ucc_tl_log.h" #include "core/ucc_service_coll.h" #include "utils/ucc_mpool.h" #include "utils/ucc_rcache.h" #include #include #include "utils/arch/cpu.h" #include "mcast/tl_mlx5_mcast.h" #ifndef UCC_TL_MLX5_DEFAULT_SCORE #define UCC_TL_MLX5_DEFAULT_SCORE 1 #endif #ifdef HAVE_PROFILING_TL_MLX5 #include "utils/profile/ucc_profile.h" #else #include "utils/profile/ucc_profile_off.h" #endif #define UCC_TL_MLX5_PROFILE_FUNC UCC_PROFILE_FUNC #define UCC_TL_MLX5_PROFILE_FUNC_VOID UCC_PROFILE_FUNC_VOID #define UCC_TL_MLX5_PROFILE_REQUEST_NEW UCC_PROFILE_REQUEST_NEW #define UCC_TL_MLX5_PROFILE_REQUEST_EVENT UCC_PROFILE_REQUEST_EVENT #define UCC_TL_MLX5_PROFILE_REQUEST_FREE UCC_PROFILE_REQUEST_FREE #define ATOMIC_IN_MEMIC 1 #define DC_KEY 1 typedef struct ucc_tl_mlx5_iface { ucc_tl_iface_t super; } ucc_tl_mlx5_iface_t; /* Extern iface should follow the pattern: ucc_tl_ */ extern ucc_tl_mlx5_iface_t ucc_tl_mlx5; typedef struct ucc_tl_mlx5_ib_qp_conf { uint8_t qp_sl; uint32_t qp_rnr_retry; uint32_t qp_rnr_timer; uint32_t qp_retry_cnt; uint32_t qp_timeout; uint32_t qp_max_atomic; } ucc_tl_mlx5_ib_qp_conf_t; typedef enum ucc_tl_mlx5_alltoall_block_shape_modes { UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_LONG, UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_WIDE, UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_SQUARE, UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_LAST, } ucc_tl_mlx5_alltoall_block_shape_modes_t; typedef struct ucc_tl_mlx5_lib_config { ucc_tl_lib_config_t super; int asr_barrier; int block_size; int num_dci_qps; int dc_threshold; size_t dm_buf_size; unsigned long dm_buf_num; int dm_host; ucc_tl_mlx5_ib_qp_conf_t qp_conf; ucc_tl_mlx5_mcast_coll_comm_init_spec_t mcast_conf; int num_serialized_batches; int num_batches_per_passage; int block_batch_size; int force_regular; ucc_tl_mlx5_alltoall_block_shape_modes_t block_shape_mode; } ucc_tl_mlx5_lib_config_t; typedef struct ucc_tl_mlx5_context_config { ucc_tl_context_config_t super; ucs_config_names_array_t devices; ucc_tl_mlx5_mcast_ctx_params_t mcast_ctx_conf; int enable_alltoall; } ucc_tl_mlx5_context_config_t; typedef struct ucc_tl_mlx5_lib { ucc_tl_lib_t super; ucc_tl_mlx5_lib_config_t cfg; } ucc_tl_mlx5_lib_t; UCC_CLASS_DECLARE(ucc_tl_mlx5_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); typedef struct ucc_tl_mlx5_context { ucc_tl_context_t super; ucc_tl_mlx5_context_config_t cfg; struct ibv_context *shared_ctx; struct ibv_pd *shared_pd; ucc_rcache_t *rcache; int is_imported; int ib_port; int sock; ucc_mpool_t req_mp; ucc_tl_mlx5_mcast_context_t mcast; uint16_t supported_mem_types; } ucc_tl_mlx5_context_t; UCC_CLASS_DECLARE(ucc_tl_mlx5_context_t, const ucc_base_context_params_t*, const ucc_base_config_t*); typedef struct ucc_tl_mlx5_task ucc_tl_mlx5_task_t; typedef struct ucc_tl_mlx5_schedule ucc_tl_mlx5_schedule_t; typedef struct ucc_tl_mlx5_dm_chunk_t { uintptr_t addr; // 0 based offset from the beginning of // memic_mr (obtained with ibv_reg_dm_mr) ucc_tl_mlx5_schedule_t *task; int posted_sends; int posted_all; int completed_sends; } ucc_tl_mlx5_dm_chunk_t; typedef struct ucc_tl_mlx5_alltoall ucc_tl_mlx5_alltoall_t; typedef enum { TL_MLX5_TEAM_STATE_ALLTOALL_CTX_CHECK, TL_MLX5_TEAM_STATE_ALLTOALL_INIT, TL_MLX5_TEAM_STATE_ALLTOALL_POSTED, TL_MLX5_TEAM_STATE_ALLTOALL_READY, TL_MLX5_TEAM_STATE_ALLTOALL_NOT_AVAILABLE, } ucc_tl_mlx5_team_a2a_state_t; typedef enum { TL_MLX5_TEAM_STATE_MCAST_CTX_CHECK, TL_MLX5_TEAM_STATE_MCAST_INIT, TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_TEST, TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_READY, TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_FAILED, TL_MLX5_TEAM_STATE_MCAST_GRP_BCAST_POST, TL_MLX5_TEAM_STATE_MCAST_RELIAB_SYNC, TL_MLX5_TEAM_STATE_MCAST_RELIABLITY, TL_MLX5_TEAM_STATE_MCAST_READY, TL_MLX5_TEAM_STATE_MCAST_NOT_AVAILABLE } ucc_tl_mlx5_team_mcast_state_t; typedef struct ucc_tl_mlx5_team_status { ucc_status_t local; ucc_status_t global; } ucc_tl_mlx5_team_status_t; #define UCC_TL_MLX5_FEATURES_COUNT 2 /* Currently only alltoall and mcast are supported */ #define UCC_TL_MLX5_A2A_STATUS_INDEX 0 #define UCC_TL_MLX5_MCAST_STATUS_INDEX 1 typedef struct ucc_tl_mlx5_team { ucc_tl_team_t super; ucc_service_coll_req_t *scoll_req; ucc_service_coll_req_t *global_sync_req; ucc_tl_mlx5_team_a2a_state_t a2a_state; ucc_tl_mlx5_team_mcast_state_t mcast_state; void *dm_offset; ucc_mpool_t dm_pool; struct ibv_dm *dm_ptr; struct ibv_mr *dm_mr; ucc_tl_mlx5_team_status_t a2a_status; ucc_tl_mlx5_alltoall_t *a2a; ucc_topo_t *topo; ucc_ep_map_t ctx_map; int local_mcast_team_ready; ucc_tl_mlx5_mcast_team_t *mcast; ucc_status_t local_status_array[UCC_TL_MLX5_FEATURES_COUNT]; ucc_status_t global_status_array[UCC_TL_MLX5_FEATURES_COUNT]; } ucc_tl_mlx5_team_t; UCC_CLASS_DECLARE(ucc_tl_mlx5_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t tl_mlx5_rcache_create(ucc_tl_mlx5_context_t *ctx); typedef struct ucc_tl_mlx5_reg { struct ibv_mr *mr; } ucc_tl_mlx5_reg_t; typedef struct ucc_tl_mlx5_rcache_region { ucc_rcache_region_t super; ucc_tl_mlx5_reg_t reg; } ucc_tl_mlx5_rcache_region_t; #define UCC_TL_MLX5_SUPPORTED_COLLS \ (UCC_COLL_TYPE_ALLTOALL | UCC_COLL_TYPE_BCAST | UCC_COLL_TYPE_ALLGATHER) #define UCC_TL_MLX5_TEAM_LIB(_team) \ (ucc_derived_of((_team)->super.super.context->lib, ucc_tl_mlx5_lib_t)) #define UCC_TL_MLX5_TEAM_CTX(_team) \ (ucc_derived_of((_team)->super.super.context, ucc_tl_mlx5_context_t)) #define UCC_TL_CTX_HAS_OOB(_ctx) \ ((_ctx)->super.super.ucc_context->params.mask & UCC_CONTEXT_PARAM_FIELD_OOB) #define UCC_TL_CTX_OOB(_ctx) ((_ctx)->super.super.ucc_context->params.oob) #define UCC_TL_CTX_LIB(_ctx) \ (ucc_derived_of((_ctx)->super.super.lib, ucc_tl_mlx5_lib_t)) #define SQUARED(_num) ((_num) * (_num)) ucc_status_t tl_mlx5_create_rcache(ucc_tl_mlx5_context_t *ctx); ucc_status_t ucc_tl_mlx5_asr_socket_init(ucc_tl_mlx5_context_t *ctx, ucc_rank_t group_size, int *socket, const char *sock_path); ucc_status_t ucc_tl_mlx5_dm_alloc_reg(struct ibv_context *ib_ctx, struct ibv_pd *pd, int dm_host, size_t buf_size, size_t *buf_num_p, struct ibv_dm **ptr, struct ibv_mr **mr, ucc_base_lib_t *lib); #endif ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_pd.h0000664000175000017500000000144115211535620021156 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef TL_MLX5_PD_H #define TL_MLX5_PD_H #include typedef struct ucc_tl_mlx5_team ucc_tl_mlx5_team_t; ucc_status_t ucc_tl_mlx5_share_ctx_pd(ucc_tl_mlx5_context_t *ctx, const char * sock_path, ucc_rank_t group_size, int is_ctx_owner, int ctx_owner_sock); ucc_status_t ucc_tl_mlx5_remove_shared_ctx_pd(ucc_tl_mlx5_context_t *ctx); ucc_status_t ucc_tl_mlx5_socket_init(ucc_tl_mlx5_context_t *ctx, ucc_rank_t group_size, int *sock_p, const char *sock_path); #endif ucc-1.8.0/src/components/tl/mlx5/tl_mlx5.c0000664000175000017500000002613415211535620020474 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5.h" ucc_status_t ucc_tl_mlx5_get_lib_attr(const ucc_base_lib_t *lib, ucc_base_lib_attr_t * base_attr); ucc_status_t ucc_tl_mlx5_get_context_attr(const ucc_base_context_t *context, ucc_base_ctx_attr_t * base_attr); ucc_status_t ucc_tl_mlx5_mem_map(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_memh_t *memh, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_mlx5_mem_unmap(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h); ucc_status_t ucc_tl_mlx5_memh_pack(const ucc_base_context_t *context, ucc_mem_map_mode_t mode, ucc_mem_map_tl_t *tl_h, void **pack_buffer); ucc_status_t ucc_tl_mlx5_get_lib_properties(ucc_base_lib_properties_t *prop); static const char *alltoall_block_shape_modes[] = { [UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_LONG] = "long", [UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_WIDE] = "wide", [UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_SQUARE] = "square", [UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_LAST] = NULL}; static ucc_config_field_t ucc_tl_mlx5_lib_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_mlx5_lib_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_lib_config_table)}, {"ASR_BARRIER", "0", "Boolean - use service barrier or p2p sync of ASRs", ucc_offsetof(ucc_tl_mlx5_lib_config_t, asr_barrier), UCC_CONFIG_TYPE_BOOL}, {"DM_BUF_SIZE", "8k", "Size of the pre-allocated DeviceMemory buffer", ucc_offsetof(ucc_tl_mlx5_lib_config_t, dm_buf_size), UCC_CONFIG_TYPE_MEMUNITS}, {"DM_BUF_NUM", "auto", "Number of DM buffers to alloc", ucc_offsetof(ucc_tl_mlx5_lib_config_t, dm_buf_num), UCC_CONFIG_TYPE_ULUNITS}, {"ALLTOALL_FORCE_REGULAR", "y", "Enforce the regular case where the block dimensions evenly divide ppn. " "This option requires BLOCK_SIZE = 0.", ucc_offsetof(ucc_tl_mlx5_lib_config_t, force_regular), UCC_CONFIG_TYPE_BOOL}, {"ALLTOALL_BLOCK_SHAPE", "long", "Shape of the blocks that are sent using blocked AlltoAll Algorithm\n" "long - blocks are more long than wide\n" "wide - blocks are more wide than long\n" "square - blocks are square", ucc_offsetof(ucc_tl_mlx5_lib_config_t, block_shape_mode), UCC_CONFIG_TYPE_ENUM(alltoall_block_shape_modes)}, {"ALLTOALL_BLOCK_SIZE", "0", "Size of the blocks that are sent using blocked AlltoAll Algorithm. " "A block size of 0 means it will be calculated automatically", ucc_offsetof(ucc_tl_mlx5_lib_config_t, block_size), UCC_CONFIG_TYPE_UINT}, {"NUM_DCI_QPS", "16", "Number of parallel DCI QPs that will be used per team", ucc_offsetof(ucc_tl_mlx5_lib_config_t, num_dci_qps), UCC_CONFIG_TYPE_UINT}, {"DC_THRESHOLD", "128", "If number of nodes >= DC_THRESHOLD then DC QPs " "are used instead of RC", ucc_offsetof(ucc_tl_mlx5_lib_config_t, dc_threshold), UCC_CONFIG_TYPE_UINT}, {"DM_HOST", "n", "Use host registered memory instead of DM for " "transpose staging", ucc_offsetof(ucc_tl_mlx5_lib_config_t, dm_host), UCC_CONFIG_TYPE_BOOL}, {"QP_RNR_RETRY", "7", "IB QP rnr retry count", ucc_offsetof(ucc_tl_mlx5_lib_config_t, qp_conf.qp_rnr_retry), UCC_CONFIG_TYPE_UINT}, {"QP_RNR_TIMER", "20", "IB QP rnr timer", ucc_offsetof(ucc_tl_mlx5_lib_config_t, qp_conf.qp_rnr_timer), UCC_CONFIG_TYPE_UINT}, {"QP_RETRY_COUNT", "7", "IB QP retry count", ucc_offsetof(ucc_tl_mlx5_lib_config_t, qp_conf.qp_retry_cnt), UCC_CONFIG_TYPE_UINT}, {"QP_TIMEOUT", "18", "IB QP timeout", ucc_offsetof(ucc_tl_mlx5_lib_config_t, qp_conf.qp_timeout), UCC_CONFIG_TYPE_UINT}, {"QP_MAX_ATOMIC", "1", "max num of outstanding atomics in IB QP", ucc_offsetof(ucc_tl_mlx5_lib_config_t, qp_conf.qp_max_atomic), UCC_CONFIG_TYPE_UINT}, {"QP_SL", "0", "IB QP Service Level", ucc_offsetof(ucc_tl_mlx5_lib_config_t, qp_conf.qp_sl), UCC_CONFIG_TYPE_UINT}, {"MCAST_SX_DEPTH", "512", "Send context depth of the Mcast comm", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.sx_depth), UCC_CONFIG_TYPE_INT}, {"MCAST_SX_INLINE", "128", "Minimal size for inline data send in Mcast", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.sx_inline), UCC_CONFIG_TYPE_MEMUNITS}, {"MCAST_RX_DEPTH", "4096", "Recv context depth of the Mcast comm", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.rx_depth), UCC_CONFIG_TYPE_INT}, {"MCAST_POST_RECV_THRESH", "64", "Threshold for posting recv into rx ctx of the Mcast comm", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.post_recv_thresh), UCC_CONFIG_TYPE_INT}, {"MCAST_WINDOW_SIZE", "64", "Reliability Mcast window size", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.wsize), UCC_CONFIG_TYPE_INT}, {"MCAST_MAX_PUSH_SEND", "16", "Max number of concurrent send wq for mcast based allgather", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.max_push_send), UCC_CONFIG_TYPE_INT}, {"MCAST_MAX_EAGER", "65536", "Max msg size to be used for Mcast with the eager protocol", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.max_eager), UCC_CONFIG_TYPE_MEMUNITS}, {"MCAST_CUDA_MEM_ENABLE", "0", "Enable GPU CUDA memory support for Mcast. GPUDirect RDMA must be enabled", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.cuda_mem_enabled), UCC_CONFIG_TYPE_BOOL}, {"MCAST_ONE_SIDED_RELIABILITY_ENABLE", "0", "Enable one sided reliability for mcast", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.one_sided_reliability_enable), UCC_CONFIG_TYPE_BOOL}, {"MCAST_ONE_SIDED_RELIABILITY_THRESHOLD", "65536", "Message threshold to toggle async to sync reliability protocol in mcast Allgather", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.reliability_scheme_msg_threshold), UCC_CONFIG_TYPE_INT}, {"MCAST_ZERO_COPY_ALLGATHER_ENABLE", "0", "Enable truly zero copy allgather design for mcast", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.truly_zero_copy_allgather_enabled), UCC_CONFIG_TYPE_BOOL}, {"MCAST_ZERO_COPY_BCAST_ENABLE", "0", "Enable truly zero copy bcast design for mcast", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.truly_zero_copy_bcast_enabled), UCC_CONFIG_TYPE_BOOL}, {"MCAST_ZERO_COPY_PREPOST_BUCKET_SIZE", "16", "Number of posted recvs during each stage of the pipeline" " in truly zero copy mcast allgather design", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.mcast_prepost_bucket_size), UCC_CONFIG_TYPE_INT}, {"MCAST_GROUP_COUNT", "1", "Number of multicast groups that can be used to increase parallelism", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.mcast_group_count), UCC_CONFIG_TYPE_INT}, {"MCAST_ZERO_COPY_COLL_MIN_MSG", "65536", "Min msg size to be used for zero copy collectives", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.truly_zero_copy_coll_min_msg), UCC_CONFIG_TYPE_UINT}, {"MCAST_HCA_COPY_ENABLE", "0", "Enable HCA-assisted copy for systems without CUDA API support", ucc_offsetof(ucc_tl_mlx5_lib_config_t, mcast_conf.hca_copy_enabled), UCC_CONFIG_TYPE_BOOL}, {"ALLTOALL_SEND_BATCH_SIZE", "2", "Number of blocks that are transposed " "on the NIC before being sent as a batch to a remote peer", ucc_offsetof(ucc_tl_mlx5_lib_config_t, block_batch_size), UCC_CONFIG_TYPE_UINT}, {"ALLTOALL_NUM_SERIALIZED_BATCHES", "4", "Number of block batches " "(within the set of blocks to be sent to a given remote peer) " "serialized on the same device memory chunk", ucc_offsetof(ucc_tl_mlx5_lib_config_t, num_serialized_batches), UCC_CONFIG_TYPE_UINT}, {"ALLTOALL_NUM_BATCHES_PER_PASSAGE", "1", "Number of batches of blocks sent to one remote node before enqueing", ucc_offsetof(ucc_tl_mlx5_lib_config_t, num_batches_per_passage), UCC_CONFIG_TYPE_UINT}, {NULL}}; static ucc_config_field_t ucc_tl_mlx5_context_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_tl_mlx5_context_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_tl_context_config_table)}, {"NET_DEVICES", "", "Specifies which network device(s) to use", ucc_offsetof(ucc_tl_mlx5_context_config_t, devices), UCC_CONFIG_TYPE_STRING_ARRAY}, {"MCAST_TIMEOUT", "1000000", "Timeout [usec] for the reliability NACK in Mcast", ucc_offsetof(ucc_tl_mlx5_context_config_t, mcast_ctx_conf.timeout), UCC_CONFIG_TYPE_INT}, {"MCAST_BCAST_ENABLE", "0", "Enable Mcast-based Bcast", ucc_offsetof(ucc_tl_mlx5_context_config_t, mcast_ctx_conf.mcast_bcast_enabled), UCC_CONFIG_TYPE_BOOL}, {"MCAST_ALLGATHER_ENABLE", "0", "Enable Mcast-based Allgather", ucc_offsetof(ucc_tl_mlx5_context_config_t, mcast_ctx_conf.mcast_allgather_enabled), UCC_CONFIG_TYPE_BOOL}, {"MCAST_ENABLE", "n", "Enable Mcast\n" "n - disable mcast entirely\n" "try - try to enable, continue without mcast if resources unavailable (silent except debug/trace)\n" "y - force enable and warn if resources unavailable, continue without mcast", ucc_offsetof(ucc_tl_mlx5_context_config_t, mcast_ctx_conf.mcast_enabled), UCC_CONFIG_TYPE_TERNARY}, {"MCAST_NET_DEVICE", "", "Specifies which network device to use for Mcast", ucc_offsetof(ucc_tl_mlx5_context_config_t, mcast_ctx_conf.ib_dev_name), UCC_CONFIG_TYPE_STRING}, {"ALLTOALL_ENABLE", "1", "Enable Accelerated alltoall", ucc_offsetof(ucc_tl_mlx5_context_config_t, enable_alltoall), UCC_CONFIG_TYPE_BOOL}, {NULL}}; UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_mlx5_lib_t, ucc_base_lib_t, const ucc_base_lib_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_mlx5_lib_t, ucc_base_lib_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_mlx5_context_t, ucc_base_context_t, const ucc_base_context_params_t *, const ucc_base_config_t *); UCC_CLASS_DEFINE_DELETE_FUNC(ucc_tl_mlx5_context_t, ucc_base_context_t); UCC_CLASS_DEFINE_NEW_FUNC(ucc_tl_mlx5_team_t, ucc_base_team_t, ucc_base_context_t *, const ucc_base_team_params_t *); ucc_status_t ucc_tl_mlx5_team_create_test(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_mlx5_team_destroy(ucc_base_team_t *tl_team); ucc_status_t ucc_tl_mlx5_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task); ucc_status_t ucc_tl_mlx5_team_get_scores(ucc_base_team_t * tl_team, ucc_coll_score_t **score); UCC_TL_IFACE_DECLARE(mlx5, MLX5); ucc_status_t ucc_tl_mlx5_context_create_epilog(ucc_base_context_t *context); __attribute__((constructor)) static void tl_mlx5_iface_init(void) { ucc_tl_mlx5.super.context.create_epilog = ucc_tl_mlx5_context_create_epilog; } ucc-1.8.0/src/components/tl/mlx5/mcast/0000775000175000017500000000000015211535620020045 5ustar alastairalastairucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_one_sided_progress.h0000664000175000017500000000274415211535620026735 0ustar alastairalastair/** * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include #include #include #include "tl_mlx5_mcast.h" #include "utils/ucc_math.h" #include "tl_mlx5_mcast_helper.h" #include "p2p/ucc_tl_mlx5_mcast_p2p.h" #ifndef TL_MLX5_MCAST_ONE_SIDED_PROGRESS_H_ #define TL_MLX5_MCAST_ONE_SIDED_PROGRESS_H_ ucc_status_t ucc_tl_mlx5_mcast_progress_one_sided_communication(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req); ucc_status_t ucc_tl_mlx5_mcast_staging_allgather_reliable_one_sided_get(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, int *completed); ucc_status_t ucc_tl_mlx5_mcast_process_packet_collective(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, struct pp_packet* pp, int coll_type); ucc_status_t ucc_tl_mlx5_mcast_reliable_zcopy_pipelined_one_sided_get(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, int *completed); #endif ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_helper.h0000664000175000017500000005774115211535620024346 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef TL_MLX5_MCAST_HELPER_H_ #define TL_MLX5_MCAST_HELPER_H_ #include "tl_mlx5_mcast_progress.h" #include "tl_mlx5_mcast_one_sided_progress.h" #include "utils/ucc_math.h" #include "tl_mlx5.h" #include "tl_mlx5_mcast_hca_copy.h" static inline ucc_status_t ucc_tl_mlx5_mcast_poll_send(ucc_tl_mlx5_mcast_coll_comm_t *comm) { int num_comp; struct pp_packet *pp; struct ibv_wc wc[POLL_PACKED]; num_comp = ibv_poll_cq(comm->mcast.scq, POLL_PACKED, &wc[0]); if (num_comp < 0) { tl_error(comm->lib, "send queue poll completion failed %d", num_comp); return UCC_ERR_NO_MESSAGE; } else if (num_comp > 0) { tl_trace(comm->lib, "polled send completions: %d", num_comp); for (int i = 0 ; i < num_comp ; i++) { if (IBV_WC_SUCCESS != wc[i].status) { tl_warn(comm->lib, "mcast_poll_send: %s err %d num_comp %d op %ld wr_id\n", ibv_wc_status_str(wc[i].status), num_comp, wc[i].opcode, wc[i].wr_id); return UCC_ERR_NO_MESSAGE; } switch (wc[i].wr_id) { case MCAST_AG_RDMA_READ_WR: /* completion of a RDMA Read to remote send buffer during * reliability protocol */ comm->one_sided.pending_reads--; tl_trace(comm->lib, "RDMA READ completion, pending reads %d", comm->one_sided.pending_reads); break; case MCAST_AG_RDMA_READ_INFO_WR: tl_trace(comm->lib, "RDMA READ remote slot info completion, pending reads %d", comm->one_sided.pending_reads); comm->one_sided.pending_reads--; break; case MCAST_BCASTSEND_WR: tl_trace(comm->lib, "completion of mcast send for bcast, opcode %d", wc[i].opcode); comm->pending_send--; break; default: tl_trace(comm->lib, "completion of mcast send for zero-copy collective, opcode %d", wc[i].opcode); comm->pending_send--; pp = (struct pp_packet*)wc[i].wr_id; assert(pp != 0); pp->context = 0; ucc_list_add_tail(&comm->bpool, &pp->super); break; } } } return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_mcast_send(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, int num_packets, const int zcopy) { struct ibv_send_wr *swr = &comm->mcast.swr; struct ibv_sge *ssg = &comm->mcast.ssg; int max_per_packet = comm->max_per_packet; int offset = req->offset; int i; struct ibv_send_wr *bad_wr; struct pp_packet *pp; int rc; int length; ucc_status_t status; ucc_memory_type_t mem_type = comm->cuda_mem_enabled ? UCC_MEMORY_TYPE_CUDA : UCC_MEMORY_TYPE_HOST; for (i = 0; i < num_packets; i++) { if (comm->params.sx_depth <= (comm->pending_send * comm->params.scq_moderation + comm->tx)) { status = ucc_tl_mlx5_mcast_poll_send(comm); if (UCC_OK != status) { return status; } break; } if (NULL == (pp = ucc_tl_mlx5_mcast_buf_get_free(comm))) { break; } ucc_assert(pp->context == 0); __builtin_prefetch((void*) pp->buf); __builtin_prefetch(PTR_OFFSET(req->ptr, offset)); length = req->to_send == 1 ? req->last_pkt_len : max_per_packet; pp->length = length; pp->psn = comm->psn; ssg[0].addr = (uintptr_t) PTR_OFFSET(req->ptr, offset); if (zcopy) { pp->context = (uintptr_t) PTR_OFFSET(req->ptr, offset); } else { status = ucc_mc_memcpy((void*) pp->buf, PTR_OFFSET(req->ptr, offset), length, mem_type, mem_type); if (ucc_unlikely(status != UCC_OK)) { tl_error(comm->lib, "failed to copy cuda buffer"); return status; } ssg[0].addr = (uint64_t) pp->buf; } ssg[0].length = length; ssg[0].lkey = zcopy ? req->mr->lkey : comm->pp_mr->lkey; swr[0].wr.ud.ah = comm->mcast.groups[0].ah; swr[0].wr_id = MCAST_BCASTSEND_WR; swr[0].imm_data = htonl(pp->psn); swr[0].send_flags = (length <= comm->max_inline) ? IBV_SEND_INLINE : 0; comm->r_window[pp->psn & (comm->bcast_comm.wsize-1)] = pp; comm->psn++; req->to_send--; offset += length; comm->tx++; if (comm->tx == comm->params.scq_moderation) { swr[0].send_flags |= IBV_SEND_SIGNALED; comm->tx = 0; comm->pending_send++; } tl_trace(comm->lib, "post_send, psn %d, length %d, zcopy %d, signaled %d", pp->psn, pp->length, zcopy, swr[0].send_flags & IBV_SEND_SIGNALED); if (0 != (rc = ibv_post_send(comm->mcast.groups[0].qp, &swr[0], &bad_wr))) { tl_error(comm->lib, "post send failed: ret %d, start_psn %d, to_send %d, " "to_recv %d, length %d, psn %d, inline %d", rc, req->start_psn, req->to_send, req->to_recv, length, pp->psn, length <= comm->max_inline); return UCC_ERR_NO_MESSAGE; } status = ucc_tl_mlx5_mcast_check_nack_requests(comm, pp->psn); if (UCC_OK != status) { return status; } } req->offset = offset; return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_mcast_process_pp(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, struct pp_packet *pp, int *num_left, int in_pending_queue) { ucc_status_t status = UCC_OK; if (PSN_RECEIVED(pp->psn, comm) || pp->psn < comm->bcast_comm.last_acked) { /* This psn was already received */ ucc_assert(pp->context == 0); if (in_pending_queue) { /* this pp belongs to pending queue so remove it */ ucc_list_del(&pp->super); } /* add pp to the free pool */ ucc_list_add_tail(&comm->bpool, &pp->super); } else if (PSN_IS_IN_RANGE(pp->psn, req, comm)) { if (*num_left <= 0 && !in_pending_queue) { /* we just received this packet and it is in order, but there is no * more space in window so we need to place this packet in the * pending queue for future processings */ ucc_list_add_tail(&comm->pending_q, &pp->super); } else { __builtin_prefetch(PTR_OFFSET(req->ptr, PSN_TO_RECV_OFFSET(pp->psn, req, comm))); __builtin_prefetch((void*) pp->buf); if (in_pending_queue) { ucc_list_del(&pp->super); } status = ucc_tl_mlx5_mcast_process_packet(comm, req, pp); if (UCC_OK != status) { return status; } (*num_left)--; } } else if (!in_pending_queue) { /* add pp to the pending queue as it is out of order */ ucc_list_add_tail(&comm->pending_q, &pp->super); } return status; } /* this function return the number of mcast recv packets that * are left or -1 in case of error */ static inline int ucc_tl_mlx5_mcast_recv(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, int num_left, int *pending_q_size) { struct pp_packet *pp; struct pp_packet *next; uint64_t id; struct ibv_wc *wc; int num_comp; int i; int real_num_comp; ucc_status_t status; /* check if we have already received something */ ucc_list_for_each_safe(pp, next, &comm->pending_q, super) { status = ucc_tl_mlx5_mcast_process_pp(comm, req, pp, &num_left, 1); if (UCC_OK != status) { return -1; } (*pending_q_size)++; } wc = ucc_malloc(sizeof(struct ibv_wc) * POLL_PACKED, "WC"); if (!wc) { tl_error(comm->lib, "ucc_malloc failed"); return -1; } while (num_left > 0) { memset(wc, 0, sizeof(struct ibv_wc) * POLL_PACKED); num_comp = ibv_poll_cq(comm->mcast.rcq, POLL_PACKED, wc); if (num_comp < 0) { tl_error(comm->lib, "recv queue poll completion failed %d", num_comp); ucc_free(wc); return -1; } else if (num_comp == 0) { break; } real_num_comp = num_comp; for (i = 0; i < real_num_comp; i++) { if (IBV_WC_SUCCESS != wc[i].status) { tl_error(comm->lib, "mcast_recv: %s err pending_recv %d wr_id %ld" " num_comp %d byte_len %d", ibv_wc_status_str(wc[i].status), comm->pending_recv, wc[i].wr_id, num_comp, wc[i].byte_len); ucc_free(wc); return -1; } id = wc[i].wr_id; pp = (struct pp_packet*) (id); pp->length = wc[i].byte_len - GRH_LENGTH; pp->psn = ntohl(wc[i].imm_data); tl_trace(comm->lib, "completion: psn %d, length %d, already_received %d, " " psn in req %d, req_start %d, req_num packets" " %d, to_send %d, to_recv %d, num_left %d", pp->psn, pp->length, PSN_RECEIVED(pp->psn, comm) > 0, PSN_IS_IN_RANGE(pp->psn, req, comm), req->start_psn, req->num_packets, req->to_send, req->to_recv, num_left); status = ucc_tl_mlx5_mcast_process_pp(comm, req, pp, &num_left, 0); if (UCC_OK != status) { return -1; } } comm->pending_recv -= num_comp; status = ucc_tl_mlx5_mcast_post_recv_buffers(comm); if (UCC_OK != status) { return -1; } } ucc_free(wc); return num_left; } static inline ucc_status_t ucc_tl_mlx5_mcast_send_collective(ucc_tl_mlx5_mcast_coll_comm_t* comm, ucc_tl_mlx5_mcast_coll_req_t *req, int num_packets, const int zcopy, int mcast_group_index, size_t send_offset) { struct ibv_send_wr *swr = &comm->mcast.swr; struct ibv_sge *ssg = &comm->mcast.ssg; size_t offset = (send_offset == SIZE_MAX) ? req->offset : send_offset; int max_commsize = ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE; int max_ag_counter = ONE_SIDED_MAX_ZCOPY_COLL_COUNTER; int i; struct ibv_send_wr *bad_wr; struct pp_packet *pp; int rc; int length; ucc_status_t status; int use_zcopy; ucc_memory_type_t src_mem_type; ucc_memory_type_t dst_mem_type; ucc_assert(mcast_group_index <= comm->mcast_group_count); swr->num_sge = 1; swr->sg_list = & comm->mcast.ssg; swr->opcode = IBV_WR_SEND_WITH_IMM; swr->wr.ud.remote_qpn = MULTICAST_QPN; swr->wr.ud.remote_qkey = DEF_QKEY; swr->next = NULL; for (i = 0; i < num_packets; i++) { if (NULL == (pp = ucc_tl_mlx5_mcast_buf_get_free(comm))) { break; } ucc_assert(pp->context == 0); __builtin_prefetch((void*) pp->buf); __builtin_prefetch(req->ptr + offset); length = (req->to_send == 1) ? (req->length - offset) : comm->max_per_packet; pp->length = length; // generate psn to be used as immediate data /* example: encapsulate packet counter (top 16 bits), collective counter (middle 8 bits), * and source rank (low 8 bits) - assuming max_commsize and * max_ag_counter are 256 */ pp->psn = (req->num_packets - req->to_send)*max_commsize*max_ag_counter + (req->ag_counter % max_ag_counter)*max_commsize + comm->rank; ssg[0].addr = (uintptr_t)req->ptr + offset; /* Enable zero-copy if we have registered CUDA memory, even with staging protocol */ use_zcopy = zcopy || (comm->cuda_mem_enabled && req->mr && req->mr != comm->pp_mr); if (use_zcopy && comm->cuda_mem_enabled && req->mr != comm->pp_mr) { tl_trace(comm->lib, "using zero-copy sending for CUDA memory, length %d", length); } if (!use_zcopy) { src_mem_type = comm->cuda_mem_enabled ? UCC_MEMORY_TYPE_CUDA : UCC_MEMORY_TYPE_HOST; dst_mem_type = UCC_MEMORY_TYPE_HOST; // staging buffer is always HOST status = ucc_tl_mlx5_mcast_memcpy((void*) pp->buf, dst_mem_type, req->ptr + offset, src_mem_type, length, comm); if (ucc_unlikely(status != UCC_OK)) { tl_error(comm->lib, "failed to copy buffer to staging area"); return status; } ssg[0].addr = (uint64_t) pp->buf; ssg[0].lkey = comm->pp_mr->lkey; } else { pp->context = (uintptr_t)req->ptr + offset; ssg[0].lkey = req->mr->lkey; } ssg[0].length = length; swr[0].wr_id = (uint64_t) pp; swr[0].imm_data = htonl(pp->psn); swr[0].send_flags = (length <= comm->max_inline) ? IBV_SEND_INLINE : 0; comm->psn ++; req->to_send --; offset += length; swr[0].send_flags |= IBV_SEND_SIGNALED; comm->pending_send++; swr[0].wr.ud.ah = comm->mcast.groups[mcast_group_index].ah; tl_trace(comm->lib, "mcast post_send, psn %d, length %d, zcopy %d, use_zcopy %d, signaled %d " "qp->state %d qp->qp_num %d qp->pd %p mcast_group_index %d", pp->psn, pp->length, zcopy, use_zcopy, swr[0].send_flags & IBV_SEND_SIGNALED, comm->mcast.groups[mcast_group_index].qp->state, comm->mcast.groups[mcast_group_index].qp->qp_num, comm->mcast.groups[mcast_group_index].qp->pd, mcast_group_index); if (0 != (rc = ibv_post_send(comm->mcast.groups[mcast_group_index].qp, &swr[0], &bad_wr))) { tl_error(comm->lib, "post send failed: ret %d, start_psn %d, to_send %d, " "to_recv %d, length %d, psn %d, inline %d", rc, req->start_psn, req->to_send, req->to_recv, length, pp->psn, length <= comm->max_inline); return UCC_ERR_NO_MESSAGE; } } if (send_offset == SIZE_MAX) { req->offset = offset; } return UCC_OK; } static inline int ucc_tl_mlx5_mcast_recv_collective(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, int num_left, int coll_type) { int max_commsize = ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE; int max_ag_counter = ONE_SIDED_MAX_ZCOPY_COLL_COUNTER; struct pp_packet *pp; struct pp_packet *next; uint64_t id; struct ibv_wc *wc; int num_comp; int i; int real_num_comp; int recv_progressed = 0; int ag_counter; ucc_status_t status; /* check if we have already received something */ ucc_list_for_each_safe(pp, next, &comm->pending_q, super) { ag_counter = (pp->psn / max_commsize) % max_ag_counter; if (ag_counter == (req->ag_counter % max_ag_counter)) { ucc_list_del(&pp->super); status = ucc_tl_mlx5_mcast_process_packet_collective(comm, req, pp, coll_type); if (UCC_OK != status) { tl_error(comm->lib, "process mcast packet failed, status %d", status); return -1; } recv_progressed++; } }; wc = ucc_malloc(sizeof(struct ibv_wc) * POLL_PACKED, "WC"); if (!wc) { recv_progressed = -1; goto exit; } while (num_left > recv_progressed) { memset(wc, 0, sizeof(struct ibv_wc) * POLL_PACKED); num_comp = ibv_poll_cq(comm->mcast.rcq, POLL_PACKED, &wc[0]); if (num_comp < 0) { tl_error(comm->lib, "recv queue poll completion failed %d", num_comp); recv_progressed = -1; goto exit; } else if (num_comp == 0) { break; } if (IBV_WC_SUCCESS != wc[0].status) { tl_error(comm->lib, "mcast_recv: %s err pending_recv %d wr_id %ld num_comp %d byte_len %d\n", ibv_wc_status_str(wc[0].status), comm->pending_recv, wc[0].wr_id, num_comp, wc[0].byte_len); recv_progressed = -1; goto exit; } real_num_comp = num_comp; for (i = 0; i < real_num_comp; i++) { ucc_assert(wc[i].status == IBV_WC_SUCCESS); id = wc[i].wr_id; pp = (struct pp_packet*) (id); pp->length = wc[i].byte_len - GRH_LENGTH; pp->psn = ntohl(wc[i].imm_data); tl_trace(comm->lib, "%d collective pkt completion: psn %d, length %d, " "req_num packets %d, to_send %d, to_recv %d, num_left %d \n", coll_type, pp->psn, pp->length, req->num_packets, req->to_send, req->to_recv, num_left); status = ucc_tl_mlx5_mcast_process_packet_collective(comm, req, pp, coll_type); if (UCC_OK != status) { tl_error(comm->lib, "process mcast packet failed, status %d", status); recv_progressed = -1; goto exit; } recv_progressed++; ucc_assert(pp->qp_id < MAX_GROUP_COUNT); } comm->pending_recv -= num_comp; status = ucc_tl_mlx5_mcast_post_recv_buffers(comm); if (UCC_OK != status) { recv_progressed = -1; goto exit; } } exit: ucc_free(wc); return recv_progressed; } static inline ucc_status_t ucc_tl_mlx5_mcast_poll_recv(ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_status_t status = UCC_OK; struct pp_packet *pp; struct ibv_wc wc; int num_comp; uint64_t id; int length; uint32_t psn; do { num_comp = ibv_poll_cq(comm->mcast.rcq, 1, &wc); if (num_comp > 0) { if (IBV_WC_SUCCESS != wc.status) { tl_error(comm->lib, "mcast_poll_recv: %s err %d num_comp", ibv_wc_status_str(wc.status), num_comp); status = UCC_ERR_NO_MESSAGE; return status; } // Make sure we received all in order. id = wc.wr_id; length = wc.byte_len - GRH_LENGTH; psn = ntohl(wc.imm_data); pp = (struct pp_packet*) id; if (psn >= comm->psn) { ucc_assert(!PSN_RECEIVED(psn, comm)); pp->psn = psn; pp->length = length; ucc_list_add_tail(&comm->pending_q, &pp->super); } else { ucc_assert(pp->context == 0); ucc_list_add_tail(&comm->bpool, &pp->super); } comm->pending_recv--; status = ucc_tl_mlx5_mcast_post_recv_buffers(comm); if (UCC_OK != status) { return status; } } else if (num_comp != 0) { tl_error(comm->lib, "mcast_poll_recv: %d num_comp", num_comp); status = UCC_ERR_NO_MESSAGE; return status; } } while (num_comp); return status; } static inline ucc_status_t ucc_tl_mlx5_mcast_reliable(ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_status_t status = UCC_OK; if (comm->bcast_comm.racks_n != comm->bcast_comm.child_n || comm->bcast_comm.sacks_n != comm->bcast_comm.parent_n || comm->bcast_comm.nack_requests) { if (comm->pending_send) { status = ucc_tl_mlx5_mcast_poll_send(comm); if (UCC_OK != status) { return status; } } if (comm->bcast_comm.parent_n) { status = ucc_tl_mlx5_mcast_poll_recv(comm); if (UCC_OK != status) { return status; } } status = ucc_tl_mlx5_mcast_check_nack_requests(comm, UINT32_MAX); if (UCC_OK != status) { return status; } } if (comm->bcast_comm.parent_n && !comm->bcast_comm.reliable_in_progress) { status = ucc_tl_mlx5_mcast_reliable_send(comm); if (UCC_OK != status) { return status; } } if (!comm->bcast_comm.reliable_in_progress) { comm->bcast_comm.reliable_in_progress = 1; } if (comm->bcast_comm.racks_n == comm->bcast_comm.child_n && comm->bcast_comm.sacks_n == comm->bcast_comm.parent_n && 0 == comm->bcast_comm.nack_requests) { // Reset for next round. memset(comm->bcast_comm.parents, 0, sizeof(comm->bcast_comm.parents)); memset(comm->bcast_comm.children, 0, sizeof(comm->bcast_comm.children)); comm->bcast_comm.racks_n = comm->bcast_comm.child_n = 0; comm->bcast_comm.sacks_n = comm->bcast_comm.parent_n = 0; comm->bcast_comm.reliable_in_progress = 0; return UCC_OK; } return UCC_INPROGRESS; } ucc_status_t ucc_tl_mlx5_probe_ip_over_ib(char* ib_dev_list, struct sockaddr_storage *addr); ucc_status_t ucc_tl_mlx5_setup_mcast(ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc_status_t ucc_tl_mlx5_mcast_init_qps(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc_status_t ucc_tl_mlx5_mcast_setup_qps(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc_status_t ucc_tl_mlx5_mcast_create_rc_qps(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc_status_t ucc_tl_mlx5_mcast_modify_rc_qps(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc_status_t ucc_tl_mlx5_clean_mcast_comm(ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc_status_t ucc_tl_mlx5_mcast_join_mcast_post(ucc_tl_mlx5_mcast_coll_context_t *ctx, struct sockaddr_in6 *net_addr, struct mcast_group *group, int is_root); ucc_status_t ucc_tl_mlx5_mcast_join_mcast_get_event(ucc_tl_mlx5_mcast_coll_context_t *ctx, struct rdma_cm_event **event); ucc_status_t ucc_tl_mlx5_leave_mcast_groups(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_coll_comm_t *comm); #endif /* TL_MLX5_MCAST_HELPER_H_ */ ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_team.c0000664000175000017500000007737615211535620024016 0ustar alastairalastair/** * Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5.h" #include "tl_mlx5_mcast_coll.h" #include "coll_score/ucc_coll_score.h" #include "tl_mlx5_mcast_helper.h" #include "p2p/ucc_tl_mlx5_mcast_p2p.h" #include "mcast/tl_mlx5_mcast_helper.h" #include "mcast/tl_mlx5_mcast_service_coll.h" #include "mcast/tl_mlx5_mcast_one_sided_reliability.h" ucc_status_t ucc_tl_mlx5_mcast_team_init(ucc_base_context_t *base_context, ucc_tl_mlx5_mcast_team_t **mcast_team, ucc_tl_mlx5_mcast_context_t *ctx, const ucc_base_team_params_t *team_params, ucc_tl_mlx5_mcast_coll_comm_init_spec_t *mcast_conf) { ucc_tl_mlx5_mcast_coll_comm_init_spec_t comm_spec = *mcast_conf; ucc_tl_mlx5_mcast_coll_context_t *mcast_context = &(ctx->mcast_context); ucc_tl_mlx5_mcast_coll_comm_init_spec_t *conf_params = &comm_spec; ucc_context_t *context = base_context->ucc_context; ucc_tl_mlx5_context_t *tl_ctx = ucc_derived_of(base_context, ucc_tl_mlx5_context_t); ucc_status_t status; ucc_subset_t set; ucc_tl_mlx5_mcast_team_t *new_mcast_team; ucc_tl_mlx5_mcast_oob_p2p_context_t *oob_p2p_ctx; ucc_tl_mlx5_mcast_coll_comm_t *comm; int i; if (!ctx->mcast_ctx_ready) { tl_debug(base_context->lib, "mcast context not available, base_context = %p", base_context ); return UCC_ERR_NO_RESOURCE; } /* Check if actual mcast resources are available (they might be NULL in FORCE mode) */ if (!mcast_context->ctx || !mcast_context->pd) { tl_mlx5_mcast_log(ctx->mcast_enabled, base_context->lib, UCC_LOG_LEVEL_WARN, "mcast context resources not available, cannot create mcast team"); return UCC_ERR_NO_RESOURCE; } new_mcast_team = ucc_calloc(1, sizeof(ucc_tl_mlx5_mcast_team_t), "new_mcast_team"); if (!new_mcast_team) { return UCC_ERR_NO_MEMORY; } new_mcast_team->mcast_context = ctx; /* init p2p interface */ conf_params->p2p_iface.send_nb = ucc_tl_mlx5_mcast_p2p_send_nb; conf_params->p2p_iface.recv_nb = ucc_tl_mlx5_mcast_p2p_recv_nb; oob_p2p_ctx = ucc_malloc(sizeof(ucc_tl_mlx5_mcast_oob_p2p_context_t), "oob_p2p_ctx"); if (!oob_p2p_ctx) { ucc_free(new_mcast_team); return UCC_ERR_NO_MEMORY; } oob_p2p_ctx->base_ctx = context; oob_p2p_ctx->base_team = team_params->team; oob_p2p_ctx->my_team_rank = team_params->rank; oob_p2p_ctx->lib = mcast_context->lib; set.myrank = team_params->rank; set.map = team_params->map; oob_p2p_ctx->subset = set; conf_params->oob = oob_p2p_ctx; conf_params->sx_sge = 1; conf_params->rx_sge = 2; conf_params->scq_moderation = 64; comm = (ucc_tl_mlx5_mcast_coll_comm_t*) ucc_calloc(1, sizeof(ucc_tl_mlx5_mcast_coll_comm_t) + sizeof(struct pp_packet*)*(conf_params->wsize-1), "ucc_tl_mlx5_mcast_coll_comm_t"); if (!comm) { ucc_free(oob_p2p_ctx); ucc_free(new_mcast_team); return UCC_ERR_NO_MEMORY; } /* Initialize HCA copy resources to NULL */ comm->hca_copy_cq = NULL; comm->hca_copy_qp = NULL; ucc_list_head_init(&comm->bpool); ucc_list_head_init(&comm->pending_q); comm->service_coll.bcast_post = ucc_tl_mlx5_mcast_service_bcast_post; comm->service_coll.allgather_post = ucc_tl_mlx5_mcast_service_allgather_post; comm->service_coll.allreduce_post = ucc_tl_mlx5_mcast_service_allreduce_post; comm->service_coll.barrier_post = ucc_tl_mlx5_mcast_service_barrier_post; comm->service_coll.coll_test = ucc_tl_mlx5_mcast_service_coll_test; memcpy(&comm->params, conf_params, sizeof(*conf_params)); comm->one_sided.reliability_enabled = conf_params->one_sided_reliability_enable; comm->one_sided.reliability_scheme_msg_threshold = conf_params->reliability_scheme_msg_threshold; comm->one_sided.hca_copy_enabled = conf_params->hca_copy_enabled; comm->max_eager = conf_params->max_eager; comm->truly_zero_copy_coll_min_msg = conf_params->truly_zero_copy_coll_min_msg; comm->cuda_mem_enabled = conf_params->cuda_mem_enabled; comm->comm_id = team_params->id; comm->ctx = mcast_context; comm->context = ctx; comm->mcast_group_count = ucc_min(conf_params->mcast_group_count, MAX_GROUP_COUNT); /* only bcast or allgather is supported not both of them together */ if (ctx->mcast_bcast_enabled) { comm->bcast_comm.mcast_prepost_bucket_size = conf_params->mcast_prepost_bucket_size; comm->bcast_comm.wsize = conf_params->wsize; comm->bcast_comm.max_push_send = conf_params->max_push_send; comm->bcast_comm.truly_zero_copy_bcast_enabled = conf_params->truly_zero_copy_bcast_enabled; } else { comm->allgather_comm.mcast_prepost_bucket_size = conf_params->mcast_prepost_bucket_size; comm->allgather_comm.truly_zero_copy_allgather_enabled = conf_params->truly_zero_copy_allgather_enabled; comm->allgather_comm.max_push_send = conf_params->max_push_send; } if (comm->cuda_mem_enabled && !(tl_ctx->supported_mem_types & UCC_BIT(UCC_MEMORY_TYPE_CUDA))) { tl_warn(mcast_context->lib, "cuda-aware mcast not available as gpu direct is not ready"); status = UCC_ERR_NO_RESOURCE; goto cleanup; } comm->mcast.rcq = ibv_create_cq(mcast_context->ctx, comm->params.rx_depth, NULL, NULL, 0); if (!comm->mcast.rcq) { tl_mlx5_mcast_log(mcast_context->params.mcast_enabled, mcast_context->lib, UCC_LOG_LEVEL_ERROR, "could not create recv cq, rx_depth %d, errno %d", comm->params.rx_depth, errno); status = UCC_ERR_NO_RESOURCE; goto cleanup; } comm->mcast.scq = ibv_create_cq(mcast_context->ctx, comm->params.sx_depth, NULL, NULL, 0); if (!comm->mcast.scq) { ibv_destroy_cq(comm->mcast.rcq); tl_mlx5_mcast_log(mcast_context->params.mcast_enabled, mcast_context->lib, UCC_LOG_LEVEL_ERROR, "could not create send cq, sx_depth %d, errno %d", comm->params.sx_depth, errno); status = UCC_ERR_NO_RESOURCE; goto cleanup; } comm->rank = team_params->rank; comm->commsize = team_params->size; comm->max_per_packet = mcast_context->mtu; comm->bcast_comm.last_acked = comm->bcast_comm.last_psn = 0; comm->bcast_comm.racks_n = comm->bcast_comm.sacks_n = 0; comm->bcast_comm.child_n = comm->bcast_comm.parent_n = 0; comm->p2p_ctx = conf_params->oob; memcpy(&comm->p2p, &conf_params->p2p_iface, sizeof(ucc_tl_mlx5_mcast_p2p_interface_t)); comm->dummy_packet.psn = UINT32_MAX; for (i=0; i< comm->bcast_comm.wsize; i++) { comm->r_window[i] = &comm->dummy_packet; } comm->lib = base_context->lib; new_mcast_team->mcast_comm = comm; *mcast_team = new_mcast_team; tl_debug(base_context->lib, "posted tl mcast team : %p", new_mcast_team); return UCC_OK; cleanup: ucc_free(comm); ucc_free(new_mcast_team); ucc_free(oob_p2p_ctx); return status; } ucc_status_t ucc_tl_mlx5_mcast_coll_setup_comm_resources(ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_status_t status; size_t page_size; int buf_size, i, ret; ucc_memory_type_t supported_mem_type; status = ucc_tl_mlx5_mcast_init_qps(comm->ctx, comm); if (UCC_OK != status) { goto error; } status = ucc_tl_mlx5_mcast_setup_qps(comm->ctx, comm); if (UCC_OK != status) { goto error; } page_size = ucc_get_page_size(); buf_size = comm->ctx->mtu; // Comm receiving buffers. ret = posix_memalign((void**)&comm->call_rwr, page_size, sizeof(struct ibv_recv_wr) * comm->params.rx_depth); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "posix_memalign failed"); return UCC_ERR_NO_MEMORY; } ret = posix_memalign((void**)&comm->call_rsgs, page_size, sizeof(struct ibv_sge) * comm->params.rx_depth * 2); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "posix_memalign failed"); return UCC_ERR_NO_MEMORY; } comm->pending_recv = 0; comm->buf_n = comm->params.rx_depth * 2; /* check the impact of creating recv posted buffers on GPU vs CPU */ /* Note: For staging-based operations, we always use HOST memory for staging buffers * to use fast memcpy operations, then do one final cudaMemcpy to user buffer. */ supported_mem_type = UCC_MEMORY_TYPE_HOST; comm->grh_buf = ucc_malloc(GRH_LENGTH * sizeof(char), "grh"); if (ucc_unlikely(!comm->grh_buf)) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "failed to allocate grh memory"); status = UCC_ERR_NO_RESOURCE; goto error; } status = ucc_mc_memset(comm->grh_buf, 0, GRH_LENGTH, UCC_MEMORY_TYPE_HOST); if (status != UCC_OK) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "could not cuda memset"); goto error; } comm->grh_mr = ibv_reg_mr(comm->ctx->pd, comm->grh_buf, GRH_LENGTH, IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE); if (!comm->grh_mr) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "could not register device memory for GRH, errno %d", errno); status = UCC_ERR_NO_RESOURCE; goto error; } status = ucc_mc_alloc(&comm->pp_buf_header, buf_size * comm->buf_n, supported_mem_type); comm->pp_buf = comm->pp_buf_header->addr; if (ucc_unlikely(status != UCC_OK)) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "failed to allocate cuda memory"); goto error; } status = ucc_mc_memset(comm->pp_buf, 0, buf_size * comm->buf_n, supported_mem_type); if (status != UCC_OK) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "could not memset"); goto error; } comm->pp_mr = ibv_reg_mr(comm->ctx->pd, comm->pp_buf, buf_size * comm->buf_n, IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE); if (!comm->pp_mr) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "could not register pp_buf device mr, errno %d", errno); status = UCC_ERR_NO_RESOURCE; goto error; } ret = posix_memalign((void**) &comm->pp, page_size, sizeof(struct pp_packet) * comm->buf_n); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "posix_memalign failed"); return UCC_ERR_NO_MEMORY; } for (i = 0; i < comm->buf_n; i++) { ucc_list_head_init(&comm->pp[i].super); comm->pp[i].buf = (uintptr_t) comm->pp_buf + i * buf_size; comm->pp[i].context = 0; ucc_list_add_tail(&comm->bpool, &comm->pp[i].super); } comm->mcast.swr.num_sge = 1; comm->mcast.swr.sg_list = &comm->mcast.ssg; comm->mcast.swr.opcode = IBV_WR_SEND_WITH_IMM; comm->mcast.swr.wr.ud.remote_qpn = MULTICAST_QPN; comm->mcast.swr.wr.ud.remote_qkey = DEF_QKEY; comm->mcast.swr.next = NULL; for (i = 0; i < comm->params.rx_depth; i++) { comm->call_rwr[i].sg_list = &comm->call_rsgs[2 * i]; comm->call_rwr[i].num_sge = 2; comm->call_rwr[i].wr_id = MCAST_BCASTRECV_WR; comm->call_rsgs[2 * i].length = GRH_LENGTH; comm->call_rsgs[2 * i].addr = (uintptr_t)comm->grh_buf; comm->call_rsgs[2 * i].lkey = comm->grh_mr->lkey; comm->call_rsgs[2 * i + 1].lkey = comm->pp_mr->lkey; comm->call_rsgs[2 * i + 1].length = comm->max_per_packet; } status = ucc_tl_mlx5_mcast_post_recv_buffers(comm); if (UCC_OK != status) { goto error; } memset(comm->bcast_comm.parents, 0, sizeof(comm->bcast_comm.parents)); memset(comm->bcast_comm.children, 0, sizeof(comm->bcast_comm.children)); comm->bcast_comm.nacks_counter = 0; comm->bcast_comm.n_mcast_reliable = 0; comm->bcast_comm.reliable_in_progress = 0; comm->bcast_comm.recv_drop_packet_in_progress = 0; comm->tx = 0; /* Mark transport ready on this rank */ comm->mcast_transport_ready = 1; /* init one-sided reliability after transport ready */ if (comm->one_sided.reliability_enabled) { status = ucc_tl_mlx5_mcast_one_sided_reliability_init(comm); if ((status != UCC_OK) && (status != UCC_ERR_NOT_SUPPORTED)) { return status; } } return UCC_OK; error: /* Skip cleanup here: resources may be only partially initialized and * ucc_tl_mlx5_clean_mcast_comm expects fully initialized structures. * The higher-level fallback logic will dispose of @comm later if needed. */ return status; } static inline ucc_status_t ucc_tl_mlx5_mcast_process_comm_event(ucc_base_team_t *team) { ucc_tl_mlx5_team_t *tl_team = ucc_derived_of(team, ucc_tl_mlx5_team_t); ucc_tl_mlx5_mcast_coll_comm_t *comm = tl_team->mcast->mcast_comm; ucc_status_t status = UCC_OK; struct mcast_group *group; int i; for (i = 0; i < comm->mcast_group_count; i++) { if (comm->mcast.groups[i].lid != 0) { continue; } status = ucc_tl_mlx5_mcast_join_mcast_get_event(comm->ctx, &comm->event); if (status != UCC_OK) { goto failed; } if (!comm->event) { tl_error(comm->lib, "received NULL event after successful get_event call"); status = UCC_ERR_NO_MESSAGE; goto failed; } group = (struct mcast_group *)comm->event->param.ud.private_data; ucc_assert(group != NULL); tl_debug(comm->lib, "found mcast group %p info in the retreived mcast join event", group); group->lid = comm->event->param.ud.ah_attr.dlid; group->mgid = comm->event->param.ud.ah_attr.grh.dgid; if (rdma_ack_cm_event(comm->event) < 0) { tl_error(comm->lib, "rdma_ack_cm_event failed"); status = UCC_ERR_NO_MESSAGE; comm->event = NULL; goto failed; } comm->event = NULL; } tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_READY; return UCC_OK; failed: if (status < 0) { tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_FAILED; } return status; } ucc_status_t ucc_tl_mlx5_mcast_team_test(ucc_base_team_t *team) { ucc_tl_mlx5_team_t *tl_team = ucc_derived_of(team, ucc_tl_mlx5_team_t); ucc_status_t status = UCC_OK; struct sockaddr_in6 net_addr = {0,}; ucc_tl_mlx5_mcast_join_info_t *data = NULL; ucc_tl_mlx5_mcast_coll_comm_t *comm = tl_team->mcast->mcast_comm; int i; if (comm->rank == 0) { switch(tl_team->mcast_state) { case TL_MLX5_TEAM_STATE_MCAST_INIT: { for (i = 0; i < comm->mcast_group_count; i++) { net_addr.sin6_family = AF_INET6; net_addr.sin6_flowinfo = comm->comm_id + i + 1; status = ucc_tl_mlx5_mcast_join_mcast_post(comm->ctx, &net_addr, &comm->mcast.groups[i], 1); if (status < 0) { tl_error(comm->lib, "rank 0 is unable to join mcast group %d error %d", i, status); tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_FAILED; return UCC_INPROGRESS; } comm->mcast.groups[i].mcast_addr = net_addr; } tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_TEST; return UCC_INPROGRESS; } case TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_TEST: { status = ucc_tl_mlx5_mcast_process_comm_event(team); if (status < UCC_OK) { tl_error(comm->lib, "mcast_process_comm_event failed"); } return UCC_INPROGRESS; } case TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_READY: case TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_FAILED: { data = ucc_calloc(1, sizeof(ucc_tl_mlx5_mcast_join_info_t), "ucc_tl_mlx5_mcast_join_info_t"); if (!data) { tl_error(comm->lib, "unable to allocate memory for group setup info"); return UCC_ERR_NO_MEMORY; } comm->group_setup_info = data; if (tl_team->mcast_state == TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_READY) { /* rank 0 bcast the lid/gid of all the groups to other processes */ data->status = UCC_OK; for (i = 0; i < comm->mcast_group_count; i++) { data->dgid[i] = comm->mcast.groups[i].mgid; data->dlid[i] = comm->mcast.groups[i].lid; } } else { /* rank 0 bcast the failed status to other processes so others do not hang */ status = ucc_tl_mlx5_leave_mcast_groups(comm->ctx, comm); if (status) { tl_error(comm->lib, "couldn't leave mcast group"); } if (comm->group_setup_info) { ucc_free(comm->group_setup_info); comm->group_setup_info = NULL; } data->status = UCC_ERR_NO_RESOURCE; } status = comm->service_coll.bcast_post(comm->p2p_ctx, data, sizeof(ucc_tl_mlx5_mcast_join_info_t), 0, &comm->group_setup_info_req); if (UCC_OK != status) { tl_error(comm->lib, "unable to post bcast for group setup info"); ucc_free(comm->group_setup_info); comm->group_setup_info = NULL; return status; } tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_GRP_BCAST_POST; return UCC_INPROGRESS; } case TL_MLX5_TEAM_STATE_MCAST_GRP_BCAST_POST: { /* rank 0 polls bcast request and wait for its completion */ status = comm->service_coll.coll_test(comm->group_setup_info_req); if (UCC_OK != status) { /* bcast is not completed yet */ if (status < 0) { ucc_free(comm->group_setup_info); comm->group_setup_info = NULL; } return status; } if (comm->group_setup_info && comm->group_setup_info->status != UCC_OK) { /* rank 0 was not able to join a mcast group so all * the ranks should return */ ucc_free(comm->group_setup_info); comm->group_setup_info = NULL; return UCC_ERR_NO_RESOURCE; } if (comm->group_setup_info) { ucc_free(comm->group_setup_info); comm->group_setup_info = NULL; } /* Try to setup per-rank multicast communication resources. If this * fails on any rank we still need to participate in the global * readiness allreduce so that OTHER ranks learn about the * failure and gracefully fall back. Therefore, don’t return * immediately on error – instead record the failure by keeping * mcast_transport_ready at 0 and continue. On success the helper * already sets mcast_transport_ready to 1. */ status = ucc_tl_mlx5_mcast_coll_setup_comm_resources(comm); if (UCC_OK != status) { tl_debug(comm->lib, "mcast_coll_setup_comm_resources failed on rank %d, will fallback", comm->rank); /* Ensure we report failure in the upcoming MIN-allreduce. */ comm->mcast_transport_ready = 0; } /* post readiness allreduce */ status = comm->service_coll.allreduce_post(comm->p2p_ctx, &comm->mcast_transport_ready, &comm->transport_ready_global, 1, UCC_DT_INT32, UCC_OP_MIN, &comm->transport_ready_req); if (status != UCC_OK) { return status; } tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_RELIAB_SYNC; return UCC_INPROGRESS; } case TL_MLX5_TEAM_STATE_MCAST_RELIAB_SYNC: { status = comm->service_coll.coll_test(comm->transport_ready_req); if (UCC_OK != status) { if (status < 0) { return status; } return status; } /* request was finalized inside coll_test; reset pointer */ comm->transport_ready_req = NULL; if (comm->transport_ready_global == 0) { /* Some ranks failed to set up multicast transport: release any * partial resources created locally so that PD can be * deallocated cleanly during context teardown. */ ucc_tl_mlx5_clean_mcast_comm(comm); return UCC_ERR_NO_RESOURCE; } tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_RELIABLITY; return UCC_INPROGRESS; } case TL_MLX5_TEAM_STATE_MCAST_RELIABLITY: { if (comm->one_sided.reliability_enabled) { status = ucc_tl_mlx5_mcast_one_sided_reliability_test(comm); if (UCC_OK != status) { if (status < 0) { tl_debug(comm->lib, "reliability test failed on rank %d", comm->rank); return status; } else { /* Still in progress */ return status; } } tl_debug(comm->lib, "reliability test succeeded on rank %d", comm->rank); } else { tl_debug(comm->lib, "reliability disabled, skipping test on rank %d", comm->rank); } tl_debug(comm->lib, "initialized tl mcast team: %p", tl_team); tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_READY; } case TL_MLX5_TEAM_STATE_MCAST_READY: case TL_MLX5_TEAM_STATE_MCAST_NOT_AVAILABLE: { return UCC_OK; } default: { tl_error(comm->lib, "unknown state during mcast team: %p create", tl_team); return UCC_ERR_NO_RESOURCE; } } } else { /* none rank 0 team create states */ switch(tl_team->mcast_state) { case TL_MLX5_TEAM_STATE_MCAST_INIT: { /* none 0 ranks bcast post to wait for rank 0 for lid/gid * of the mcast group */ data = ucc_calloc(1, sizeof(ucc_tl_mlx5_mcast_join_info_t), "ucc_tl_mlx5_mcast_join_info_t"); if (!data) { tl_error(comm->lib, "unable to allocate memory for group setup info"); return UCC_ERR_NO_MEMORY; } status = comm->service_coll.bcast_post(comm->p2p_ctx, data, sizeof(ucc_tl_mlx5_mcast_join_info_t), 0, &comm->group_setup_info_req); if (UCC_OK != status) { tl_error(comm->lib, "unable to post bcast for group setup info"); ucc_free(data); return status; } comm->group_setup_info = data; tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_GRP_BCAST_POST; return UCC_INPROGRESS; } case TL_MLX5_TEAM_STATE_MCAST_GRP_BCAST_POST: { /* none rank 0 processes poll bcast request and wait for its completion */ status = comm->service_coll.coll_test(comm->group_setup_info_req); if (UCC_OK != status) { /* bcast is not completed yet */ if (status < 0) { ucc_free(comm->group_setup_info); comm->group_setup_info = NULL; } return status; } data = comm->group_setup_info; if (!data) { tl_error(comm->lib, "group_setup_info is NULL after successful coll_test"); return UCC_ERR_NO_RESOURCE; } status = data->status; if (UCC_OK != status) { /* rank 0 was not able to join a mcast group so all * the ranks should return */ ucc_free(data); comm->group_setup_info = NULL; return status; } /* now it is time for none rank 0 to call rdma_join_multicast() * for all the mcast groups */ for (i = 0; i < comm->mcast_group_count; i++) { memcpy(&net_addr.sin6_addr, &(data->dgid[i]), sizeof(struct in6_addr)); net_addr.sin6_family = AF_INET6; status = ucc_tl_mlx5_mcast_join_mcast_post(comm->ctx, &net_addr, &comm->mcast.groups[i], 0); if (status < 0) { tl_error(comm->lib, "none-root rank is unable to join mcast group error %d", status); ucc_free(data); comm->group_setup_info = NULL; return status; } comm->mcast.groups[i].mcast_addr = net_addr; } tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_TEST; return UCC_INPROGRESS; } case TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_TEST: { status = ucc_tl_mlx5_mcast_process_comm_event(team); if (status < UCC_OK) { tl_error(comm->lib, "mcast_process_comm_event failed"); } return UCC_INPROGRESS; } case TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_FAILED: { status = ucc_tl_mlx5_leave_mcast_groups(comm->ctx, comm); if (status) { tl_error(comm->lib, "couldn't leave mcast group"); } return UCC_ERR_NO_RESOURCE; } case TL_MLX5_TEAM_STATE_MCAST_GRP_JOIN_READY: { status = ucc_tl_mlx5_mcast_coll_setup_comm_resources(comm); if (UCC_OK != status) { tl_debug(comm->lib, "mcast_coll_setup_comm_resources failed on rank %d, will fallback", comm->rank); comm->mcast_transport_ready = 0; } status = comm->service_coll.allreduce_post(comm->p2p_ctx, &comm->mcast_transport_ready, &comm->transport_ready_global, 1, UCC_DT_INT32, UCC_OP_MIN, &comm->transport_ready_req); if (status != UCC_OK) { return status; } tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_RELIAB_SYNC; return UCC_INPROGRESS; } case TL_MLX5_TEAM_STATE_MCAST_RELIAB_SYNC: { status = comm->service_coll.coll_test(comm->transport_ready_req); if (UCC_OK != status) { if (status < 0) { return status; } return status; } /* request finalized inside coll_test */ comm->transport_ready_req = NULL; if (comm->transport_ready_global == 0) { ucc_tl_mlx5_clean_mcast_comm(comm); return UCC_ERR_NO_RESOURCE; } tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_RELIABLITY; return UCC_INPROGRESS; } case TL_MLX5_TEAM_STATE_MCAST_RELIABLITY: { if (comm->one_sided.reliability_enabled) { status = ucc_tl_mlx5_mcast_one_sided_reliability_test(comm); if (UCC_OK != status) { if (status < 0) { tl_debug(comm->lib, "reliability test failed on rank %d", comm->rank); return status; } else { /* Still in progress */ return status; } } tl_debug(comm->lib, "reliability test succeeded on rank %d", comm->rank); } else { tl_debug(comm->lib, "reliability disabled, skipping test on rank %d", comm->rank); } tl_debug(comm->lib, "initialized tl mcast team: %p", tl_team); tl_team->mcast_state = TL_MLX5_TEAM_STATE_MCAST_READY; } case TL_MLX5_TEAM_STATE_MCAST_READY: case TL_MLX5_TEAM_STATE_MCAST_NOT_AVAILABLE: { return UCC_OK; } default: { tl_error(comm->lib, "unknown state during mcast team: %p create", tl_team); return UCC_ERR_NO_RESOURCE; } } } } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_one_sided_reliability.c0000664000175000017500000002065615211535620027377 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_mcast_one_sided_reliability.h" inline static ucc_status_t ucc_tl_mlx5_mcast_one_sided_setup_reliability_buffers(ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_status_t status = UCC_OK; int one_sided_total_slots_size, i; /* this array keeps track of the number of recv packets from each process * used in all the protocols */ comm->one_sided.recvd_pkts_tracker = ucc_calloc(1, comm->commsize * sizeof(uint32_t), "one_sided.recvd_pkts_tracker"); if (!comm->one_sided.recvd_pkts_tracker) { tl_error(comm->lib, "unable to malloc for one_sided.recvd_pkts_tracker"); status = UCC_ERR_NO_MEMORY; goto failed; } comm->one_sided.sendbuf_memkey_list = ucc_calloc (1, comm->commsize * sizeof(ucc_tl_mlx5_mcast_slot_mem_info_t), "one_sided.sendbuf_memkey_list"); if (!comm->one_sided.sendbuf_memkey_list) { tl_error(comm->lib, "unable to malloc for one_sided.sendbuf_memkey_list"); status = UCC_ERR_NO_MEMORY; goto failed; } /* below data structures are used in async design only */ comm->one_sided.slot_size = comm->one_sided.reliability_scheme_msg_threshold + ONE_SIDED_SLOTS_INFO_SIZE; one_sided_total_slots_size = comm->one_sided.slot_size * ONE_SIDED_SLOTS_COUNT * sizeof(char); comm->one_sided.slots_buffer = (char *)ucc_calloc(1, one_sided_total_slots_size, "one_sided.slots_buffer"); if (!comm->one_sided.slots_buffer) { tl_error(comm->lib, "unable to malloc for one_sided.slots_buffer"); status = UCC_ERR_NO_MEMORY; goto failed; } comm->one_sided.slots_mr = ibv_reg_mr(comm->ctx->pd, comm->one_sided.slots_buffer, one_sided_total_slots_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE); if (!comm->one_sided.slots_mr) { tl_error(comm->lib, "unable to register for one_sided.slots_mr"); status = UCC_ERR_NO_RESOURCE; goto failed; } /* this array holds local information about the slot status that was read from remote ranks */ comm->one_sided.remote_slot_info = (int *)ucc_calloc(comm->commsize, ONE_SIDED_SLOTS_INFO_SIZE, "one_sided.remote_slot_info"); if (!comm->one_sided.remote_slot_info) { tl_error(comm->lib, "unable to malloc for one_sided.remote_slot_info"); status = UCC_ERR_NO_MEMORY; goto failed; } comm->one_sided.remote_slot_info_mr = ibv_reg_mr(comm->ctx->pd, comm->one_sided.remote_slot_info, comm->commsize * ONE_SIDED_SLOTS_INFO_SIZE, IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ); if (!comm->one_sided.remote_slot_info_mr) { tl_error(comm->lib, "unable to register for one_sided.remote_slot_info_mr"); status = UCC_ERR_NO_RESOURCE; goto failed; } comm->one_sided.info = ucc_calloc(1, sizeof(ucc_tl_mlx5_one_sided_reliable_team_info_t) * comm->commsize, "one_sided.info"); if (!comm->one_sided.info) { tl_error(comm->lib, "unable to allocate mem for one_sided.info"); status = UCC_ERR_NO_MEMORY; goto failed; } status = ucc_tl_mlx5_mcast_create_rc_qps(comm->ctx, comm); if (UCC_OK != status) { tl_error(comm->lib, "RC qp create failed"); goto failed; } /* below holds the remote addr/rkey to local slot field of all the * processes used in async protocol */ comm->one_sided.info[comm->rank].slot_mem.rkey = comm->one_sided.slots_mr->rkey; comm->one_sided.info[comm->rank].slot_mem.remote_addr = (uint64_t)comm->one_sided.slots_buffer; comm->one_sided.info[comm->rank].port_lid = comm->ctx->port_lid; for (i = 0; i < comm->commsize; i++) { comm->one_sided.info[comm->rank].rc_qp_num[i] = comm->mcast.rc_qp[i]->qp_num; } return UCC_OK; failed: return status; } ucc_status_t ucc_tl_mlx5_mcast_one_sided_cleanup(ucc_tl_mlx5_mcast_coll_comm_t *comm) { int j; if (comm->mcast.rc_qp != NULL) { for (j=0; jcommsize; j++) { if (comm->mcast.rc_qp[j] != NULL && ibv_destroy_qp(comm->mcast.rc_qp[j])) { tl_error(comm->lib, "ibv_destroy_qp failed"); return UCC_ERR_NO_RESOURCE; } comm->mcast.rc_qp[j] = NULL; } ucc_free(comm->mcast.rc_qp); comm->mcast.rc_qp = NULL; } if (comm->mcast.srq != NULL && ibv_destroy_srq(comm->mcast.srq)) { tl_error(comm->lib, "ibv_destroy_srq failed"); return UCC_ERR_NO_RESOURCE; } comm->mcast.srq = NULL; if (comm->one_sided.slots_mr) { ibv_dereg_mr(comm->one_sided.slots_mr); comm->one_sided.slots_mr = 0; } if (comm->one_sided.remote_slot_info_mr) { ibv_dereg_mr(comm->one_sided.remote_slot_info_mr); comm->one_sided.remote_slot_info_mr = 0; } if (comm->one_sided.slots_buffer) { ucc_free(comm->one_sided.slots_buffer); comm->one_sided.slots_buffer = NULL; } if (comm->one_sided.recvd_pkts_tracker) { ucc_free(comm->one_sided.recvd_pkts_tracker); comm->one_sided.recvd_pkts_tracker = NULL; } if (comm->one_sided.sendbuf_memkey_list) { ucc_free(comm->one_sided.sendbuf_memkey_list); comm->one_sided.sendbuf_memkey_list = NULL; } if (comm->one_sided.remote_slot_info) { ucc_free(comm->one_sided.remote_slot_info); comm->one_sided.remote_slot_info = NULL; } if (comm->one_sided.info) { ucc_free(comm->one_sided.info); comm->one_sided.info = NULL; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_one_sided_reliability_init(ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_status_t status = UCC_OK; if (!comm->one_sided.reliability_enabled) { return UCC_OK; } if (comm->commsize > ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE) { tl_warn(comm->lib, "team size is %d but max supported team size of mcast one-sided reliability is %d", comm->commsize, ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE); return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_mlx5_mcast_one_sided_setup_reliability_buffers(comm); if (status != UCC_OK) { tl_error(comm->lib, "setup reliability resources failed"); } return status; } ucc_status_t ucc_tl_mlx5_mcast_one_sided_reliability_test(ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_status_t status = UCC_OK; if (!comm->one_sided.reliability_enabled) { return UCC_OK; } if (comm->one_sided.reliability_req == NULL) { status = comm->service_coll.allgather_post(comm->p2p_ctx, &(comm->one_sided.info[comm->rank]), comm->one_sided.info, sizeof(ucc_tl_mlx5_one_sided_reliable_team_info_t), &comm->one_sided.reliability_req); if (UCC_OK != status) { tl_error(comm->lib, "oob allgather failed during one-sided reliability init"); goto failed; } } status = comm->service_coll.coll_test(comm->one_sided.reliability_req); if (UCC_OK != status) { if (status < 0) { tl_error(comm->lib, "one sided config info exchange failed"); goto failed; } return status; } /* we have all the info to make the reliable connections */ status = ucc_tl_mlx5_mcast_modify_rc_qps(comm->ctx, comm); if (UCC_OK != status) { tl_error(comm->lib, "RC qp modify failed"); goto failed; } tl_debug(comm->lib, "support for allgather reliability is enabled"); comm->one_sided.reliability_req = NULL; return UCC_OK; failed: if (UCC_OK != ucc_tl_mlx5_mcast_one_sided_cleanup(comm)) { tl_error(comm->lib, "mcast one-sided reliability resource cleanup failed"); } return status; } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_hca_copy.h0000664000175000017500000000371215211535620024641 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_MLX5_MCAST_HCA_COPY_H_ #define UCC_TL_MLX5_MCAST_HCA_COPY_H_ #include "tl_mlx5_mcast.h" typedef struct ucc_tl_mlx5_mcast_hca_copy_task { void *dst; void *src; size_t size; ucc_memory_type_t dst_mtype; ucc_memory_type_t src_mtype; ucc_tl_mlx5_mcast_coll_comm_t *comm; /* Use RDMA write for HCA copy with rcache optimization */ ucc_tl_mlx5_mcast_reg_t *src_reg; ucc_tl_mlx5_mcast_reg_t *dst_reg; struct ibv_send_wr rdma_wr; struct ibv_sge rdma_sge; volatile int completed; ucc_status_t status; ucc_rank_t target_rank; } ucc_tl_mlx5_mcast_hca_copy_task_t; /* HCA copy API functions */ ucc_status_t ucc_tl_mlx5_mcast_hca_copy_post(void *dst, ucc_memory_type_t dst_mtype, void *src, ucc_memory_type_t src_mtype, size_t size, ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_hca_copy_task_t **copy_task); ucc_status_t ucc_tl_mlx5_mcast_hca_copy_test(ucc_tl_mlx5_mcast_hca_copy_task_t *copy_task); ucc_status_t ucc_tl_mlx5_mcast_hca_copy_finalize(ucc_tl_mlx5_mcast_hca_copy_task_t *copy_task); /* Helper function to choose between HCA copy and mc copy */ ucc_status_t ucc_tl_mlx5_mcast_memcpy(void *dst, ucc_memory_type_t dst_mtype, void *src, ucc_memory_type_t src_mtype, size_t size, ucc_tl_mlx5_mcast_coll_comm_t *comm); #endif /* UCC_TL_MLX5_MCAST_HCA_COPY_H_ */ ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_context.c0000664000175000017500000002775315211535620024546 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include #include "tl_mlx5_mcast.h" #include "utils/arch/cpu.h" #include #include "core/ucc_service_coll.h" #include "tl_mlx5.h" #include "tl_mlx5_mcast_helper.h" #include "tl_mlx5_mcast_rcache.h" #define UCC_TL_MLX5_MCAST_MAX_MTU_COUNT 5 int mtu_lookup[UCC_TL_MLX5_MCAST_MAX_MTU_COUNT][2] = { {256, IBV_MTU_256}, {512, IBV_MTU_512}, {1024, IBV_MTU_1024}, {2048, IBV_MTU_2048}, {4096, IBV_MTU_4096} }; ucc_status_t ucc_tl_mlx5_mcast_context_init(ucc_tl_mlx5_mcast_context_t *context, ucc_tl_mlx5_mcast_ctx_params_t *mcast_ctx_conf) { ucc_status_t status = UCC_OK; struct ibv_device **device_list = NULL; struct ibv_device *dev = NULL; char *devname = NULL; int is_ipv4 = 0; struct sockaddr_in *in_src_addr = NULL; struct rdma_cm_event *revent = NULL; int active_mtu = 4096; int max_mtu = 4096; ucc_tl_mlx5_mcast_coll_context_t *ctx = NULL; char *ib_devname = NULL; int devname_len = 0, ib_port = 1; struct ibv_port_attr port_attr; struct ibv_device_attr device_attr; struct sockaddr_storage ip_oib_addr; struct sockaddr_storage dst_addr; int num_devices; char addrstr[128]; ucc_tl_mlx5_context_t *mlx5_ctx; ucc_base_lib_t *lib; int i; int ib_valid; const char *dst; char tmp[128], *pos, *end_pos; mlx5_ctx = ucc_container_of(context, ucc_tl_mlx5_context_t, mcast); lib = mlx5_ctx->super.super.lib; context->mcast_enabled = mcast_ctx_conf->mcast_enabled; context->mcast_bcast_enabled = mcast_ctx_conf->mcast_bcast_enabled; context->mcast_allgather_enabled = mcast_ctx_conf->mcast_allgather_enabled; if (context->mcast_bcast_enabled && context->mcast_allgather_enabled) { /* only a single collective type is supported at a time */ context->mcast_allgather_enabled = 0; } if (mcast_ctx_conf->mcast_enabled == UCC_NO) { tl_debug(lib, "Mcast is disabled by the user"); return UCC_ERR_NO_RESOURCE; } ctx = &(context->mcast_context); memset(ctx, 0, sizeof(ucc_tl_mlx5_mcast_coll_context_t)); memcpy(&ctx->params, mcast_ctx_conf, sizeof(ucc_tl_mlx5_mcast_ctx_params_t)); ctx->lib = lib; /* TODO unify all the contexts under TL mlx5 */ device_list = ibv_get_device_list(&num_devices); if (!device_list || !num_devices) { tl_debug(lib, "no ib devices available"); status = UCC_ERR_NOT_SUPPORTED; goto error; } if (!strcmp(mcast_ctx_conf->ib_dev_name, "")) { dev = device_list[0]; devname = (char *)ibv_get_device_name(dev); ctx->devname = ucc_malloc(strlen(devname)+3, "devname"); if (!ctx->devname) { status = UCC_ERR_NO_MEMORY; goto error; } memset(ctx->devname, 0, strlen(devname)+3); memcpy(ctx->devname, devname, strlen(devname)); strncat(ctx->devname, ":1", 3); ctx->user_provided_ib = 0; ctx->ib_port = 1; } else { ib_valid = 0; /* user has provided the devname now make sure it is valid */ /* check if port number is also included and extract devname from user str */ ib_devname = mcast_ctx_conf->ib_dev_name; pos = strstr(ib_devname, ":"); if (!pos) { devname_len = sizeof(tmp) - 1; } else { devname_len = (int)(pos - ib_devname); pos++; errno = 0; ib_port = (int)strtol(pos, &end_pos, 0); if (errno != 0 || pos == end_pos || strcmp(end_pos,"\0") || ib_port < 0 || ib_port > UINT8_MAX ) { tl_warn(lib, "wrong device's port number"); return UCC_ERR_INVALID_PARAM; } } ctx->ib_port = ib_port; strncpy(tmp, ib_devname, devname_len); tmp[devname_len] = '\0'; ib_devname = tmp; for (i = 0; device_list[i]; ++i) { if (!strcmp(ibv_get_device_name(device_list[i]), ib_devname)) { ib_valid = 1; break; } } if (!ib_valid) { tl_warn(lib, "ib device %s not found", mcast_ctx_conf->ib_dev_name); status = UCC_ERR_NOT_FOUND; ibv_free_device_list(device_list); goto error; } ctx->devname = mcast_ctx_conf->ib_dev_name; ctx->user_provided_ib = 1; } ibv_free_device_list(device_list); status = ucc_tl_mlx5_probe_ip_over_ib(ctx->devname, &ip_oib_addr); if (UCC_OK != status) { tl_debug(lib, "failed to get ipoib interface for devname %s", ctx->devname); if (!ctx->user_provided_ib) { ucc_free(ctx->devname); } goto error; } is_ipv4 = (ip_oib_addr.ss_family == AF_INET) ? 1 : 0; in_src_addr = (struct sockaddr_in*)&ip_oib_addr; dst = inet_ntop((is_ipv4) ? AF_INET : AF_INET6, &in_src_addr->sin_addr, addrstr, sizeof(addrstr) - 1); if (NULL == dst) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "inet_ntop failed"); status = UCC_ERR_NO_RESOURCE; goto error; } tl_debug(ctx->lib, "devname %s, ipoib %s", ctx->devname, addrstr); ctx->channel = rdma_create_event_channel(); if (!ctx->channel) { tl_debug(lib, "rdma_create_event_channel failed, errno %d", errno); status = UCC_ERR_NO_RESOURCE; goto error; } memset(&dst_addr, 0, sizeof(struct sockaddr_storage)); dst_addr.ss_family = is_ipv4 ? AF_INET : AF_INET6; if (rdma_create_id(ctx->channel, &ctx->id, NULL, RDMA_PS_UDP)) { tl_debug(lib, "failed to create rdma id, errno %d", errno); status = UCC_ERR_NOT_SUPPORTED; goto error; } if (0 != rdma_resolve_addr(ctx->id, (struct sockaddr *)&ip_oib_addr, (struct sockaddr *) &dst_addr, 1000)) { tl_debug(lib, "failed to resolve rdma addr, errno %d", errno); status = UCC_ERR_NOT_SUPPORTED; goto error; } if (rdma_get_cm_event(ctx->channel, &revent) < 0) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "failed to get cm event, errno %d", errno); status = UCC_ERR_NO_RESOURCE; goto error; } else if (revent->event != RDMA_CM_EVENT_ADDR_RESOLVED) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "cm event is not resolved"); if (rdma_ack_cm_event(revent) < 0) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "rdma_ack_cm_event failed"); } status = UCC_ERR_NO_RESOURCE; goto error; } if (rdma_ack_cm_event(revent) < 0) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "rdma_ack_cm_event failed"); status = UCC_ERR_NO_RESOURCE; goto error; } ctx->ctx = ctx->id->verbs; ctx->pd = ibv_alloc_pd(ctx->ctx); if (!ctx->pd) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "failed to allocate pd"); status = UCC_ERR_NO_RESOURCE; goto error; } /* Determine MTU */ if (ibv_query_port(ctx->ctx, ctx->ib_port, &port_attr)) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "couldn't query port in ctx create, errno %d", errno); status = UCC_ERR_NO_RESOURCE; goto error; } for (i = 0; i < UCC_TL_MLX5_MCAST_MAX_MTU_COUNT; i++) { if (mtu_lookup[i][1] == port_attr.max_mtu) { max_mtu = mtu_lookup[i][0]; } if (mtu_lookup[i][1] == port_attr.active_mtu) { active_mtu = mtu_lookup[i][0]; } } ctx->mtu = active_mtu; ctx->port_lid = port_attr.lid; tl_debug(ctx->lib, "port active MTU is %d and port max MTU is %d", active_mtu, max_mtu); if (port_attr.max_mtu < port_attr.active_mtu) { tl_debug(ctx->lib, "port active MTU (%d) is smaller than port max MTU (%d)", active_mtu, max_mtu); } if (ibv_query_device(ctx->ctx, &device_attr)) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "failed to query device in ctx create, errno %d", errno); status = UCC_ERR_NO_RESOURCE; goto error; } tl_debug(ctx->lib, "MTU %d, MAX QP WR: %d, max sqr_wr: %d, max cq: %d, max cqe: %d", ctx->mtu, device_attr.max_qp_wr, device_attr.max_srq_wr, device_attr.max_cq, device_attr.max_cqe); ctx->max_qp_wr = device_attr.max_qp_wr; status = ucc_mpool_init(&ctx->compl_objects_mp, 0, sizeof(ucc_tl_mlx5_mcast_p2p_completion_obj_t), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_coll_task_mpool_ops, UCC_THREAD_SINGLE, "ucc_tl_mlx5_mcast_p2p_completion_obj_t"); if (ucc_unlikely(UCC_OK != status)) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "failed to initialize compl_objects_mp mpool"); status = UCC_ERR_NO_MEMORY; goto error; } status = ucc_mpool_init(&ctx->mcast_req_mp, 0, sizeof(ucc_tl_mlx5_mcast_coll_req_t), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_coll_task_mpool_ops, UCC_THREAD_SINGLE, "ucc_tl_mlx5_mcast_coll_req_t"); if (ucc_unlikely(UCC_OK != status)) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "failed to initialize mcast_req_mp mpool"); status = UCC_ERR_NO_MEMORY; goto error; } ctx->rcache = NULL; status = ucc_tl_mlx5_mcast_setup_rcache(ctx); if (UCC_OK != status) { tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "failed to setup rcache"); goto error; } tl_debug(ctx->lib, "multicast context setup complete: ctx %p", ctx); return UCC_OK; error: if (ctx->pd) { ibv_dealloc_pd(ctx->pd); ctx->pd = NULL; } if (ctx->id) { rdma_destroy_id(ctx->id); ctx->id = NULL; } if (ctx->channel) { rdma_destroy_event_channel(ctx->channel); ctx->channel = NULL; } /* Context initialization failed */ tl_mlx5_mcast_log(context->mcast_enabled, lib, UCC_LOG_LEVEL_WARN, "mcast context initialization failed (status: %d)", status); return UCC_ERR_NO_RESOURCE; } ucc_status_t ucc_tl_mlx5_mcast_clean_ctx(ucc_tl_mlx5_mcast_coll_context_t *ctx) { tl_debug(ctx->lib, "cleaning mcast ctx: %p", ctx); if (ctx->rcache) { ucc_rcache_destroy(ctx->rcache); ctx->rcache = NULL; } if (ctx->pd) { if (ibv_dealloc_pd(ctx->pd)) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_ERROR, "ibv_dealloc_pd failed errno %d", errno); return UCC_ERR_NO_RESOURCE; } ctx->pd = NULL; } if (ctx->id && rdma_destroy_id(ctx->id)) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_ERROR, "rdma_destroy_id failed errno %d", errno); return UCC_ERR_NO_RESOURCE; } ctx->id = NULL; if (ctx->channel) { rdma_destroy_event_channel(ctx->channel); ctx->channel = NULL; } if (ctx->devname && !ctx->user_provided_ib) { ucc_free(ctx->devname); ctx->devname = NULL; } return UCC_OK; } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_helper.c0000664000175000017500000006735115211535620024337 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_mcast_helper.h" #include #include #include #include "tl_mlx5_mcast_one_sided_reliability.h" #define PREF "/sys/class/net/" #define SUFF "/device/resource" #define MAX_STR_LEN 128 static ucc_status_t ucc_tl_mlx5_get_ipoib_ip(char *ifname, struct sockaddr_storage *addr) { ucc_status_t status = UCC_ERR_NO_RESOURCE; struct ifaddrs *ifaddr = NULL; struct ifaddrs *ifa = NULL; int is_ipv4 = 0; int family; int n; int is_up; if (getifaddrs(&ifaddr) == -1) { return UCC_ERR_NO_RESOURCE; } for (ifa = ifaddr, n = 0; ifa != NULL; ifa=ifa->ifa_next, n++) { if (ifa->ifa_addr == NULL) { continue; } family = ifa->ifa_addr->sa_family; if (family != AF_INET && family != AF_INET6) { continue; } is_up = (ifa->ifa_flags & IFF_UP) == IFF_UP; is_ipv4 = (family == AF_INET) ? 1 : 0; if (is_up && !strncmp(ifa->ifa_name, ifname, strlen(ifname)) ) { if (is_ipv4) { memcpy((struct sockaddr_in *) addr, (struct sockaddr_in *) ifa->ifa_addr, sizeof(struct sockaddr_in)); } else { memcpy((struct sockaddr_in6 *) addr, (struct sockaddr_in6 *) ifa->ifa_addr, sizeof(struct sockaddr_in6)); } status = UCC_OK; break; } } freeifaddrs(ifaddr); return status; } static int cmp_files(char *f1, char *f2) { int answer = 0; FILE *fp1; FILE *fp2; int ch1; int ch2; if ((fp1 = fopen(f1, "r")) == NULL) { goto out; } else if ((fp2 = fopen(f2, "r")) == NULL) { goto close; } do { ch1 = getc(fp1); ch2 = getc(fp2); } while((ch1 != EOF) && (ch2 != EOF) && (ch1 == ch2)); if (ch1 == ch2) { answer = 1; } if (fclose(fp2) != 0) { fclose(fp1); return 0; } close: if (fclose(fp1) != 0) { return 0; } out: return answer; } static int port_from_file(char *port_file) { int res = -1; char buf1[MAX_STR_LEN]; char buf2[MAX_STR_LEN]; FILE *fp; int len; if ((fp = fopen(port_file, "r")) == NULL) { return -1; } if (fgets(buf1, MAX_STR_LEN - 1, fp) == NULL) { goto out; } len = strlen(buf1) - 2; strncpy(buf2, buf1 + 2, len); buf2[len] = 0; res = atoi(buf2); out: if (fclose(fp) != 0) { return -1; } return res; } static ucc_status_t dev2if(char *dev_name, char *port, struct sockaddr_storage *rdma_src_addr) { ucc_status_t status = UCC_OK; glob_t glob_el = {0,}; char dev_file [MAX_STR_LEN]; char port_file[MAX_STR_LEN]; char net_file [MAX_STR_LEN]; char if_name [MAX_STR_LEN]; char glob_path[MAX_STR_LEN]; int i; char **p; int len; sprintf(glob_path, PREF"*"); sprintf(dev_file, "/sys/class/infiniband/%s"SUFF, dev_name); if (glob(glob_path, 0, 0, &glob_el)) { return UCC_ERR_NO_RESOURCE; } p = glob_el.gl_pathv; if (glob_el.gl_pathc >= 1) { for (i = 0; i < glob_el.gl_pathc; i++, p++) { sprintf(port_file, "%s/dev_id", *p); sprintf(net_file, "%s"SUFF, *p); if(cmp_files(net_file, dev_file) && port != NULL && port_from_file(port_file) == atoi(port) - 1) { len = strlen(net_file) - strlen(PREF) - strlen(SUFF); strncpy(if_name, net_file + strlen(PREF), len); if_name[len] = 0; status = ucc_tl_mlx5_get_ipoib_ip(if_name, rdma_src_addr); if (UCC_OK == status) { break; } } } } globfree(&glob_el); return status; } ucc_status_t ucc_tl_mlx5_probe_ip_over_ib(char* ib_dev, struct sockaddr_storage *addr) { char *ib_name = NULL; char *port = NULL; char *ib = NULL; ucc_status_t status; struct sockaddr_storage rdma_src_addr; if (ib_dev == NULL) { return UCC_ERR_NO_RESOURCE; } ib = strdup(ib_dev); if (!ib) { return UCC_ERR_NO_MEMORY; } ucc_string_split(ib, ":", 2, &ib_name, &port); status = dev2if(ib_name, port, &rdma_src_addr); if (UCC_OK == status) { *addr = rdma_src_addr; } ucc_free(ib); return status; } ucc_status_t ucc_tl_mlx5_mcast_join_mcast_post(ucc_tl_mlx5_mcast_coll_context_t *ctx, struct sockaddr_in6 *net_addr, struct mcast_group *group, int is_root) { char buf[INET6_ADDRSTRLEN]; const char *dst; dst = inet_ntop(AF_INET6, net_addr, buf, INET6_ADDRSTRLEN); if (NULL == dst) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_WARN, "inet_ntop failed"); return UCC_ERR_NO_RESOURCE; } tl_debug(ctx->lib, "joining addr: %s is_root %d group %p", buf, is_root, group); if (rdma_join_multicast(ctx->id, (struct sockaddr*)net_addr, (void *)group)) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_WARN, "rdma_join_multicast failed errno %d", errno); return UCC_ERR_NO_RESOURCE; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_join_mcast_get_event(ucc_tl_mlx5_mcast_coll_context_t *ctx, struct rdma_cm_event **event) { char buf[INET6_ADDRSTRLEN]; const char *dst; if (rdma_get_cm_event(ctx->channel, event) < 0) { if (EINTR != errno) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_WARN, "rdma_get_cm_event failed, errno %d %s", errno, strerror(errno)); return UCC_ERR_NO_RESOURCE; } else { /* need to retry again */ return UCC_INPROGRESS; } } if (RDMA_CM_EVENT_MULTICAST_JOIN != (*event)->event) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_WARN, "failed to join multicast, unexpected event was" " received: event=%d, str=%s, status=%d", (*event)->event, rdma_event_str((*event)->event), (*event)->status); if (rdma_ack_cm_event(*event) < 0) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_WARN, "rdma_ack_cm_event failed"); } return UCC_ERR_NO_RESOURCE; } dst = inet_ntop(AF_INET6, (*event)->param.ud.ah_attr.grh.dgid.raw, buf, INET6_ADDRSTRLEN); if (NULL == dst) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_WARN, "inet_ntop failed"); return UCC_ERR_NO_RESOURCE; } tl_debug(ctx->lib, "joined dgid: %s, mlid 0x%x, sl %d", buf, (*event)->param.ud.ah_attr.dlid, (*event)->param.ud.ah_attr.sl); return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_init_qps(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_coll_comm_t *comm) { int max_inline = INT_MAX; struct ibv_qp_init_attr qp_init_attr = {0}; int i; int j; qp_init_attr.qp_type = IBV_QPT_UD; qp_init_attr.send_cq = comm->mcast.scq; //cq can be shared between multiple QPs qp_init_attr.recv_cq = comm->mcast.rcq; qp_init_attr.sq_sig_all = 0; qp_init_attr.cap.max_send_wr = comm->params.sx_depth; qp_init_attr.cap.max_recv_wr = comm->params.rx_depth; qp_init_attr.cap.max_inline_data = comm->params.sx_inline; qp_init_attr.cap.max_send_sge = comm->params.sx_sge; qp_init_attr.cap.max_recv_sge = comm->params.rx_sge; for (i = 0; i < comm->mcast_group_count; i++) { ucc_list_head_init(&comm->one_sided.posted_recv[i].posted_recv_bufs); comm->mcast.groups[i].qp = ibv_create_qp(ctx->pd, &qp_init_attr); if (!comm->mcast.groups[i].qp) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "Failed to create mcast UD qp index %d, errno %d", i, errno); goto error; } if (qp_init_attr.cap.max_inline_data < max_inline) { max_inline = qp_init_attr.cap.max_inline_data; } } if (comm->cuda_mem_enabled) { /* max inline send otherwise it segfault during ibv send */ comm->max_inline = 0; } else { comm->max_inline = max_inline; } return UCC_OK; error: for (j = 0; j < i; j++) { ibv_destroy_qp(comm->mcast.groups[j].qp); comm->mcast.groups[j].qp = NULL; } return UCC_ERR_NO_RESOURCE; } static ucc_status_t ucc_tl_mlx5_mcast_create_ah(ucc_tl_mlx5_mcast_coll_comm_t *comm) { int i, j, ret; struct ibv_ah_attr ah_attr = { .is_global = 1, .grh = {.sgid_index = 0}, .sl = DEF_SL, .src_path_bits = DEF_SRC_PATH_BITS, .port_num = comm->ctx->ib_port }; for (i = 0; i < comm->mcast_group_count; i ++) { ah_attr.dlid = comm->mcast.groups[i].lid; memcpy(ah_attr.grh.dgid.raw, &comm->mcast.groups[i].mgid, sizeof(ah_attr.grh.dgid.raw)); comm->mcast.groups[i].ah = ibv_create_ah(comm->ctx->pd, &ah_attr); if (!comm->mcast.groups[i].ah) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "failed to create AH index %d", i); goto error; } } return UCC_OK; error: for (j = 0; j < i; j++) { ret = ibv_destroy_ah(comm->mcast.groups[j].ah); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "couldn't destroy ah"); return UCC_ERR_NO_RESOURCE; } comm->mcast.groups[j].ah = NULL; } return UCC_ERR_NO_RESOURCE; } ucc_status_t ucc_tl_mlx5_mcast_setup_qps(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_coll_comm_t *comm) { struct ibv_port_attr port_attr; struct ibv_qp_attr attr; uint16_t pkey; int i; int retry_count; const int max_retries = 5; // More attempts for large scale const int base_delay_us = 5000; // Longer base delay (5ms) int attach_result; int attached_groups = 0; // Track successfully attached groups ibv_query_port(ctx->ctx, ctx->ib_port, &port_attr); for (ctx->pkey_index = 0; ctx->pkey_index < port_attr.pkey_tbl_len; ++ctx->pkey_index) { ibv_query_pkey(ctx->ctx, ctx->ib_port, ctx->pkey_index, &pkey); if (pkey == DEF_PKEY) break; } if (ctx->pkey_index >= port_attr.pkey_tbl_len) { ctx->pkey_index = 0; ibv_query_pkey(ctx->ctx, ctx->ib_port, ctx->pkey_index, &pkey); if (!pkey) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_WARN, "cannot find valid PKEY"); return UCC_ERR_NO_RESOURCE; } tl_debug(ctx->lib, "cannot find default pkey 0x%04x on port %d, using " "index 0 pkey:0x%04x", DEF_PKEY, ctx->ib_port, pkey); } for (i = 0; i < comm->mcast_group_count; i++) { attr.qp_state = IBV_QPS_INIT; attr.pkey_index = ctx->pkey_index; attr.port_num = ctx->ib_port; attr.qkey = DEF_QKEY; if (ibv_modify_qp(comm->mcast.groups[i].qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY)) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "failed to move mcast qp to INIT, errno %d", errno); goto error; } // Add retry logic for multicast attach - enhanced for large scale retry_count = 0; attach_result = -1; while (retry_count < max_retries) { attach_result = ibv_attach_mcast(comm->mcast.groups[i].qp, &comm->mcast.groups[i].mgid, comm->mcast.groups[i].lid); if (attach_result == 0) { break; // Success } retry_count++; if (retry_count < max_retries) { // Add randomization to prevent thundering herd int random_jitter = rand() % 1000; // 0-1ms random jitter int delay = base_delay_us * retry_count + random_jitter; usleep(delay); tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_DEBUG, "retrying ibv_attach_mcast for group %d, attempt %d/%d, errno %d, delay %dus", i, retry_count + 1, max_retries, errno, delay); } } if (attach_result != 0) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "failed to attach QP to the mcast group with mcast_lid %d after %d attempts, errno %d", comm->mcast.groups[i].lid, max_retries, errno); goto error; } attached_groups++; // Track successful attachment if (retry_count > 0) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_DEBUG, "successfully attached QP to mcast group %d after %d retries", i, retry_count); } attr.qp_state = IBV_QPS_RTR; if (ibv_modify_qp(comm->mcast.groups[i].qp, &attr, IBV_QP_STATE)) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "failed to modify QP to RTR, errno %d", errno); goto error; } attr.qp_state = IBV_QPS_RTS; attr.sq_psn = DEF_PSN; if (ibv_modify_qp(comm->mcast.groups[i].qp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN)) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "failed to modify QP to RTS, errno %d", errno); goto error; } tl_debug(ctx->lib, "modified UD QP to RTS and RTR for mcast group id %d", i); } /* create the address handle */ if (UCC_OK != ucc_tl_mlx5_mcast_create_ah(comm)) { tl_warn(ctx->lib, "failed to create adress handle"); goto error; } return UCC_OK; error: /* Detach any successfully attached groups to prevent ghost attachments */ for (i = 0; i < attached_groups; i++) { if (ibv_detach_mcast(comm->mcast.groups[i].qp, &comm->mcast.groups[i].mgid, comm->mcast.groups[i].lid)) { tl_warn(ctx->lib, "failed to detach QP from mcast group %d during cleanup", i); } } for (i = 0; i < comm->mcast_group_count; i++) { ibv_destroy_qp(comm->mcast.groups[i].qp); comm->mcast.groups[i].qp = NULL; } return UCC_ERR_NO_RESOURCE; } ucc_status_t ucc_tl_mlx5_mcast_create_rc_qps(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_coll_comm_t *comm) { int i = 0, j = 0; struct ibv_srq_init_attr srq_init_attr; struct ibv_qp_init_attr qp_init_attr; /* create srq for this RC connection */ memset(&srq_init_attr, 0, sizeof(srq_init_attr)); srq_init_attr.attr.max_wr = comm->params.rx_depth; srq_init_attr.attr.max_sge = 2; comm->mcast.srq = ibv_create_srq(ctx->pd, &srq_init_attr); if (!comm->mcast.srq) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_ERROR, "ibv_create_srq() failed"); return UCC_ERR_NO_RESOURCE; } comm->mcast.rc_qp = ucc_calloc(1, comm->commsize * sizeof(struct ibv_qp *), "ibv_qp* list"); if (!comm->mcast.rc_qp) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_ERROR, "failed to allocate memory for ibv_qp*"); goto failed; } /* create RC qp */ for (i = 0; i < comm->commsize; i++) { memset(&qp_init_attr, 0, sizeof(qp_init_attr)); qp_init_attr.srq = comm->mcast.srq; qp_init_attr.qp_type = IBV_QPT_RC; qp_init_attr.send_cq = comm->mcast.scq; qp_init_attr.recv_cq = comm->mcast.rcq; qp_init_attr.sq_sig_all = 0; qp_init_attr.cap.max_send_wr = comm->params.sx_depth; qp_init_attr.cap.max_recv_wr = 0; // has srq qp_init_attr.cap.max_inline_data = 0; qp_init_attr.cap.max_send_sge = comm->params.sx_sge; qp_init_attr.cap.max_recv_sge = comm->params.rx_sge; comm->mcast.rc_qp[i] = ibv_create_qp(ctx->pd, &qp_init_attr); if (!comm->mcast.rc_qp[i]) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_ERROR, "Failed to create mcast RC qp index %d, errno %d", i, errno); goto failed; } } return UCC_OK; failed: for (j=0; jmcast.rc_qp[j])) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "ibv_destroy_qp failed"); return UCC_ERR_NO_RESOURCE; } } if (ibv_destroy_srq(comm->mcast.srq)) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "ibv_destroy_srq failed"); return UCC_ERR_NO_RESOURCE; } ucc_free(comm->mcast.rc_qp); return UCC_ERR_NO_RESOURCE; } ucc_status_t ucc_tl_mlx5_mcast_modify_rc_qps(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_rank_t my_rank = comm->rank; struct ibv_qp_attr attr; int i; for (i = 0; i < comm->commsize; i++) { memset(&attr, 0, sizeof(attr)); attr.qp_state = IBV_QPS_INIT; attr.pkey_index = 0; attr.port_num = ctx->ib_port; attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC; if (ibv_modify_qp(comm->mcast.rc_qp[i], &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_ERROR, "Failed to move rc qp to INIT, errno %d", errno); return UCC_ERR_NO_RESOURCE; } memset(&attr, 0, sizeof(attr)); attr.qp_state = IBV_QPS_RTR; attr.path_mtu = IBV_MTU_4096; attr.dest_qp_num = comm->one_sided.info[i].rc_qp_num[my_rank]; attr.rq_psn = DEF_PSN; attr.max_dest_rd_atomic = 16; attr.min_rnr_timer = 12; attr.ah_attr.is_global = 0; attr.ah_attr.dlid = comm->one_sided.info[i].port_lid; attr.ah_attr.sl = DEF_SL; attr.ah_attr.src_path_bits = 0; attr.ah_attr.port_num = ctx->ib_port; tl_debug(comm->lib, "Connecting to rc qp to rank %d with lid %d qp_num %d port_num %d", i, attr.ah_attr.dlid, attr.dest_qp_num, attr.ah_attr.port_num); if (ibv_modify_qp(comm->mcast.rc_qp[i], &attr, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_ERROR, "Failed to modify rc QP index %d to RTR, errno %d", i, errno); return UCC_ERR_NO_RESOURCE; } memset(&attr, 0, sizeof(attr)); attr.qp_state = IBV_QPS_RTS; attr.sq_psn = DEF_PSN; attr.timeout = 14; attr.retry_cnt = 7; attr.rnr_retry = 7; /* infinite */ attr.max_rd_atomic = 1; if (ibv_modify_qp(comm->mcast.rc_qp[i], &attr, IBV_QP_STATE | IBV_QP_SQ_PSN | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC)) { tl_mlx5_mcast_log(ctx->params.mcast_enabled, ctx->lib, UCC_LOG_LEVEL_ERROR, "Failed to modify rc QP index %i to RTS, errno %d", i, errno); return UCC_ERR_NO_RESOURCE; } } return UCC_OK; } ucc_status_t ucc_tl_mlx5_leave_mcast_groups(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_status_t status = UCC_OK; char buf[INET6_ADDRSTRLEN]; const char *dst; int i; for (i = 0; i < comm->mcast_group_count; i++) { if (comm->mcast.groups[i].mcast_addr.sin6_flowinfo != 0) { dst = inet_ntop(AF_INET6, &comm->mcast.groups[i].mcast_addr, buf, INET6_ADDRSTRLEN); if (NULL == dst) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "inet_ntop failed for group %d during mcast leave group", i); status = UCC_ERR_NO_RESOURCE; continue; } tl_debug(ctx->lib, "mcast leave: ctx %p, comm %p, dgid: %s group %d", ctx, comm, buf, i); if (rdma_leave_multicast(ctx->id, (struct sockaddr*)&comm->mcast.groups[i].mcast_addr)) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "mcast rmda_leave_multicast failed for group %d", i); status = UCC_ERR_NO_RESOURCE; } } } return status; } ucc_status_t ucc_tl_mlx5_clean_mcast_comm(ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_tl_mlx5_mcast_context_t *mcast_ctx = ucc_container_of(comm->ctx, ucc_tl_mlx5_mcast_context_t, mcast_context); ucc_tl_mlx5_context_t *mlx5_ctx = ucc_container_of(mcast_ctx, ucc_tl_mlx5_context_t, mcast); ucc_context_h context = mlx5_ctx->super.super.ucc_context; int ret, i; ucc_status_t status; tl_debug(comm->lib, "cleaning mcast comm: %p, id %d", comm, comm->comm_id); while (UCC_INPROGRESS == (status = ucc_tl_mlx5_mcast_reliable(comm))) { ucc_context_progress(context); } if (UCC_OK != status) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "failed to clean mcast team: relibality progress status %d", status); return status; } for (i = 0; i < comm->mcast_group_count; i++) { if (comm->mcast.groups[i].qp) { ret = ibv_detach_mcast(comm->mcast.groups[i].qp, &(comm->mcast.groups[i].mgid), comm->mcast.groups[i].lid); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "couldn't detach QP, ret %d, errno %d", ret, errno); return UCC_ERR_NO_RESOURCE; } ret = ibv_destroy_qp(comm->mcast.groups[i].qp); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "failed to destroy QP %d", ret); return UCC_ERR_NO_RESOURCE; } comm->mcast.groups[i].qp = NULL; } if (comm->mcast.groups[i].ah) { ret = ibv_destroy_ah(comm->mcast.groups[i].ah); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "couldn't destroy ah"); return UCC_ERR_NO_RESOURCE; } comm->mcast.groups[i].ah = NULL; } } status = ucc_tl_mlx5_leave_mcast_groups(comm->ctx, comm); if (status) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "couldn't leave mcast group"); return status; } if (comm->one_sided.reliability_enabled) { ucc_tl_mlx5_mcast_one_sided_cleanup(comm); } if (comm->mcast.rcq) { ret = ibv_destroy_cq(comm->mcast.rcq); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "couldn't destroy rcq"); return UCC_ERR_NO_RESOURCE; } } if (comm->mcast.scq) { ret = ibv_destroy_cq(comm->mcast.scq); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "couldn't destroy scq"); return UCC_ERR_NO_RESOURCE; } } if (comm->grh_mr) { ret = ibv_dereg_mr(comm->grh_mr); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "couldn't destroy grh mr"); return UCC_ERR_NO_RESOURCE; } } if (comm->grh_buf) { ucc_free(comm->grh_buf); } if (comm->pp) { ucc_free(comm->pp); } if (comm->pp_mr) { ret = ibv_dereg_mr(comm->pp_mr); if (ret) { tl_mlx5_mcast_log(comm->context->mcast_enabled, comm->lib, UCC_LOG_LEVEL_ERROR, "couldn't destroy pp mr"); return UCC_ERR_NO_RESOURCE; } } if (comm->pp_buf) { ucc_mc_free(comm->pp_buf_header); } if (comm->call_rwr) { ucc_free(comm->call_rwr); } if (comm->call_rsgs) { ucc_free(comm->call_rsgs); } if (comm->ctx->params.print_nack_stats) { tl_debug(comm->lib, "comm_id %d, comm_size %d, comm->psn %d, rank %d, " "nacks counter %d, n_mcast_rel %d", comm->comm_id, comm->commsize, comm->psn, comm->rank, comm->bcast_comm.nacks_counter, comm->bcast_comm.n_mcast_reliable); } if (comm->p2p_ctx != NULL) { ucc_free(comm->p2p_ctx); } /* Cleanup HCA copy resources */ if (comm->hca_copy_qp) { ret = ibv_destroy_qp(comm->hca_copy_qp); if (ret) { tl_error(comm->lib, "couldn't destroy HCA copy QP"); return UCC_ERR_NO_RESOURCE; } } if (comm->hca_copy_cq) { ret = ibv_destroy_cq(comm->hca_copy_cq); if (ret) { tl_error(comm->lib, "couldn't destroy HCA copy CQ"); return UCC_ERR_NO_RESOURCE; } } ucc_free(comm); return UCC_OK; } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_one_sided_progress.c0000664000175000017500000005720315211535620026730 0ustar alastairalastair/** * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_mcast_one_sided_progress.h" #include "tl_mlx5_mcast_rcache.h" #include "tl_mlx5_mcast_hca_copy.h" #include ucc_status_t ucc_tl_mlx5_mcast_staging_allgather_reliable_one_sided_get(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, int *completed) { int target_completed = 0; int issued = 0; ucc_status_t status; ucc_rank_t target; ucc_tl_mlx5_mcast_reg_t *reg; void *src_addr; void *remote_addr; uint32_t rkey; uint32_t lkey; size_t size; uint64_t wr; /* in sync design this function is only called once */ ucc_assert(!(ONE_SIDED_SYNCHRONOUS_PROTO == req->one_sided_reliability_scheme && comm->one_sided.pending_reads)); /* When reliability protocol starts, copy scratch buffer to user buffer before RDMA reads. * This preserves received multicast packets while missing packets are retrieved via RDMA. */ if (req->scratch_buf && req->scratch_packets_received > 0) { tl_trace(comm->lib, "reliability protocol starting - copying scratch buffer (%d packets received) " "to user buffer before RDMA reads", req->scratch_packets_received); /* Copy the entire scratch buffer to user buffer. Successfully received packets * will be preserved, and missing packets will be overwritten by RDMA reads. */ status = ucc_tl_mlx5_mcast_memcpy(req->rptr, UCC_MEMORY_TYPE_CUDA, req->scratch_buf, UCC_MEMORY_TYPE_HOST, req->length * comm->commsize, comm); if (ucc_unlikely(status != UCC_OK)) { tl_error(comm->lib, "failed to copy scratch buffer to user buffer before reliability protocol"); return status; } /* Mark scratch buffer as copied to avoid further operations on it */ req->scratch_packets_received = -1; tl_trace(comm->lib, "successfully copied scratch buffer to user buffer before reliability protocol"); } for (target = 0; target < comm->commsize; target++) { if (comm->one_sided.recvd_pkts_tracker[target] == req->num_packets) { target_completed++; continue; } if (NULL == req->recv_rreg) { /* For reliability protocol, always register the user buffer to avoid memory access issues */ tl_debug(comm->lib, "registering recv buf of size %ld", comm->commsize * req->length); status = ucc_tl_mlx5_mcast_mem_register(comm->ctx, req->rptr, comm->commsize * req->length, ®); if (UCC_OK != status) { return status; } req->recv_rreg = reg; req->recv_mr = reg->mr; } switch(req->one_sided_reliability_scheme) { case ONE_SIDED_ASYNCHRONOUS_PROTO: /* first check if the remote slot is valid in this design, if * reliability protocol is kicked, allgather is completed once * all the values in one_sided.recvd_pkts_tracker[] is set to * req->num_packets and comm->pending_reads is set to 0 */ if (req->ag_counter == comm->one_sided.remote_slot_info[target]) { /* read remote data from remote slot * the content of this data is copied from send buffer by remote * process */ /* Always read to user buffer for reliability protocol */ src_addr = PTR_OFFSET(req->rptr, (req->length * target)); remote_addr = PTR_OFFSET(comm->one_sided.info[target].slot_mem.remote_addr, ((req->ag_counter % ONE_SIDED_SLOTS_COUNT) * comm->one_sided.slot_size + ONE_SIDED_SLOTS_INFO_SIZE)); lkey = req->recv_mr->lkey; rkey = comm->one_sided.info[target].slot_mem.rkey; size = req->length; wr = MCAST_AG_RDMA_READ_WR; comm->one_sided.pending_reads++; target_completed++; comm->one_sided.remote_slot_info[target] = ONE_SIDED_PENDING_DATA; comm->one_sided.recvd_pkts_tracker[target] = req->num_packets; } else if (ONE_SIDED_PENDING_INFO != comm->one_sided.remote_slot_info[target] && ONE_SIDED_PENDING_DATA != comm->one_sided.remote_slot_info[target]) { /* remote slot is not valid yet. Need to do an rdma * read to check the latest state */ src_addr = &comm->one_sided.remote_slot_info[target]; remote_addr = PTR_OFFSET(comm->one_sided.info[target].slot_mem.remote_addr, ((req->ag_counter % ONE_SIDED_SLOTS_COUNT) * comm->one_sided.slot_size)); lkey = comm->one_sided.remote_slot_info_mr->lkey; rkey = comm->one_sided.info[target].slot_mem.rkey; size = ONE_SIDED_SLOTS_INFO_SIZE; wr = MCAST_AG_RDMA_READ_INFO_WR; comm->one_sided.pending_reads++; comm->one_sided.remote_slot_info[target] = ONE_SIDED_PENDING_INFO; } else { /* rdma read to remote info or data has already been issue but it * has not been completed */ continue; } break; case ONE_SIDED_SYNCHRONOUS_PROTO: /* read the whole remote send buffer */ /* Always read to user buffer for reliability protocol */ src_addr = PTR_OFFSET(req->rptr, (req->length * target)); remote_addr = (void *)comm->one_sided.sendbuf_memkey_list[target].remote_addr; rkey = comm->one_sided.sendbuf_memkey_list[target].rkey; lkey = req->recv_mr->lkey; size = req->length; wr = MCAST_AG_RDMA_READ_WR; comm->one_sided.pending_reads++; target_completed++; break; default: return UCC_ERR_NOT_IMPLEMENTED; } issued++; status = ucc_tl_mlx5_one_sided_p2p_get(src_addr, remote_addr, size, lkey, rkey, target, wr, comm); if (UCC_OK != status) { return status; } } if (completed) { *completed = target_completed; } if (issued) { tl_debug(comm->lib, "issued %d RDMA READ to remote INFO/DATA. Number of target ranks completed: %d", issued, target_completed); } return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_progress_one_sided_communication(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { int completed = 0; ucc_status_t status; ucc_assert(comm->one_sided.pending_reads); if (!req->to_send && !req->to_recv) { // need to wait until all the rdma reads are done to avoid data invalidation tl_trace(comm->lib, "all the mcast packets arrived during the reliability protocol " "current timeout is %d usec", comm->ctx->params.timeout); } if (ucc_tl_mlx5_mcast_poll_send(comm) < 0) { return UCC_ERR_NO_MESSAGE; } // check if all the rdma read have been completed and return UCC_OK if so switch(req->one_sided_reliability_scheme) { case ONE_SIDED_ASYNCHRONOUS_PROTO: /* only applicable for allgather collectives */ status = ucc_tl_mlx5_mcast_staging_allgather_reliable_one_sided_get(comm, req, &completed); if (UCC_OK != status) { return status; } if (!comm->one_sided.pending_reads && (completed == comm->commsize)) { tl_trace(comm->lib, "all the pending RDMA READ are comepleted in async reliability protocol"); req->to_recv = 0; return UCC_OK; } break; case ONE_SIDED_SYNCHRONOUS_PROTO: if (!comm->one_sided.pending_reads) { tl_trace(comm->lib, "all the pending RDMA READ are comepleted in sync reliability protocol"); if (!comm->allgather_comm.truly_zero_copy_allgather_enabled && !comm->bcast_comm.truly_zero_copy_bcast_enabled) { req->to_recv = 0; } return UCC_OK; } break; default: return UCC_ERR_NOT_IMPLEMENTED; } return UCC_INPROGRESS; } ucc_status_t ucc_tl_mlx5_mcast_process_packet_collective(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, struct pp_packet *pp, int coll_type) { int out_of_order_recvd = 0; void *dest; int offset; int source_rank; uint32_t ag_counter; ucc_memory_type_t dst_mem_type; ucc_memory_type_t src_mem_type; ucc_status_t status; ucc_assert(pp->context == 0); // making sure it's a recv packet not send // process the immediate value saved in pp->psn source_rank = pp->psn % ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE; ag_counter = (pp->psn / ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE) % ONE_SIDED_MAX_ZCOPY_COLL_COUNTER; offset = (pp->psn / (ONE_SIDED_MAX_ZCOPY_COLL_COUNTER * ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE)); tl_trace(comm->lib, "processing a recvd packet with length %d source_rank" " %d ag_counter %d offset %d", pp->length, source_rank, ag_counter, offset); ucc_assert(offset < req->num_packets); // there are scenarios where we receive a packet with same offset/rank more than one time // this means that a packet which was considered dropped in previous run has not just arrived // need to check the allgather call counter and ignore this packet if it does not match if (ag_counter == (req->ag_counter % ONE_SIDED_MAX_ZCOPY_COLL_COUNTER)) { /* Update reliability tracking FIRST before any data processing */ if (comm->one_sided.reliability_enabled) { /* out of order recv'd packet that happen that is fatal in zero-copy * design is considered just like dropped packet */ if (offset != pp->packet_counter) { out_of_order_recvd = 1; } if (out_of_order_recvd == 0) { comm->one_sided.recvd_pkts_tracker[source_rank]++; } if (comm->one_sided.recvd_pkts_tracker[source_rank] > req->num_packets) { tl_error(comm->lib, "reliability failed: comm->one_sided.recvd_pkts_tracker[%d] %d" " req->num_packets %d offset %d" " comm->allgather_comm.under_progress_counter %d req->ag_counter" " %d \n", source_rank, comm->one_sided.recvd_pkts_tracker[source_rank], req->num_packets, offset, comm->allgather_comm.under_progress_counter, req->ag_counter); return UCC_ERR_NO_MESSAGE; } if (comm->allgather_comm.truly_zero_copy_allgather_enabled || comm->bcast_comm.truly_zero_copy_bcast_enabled) { /* remove pp from posted_recv_bufs queue */ ucc_list_del(&pp->super); } } if (comm->allgather_comm.truly_zero_copy_allgather_enabled || comm->bcast_comm.truly_zero_copy_bcast_enabled) { /* no need for memcopy - packet must be delivered by hca into * the user buffer double check that ordering is correct */ if (offset != pp->packet_counter) { /* recevied out of order packet */ tl_trace(comm->lib, "recevied out of order: packet counter %d expected recv counter %d with pp %p", offset, pp->packet_counter, pp); } } else if (pp->length) { /* staging based allgather */ ucc_assert(coll_type == UCC_COLL_TYPE_ALLGATHER); /* Use scratch buffer optimization when available for CUDA memory. * Both reliability and non-reliability paths coordinate properly with scratch buffer. */ if (req->scratch_buf && comm->cuda_mem_enabled) { /* CUDA staging with scratch buffer optimization */ if (pp->length == comm->max_per_packet) { dest = req->scratch_buf + (offset * pp->length + source_rank * req->length); } else { dest = req->scratch_buf + ((req->length - pp->length) + source_rank * req->length); } /* Fast HOST-to-HOST memcpy to scratch buffer */ memcpy(dest, (void*) pp->buf, pp->length); /* Only increment counter if we haven't already completed the copy */ if (req->scratch_packets_received >= 0) { req->scratch_packets_received++; } /* Check if all packets received - if so, copy scratch buffer to user buffer */ if (req->scratch_packets_received == (req->comm->commsize * req->num_packets)) { status = ucc_tl_mlx5_mcast_memcpy(req->rptr, UCC_MEMORY_TYPE_CUDA, req->scratch_buf, UCC_MEMORY_TYPE_HOST, req->length * req->comm->commsize, comm); if (ucc_unlikely(status != UCC_OK)) { tl_error(comm->lib, "failed to copy scratch buffer to user buffer"); return status; } req->scratch_packets_received = -1; tl_trace(comm->lib, "all packets received - copied scratch buffer to user buffer"); } } else { /* Staging logic fallback - used when scratch buffer is not available or CUDA is disabled */ if (pp->length == comm->max_per_packet) { dest = PTR_OFFSET(req->rptr, (offset * pp->length + source_rank * req->length)); } else { dest = PTR_OFFSET(req->rptr, ((req->length - pp->length) + source_rank * req->length)); } dst_mem_type = comm->cuda_mem_enabled ? UCC_MEMORY_TYPE_CUDA : UCC_MEMORY_TYPE_HOST; src_mem_type = UCC_MEMORY_TYPE_HOST; // staging buffer is always HOST status = ucc_tl_mlx5_mcast_memcpy(dest, dst_mem_type, (void*) pp->buf, src_mem_type, pp->length, comm); if (ucc_unlikely(status != UCC_OK)) { tl_error(comm->lib, "failed to copy buffer"); return status; } } } req->to_recv--; comm->psn++; pp->context = 0; ucc_list_add_tail(&comm->bpool, &pp->super); comm->one_sided.posted_recv[pp->qp_id].posted_recvs_count--; } else if (ag_counter > (req->ag_counter % ONE_SIDED_MAX_ZCOPY_COLL_COUNTER)) { /* received out of order allgather/bcast packet - add it to queue for future * processing */ ucc_list_add_tail(&comm->pending_q, &pp->super); } else { /* received a packet which was left from previous iterations * it is due to the fact that reliability protocol was initiated. * return the posted receive buffer back to the pool */ ucc_assert(comm->one_sided.reliability_enabled); pp->context = 0; ucc_list_add_tail(&comm->bpool, &pp->super); } return UCC_OK; } /* QP drain and reattach functionality - tested and working properly */ static inline ucc_status_t ucc_tl_mlx5_mcast_drain_recv_wr(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_context_t *ctx, int qp_id) { struct ibv_qp_attr attr = { .qp_state = IBV_QPS_ERR, .pkey_index = ctx->pkey_index, .port_num = ctx->ib_port, .qkey = DEF_QKEY }; /* set the qp state to ERR to flush the posted recv buffers */ if (ibv_modify_qp(comm->mcast.groups[qp_id].qp, &attr, IBV_QP_STATE)) { tl_error(comm->lib, "failed to move mcast qp to ERR, errno %d", errno); return UCC_ERR_NO_RESOURCE; } attr.qp_state = IBV_QPS_RESET; if (ibv_modify_qp(comm->mcast.groups[qp_id].qp, &attr, IBV_QP_STATE)) { tl_error(comm->lib, "failed to move mcast qp to RESET, errno %d", errno); return UCC_ERR_NO_RESOURCE; } attr.qp_state = IBV_QPS_INIT; if (ibv_modify_qp(comm->mcast.groups[qp_id].qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY)) { tl_error(comm->lib, "failed to move mcast qp to INIT, errno %d", errno); return UCC_ERR_NO_RESOURCE; } attr.qp_state = IBV_QPS_RTR; if (ibv_modify_qp(comm->mcast.groups[qp_id].qp, &attr, IBV_QP_STATE)) { tl_error(comm->lib, "failed to modify QP to RTR, errno %d", errno); return UCC_ERR_NO_RESOURCE; } attr.qp_state = IBV_QPS_RTS; attr.sq_psn = DEF_PSN; if (ibv_modify_qp(comm->mcast.groups[qp_id].qp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN)) { tl_error(comm->lib, "failed to modify QP to RTS, errno %d", errno); return UCC_ERR_NO_RESOURCE; } if (ibv_attach_mcast(comm->mcast.groups[qp_id].qp, &comm->mcast.groups[qp_id].mgid, comm->mcast.groups[qp_id].lid)) { tl_error(comm->lib, "failed to attach QP %d to the mcast group with mcast_lid %d, errno %d", qp_id, comm->mcast.groups[qp_id].lid, errno); return UCC_ERR_NO_RESOURCE; } tl_trace(comm->lib, "drained recv queue of QP %d", qp_id); return UCC_OK; } /* RDMA read for pipelined zero-copy reliability protocol */ ucc_status_t ucc_tl_mlx5_mcast_reliable_zcopy_pipelined_one_sided_get(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, int *completed) { ucc_tl_mlx5_mcast_pipelined_ag_schedule_t *sched = req->ag_schedule; ucc_status_t status; struct pp_packet *pp; struct pp_packet *next; void *src_addr; void *remote_addr; uint32_t rkey; uint32_t lkey; size_t size; uint64_t wr; int target_completed = 0; int issued = 0; int j; int root; int qp_id; ucc_assert(!comm->one_sided.pending_reads); for (j = 0; j < req->concurrency_level; j++) { root = sched[req->step].multicast_op[j].root; /* comm->one_sided.recvd_pkts_tracker[root] will not be incremented if there is out of order packet */ if (comm->one_sided.recvd_pkts_tracker[root] == sched[req->step].multicast_op[j].to_recv) { target_completed++; continue; } qp_id = sched[req->step].prepost_buf_op[j].group_id; status = ucc_tl_mlx5_mcast_drain_recv_wr(comm, comm->ctx, qp_id); if (UCC_OK != status) { tl_error(comm->lib, "unable to drain the posted recv wr on qp %d", qp_id); return status; } ucc_list_for_each_safe(pp, next, &comm->one_sided.posted_recv[qp_id].posted_recv_bufs, super) { /* return the recv pp this list back to free pool */ ucc_list_del(&pp->super); pp->context = 0; ucc_list_add_tail(&comm->bpool, &pp->super); } tl_trace(comm->lib, "RDMA READ for step %d total steps %d" " posted_recvs_count %d recvd_pkts_tracker for root=%d is %d " "to_recv for this step %d req->to_recv %d comm->pending_recv %d QP %d", req->step, req->total_steps, comm->one_sided.posted_recv[qp_id].posted_recvs_count, root, comm->one_sided.recvd_pkts_tracker[root], sched[req->step].multicast_op[j].to_recv, req->to_recv, comm->pending_recv, qp_id); comm->pending_recv -= comm->one_sided.posted_recv[qp_id].posted_recvs_count; req->to_recv -= comm->one_sided.posted_recv[qp_id].posted_recvs_count; comm->one_sided.recvd_pkts_tracker[root] = sched[req->step].multicast_op[j].to_recv; comm->one_sided.posted_recv[qp_id].posted_recvs_count = 0; ucc_assert(comm->pending_recv >= 0 && req->to_recv >= 0); /* issue RDMA Read to this root and read a piece of sendbuf * from related to this step*/ if (comm->bcast_comm.truly_zero_copy_bcast_enabled) { src_addr = PTR_OFFSET(req->ptr, req->length * root + sched[req->step].multicast_op[j].offset); } else { ucc_assert(comm->allgather_comm.truly_zero_copy_allgather_enabled); src_addr = PTR_OFFSET(req->rptr, req->length * root + sched[req->step].multicast_op[j].offset); } remote_addr = PTR_OFFSET(comm->one_sided.sendbuf_memkey_list[root].remote_addr, sched[req->step].multicast_op[j].offset); rkey = comm->one_sided.sendbuf_memkey_list[root].rkey; lkey = req->recv_mr->lkey; size = sched[req->step].multicast_op[j].to_recv * comm->max_per_packet; wr = MCAST_AG_RDMA_READ_WR; ucc_assert(size != 0); comm->one_sided.pending_reads++; issued++; status = ucc_tl_mlx5_one_sided_p2p_get(src_addr, remote_addr, size, lkey, rkey, root, wr, comm); if (UCC_OK != status) { return status; } } sched[req->step].num_recvd = sched[req->step].to_recv; if (completed) { *completed = target_completed; } if (issued) { tl_debug(comm->lib, "issued %d RDMA READ to remote DATA for step %d. Number of" " target ranks completed: %d", req->step, issued, target_completed); } return UCC_OK; } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_progress.c0000664000175000017500000004033715211535620024717 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_mcast_progress.h" static ucc_status_t ucc_tl_mlx5_mcast_recv_completion(ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj); static ucc_status_t ucc_tl_mlx5_mcast_send_completion(ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj); static ucc_status_t ucc_tl_mlx5_mcast_reliability_send_completion(ucc_tl_mlx5_mcast_p2p_completion_obj_t *comp_obj) { ucc_tl_mlx5_mcast_coll_comm_t *comm = (ucc_tl_mlx5_mcast_coll_comm_t*)comp_obj->data[0]; unsigned int pkt_id = comp_obj->data[2]; struct packet *p = (struct packet *)comp_obj->data[1]; ucc_status_t status; if (p != NULL) { /* it was a nack packet to our parent */ ucc_free(p); } if (pkt_id != UINT_MAX) { /* we sent the real data to our child so reduce the nack reqs */ ucc_assert(comm->bcast_comm.nack_requests > 0); ucc_assert(comm->bcast_comm.p2p_pkt[pkt_id].type == MCAST_P2P_NACK_SEND_PENDING); comm->bcast_comm.p2p_pkt[pkt_id].type = MCAST_P2P_ACK; comm->bcast_comm.nack_requests--; status = comm->params.p2p_iface.recv_nb(&comm->bcast_comm.p2p_pkt[pkt_id], sizeof(struct packet), comm->bcast_comm.p2p_pkt[pkt_id].from, UCC_MEMORY_TYPE_HOST, comm->p2p_ctx, GET_COMPL_OBJ(comm, ucc_tl_mlx5_mcast_recv_completion, pkt_id, NULL)); if (status < 0) { return status; } } ucc_mpool_put(comp_obj); return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_mcast_resend_packet_reliable(ucc_tl_mlx5_mcast_coll_comm_t *comm, int p2p_pkt_id) { uint32_t psn = comm->bcast_comm.p2p_pkt[p2p_pkt_id].psn; struct pp_packet *pp = comm->r_window[psn % comm->bcast_comm.wsize]; ucc_status_t status; ucc_memory_type_t mem_type; ucc_assert(pp->psn == psn); ucc_assert(comm->bcast_comm.p2p_pkt[p2p_pkt_id].type == MCAST_P2P_NEED_NACK_SEND); comm->bcast_comm.p2p_pkt[p2p_pkt_id].type = MCAST_P2P_NACK_SEND_PENDING; tl_trace(comm->lib, "[comm %d, rank %d] Send data NACK: to %d, psn %d, context %ld nack_requests %d \n", comm->comm_id, comm->rank, comm->bcast_comm.p2p_pkt[p2p_pkt_id].from, psn, pp->context, comm->bcast_comm.nack_requests); if (comm->cuda_mem_enabled) { mem_type = UCC_MEMORY_TYPE_CUDA; } else { mem_type = UCC_MEMORY_TYPE_HOST; } status = comm->params.p2p_iface.send_nb((void*) (pp->context ? pp->context : pp->buf), pp->length, comm->bcast_comm.p2p_pkt[p2p_pkt_id].from, mem_type, comm->p2p_ctx, GET_COMPL_OBJ(comm, ucc_tl_mlx5_mcast_reliability_send_completion, NULL, p2p_pkt_id)); if (status < 0) { return status; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_check_nack_requests(ucc_tl_mlx5_mcast_coll_comm_t *comm, uint32_t psn) { ucc_status_t status = UCC_OK; int i; struct pp_packet *pp; if (!comm->bcast_comm.nack_requests) { return UCC_OK; } if (psn != UINT32_MAX) { for (i=0; ibcast_comm.child_n; i++) { if (psn == comm->bcast_comm.p2p_pkt[i].psn && comm->bcast_comm.p2p_pkt[i].type == MCAST_P2P_NEED_NACK_SEND) { status = ucc_tl_mlx5_mcast_resend_packet_reliable(comm, i); if (status != UCC_OK) { break; } } } } else { for (i=0; ibcast_comm.child_n; i++){ if (comm->bcast_comm.p2p_pkt[i].type == MCAST_P2P_NEED_NACK_SEND) { psn = comm->bcast_comm.p2p_pkt[i].psn; pp = comm->r_window[psn % comm->bcast_comm.wsize]; if (psn == pp->psn) { status = ucc_tl_mlx5_mcast_resend_packet_reliable(comm, i); if (status < 0) { break; } } } } } return status; } static inline int ucc_tl_mlx5_mcast_find_nack_psn(ucc_tl_mlx5_mcast_coll_comm_t* comm, ucc_tl_mlx5_mcast_coll_req_t *req) { int psn = ucc_max(comm->bcast_comm.last_acked, req->start_psn); int max_search_psn = ucc_min(req->start_psn + req->num_packets, comm->bcast_comm.last_acked + comm->bcast_comm.wsize + 1); for (; psn < max_search_psn; psn++) { if (!PSN_RECEIVED(psn, comm)) { break; } } ucc_assert(psn < max_search_psn); return psn; } static inline ucc_rank_t ucc_tl_mlx5_mcast_get_nack_parent(ucc_tl_mlx5_mcast_coll_req_t *req) { return req->parent; } /* When parent resend the lost packet to a child, this function is called at child side */ static ucc_status_t ucc_tl_mlx5_mcast_recv_data_completion(ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj) { ucc_status_t status = UCC_OK; ucc_tl_mlx5_mcast_coll_comm_t *comm = (ucc_tl_mlx5_mcast_coll_comm_t *)obj->data[0]; struct pp_packet *pp = (struct pp_packet *)obj->data[1]; ucc_tl_mlx5_mcast_coll_req_t *req = (ucc_tl_mlx5_mcast_coll_req_t *)obj->data[2]; void *dest; ucc_memory_type_t dst_mem_type; ucc_memory_type_t src_mem_type; tl_trace(comm->lib, "[comm %d, rank %d] Recved data psn %d", comm->comm_id, comm->rank, pp->psn); dest = req->ptr + PSN_TO_RECV_OFFSET(pp->psn, req, comm); dst_mem_type = comm->cuda_mem_enabled ? UCC_MEMORY_TYPE_CUDA : UCC_MEMORY_TYPE_HOST; src_mem_type = UCC_MEMORY_TYPE_HOST; // staging buffer is always HOST status = ucc_mc_memcpy(dest, (void*) pp->buf, pp->length, dst_mem_type, src_mem_type); if (ucc_unlikely(status != UCC_OK)) { tl_error(comm->lib, "failed to copy buffer"); return status; } req->to_recv--; comm->r_window[pp->psn % comm->bcast_comm.wsize] = pp; status = ucc_tl_mlx5_mcast_check_nack_requests(comm, pp->psn); if (status < 0) { return status; } comm->psn++; comm->bcast_comm.recv_drop_packet_in_progress = false; return status; } static inline ucc_status_t ucc_tl_mlx5_mcast_reliable_send_NACK(ucc_tl_mlx5_mcast_coll_comm_t* comm, ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_status_t status = UCC_OK; uint32_t psn = ucc_tl_mlx5_mcast_find_nack_psn(comm, req); struct pp_packet *pp; ucc_rank_t parent; struct packet *p; p = ucc_calloc(1, sizeof(struct packet)); p->type = MCAST_P2P_NACK; p->psn = psn; p->from = comm->rank; p->comm_id = comm->comm_id; parent = ucc_tl_mlx5_mcast_get_nack_parent(req); comm->bcast_comm.nacks_counter++; status = comm->params.p2p_iface.send_nb(p, sizeof(struct packet), parent, UCC_MEMORY_TYPE_HOST, comm->p2p_ctx, GET_COMPL_OBJ(comm, ucc_tl_mlx5_mcast_reliability_send_completion, p, UINT_MAX)); if (status < 0) { return status; } tl_trace(comm->lib, "[comm %d, rank %d] Sent NAK : parent %d, psn %d", comm->comm_id, comm->rank, parent, psn); // Prepare to obtain the data. pp = ucc_tl_mlx5_mcast_buf_get_free(comm); if (ucc_unlikely(pp == NULL)) { tl_error(comm->lib, "failed to get free buffer for NACK"); return UCC_ERR_NO_MEMORY; } pp->psn = psn; pp->length = PSN_TO_RECV_LEN(pp->psn, req, comm); comm->bcast_comm.recv_drop_packet_in_progress = true; ucc_memory_type_t mem_type = comm->cuda_mem_enabled ? UCC_MEMORY_TYPE_CUDA : UCC_MEMORY_TYPE_HOST; status = comm->params.p2p_iface.recv_nb((void*) pp->buf, pp->length, parent, mem_type, comm->p2p_ctx, GET_COMPL_OBJ(comm, ucc_tl_mlx5_mcast_recv_data_completion, pp, req)); if (status < 0) { return status; } return UCC_INPROGRESS; } ucc_status_t ucc_tl_mlx5_mcast_reliable_send(ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_rank_t i; ucc_rank_t parent; ucc_status_t status; tl_trace(comm->lib, "comm %p, psn %d, last_acked %d, n_parent %d", comm, comm->psn, comm->bcast_comm.last_acked, comm->bcast_comm.parent_n); ucc_assert(!comm->bcast_comm.reliable_in_progress); for (i=0; ibcast_comm.parent_n; i++) { parent = comm->bcast_comm.parents[i]; comm->bcast_comm.p2p_spkt[i].type = MCAST_P2P_ACK; comm->bcast_comm.p2p_spkt[i].psn = comm->bcast_comm.last_acked + comm->bcast_comm.wsize; comm->bcast_comm.p2p_spkt[i].comm_id = comm->comm_id; tl_trace(comm->lib, "rank %d, Posting SEND to parent %d, n_parent %d, psn %d", comm->rank, parent, comm->bcast_comm.parent_n, comm->psn); status = comm->params.p2p_iface.send_nb(&comm->bcast_comm.p2p_spkt[i], sizeof(struct packet), parent, UCC_MEMORY_TYPE_HOST, comm->p2p_ctx, GET_COMPL_OBJ(comm, ucc_tl_mlx5_mcast_send_completion, i, NULL)); if (status < 0) { return status; } } return UCC_OK; } static ucc_status_t ucc_tl_mlx5_mcast_recv_completion(ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj) { ucc_tl_mlx5_mcast_coll_comm_t *comm = (ucc_tl_mlx5_mcast_coll_comm_t*)obj->data[0]; int pkt_id = (int)obj->data[1]; uint32_t psn; struct pp_packet *pp; ucc_status_t status; ucc_assert(comm->comm_id == comm->bcast_comm.p2p_pkt[pkt_id].comm_id); if (comm->bcast_comm.p2p_pkt[pkt_id].type != MCAST_P2P_ACK) { ucc_assert(comm->bcast_comm.p2p_pkt[pkt_id].type == MCAST_P2P_NACK); psn = comm->bcast_comm.p2p_pkt[pkt_id].psn; pp = comm->r_window[psn % comm->bcast_comm.wsize]; tl_trace(comm->lib, "[comm %d, rank %d] Got NACK: from %d, psn %d, avail %d pkt_id %d", comm->comm_id, comm->rank, comm->bcast_comm.p2p_pkt[pkt_id].from, psn, pp->psn == psn, pkt_id); comm->bcast_comm.p2p_pkt[pkt_id].type = MCAST_P2P_NEED_NACK_SEND; comm->bcast_comm.nack_requests++; if (pp->psn == psn) { /* parent already has this packet so it is ready to forward it to its child */ status = ucc_tl_mlx5_mcast_resend_packet_reliable(comm, pkt_id); if (status != UCC_OK) { return status; } } } else { ucc_assert(comm->bcast_comm.p2p_pkt[pkt_id].type == MCAST_P2P_ACK); comm->bcast_comm.racks_n++; } ucc_mpool_put(obj); /* return the completion object back to the mem pool compl_objects_mp */ return UCC_OK; } static ucc_status_t ucc_tl_mlx5_mcast_send_completion(ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj) { ucc_tl_mlx5_mcast_coll_comm_t *comm = (ucc_tl_mlx5_mcast_coll_comm_t*)obj->data[0]; comm->bcast_comm.sacks_n++; ucc_mpool_put(obj); return UCC_OK; } static inline int add_uniq(ucc_rank_t *arr, uint32_t *len, ucc_rank_t value) { int i; for (i=0; i<(*len); i++) { if (arr[i] == value) { return 0; } } arr[*len] = value; (*len)++; return 1; } ucc_status_t ucc_tl_mlx5_mcast_prepare_reliable(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, ucc_rank_t root) { ucc_rank_t mask = 1; ucc_rank_t vrank = TO_VIRTUAL(comm->rank, comm->commsize, root); ucc_rank_t child; ucc_status_t status; ucc_assert(comm->commsize <= pow(2, MAX_COMM_POW2)); while (mask < comm->commsize) { if (vrank & mask) { req->parent = TO_ORIGINAL((vrank ^ mask), comm->commsize, root); add_uniq(comm->bcast_comm.parents, &comm->bcast_comm.parent_n, req->parent); break; } else { child = vrank ^ mask; if (child < comm->commsize) { child = TO_ORIGINAL(child, comm->commsize, root); if (add_uniq(comm->bcast_comm.children, &comm->bcast_comm.child_n, child)) { tl_trace(comm->lib, "rank %d, Posting RECV from child %d, n_child %d, psn %d", comm->rank, child, comm->bcast_comm.child_n, comm->psn); status = comm->params.p2p_iface.recv_nb(&comm->bcast_comm.p2p_pkt[comm->bcast_comm.child_n - 1], sizeof(struct packet), child, UCC_MEMORY_TYPE_HOST, comm->p2p_ctx, GET_COMPL_OBJ(comm, ucc_tl_mlx5_mcast_recv_completion, comm->bcast_comm.child_n - 1, req)); if (status < 0) { return status; } } } } mask <<= 1; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_bcast_check_drop(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_status_t status = UCC_OK; if (comm->timer == 0) { comm->timer = ucc_tl_mlx5_mcast_get_timer(); } else { if (ucc_tl_mlx5_mcast_get_timer() - comm->timer >= comm->ctx->params.timeout) { tl_trace(comm->lib, "[REL] time out %d", comm->psn); status = ucc_tl_mlx5_mcast_reliable_send_NACK(comm, req); comm->timer = 0; } } return status; } ucc_status_t ucc_tl_mlx5_mcast_process_packet(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, struct pp_packet* pp) { ucc_status_t status = UCC_OK; void *dest; ucc_ee_executor_task_args_t eargs; ucc_ee_executor_t *exec; ucc_assert(pp->psn >= req->start_psn && pp->psn < req->start_psn + req->num_packets); ucc_assert(pp->length == PSN_TO_RECV_LEN(pp->psn, req, comm)); ucc_assert(pp->context == 0); if (pp->length > 0 ) { dest = req->ptr + PSN_TO_RECV_OFFSET(pp->psn, req, comm); while (req->exec_task != NULL) { EXEC_TASK_TEST("failed to complete the nb memcpy", req->exec_task, comm->lib); } /* for cuda copy, exec is nonblocking but for host copy it is blocking */ status = ucc_coll_task_get_executor(req->coll_task, &exec); if (ucc_unlikely(status != UCC_OK)) { return status; } eargs.task_type = UCC_EE_EXECUTOR_TASK_COPY; eargs.copy.src = (void*) pp->buf; eargs.copy.dst = dest; eargs.copy.len = pp->length; assert(req->exec_task == NULL); status = ucc_ee_executor_task_post(exec, &eargs, &req->exec_task); if (ucc_unlikely(status != UCC_OK)) { return status; } if (req->exec_task != NULL) { EXEC_TASK_TEST("failed to progress the memcpy", req->exec_task, comm->lib); } } comm->r_window[pp->psn & (comm->bcast_comm.wsize-1)] = pp; status = ucc_tl_mlx5_mcast_check_nack_requests(comm, pp->psn); if (status < 0) { return status; } req->to_recv--; comm->psn++; ucc_assert(comm->bcast_comm.recv_drop_packet_in_progress == false); return status; } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_service_coll.h0000664000175000017500000000205515211535620025524 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_mcast_coll.h" ucc_status_t ucc_tl_mlx5_mcast_service_bcast_post(void *arg, void *buf, size_t size, ucc_rank_t root, ucc_service_coll_req_t **bcast_req); ucc_status_t ucc_tl_mlx5_mcast_service_allgather_post(void *arg, void *sbuf, void *rbuf, size_t size, ucc_service_coll_req_t **ag_req); ucc_status_t ucc_tl_mlx5_mcast_service_barrier_post(void *arg, ucc_service_coll_req_t **barrier_req); ucc_status_t ucc_tl_mlx5_mcast_service_allreduce_post(void *arg, void *sbuf, void *rbuf, size_t count, ucc_datatype_t dt, ucc_reduction_op_t op, ucc_service_coll_req_t **red_req); ucc_status_t ucc_tl_mlx5_mcast_service_coll_test(ucc_service_coll_req_t *req); ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_allgather.h0000664000175000017500000000052315211535620025014 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_MLX5_MCAST_ALLGATHER_H_ #define UCC_TL_MLX5_MCAST_ALLGATHER_H_ #include "tl_mlx5_mcast.h" #include "tl_mlx5_coll.h" ucc_status_t ucc_tl_mlx5_mcast_allgather_init(ucc_tl_mlx5_task_t *task); #endif ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_allgather.c0000664000175000017500000010653515211535620025021 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_mcast_helper.h" #include "tl_mlx5_coll.h" #include "tl_mlx5_mcast_rcache.h" #include "tl_mlx5_mcast_progress.h" #include "tl_mlx5_mcast_allgather.h" #include "tl_mlx5_mcast_one_sided_progress.h" #include "tl_mlx5_mcast_hca_copy.h" #include /* 32 here is the bit count of ib mcast packet's immediate data */ #define TL_MLX5_MCAST_IB_IMMEDIATE_PACKET_BIT_COUNT 32u #define MCAST_GET_MAX_ALLGATHER_PACKET_COUNT(_max_count, _max_team, _max_counter) \ do { \ _max_count = 2 << (TL_MLX5_MCAST_IB_IMMEDIATE_PACKET_BIT_COUNT - \ ucc_ilog2(_max_team) - \ ucc_ilog2(_max_counter)); \ } while (0); #define MCAST_ALLGATHER_IN_PROGRESS(_req, _comm) \ (_req->to_send || _req->to_recv || _comm->pending_send || \ _comm->one_sided.pending_reads || (NULL != _req->allgather_rkeys_req) || \ (_req->ag_schedule != NULL && _req->step != _req->ag_schedule->total_steps)) static inline ucc_status_t ucc_tl_mlx5_mcast_check_staging_based_collective(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_status_t status; ucc_assert(comm->one_sided.reliability_ready); ucc_assert(req->allgather_rkeys_req == NULL); if (comm->one_sided.pending_reads) { return ucc_tl_mlx5_mcast_progress_one_sided_communication(comm, req); } if (!req->to_send && !req->to_recv) { // all have been received, nothing to do return UCC_OK; } else if (req->to_send) { // it is not yet the time to start the reliability protocol return UCC_INPROGRESS; } if (!comm->timer) { if (comm->stalled < DROP_THRESHOLD) { comm->stalled++; } else { // kick the timer comm->timer = ucc_tl_mlx5_mcast_get_timer(); comm->stalled = 0; } } else { if (comm->stalled < DROP_THRESHOLD) { comm->stalled++; } else { // calcuate the current time and check if it's time to do RDMA READ if (ucc_tl_mlx5_mcast_get_timer() - comm->timer >= comm->ctx->params.timeout) { tl_debug(comm->lib, "[REL] time out req->to_recv %d left out of total of %d packets", req->to_recv, req->num_packets * comm->commsize); status = ucc_tl_mlx5_mcast_staging_allgather_reliable_one_sided_get(comm, req, NULL); if (UCC_OK != status) { return status; } } else { comm->stalled = 0; } } } return UCC_INPROGRESS; } static inline ucc_status_t ucc_tl_mlx5_mcast_allgather_reliability_ready(ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_tl_mlx5_mcast_coll_comm_t *comm = req->comm; ucc_tl_mlx5_mcast_reg_t *reg = NULL; ucc_status_t status; ucc_assert(req->ag_counter == comm->allgather_comm.under_progress_counter); if (!comm->one_sided.reliability_enabled || comm->one_sided.reliability_ready) { return UCC_OK; } if (req->allgather_rkeys_req) { status = comm->service_coll.coll_test(req->allgather_rkeys_req); if (status == UCC_OK) { ucc_assert(ONE_SIDED_SYNCHRONOUS_PROTO == req->one_sided_reliability_scheme); req->allgather_rkeys_req = NULL; tl_trace(comm->lib, "allgather for remote_addr/rkey is completed"); comm->one_sided.reliability_ready = 1; } return status; } /* initialize the structures needed by reliability protocol */ memset(comm->one_sided.recvd_pkts_tracker, 0, comm->commsize * sizeof(uint32_t)); memset(comm->one_sided.remote_slot_info, ONE_SIDED_INVALID, comm->commsize * sizeof(int)); /* local slots state */ comm->one_sided.slots_state = ONE_SIDED_INVALID; if (ONE_SIDED_SYNCHRONOUS_PROTO == req->one_sided_reliability_scheme) { /* do nonblocking allgather over remote addresses/keys */ if (!req->rreg) { /* register sbuf if it is not registered before */ status = ucc_tl_mlx5_mcast_mem_register(comm->ctx, req->ptr, req->length, ®); if (UCC_OK != status) { return status; } req->rreg = reg; req->mr = reg->mr; } comm->one_sided.sendbuf_memkey_list[comm->rank].rkey = req->mr->rkey; comm->one_sided.sendbuf_memkey_list[comm->rank].remote_addr = (uint64_t)req->ptr; tl_trace(comm->lib, "allgather over sendbuf addresses/rkey: address %p rkey %d", req->ptr, req->mr->rkey); status = comm->service_coll.allgather_post(comm->p2p_ctx, &(comm->one_sided.sendbuf_memkey_list[comm->rank]), comm->one_sided.sendbuf_memkey_list, sizeof(ucc_tl_mlx5_mcast_slot_mem_info_t), &req->allgather_rkeys_req); if (UCC_OK != status) { tl_error(comm->lib, "oob allgather failed during one-sided reliability reset of a collective call"); return status; } return UCC_INPROGRESS; } else { comm->one_sided.reliability_ready = 1; } return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_mcast_init_async_reliability_slots(ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_tl_mlx5_mcast_coll_comm_t *comm = req->comm; char *dest; ucc_assert(req->ag_counter == comm->allgather_comm.under_progress_counter); if (ONE_SIDED_ASYNCHRONOUS_PROTO == req->one_sided_reliability_scheme && ONE_SIDED_INVALID == comm->one_sided.slots_state) { /* copy the sendbuf and seqnum to the internal temp buf in case other processes need * to read from it */ ucc_assert(req->length <= comm->one_sided.reliability_scheme_msg_threshold); dest = PTR_OFFSET(comm->one_sided.slots_buffer, (req->ag_counter % ONE_SIDED_SLOTS_COUNT) * comm->one_sided.slot_size); /* Copy from user buffer to reliability slots - handle CUDA memory safely */ if (comm->cuda_mem_enabled) { /* Use HCA copy if enabled, otherwise use CUDA-aware memory copy */ ucc_status_t status = ucc_tl_mlx5_mcast_memcpy(PTR_OFFSET(dest, ONE_SIDED_SLOTS_INFO_SIZE), UCC_MEMORY_TYPE_HOST, req->ptr, UCC_MEMORY_TYPE_CUDA, req->length, comm); if (status != UCC_OK) { tl_error(comm->lib, "memory copy failed in reliability slots"); return status; } } else { memcpy(PTR_OFFSET(dest, ONE_SIDED_SLOTS_INFO_SIZE), req->ptr, req->length); } memcpy(dest, &req->ag_counter, ONE_SIDED_SLOTS_INFO_SIZE); comm->one_sided.slots_state = ONE_SIDED_VALID; } return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_mcast_do_staging_based_allgather(void *req_handle) { ucc_status_t status = UCC_OK; ucc_tl_mlx5_mcast_coll_req_t *req = (ucc_tl_mlx5_mcast_coll_req_t *)req_handle; ucc_tl_mlx5_mcast_coll_comm_t *comm = req->comm; const int zcopy = req->proto != MCAST_PROTO_EAGER; int num_recvd; ucc_assert(req->to_recv >= 0 && req->to_send >= 0); status = ucc_tl_mlx5_mcast_allgather_reliability_ready(req); if (UCC_OK != status) { return status; } if (req->to_send || req->to_recv) { ucc_assert(comm->allgather_comm.max_push_send >= comm->pending_send); if (req->to_send && (comm->allgather_comm.max_push_send - comm->pending_send) > 0) { status = ucc_tl_mlx5_mcast_send_collective(comm, req, ucc_min(comm->allgather_comm.max_push_send - comm->pending_send, req->to_send), zcopy, 0, SIZE_MAX); if (status < 0) { tl_error(comm->lib, "a failure happend during send packets"); return status; } } status = ucc_tl_mlx5_mcast_init_async_reliability_slots(req); if (status != UCC_OK) { tl_error(comm->lib, "failed to initialize async reliability slots"); return status; } if (req->to_recv) { num_recvd = ucc_tl_mlx5_mcast_recv_collective(comm, req, req->to_recv, UCC_COLL_TYPE_ALLGATHER); if (num_recvd < 0) { tl_error(comm->lib, "a failure happend during cq polling"); status = UCC_ERR_NO_MESSAGE; return status; } } } if (comm->pending_send) { status = ucc_tl_mlx5_mcast_poll_send(comm); if (status != UCC_OK) { return status; } } if (comm->one_sided.reliability_enabled) { status = ucc_tl_mlx5_mcast_check_staging_based_collective(comm, req); if (status < 0) { return status; } } if (MCAST_ALLGATHER_IN_PROGRESS(req, comm)) { return UCC_INPROGRESS; } if (ONE_SIDED_SYNCHRONOUS_PROTO == req->one_sided_reliability_scheme) { /* mcast operations are all done, now wait until all the processes * are done with their mcast operations */ if (!req->barrier_req) { // mcast operations are done and now go to barrier status = comm->service_coll.barrier_post(comm->p2p_ctx, &req->barrier_req); if (status != UCC_OK) { return status; } tl_trace(comm->lib, "mcast operations are done and now go to barrier"); } status = comm->service_coll.coll_test(req->barrier_req); if (status == UCC_OK) { req->barrier_req = NULL; tl_trace(comm->lib, "barrier at the end of mcast allgather is completed"); } else { return status; } } /* this task is completed */ return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_allgather_start(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_task_t *task = ucc_derived_of(coll_task, ucc_tl_mlx5_task_t); ucc_tl_mlx5_team_t *mlx5_team = TASK_TEAM(task); return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(mlx5_team)->pq, &task->super); } void ucc_tl_mlx5_mcast_allgather_progress(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_task_t *task = ucc_derived_of(coll_task, ucc_tl_mlx5_task_t); ucc_tl_mlx5_mcast_coll_req_t *req = task->coll_mcast.req_handle; ucc_assert(req != NULL); if (req->ag_counter != req->comm->allgather_comm.under_progress_counter) { /* it is not this task's turn for progress */ ucc_assert(req->comm->allgather_comm.under_progress_counter < req->ag_counter); return; } coll_task->status = (req->progress)(req); if (coll_task->status < 0) { tl_error(UCC_TASK_LIB(task), "progress mcast allgather failed:%d", coll_task->status); } } static inline ucc_status_t ucc_tl_mlx5_mcast_validate_zero_copy_allgather_params(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { if (req->concurrency_level % 2 == 0 && req->num_packets % req->mcast_prepost_bucket_size != 0) { tl_debug(comm->lib, "Pipelined mcast allgather not supported: " "num_packets (%d) must be a multiple of mcast_prepost_bucket_size (%d) " "when concurrency_level (%d) is even.", req->num_packets, req->mcast_prepost_bucket_size, req->concurrency_level); return UCC_ERR_NOT_SUPPORTED; } if (comm->commsize % req->concurrency_level != 0) { tl_debug(comm->lib, "Pipelined mcast allgather not supported: " "team size (%d) must be a multiple of concurrency_level (%d).", comm->commsize, req->concurrency_level); return UCC_ERR_NOT_SUPPORTED; } /* Allow small messages (< max_per_packet) to use truly zero-copy * For small messages, we have num_packets = 1 and the message fits in one packet */ if (req->length >= comm->max_per_packet && req->length % comm->max_per_packet != 0) { tl_debug(comm->lib, "Pipelined mcast allgather not supported: " "length (%ld) must be a multiple of max_per_packet (%d) " "for messages >= max_per_packet.", req->length, comm->max_per_packet); return UCC_ERR_NOT_SUPPORTED; } if (req->mcast_prepost_bucket_size * req->concurrency_level * 2 > comm->params.rx_depth) { tl_debug(comm->lib, "Pipelined mcast allgather not supported: " "we only support the case prepost_bucket_size * concurrency_level * 2 > rx_depth, " "but got: prepost_bucket_size=%d, concurrency_level=%d, " "rx_depth=%d", req->mcast_prepost_bucket_size, req->concurrency_level, comm->params.rx_depth); return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } /* * at each stage half of the mcast groups are ready for receiving mcast * packets while the other half are getting prepared by preposting recv * buffers */ static inline ucc_status_t ucc_tl_mlx5_mcast_prepare_zero_copy_allgather(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_tl_mlx5_mcast_reg_t *reg = NULL; ucc_rank_t root = 0; int offset = 0; ucc_status_t status; ucc_rank_t j, i; int total_steps; ucc_tl_mlx5_mcast_pipelined_ag_schedule_t *schedule; ucc_assert(comm->allgather_comm.truly_zero_copy_allgather_enabled); req->concurrency_level = comm->mcast_group_count / 2; req->concurrency_level = ucc_min(req->concurrency_level, ONE_SIDED_MAX_CONCURRENT_LEVEL); req->concurrency_level = ucc_min(req->concurrency_level, comm->commsize); if (req->concurrency_level == 0) { tl_warn(comm->lib, "not enough concurreny level to enable zcopy pipeline allgather"); tl_debug(comm->lib, "truly zero-copy allgather requires at least 2 multicast groups. " "Current mcast_group_count=%d, concurrency_level=%d. " "Set UCC_TL_MLX5_MCAST_GROUP_COUNT=2 or higher " "to enable truly zero-copy.", comm->mcast_group_count, req->concurrency_level); return UCC_ERR_NOT_SUPPORTED; } req->mcast_prepost_bucket_size = ucc_min(req->num_packets, comm->allgather_comm.mcast_prepost_bucket_size); status = ucc_tl_mlx5_mcast_validate_zero_copy_allgather_params(comm, req); if (status != UCC_OK) { return status; } /* calculate the schedule and details of what we should * mcast and prepost to which mcast group at each stage*/ total_steps = req->num_packets * (comm->commsize / req->concurrency_level) / req->mcast_prepost_bucket_size + 1; schedule = ucc_calloc(1, sizeof(ucc_tl_mlx5_mcast_pipelined_ag_schedule_t) * total_steps, "sched"); if (!schedule) { tl_warn(comm->lib, "cannot allocate memory for schedule list"); return UCC_ERR_NO_MEMORY; } /* generate schedule */ for (i = 0; i < total_steps; i++) { if (i < total_steps - 1) { for (j = 0; j < req->concurrency_level; j++) { schedule[i].prepost_buf_op[j].group_id = j + req->concurrency_level * (i % 2); schedule[i].prepost_buf_op[j].offset = offset * comm->max_per_packet; schedule[i].prepost_buf_op[j].root = root + j; schedule[i].prepost_buf_op[j].count = req->mcast_prepost_bucket_size; } } else { schedule[i].prepost_buf_op_done = 1; } if (i > 0) { for (j = 0; j < req->concurrency_level; j++) { schedule[i].multicast_op[j].group_id = schedule[i - 1].prepost_buf_op[j].group_id; schedule[i].multicast_op[j].offset = schedule[i - 1].prepost_buf_op[j].offset; schedule[i].multicast_op[j].offset_left = schedule[i - 1].prepost_buf_op[j].offset; schedule[i].multicast_op[j].root = schedule[i - 1].prepost_buf_op[j].root; schedule[i].multicast_op[j].to_send_left = schedule[i - 1].prepost_buf_op[j].count; schedule[i].multicast_op[j].to_recv = schedule[i - 1].prepost_buf_op[j].count; schedule[i].to_recv += schedule[i].multicast_op[j].to_recv; if (schedule[i].multicast_op[j].root == comm->rank) { schedule[i].to_send += schedule[i].multicast_op[j].to_send_left; } } } if (!schedule[i].to_send || !schedule[i].to_recv) { schedule[i].multicast_op_done = 1; } offset += req->mcast_prepost_bucket_size; if (offset == req->num_packets) { offset = 0; root = (root + req->concurrency_level) % comm->commsize; } } tl_trace(comm->lib, "generated the schedule for pipelined zero copy allgather with total_steps %d", total_steps); schedule->total_steps = total_steps; req->total_steps = total_steps; req->ag_schedule = schedule; tl_trace(comm->lib, "registering recv buf of size %ld", req->length * comm->commsize); ucc_assert(req->recv_rreg == NULL); status = ucc_tl_mlx5_mcast_mem_register(comm->ctx, req->rptr, req->length * comm->commsize, ®); if (UCC_OK != status) { tl_warn(comm->lib, "unable to register receive buffer %p of size %ld", req->rptr, req->length * comm->commsize); ucc_free(schedule); return status; } req->recv_rreg = reg; req->recv_mr = reg->mr; return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_mcast_check_zcopy_allgather_collective(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_tl_mlx5_mcast_pipelined_ag_schedule_t *sched = req->ag_schedule; ucc_assert(req->allgather_rkeys_req == NULL); if (comm->one_sided.pending_reads) { return ucc_tl_mlx5_mcast_progress_one_sided_communication(comm, req); } if (req->step == req->total_steps) { return UCC_OK; } if (!sched[req->step].prepost_buf_op_done || !sched[req->step].multicast_op_done) { // it is not yet the time to start the reliability protocol return UCC_INPROGRESS; } if (sched[req->step].num_recvd == sched[req->step].to_recv) { /* check for out of order packets, if any root sent a out of order * packet to us in the current step, go ahead and issue RDMA READ * from that root for this specific piece of send buffer */ return ucc_tl_mlx5_mcast_reliable_zcopy_pipelined_one_sided_get(comm, req, NULL); } else if (!comm->timer) { if (comm->stalled < DROP_THRESHOLD) { comm->stalled++; } else { // kick the timer comm->timer = ucc_tl_mlx5_mcast_get_timer(); comm->stalled = 0; } } else { if (comm->stalled < DROP_THRESHOLD) { comm->stalled++; } else { // calcuate the current time and check if it's time to do RDMA READ if (ucc_tl_mlx5_mcast_get_timer() - comm->timer >= comm->ctx->params.timeout) { comm->timer = 0; ucc_assert(sched[req->step].to_recv >= sched[req->step].num_recvd); tl_debug(comm->lib, "allgather timeout %d pending packets to recv %d on step %d", comm->ctx->params.timeout, sched[req->step].to_recv - sched[req->step].num_recvd, req->step); return ucc_tl_mlx5_mcast_reliable_zcopy_pipelined_one_sided_get(comm, req, NULL); } else { comm->stalled = 0; } } } return UCC_INPROGRESS; } static inline ucc_status_t ucc_tl_mlx5_mcast_do_zero_copy_pipelined_allgather(void *req_handle) { ucc_tl_mlx5_mcast_coll_req_t *req = (ucc_tl_mlx5_mcast_coll_req_t *)req_handle; ucc_tl_mlx5_mcast_coll_comm_t *comm = req->comm; const int zcopy = req->proto != MCAST_PROTO_EAGER; ucc_tl_mlx5_mcast_pipelined_ag_schedule_t *sched = req->ag_schedule; int root = 0; int num_recvd = 0; int to_send_left = 0; int j = 0; int group_id = 0; int num_packets = 0; int count = 0; size_t offset = 0; size_t offset_left = 0; ucc_status_t status; status = ucc_tl_mlx5_mcast_allgather_reliability_ready(req); if (UCC_OK != status) { return status; } ucc_assert(req->to_recv>=0 && req->to_send >=0); if (req->barrier_req) { status = comm->service_coll.coll_test(req->barrier_req); if (status != UCC_OK) { return status; } tl_trace(comm->lib, "barrier at end of req->step %d is completed", req->step); req->barrier_req = NULL; req->step++; if (comm->one_sided.reliability_enabled) { memset(comm->one_sided.recvd_pkts_tracker, 0, comm->commsize * sizeof(uint32_t)); } } if (req->step < sched->total_steps) { if (!sched[req->step].multicast_op_done) { for (j = 0; j < req->concurrency_level; j++) { root = sched[req->step].multicast_op[j].root; if (comm->rank == root) { /* it's my turn to place mcast packets on wire */ group_id = sched[req->step].multicast_op[j].group_id; to_send_left = sched[req->step].multicast_op[j].to_send_left; offset_left = sched[req->step].multicast_op[j].offset_left; num_packets = ucc_min(comm->allgather_comm.max_push_send - comm->pending_send, to_send_left); if (to_send_left && (comm->allgather_comm.max_push_send - comm->pending_send) > 0) { status = ucc_tl_mlx5_mcast_send_collective(comm, req, num_packets, zcopy, group_id, offset_left); if (UCC_OK != status) { return status; } sched[req->step].multicast_op[j].to_send_left -= num_packets; sched[req->step].multicast_op[j].offset_left += (num_packets * comm->max_per_packet); } if (comm->pending_send) { status = ucc_tl_mlx5_mcast_poll_send(comm); if (status != UCC_OK) { return status; } } if (!sched[req->step].multicast_op[j].to_send_left && !comm->pending_send) { tl_trace(comm->lib, "done with mcast ops step %d group id %d to_send %d", req->step, group_id, sched[req->step].to_send); sched[req->step].multicast_op_done = 1; break; } } } } if (!sched[req->step].prepost_buf_op_done) { /* prepost the user buffers for a set of processes */ for (j = 0; j < req->concurrency_level; j++) { root = sched[req->step].prepost_buf_op[j].root; group_id = sched[req->step].prepost_buf_op[j].group_id; count = sched[req->step].prepost_buf_op[j].count; offset = sched[req->step].prepost_buf_op[j].offset; status = ucc_tl_mlx5_mcast_post_user_recv_buffers(comm, req, group_id, root, UCC_COLL_TYPE_ALLGATHER, count, offset); if (UCC_OK != status) { return status; } /* progress the recvd packets in between */ if (req->to_recv) { num_recvd = ucc_tl_mlx5_mcast_recv_collective(comm, req, req->to_recv, UCC_COLL_TYPE_ALLGATHER); if (num_recvd < 0) { tl_error(comm->lib, "a failure happend during cq polling"); status = UCC_ERR_NO_MESSAGE; return status; } if (sched[req->step].num_recvd != sched[req->step].to_recv) { /* if to_recv and num_recvd are equal then it means * we already kicked the reliabilty protocol for this step * and we have read the remote sendbuf for this step, therefore * simply ignore a packet that we received now. This happens * when timeout is set to low value */ sched[req->step].num_recvd += num_recvd; } } tl_trace(comm->lib, "preposted bufs step %d group id %d count %d offset %ld root %d", req->step, group_id, count, offset, root); } tl_trace(comm->lib, "done with prepost bufs step %d group id %d root %d", req->step, group_id, root); sched[req->step].prepost_buf_op_done = 1; } if (req->to_recv) { num_recvd = ucc_tl_mlx5_mcast_recv_collective(comm, req, req->to_recv, UCC_COLL_TYPE_ALLGATHER); if (num_recvd < 0) { tl_error(comm->lib, "a failure happend during cq polling"); status = UCC_ERR_NO_MESSAGE; return status; } if (sched[req->step].num_recvd != sched[req->step].to_recv) { sched[req->step].num_recvd += num_recvd; } } if (sched[req->step].prepost_buf_op_done && sched[req->step].multicast_op_done && sched[req->step].num_recvd == sched[req->step].to_recv) { // current step done tl_trace(comm->lib, "init global sync req->step %d", req->step); ucc_assert(sched[req->step].prepost_buf_op_done && sched[req->step].multicast_op_done); ucc_assert(req->barrier_req == NULL); status = comm->service_coll.barrier_post(comm->p2p_ctx, &req->barrier_req); if (status != UCC_OK) { return status; } } } if (comm->one_sided.reliability_enabled) { status = ucc_tl_mlx5_mcast_check_zcopy_allgather_collective(comm, req); if (status < 0) { return status; } } if (req->barrier_req != NULL || MCAST_ALLGATHER_IN_PROGRESS(req, comm)) { return UCC_INPROGRESS; } /* all completed */ assert(req->step == sched->total_steps); return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_allgather_init(ucc_tl_mlx5_task_t *task) { ucc_coll_task_t *coll_task = &(task->super); ucc_tl_mlx5_team_t *mlx5_team = TASK_TEAM(task); ucc_tl_mlx5_mcast_team_t *team = mlx5_team->mcast; ucc_coll_args_t *args = &TASK_ARGS(task); ucc_datatype_t dt = args->src.info.datatype; size_t count = args->src.info.count; ucc_status_t status = UCC_OK; size_t data_size = ucc_dt_size(dt) * count; void *sbuf = args->src.info.buffer; void *rbuf = args->dst.info.buffer; ucc_tl_mlx5_mcast_coll_comm_t *comm = team->mcast_comm; ucc_tl_mlx5_mcast_reg_t *reg = NULL; ucc_rank_t max_team = ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE; int max_ctr = ONE_SIDED_MAX_ZCOPY_COLL_COUNTER; ucc_tl_mlx5_mcast_coll_req_t *req; task->coll_mcast.req_handle = NULL; tl_trace(comm->lib, "MCAST allgather init, sbuf %p, rbuf %p, size %ld, comm %d, " "comm_size %d, counter %d", sbuf, rbuf, data_size, comm->comm_id, comm->commsize, comm->allgather_comm.coll_counter); req = ucc_mpool_get(&comm->ctx->mcast_req_mp); if (!req) { tl_error(comm->lib, "failed to get a mcast req"); status = UCC_ERR_NO_MEMORY; goto failed; } memset(req, 0, sizeof(ucc_tl_mlx5_mcast_coll_req_t)); req->comm = comm; req->ptr = sbuf; req->rptr = rbuf; req->length = data_size; req->mr = comm->pp_mr; req->rreg = NULL; req->scratch_buf = NULL; req->scratch_buf_header = NULL; req->scratch_packets_received = 0; /* - zero copy protocol only provides zero copy design at sender side * - truly zero copy protocol provides zero copy design at receiver side as well * here we select the sender side protocol * - For CUDA staging with scratch buffer optimization, we use EAGER protocol * - For non-CUDA or small messages, we use EAGER protocol * - For large non-CUDA messages, we use ZCOPY protocol */ if (comm->cuda_mem_enabled) { /* For CUDA memory: Use truly zero-copy if enabled, otherwise use staging with scratch buffer */ if (comm->allgather_comm.truly_zero_copy_allgather_enabled) { req->proto = MCAST_PROTO_ZCOPY; tl_trace(comm->lib, "CUDA message size %zu: using truly zero-copy (including small messages)", req->length); } else { req->proto = MCAST_PROTO_EAGER; tl_trace(comm->lib, "CUDA message size %zu: using staging with scratch buffer", req->length); } } else { /* Use eager for small messages, zcopy for large messages */ req->proto = (req->length < comm->max_eager) ? MCAST_PROTO_EAGER : MCAST_PROTO_ZCOPY; } assert(comm->commsize <= ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE); req->offset = 0; req->num_packets = ucc_div_round_up(req->length, comm->max_per_packet); MCAST_GET_MAX_ALLGATHER_PACKET_COUNT(comm->allgather_comm.max_num_packets, max_team, max_ctr); if (comm->allgather_comm.max_num_packets < req->num_packets) { tl_warn(comm->lib, "msg size is %ld but max supported msg size of mcast allgather is %d", req->length, comm->allgather_comm.max_num_packets * comm->max_per_packet); status = UCC_ERR_NOT_SUPPORTED; goto failed; } if (req->proto == MCAST_PROTO_EAGER && comm->cuda_mem_enabled) { /* For CUDA staging protocol, allocate scratch buffer for message assembly */ size_t scratch_size = req->length * comm->commsize; status = ucc_mc_alloc(&req->scratch_buf_header, scratch_size, UCC_MEMORY_TYPE_HOST); if (UCC_OK != status) { tl_error(comm->lib, "failed to allocate scratch buffer of size %zu", scratch_size); goto failed; } req->scratch_buf = req->scratch_buf_header->addr; tl_trace(comm->lib, "allocated scratch buffer of size %zu for CUDA staging", scratch_size); } /* Register the send buffer for both zero-copy and CUDA staging protocols */ status = ucc_tl_mlx5_mcast_mem_register(comm->ctx, req->ptr, req->length, ®); if (UCC_OK != status) { tl_error(comm->lib, "sendbuf registration failed"); goto failed; } req->rreg = reg; req->mr = reg->mr; if (comm->one_sided.reliability_enabled) { req->one_sided_reliability_scheme = (req->length < comm->one_sided.reliability_scheme_msg_threshold) ? ONE_SIDED_ASYNCHRONOUS_PROTO : ONE_SIDED_SYNCHRONOUS_PROTO; if (comm->allgather_comm.truly_zero_copy_allgather_enabled) { req->one_sided_reliability_scheme = ONE_SIDED_SYNCHRONOUS_PROTO; } } else { req->one_sided_reliability_scheme = ONE_SIDED_NO_RELIABILITY; } req->ag_counter = comm->allgather_comm.coll_counter; req->to_send = req->num_packets; req->to_recv = comm->commsize * req->num_packets; req->progress = ucc_tl_mlx5_mcast_do_staging_based_allgather; if (comm->allgather_comm.truly_zero_copy_allgather_enabled) { status = ucc_tl_mlx5_mcast_prepare_zero_copy_allgather(comm, req); if (UCC_OK != status) { tl_trace(comm->lib, "truly zero-copy allgather failed to prepare, falling back to staging: %s", ucc_status_string(status)); tl_trace(comm->lib, "using staging protocol as fallback for message size %zu", req->length); } else { req->progress = ucc_tl_mlx5_mcast_do_zero_copy_pipelined_allgather; tl_trace(comm->lib, "successfully enabled truly zero-copy allgather for message size %zu", req->length); } } else { tl_trace(comm->lib, "truly zero-copy allgather not enabled, using staging protocol for message size %zu", req->length); } comm->allgather_comm.coll_counter++; task->coll_mcast.req_handle = req; coll_task->status = UCC_OPERATION_INITIALIZED; task->super.post = ucc_tl_mlx5_mcast_allgather_start; task->super.progress = ucc_tl_mlx5_mcast_allgather_progress; return UCC_OK; failed: tl_warn(UCC_TASK_LIB(task), "mcast init allgather failed:%d", status); if (req) { if (req->rreg) { ucc_tl_mlx5_mcast_mem_deregister(comm->ctx, req->rreg); } if (req->scratch_buf_header) { ucc_mc_free(req->scratch_buf_header); } ucc_mpool_put(req); } return status; } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.h0000664000175000017500000000120415211535620024273 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_mcast.h" #include "utils/ucc_rcache.h" ucc_status_t ucc_tl_mlx5_mcast_setup_rcache(ucc_tl_mlx5_mcast_coll_context_t *ctx); ucc_status_t ucc_tl_mlx5_mcast_mem_register(ucc_tl_mlx5_mcast_coll_context_t *ctx, void *addr, size_t length, ucc_tl_mlx5_mcast_reg_t **reg); void ucc_tl_mlx5_mcast_mem_deregister(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_reg_t *reg); ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_one_sided_reliability.h0000664000175000017500000000122415211535620027372 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5.h" #include "tl_mlx5_mcast_coll.h" #include "coll_score/ucc_coll_score.h" #include "tl_mlx5_mcast_helper.h" #include "p2p/ucc_tl_mlx5_mcast_p2p.h" #include "mcast/tl_mlx5_mcast_helper.h" #include "mcast/tl_mlx5_mcast_service_coll.h" ucc_status_t ucc_tl_mlx5_mcast_one_sided_reliability_init(ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc_status_t ucc_tl_mlx5_mcast_one_sided_reliability_test(ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc_status_t ucc_tl_mlx5_mcast_one_sided_cleanup(ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_progress.h0000664000175000017500000000460315211535620024720 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_mcast.h" #include "tl_mlx5_mcast_helper.h" #ifndef TL_MLX5_MCAST_PROGRESS_H_ #define TL_MLX5_MCAST_PROGRESS_H_ #define TO_VIRTUAL(_rank, _size, _root) ((_rank + _size - _root) % _size) #define TO_ORIGINAL(_rank, _size, _root) ((_rank + _root) % _size) #define ACK 1 #define GET_COMPL_OBJ(_comm, _compl_fn, _pkt_id, _req) \ ({ \ void* item; \ ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj; \ item = ucc_mpool_get(&(_comm)->ctx->compl_objects_mp); \ obj = (ucc_tl_mlx5_mcast_p2p_completion_obj_t *)item; \ \ obj->data[0] = (uintptr_t)_comm; \ obj->compl_cb = _compl_fn; \ obj->data[1] = (uintptr_t)_pkt_id; \ obj->data[2] = (uintptr_t)_req; \ obj; \ }) ucc_status_t ucc_tl_mlx5_mcast_prepare_reliable(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, ucc_rank_t root); ucc_status_t ucc_tl_mlx5_mcast_bcast_check_drop(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req); ucc_status_t ucc_tl_mlx5_mcast_process_packet(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, struct pp_packet* pp); ucc_status_t ucc_tl_mlx5_mcast_check_nack_requests(ucc_tl_mlx5_mcast_coll_comm_t *comm, uint32_t psn); ucc_status_t ucc_tl_mlx5_mcast_reliable_send(ucc_tl_mlx5_mcast_coll_comm_t* comm); ucc_status_t ucc_tl_mlx5_mcast_check_nack_requests(ucc_tl_mlx5_mcast_coll_comm_t* comm, uint32_t psn); #endif /* ifndef TL_MLX5_MCAST_PROGRESS_H_ */ ucc-1.8.0/src/components/tl/mlx5/mcast/p2p/0000775000175000017500000000000015211535620020546 5ustar alastairalastairucc-1.8.0/src/components/tl/mlx5/mcast/p2p/ucc_tl_mlx5_mcast_p2p.c0000664000175000017500000002107515211535620025105 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_tl_mlx5_mcast_p2p.h" static inline void ucc_tl_mlx5_mcast_p2p_completion_cb(void* context) { ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj = (ucc_tl_mlx5_mcast_p2p_completion_obj_t *)context; ucc_assert(obj != NULL && obj->compl_cb != NULL); obj->compl_cb(obj); ucc_assert(obj->req != NULL); ucc_collective_finalize(obj->req); } void ucc_tl_mlx5_mcast_completion_cb(void* context, ucc_status_t status) //NOLINT { ucc_tl_mlx5_mcast_p2p_completion_cb(context); } static inline ucc_status_t ucc_tl_mlx5_mcast_do_p2p_bcast_nb(void *buf, size_t len, ucc_rank_t my_team_rank, ucc_rank_t dest, ucc_memory_type_t mem_type, ucc_team_h team, ucc_coll_callback_t *callback, ucc_coll_req_h *p2p_req, int is_send, ucc_base_lib_t *lib) { ucc_status_t status = UCC_OK; ucc_coll_req_h req = NULL; ucc_coll_args_t args = {0}; args.mask = UCC_COLL_ARGS_FIELD_ACTIVE_SET | UCC_COLL_ARGS_FIELD_CB; args.coll_type = UCC_COLL_TYPE_BCAST; args.src.info.buffer = buf; args.src.info.count = len; args.src.info.datatype = UCC_DT_INT8; args.src.info.mem_type = mem_type; args.root = is_send ? my_team_rank : dest; args.cb.cb = callback->cb; args.cb.data = callback->data; args.active_set.size = 2; args.active_set.start = my_team_rank; args.active_set.stride = (int)dest - (int)my_team_rank; status = ucc_collective_init(&args, &req, team); if (ucc_unlikely(UCC_OK != status)) { tl_error(lib, "nonblocking p2p init failed"); return status; } ((ucc_tl_mlx5_mcast_p2p_completion_obj_t *)args.cb.data)->req = req; status = ucc_collective_post(req); if (ucc_unlikely(UCC_OK != status)) { tl_error(lib, "nonblocking p2p post failed"); return status; } *p2p_req = req; return status; } static inline ucc_status_t do_send_nb(void *sbuf, size_t len, ucc_rank_t my_team_rank, ucc_rank_t dest, ucc_memory_type_t mem_type, ucc_team_h team, ucc_coll_callback_t *callback, ucc_coll_req_h *req, ucc_base_lib_t *lib) { return ucc_tl_mlx5_mcast_do_p2p_bcast_nb(sbuf, len, my_team_rank, dest, mem_type, team, callback, req, 1, lib); } static inline ucc_status_t do_recv_nb(void *rbuf, size_t len, ucc_rank_t my_team_rank, ucc_rank_t dest, ucc_memory_type_t mem_type, ucc_team_h team, ucc_coll_callback_t *callback, ucc_coll_req_h *req, ucc_base_lib_t *lib) { return ucc_tl_mlx5_mcast_do_p2p_bcast_nb(rbuf, len, my_team_rank, dest, mem_type, team, callback, req, 0, lib); } ucc_status_t ucc_tl_mlx5_mcast_p2p_send_nb(void* src, size_t size, ucc_rank_t rank, ucc_memory_type_t mem_type, void *context, ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj) { ucc_tl_mlx5_mcast_oob_p2p_context_t *oob_p2p_ctx = (ucc_tl_mlx5_mcast_oob_p2p_context_t *)context; ucc_status_t status = UCC_OK; ucc_coll_req_h req = NULL; ucc_rank_t my_team_rank = oob_p2p_ctx->my_team_rank; ucc_team_h team = oob_p2p_ctx->base_team; ucc_coll_callback_t callback; callback.cb = ucc_tl_mlx5_mcast_completion_cb; callback.data = obj; tl_trace(oob_p2p_ctx->lib, "P2P: SEND to %d Msg Size %ld", rank, size); status = do_send_nb(src, size, my_team_rank, rank, mem_type, team, &callback, &req, oob_p2p_ctx->lib); if (status < 0) { tl_error(oob_p2p_ctx->lib, "nonblocking p2p send failed"); return status; } return status; } ucc_status_t ucc_tl_mlx5_mcast_p2p_recv_nb(void *dst, size_t size, ucc_rank_t rank, ucc_memory_type_t mem_type, void *context, ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj) { ucc_tl_mlx5_mcast_oob_p2p_context_t *oob_p2p_ctx = (ucc_tl_mlx5_mcast_oob_p2p_context_t *)context; ucc_status_t status = UCC_OK; ucc_coll_req_h req = NULL; ucc_rank_t my_team_rank = oob_p2p_ctx->my_team_rank; ucc_team_h team = oob_p2p_ctx->base_team; ucc_coll_callback_t callback; callback.cb = ucc_tl_mlx5_mcast_completion_cb; callback.data = obj; tl_trace(oob_p2p_ctx->lib, "P2P: RECV to %d Msg Size %ld", rank, size); status = do_recv_nb(dst, size, my_team_rank, rank, mem_type, team, &callback, &req, oob_p2p_ctx->lib); if (status < 0) { tl_error(oob_p2p_ctx->lib, "nonblocking p2p recv failed"); return status; } return status; } ucc_status_t ucc_tl_mlx5_one_sided_p2p_put(void* src, void* remote_addr, size_t length, uint32_t lkey, uint32_t rkey, ucc_rank_t target_rank, uint64_t wr_id, ucc_tl_mlx5_mcast_coll_comm_t *comm) { struct ibv_send_wr swr = {0}; struct ibv_sge ssg = {0}; struct ibv_send_wr *bad_wr; int rc; if (UINT32_MAX < length) { tl_error(comm->lib, "msg too large for p2p put"); return UCC_ERR_NOT_SUPPORTED; } ssg.addr = (uint64_t)src; ssg.length = (uint32_t)length; ssg.lkey = lkey; swr.sg_list = &ssg; swr.num_sge = 1; swr.opcode = IBV_WR_RDMA_WRITE; swr.wr_id = wr_id; swr.send_flags = IBV_SEND_SIGNALED; swr.wr.rdma.remote_addr = (uint64_t)remote_addr; swr.wr.rdma.rkey = rkey; swr.next = NULL; tl_trace(comm->lib, "RDMA WRITE to rank %d size length %ld remote address %p rkey %d lkey %d src %p", target_rank, length, remote_addr, rkey, lkey, src); if (0 != (rc = ibv_post_send(comm->mcast.rc_qp[target_rank], &swr, &bad_wr))) { tl_error(comm->lib, "RDMA Write failed rc %d rank %d remote addresss %p rkey %d", rc, target_rank, remote_addr, rkey); return UCC_ERR_NO_MESSAGE; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_one_sided_p2p_get(void* src, void* remote_addr, size_t length, uint32_t lkey, uint32_t rkey, ucc_rank_t target_rank, uint64_t wr_id, ucc_tl_mlx5_mcast_coll_comm_t *comm) { struct ibv_send_wr swr = {0}; struct ibv_sge ssg = {0}; struct ibv_send_wr *bad_wr; int rc; if (UINT32_MAX < length) { tl_error(comm->lib, "msg too large for p2p get"); return UCC_ERR_NOT_SUPPORTED; } ssg.addr = (uint64_t)src; ssg.length = (uint32_t)length; ssg.lkey = lkey; swr.sg_list = &ssg; swr.num_sge = 1; swr.opcode = IBV_WR_RDMA_READ; swr.wr_id = wr_id; swr.send_flags = IBV_SEND_SIGNALED; swr.wr.rdma.remote_addr = (uint64_t)remote_addr; swr.wr.rdma.rkey = rkey; swr.next = NULL; tl_trace(comm->lib, "RDMA READ to rank %d size length %ld remote address %p rkey %d lkey %d src %p", target_rank, length, remote_addr, rkey, lkey, src); if (0 != (rc = ibv_post_send(comm->mcast.rc_qp[target_rank], &swr, &bad_wr))) { tl_error(comm->lib, "RDMA Read failed rc %d rank %d remote addresss %p rkey %d", rc, target_rank, remote_addr, rkey); return UCC_ERR_NO_MESSAGE; } return UCC_OK; } ucc-1.8.0/src/components/tl/mlx5/mcast/p2p/ucc_tl_mlx5_mcast_p2p.h0000664000175000017500000000253715211535620025114 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include #include "components/tl/mlx5/mcast/tl_mlx5_mcast.h" ucc_status_t ucc_tl_mlx5_mcast_p2p_send_nb(void* src, size_t size, ucc_rank_t rank, ucc_memory_type_t mem_type, void *context, ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj); ucc_status_t ucc_tl_mlx5_mcast_p2p_recv_nb(void* dst, size_t size, ucc_rank_t rank, ucc_memory_type_t mem_type, void *context, ucc_tl_mlx5_mcast_p2p_completion_obj_t *obj); ucc_status_t ucc_tl_mlx5_one_sided_p2p_put(void* src, void* remote_addr, size_t length, uint32_t lkey, uint32_t rkey, ucc_rank_t target_rank, uint64_t wr_id, ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc_status_t ucc_tl_mlx5_one_sided_p2p_get(void* src, void* remote_addr, size_t length, uint32_t lkey, uint32_t rkey, ucc_rank_t target_rank, uint64_t wr_id, ucc_tl_mlx5_mcast_coll_comm_t *comm); ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_coll.h0000664000175000017500000000104315211535620024000 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_MLX5_MCAST_COLL_H_ #define UCC_TL_MLX5_MCAST_COLL_H_ #include "tl_mlx5_mcast.h" #include "tl_mlx5_coll.h" ucc_status_t ucc_tl_mlx5_mcast_bcast_init(ucc_tl_mlx5_task_t *task); ucc_status_t ucc_tl_mlx5_mcast_test(ucc_tl_mlx5_mcast_coll_req_t* _req); ucc_status_t ucc_tl_mlx5_mcast_check_support(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team); #endif ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_hca_copy.c0000664000175000017500000003004115211535620024627 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_mcast_hca_copy.h" #include "tl_mlx5_mcast_helper.h" #include "tl_mlx5_mcast_rcache.h" #include /* Create dedicated CQ and QP for HCA copy to avoid interference with mcast operations */ static ucc_status_t ucc_tl_mlx5_mcast_hca_copy_setup_resources(ucc_tl_mlx5_mcast_coll_comm_t *comm) { struct ibv_qp_init_attr qp_attr; /* Create dedicated completion queue for HCA copy operations */ if (!comm->hca_copy_cq) { comm->hca_copy_cq = ibv_create_cq(comm->ctx->ctx, 16, NULL, NULL, 0); if (!comm->hca_copy_cq) { tl_error(comm->lib, "failed to create HCA copy CQ, errno %d", errno); return UCC_ERR_NO_RESOURCE; } tl_debug(comm->lib, "created dedicated HCA copy CQ"); } /* Create dedicated RC QP for HCA copy loopback */ if (!comm->hca_copy_qp) { memset(&qp_attr, 0, sizeof(qp_attr)); qp_attr.qp_type = IBV_QPT_RC; qp_attr.send_cq = comm->hca_copy_cq; qp_attr.recv_cq = comm->hca_copy_cq; qp_attr.sq_sig_all = 1; /* Signal all sends for HCA copy */ qp_attr.cap.max_send_wr = 16; qp_attr.cap.max_recv_wr = 16; qp_attr.cap.max_send_sge = 1; qp_attr.cap.max_recv_sge = 1; qp_attr.cap.max_inline_data = 0; comm->hca_copy_qp = ibv_create_qp(comm->ctx->pd, &qp_attr); if (!comm->hca_copy_qp) { tl_error(comm->lib, "failed to create HCA copy QP, errno %d", errno); ibv_destroy_cq(comm->hca_copy_cq); comm->hca_copy_cq = NULL; return UCC_ERR_NO_RESOURCE; } tl_debug(comm->lib, "created dedicated HCA copy QP"); /* Initialize QP to INIT state */ struct ibv_qp_attr attr; memset(&attr, 0, sizeof(attr)); attr.qp_state = IBV_QPS_INIT; attr.pkey_index = 0; attr.port_num = comm->ctx->ib_port; attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; if (ibv_modify_qp(comm->hca_copy_qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) { tl_error(comm->lib, "failed to move HCA copy QP to INIT"); ibv_destroy_qp(comm->hca_copy_qp); ibv_destroy_cq(comm->hca_copy_cq); comm->hca_copy_qp = NULL; comm->hca_copy_cq = NULL; return UCC_ERR_NO_RESOURCE; } /* Move to RTR state for loopback */ memset(&attr, 0, sizeof(attr)); attr.qp_state = IBV_QPS_RTR; attr.path_mtu = IBV_MTU_1024; attr.dest_qp_num = comm->hca_copy_qp->qp_num; /* Self-connection */ attr.rq_psn = 0; attr.max_dest_rd_atomic = 1; attr.min_rnr_timer = 12; attr.ah_attr.is_global = 0; attr.ah_attr.dlid = comm->ctx->port_lid; /* Self LID */ attr.ah_attr.sl = 0; attr.ah_attr.src_path_bits = 0; attr.ah_attr.port_num = comm->ctx->ib_port; if (ibv_modify_qp(comm->hca_copy_qp, &attr, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) { tl_error(comm->lib, "failed to move HCA copy QP to RTR"); ibv_destroy_qp(comm->hca_copy_qp); ibv_destroy_cq(comm->hca_copy_cq); comm->hca_copy_qp = NULL; comm->hca_copy_cq = NULL; return UCC_ERR_NO_RESOURCE; } /* Move to RTS state */ memset(&attr, 0, sizeof(attr)); attr.qp_state = IBV_QPS_RTS; attr.sq_psn = 0; attr.timeout = 14; attr.retry_cnt = 7; attr.rnr_retry = 7; attr.max_rd_atomic = 1; if (ibv_modify_qp(comm->hca_copy_qp, &attr, IBV_QP_STATE | IBV_QP_SQ_PSN | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC)) { tl_error(comm->lib, "failed to move HCA copy QP to RTS"); ibv_destroy_qp(comm->hca_copy_qp); ibv_destroy_cq(comm->hca_copy_cq); comm->hca_copy_qp = NULL; comm->hca_copy_cq = NULL; return UCC_ERR_NO_RESOURCE; } tl_trace(comm->lib, "HCA copy QP ready for loopback RDMA operations"); } return UCC_OK; } /* Completion handler for HCA copy RDMA operations */ static void ucc_tl_mlx5_mcast_hca_copy_completion(struct ibv_wc *wc, void *arg) { ucc_tl_mlx5_mcast_hca_copy_task_t *task = (ucc_tl_mlx5_mcast_hca_copy_task_t *)arg; if (wc->status != IBV_WC_SUCCESS) { task->status = UCC_ERR_NO_MESSAGE; } else { task->status = UCC_OK; } task->completed = 1; } /* Post HCA copy operation - uses dedicated QP for true HCA-assisted copy * with rcache optimization */ ucc_status_t ucc_tl_mlx5_mcast_hca_copy_post(void *dst, ucc_memory_type_t dst_mtype, void *src, ucc_memory_type_t src_mtype, size_t size, ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_hca_copy_task_t **copy_task) { ucc_tl_mlx5_mcast_hca_copy_task_t *task; struct ibv_send_wr *bad_wr; int ret; ucc_status_t status; if (!comm->one_sided.hca_copy_enabled) { return UCC_ERR_NOT_SUPPORTED; } /* Only use HCA copy for CUDA memory */ if (src_mtype != UCC_MEMORY_TYPE_CUDA && dst_mtype != UCC_MEMORY_TYPE_CUDA) { return UCC_ERR_NOT_SUPPORTED; } /* Ensure dedicated HCA copy resources are set up */ status = ucc_tl_mlx5_mcast_hca_copy_setup_resources(comm); if (status != UCC_OK) { return status; } task = ucc_malloc(sizeof(*task), "hca_copy_task"); if (!task) { return UCC_ERR_NO_MEMORY; } task->dst = dst; task->src = src; task->size = size; task->dst_mtype = dst_mtype; task->src_mtype = src_mtype; task->comm = comm; task->target_rank = comm->rank; task->completed = 0; task->status = UCC_OK; task->src_reg = NULL; task->dst_reg = NULL; /* Use registration cache for better performance */ status = ucc_tl_mlx5_mcast_mem_register(comm->ctx, src, size, &task->src_reg); if (status != UCC_OK) { tl_error(comm->lib, "failed to register source buffer via rcache for HCA copy"); ucc_free(task); return status; } status = ucc_tl_mlx5_mcast_mem_register(comm->ctx, dst, size, &task->dst_reg); if (status != UCC_OK) { tl_error(comm->lib, "failed to register destination buffer via rcache for HCA copy"); ucc_tl_mlx5_mcast_mem_deregister(comm->ctx, task->src_reg); ucc_free(task); return status; } /* Setup RDMA write work request for HCA-assisted loopback copy */ task->rdma_sge.addr = (uintptr_t)src; task->rdma_sge.length = size; task->rdma_sge.lkey = ((struct ibv_mr*)task->src_reg->mr)->lkey; task->rdma_wr.wr_id = (uintptr_t)task; task->rdma_wr.sg_list = &task->rdma_sge; task->rdma_wr.num_sge = 1; task->rdma_wr.opcode = IBV_WR_RDMA_WRITE; task->rdma_wr.send_flags = IBV_SEND_SIGNALED; task->rdma_wr.next = NULL; /* Set up RDMA write parameters for self-copy via HCA */ task->rdma_wr.wr.rdma.remote_addr = (uintptr_t)dst; /* For loopback operations, use destination's lkey as rkey */ task->rdma_wr.wr.rdma.rkey = ((struct ibv_mr*)task->dst_reg->mr)->lkey; /* Use dedicated HCA copy QP */ ret = ibv_post_send(comm->hca_copy_qp, &task->rdma_wr, &bad_wr); if (ret) { tl_error(comm->lib, "failed to post HCA copy RDMA write, errno %d", ret); ucc_tl_mlx5_mcast_mem_deregister(comm->ctx, task->src_reg); ucc_tl_mlx5_mcast_mem_deregister(comm->ctx, task->dst_reg); ucc_free(task); return UCC_ERR_NO_RESOURCE; } *copy_task = task; tl_trace(comm->lib, "posted HCA RDMA write operation via rcache: %p -> %p, size %zu", src, dst, size); return UCC_OK; } /* Test HCA copy completion using dedicated CQ */ ucc_status_t ucc_tl_mlx5_mcast_hca_copy_test(ucc_tl_mlx5_mcast_hca_copy_task_t *copy_task) { struct ibv_wc wc; int ne; if (!copy_task) { return UCC_ERR_INVALID_PARAM; } /* Check if already completed */ if (copy_task->completed) { return copy_task->status; } /* Poll dedicated HCA copy completion queue */ ne = ibv_poll_cq(copy_task->comm->hca_copy_cq, 1, &wc); if (ne < 0) { tl_error(copy_task->comm->lib, "failed to poll HCA copy CQ"); return UCC_ERR_NO_MESSAGE; } if (ne > 0 && wc.wr_id == (uintptr_t)copy_task) { ucc_tl_mlx5_mcast_hca_copy_completion(&wc, copy_task); } /* Check completion */ if (copy_task->completed) { if (copy_task->status == UCC_OK) { tl_trace(copy_task->comm->lib, "HCA copy RDMA write completed successfully"); } else { tl_trace(copy_task->comm->lib, "HCA copy RDMA write completed with error"); } return copy_task->status; } return UCC_INPROGRESS; } /* Finalize HCA copy task - uses rcache deregistration */ ucc_status_t ucc_tl_mlx5_mcast_hca_copy_finalize(ucc_tl_mlx5_mcast_hca_copy_task_t *copy_task) { if (!copy_task) { return UCC_ERR_INVALID_PARAM; } /* Deregister memory regions via rcache */ if (copy_task->src_reg) { ucc_tl_mlx5_mcast_mem_deregister(copy_task->comm->ctx, copy_task->src_reg); } if (copy_task->dst_reg) { ucc_tl_mlx5_mcast_mem_deregister(copy_task->comm->ctx, copy_task->dst_reg); } /* Free task */ ucc_free(copy_task); return UCC_OK; } /* Helper function to choose between HCA copy and mc copy */ ucc_status_t ucc_tl_mlx5_mcast_memcpy(void *dst, ucc_memory_type_t dst_mtype, void *src, ucc_memory_type_t src_mtype, size_t size, ucc_tl_mlx5_mcast_coll_comm_t *comm) { ucc_tl_mlx5_mcast_hca_copy_task_t *copy_task; ucc_status_t status; /* Use HCA copy if enabled and CUDA memory is involved */ if (comm->one_sided.hca_copy_enabled && (src_mtype == UCC_MEMORY_TYPE_CUDA || dst_mtype == UCC_MEMORY_TYPE_CUDA)) { tl_trace(comm->lib, "using HCA-assisted copy with rcache for CUDA memory transfer (src: %s, dst: %s)", src_mtype == UCC_MEMORY_TYPE_CUDA ? "CUDA" : "HOST", dst_mtype == UCC_MEMORY_TYPE_CUDA ? "CUDA" : "HOST"); /* Post HCA copy operation */ status = ucc_tl_mlx5_mcast_hca_copy_post(dst, dst_mtype, src, src_mtype, size, comm, ©_task); if (status != UCC_OK) { tl_warn(comm->lib, "HCA copy post failed, falling back to mc copy"); return ucc_mc_memcpy(dst, src, size, dst_mtype, src_mtype); } /* Wait for completion */ while ((status = ucc_tl_mlx5_mcast_hca_copy_test(copy_task)) == UCC_INPROGRESS) { /* Keep polling until completion */ } /* Finalize the task */ ucc_tl_mlx5_mcast_hca_copy_finalize(copy_task); if (status == UCC_OK) { tl_trace(comm->lib, "HCA copy completed successfully"); return UCC_OK; } else { tl_warn(comm->lib, "HCA copy failed, falling back to mc copy"); return ucc_mc_memcpy(dst, src, size, dst_mtype, src_mtype); } } else { /* Use regular mc copy for non-CUDA memory */ return ucc_mc_memcpy(dst, src, size, dst_mtype, src_mtype); } } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_service_coll.c0000664000175000017500000000764415211535620025530 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "mcast/tl_mlx5_mcast_service_coll.h" #include "core/ucc_service_coll.h" ucc_status_t ucc_tl_mlx5_mcast_service_bcast_post(void *arg, void *buf, size_t size, ucc_rank_t root, ucc_service_coll_req_t **bcast_req) { ucc_tl_mlx5_mcast_oob_p2p_context_t *ctx = (ucc_tl_mlx5_mcast_oob_p2p_context_t *)arg; ucc_status_t status = UCC_OK; ucc_team_t *team = ctx->base_team; ucc_subset_t subset = ctx->subset; ucc_service_coll_req_t *req = NULL; status = ucc_service_bcast(team, buf, size, root, subset, &req); if (ucc_unlikely(UCC_OK != status)) { tl_error(ctx->base_ctx->lib, "tl service mcast bcast failed"); return status; } *bcast_req = req; return status; } ucc_status_t ucc_tl_mlx5_mcast_service_allgather_post(void *arg, void *sbuf, void *rbuf, size_t size, ucc_service_coll_req_t **ag_req) { ucc_tl_mlx5_mcast_oob_p2p_context_t *ctx = (ucc_tl_mlx5_mcast_oob_p2p_context_t *)arg; ucc_status_t status = UCC_OK; ucc_team_t *team = ctx->base_team; ucc_subset_t subset = ctx->subset; ucc_service_coll_req_t *req = NULL; status = ucc_service_allgather(team, sbuf, rbuf, size, subset, &req); if (ucc_unlikely(UCC_OK != status)) { tl_error(ctx->base_ctx->lib, "tl service mcast allgather failed"); return status; } *ag_req = req; return status; } ucc_status_t ucc_tl_mlx5_mcast_service_barrier_post(void *arg, ucc_service_coll_req_t **barrier_req) { ucc_tl_mlx5_mcast_oob_p2p_context_t *ctx = (ucc_tl_mlx5_mcast_oob_p2p_context_t *)arg; ucc_status_t status = UCC_OK; ucc_team_t *team = ctx->base_team; ucc_subset_t subset = ctx->subset; ucc_service_coll_req_t *req = NULL; ctx->tmp_sbuf = 0; ctx->tmp_rbuf = 0; status = ucc_service_allreduce(team, &ctx->tmp_sbuf, &ctx->tmp_rbuf, UCC_DT_INT8, 1, UCC_OP_SUM, subset, &req); if (ucc_unlikely(UCC_OK != status)) { tl_error(ctx->base_ctx->lib, "tl service mcast barrier failed"); return status; } *barrier_req = req; return status; } ucc_status_t ucc_tl_mlx5_mcast_service_allreduce_post(void *arg, void *sendbuf, void *recvbuf, size_t count, ucc_datatype_t dt, ucc_reduction_op_t op, ucc_service_coll_req_t **allred_req) { ucc_tl_mlx5_mcast_oob_p2p_context_t *ctx = (ucc_tl_mlx5_mcast_oob_p2p_context_t *)arg; ucc_status_t status = UCC_OK; ucc_team_t *team = ctx->base_team; ucc_subset_t subset = ctx->subset; ucc_service_coll_req_t *req = NULL; status = ucc_service_allreduce(team, sendbuf, recvbuf, dt, count, op, subset, &req); if (UCC_OK != status) { tl_error(ctx->base_ctx->lib, "tl service mcast allreduce failed"); return status; } *allred_req = req; return status; } ucc_status_t ucc_tl_mlx5_mcast_service_coll_test(ucc_service_coll_req_t *req) { ucc_status_t status = UCC_OK; status = ucc_collective_test(&req->task->super); if (UCC_INPROGRESS != status) { if (status < 0) { ucc_error("oob service coll progress failed"); } ucc_service_coll_finalize(req); } return status; } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c0000664000175000017500000001242515211535620024275 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_mcast_rcache.h" static ucs_status_t ucc_tl_mlx5_mcast_coll_reg_mr(ucc_tl_mlx5_mcast_coll_context_t *ctx, void *data, size_t data_size, void **mr) { *mr = ibv_reg_mr(ctx->pd, data, data_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE); tl_trace(ctx->lib, "external memory register: ptr %p, len %zd, mr %p", data, data_size, (*mr)); if (!*mr) { tl_error(ctx->lib, "failed to register MR"); return UCS_ERR_NO_MEMORY; } return UCS_OK; } static ucc_status_t ucc_tl_mlx5_mcast_coll_dereg_mr(ucc_tl_mlx5_mcast_coll_context_t *ctx, void *mr) { if (ucc_unlikely(NULL == mr)) { tl_debug(ctx->lib, "external memory mr %p already deregistered", mr); return UCC_OK; } tl_trace(ctx->lib, "external memory deregister: mr %p", mr); if (ibv_dereg_mr(mr)) { tl_error(ctx->lib, "couldn't destroy mr %p", mr); return UCC_ERR_NO_RESOURCE; } return UCC_OK; } static ucs_status_t ucc_tl_mlx5_mcast_rcache_mem_reg_cb(void *context, ucs_rcache_t *rcache, //NOLINT void *arg, ucs_rcache_region_t *rregion, //NOLINT uint16_t flags) //NOLINT { ucc_tl_mlx5_mcast_rcache_region_t *region; void *address; size_t length; address = (void*)rregion->super.start; length = (size_t)(rregion->super.end - rregion->super.start); region = ucc_derived_of(rregion, ucc_tl_mlx5_mcast_rcache_region_t); return ucc_tl_mlx5_mcast_coll_reg_mr((ucc_tl_mlx5_mcast_coll_context_t *)context, address, length, ®ion->reg.mr); } static void ucc_tl_mlx5_mcast_rcache_mem_dereg_cb(void *context, ucc_rcache_t //NOLINT *rcache, ucc_rcache_region_t *rregion) //NOLINT { ucc_tl_mlx5_mcast_rcache_region_t *region = ucc_derived_of(rregion, ucc_tl_mlx5_mcast_rcache_region_t); ucc_tl_mlx5_mcast_coll_dereg_mr((ucc_tl_mlx5_mcast_coll_context_t *)context, region->reg.mr); } static void ucc_tl_mlx5_mcast_rcache_dump_region_cb(void *context, //NOLINT ucc_rcache_t *rcache, //NOLINT ucc_rcache_region_t *rregion, //NOLINT char *buf, //NOLINT size_t max) //NOLINT { ucc_tl_mlx5_mcast_rcache_region_t *region = ucc_derived_of(rregion, ucc_tl_mlx5_mcast_rcache_region_t); snprintf(buf, max, "bar ptr:%p", region->reg.mr); } ucc_status_t ucc_tl_mlx5_mcast_mem_register(ucc_tl_mlx5_mcast_coll_context_t *ctx, void *addr, size_t length, ucc_tl_mlx5_mcast_reg_t **reg) { ucc_rcache_region_t *rregion; ucc_tl_mlx5_mcast_rcache_region_t *region; ucc_status_t status; ucc_rcache_t *rcache; rcache = ctx->rcache; ucc_assert(rcache != NULL); status = ucc_rcache_get(rcache, (void *)addr, length, NULL, &rregion); if (ucc_unlikely(UCC_OK != status)) { tl_error(ctx->lib, "ucc_rcache_get failed"); return status; } region = ucc_derived_of(rregion, ucc_tl_mlx5_mcast_rcache_region_t); *reg = ®ion->reg; tl_trace(ctx->lib, "memory register mr %p", (*reg)->mr); return UCC_OK; } void ucc_tl_mlx5_mcast_mem_deregister(ucc_tl_mlx5_mcast_coll_context_t *ctx, ucc_tl_mlx5_mcast_reg_t *reg) { ucc_tl_mlx5_mcast_rcache_region_t *region; ucc_rcache_t *rcache; rcache = ctx->rcache; if (reg == NULL) { return; } ucc_assert(rcache != NULL); tl_trace(ctx->lib, "memory deregister mr %p", reg->mr); region = ucc_container_of(reg, ucc_tl_mlx5_mcast_rcache_region_t, reg); ucc_rcache_region_put(rcache, ®ion->super); } static ucc_rcache_ops_t ucc_tl_mlx5_rcache_ops = { .mem_reg = ucc_tl_mlx5_mcast_rcache_mem_reg_cb, .mem_dereg = ucc_tl_mlx5_mcast_rcache_mem_dereg_cb, .dump_region = ucc_tl_mlx5_mcast_rcache_dump_region_cb, #ifdef UCS_HAVE_RCACHE_MERGE_CB .merge = ucc_rcache_merge_cb_empty #endif }; ucc_status_t ucc_tl_mlx5_mcast_setup_rcache(ucc_tl_mlx5_mcast_coll_context_t *ctx) { ucc_rcache_params_t rcache_params; ucc_rcache_set_default_params(&rcache_params); rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_mcast_rcache_region_t); rcache_params.context = ctx; rcache_params.ops = &ucc_tl_mlx5_rcache_ops; rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED | UCM_EVENT_MEM_TYPE_FREE; return ucc_rcache_create(&rcache_params, "MLX5_MCAST", &ctx->rcache); } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast_coll.c0000664000175000017500000010465515211535620024010 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_coll.h" #include "tl_mlx5_mcast_helper.h" #include "tl_mlx5_mcast_rcache.h" #define MCAST_BCAST_IN_PROGRESS(_req, _comm) \ (_req->to_send || _req->to_recv || _comm->pending_send || \ _comm->one_sided.pending_reads || (NULL != _req->bcast_rkeys_req) || \ (_req->ag_schedule != NULL && _req->step != _req->ag_schedule->total_steps)) static inline ucc_status_t ucc_tl_mlx5_mcast_r_window_recycle(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_status_t status = UCC_OK; int wsize = comm->bcast_comm.wsize; int num_free_win = wsize - (comm->psn - comm->bcast_comm.last_acked); int req_completed = (req->to_send == 0 && req->to_recv == 0); struct pp_packet *pp = NULL; ucc_assert(comm->bcast_comm.recv_drop_packet_in_progress == false); ucc_assert(req->to_send >= 0); /* When do we need to perform reliability protocol: 1. Always in the end of the window 2. For the zcopy case: in the end of collective, because we can't signal completion before made sure that children received the data - user can modify buffer */ ucc_assert(num_free_win >= 0); if (!num_free_win || (req->proto == MCAST_PROTO_ZCOPY && req_completed)) { status = ucc_tl_mlx5_mcast_reliable(comm); if (UCC_OK != status) { return status; } while (req->exec_task != NULL) { EXEC_TASK_TEST("failed to complete the nb memcpy", req->exec_task, comm->lib); } comm->bcast_comm.n_mcast_reliable++; for (; comm->bcast_comm.last_acked < comm->psn; comm->bcast_comm.last_acked++) { pp = comm->r_window[comm->bcast_comm.last_acked & (wsize-1)]; ucc_assert(pp != &comm->dummy_packet); comm->r_window[comm->bcast_comm.last_acked & (wsize-1)] = &comm->dummy_packet; pp->context = 0; ucc_list_add_tail(&comm->bpool, &pp->super); } if (!req_completed) { status = ucc_tl_mlx5_mcast_prepare_reliable(comm, req, req->root); if (ucc_unlikely(UCC_OK != status)) { return status; } } } return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_mcast_do_bcast(void *req_handle) { ucc_status_t status = UCC_OK; ucc_tl_mlx5_mcast_coll_req_t *req = (ucc_tl_mlx5_mcast_coll_req_t *)req_handle; ucc_tl_mlx5_mcast_coll_comm_t *comm = req->comm; int zcopy = req->proto != MCAST_PROTO_EAGER; int wsize = comm->bcast_comm.wsize; int num_free_win; int num_sent; int to_send; int to_recv; int to_recv_left; int pending_q_size; ucc_assert(req->comm->psn >= req->start_psn); status = ucc_tl_mlx5_mcast_check_nack_requests(comm, UINT32_MAX); if (status < 0) { return status; } if (ucc_unlikely(comm->bcast_comm.recv_drop_packet_in_progress)) { /* wait till parent resend the dropped packet */ return UCC_INPROGRESS; } if (req->to_send || req->to_recv) { num_free_win = wsize - (comm->psn - comm->bcast_comm.last_acked); /* Send data if i'm root and there is a space in the window */ if (num_free_win && req->am_root) { num_sent = req->num_packets - req->to_send; ucc_assert(req->to_send > 0); ucc_assert(req->first_send_psn + num_sent < comm->bcast_comm.last_acked + wsize); if (req->first_send_psn + num_sent < comm->bcast_comm.last_acked + wsize && req->to_send) { /* How many to send: either all that are left (if they fit into window) or up to the window limit */ to_send = ucc_min(req->to_send, comm->bcast_comm.last_acked + wsize - (req->first_send_psn + num_sent)); ucc_tl_mlx5_mcast_send(comm, req, to_send, zcopy); num_free_win = wsize - (comm->psn - comm->bcast_comm.last_acked); } } status = ucc_tl_mlx5_mcast_prepare_reliable(comm, req, req->root); if (ucc_unlikely(UCC_OK != status)) { return status; } if (num_free_win && req->to_recv) { /* How many to recv: either all that are left or up to the window limit. */ pending_q_size = 0; to_recv = ucc_min(num_free_win, req->to_recv); to_recv_left = ucc_tl_mlx5_mcast_recv(comm, req, to_recv, &pending_q_size); if (to_recv == to_recv_left) { /* We didn't receive anything: increase the stalled counter and get ready for drop event */ if (comm->stalled++ >= DROP_THRESHOLD) { tl_trace(comm->lib, "Did not receive the packet with psn in" " current window range, so get ready for drop" " event. pending_q_size %d current comm psn %d" " bcast_comm.last_acked psn %d stall threshold %d ", pending_q_size, comm->psn, comm->bcast_comm.last_acked, DROP_THRESHOLD); status = ucc_tl_mlx5_mcast_bcast_check_drop(comm, req); if (UCC_INPROGRESS == status) { return status; } } } else if (to_recv_left < 0) { /* a failure happend during cq polling */ return UCC_ERR_NO_MESSAGE; } else { comm->stalled = 0; comm->timer = 0; } } } /* This function will check if we have to do a round of reliability protocol */ status = ucc_tl_mlx5_mcast_r_window_recycle(comm, req); if (UCC_OK != status) { return status; } if (req->to_send || req->to_recv || (zcopy && comm->psn != comm->bcast_comm.last_acked)) { return UCC_INPROGRESS; } else { return status; } } static inline ucc_status_t ucc_tl_mlx5_mcast_validate_zero_copy_bcast_params(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { if (req->num_packets % req->mcast_prepost_bucket_size != 0) { tl_debug(comm->lib, "Pipelined mcast bcast not supported: " "num_packets (%d) must be a multiple of mcast_prepost_bucket_size (%d) ", req->num_packets, req->mcast_prepost_bucket_size); return UCC_ERR_NOT_SUPPORTED; } if (comm->commsize % req->concurrency_level != 0) { tl_debug(comm->lib, "Pipelined mcast bcast not supported: " "team size (%d) must be a multiple of concurrency_level (%d).", comm->commsize, req->concurrency_level); return UCC_ERR_NOT_SUPPORTED; } if (req->length % comm->max_per_packet != 0) { tl_debug(comm->lib, "Pipelined mcast bcast not supported: " "length (%ld) must be a multiple of max_per_packet (%d).", req->length, comm->max_per_packet); return UCC_ERR_NOT_SUPPORTED; } if (req->mcast_prepost_bucket_size * req->concurrency_level * 2 > comm->params.rx_depth) { tl_debug(comm->lib, "Pipelined mcast bcast not supported: " "we only support the case prepost_bucket_size * concurrency_level * 2 > rx_depth, " "but got: prepost_bucket_size=%d, concurrency_level=%d, " "rx_depth=%d", req->mcast_prepost_bucket_size, req->concurrency_level, comm->params.rx_depth); return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_mcast_prepare_zero_copy_bcast(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_tl_mlx5_mcast_reg_t *reg = NULL; ucc_rank_t root = req->root; int offset = 0; ucc_status_t status; ucc_rank_t j, i; int total_steps; ucc_tl_mlx5_mcast_pipelined_ag_schedule_t *schedule; ucc_tl_mlx5_mcast_pipelined_ag_schedule_t *curr_schedule; ucc_tl_mlx5_mcast_pipelined_ag_schedule_t *prev_schedule; ucc_assert(comm->bcast_comm.truly_zero_copy_bcast_enabled); if (comm->commsize == 1) return UCC_OK; if (comm->mcast_group_count != 2) { tl_warn(comm->lib, "need exactly two mcast groups to enable zcopy bcast"); return UCC_ERR_NOT_SUPPORTED; } req->concurrency_level = 1; req->mcast_prepost_bucket_size = ucc_min(req->num_packets, comm->bcast_comm.mcast_prepost_bucket_size); status = ucc_tl_mlx5_mcast_validate_zero_copy_bcast_params(comm, req); if (status != UCC_OK) { return status; } /* calculate the schedule and details of what we should * mcast and prepost to which mcast group at each stage*/ total_steps = req->num_packets / (req->concurrency_level * req->mcast_prepost_bucket_size) + 1; schedule = ucc_calloc(1, sizeof(ucc_tl_mlx5_mcast_pipelined_ag_schedule_t) * total_steps, "sched"); if (!schedule) { tl_warn(comm->lib, "cannot allocate memory for schedule list"); return UCC_ERR_NO_MEMORY; } /* generate schedule */ for (i = 0; i < total_steps; i++) { curr_schedule = &(schedule[i]); if (i < total_steps - 1) { for (j = 0; j < req->concurrency_level; j++) { curr_schedule->prepost_buf_op[j].group_id = j + req->concurrency_level * (i % 2); curr_schedule->prepost_buf_op[j].offset = (offset + j * req->mcast_prepost_bucket_size) * comm->max_per_packet; curr_schedule->prepost_buf_op[j].root = root; curr_schedule->prepost_buf_op[j].count = req->mcast_prepost_bucket_size; } } if (i > 0) { prev_schedule = &(schedule[i - 1]); for (j = 0; j < req->concurrency_level; j++) { curr_schedule->multicast_op[j].group_id = prev_schedule->prepost_buf_op[j].group_id; curr_schedule->multicast_op[j].offset = prev_schedule->prepost_buf_op[j].offset; curr_schedule->multicast_op[j].offset_left = prev_schedule->prepost_buf_op[j].offset; curr_schedule->multicast_op[j].root = root; if (!req->am_root) { curr_schedule->multicast_op[j].to_recv = prev_schedule->prepost_buf_op[j].count; } curr_schedule->to_recv += curr_schedule->multicast_op[j].to_recv; if (curr_schedule->multicast_op[j].root == comm->rank) { curr_schedule->multicast_op[j].to_send_left = prev_schedule->prepost_buf_op[j].count; curr_schedule->to_send += curr_schedule->multicast_op[j].to_send_left; } } } if (req->am_root) { curr_schedule->prepost_buf_op_done = 1; } offset += req->mcast_prepost_bucket_size * req->concurrency_level; } tl_trace(comm->lib, "generated the schedule for pipelined zero copy bcast with total_steps %d", total_steps); schedule->total_steps = total_steps; req->total_steps = total_steps; req->ag_schedule = schedule; req->ag_counter = comm->bcast_comm.coll_counter; comm->bcast_comm.coll_counter++; if (!req->am_root) { tl_trace(comm->lib, "registering recv buf of size %ld", req->length); ucc_assert(req->recv_rreg == NULL); status = ucc_tl_mlx5_mcast_mem_register(comm->ctx, req->ptr, req->length, ®); if (UCC_OK != status) { tl_warn(comm->lib, "unable to register receive buffer %p of size %ld", req->ptr, req->length); ucc_free(schedule); return status; } req->recv_rreg = reg; req->recv_mr = reg->mr; } if (comm->one_sided.reliability_enabled) { req->one_sided_reliability_scheme = ONE_SIDED_SYNCHRONOUS_PROTO; } else { req->one_sided_reliability_scheme = ONE_SIDED_NO_RELIABILITY; } return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_mcast_bcast_reliability_ready(ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_tl_mlx5_mcast_coll_comm_t *comm = req->comm; ucc_tl_mlx5_mcast_reg_t *reg = NULL; ucc_status_t status; ucc_assert(req->ag_counter == comm->bcast_comm.under_progress_counter); if (!comm->one_sided.reliability_enabled || comm->one_sided.reliability_ready) { return UCC_OK; } if (req->bcast_rkeys_req) { status = comm->service_coll.coll_test(req->bcast_rkeys_req); if (status == UCC_OK) { req->bcast_rkeys_req = NULL; tl_trace(comm->lib, "bcast for remote_addr/rkey is completed"); comm->one_sided.reliability_ready = 1; } return status; } /* initialize the structures needed by reliability protocol */ memset(comm->one_sided.recvd_pkts_tracker, 0, comm->commsize * sizeof(uint32_t)); memset(comm->one_sided.remote_slot_info, ONE_SIDED_INVALID, comm->commsize * sizeof(int)); /* local slots state */ comm->one_sided.slots_state = ONE_SIDED_INVALID; /* do nonblocking bcast over remote addresses/keys */ if (!req->rreg) { /* register sbuf if it is not registered before */ status = ucc_tl_mlx5_mcast_mem_register(comm->ctx, req->ptr, req->length, ®); if (UCC_OK != status) { return status; } req->rreg = reg; req->mr = reg->mr; } if (req->am_root) { comm->one_sided.sendbuf_memkey_list[comm->rank].rkey = req->mr->rkey; comm->one_sided.sendbuf_memkey_list[comm->rank].remote_addr = (uint64_t)req->ptr; } tl_trace(comm->lib, "bcast over buffer addresses/rkey: address %p rkey %d", req->ptr, req->mr->rkey); status = comm->service_coll.bcast_post(comm->p2p_ctx, &(comm->one_sided.sendbuf_memkey_list[req->root]), sizeof(ucc_tl_mlx5_mcast_slot_mem_info_t), req->root, &req->bcast_rkeys_req); if (UCC_OK != status) { tl_error(comm->lib, "oob bcast failed during one-sided reliability reset of a collective call"); return status; } return UCC_INPROGRESS; } static inline ucc_status_t ucc_tl_mlx5_mcast_check_zcopy_bcast_collective(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_tl_mlx5_mcast_pipelined_ag_schedule_t *sched = req->ag_schedule; ucc_assert(req->bcast_rkeys_req == NULL); if (comm->one_sided.pending_reads) { return ucc_tl_mlx5_mcast_progress_one_sided_communication(comm, req); } if (req->step == req->total_steps) { return UCC_OK; } if (!sched[req->step].prepost_buf_op_done || !sched[req->step].multicast_op_done) { // it is not yet the time to start the reliability protocol return UCC_INPROGRESS; } if (sched[req->step].num_recvd == sched[req->step].to_recv) { /* check for out of order packets, if any root sent a out of order * packet to us in the current step, go ahead and issue RDMA READ * from that root for this specific piece of send buffer */ return ucc_tl_mlx5_mcast_reliable_zcopy_pipelined_one_sided_get(comm, req, NULL); } else if (!comm->timer) { if (comm->stalled < DROP_THRESHOLD) { comm->stalled++; } else { // kick the timer comm->timer = ucc_tl_mlx5_mcast_get_timer(); comm->stalled = 0; } } else { if (comm->stalled < DROP_THRESHOLD) { comm->stalled++; } else { // calcuate the current time and check if it's time to do RDMA READ if (ucc_tl_mlx5_mcast_get_timer() - comm->timer >= comm->ctx->params.timeout) { comm->timer = 0; ucc_assert(sched[req->step].to_recv >= sched[req->step].num_recvd); tl_debug(comm->lib, "zcopy bcast timeout %d pending packets to recv %d on step %d", comm->ctx->params.timeout, sched[req->step].to_recv - sched[req->step].num_recvd, req->step); return ucc_tl_mlx5_mcast_reliable_zcopy_pipelined_one_sided_get(comm, req, NULL); } else { comm->stalled = 0; } } } return UCC_INPROGRESS; } static inline ucc_status_t ucc_tl_mlx5_mcast_do_zero_copy_pipelined_bcast(void *req_handle) { ucc_tl_mlx5_mcast_coll_req_t *req = (ucc_tl_mlx5_mcast_coll_req_t *)req_handle; ucc_tl_mlx5_mcast_coll_comm_t *comm = req->comm; const int zcopy = req->proto != MCAST_PROTO_EAGER; ucc_tl_mlx5_mcast_pipelined_ag_schedule_t *sched = req->ag_schedule; int root = req->root; int num_recvd = 0; int to_send_left = 0; int j = 0; int group_id = 0; int num_packets = 0; int count = 0; size_t offset = 0; size_t offset_left = 0; ucc_status_t status; if (req->ag_counter != req->comm->bcast_comm.under_progress_counter) { /* it is not this task's turn for progress */ ucc_assert(req->comm->bcast_comm.under_progress_counter < req->ag_counter); return UCC_INPROGRESS; } status = ucc_tl_mlx5_mcast_bcast_reliability_ready(req); if (UCC_OK != status) { return status; } ucc_assert(req->to_recv >= 0 && req->to_send >= 0); if (req->barrier_req) { status = comm->service_coll.coll_test(req->barrier_req); if (status != UCC_OK) { return status; } tl_trace(comm->lib, "barrier at end of req->step %d is completed", req->step); req->barrier_req = NULL; req->step++; if (comm->one_sided.reliability_enabled) { memset(comm->one_sided.recvd_pkts_tracker, 0, comm->commsize * sizeof(uint32_t)); } } if (req->step < sched->total_steps) { if (!sched[req->step].multicast_op_done) { for (j = 0; j < req->concurrency_level; j++) { ucc_assert(root == sched[req->step].multicast_op[j].root); group_id = sched[req->step].multicast_op[j].group_id; to_send_left = sched[req->step].multicast_op[j].to_send_left; offset_left = sched[req->step].multicast_op[j].offset_left; num_packets = ucc_min(comm->bcast_comm.max_push_send - comm->pending_send, to_send_left); if (to_send_left && (comm->bcast_comm.max_push_send - comm->pending_send) > 0) { ucc_assert(req->am_root); status = ucc_tl_mlx5_mcast_send_collective(comm, req, num_packets, zcopy, group_id, offset_left); if (UCC_OK != status) { return status; } sched[req->step].multicast_op[j].to_send_left -= num_packets; sched[req->step].multicast_op[j].offset_left += (num_packets * comm->max_per_packet); } if (comm->pending_send) { status = ucc_tl_mlx5_mcast_poll_send(comm); if (status != UCC_OK) { return status; } } if (!sched[req->step].multicast_op[j].to_send_left && !comm->pending_send) { tl_trace(comm->lib, "done with mcast ops step %d group id %d to_send %d", req->step, group_id, sched[req->step].to_send); sched[req->step].multicast_op_done = 1; break; } } } if (!sched[req->step].prepost_buf_op_done) { /* prepost the user buffers for none roots */ for (j = 0; j < req->concurrency_level; j++) { group_id = sched[req->step].prepost_buf_op[j].group_id; count = sched[req->step].prepost_buf_op[j].count; offset = sched[req->step].prepost_buf_op[j].offset; if (count) { ucc_assert(!req->am_root); status = ucc_tl_mlx5_mcast_post_user_recv_buffers(comm, req, group_id, root, UCC_COLL_TYPE_BCAST, count, offset); if (UCC_OK != status) { return status; } } /* progress the recvd packets in between */ if (req->to_recv) { num_recvd = ucc_tl_mlx5_mcast_recv_collective(comm, req, req->to_recv, UCC_COLL_TYPE_BCAST); if (num_recvd < 0) { tl_error(comm->lib, "a failure happend during cq polling"); status = UCC_ERR_NO_MESSAGE; return status; } sched[req->step].num_recvd += num_recvd; } } tl_trace(comm->lib, "done with prepost bufs step %d group id %d count %d offset %ld root %d", req->step, group_id, count, offset, root); sched[req->step].prepost_buf_op_done = 1; } if (req->to_recv) { num_recvd = ucc_tl_mlx5_mcast_recv_collective(comm, req, req->to_recv, UCC_COLL_TYPE_BCAST); if (num_recvd < 0) { tl_error(comm->lib, "a failure happend during cq polling"); status = UCC_ERR_NO_MESSAGE; return status; } sched[req->step].num_recvd += num_recvd; } if (sched[req->step].prepost_buf_op_done && sched[req->step].multicast_op_done && sched[req->step].num_recvd == sched[req->step].to_recv) { // current step done ucc_assert(sched[req->step].prepost_buf_op_done && sched[req->step].multicast_op_done); ucc_assert(req->barrier_req == NULL); status = comm->service_coll.barrier_post(comm->p2p_ctx, &req->barrier_req); if (status != UCC_OK) { return status; } tl_trace(comm->lib, "init global sync req->step %d", req->step); } } if (comm->one_sided.reliability_enabled) { status = ucc_tl_mlx5_mcast_check_zcopy_bcast_collective(comm, req); if (status < 0) { return status; } } if (req->barrier_req != NULL || MCAST_BCAST_IN_PROGRESS(req, comm)) { return UCC_INPROGRESS; } /* all completed */ assert(req->step == sched->total_steps); return UCC_OK; } static inline ucc_status_t ucc_tl_mlx5_mcast_prepare_bcast(void* buf, size_t size, ucc_rank_t root, ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req) { ucc_status_t status; ucc_tl_mlx5_mcast_reg_t *reg; req->comm = comm; req->ptr = buf; req->length = size; req->root = root; req->am_root = (root == comm->rank); req->mr = comm->pp_mr; req->rreg = NULL; /* cost of copy is too high in cuda buffers */ req->proto = (req->length < comm->max_eager && !comm->cuda_mem_enabled) ? MCAST_PROTO_EAGER : MCAST_PROTO_ZCOPY; req->num_packets = ucc_div_round_up(req->length, comm->max_per_packet); req->offset = 0; req->to_send = req->am_root ? req->num_packets : 0; req->to_recv = req->am_root ? 0 : req->num_packets; if (comm->bcast_comm.truly_zero_copy_bcast_enabled) { status = ucc_tl_mlx5_mcast_prepare_zero_copy_bcast(comm, req); if (UCC_OK != status) { return status; } req->progress = ucc_tl_mlx5_mcast_do_zero_copy_pipelined_bcast; } else { status = ucc_tl_mlx5_mcast_prepare_reliable(comm, req, req->root); if (ucc_unlikely(UCC_OK != status)) { return status; } req->start_psn = comm->bcast_comm.last_psn; req->last_pkt_len = req->length - (req->num_packets - 1) * comm->max_per_packet; ucc_assert(req->last_pkt_len > 0 && req->last_pkt_len <= comm->max_per_packet); comm->bcast_comm.last_psn += req->num_packets; req->first_send_psn = req->start_psn; req->progress = ucc_tl_mlx5_mcast_do_bcast; } if (req->am_root) { if (req->proto != MCAST_PROTO_EAGER) { status = ucc_tl_mlx5_mcast_mem_register(comm->ctx, req->ptr, req->length, ®); if (UCC_OK != status) { if (req->ag_schedule) { ucc_free(req->ag_schedule); } return status; } req->rreg = reg; req->mr = reg->mr; } } return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_coll_do_bcast(void* buf, size_t size, ucc_rank_t root, ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t **task_req_handle) { ucc_status_t status; ucc_tl_mlx5_mcast_coll_req_t *req; tl_trace(comm->lib, "MCAST bcast start, buf %p, size %ld, root %d, comm %d, " "comm_size %d, am_i_root %d comm->psn = %d \n", buf, size, root, comm->comm_id, comm->commsize, comm->rank == root, comm->psn ); req = ucc_mpool_get(&comm->ctx->mcast_req_mp); if (!req) { tl_error(comm->lib, "failed to get mcast req"); return UCC_ERR_NO_MEMORY; } memset(req, 0, sizeof(ucc_tl_mlx5_mcast_coll_req_t)); status = ucc_tl_mlx5_mcast_prepare_bcast(buf, size, root, comm, req); if (UCC_OK != status) { ucc_mpool_put(req); return status; } status = UCC_INPROGRESS; *task_req_handle = req; return status; } ucc_status_t ucc_tl_mlx5_mcast_bcast_start(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_task_t *task = ucc_derived_of(coll_task, ucc_tl_mlx5_task_t); ucc_tl_mlx5_team_t *mlx5_team = TASK_TEAM(task); ucc_tl_mlx5_mcast_team_t *team = mlx5_team->mcast; ucc_coll_args_t *args = &TASK_ARGS(task); ucc_datatype_t dt = args->src.info.datatype; size_t count = args->src.info.count; ucc_rank_t root = args->root; ucc_status_t status = UCC_OK; size_t data_size = ucc_dt_size(dt) * count; void *buf = args->src.info.buffer; ucc_tl_mlx5_mcast_coll_comm_t *comm = team->mcast_comm; task->coll_mcast.req_handle = NULL; status = ucc_tl_mlx5_mcast_coll_do_bcast(buf, data_size, root, comm, &task->coll_mcast.req_handle); if (status < 0) { tl_error(UCC_TASK_LIB(task), "mcast_coll_do_bcast failed:%d", status); coll_task->status = status; return ucc_task_complete(coll_task); } ucc_assert(task->coll_mcast.req_handle != NULL); coll_task->status = status; task->coll_mcast.req_handle->coll_task = coll_task; return ucc_progress_queue_enqueue(UCC_TL_CORE_CTX(mlx5_team)->pq, &task->super); } static inline void ucc_tl_mlx5_mcast_bcast_progress(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_task_t *task = ucc_derived_of(coll_task, ucc_tl_mlx5_task_t); ucc_tl_mlx5_mcast_coll_req_t *req = task->coll_mcast.req_handle; ucc_assert(req != NULL); coll_task->status = (req->progress)(req); if (coll_task->status < 0) { tl_error(UCC_TASK_LIB(task), "progress mcast bcast failed:%d", coll_task->status); } } static inline ucc_status_t ucc_tl_mlx5_mcast_check_comm_level_cap(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team) { ucc_tl_mlx5_team_t *mlx5_team = ucc_derived_of(team, ucc_tl_mlx5_team_t); ucc_tl_mlx5_mcast_coll_comm_t *comm = mlx5_team->mcast->mcast_comm; ucc_coll_args_t *args = &coll_args->args; size_t buf_size = ucc_dt_size(args->src.info.datatype) * args->src.info.count; ucc_memory_type_t mem_type = args->src.info.mem_type; ucc_coll_type_t coll_type = args->coll_type; bool mcast_enabled = false; bool zero_copy_supported = false; /* Check supported memory types */ if (mem_type != UCC_MEMORY_TYPE_CUDA && mem_type != UCC_MEMORY_TYPE_HOST) { tl_trace(comm->lib, "unsupported memory type %d for mlx5 multicast", mem_type); return UCC_ERR_NO_RESOURCE; } /* Check CUDA memory enablement consistency */ if ((comm->cuda_mem_enabled && mem_type == UCC_MEMORY_TYPE_HOST) || (!comm->cuda_mem_enabled && mem_type == UCC_MEMORY_TYPE_CUDA)) { tl_trace(comm->lib, "CUDA memory usage inconsistent with configuration " "(enabled: %d, requested: %d)", comm->cuda_mem_enabled, mem_type == UCC_MEMORY_TYPE_CUDA); return UCC_ERR_NO_RESOURCE; } /* Check if multicast is enabled for the specific collective type */ if (coll_type == UCC_COLL_TYPE_BCAST && comm->context->mcast_bcast_enabled) { mcast_enabled = true; } else if (coll_type == UCC_COLL_TYPE_ALLGATHER && comm->context->mcast_allgather_enabled) { mcast_enabled = true; } if (!mcast_enabled) { tl_trace(comm->lib, "multicast for collective type %s is disabled", ucc_coll_type_str(coll_type)); return UCC_ERR_NO_RESOURCE; } /* Check zero-copy requirements */ if (coll_type == UCC_COLL_TYPE_BCAST && comm->bcast_comm.truly_zero_copy_bcast_enabled) { zero_copy_supported = true; } else if (coll_type == UCC_COLL_TYPE_ALLGATHER && comm->allgather_comm.truly_zero_copy_allgather_enabled) { zero_copy_supported = true; } if (zero_copy_supported) { if (coll_type == UCC_COLL_TYPE_BCAST && (buf_size < comm->truly_zero_copy_coll_min_msg || buf_size % comm->max_per_packet != 0)) { tl_trace(comm->lib, "zero-copy bcast requirements not met " "(buf_size: %zu, min_msg: %d, max_packet: %d)", buf_size, comm->truly_zero_copy_coll_min_msg, comm->max_per_packet); return UCC_ERR_NO_RESOURCE; } else if (coll_type == UCC_COLL_TYPE_ALLGATHER && buf_size >= comm->max_per_packet && buf_size % comm->max_per_packet != 0) { /* For allgather, allow small messages (< max_per_packet) and * only enforce packet size multiple for large messages */ tl_trace(comm->lib, "zero-copy allgather requirements not met: " "large message (>= %d bytes) must be multiple of max_per_packet (%d)", comm->max_per_packet, comm->max_per_packet); return UCC_ERR_NO_RESOURCE; } } /* Specific checks for CUDA BCAST without zero-copy */ if (mem_type == UCC_MEMORY_TYPE_CUDA && coll_type == UCC_COLL_TYPE_BCAST && buf_size > CUDA_MEM_MCAST_BCAST_MAX_MSG && !comm->bcast_comm.truly_zero_copy_bcast_enabled) { tl_trace(comm->lib, "mcast cuda bcast with size %zu not supported without " "zero copy enabled (max size: %d)", buf_size, CUDA_MEM_MCAST_BCAST_MAX_MSG); return UCC_ERR_NO_RESOURCE; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_check_support(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team) { if (UCC_COLL_ARGS_ACTIVE_SET(&coll_args->args) || ((coll_args->args.coll_type == UCC_COLL_TYPE_ALLGATHER) && (UCC_IS_INPLACE(coll_args->args) || UCC_IS_PERSISTENT(coll_args->args)))) { tl_trace(team->context->lib, "mcast collective not supported"); return UCC_ERR_NOT_SUPPORTED; } if (UCC_OK != ucc_tl_mlx5_mcast_check_comm_level_cap(coll_args, team)) { tl_trace(team->context->lib, "mcast collective not supported"); return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_mcast_bcast_init(ucc_tl_mlx5_task_t *task) { task->super.post = ucc_tl_mlx5_mcast_bcast_start; task->super.progress = ucc_tl_mlx5_mcast_bcast_progress; task->super.flags = UCC_COLL_TASK_FLAG_EXECUTOR; return UCC_OK; } ucc-1.8.0/src/components/tl/mlx5/mcast/tl_mlx5_mcast.h0000664000175000017500000007124115211535620022776 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_MCAST_H #define UCC_MCAST_H #include #include #include #include #include "utils/ucc_list.h" #include "utils/ucc_mpool.h" #include "components/tl/ucc_tl.h" #include "components/tl/ucc_tl_log.h" #include "utils/ucc_rcache.h" #include "core/ucc_service_coll.h" #include "components/mc/ucc_mc.h" #define POLL_PACKED 16 #define REL_DONE ((void*)-1) #define NB_POLL 8 #define NB_POLL_LARGE 32 #define MULTICAST_QPN 0xFFFFFF /* default parameters during modify QP */ #define DEF_QKEY 0x1a1a1a1a #define DEF_PKEY 0xffff #define DEF_PSN 0 #define DEF_SL 0 #define DEF_SRC_PATH_BITS 0 #define GRH_LENGTH 40 #define DROP_THRESHOLD 10000 #define MAX_COMM_POW2 32 #define MAX_GROUP_COUNT 64 /* Allgather RDMA-based reliability designs */ #define ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE 1024u #define ONE_SIDED_SLOTS_COUNT 2 /* number of memory slots during async design */ #define ONE_SIDED_SLOTS_INFO_SIZE sizeof(int) /* size of metadata prepended to each slots in bytes */ #define ONE_SIDED_MAX_ZCOPY_COLL_COUNTER 32u #define ONE_SIDED_MAX_CONCURRENT_LEVEL 64 enum ucc_tl_mlx5_mcast_one_sided_slot_states { ONE_SIDED_INVALID = -4, ONE_SIDED_VALID, ONE_SIDED_PENDING_INFO, ONE_SIDED_PENDING_DATA, }; enum ucc_tl_mlx5_mcast_one_sided_reliability_scheme { ONE_SIDED_NO_RELIABILITY = 0, ONE_SIDED_SYNCHRONOUS_PROTO, ONE_SIDED_ASYNCHRONOUS_PROTO }; #define CUDA_MEM_MCAST_BCAST_MAX_MSG 4096 enum { MCAST_PROTO_EAGER, /* Internal staging buffers */ MCAST_PROTO_ZCOPY }; enum { MCAST_P2P_NACK, MCAST_P2P_ACK, MCAST_P2P_NEED_NACK_SEND, MCAST_P2P_NACK_SEND_PENDING }; enum { MCAST_RECV_WR = 1, MCAST_WAIT_RECV_WR, MCAST_SEND_WR, MCAST_CALC_WR, MCAST_BCASTRECV_WR, MCAST_BCASTSEND_WR, MCAST_AG_RDMA_READ_INFO_WR, MCAST_AG_RDMA_READ_WR, }; struct ucc_tl_mlx5_mcast_p2p_completion_obj; typedef int (*ucc_tl_mlx5_mcast_p2p_completion_cb_fn_t)(struct ucc_tl_mlx5_mcast_p2p_completion_obj *obj); typedef struct ucc_tl_mlx5_mcast_p2p_completion_obj { ucc_list_link_t super; ucc_tl_mlx5_mcast_p2p_completion_cb_fn_t compl_cb; uint64_t data[3]; ucc_coll_req_h req; } ucc_tl_mlx5_mcast_p2p_completion_obj_t; typedef int (*ucc_tl_mlx5_mcast_p2p_wait_cb_fn_t)(void *wait_arg); typedef ucc_status_t (*ucc_tl_mlx5_mcast_p2p_send_nb_fn_t)(void* src, size_t size, ucc_rank_t rank, ucc_memory_type_t mem_type, void *context, ucc_tl_mlx5_mcast_p2p_completion_obj_t *compl_obj); typedef ucc_status_t (*ucc_tl_mlx5_mcast_p2p_recv_nb_fn_t)(void* src, size_t size, ucc_rank_t rank, ucc_memory_type_t mem_type, void *context, ucc_tl_mlx5_mcast_p2p_completion_obj_t *compl_obj); typedef struct ucc_tl_mlx5_mcast_p2p_interface { ucc_tl_mlx5_mcast_p2p_send_nb_fn_t send_nb; ucc_tl_mlx5_mcast_p2p_recv_nb_fn_t recv_nb; } ucc_tl_mlx5_mcast_p2p_interface_t; typedef struct ucc_tl_mlx5_mcast_coll_comm_init_spec { ucc_tl_mlx5_mcast_p2p_interface_t p2p_iface; int sx_depth; int rx_depth; int sx_sge; int rx_sge; int sx_inline; int post_recv_thresh; int scq_moderation; int wsize; int mcast_group_count; int max_push_send; int max_eager; int cuda_mem_enabled; int one_sided_reliability_enable; int reliability_scheme_msg_threshold; int truly_zero_copy_allgather_enabled; int truly_zero_copy_bcast_enabled; int truly_zero_copy_coll_min_msg; int mcast_prepost_bucket_size; int hca_copy_enabled; void *oob; } ucc_tl_mlx5_mcast_coll_comm_init_spec_t; typedef struct ucc_tl_mlx5_mcast_context_config { ucc_tl_context_config_t super; char *dev_list; int use_rcache; size_t reg_threshold; unsigned int rand_seed; unsigned int uprogress_num_polls; int context_per_team; } ucc_tl_mlx5_mcast_context_config_t; typedef struct ucc_tl_mlx5_mcast_oob_ctx { void *ctx; union { ucc_oob_coll_t *oob; ucc_subset_t subset; }; } ucc_tl_mlx5_mcast_oob_ctx_t; typedef struct ucc_tl_mlx5_mcast_reg { void *mr; } ucc_tl_mlx5_mcast_reg_t; typedef struct ucc_tl_mlx5_mcast_rcache_region { ucc_rcache_region_t super; ucc_tl_mlx5_mcast_reg_t reg; } ucc_tl_mlx5_mcast_rcache_region_t; typedef struct ucc_tl_mlx5_mcast_ctx_params { ucc_ternary_auto_value_t mcast_enabled; char *ib_dev_name; int print_nack_stats; int timeout; int mcast_bcast_enabled; int mcast_allgather_enabled; } ucc_tl_mlx5_mcast_ctx_params_t; typedef struct ucc_tl_mlx5_mcast_coll_context { struct ibv_context *ctx; struct ibv_pd *pd; char *devname; int max_qp_wr; int user_provided_ib; int ib_port; int pkey_index; int mtu; uint16_t port_lid; struct rdma_cm_id *id; struct rdma_event_channel *channel; ucc_mpool_t compl_objects_mp; ucc_mpool_t mcast_req_mp; ucc_list_link_t pending_nacks_list; ucc_rcache_t *rcache; ucc_tl_mlx5_mcast_ctx_params_t params; ucc_base_lib_t *lib; } ucc_tl_mlx5_mcast_coll_context_t; typedef struct ucc_tl_mlx5_mcast_join_info_t { ucc_status_t status; uint16_t dlid[MAX_GROUP_COUNT]; union ibv_gid dgid[MAX_GROUP_COUNT]; } ucc_tl_mlx5_mcast_join_info_t; typedef struct ucc_tl_mlx5_mcast_context { ucc_thread_mode_t tm; ucc_tl_mlx5_mcast_coll_context_t mcast_context; ucc_tl_mlx5_mcast_context_config_t cfg; ucc_ternary_auto_value_t mcast_enabled; int mcast_ctx_ready; ucc_tl_mlx5_mcast_oob_ctx_t oob_ctx; int mcast_bcast_enabled; int mcast_allgather_enabled; } ucc_tl_mlx5_mcast_context_t; struct pp_packet { ucc_list_link_t super; uint32_t psn; int length; int packet_counter; uintptr_t context; int qp_id; uintptr_t buf; // buffer address, initialized once }; struct mcast_group { struct ibv_qp *qp; struct ibv_ah *ah; uint16_t lid; union ibv_gid mgid; struct sockaddr_in6 mcast_addr; }; struct mcast_ctx { struct ibv_send_wr swr; struct ibv_sge ssg; struct ibv_cq *scq; struct ibv_cq *rcq; struct ibv_srq *srq; struct mcast_group groups[MAX_GROUP_COUNT]; // RC connection info for supporing one-sided based relibality struct ibv_qp **rc_qp; }; struct packet { int type; ucc_rank_t from; uint32_t psn; int comm_id; }; typedef struct ucc_tl_mlx5_mcast_slot_mem_info { uint64_t remote_addr; uint32_t rkey; } ucc_tl_mlx5_mcast_slot_mem_info_t; typedef struct ucc_tl_mlx5_one_sided_reliable_team_info { ucc_tl_mlx5_mcast_slot_mem_info_t slot_mem; uint16_t port_lid; uint32_t rc_qp_num[ONE_SIDED_RELIABILITY_MAX_TEAM_SIZE]; } ucc_tl_mlx5_one_sided_reliable_team_info_t; typedef struct ucc_tl_mlx5_mcast_per_qp_posted_recv_info { ucc_list_link_t posted_recv_bufs; int posted_recvs_count; } ucc_tl_mlx5_mcast_per_qp_posted_recv_info_t; typedef struct ucc_tl_mlx5_mcast_one_sided_reliability_comm { /* all the info required for establishing a reliable connection as * well as temp slots memkeys that all processes in the team need * to be aware of*/ ucc_tl_mlx5_one_sided_reliable_team_info_t *info; /* holds all the remote-addr/rkey of sendbuf from processes in the team * used in sync design. it needs to be set during each mcast-allgather call * after sendbuf registration */ ucc_tl_mlx5_mcast_slot_mem_info_t *sendbuf_memkey_list; /* counter for each target recv packet */ uint32_t *recvd_pkts_tracker; /* holds the remote targets' collective call counter. it is used to check * if remote temp slot is ready for RDMA READ in async design */ int *remote_slot_info; struct ibv_mr *remote_slot_info_mr; int reliability_scheme_msg_threshold; /* mem address and mem keys of the temp slots in async design */ char *slots_buffer; struct ibv_mr *slots_mr; /* size of a temp slot in async design */ int slot_size; /* coll req that is used during the oob service calls */ ucc_service_coll_req_t *reliability_req; int reliability_enabled; int reliability_ready; int pending_reads; int hca_copy_enabled; enum ucc_tl_mlx5_mcast_one_sided_slot_states slots_state; ucc_tl_mlx5_mcast_per_qp_posted_recv_info_t posted_recv[MAX_GROUP_COUNT]; } ucc_tl_mlx5_mcast_one_sided_reliability_comm_t; typedef struct ucc_tl_mlx5_mcast_service_coll { ucc_status_t (*bcast_post) (void*, void*, size_t, ucc_rank_t, ucc_service_coll_req_t**); ucc_status_t (*allgather_post) (void*, void*, void*, size_t, ucc_service_coll_req_t**); ucc_status_t (*allreduce_post) (void*, void*, void*, size_t, ucc_datatype_t, ucc_reduction_op_t, ucc_service_coll_req_t**); ucc_status_t (*barrier_post) (void*, ucc_service_coll_req_t**); ucc_status_t (*coll_test) (ucc_service_coll_req_t*); } ucc_tl_mlx5_mcast_service_coll_t; typedef struct ucc_tl_mlx5_mcast_allgather_comm { uint32_t under_progress_counter; uint32_t coll_counter; uint32_t max_num_packets; uint32_t max_push_send; uint8_t truly_zero_copy_allgather_enabled; uint32_t mcast_prepost_bucket_size; } ucc_tl_mlx5_mcast_allgather_comm_t; typedef struct ucc_tl_mlx5_mcast_bcast_comm { uint32_t under_progress_counter; uint32_t coll_counter; uint32_t last_psn; uint32_t racks_n; uint32_t sacks_n; uint32_t last_acked; uint32_t child_n; uint32_t parent_n; struct packet p2p_pkt[MAX_COMM_POW2]; struct packet p2p_spkt[MAX_COMM_POW2]; int reliable_in_progress; int recv_drop_packet_in_progress; ucc_rank_t parents[MAX_COMM_POW2]; ucc_rank_t children[MAX_COMM_POW2]; int nack_requests; int nacks_counter; int n_mcast_reliable; int wsize; uint32_t mcast_prepost_bucket_size; uint8_t truly_zero_copy_bcast_enabled; uint32_t max_push_send; } ucc_tl_mlx5_mcast_bcast_comm_t; typedef struct ucc_tl_mlx5_mcast_coll_comm { struct pp_packet dummy_packet; ucc_tl_mlx5_mcast_coll_context_t *ctx; ucc_tl_mlx5_mcast_coll_comm_init_spec_t params; ucc_tl_mlx5_mcast_p2p_interface_t p2p; int tx; ucc_rank_t rank; ucc_rank_t commsize; char *grh_buf; struct ibv_mr *grh_mr; unsigned max_inline; size_t max_eager; int max_per_packet; int pending_send; int pending_recv; struct ibv_mr *pp_mr; char *pp_buf; ucc_mc_buffer_header_t *pp_buf_header; struct pp_packet *pp; uint32_t psn; int buf_n; ucc_list_link_t bpool; ucc_list_link_t pending_q; struct mcast_ctx mcast; struct ibv_recv_wr *call_rwr; struct ibv_sge *call_rsgs; uint64_t timer; int stalled; int comm_id; void *p2p_ctx; ucc_base_lib_t *lib; int cuda_mem_enabled; ucc_tl_mlx5_mcast_join_info_t *group_setup_info; ucc_service_coll_req_t *group_setup_info_req; int mcast_transport_ready; int transport_ready_global; ucc_service_coll_req_t *transport_ready_req; ucc_tl_mlx5_mcast_service_coll_t service_coll; struct rdma_cm_event *event; ucc_tl_mlx5_mcast_one_sided_reliability_comm_t one_sided; int mcast_group_count; ucc_tl_mlx5_mcast_allgather_comm_t allgather_comm; ucc_tl_mlx5_mcast_bcast_comm_t bcast_comm; uint32_t truly_zero_copy_coll_min_msg; ucc_tl_mlx5_mcast_context_t *context; /* Dedicated HCA copy resources - separate from mcast operations */ struct ibv_cq *hca_copy_cq; /* Dedicated CQ for HCA copy */ struct ibv_qp *hca_copy_qp; /* Dedicated QP for HCA copy */ struct pp_packet *r_window[1]; // note: do not add any new variable after here } ucc_tl_mlx5_mcast_coll_comm_t; typedef struct ucc_tl_mlx5_mcast_team { ucc_tl_mlx5_mcast_context_t *mcast_context; struct ucc_tl_mlx5_mcast_coll_comm *mcast_comm; ucc_tl_mlx5_mcast_oob_ctx_t oob_ctx; } ucc_tl_mlx5_mcast_team_t; typedef struct ucc_tl_mlx5_mcast_nack_req { ucc_list_link_t super; int pkt_id; ucc_tl_mlx5_mcast_coll_comm_t *comm; } ucc_tl_mlx5_mcast_nack_req_t; #define PSN_IS_IN_RANGE(_psn, _call, _comm) \ ( \ ((_psn >= _call->start_psn) && \ (_psn < _call->start_psn + _call->num_packets) && \ (_psn >= _comm->bcast_comm.last_acked) && \ (_psn < _comm->bcast_comm.last_acked + _comm->bcast_comm.wsize)) \ ) #define PSN_TO_RECV_OFFSET(_psn, _call, _comm) \ ( \ ((ptrdiff_t)((_psn - _call->start_psn) \ * (_comm->max_per_packet))) \ ) #define PSN_TO_RECV_LEN(_psn, _call, _comm) \ ( \ ((_psn - _call->start_psn + 1) % \ _call->num_packets == 0 ? _call->last_pkt_len : \ _comm->max_per_packet) \ ) #define PSN_RECEIVED(_psn, _comm) \ ( \ (_comm->r_window[(_psn) % \ _comm->bcast_comm.wsize]->psn == (_psn)) \ ) typedef struct ucc_tl_mlx5_mcast_tensor { int group_id; size_t offset; size_t offset_left; int root; int count; int to_recv; int to_send_left; } ucc_tl_mlx5_mcast_tensor_t; typedef struct ucc_tl_mlx5_mcast_pipelined_ag_schedule { ucc_tl_mlx5_mcast_tensor_t multicast_op[ONE_SIDED_MAX_CONCURRENT_LEVEL]; ucc_tl_mlx5_mcast_tensor_t prepost_buf_op[ONE_SIDED_MAX_CONCURRENT_LEVEL]; int prepost_buf_op_done; int multicast_op_done; int total_steps; int num_recvd; int to_recv; int to_send; } ucc_tl_mlx5_mcast_pipelined_ag_schedule_t; typedef struct ucc_tl_mlx5_mcast_coll_req { ucc_tl_mlx5_mcast_coll_comm_t *comm; size_t length; int proto; struct ibv_mr *mr; struct ibv_mr *recv_mr; struct ibv_recv_wr *rwr; struct ibv_sge *rsgs; void *rreg; char *ptr; char *rptr; int am_root; ucc_rank_t root; void **rbufs; int first_send_psn; int to_send; int to_recv; ucc_rank_t parent; uint32_t start_psn; int num_packets; int last_pkt_len; int offset; ucc_memory_type_t buf_mem_type; enum ucc_tl_mlx5_mcast_one_sided_reliability_scheme one_sided_reliability_scheme; uint32_t ag_counter; int concurrency_level; int mcast_prepost_bucket_size; int state; ucc_tl_mlx5_mcast_pipelined_ag_schedule_t *ag_schedule; int total_steps; int step; ucc_service_coll_req_t *allgather_rkeys_req; ucc_service_coll_req_t *bcast_rkeys_req; ucc_service_coll_req_t *barrier_req; void *recv_rreg; ucc_ee_executor_task_t *exec_task; ucc_coll_task_t *coll_task; ucc_status_t (*progress) (void *req); /* Scratch buffer for efficient CUDA memory assembly */ char *scratch_buf; ucc_mc_buffer_header_t *scratch_buf_header; int scratch_packets_received; } ucc_tl_mlx5_mcast_coll_req_t; typedef struct ucc_tl_mlx5_mcast_oob_p2p_context { ucc_context_h base_ctx; ucc_team_h base_team; ucc_rank_t my_team_rank; ucc_subset_t subset; ucc_base_lib_t *lib; int tmp_sbuf; int tmp_rbuf; } ucc_tl_mlx5_mcast_oob_p2p_context_t; static inline struct pp_packet* ucc_tl_mlx5_mcast_buf_get_free(ucc_tl_mlx5_mcast_coll_comm_t* comm) { struct pp_packet* pp; pp = ucc_list_extract_head(&comm->bpool, struct pp_packet, super); ucc_assert(pp == NULL || pp->context == 0); return pp; } static inline ucc_status_t ucc_tl_mlx5_mcast_post_recv_buffers(ucc_tl_mlx5_mcast_coll_comm_t* comm) { struct ibv_recv_wr *bad_wr = NULL; struct ibv_recv_wr *rwr = comm->call_rwr; struct ibv_sge *sge = comm->call_rsgs; struct pp_packet *pp = NULL; int i; int count; int count_per_qp; count = comm->params.rx_depth - comm->pending_recv; if (comm->allgather_comm.truly_zero_copy_allgather_enabled || comm->bcast_comm.truly_zero_copy_bcast_enabled || count <= comm->params.post_recv_thresh) { return UCC_OK; } count_per_qp = count / comm->mcast_group_count; for (i = 0; i < count_per_qp; i++) { if (NULL == (pp = ucc_tl_mlx5_mcast_buf_get_free(comm))) { break; } rwr[i].wr_id = ((uint64_t) pp); rwr[i].next = &rwr[i+1]; sge[2*i + 1].addr = pp->buf; assert((uint64_t)comm->pp <= rwr[i].wr_id && ((uint64_t)comm->pp + comm->buf_n * sizeof(struct pp_packet)) > rwr[i].wr_id); } if (i > 0) { rwr[i-1].next = NULL; if (ibv_post_recv(comm->mcast.groups[0].qp, &rwr[0], &bad_wr)) { tl_error(comm->lib, "Failed to prepost recvs: errno %d qp index %d buffer count %d", errno, 0, i); return UCC_ERR_NO_RESOURCE; } comm->pending_recv += i; } return UCC_OK; } static inline uint64_t ucc_tl_mlx5_mcast_get_timer(void) { double t_second = ucc_get_time(); return (uint64_t) (t_second * 1000000); } static inline ucc_status_t ucc_tl_mlx5_mcast_post_user_recv_buffers(ucc_tl_mlx5_mcast_coll_comm_t *comm, ucc_tl_mlx5_mcast_coll_req_t *req, int group_id, ucc_rank_t root, int coll_type, int count, size_t offset) { struct ibv_recv_wr *bad_wr = NULL; struct ibv_recv_wr *rwr = comm->call_rwr; struct ibv_sge *sge = comm->call_rsgs; struct pp_packet *pp = NULL; uint32_t i; for (i = 0; i < count; i++) { if (NULL == (pp = ucc_tl_mlx5_mcast_buf_get_free(comm))) { tl_error(comm->lib, "not enought free pp packets to cover the entire message"); return UCC_ERR_NO_RESOURCE; } if (comm->one_sided.reliability_enabled && (comm->allgather_comm.truly_zero_copy_allgather_enabled || comm->bcast_comm.truly_zero_copy_bcast_enabled)) { /* need to keep track to cancel the posted recv if needed */ ucc_list_add_tail(&comm->one_sided.posted_recv[group_id].posted_recv_bufs, &pp->super); } assert(offset % comm->max_per_packet == 0); pp->packet_counter = offset / comm->max_per_packet; pp->qp_id = group_id; rwr[i].wr_id = ((uint64_t) pp); if (coll_type == UCC_COLL_TYPE_ALLGATHER) { sge[2*i + 1].addr = (uint64_t)req->rptr + root * req->length + offset; } else { ucc_assert(UCC_COLL_TYPE_BCAST == coll_type); sge[2*i + 1].addr = (uint64_t)req->ptr + offset; } sge[2*i + 1].lkey = req->recv_mr->lkey; offset += comm->max_per_packet; sge[2*i + 1].length = comm->max_per_packet; if (i < count - 1) { rwr[i].next = &rwr[i+1]; } } if (i > 0) { rwr[i-1].next = NULL; if (ibv_post_recv(comm->mcast.groups[group_id].qp, &rwr[0], &bad_wr)) { tl_error(comm->lib, "failed to prepost recvs: errno %d buffer count %d", errno, i); return UCC_ERR_NO_RESOURCE; } comm->pending_recv += i; comm->one_sided.posted_recv[group_id].posted_recvs_count += i; tl_trace(comm->lib, "posted %d buffers into recv queue with root %d mcast group %d" " with packet counter ranging from %ld to %ld", i, root, group_id, (offset-(i*comm->max_per_packet))/comm->max_per_packet, offset/comm->max_per_packet); } return UCC_OK; } #define EXEC_TASK_TEST(_errmsg, _etask, _lib) do { \ if (_etask != NULL) { \ status = ucc_ee_executor_task_test(_etask); \ if (status > 0) { \ return status; \ } \ ucc_ee_executor_task_finalize(_etask); \ _etask = NULL; \ if (ucc_unlikely(status < 0)) { \ tl_error(_lib, _errmsg); \ return status; \ } \ } \ } while(0) ucc_status_t ucc_tl_mlx5_mcast_team_init(ucc_base_context_t *tl_context, ucc_tl_mlx5_mcast_team_t **mcast_team, ucc_tl_mlx5_mcast_context_t *ctx, const ucc_base_team_params_t *params, ucc_tl_mlx5_mcast_coll_comm_init_spec_t *mcast_conf); ucc_status_t ucc_tl_mlx5_mcast_team_test(ucc_base_team_t *team); ucc_status_t ucc_tl_mlx5_mcast_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_mlx5_mcast_context_init(ucc_tl_mlx5_mcast_context_t *mcast_ctx, ucc_tl_mlx5_mcast_ctx_params_t *mcast_ctx_conf); ucc_status_t ucc_tl_mlx5_mcast_clean_ctx(ucc_tl_mlx5_mcast_coll_context_t *ctx); /* Conditional logging macro for mcast operations */ #define tl_mlx5_mcast_log(_mcast_enabled, _lib, _intended_level, _fmt, ...) \ do { \ if ((_mcast_enabled) == UCC_YES) { \ ucc_log_component((_intended_level), &((_lib)->log_component), (_fmt), ##__VA_ARGS__); \ } else if ((_mcast_enabled) == UCC_TRY) { \ ucc_log_component(UCC_LOG_LEVEL_DEBUG, &((_lib)->log_component), (_fmt), ##__VA_ARGS__); \ } \ } while(0) #endif ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_ib.c0000664000175000017500000004230215211535620021141 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_ib.h" #define UCC_QP_PKEY_INDEX 0 #define UCC_QP_PSN 0x123 #define UCC_QP_ACCESS_FLAGS (IBV_ACCESS_LOCAL_WRITE | \ IBV_ACCESS_REMOTE_READ | \ IBV_ACCESS_REMOTE_WRITE | \ IBV_ACCESS_REMOTE_ATOMIC) static ucc_status_t create_ctx_for_dev(struct ibv_device * ib_dev, struct ibv_context **ib_ctx) { struct mlx5dv_context_attr attr = {}; struct ibv_context *ctx; int port; /* Need to open the device with `MLX5DV_CONTEXT_FLAGS_DEVX` flag, since DEVX interfaced will be used*/ attr.flags = MLX5DV_CONTEXT_FLAGS_DEVX; ctx = mlx5dv_open_device(ib_dev, &attr); if (!ctx) { return UCC_ERR_NO_MESSAGE; } port = ucc_tl_mlx5_get_active_port(ctx); if (port < 0) { /* no active ports on this device */ ibv_close_device(ctx); return UCC_ERR_NO_MESSAGE; } *ib_ctx = ctx; return UCC_OK; } ucc_status_t ucc_tl_mlx5_create_ibv_ctx(char **ib_devname, struct ibv_context **ctx, ucc_base_lib_t *lib) { struct ibv_device **dev_list = ibv_get_device_list(NULL); ucc_status_t status; int i; if (!dev_list) { tl_debug(lib, "no IB devices are available"); return UCC_ERR_NOT_FOUND; } if (!(*ib_devname)) { /* If no device was specified by name, loop through all available until first with active port */ for (i = 0; dev_list[i]; ++i) { status = create_ctx_for_dev(dev_list[i], ctx); if (UCC_OK == status) { *ib_devname = (char *)ibv_get_device_name(dev_list[i]); status = UCC_OK; goto out; } } tl_debug(lib, "no IB devices found"); status = UCC_ERR_NOT_FOUND; goto out; } else { for (i = 0; dev_list[i]; ++i) { if (!strcmp(ibv_get_device_name(dev_list[i]), *ib_devname)) { break; } } if (!dev_list[i]) { tl_debug(lib, "IB device %s not found", *ib_devname); status = UCC_ERR_NOT_FOUND; goto out; } status = create_ctx_for_dev(dev_list[i], ctx); } out: if (dev_list) { ibv_free_device_list(dev_list); } return status; } int ucc_tl_mlx5_check_port_active(struct ibv_context *ctx, int port_num) { struct ibv_port_attr port_attr; ibv_query_port(ctx, port_num, &port_attr); if (port_attr.state == IBV_PORT_ACTIVE && port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) { return 1; } return 0; } int ucc_tl_mlx5_get_active_port(struct ibv_context *ctx) { struct ibv_device_attr device_attr; int i; ibv_query_device(ctx, &device_attr); for (i = 1; i <= device_attr.phys_port_cnt; i++) { if (ucc_tl_mlx5_check_port_active(ctx, i)) { return i; } } return -1; } ucc_status_t ucc_tl_mlx5_qp_connect(struct ibv_qp *qp, uint32_t qp_num, uint16_t lid, int port, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib) { struct ibv_qp_attr qp_attr; memset(&qp_attr, 0, sizeof(qp_attr)); qp_attr.qp_state = IBV_QPS_INIT; qp_attr.pkey_index = UCC_QP_PKEY_INDEX; qp_attr.port_num = port; qp_attr.qp_access_flags = UCC_QP_ACCESS_FLAGS; if (ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS) != 0) { tl_debug(lib, "QP RESET->INIT failed, %m"); return UCC_ERR_NO_MESSAGE; } memset((void *)&qp_attr, 0, sizeof(qp_attr)); qp_attr.qp_state = IBV_QPS_RTR; qp_attr.path_mtu = IBV_MTU_4096; qp_attr.dest_qp_num = qp_num; qp_attr.rq_psn = UCC_QP_PSN; qp_attr.min_rnr_timer = qp_conf->qp_rnr_timer; qp_attr.max_dest_rd_atomic = qp_conf->qp_max_atomic; qp_attr.ah_attr.dlid = lid; qp_attr.ah_attr.sl = qp_conf->qp_sl; qp_attr.ah_attr.src_path_bits = 0; qp_attr.ah_attr.port_num = port; if (ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) { tl_debug(lib, "QP INIT->RTR failed, %m"); return UCC_ERR_NO_MESSAGE; } // Modify QP to RTS qp_attr.qp_state = IBV_QPS_RTS; qp_attr.timeout = qp_conf->qp_timeout; qp_attr.retry_cnt = qp_conf->qp_retry_cnt; qp_attr.rnr_retry = qp_conf->qp_rnr_retry; qp_attr.sq_psn = UCC_QP_PSN; qp_attr.max_rd_atomic = qp_conf->qp_max_atomic; if (ibv_modify_qp(qp, &qp_attr, IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC)) { tl_debug(lib, "QP RTR->RTS failed, %m"); return UCC_ERR_NO_MESSAGE; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_init_dct(struct ibv_pd *pd, struct ibv_context *ctx, struct ibv_cq *cq, struct ibv_srq *srq, uint8_t port_num, struct ibv_qp **dct_qp, uint32_t *qpn, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib) { struct ibv_qp_init_attr_ex attr_ex; struct mlx5dv_qp_init_attr attr_dv; struct ibv_qp_attr qp_attr_to_init; struct ibv_qp_attr qp_attr_to_rtr; struct ibv_qp * qp; memset(&attr_ex, 0, sizeof(struct ibv_qp_init_attr_ex)); memset(&attr_dv, 0, sizeof(struct mlx5dv_qp_init_attr)); memset(&qp_attr_to_init, 0, sizeof(qp_attr_to_init)); memset(&qp_attr_to_rtr, 0, sizeof(qp_attr_to_rtr)); qp_attr_to_init.qp_state = IBV_QPS_INIT; qp_attr_to_init.pkey_index = UCC_QP_PKEY_INDEX; qp_attr_to_init.port_num = port_num; qp_attr_to_init.qp_access_flags = UCC_QP_ACCESS_FLAGS; qp_attr_to_rtr.qp_state = IBV_QPS_RTR; qp_attr_to_rtr.path_mtu = IBV_MTU_4096; qp_attr_to_rtr.min_rnr_timer = qp_conf->qp_rnr_timer; qp_attr_to_rtr.ah_attr.port_num = port_num; qp_attr_to_rtr.ah_attr.is_global = 0; qp_attr_to_rtr.ah_attr.sl = qp_conf->qp_sl; attr_ex.qp_type = IBV_QPT_DRIVER; attr_ex.send_cq = cq; attr_ex.recv_cq = cq; attr_ex.comp_mask |= IBV_QP_INIT_ATTR_PD; attr_ex.pd = pd; attr_ex.srq = srq; attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_DC; attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCT; attr_dv.dc_init_attr.dct_access_key = DC_KEY; qp = mlx5dv_create_qp(ctx, &attr_ex, &attr_dv); if (qp == NULL) { tl_debug(lib, "couldn't create DCT QP, %m"); return UCC_ERR_NO_MESSAGE; } if (ibv_modify_qp(qp, &qp_attr_to_init, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS) != 0) { tl_debug(lib, "failed to modify init qp, %m"); goto fail; } if (ibv_modify_qp(qp, &qp_attr_to_rtr, IBV_QP_STATE | IBV_QP_PATH_MTU | IBV_QP_AV | IBV_QP_MIN_RNR_TIMER) != 0) { tl_debug(lib, "failed to modify init qp, %m"); goto fail; } *dct_qp = qp; *qpn = qp->qp_num; return UCC_OK; fail: if (ibv_destroy_qp(qp)) { tl_debug(lib, "couldn't destroy QP, %m"); } return UCC_ERR_NO_MESSAGE; } ucc_status_t ucc_tl_mlx5_init_dci(ucc_tl_mlx5_dci_t *dci, struct ibv_pd *pd, struct ibv_context *ctx, struct ibv_cq *cq, uint8_t port_num, int tx_depth, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib) { struct ibv_qp_init_attr_ex attr_ex; struct mlx5dv_qp_init_attr attr_dv; struct ibv_qp_attr qp_attr_to_init; struct ibv_qp_attr qp_attr_to_rtr; struct ibv_qp_attr qp_attr_to_rts; memset(&attr_ex, 0, sizeof(attr_ex)); memset(&attr_dv, 0, sizeof(attr_dv)); memset(&qp_attr_to_init, 0, sizeof(qp_attr_to_init)); memset(&qp_attr_to_rtr, 0, sizeof(qp_attr_to_rtr)); memset(&qp_attr_to_rts, 0, sizeof(qp_attr_to_rts)); attr_ex.qp_type = IBV_QPT_DRIVER; attr_ex.send_cq = cq; attr_ex.recv_cq = cq; attr_ex.pd = pd; attr_ex.cap.max_send_wr = tx_depth; attr_ex.cap.max_send_sge = 1; attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | IBV_QP_INIT_ATTR_PD; attr_ex.send_ops_flags = IBV_QP_EX_WITH_RDMA_WRITE | IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM | IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD; attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_DC | MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS | MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS; attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI; attr_dv.create_flags |= MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE; attr_dv.send_ops_flags = MLX5DV_QP_EX_WITH_RAW_WQE; qp_attr_to_init.qp_state = IBV_QPS_INIT; qp_attr_to_init.pkey_index = UCC_QP_PKEY_INDEX; qp_attr_to_init.port_num = port_num; qp_attr_to_init.qp_access_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_ATOMIC; qp_attr_to_rtr.qp_state = IBV_QPS_RTR; qp_attr_to_rtr.path_mtu = IBV_MTU_4096; qp_attr_to_rtr.min_rnr_timer = qp_conf->qp_rnr_timer; qp_attr_to_rtr.ah_attr.port_num = port_num; qp_attr_to_rtr.ah_attr.is_global = 0; qp_attr_to_rtr.ah_attr.sl = qp_conf->qp_sl; qp_attr_to_rts.qp_state = IBV_QPS_RTS; qp_attr_to_rts.timeout = qp_conf->qp_timeout; qp_attr_to_rts.retry_cnt = qp_conf->qp_retry_cnt; qp_attr_to_rts.rnr_retry = qp_conf->qp_rnr_retry; qp_attr_to_rts.sq_psn = UCC_QP_PSN; qp_attr_to_rts.max_rd_atomic = qp_conf->qp_max_atomic; dci->dci_qp = mlx5dv_create_qp(ctx, &attr_ex, &attr_dv); if (!dci->dci_qp) { tl_debug(lib, "couldn't create DCI QP, %m"); return UCC_ERR_NO_MESSAGE; } // Turn DCI ibv_qp to ibv_qpex and ibv_mqpex dci->dc_qpex = ibv_qp_to_qp_ex(dci->dci_qp); if (!dci->dc_qpex) { tl_debug(lib, "failed turn ibv_qp to ibv_qp_ex, %m"); goto fail; } dci->dc_mqpex = mlx5dv_qp_ex_from_ibv_qp_ex(dci->dc_qpex); if (!dci->dc_mqpex) { tl_debug(lib, "failed turn ibv_qp_ex to mlx5dv_qp_ex, %m"); goto fail; } if (ibv_modify_qp(dci->dci_qp, &qp_attr_to_init, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT) != 0) { tl_debug(lib, "failed to modify init qp, %m"); goto fail; } if (ibv_modify_qp(dci->dci_qp, &qp_attr_to_rtr, IBV_QP_STATE | IBV_QP_PATH_MTU | IBV_QP_AV) != 0) { tl_debug(lib, "failed to modify qp to rtr, %m"); goto fail; } if (ibv_modify_qp(dci->dci_qp, &qp_attr_to_rts, IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER) != 0) { tl_debug(lib, "failed to modify qp to rts, %m"); goto fail; } return UCC_OK; fail: if (ibv_destroy_qp(dci->dci_qp)) { tl_debug(lib, "couldn't destroy qp, %m"); } return UCC_ERR_NO_MESSAGE; } ucc_status_t ucc_tl_mlx5_create_rc_qp(struct ibv_context *ctx, struct ibv_pd *pd, struct ibv_cq *cq, int tx_depth, ucc_tl_mlx5_qp_t *qp, uint32_t *qpn, ucc_base_lib_t *lib) { struct ibv_qp_init_attr_ex attr_ex; struct mlx5dv_qp_init_attr attr_dv; memset(&attr_ex, 0, sizeof(attr_ex)); memset(&attr_dv, 0, sizeof(attr_dv)); attr_ex.qp_type = IBV_QPT_RC; attr_ex.send_cq = cq; attr_ex.recv_cq = cq; attr_ex.pd = pd; attr_ex.cap.max_send_wr = tx_depth; attr_ex.cap.max_send_sge = 1; attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | IBV_QP_INIT_ATTR_PD; attr_ex.send_ops_flags = IBV_QP_EX_WITH_RDMA_WRITE | IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM | IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD; attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS | MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS; attr_dv.create_flags |= MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE; attr_dv.send_ops_flags = MLX5DV_QP_EX_WITH_RAW_WQE; qp->qp = mlx5dv_create_qp(ctx, &attr_ex, &attr_dv); if (!qp->qp) { tl_debug(lib, "failed to create RC QP, %m"); return UCC_ERR_NO_MESSAGE; } qp->qp_ex = ibv_qp_to_qp_ex(qp->qp); *qpn = qp->qp->qp_num; return UCC_OK; } ucc_status_t ucc_tl_mlx5_create_ah(struct ibv_pd *pd, uint16_t lid, uint8_t port_num, struct ibv_ah **ah_ptr, ucc_base_lib_t *lib) { struct ibv_ah_attr ah_attr; memset(&ah_attr, 0, sizeof(struct ibv_ah_attr)); ah_attr.dlid = lid; ah_attr.port_num = port_num; ah_attr.is_global = 0; ah_attr.grh.hop_limit = 0; *ah_ptr = ibv_create_ah(pd, &ah_attr); if (!(*ah_ptr)) { tl_debug(lib, "failed to create ah, %m"); return UCC_ERR_NO_MESSAGE; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_create_umr_qp(struct ibv_context *ctx, struct ibv_pd *pd, struct ibv_cq *cq, int ib_port, struct ibv_qp **qp, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib) { ucc_status_t status = UCC_OK; struct ibv_qp_ex * qp_ex = NULL; struct ibv_qp_init_attr_ex umr_init_attr_ex; struct mlx5dv_qp_init_attr umr_mlx5dv_qp_attr; struct ibv_port_attr port_attr; memset(&umr_mlx5dv_qp_attr, 0, sizeof(umr_mlx5dv_qp_attr)); memset(&umr_init_attr_ex, 0, sizeof(umr_init_attr_ex)); umr_mlx5dv_qp_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS; umr_mlx5dv_qp_attr.send_ops_flags = MLX5DV_QP_EX_WITH_MR_LIST | MLX5DV_QP_EX_WITH_MR_INTERLEAVED | MLX5DV_QP_EX_WITH_RAW_WQE; umr_init_attr_ex.send_cq = cq; umr_init_attr_ex.recv_cq = cq; umr_init_attr_ex.cap.max_send_wr = 1; umr_init_attr_ex.cap.max_recv_wr = 1; umr_init_attr_ex.cap.max_send_sge = 1; umr_init_attr_ex.cap.max_recv_sge = 1; umr_init_attr_ex.qp_type = IBV_QPT_RC; umr_init_attr_ex.comp_mask = IBV_QP_INIT_ATTR_SEND_OPS_FLAGS | IBV_QP_INIT_ATTR_PD; umr_init_attr_ex.pd = pd; umr_init_attr_ex.send_ops_flags |= IBV_QP_EX_WITH_SEND; /* In order to create UMRs using mlx5dv_wr_mr_list we need to have inline data support. Try to find max supported inline data size. Loop starting from 828 (expected max for cx7), if not supported decrease and go by multiples of 128 */ umr_init_attr_ex.cap.max_inline_data = 828; do { *qp = mlx5dv_create_qp(ctx, &umr_init_attr_ex, &umr_mlx5dv_qp_attr); if (umr_init_attr_ex.cap.max_inline_data == 828) { umr_init_attr_ex.cap.max_inline_data = 768; } else { umr_init_attr_ex.cap.max_inline_data -= 128; } } while (*qp == NULL && umr_init_attr_ex.cap.max_inline_data > 0); if (*qp == NULL) { tl_debug(lib, "failed to create UMR QP with inline_data, %m"); return UCC_ERR_NO_MESSAGE; } qp_ex = ibv_qp_to_qp_ex(*qp); if (qp_ex == NULL) { tl_debug(lib, "failed to create UMR qp_ex, %m"); status = UCC_ERR_NO_MESSAGE; goto failure; } qp_ex->wr_flags = IBV_SEND_INLINE | IBV_SEND_SIGNALED; if (ibv_query_port(ctx, ib_port, &port_attr)) { tl_debug(lib, "failed to get port info, %m"); status = UCC_ERR_NO_MESSAGE; goto failure; } status = ucc_tl_mlx5_qp_connect(*qp, (*qp)->qp_num, port_attr.lid, ib_port, qp_conf, lib); if (status != UCC_OK) { goto failure; } tl_debug(lib, "created UMR QP, cap.max_inline_data = %d", umr_init_attr_ex.cap.max_inline_data); return UCC_OK; failure: if (*qp) { if (ibv_destroy_qp(*qp)) { tl_debug(lib, "failed to destroy UMR QP, %m"); } } *qp = NULL; return status; } ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_coll.c0000664000175000017500000001103515211535620021477 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_coll.h" #include "mcast/tl_mlx5_mcast_coll.h" #include "mcast/tl_mlx5_mcast_allgather.h" #include "mcast/tl_mlx5_mcast_rcache.h" #include "alltoall/alltoall.h" ucc_status_t ucc_tl_mlx5_coll_mcast_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_status_t status = UCC_OK; ucc_tl_mlx5_task_t *task = NULL; status = ucc_tl_mlx5_mcast_check_support(coll_args, team); if (UCC_OK != status) { return status; } task = ucc_tl_mlx5_get_task(coll_args, team); if (ucc_unlikely(!task)) { return UCC_ERR_NO_MEMORY; } task->super.finalize = ucc_tl_mlx5_task_finalize; switch (coll_args->args.coll_type) { case UCC_COLL_TYPE_BCAST: status = ucc_tl_mlx5_mcast_bcast_init(task); if (ucc_unlikely(UCC_OK != status)) { goto free_task; } *task_h = &(task->super); break; case UCC_COLL_TYPE_ALLGATHER: status = ucc_tl_mlx5_mcast_allgather_init(task); if (ucc_unlikely(UCC_OK != status)) { goto free_task; } *task_h = &(task->super); break; default: status = UCC_ERR_NOT_SUPPORTED; tl_trace(team->context->lib, "mcast not supported for this collective type"); goto free_task; } tl_trace(UCC_TASK_LIB(task), "initialized mcast collective task %p", task); return UCC_OK; free_task: ucc_mpool_put(task); return status; } ucc_status_t ucc_tl_mlx5_task_finalize(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_task_t *task = ucc_derived_of(coll_task, ucc_tl_mlx5_task_t); ucc_tl_mlx5_mcast_coll_req_t *req = task->coll_mcast.req_handle; if (req != NULL) { ucc_assert(coll_task->status != UCC_INPROGRESS); ucc_assert(req->comm->ctx != NULL); if (coll_task->status == UCC_OK) { if (coll_task->bargs.args.coll_type == UCC_COLL_TYPE_ALLGATHER) { req->comm->allgather_comm.under_progress_counter++; } else { ucc_assert(coll_task->bargs.args.coll_type == UCC_COLL_TYPE_BCAST); req->comm->bcast_comm.under_progress_counter++; } /* reset the reliability structures so that it gets initialized again for the next * allgather */ req->comm->one_sided.reliability_ready = 0; req->comm->stalled = 0; req->comm->timer = 0; } if (req->rreg != NULL) { ucc_tl_mlx5_mcast_mem_deregister(req->comm->ctx, req->rreg); req->rreg = NULL; } if (req->recv_rreg != NULL) { ucc_tl_mlx5_mcast_mem_deregister(req->comm->ctx, req->recv_rreg); req->recv_rreg = NULL; } if (req->ag_schedule) { ucc_free(req->ag_schedule); req->ag_schedule = NULL; } if (req->scratch_buf_header) { ucc_mc_free(req->scratch_buf_header); req->scratch_buf_header = NULL; req->scratch_buf = NULL; } ucc_mpool_put(req); tl_trace(UCC_TASK_LIB(task), "finalizing an mcast task %p", task); task->coll_mcast.req_handle = NULL; } tl_trace(UCC_TASK_LIB(task), "finalizing task %p", task); ucc_tl_mlx5_put_task(task); return UCC_OK; } ucc_tl_mlx5_task_t* ucc_tl_mlx5_init_task(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_schedule_t *schedule) { ucc_tl_mlx5_task_t *task = ucc_tl_mlx5_get_task(coll_args, team); task->super.schedule = schedule; task->super.finalize = ucc_tl_mlx5_task_finalize; return task; } ucc_status_t ucc_tl_mlx5_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_status_t status = UCC_OK; switch (coll_args->args.coll_type) { case UCC_COLL_TYPE_ALLTOALL: status = ucc_tl_mlx5_alltoall_init(coll_args, team, task_h); break; case UCC_COLL_TYPE_BCAST: case UCC_COLL_TYPE_ALLGATHER: status = ucc_tl_mlx5_coll_mcast_init(coll_args, team, task_h); break; default: status = UCC_ERR_NOT_SUPPORTED; } return status; } ucc-1.8.0/src/components/tl/mlx5/configure.m40000664000175000017500000000170415211535620021163 0ustar alastairalastair# # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # tl_mlx5_enabled=n CHECK_TLS_REQUIRED(["mlx5"]) AS_IF([test "$CHECKED_TL_REQUIRED" = "y"], [ CHECK_RDMACM AC_MSG_RESULT([RDMACM support: $rdmacm_happy]) mlx5_happy=no if test "x$mlx5dv_happy" = "xyes" -a "x$have_mlx5dv_wr_raw_wqe" = "xyes" -a "x$rdmacm_happy" = "xyes"; then mlx5_happy=yes fi AC_MSG_RESULT([MLX5 support: $mlx5_happy]) if test $mlx5_happy = "yes"; then tl_modules="${tl_modules}:mlx5" tl_mlx5_enabled=y CHECK_NEED_TL_PROFILING(["tl_mlx5"]) AS_IF([test "$TL_PROFILING_REQUIRED" = "y"], [ AC_DEFINE([HAVE_PROFILING_TL_MLX5], [1], [Enable profiling for TL MLX5]) prof_modules="${prof_modules}:tl_mlx5" ], []) fi ], []) AM_CONDITIONAL([TL_MLX5_ENABLED], [test "$tl_mlx5_enabled" = "y"]) AC_CONFIG_FILES([src/components/tl/mlx5/Makefile]) ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_pd.c0000664000175000017500000002332515211535620021156 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5.h" #include "tl_mlx5_pd.h" typedef struct { int sock, fd; uint32_t pd_handle; } connection_t; ucc_status_t do_sendmsg(connection_t *conn) { struct msghdr msg = {}; struct cmsghdr *cmsghdr; struct iovec iov; ssize_t nbytes; int * p; char buf[CMSG_SPACE(sizeof(int))]; uint32_t handle; handle = conn->pd_handle; iov.iov_base = &handle; iov.iov_len = sizeof(handle); memset(buf, 0x0b, sizeof(buf)); cmsghdr = (struct cmsghdr *)buf; cmsghdr->cmsg_len = CMSG_LEN(sizeof(int)); cmsghdr->cmsg_level = SOL_SOCKET; cmsghdr->cmsg_type = SCM_RIGHTS; msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_iov = &iov; msg.msg_iovlen = 1; msg.msg_control = cmsghdr; msg.msg_controllen = CMSG_LEN(sizeof(int)); msg.msg_flags = 0; p = (int *)CMSG_DATA(cmsghdr); *p = conn->fd; nbytes = sendmsg(conn->sock, &msg, 0); if (nbytes == -1) { return UCC_ERR_NO_MESSAGE; } return UCC_OK; } static ucc_status_t do_recvmsg(int sock, int *shared_cmd_fd, uint32_t *shared_pd_handle) { uint32_t handle; struct msghdr msg; struct cmsghdr *cmsghdr; struct iovec iov; ssize_t nbytes; int * p; char buf[CMSG_SPACE(sizeof(int))]; iov.iov_base = &handle; iov.iov_len = sizeof(handle); memset(buf, 0x0d, sizeof(buf)); cmsghdr = (struct cmsghdr *)buf; cmsghdr->cmsg_len = CMSG_LEN(sizeof(int)); cmsghdr->cmsg_level = SOL_SOCKET; cmsghdr->cmsg_type = SCM_RIGHTS; msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_iov = &iov; msg.msg_iovlen = 1; msg.msg_control = cmsghdr; msg.msg_controllen = CMSG_LEN(sizeof(int)); msg.msg_flags = 0; nbytes = recvmsg(sock, &msg, 0); if (nbytes == -1) { return UCC_ERR_NO_MESSAGE; } p = (int *)CMSG_DATA(cmsghdr); *shared_cmd_fd = *p; *shared_pd_handle = handle; return UCC_OK; } ucc_status_t ucc_tl_mlx5_socket_init(ucc_tl_mlx5_context_t *ctx, ucc_rank_t group_size, int *sock_p, const char *sock_path) { struct sockaddr_storage storage = {}; struct sockaddr_un * addr; int sock; sock = socket(PF_LOCAL, SOCK_STREAM, 0); if (sock == -1) { tl_debug(ctx->super.super.lib, "failed to create server socket errno %d", errno); return UCC_ERR_NO_MESSAGE; } addr = (struct sockaddr_un *)&storage; addr->sun_family = AF_UNIX; strncpy(addr->sun_path, sock_path, sizeof(addr->sun_path)); addr->sun_path[sizeof(addr->sun_path) - 1] = '\0'; if (bind(sock, (struct sockaddr *)addr, sizeof(struct sockaddr_un)) == -1) { tl_debug(ctx->super.super.lib, "failed to bind server socket errno %d", errno); goto out; } if (listen(sock, group_size) == -1) { tl_debug(ctx->super.super.lib, "failed to listen to server socket errno %d", errno); goto out; } *sock_p = sock; return UCC_OK; out: close(sock); return UCC_ERR_NO_MESSAGE; } static ucc_status_t client_recv_data(int *shared_cmd_fd, uint32_t *shared_pd_handle, const char *sock_path, int *sock_p, ucc_tl_mlx5_lib_t *lib) { struct sockaddr_storage sockaddr = {}; ucc_status_t status = UCC_OK; struct sockaddr_un * addr; int sock; sock = socket(PF_LOCAL, SOCK_STREAM, 0); if (sock == -1) { tl_debug(lib, "failed to create client socket errno %d", errno); return UCC_ERR_NO_MESSAGE; } addr = (struct sockaddr_un *)&sockaddr; addr->sun_family = AF_UNIX; strncpy(addr->sun_path, sock_path, sizeof(addr->sun_path)); addr->sun_path[sizeof(addr->sun_path) - 1] = '\0'; while (connect(sock, (struct sockaddr *)addr, SUN_LEN(addr)) == -1) { if (errno != ENOENT) { tl_debug(lib, "failed to connect client socket errno %d", errno); status = UCC_ERR_NO_MESSAGE; goto out; } } if (do_recvmsg(sock, shared_cmd_fd, shared_pd_handle) != UCC_OK) { tl_debug(lib, "Failed to recv msg"); status = UCC_ERR_NO_MESSAGE; goto out; } *sock_p = sock; return UCC_OK; out: if (close(sock) == -1) { tl_debug(lib, "Failed to close client socket errno %d", errno); status = UCC_ERR_NO_MESSAGE; } return status; } static ucc_status_t server_send_data(int command_fd, uint32_t pd_handle, ucc_rank_t group_size, int sock, ucc_tl_mlx5_lib_t *lib) { ucc_status_t status = UCC_OK; int i; connection_t connection[group_size]; struct sockaddr_un addr; socklen_t addrlen; for (i = 0; i < group_size; i++) { /* accept incoming connections */ connection[i].fd = command_fd; connection[i].pd_handle = pd_handle; connection[i].sock = accept(sock, NULL, 0); if (connection[i].sock == -1) { tl_debug(lib, "failed to accept socket connection request %d," " errno %d", i, errno); goto listen_fail; } status = do_sendmsg(&connection[i]); if (status != UCC_OK) { tl_debug(lib, "failed to send cmd_fd"); goto listen_fail; } } addrlen = sizeof(addr); getsockname(sock, (struct sockaddr *)&addr, &addrlen); if (remove(addr.sun_path) == -1) { tl_debug(lib, "socket file removal failed"); status = UCC_ERR_NO_MESSAGE; } return status; listen_fail: if (close(sock) == -1) { tl_debug(lib, "failed to close server socket errno %d", errno); status = UCC_ERR_NO_MESSAGE; } return status; } ucc_status_t ucc_tl_mlx5_share_ctx_pd(ucc_tl_mlx5_context_t *ctx, const char * sock_path, ucc_rank_t group_size, int is_ctx_owner, int ctx_owner_sock) { ucc_tl_mlx5_lib_t *lib = ucc_derived_of(ctx->super.super.lib, ucc_tl_mlx5_lib_t); int ctx_fd; uint32_t pd_handle; ucc_status_t status; if (!is_ctx_owner) { status = client_recv_data(&ctx_fd, &pd_handle, sock_path, &ctx->sock, lib); if (UCC_OK != status) { tl_debug(lib, "failed to share ctx & pd from client side"); return status; } ctx->shared_ctx = ibv_import_device(ctx_fd); if (!ctx->shared_ctx) { tl_debug(lib, "import context failed"); return UCC_ERR_NO_MESSAGE; } ctx->shared_pd = ibv_import_pd(ctx->shared_ctx, pd_handle); if (!ctx->shared_pd) { tl_debug(lib, "import PD failed"); if (ibv_close_device(ctx->shared_ctx)) { tl_debug(lib, "imported context close failed"); } return UCC_ERR_NO_MESSAGE; } } else { ctx_fd = ctx->shared_ctx->cmd_fd; pd_handle = ctx->shared_pd->handle; status = server_send_data(ctx_fd, pd_handle, group_size - 1, ctx_owner_sock, lib); if (UCC_OK != status) { tl_debug(lib, "failed to share ctx & pd from server side"); return status; } } return UCC_OK; } static void ucc_tl_mlx5_context_barrier(ucc_context_oob_coll_t *oob, ucc_context_t *core_ctx, ucc_base_lib_t *lib) { char *rbuf; char sbuf; void *req; ucc_status_t status; if (ucc_unlikely(oob->n_oob_eps < 2)) { return; } rbuf = ucc_malloc(sizeof(char) * oob->n_oob_eps, "tmp_barrier"); if (!rbuf) { tl_debug(lib, "failed to allocate %zd bytes for tmp barrier array", sizeof(char) * oob->n_oob_eps); return; } if (UCC_OK == oob->allgather(&sbuf, rbuf, sizeof(char), oob->coll_info, &req)) { ucc_assert(req != NULL); while (UCC_OK != (status = oob->req_test(req))) { ucc_context_progress(core_ctx); if (status < 0) { tl_debug(lib, "failed to test oob req"); break; } } oob->req_free(req); } ucc_free(rbuf); } ucc_status_t ucc_tl_mlx5_remove_shared_ctx_pd(ucc_tl_mlx5_context_t *ctx) { ucc_base_lib_t *lib = ctx->super.super.lib; ucc_status_t status = UCC_OK; int err; if (ctx->shared_pd && ctx->is_imported) { ibv_unimport_pd(ctx->shared_pd); } ucc_tl_mlx5_context_barrier(&UCC_TL_CTX_OOB(ctx), ctx->super.super.ucc_context, lib); if (ctx->shared_pd && !ctx->is_imported) { err = ibv_dealloc_pd(ctx->shared_pd); if (err) { tl_debug(lib, "failed to dealloc PD, errno %d", err); status = UCC_ERR_NO_MESSAGE; } } if (ctx->shared_ctx) { if (ibv_close_device(ctx->shared_ctx)) { tl_debug(lib, "failed to close ib ctx"); status = UCC_ERR_NO_MESSAGE; } } return status; } ucc-1.8.0/src/components/tl/mlx5/alltoall/0000775000175000017500000000000015211535620020542 5ustar alastairalastairucc-1.8.0/src/components/tl/mlx5/alltoall/alltoall.h0000664000175000017500000001617015211535620022524 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef ALLTOALL_H_ #define ALLTOALL_H_ #include "tl_mlx5.h" #include "tl_mlx5_ib.h" #include "tl_mlx5_dm.h" #define SEQ_INDEX(_seq_num) ((_seq_num) % MAX_OUTSTANDING_OPS) #define MLX5_ASR_RANK 0 // has to be 0 with current implementation of UCC_SBGP_NODE_LEADERS #define MLX5_NUM_OF_BLOCKS_SIZE_BINS 8 #define MAX_TRANSPOSE_SIZE 8192 // HW transpose unit is limited to matrix size #define MAX_MSG_SIZE 128 // HW transpose unit is limited to element size #define MAX_BLOCK_SIZE 64 // from limit of Transpose unit capabilities #define MAX_OUTSTANDING_OPS 1 //todo change - according to limitations (52 top) #define MIN_POLL_WC 8 typedef uint64_t tl_mlx5_atomic_t; typedef uint64_t tl_mlx5_barrier_t; enum { UCC_MLX5_NEED_SEND_MKEY_UPDATE = UCC_BIT(1), UCC_MLX5_NEED_RECV_MKEY_UPDATE = UCC_BIT(2), }; typedef struct ucc_tl_mlx5_alltoall_ctrl { union { struct { volatile int seq_num; int mkey_cache_flag; }; char tmp[UCC_CACHE_LINE_SIZE]; }; } ucc_tl_mlx5_alltoall_ctrl_t; typedef struct ucc_tl_mlx5_alltoall_op { ucc_tl_mlx5_alltoall_ctrl_t *ctrl; ucc_tl_mlx5_alltoall_ctrl_t *my_ctrl; struct mlx5dv_mkey ** send_mkeys; struct mlx5dv_mkey ** recv_mkeys; int * blocks_sent; } ucc_tl_mlx5_alltoall_op_t; /* This structure holds resources and data related to the "in-node" part of the algorithm. */ typedef struct ucc_tl_mlx5_alltoall_node { int asr_rank; ucc_sbgp_t *sbgp; void *storage; ucc_tl_mlx5_alltoall_op_t ops[MAX_OUTSTANDING_OPS]; struct mlx5dv_mkey *team_recv_mkey; void *umr_entries_buf; struct ibv_mr *umr_entries_mr; } ucc_tl_mlx5_alltoall_node_t; typedef struct alltoall_net_ctrl { struct { void *addr; uint32_t rkey; } atomic; struct { void *addr; uint32_t rkey; } barrier; } alltoall_net_ctrl_t; typedef struct ucc_tl_mlx5_alltoall_net { ucc_sbgp_t *sbgp; int net_size; int *rank_map; ucc_tl_mlx5_qp_t *rc_qps; struct ibv_qp *dct_qp; struct ibv_srq *srq; uint32_t *remote_dctns; struct ibv_ah **ahs; struct ibv_cq *cq; struct ibv_cq *umr_cq; struct ibv_qp *umr_qp; struct ibv_mr *ctrl_mr; struct { #if ATOMIC_IN_MEMIC struct ibv_dm *counters; #else tl_mlx5_atomic_t *counters; #endif struct ibv_mr *mr; } atomic; struct { tl_mlx5_barrier_t *flags; struct ibv_mr *mr; } barrier; int *blocks_sent; alltoall_net_ctrl_t *remote_ctrl; uint32_t *rkeys; ucc_tl_mlx5_dci_t *dcis; } ucc_tl_mlx5_alltoall_net_t; typedef struct ucc_tl_mlx5_a2a_bcast_data { int shmid; int net_size; } ucc_tl_mlx5_a2a_bcast_data_t; enum { TL_MLX5_ALLTOALL_STATE_SHMID, TL_MLX5_ALLTOALL_STATE_EXCHANGE_PROGRESS, TL_MLX5_ALLTOALL_STATE_EXCHANGE_DONE }; typedef struct ucc_tl_mlx5_alltoall { struct ibv_pd *pd; struct ibv_context *ctx; int ib_port; int state; uint64_t max_msg_size; ucc_tl_mlx5_alltoall_node_t node; ucc_tl_mlx5_alltoall_net_t net; int sequence_number; int op_busy[MAX_OUTSTANDING_OPS]; int num_dci_qps; uint8_t is_dc; int previous_msg_size[MAX_OUTSTANDING_OPS]; void *previous_send_address[MAX_OUTSTANDING_OPS]; void *previous_recv_address[MAX_OUTSTANDING_OPS]; uint64_t atomic_scratch_bf; int requested_block_size; int max_num_of_columns; struct ibv_mr *atomic_scratch_bf_mr; ucc_rank_t node_size; ucc_tl_mlx5_a2a_bcast_data_t bcast_data; } ucc_tl_mlx5_alltoall_t; void ucc_tl_mlx5_topo_cleanup(ucc_tl_mlx5_team_t *team); ucc_status_t ucc_tl_mlx5_team_init_alltoall(ucc_tl_mlx5_team_t *team); ucc_status_t ucc_tl_mlx5_team_test_alltoall_start(ucc_tl_mlx5_team_t *team); ucc_status_t ucc_tl_mlx5_team_test_alltoall_progress(ucc_tl_mlx5_team_t *team); ucc_status_t ucc_tl_mlx5_alltoall_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t * team, ucc_coll_task_t ** task_h); void ucc_tl_mlx5_alltoall_cleanup(ucc_tl_mlx5_team_t *team); static inline ucc_tl_mlx5_alltoall_ctrl_t* ucc_tl_mlx5_get_ctrl(ucc_tl_mlx5_alltoall_t *a2a, int op_index, int rank) { ucc_tl_mlx5_alltoall_ctrl_t *ctrl = PTR_OFFSET(a2a->node.ops[op_index].ctrl, sizeof(ucc_tl_mlx5_alltoall_ctrl_t) * rank); return ctrl; } static inline ucc_tl_mlx5_alltoall_ctrl_t* ucc_tl_mlx5_get_my_ctrl(ucc_tl_mlx5_alltoall_t *a2a, int op_index) { int my_rank = a2a->node.sbgp->group_rank; return ucc_tl_mlx5_get_ctrl(a2a, op_index, my_rank); } #define OP_SEGMENT_SIZE(_a2a) \ (sizeof(ucc_tl_mlx5_alltoall_ctrl_t) * (_a2a)->node_size + \ (sizeof(umr_t) * (_a2a)->max_num_of_columns * (_a2a)->node_size) * 2) #define UMR_DATA_OFFSET(_a2a) \ (sizeof(ucc_tl_mlx5_alltoall_ctrl_t) * (_a2a)->node_size) #define OP_SEGMENT_STORAGE(_req, _a2a) \ PTR_OFFSET((_a2a)->node.storage, \ OP_SEGMENT_SIZE(_a2a) * (_req)->alltoall.seq_index) #define OP_UMR_DATA(_req, _a2a) \ PTR_OFFSET(OP_SEGMENT_STORAGE(_req, _a2a), UMR_DATA_OFFSET(_a2a)) #define SEND_UMR_DATA(_req, _a2a, _col) \ PTR_OFFSET(OP_UMR_DATA(_req, _a2a), \ _col *(_a2a)->node.sbgp->group_size * sizeof(umr_t)) #define RECV_UMR_DATA(_req, _a2a, _col) \ PTR_OFFSET(OP_UMR_DATA(_req, _a2a), \ (_a2a)->max_num_of_columns *(_a2a)->node.sbgp->group_size * \ sizeof(umr_t) + \ _col * (_a2a)->node.sbgp->group_size * sizeof(umr_t)) #define MY_SEND_UMR_DATA(_req, _a2a, _col) \ PTR_OFFSET(SEND_UMR_DATA(_req, _a2a, _col), \ (_a2a)->node.sbgp->group_rank * sizeof(umr_t)) #define MY_RECV_UMR_DATA(_req, _a2a, _col) \ PTR_OFFSET(RECV_UMR_DATA(_req, _a2a, _col), \ (_a2a)->node.sbgp->group_rank * sizeof(umr_t)) #endif ucc-1.8.0/src/components/tl/mlx5/alltoall/alltoall_mkeys.h0000664000175000017500000000272615211535620023736 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_MLX5_MKEYS_H #define UCC_TL_MLX5_MKEYS_H typedef struct ucc_tl_mlx5_context ucc_tl_mlx5_context_t; typedef struct ucc_tl_mlx5_alltoall ucc_tl_mlx5_alltoall_t; typedef struct ucc_tl_mlx5_schedule ucc_tl_mlx5_schedule_t; typedef struct ucc_tl_mlx5_team ucc_tl_mlx5_team_t; typedef struct ucc_tl_mlx5_lib ucc_tl_mlx5_lib_t; typedef struct ucc_base_lib ucc_base_lib_t; #define UMR_CQ_SIZE 8 ucc_status_t ucc_tl_mlx5_alltoall_init_umr(ucc_tl_mlx5_alltoall_t *a2a, ucc_base_lib_t *lib); ucc_status_t ucc_tl_mlx5_init_mkeys(ucc_tl_mlx5_team_t *team, ucc_base_lib_t *lib); ucc_status_t ucc_tl_mlx5_populate_send_recv_mkeys(ucc_tl_mlx5_team_t *team, ucc_tl_mlx5_schedule_t *req); ucc_status_t ucc_tl_mlx5_update_mkeys_entries(ucc_tl_mlx5_alltoall_t *a2a, ucc_tl_mlx5_schedule_t *req, int flag); ucc_status_t ucc_tl_mlx5_destroy_umr(ucc_tl_mlx5_alltoall_t *a2a, ucc_base_lib_t *lib); ucc_status_t ucc_tl_mlx5_destroy_mkeys(ucc_tl_mlx5_alltoall_t *a2a, int error_mode, ucc_base_lib_t *lib); #endif ucc-1.8.0/src/components/tl/mlx5/alltoall/alltoall_mkeys.c0000664000175000017500000004220215211535620023722 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5.h" #include "tl_mlx5_ib.h" #include "tl_mlx5_coll.h" #include "tl_mlx5_wqe.h" #include "alltoall/alltoall.h" #include "alltoall/alltoall_mkeys.h" #include /** * Create and connect UMR qp & cq. */ ucc_status_t ucc_tl_mlx5_alltoall_init_umr(ucc_tl_mlx5_alltoall_t *a2a, ucc_base_lib_t *lib) { ucc_tl_mlx5_lib_config_t cfg = ucc_derived_of(lib, ucc_tl_mlx5_lib_t)->cfg; ucc_status_t status = UCC_OK; a2a->net.umr_cq = ibv_create_cq(a2a->ctx, UMR_CQ_SIZE, NULL, NULL, 0); if (a2a->net.umr_cq == NULL) { tl_error(lib, "failed to create UMR (errno %d)", errno); return UCC_ERR_NO_MESSAGE; } status = ucc_tl_mlx5_create_umr_qp(a2a->ctx, a2a->pd, a2a->net.umr_cq, a2a->ib_port, &a2a->net.umr_qp, &cfg.qp_conf, lib); if (status != UCC_OK) { goto err; } return status; err: if (ibv_destroy_cq(a2a->net.umr_cq)) { tl_error(lib, "failed to destroy UMR CQ (errno=%d)", errno); } return status; } static ucc_status_t create_master_key(int num_of_entries, struct ibv_pd *pd, struct mlx5dv_mkey **mkey_ptr, ucc_base_lib_t * lib) { struct mlx5dv_mkey * mkey; struct mlx5dv_mkey_init_attr umr_mkey_init_attr; memset(&umr_mkey_init_attr, 0, sizeof(umr_mkey_init_attr)); umr_mkey_init_attr.pd = pd; umr_mkey_init_attr.create_flags = MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT; // Defines how many entries the Mkey will support. // In case the MKey is used as "strided-KLM based MKey", the number // of entries that is needed is increased by one because one entry is // consumed by the "strided header" (see mlx5dv_wr_post manual). umr_mkey_init_attr.max_entries = num_of_entries; mkey = mlx5dv_create_mkey(&umr_mkey_init_attr); if (mkey == NULL) { tl_error(lib, "MasterMKey creation failed (errno=%d)", errno); return UCC_ERR_NO_MESSAGE; } tl_trace(lib, "umr_master_key_dv_mkey: lkey=0x%x, with %d entries", mkey->lkey, num_of_entries); *mkey_ptr = mkey; return UCC_OK; } static ucc_status_t poll_umr_cq(struct ibv_cq *cq, ucc_base_lib_t *lib) { struct ibv_wc wc; int ret = 0; while (!ret) { ret = ibv_poll_cq(cq, 1, &wc); if (ret < 0) { tl_error(lib, "ibv_poll_cq() failed for UMR execution"); return UCC_ERR_NO_MESSAGE; } else if (ret > 0) { if (wc.status != IBV_WC_SUCCESS) { tl_error(lib, "umr cq returned incorrect completion: status " "%s", ibv_wc_status_str(wc.status)); return UCC_ERR_NO_MESSAGE; } } } return UCC_OK; } // Execute the UMR WQE for populating the UMR's MasterMKey static ucc_status_t populate_non_strided_mkey(struct ibv_qp *umr_qp, struct ibv_cq *umr_cq, int mem_access_flags, struct mlx5dv_mkey *mkey, void *mkey_entries, int n_entries, ucc_base_lib_t *lib) { struct ibv_qp_ex * qp_ex = ibv_qp_to_qp_ex(umr_qp); struct mlx5dv_qp_ex *mqp = mlx5dv_qp_ex_from_ibv_qp_ex(qp_ex); ucc_status_t status; ibv_wr_start(qp_ex); qp_ex->wr_id = 1; // First (and only) WR mlx5dv_wr_mr_list(mqp, mkey, mem_access_flags, n_entries, (struct ibv_sge *)mkey_entries); if (ibv_wr_complete(qp_ex)) { tl_error(lib, "UMR WQE failed (errno=%d)", errno); return UCC_ERR_NO_MESSAGE; } status = poll_umr_cq(umr_cq, lib); if (status != UCC_OK) { return status; } return UCC_OK; } // Execute the UMR WQE for populating the UMR's MasterMKey static ucc_status_t populate_strided_mkey(ucc_tl_mlx5_alltoall_t *a2a, int mem_access_flags, struct mlx5dv_mkey *mkey, void *mkey_entries, int repeat_count, ucc_base_lib_t *lib) { ucc_status_t status; ucc_tl_mlx5_alltoall_net_t * net = &a2a->net; ucc_tl_mlx5_alltoall_node_t *node = &a2a->node; ucc_tl_mlx5_post_umr(net->umr_qp, mkey, mem_access_flags, repeat_count, node->sbgp->group_size, (struct mlx5dv_mr_interleaved *)mkey_entries, node->umr_entries_mr->lkey, node->umr_entries_buf); status = poll_umr_cq(net->umr_cq, lib); if (status != UCC_OK) { tl_error(lib, "failed to populate strided UMR mkey (errno=%d)", errno); return status; } return UCC_OK; } static ucc_status_t create_and_populate_recv_team_mkey(ucc_tl_mlx5_team_t *team, ucc_base_lib_t *lib) { int team_size = UCC_TL_TEAM_SIZE(team); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; ucc_tl_mlx5_alltoall_node_t *node = &a2a->node; int mnc = a2a->max_num_of_columns; ucc_status_t status = UCC_OK; int i, j, index; status = create_master_key(MAX_OUTSTANDING_OPS * mnc, a2a->pd, &node->team_recv_mkey, lib); if (status != UCC_OK) { return status; } struct ibv_sge *team_mkey_klm_entries = (struct ibv_sge *)calloc( MAX_OUTSTANDING_OPS * mnc, sizeof(struct ibv_sge)); if (!team_mkey_klm_entries) { tl_error(lib, "failed to allocate team_mkey_klm_entries"); status = UCC_ERR_NO_MEMORY; goto err_calloc; } for (i = 0; i < MAX_OUTSTANDING_OPS; i++) { for (j = 0; j < mnc; j++) { index = i * mnc + j; team_mkey_klm_entries[index].addr = 0; //length could be minimized for all mkeys beside the first, but no need because address space is big enough team_mkey_klm_entries[index].length = node->sbgp->group_size * a2a->max_msg_size * team_size; team_mkey_klm_entries[index].lkey = node->ops[i].recv_mkeys[j]->rkey; } } status = populate_non_strided_mkey( a2a->net.umr_qp, a2a->net.umr_cq, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE, node->team_recv_mkey, team_mkey_klm_entries, MAX_OUTSTANDING_OPS * mnc, lib); if (status != UCC_OK) { tl_error(a2a, "failed to populate team mkey"); goto err_mkey; } ucc_free(team_mkey_klm_entries); return status; err_mkey: ucc_free(team_mkey_klm_entries); err_calloc: if (mlx5dv_destroy_mkey(node->team_recv_mkey)) { tl_error(lib, "mkey destroy failed(errno=%d)", errno); } return status; } /** * Create mkeys for all outstanding AlltoAll ops in each rank. Creats team mkey, and execute the team mkey's * population WQE */ ucc_status_t ucc_tl_mlx5_init_mkeys(ucc_tl_mlx5_team_t *team, ucc_base_lib_t *lib) { ucc_tl_mlx5_alltoall_t *a2a = team->a2a; ucc_tl_mlx5_alltoall_node_t *node = &a2a->node; int i, j, k, l; ucc_status_t status; for (i = 0; i < MAX_OUTSTANDING_OPS; i++) { node->ops[i].send_mkeys = (struct mlx5dv_mkey **)ucc_malloc( sizeof(struct mlx5dv_mkey *) * a2a->max_num_of_columns); if (!node->ops[i].send_mkeys) { tl_error(lib, "failed to malloc"); goto err_malloc; } node->ops[i].recv_mkeys = (struct mlx5dv_mkey **)ucc_malloc( sizeof(struct mlx5dv_mkey *) * a2a->max_num_of_columns); if (!node->ops[i].recv_mkeys) { tl_error(lib, "failed to malloc"); ucc_free(node->ops[i].send_mkeys); goto err_malloc; } for (j = 0; j < a2a->max_num_of_columns; j++) { status = create_master_key(node->sbgp->group_size + 1, a2a->pd, &node->ops[i].send_mkeys[j], lib); if (status != UCC_OK) { tl_error(lib, "failed to create send masterkey [%d,%d]", i, j); goto err_create_mkey; } status = create_master_key(node->sbgp->group_size + 1, a2a->pd, &node->ops[i].recv_mkeys[j], lib); if (status != UCC_OK) { tl_error(lib, "failed to create recv masterkey [%d,%d]", i, j); if (!mlx5dv_destroy_mkey(node->ops[i].send_mkeys[j])) { tl_error(lib, "mkey destroy failed(errno=%d)", errno); } goto err_create_mkey; } } } status = create_and_populate_recv_team_mkey(team, lib); if (status != UCC_OK) { tl_error(lib, "failed to create recv top masterkey"); goto err_malloc; } return UCC_OK; err_create_mkey: for (l = 0; l < j; l++) { if (!mlx5dv_destroy_mkey(node->ops[i].recv_mkeys[l])) { tl_error(lib, "mkey destroy failed(errno=%d)", errno); } } ucc_free(node->ops[i].recv_mkeys); ucc_free(node->ops[i].send_mkeys); err_malloc: for (k = 0; k < i; k++) { for (l = 0; l < a2a->max_num_of_columns; l++) { if (!mlx5dv_destroy_mkey(node->ops[k].send_mkeys[l])) { tl_error(lib, "mkey destroy failed(errno=%d)", errno); } if (!mlx5dv_destroy_mkey(node->ops[k].recv_mkeys[l])) { tl_error(lib, "mkey destroy failed(errno=%d)", errno); } } ucc_free(node->ops[k].send_mkeys); ucc_free(node->ops[k].recv_mkeys); } return UCC_ERR_NO_MESSAGE; } /** * Execute UMR WQE to populate mkey of specific AlltoAll operation, after the mkey entries were already updated * @param team struct of the current team * @param req current AlltoAll operation request */ ucc_status_t ucc_tl_mlx5_populate_send_recv_mkeys(ucc_tl_mlx5_team_t * team, ucc_tl_mlx5_schedule_t *req) { int send_mem_access_flags = 0; ucc_tl_mlx5_alltoall_t * a2a = team->a2a; ucc_tl_mlx5_alltoall_node_t *node = &a2a->node; int recv_mem_access_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE; int nbc = req->alltoall.num_of_blocks_columns; int seq_index = req->alltoall.seq_index; int n_mkeys = nbc ? nbc : 1; int repeat_count; int i; ucc_status_t status; if (ucc_tl_mlx5_get_my_ctrl(a2a, seq_index)->mkey_cache_flag & UCC_MLX5_NEED_SEND_MKEY_UPDATE) { repeat_count = nbc ? a2a->net.sbgp->group_size : UCC_TL_TEAM_SIZE(team) / req->alltoall.block_width; for (i = 0; i < n_mkeys; i++) { status = populate_strided_mkey(a2a, send_mem_access_flags, node->ops[seq_index].send_mkeys[i], SEND_UMR_DATA(req, a2a, i), repeat_count, UCC_TL_TEAM_LIB(team)); if (status != UCC_OK) { tl_error(UCC_TL_MLX5_TEAM_LIB(team), "Failed to populate send umr[%d,%d]", seq_index, i); return status; } } } if (ucc_tl_mlx5_get_my_ctrl(a2a, seq_index)->mkey_cache_flag & UCC_MLX5_NEED_RECV_MKEY_UPDATE) { repeat_count = nbc ? a2a->net.sbgp->group_size : UCC_TL_TEAM_SIZE(team) / req->alltoall.block_height; for (i = 0; i < n_mkeys; i++) { status = populate_strided_mkey(a2a, recv_mem_access_flags, node->ops[seq_index].recv_mkeys[i], RECV_UMR_DATA(req, a2a, i), repeat_count, UCC_TL_TEAM_LIB(team)); if (status != UCC_OK) { tl_error(UCC_TL_MLX5_TEAM_LIB(team), "Failed to populate recv umr[%d,%d]", seq_index, i); return status; } } } return UCC_OK; } static void update_mkey_entry(ucc_tl_mlx5_alltoall_t *a2a, ucc_tl_mlx5_schedule_t *req, int direction_send) { ucc_tl_mlx5_alltoall_node_t *node = &a2a->node; int block_height = req->alltoall.block_height; int block_width = req->alltoall.block_width; size_t msg_size = req->alltoall.msg_size; int nbc = req->alltoall.num_of_blocks_columns; struct ibv_mr *buff = direction_send ? req->alltoall.send_rcache_region_p->reg.mr : req->alltoall.recv_rcache_region_p->reg.mr; struct mlx5dv_mr_interleaved *mkey_entry; int i; if (!nbc) { mkey_entry = (umr_t *)(direction_send ? MY_SEND_UMR_DATA(req, a2a, 0) : MY_RECV_UMR_DATA(req, a2a, 0)); mkey_entry->addr = (uintptr_t)buff->addr; mkey_entry->bytes_count = (direction_send ? block_width : block_height) * msg_size; mkey_entry->bytes_skip = 0; mkey_entry->lkey = direction_send ? buff->lkey : buff->rkey; } else { for (i = 0; i < nbc; i++) { ucc_assert(block_height == block_width); mkey_entry = (umr_t *)(direction_send ? MY_SEND_UMR_DATA(req, a2a, i) : MY_RECV_UMR_DATA(req, a2a, i)); mkey_entry->addr = (uintptr_t)buff->addr + i * (block_height * msg_size); mkey_entry->bytes_count = (i == (nbc - 1)) ? ((node->sbgp->group_size % block_height) * msg_size) : (block_height * msg_size); mkey_entry->bytes_skip = (i == (nbc - 1)) ? ((node->sbgp->group_size - (node->sbgp->group_size % block_height)) * msg_size) : ((node->sbgp->group_size - block_height) * msg_size); mkey_entry->lkey = direction_send ? buff->lkey : buff->rkey; } } } /** * Update the UMR klm entry (ranks send & receive buffers) for specific AlltoAll operation * @param node struct of the current process's node * @param req AlltoAll operation request object */ ucc_status_t ucc_tl_mlx5_update_mkeys_entries(ucc_tl_mlx5_alltoall_t *a2a, ucc_tl_mlx5_schedule_t *req, int flag) { if (flag & UCC_MLX5_NEED_SEND_MKEY_UPDATE) { update_mkey_entry(a2a, req, 1); } if (flag & UCC_MLX5_NEED_RECV_MKEY_UPDATE) { update_mkey_entry(a2a, req, 0); } return UCC_OK; } /** * Clean UMR qp & cq */ ucc_status_t ucc_tl_mlx5_destroy_umr(ucc_tl_mlx5_alltoall_t *a2a, ucc_base_lib_t *lib) { if (ibv_destroy_qp(a2a->net.umr_qp)) { tl_error(lib, "umr qp destroy failed (errno=%d)", errno); return UCC_ERR_NO_MESSAGE; } if (ibv_destroy_cq(a2a->net.umr_cq)) { tl_error(lib, "umr cq destroy failed (errno=%d)", errno); return UCC_ERR_NO_MESSAGE; } return UCC_OK; } /** * Clean all mkeys - operation mkeys, team mkeys */ ucc_status_t ucc_tl_mlx5_destroy_mkeys(ucc_tl_mlx5_alltoall_t *a2a, int error_mode, ucc_base_lib_t *lib) { int i, j; ucc_tl_mlx5_alltoall_node_t *node = &a2a->node; ucc_status_t status = UCC_OK; for (i = 0; i < MAX_OUTSTANDING_OPS; i++) { for (j = 0; j < a2a->max_num_of_columns; j++) { if (mlx5dv_destroy_mkey(node->ops[i].send_mkeys[j])) { if (!error_mode) { tl_error(lib, "mkey destroy failed(errno=%d)", errno); status = UCC_ERR_NO_MESSAGE; } } if (mlx5dv_destroy_mkey(node->ops[i].recv_mkeys[j])) { if (!error_mode) { tl_error(lib, "mkey destroy failed(errno=%d)", errno); status = UCC_ERR_NO_MESSAGE; } } } ucc_free(node->ops[i].send_mkeys); ucc_free(node->ops[i].recv_mkeys); } if (mlx5dv_destroy_mkey(node->team_recv_mkey)) { if (!error_mode) { tl_error(lib, "mkey destroy failed(errno=%d)", errno); status = UCC_ERR_NO_MESSAGE; } } return status; } ucc-1.8.0/src/components/tl/mlx5/alltoall/alltoall_coll.c0000664000175000017500000012205415211535620023527 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_coll.h" #include "alltoall/alltoall.h" #include "alltoall/alltoall_mkeys.h" #include "alltoall/alltoall_inline.h" #include "components/mc/ucc_mc.h" #include "core/ucc_team.h" #include "tl_mlx5_wqe.h" #include "tl_mlx5_ib.h" static ucc_status_t ucc_tl_mlx5_poll_free_op_slot_start(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; int seq_index = task->alltoall.seq_index; if (a2a->op_busy[seq_index] && !task->alltoall.started) { tl_debug( UCC_TL_TEAM_LIB(team), "Operation num %d must wait for previous outstanding to complete", task->alltoall.seq_num); } coll_task->status = UCC_INPROGRESS; coll_task->super.status = UCC_INPROGRESS; ucc_progress_enqueue(UCC_TL_CORE_CTX(team)->pq, coll_task); return UCC_OK; } void ucc_tl_mlx5_poll_free_op_slot_progress(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; int seq_index = task->alltoall.seq_index; if (a2a->op_busy[seq_index] && !task->alltoall.started) { coll_task->status = UCC_INPROGRESS; return; } //wait for slot to be open a2a->op_busy[seq_index] = 1; task->alltoall.started = 1; coll_task->status = UCC_OK; tl_debug(UCC_TL_TEAM_LIB(team), "Operation num %d started", task->alltoall.seq_num); } static ucc_status_t ucc_tl_mlx5_poll_cq(struct ibv_cq *cq, ucc_base_lib_t *lib) { int i, completions_num; struct ibv_wc wcs[MIN_POLL_WC]; completions_num = ibv_poll_cq(cq, MIN_POLL_WC, wcs); if (completions_num < 0) { tl_error(lib, "ibv_poll_cq() failed, errno %d", errno); return UCC_ERR_NO_MESSAGE; } for (i = 0; i < completions_num; i++) { if (wcs[i].status != IBV_WC_SUCCESS) { tl_error(lib, "bad work completion status %s, wr_id %zu", ibv_wc_status_str(wcs[i].status), wcs[i].wr_id); return UCC_ERR_NO_MESSAGE; } ucc_assert(wcs[i].opcode == IBV_WC_DRIVER2); if (wcs[i].wr_id == 0) { /* signalled transpose */ continue; } else if (wcs[i].wr_id & 0x1) { ucc_tl_mlx5_schedule_t *task = (ucc_tl_mlx5_schedule_t *)(uintptr_t)(wcs[i].wr_id & (~(uint64_t)0x1)); task->alltoall.wait_wc = 1; } else { ucc_tl_mlx5_dm_chunk_t *dm = (ucc_tl_mlx5_dm_chunk_t *)wcs[i].wr_id; dm->task->alltoall.blocks_completed++; dm->completed_sends++; if (dm->posted_all && dm->completed_sends == dm->posted_sends) { ucc_mpool_put(dm); } } } return UCC_OK; } static ucc_status_t ucc_tl_mlx5_node_fanin(ucc_tl_mlx5_team_t *team, ucc_tl_mlx5_schedule_t *task) { ucc_tl_mlx5_alltoall_t *a2a = team->a2a; int seq_index = task->alltoall.seq_index; int i; ucc_tl_mlx5_alltoall_ctrl_t *ctrl_v; if (a2a->node.sbgp->group_rank != a2a->node.asr_rank) { ucc_tl_mlx5_get_my_ctrl(a2a, seq_index)->seq_num = task->alltoall.seq_num; } else { for (i = 0; i < a2a->node.sbgp->group_size; i++) { if (i == a2a->node.sbgp->group_rank) { continue; } ctrl_v = ucc_tl_mlx5_get_ctrl(a2a, seq_index, i); if (ctrl_v->seq_num != task->alltoall.seq_num) { return UCC_INPROGRESS; } } for (i = 0; i < a2a->node.sbgp->group_size; i++) { if (i == a2a->node.sbgp->group_rank) { continue; } ctrl_v = ucc_tl_mlx5_get_ctrl(a2a, seq_index, i); ucc_tl_mlx5_get_my_ctrl(a2a, seq_index)->mkey_cache_flag |= ctrl_v->mkey_cache_flag; } } UCC_TL_MLX5_PROFILE_REQUEST_EVENT(task, "mlx5_alltoall_fanin_done", 0); return UCC_OK; } /* Each rank registers sbuf and rbuf and place the registration data in the shared memory location. Next, all rank in node nitify the ASR the registration data is ready using SHM Fanin */ static ucc_status_t ucc_tl_mlx5_reg_fanin_start(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(team); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; int reg_change_flag = 0; int seq_index = task->alltoall.seq_index; int flag; ucc_tl_mlx5_rcache_region_t *send_ptr; ucc_tl_mlx5_rcache_region_t *recv_ptr; UCC_TL_MLX5_PROFILE_REQUEST_EVENT(task, "mlx5_alltoall_fanin_start", 0); tl_debug(UCC_TASK_LIB(task), "register memory buffers"); coll_task->status = UCC_INPROGRESS; coll_task->super.status = UCC_INPROGRESS; errno = 0; if (UCC_OK != ucc_rcache_get(ctx->rcache, (void *)SCHEDULE_ARGS(task).src.info.buffer, task->alltoall.msg_size * UCC_TL_TEAM_SIZE(team), ®_change_flag, (ucc_rcache_region_t **)&send_ptr)) { tl_error(UCC_TASK_LIB(task), "Failed to register send_bf memory (errno=%d)", errno); return UCC_ERR_NO_RESOURCE; } task->alltoall.send_rcache_region_p = send_ptr; /* NOTE: we don't support alternating block_size for the same msg size - TODO Will need to add the possibility of block_size change into consideration when initializing the mkey_cache_flag */ flag = (task->alltoall.msg_size == a2a->previous_msg_size[seq_index]) ? 0 : (UCC_MLX5_NEED_SEND_MKEY_UPDATE | UCC_MLX5_NEED_RECV_MKEY_UPDATE); if (reg_change_flag || (task->alltoall.send_rcache_region_p->reg.mr->addr != a2a->previous_send_address[seq_index])) { flag |= UCC_MLX5_NEED_SEND_MKEY_UPDATE; } reg_change_flag = 0; if (UCC_OK != ucc_rcache_get(ctx->rcache, (void *)SCHEDULE_ARGS(task).dst.info.buffer, task->alltoall.msg_size * UCC_TL_TEAM_SIZE(team), ®_change_flag, (ucc_rcache_region_t **)&recv_ptr)) { tl_error(UCC_TASK_LIB(task), "Failed to register receive_bf memory"); ucc_rcache_region_put(ctx->rcache, &task->alltoall.send_rcache_region_p->super); return UCC_ERR_NO_RESOURCE; } task->alltoall.recv_rcache_region_p = recv_ptr; if (reg_change_flag || (task->alltoall.recv_rcache_region_p->reg.mr->addr != a2a->previous_recv_address[seq_index])) { flag |= UCC_MLX5_NEED_RECV_MKEY_UPDATE; } tl_debug(UCC_TL_MLX5_TEAM_LIB(team), "fanin start"); /* start task if completion event received */ /* Start fanin */ ucc_tl_mlx5_get_my_ctrl(a2a, seq_index)->mkey_cache_flag = flag; ucc_tl_mlx5_update_mkeys_entries(a2a, task, flag); if (UCC_OK == ucc_tl_mlx5_node_fanin(team, task)) { tl_debug(UCC_TL_MLX5_TEAM_LIB(team), "fanin complete"); coll_task->status = UCC_OK; UCC_TL_MLX5_PROFILE_REQUEST_EVENT(task, "mlx5_alltoall_fanin_done", 0); return ucc_task_complete(coll_task); } ucc_progress_enqueue(UCC_TL_CORE_CTX(team)->pq, coll_task); return UCC_OK; } void ucc_tl_mlx5_reg_fanin_progress(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_assert(team->a2a->node.sbgp->group_rank == team->a2a->node.asr_rank); if (UCC_OK == ucc_tl_mlx5_node_fanin(team, task)) { tl_debug(UCC_TL_MLX5_TEAM_LIB(team), "fanin complete"); coll_task->status = UCC_OK; } } static ucc_status_t ucc_tl_mlx5_node_fanout(ucc_tl_mlx5_team_t *team, ucc_tl_mlx5_schedule_t *task) { ucc_tl_mlx5_alltoall_t *a2a = team->a2a; ucc_tl_mlx5_alltoall_ctrl_t *ctrl_v; /* First phase of fanout: asr signals it completed local ops and other ranks wait for asr */ if (a2a->node.sbgp->group_rank == a2a->node.asr_rank) { /* no need to check counter - we wait on data in device */ ucc_tl_mlx5_get_my_ctrl(a2a, task->alltoall.seq_index)->seq_num = task->alltoall.seq_num; } else { ctrl_v = ucc_tl_mlx5_get_ctrl(a2a, task->alltoall.seq_index, a2a->node.asr_rank); if (ctrl_v->seq_num != task->alltoall.seq_num) { return UCC_INPROGRESS; } } return UCC_OK; } static ucc_status_t ucc_tl_mlx5_fanout_start(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); coll_task->status = UCC_INPROGRESS; coll_task->super.status = UCC_INPROGRESS; tl_debug(UCC_TASK_LIB(task), "fanout start"); /* start task if completion event received */ UCC_TL_MLX5_PROFILE_REQUEST_EVENT(task, "mlx5_alltoall_fanout_start", 0); if (team->a2a->node.sbgp->group_rank == team->a2a->node.asr_rank) { UCC_TL_MLX5_PROFILE_REQUEST_EVENT( task, "mlx5_alltoall_wait-on-data_start", 0); } /* Start fanout */ ucc_progress_enqueue(UCC_TL_CORE_CTX(team)->pq, coll_task); return UCC_OK; } static void ucc_tl_mlx5_fanout_progress(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; ucc_status_t status; if (a2a->node.sbgp->group_rank == a2a->node.asr_rank) { status = ucc_tl_mlx5_poll_cq(a2a->net.umr_cq, UCC_TASK_LIB(task)); if (UCC_OK != status) { coll_task->status = status; return; } if (!task->alltoall.wait_wc) { coll_task->status = UCC_INPROGRESS; return; } UCC_TL_MLX5_PROFILE_REQUEST_EVENT( task, "mlx5_alltoall_wait-on-data_complete, fanout_start", 0); } if (UCC_OK == ucc_tl_mlx5_node_fanout(team, task)) { /*Cleanup alg resources - all done */ tl_debug(UCC_TASK_LIB(task), "Algorithm completion"); a2a->op_busy[task->alltoall.seq_index] = 0; coll_task->status = UCC_OK; UCC_TL_MLX5_PROFILE_REQUEST_EVENT(task, "mlx5_alltoall_fanout_done", 0); } } static ucc_status_t ucc_tl_mlx5_asr_barrier_start(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; tl_mlx5_barrier_t *local = tl_mlx5_barrier_local_addr(task); ucc_status_t status; int i; coll_task->status = UCC_INPROGRESS; coll_task->super.status = UCC_INPROGRESS; task->alltoall.started = 0; UCC_TL_MLX5_PROFILE_REQUEST_EVENT(task, "mlx5_alltoall_barrier_start", 0); // despite while statement in poll_umr_cq, non blocking because have independent cq, // will be finished in a finite time ucc_tl_mlx5_populate_send_recv_mkeys(team, task); //Reset atomic notification counter to 0 #if ATOMIC_IN_MEMIC tl_mlx5_atomic_t zero = 0; if (0 != ibv_memcpy_to_dm(a2a->net.atomic.counters, task->alltoall.seq_index * sizeof(tl_mlx5_atomic_t), &zero, sizeof(tl_mlx5_atomic_t))) { tl_error(UCC_TASK_LIB(task), "failed to reset atomic in memic"); return UCC_ERR_NO_MESSAGE; } #else a2a->net.atomic.counters[task->alltoall.seq_index] = 0; #endif if (UCC_TL_MLX5_TEAM_LIB(team)->cfg.asr_barrier) { tl_debug(UCC_TASK_LIB(task), "asr barrier start"); status = ucc_service_allreduce( UCC_TL_CORE_TEAM(team), &task->alltoall.barrier_scratch[0], &task->alltoall.barrier_scratch[1], UCC_DT_INT32, 1, UCC_OP_SUM, ucc_sbgp_to_subset(a2a->net.sbgp), &task->alltoall.barrier_req); if (status < 0) { tl_error(UCC_TASK_LIB(task), "failed to start asr barrier"); } for (i = 0; i < a2a->net.net_size; i++) { task->alltoall.op->blocks_sent[i] = 0; a2a->net.barrier .flags[(a2a->net.net_size + 1) * task->alltoall.seq_index + i] = task->alltoall.seq_num; } ucc_progress_enqueue(UCC_TL_CORE_CTX(team)->pq, coll_task); } else { *local = task->alltoall.seq_num; for (i = 0; i < a2a->net.net_size; i++) { task->alltoall.op->blocks_sent[i] = 0; if (i == a2a->net.sbgp->group_rank) { tl_mlx5_barrier_flag_set(task, i); continue; } send_start(team, i); status = send_block_data( a2a, i, (uintptr_t)local, sizeof(tl_mlx5_barrier_t), a2a->net.barrier.mr->lkey, tl_mlx5_barrier_my_remote_addr(task, i), tl_mlx5_barrier_remote_rkey(task, i), 0, NULL); if (UCC_OK == status) { status = send_done(team, i); } if (status != UCC_OK) { tl_error(UCC_TASK_LIB(task), "failed sending barrier notice"); return status; } UCC_TL_MLX5_PROFILE_REQUEST_EVENT( task, "mlx5_alltoall_barrier_send_posted", 0); } coll_task->status = UCC_OK; UCC_TL_MLX5_PROFILE_REQUEST_EVENT(task, "mlx5_alltoall_barrier_done", 0); return ucc_task_complete(coll_task); } return UCC_OK; } static void ucc_tl_mlx5_asr_barrier_progress(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_status_t status; status = ucc_collective_test(&task->alltoall.barrier_req->task->super); if (status < 0) { tl_error(UCC_TASK_LIB(task), "failure during asr barrier"); } else if (UCC_OK == status) { tl_debug(UCC_TASK_LIB(task), "asr barrier done"); UCC_TL_MLX5_PROFILE_REQUEST_EVENT(task, "mlx5_alltoall_barreir_done", 0); ucc_service_coll_finalize(task->alltoall.barrier_req); coll_task->status = UCC_OK; } } ucc_tl_mlx5_dm_chunk_t * ucc_tl_mlx5_a2a_wait_for_dm_chunk(ucc_tl_mlx5_schedule_t *task) { ucc_base_lib_t *lib = UCC_TASK_LIB(task); ucc_tl_mlx5_team_t *team = TASK_TEAM(&task->super); ucc_tl_mlx5_dm_chunk_t *dm = NULL; dm = ucc_mpool_get(&team->dm_pool); while (!dm) { if (UCC_OK != ucc_tl_mlx5_poll_cq(team->a2a->net.cq, lib)) { return NULL; } dm = ucc_mpool_get(&team->dm_pool); } dm->task = task; return dm; } // add polling mechanism for blocks in order to maintain const qp tx rx static ucc_status_t ucc_tl_mlx5_send_blocks_start(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_base_lib_t * lib = UCC_TASK_LIB(task); ucc_tl_mlx5_team_t * team = TASK_TEAM(&task->super); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; ucc_rank_t node_size = a2a->node.sbgp->group_size; ucc_rank_t net_size = a2a->net.sbgp->group_size; size_t op_msgsize = node_size * a2a->max_msg_size * UCC_TL_TEAM_SIZE(team) * a2a->max_num_of_columns; size_t node_msgsize = SQUARED(node_size) * task->alltoall.msg_size; int block_h = task->alltoall.block_height; int block_w = task->alltoall.block_width; size_t col_msgsize = task->alltoall.msg_size * block_w * node_size; size_t line_msgsize = task->alltoall.msg_size * block_h * node_size; size_t block_msgsize = block_h * block_w * task->alltoall.msg_size; ucc_status_t status = UCC_OK; int node_grid_w = node_size / block_w; int node_nbr_blocks = (node_size * node_size) / (block_h * block_w); int seq_index = task->alltoall.seq_index; int block_row = 0; int block_col = 0; uint64_t remote_addr = 0; ucc_tl_mlx5_dm_chunk_t *dm = NULL; int batch_size = UCC_TL_MLX5_TEAM_LIB(team)->cfg.block_batch_size; int num_serialized_batches = UCC_TL_MLX5_TEAM_LIB(team)->cfg.num_serialized_batches; int num_batches_per_passage = UCC_TL_MLX5_TEAM_LIB(team)->cfg.num_batches_per_passage; int i, j, k, send_to_self, block_idx, rank, dest_rank, cyc_rank, node_idx; uint64_t src_addr; coll_task->status = UCC_INPROGRESS; coll_task->super.status = UCC_INPROGRESS; tl_debug(lib, "send blocks start"); rank = a2a->net.rank_map[a2a->net.sbgp->group_rank]; if (!task->alltoall.send_blocks_enqueued) { UCC_TL_MLX5_PROFILE_REQUEST_EVENT(task, "mlx5_alltoall_block_send_start", 0); } for (j = 0; j < num_batches_per_passage; j++) { for (node_idx = 0; node_idx < net_size; node_idx++) { cyc_rank = (node_idx + a2a->net.sbgp->group_rank) % net_size; dest_rank = a2a->net.rank_map[cyc_rank]; send_to_self = (cyc_rank == a2a->net.sbgp->group_rank); if (tl_mlx5_barrier_flag(task, cyc_rank) != task->alltoall.seq_num) { continue; } dm = NULL; if (!send_to_self && task->alltoall.op->blocks_sent[cyc_rank] < node_nbr_blocks) { dm = ucc_tl_mlx5_a2a_wait_for_dm_chunk(task); } send_start(team, cyc_rank); for (i = 0; i < num_serialized_batches; i++) { for (k = 0; k < batch_size && task->alltoall.op->blocks_sent[cyc_rank] < node_nbr_blocks; k++, task->alltoall.op->blocks_sent[cyc_rank]++) { block_idx = task->alltoall.op->blocks_sent[cyc_rank]; block_col = block_idx % node_grid_w; block_row = block_idx / node_grid_w; src_addr = (uintptr_t)( op_msgsize * seq_index + node_msgsize * dest_rank + col_msgsize * block_col + block_msgsize * block_row); if (send_to_self || !k) { remote_addr = (uintptr_t)(op_msgsize * seq_index + node_msgsize * rank + block_msgsize * block_col + line_msgsize * block_row); } if (send_to_self) { status = ucc_tl_mlx5_post_transpose( tl_mlx5_get_qp(a2a, cyc_rank), a2a->node.ops[seq_index].send_mkeys[0]->lkey, a2a->net.rkeys[cyc_rank], src_addr, remote_addr, task->alltoall.msg_size, block_w, block_h, (block_row == 0 && block_col == 0) ? IBV_SEND_SIGNALED : 0); if (UCC_OK != status) { return status; } } else { ucc_assert(dm != NULL); status = ucc_tl_mlx5_post_transpose( tl_mlx5_get_qp(a2a, cyc_rank), a2a->node.ops[seq_index].send_mkeys[0]->lkey, team->dm_mr->rkey, src_addr, dm->addr + k * block_msgsize, task->alltoall.msg_size, block_w, block_h, 0); if (UCC_OK != status) { return status; } } } if (!send_to_self && k) { status = send_block_data( a2a, cyc_rank, dm->addr, block_msgsize * k, team->dm_mr->lkey, remote_addr, a2a->net.rkeys[cyc_rank], IBV_SEND_SIGNALED, dm); if (status != UCC_OK) { tl_error(lib, "failed sending block [%d,%d,%d]", node_idx, block_row, block_col); return status; } dm->posted_sends++; } } status = send_done(team, cyc_rank); if (status != UCC_OK) { return status; } if (dm) { dm->posted_all = 1; } if (task->alltoall.op->blocks_sent[cyc_rank] == node_nbr_blocks) { send_start(team, cyc_rank); status = send_atomic(a2a, cyc_rank, tl_mlx5_atomic_addr(task, cyc_rank), tl_mlx5_atomic_rkey(task, cyc_rank)); if (status != UCC_OK) { tl_error(UCC_TASK_LIB(task), "Failed sending atomic to node [%d]", cyc_rank); return status; } status = send_done(team, cyc_rank); if (UCC_OK != status) { tl_error(lib, "Failed sending atomic to node %d", node_idx); return status; } task->alltoall.op->blocks_sent[cyc_rank]++; task->alltoall.started++; } } } if (!task->alltoall.send_blocks_enqueued) { ucc_progress_enqueue(UCC_TL_CORE_CTX(team)->pq, coll_task); task->alltoall.send_blocks_enqueued = 1; } if (task->alltoall.started == a2a->net.net_size) { UCC_TL_MLX5_PROFILE_REQUEST_EVENT(task, "mlx5_alltoall_block_send_done", 0); status = ucc_tl_mlx5_post_wait_on_data( a2a->net.umr_qp, a2a->net.net_size, a2a->net.atomic.mr->lkey, (uintptr_t) #if ATOMIC_IN_MEMIC PTR_OFFSET(0, #else PTR_OFFSET(a2a->net.atomic.counters, #endif seq_index * sizeof(tl_mlx5_atomic_t)), task); UCC_TL_MLX5_PROFILE_REQUEST_EVENT( task, "mlx5_alltoall_block_post_wait_on_data_done", 0); } return status; } static ucc_status_t ucc_tl_mlx5_send_blocks_leftovers_start(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; int node_size = a2a->node.sbgp->group_size; int net_size = a2a->net.sbgp->group_size; int seq_index = task->alltoall.seq_index; size_t msg_size = task->alltoall.msg_size; size_t op_msgsize = node_size * a2a->max_msg_size * UCC_TL_TEAM_SIZE(team) * a2a->max_num_of_columns; size_t mkey_msgsize = node_size * a2a->max_msg_size * UCC_TL_TEAM_SIZE(team); int block_size = task->alltoall.block_height; size_t col_msgsize = msg_size * block_size * node_size; size_t block_msgsize = SQUARED(block_size) * msg_size; int block_size_leftovers_side = node_size % block_size; size_t col_msgsize_leftovers = msg_size * block_size_leftovers_side * node_size; size_t block_msgsize_leftovers = block_size_leftovers_side * block_size * msg_size; size_t corner_msgsize = SQUARED(block_size_leftovers_side) * msg_size; ucc_status_t status = UCC_OK; ucc_base_lib_t *lib = UCC_TASK_LIB(coll_task); int nbc = task->alltoall.num_of_blocks_columns; int i, j, k, dest_rank, rank, cyc_rank, bs_x, bs_y; size_t current_block_msgsize; uint64_t src_addr, remote_addr; ucc_tl_mlx5_dm_chunk_t *dm; uintptr_t dm_addr; coll_task->status = UCC_INPROGRESS; coll_task->super.status = UCC_INPROGRESS; tl_debug(UCC_TASK_LIB(task), "send blocks start"); rank = a2a->net.rank_map[a2a->net.sbgp->group_rank]; for (i = 0; i < net_size; i++) { cyc_rank = (i + a2a->net.sbgp->group_rank) % net_size; dest_rank = a2a->net.rank_map[cyc_rank]; if (task->alltoall.op->blocks_sent[cyc_rank] || tl_mlx5_barrier_flag(task, cyc_rank) != task->alltoall.seq_num) { continue; } //send all blocks from curr node to some ARR for (j = 0; j < nbc; j++) { for (k = 0; k < nbc; k++) { if (j != (nbc - 1)) { src_addr = (uintptr_t)(col_msgsize * dest_rank + block_msgsize * k); } else { src_addr = (uintptr_t)(col_msgsize_leftovers * dest_rank + block_msgsize_leftovers * k); } if (k != (nbc - 1)) { remote_addr = (uintptr_t)( op_msgsize * seq_index + col_msgsize * rank + block_msgsize * j + mkey_msgsize * k); current_block_msgsize = (j != (nbc - 1)) ? block_msgsize : block_msgsize_leftovers; } else { remote_addr = (uintptr_t)( op_msgsize * seq_index + col_msgsize_leftovers * rank + block_msgsize_leftovers * j + mkey_msgsize * k); current_block_msgsize = (j != (nbc - 1)) ? block_msgsize_leftovers : corner_msgsize; } bs_x = k < nbc - 1 ? block_size : block_size_leftovers_side; bs_y = j < nbc - 1 ? block_size : block_size_leftovers_side; send_start(team, cyc_rank); //todo : start/end for RC ? if (bs_x == 1 || bs_y == 1) { status = send_block_data( a2a, cyc_rank, src_addr, current_block_msgsize, a2a->node.ops[seq_index].send_mkeys[j]->lkey, remote_addr, a2a->net.rkeys[cyc_rank], 0, NULL); } else { dm = ucc_mpool_get(&team->dm_pool); while (!dm) { status = send_done(team, cyc_rank); if (UCC_OK != status) { return status; } status = ucc_tl_mlx5_poll_cq(a2a->net.cq, lib); if (UCC_OK != status) { return status; } dm = ucc_mpool_get(&team->dm_pool); send_start(team, cyc_rank); } dm_addr = dm->addr; dm->task = task; status = ucc_tl_mlx5_post_transpose( tl_mlx5_get_qp(a2a, cyc_rank), a2a->node.ops[seq_index].send_mkeys[j]->lkey, team->dm_mr->rkey, src_addr, dm_addr, msg_size, bs_x, bs_y, 0); if (UCC_OK != status) { return status; } status = send_block_data( a2a, cyc_rank, dm_addr, current_block_msgsize, team->dm_mr->lkey, remote_addr, a2a->net.rkeys[cyc_rank], IBV_SEND_SIGNALED, dm); } if (status != UCC_OK) { tl_error(UCC_TASK_LIB(task), "Failed sending block [%d,%d,%d]", i, j, k); return status; } status = send_done(team, cyc_rank); if (UCC_OK != status) { return status; } } } send_start(team, cyc_rank); status = send_atomic(a2a, cyc_rank, tl_mlx5_atomic_addr(task, cyc_rank), tl_mlx5_atomic_rkey(task, cyc_rank)); if (UCC_OK == status) { status = send_done(team, cyc_rank); } task->alltoall.op->blocks_sent[cyc_rank] = 1; task->alltoall.started++; if (status != UCC_OK) { tl_error(UCC_TASK_LIB(task), "Failed sending atomic to node [%d]", cyc_rank); return status; } } if (!task->alltoall.send_blocks_enqueued) { ucc_progress_enqueue(UCC_TL_CORE_CTX(team)->pq, coll_task); task->alltoall.send_blocks_enqueued = 1; } if (task->alltoall.started == a2a->net.net_size) { status = ucc_tl_mlx5_post_wait_on_data( a2a->net.umr_qp, a2a->net.net_size, a2a->net.atomic.mr->lkey, (uintptr_t) #if ATOMIC_IN_MEMIC PTR_OFFSET(0, #else PTR_OFFSET(a2a->net.atomic.counters, #endif seq_index * sizeof(tl_mlx5_atomic_t)), task); } return status; } static void ucc_tl_mlx5_send_blocks_progress(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = TASK_SCHEDULE(coll_task); ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; ucc_status_t status; if (task->alltoall.started != a2a->net.net_size) { coll_task->post(coll_task); return; } status = ucc_tl_mlx5_poll_cq(a2a->net.cq, UCC_TASK_LIB(coll_task)); if (UCC_OK != status) { coll_task->status = status; return; } if (task->alltoall.blocks_sent == task->alltoall.blocks_completed) { UCC_TL_MLX5_PROFILE_REQUEST_EVENT( task, "mlx5_alltoall_all_blocks_completed", 0); coll_task->status = UCC_OK; } } ucc_status_t ucc_tl_mlx5_alltoall_start(ucc_coll_task_t *coll_task) { UCC_TL_MLX5_PROFILE_REQUEST_EVENT(coll_task, "mlx5_alltoall_start", 0); return ucc_schedule_start(coll_task); } ucc_status_t ucc_tl_mlx5_alltoall_finalize(ucc_coll_task_t *coll_task) { ucc_tl_mlx5_schedule_t *task = ucc_derived_of(coll_task, ucc_tl_mlx5_schedule_t); ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; ucc_tl_mlx5_context_t *ctx = SCHEDULE_CTX(task); int seq_index = task->alltoall.seq_index; ucc_status_t status; a2a->previous_msg_size[seq_index] = task->alltoall.msg_size; a2a->previous_send_address[seq_index] = task->alltoall.send_rcache_region_p->reg.mr->addr; a2a->previous_recv_address[seq_index] = task->alltoall.recv_rcache_region_p->reg.mr->addr; ucc_rcache_region_put(ctx->rcache, &task->alltoall.send_rcache_region_p->super); ucc_rcache_region_put(ctx->rcache, &task->alltoall.recv_rcache_region_p->super); UCC_TL_MLX5_PROFILE_REQUEST_EVENT(coll_task, "mlx5_alltoall_done", 0); status = ucc_schedule_finalize(coll_task); ucc_tl_mlx5_put_schedule(task); return status; } static inline int block_size_fits(size_t msgsize, int height, int width) { int t; if (msgsize > MAX_MSG_SIZE || height > MAX_BLOCK_SIZE || width > MAX_BLOCK_SIZE) { return false; } t = ucc_round_up_power2(ucc_max(msgsize, 8)); return height * ucc_max(ucc_round_up_power2(width) * t, MAX_MSG_SIZE) <= MAX_TRANSPOSE_SIZE; } #define MAYBE_CONTINUE_OR_BREAK_IF_REGULAR(x) \ if (force_regular) { \ if (x > ppn) { \ break; \ } else if (ppn % x) { \ continue; \ } \ } static inline void get_block_dimensions(int ppn, int msgsize, int force_regular, ucc_tl_mlx5_alltoall_block_shape_modes_t block_shape_mode, int *block_height, int *block_width) { int h_best = 1; int w_best = 1; int h, w, w_min, w_max; for (h = 1; h <= MAX_BLOCK_SIZE; h++) { MAYBE_CONTINUE_OR_BREAK_IF_REGULAR(h); w_max = block_shape_mode != UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_LONG ? MAX_BLOCK_SIZE : h; w_min = block_shape_mode != UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_WIDE ? 1 : h; w_min = ucc_max(w_min, h_best * w_best / h); for (w = w_min; w <= w_max; w++) { MAYBE_CONTINUE_OR_BREAK_IF_REGULAR(w); if (block_size_fits(msgsize, h, w)) { if (h * w > h_best * w_best || abs(h / w - 1) < abs(h_best / w_best - 1)) { h_best = h; w_best = w; } } } } *block_height = h_best; *block_width = w_best; } UCC_TL_MLX5_PROFILE_FUNC(ucc_status_t, ucc_tl_mlx5_alltoall_init, (coll_args, team, task_h), ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h) { ucc_tl_mlx5_team_t *tl_team = ucc_derived_of(team, ucc_tl_mlx5_team_t); ucc_tl_mlx5_alltoall_t *a2a = tl_team->a2a; ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(tl_team); int is_asr = (a2a->node.sbgp->group_rank == a2a->node.asr_rank); int n_tasks = is_asr ? 5 : 3; int curr_task = 0; int ppn = tl_team->a2a->node.sbgp->group_size; ucc_tl_mlx5_lib_config_t *cfg = &UCC_TL_MLX5_TEAM_LIB(tl_team)->cfg; ucc_schedule_t *schedule; ucc_tl_mlx5_schedule_t *task; size_t msg_size; int block_size, i; ucc_coll_task_t *tasks[5]; ucc_status_t status; size_t bytes_count, bytes_count_last, bytes_skip, bytes_skip_last; if (!ctx->cfg.enable_alltoall) { return UCC_ERR_NOT_SUPPORTED; } if (UCC_IS_INPLACE(coll_args->args)) { return UCC_ERR_NOT_SUPPORTED; } msg_size = (coll_args->args.src.info.count * ucc_dt_size(coll_args->args.src.info.datatype)) / UCC_TL_TEAM_SIZE(tl_team); if (!msg_size) { tl_trace(UCC_TL_TEAM_LIB(tl_team), "msg size too short, reduces to nullop"); return UCC_ERR_NOT_SUPPORTED; } if (msg_size > a2a->max_msg_size) { tl_trace(UCC_TL_TEAM_LIB(tl_team), "msg size too long"); return UCC_ERR_NOT_SUPPORTED; } status = ucc_tl_mlx5_get_schedule(tl_team, coll_args, &task); if (ucc_unlikely(UCC_OK != status)) { return status; } schedule = &task->super; for (i = 0; i < n_tasks; i++) { tasks[i] = &ucc_tl_mlx5_init_task(coll_args, team, schedule)->super; } task->alltoall.send_blocks_enqueued = 0; task->alltoall.started = 0; task->alltoall.wait_wc = 0; task->alltoall.blocks_sent = 0; task->alltoall.blocks_completed = 0; task->alltoall.seq_num = a2a->sequence_number; task->alltoall.seq_index = SEQ_INDEX(a2a->sequence_number); task->alltoall.op = &a2a->node.ops[ task->alltoall.seq_index]; task->alltoall.msg_size = msg_size; tl_trace(UCC_TL_TEAM_LIB(tl_team), "Seq num is %d", task->alltoall.seq_num); a2a->sequence_number += 1; if (a2a->requested_block_size) { task->alltoall.block_height = task->alltoall.block_width = a2a->requested_block_size; if (cfg->force_regular && ((ppn % task->alltoall.block_height) || (ppn % task->alltoall.block_width))) { tl_debug(UCC_TL_TEAM_LIB(tl_team), "the requested block size implies irregular case" "consider changing the block size or turn off the config " "FORCE_REGULAR"); return UCC_ERR_INVALID_PARAM; } } else { if (!cfg->force_regular) { if (!(cfg->block_shape_mode != UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_SQUARE)) { tl_debug(UCC_TL_TEAM_LIB(tl_team), "turning off FORCE_REGULAR automatically forces the " "blocks to be square"); cfg->block_shape_mode = UCC_TL_MLX5_ALLTOALL_BLOCK_SHAPE_SQUARE; } } get_block_dimensions(ppn, task->alltoall.msg_size, cfg->force_regular, cfg->block_shape_mode, &task->alltoall.block_height, &task->alltoall.block_width); } tl_debug(UCC_TL_TEAM_LIB(tl_team), "block dimensions: [%d,%d]", task->alltoall.block_height, task->alltoall.block_width); //todo following section correct assuming homogenous PPN across all nodes task->alltoall.num_of_blocks_columns = (a2a->node.sbgp->group_size % task->alltoall.block_height) ? ucc_div_round_up(a2a->node.sbgp->group_size, task->alltoall.block_height) : 0; // TODO remove for connectX-7 - this is mkey_entry->stride (count+skip) limitation - only 16 bits if (task->alltoall .num_of_blocks_columns) { // for other case we will never reach limit - we use bytes skip only for the "leftovers" mode, which means when // num_of_blocks_columns != 0 size_t limit = (1ULL << 16); // TODO We need to query this from device (or device type) and not user hardcoded values ucc_assert(task->alltoall.block_height == task->alltoall.block_width); block_size = task->alltoall.block_height; ucc_assert(task->alltoall.block_height == task->alltoall.block_width); bytes_count_last = (ppn % block_size) * msg_size; bytes_skip_last = (ppn - (ppn % block_size)) * msg_size; bytes_count = block_size * msg_size; bytes_skip = (ppn - block_size) * msg_size; if ((bytes_count + bytes_skip >= limit) || (bytes_count_last + bytes_skip_last >= limit)) { tl_debug(UCC_TL_TEAM_LIB(tl_team), "unsupported operation"); status = UCC_ERR_NOT_SUPPORTED; goto put_schedule; } } ucc_schedule_add_task(schedule, tasks[0]); ucc_event_manager_subscribe(&schedule->super, UCC_EVENT_SCHEDULE_STARTED, tasks[0], ucc_task_start_handler); for (i = 0; i < (n_tasks - 1); i++) { ucc_schedule_add_task(schedule, tasks[i + 1]); ucc_event_manager_subscribe(tasks[i], UCC_EVENT_COMPLETED, tasks[i + 1], ucc_task_start_handler); } tasks[curr_task]->post = ucc_tl_mlx5_poll_free_op_slot_start; tasks[curr_task]->progress = ucc_tl_mlx5_poll_free_op_slot_progress; curr_task++; tasks[curr_task]->post = ucc_tl_mlx5_reg_fanin_start; tasks[curr_task]->progress = ucc_tl_mlx5_reg_fanin_progress; curr_task++; if (is_asr) { tasks[curr_task]->post = ucc_tl_mlx5_asr_barrier_start; tasks[curr_task]->progress = ucc_tl_mlx5_asr_barrier_progress; curr_task++; if (task->alltoall.num_of_blocks_columns) { tasks[curr_task]->post = ucc_tl_mlx5_send_blocks_leftovers_start; } else { tasks[curr_task]->post = ucc_tl_mlx5_send_blocks_start; } tasks[curr_task]->progress = ucc_tl_mlx5_send_blocks_progress; curr_task++; } tasks[curr_task]->post = ucc_tl_mlx5_fanout_start; tasks[curr_task]->progress = ucc_tl_mlx5_fanout_progress; schedule->super.post = ucc_tl_mlx5_alltoall_start; schedule->super.progress = NULL; schedule->super.finalize = ucc_tl_mlx5_alltoall_finalize; schedule->super.triggered_post = NULL; *task_h = &schedule->super; return status; put_schedule: ucc_tl_mlx5_put_schedule(task); return status; } ucc-1.8.0/src/components/tl/mlx5/alltoall/alltoall_inline.h0000664000175000017500000001254615211535620024065 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_MLX5_INLINE_H_ #define UCC_TL_MLX5_INLINE_H_ #include "tl_mlx5_ib.h" #include "tl_mlx5_coll.h" #include "tl_mlx5_wqe.h" #include "alltoall/alltoall.h" static inline struct ibv_qp_ex *tl_mlx5_get_qp_ex(ucc_tl_mlx5_alltoall_t *a2a, ucc_rank_t rank) { if (a2a->is_dc) { return a2a->net.dcis[rank % a2a->num_dci_qps].dc_qpex; } return a2a->net.rc_qps[rank].qp_ex; } static inline struct ibv_qp *tl_mlx5_get_qp(ucc_tl_mlx5_alltoall_t *a2a, ucc_rank_t rank) { if (a2a->is_dc) { return a2a->net.dcis[rank % a2a->num_dci_qps].dci_qp; } return a2a->net.rc_qps[rank].qp; } static inline ucc_status_t send_block_data(ucc_tl_mlx5_alltoall_t *a2a, ucc_rank_t rank, uint64_t src_addr, uint32_t msg_size, uint32_t lkey, uint64_t remote_addr, uint32_t rkey, int send_flags, ucc_tl_mlx5_dm_chunk_t *dm) { struct ibv_qp *qp = tl_mlx5_get_qp(a2a, rank); struct ibv_ah *ah = NULL; uint32_t dctn = 0; if (dm) { dm->task->alltoall.blocks_sent++; } if (a2a->is_dc) { ah = a2a->net.ahs[rank]; dctn = a2a->net.remote_dctns[rank]; } return ucc_tl_mlx5_post_rdma(qp, dctn, ah, src_addr, msg_size, lkey, remote_addr, rkey, send_flags, (uintptr_t)dm); } static inline void send_start(ucc_tl_mlx5_team_t *team, ucc_rank_t rank) { ibv_wr_start(tl_mlx5_get_qp_ex(team->a2a, rank)); } static inline ucc_status_t send_done(ucc_tl_mlx5_team_t *team, ucc_rank_t rank) { if (ibv_wr_complete(tl_mlx5_get_qp_ex(team->a2a, rank))) { tl_error(UCC_TL_TEAM_LIB(team), "ibv_wr_complete failed, errno %d", errno); return UCC_ERR_NO_MESSAGE; } return UCC_OK; } static inline ucc_status_t send_atomic(ucc_tl_mlx5_alltoall_t *a2a, ucc_rank_t rank, void *remote_addr, uint32_t rkey) { struct ibv_qp_ex *qp_ex; struct mlx5dv_qp_ex *qp_dv; qp_ex = tl_mlx5_get_qp_ex(a2a, rank); qp_ex->wr_flags = 0; ibv_wr_atomic_fetch_add(qp_ex, rkey, (uintptr_t)remote_addr, 1ULL); if (a2a->is_dc) { qp_dv = mlx5dv_qp_ex_from_ibv_qp_ex(qp_ex); mlx5dv_wr_set_dc_addr(qp_dv, a2a->net.ahs[rank], a2a->net.remote_dctns[rank], DC_KEY); } ibv_wr_set_sge(qp_ex, a2a->atomic_scratch_bf_mr->lkey, (uint64_t)a2a->atomic_scratch_bf_mr->addr, a2a->atomic_scratch_bf_mr->length); return UCC_OK; } static inline void *tl_mlx5_atomic_addr(ucc_tl_mlx5_schedule_t *task, ucc_rank_t rank) { void *remote_atomic = NULL; #if !ATOMIC_IN_MEMIC ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); remote_atomic = team->a2a->net.remote_ctrl[rank].atomic.addr; #endif return PTR_OFFSET(remote_atomic, task->alltoall.seq_index * sizeof(tl_mlx5_atomic_t)); } static inline uint32_t tl_mlx5_atomic_rkey(ucc_tl_mlx5_schedule_t *task, ucc_rank_t rank) { ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); return team->a2a->net.remote_ctrl[rank].atomic.rkey; } static inline tl_mlx5_barrier_t tl_mlx5_barrier_flag(ucc_tl_mlx5_schedule_t *task, ucc_rank_t rank) { ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_rank_t net_size = team->a2a->net.net_size; return team->a2a->net.barrier .flags[(net_size + 1) * task->alltoall.seq_index + rank]; } static inline void tl_mlx5_barrier_flag_set(ucc_tl_mlx5_schedule_t *task, ucc_rank_t rank) { ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_rank_t net_size = team->a2a->net.net_size; team->a2a->net.barrier .flags[(net_size + 1) * task->alltoall.seq_index + rank] = task->alltoall.seq_num; } static inline tl_mlx5_barrier_t * tl_mlx5_barrier_local_addr(ucc_tl_mlx5_schedule_t *task) { ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_rank_t net_size = team->a2a->net.net_size; return &team->a2a->net.barrier .flags[(net_size + 1) * task->alltoall.seq_index + net_size]; } static inline uintptr_t tl_mlx5_barrier_my_remote_addr(ucc_tl_mlx5_schedule_t *task, ucc_rank_t rank) { ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); ucc_rank_t net_size = team->a2a->net.net_size; ucc_rank_t net_rank = team->a2a->net.sbgp->group_rank; void * remote_barrier; ptrdiff_t offset; remote_barrier = team->a2a->net.remote_ctrl[rank].barrier.addr; offset = (task->alltoall.seq_index * (net_size + 1) + net_rank) * sizeof(tl_mlx5_barrier_t); return (uintptr_t)PTR_OFFSET(remote_barrier, offset); } static inline uint32_t tl_mlx5_barrier_remote_rkey(ucc_tl_mlx5_schedule_t *task, ucc_rank_t rank) { ucc_tl_mlx5_team_t *team = SCHEDULE_TEAM(task); return team->a2a->net.remote_ctrl[rank].barrier.rkey; } #endif ucc-1.8.0/src/components/tl/mlx5/alltoall/alltoall.c0000664000175000017500000006325515211535620022525 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "alltoall.h" #include "alltoall_mkeys.h" #include "tl_mlx5_ib.h" #include "core/ucc_team.h" #include struct rank_data { ucc_rank_t team_rank; ucc_rank_t sbgp_rank; }; typedef struct net_exchage { struct rank_data rd; alltoall_net_ctrl_t net_ctrl; uint32_t port_lid; uint32_t recv_mkey_rkey; uint32_t qpn[1]; } net_exchange_t; static int compare_rank_data(const void *a, const void *b) { const struct rank_data *d1 = (const struct rank_data *)a; const struct rank_data *d2 = (const struct rank_data *)b; return d1->team_rank > d2->team_rank ? 1 : -1; } static ucc_status_t build_rank_map(ucc_tl_mlx5_alltoall_t *a2a, net_exchange_t *global_data, size_t local_data_size) { struct rank_data *data; net_exchange_t *d; int i; data = ucc_malloc(sizeof(*data) * a2a->net.net_size); if (!data) { return UCC_ERR_NO_MEMORY; } for (i = 0; i < a2a->net.net_size; i++) { d = PTR_OFFSET(global_data, i * local_data_size); data[i].team_rank = d->rd.team_rank; data[i].sbgp_rank = d->rd.sbgp_rank; } a2a->net.rank_map = ucc_malloc(sizeof(int) * a2a->net.net_size); if (!a2a->net.rank_map) { ucc_free(data); return UCC_ERR_NO_MEMORY; } qsort(data, a2a->net.net_size, sizeof(*data), compare_rank_data); for (i = 0; i < a2a->net.net_size; i++) { a2a->net.rank_map[data[i].sbgp_rank] = i; } ucc_free(data); return UCC_OK; } ucc_status_t ucc_tl_mlx5_team_init_alltoall(ucc_tl_mlx5_team_t *team) { ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(team); ucc_tl_mlx5_alltoall_t *a2a; ucc_sbgp_t *node, *net; int i, j, node_size, ppn, team_size, nnodes; ucc_topo_t *topo; team->a2a = NULL; team->dm_ptr = NULL; team->a2a_status.local = UCC_OK; topo = team->topo; node = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); net = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); node_size = node->group_size; nnodes = ucc_topo_nnodes(topo); team_size = UCC_TL_TEAM_SIZE(team); if (!ucc_topo_isoppn(topo)) { tl_debug(ctx->super.super.lib, "disabling mlx5 a2a for team with non-uniform ppn, " "min_ppn %d, max_ppn %d", ucc_topo_min_ppn(topo), ucc_topo_max_ppn(topo)); goto non_fatal_error; } ppn = ucc_topo_max_ppn(topo); if (net->status == UCC_SBGP_NOT_EXISTS) { tl_debug(ctx->super.super.lib, "disabling mlx5 a2a for single node team"); goto non_fatal_error; } if (nnodes == team_size) { tl_debug(ctx->super.super.lib, "disabling mlx5 a2a for ppn=1 case, not supported so far"); goto non_fatal_error; } for (i = 0; i < nnodes; i++) { for (j = 1; j < ppn; j++) { if (!ucc_team_ranks_on_same_node(i * ppn, i * ppn + j, UCC_TL_CORE_TEAM(team))) { tl_debug(ctx->super.super.lib, "disabling mlx5 a2a for team with non contiguous " "ranks-per-node placement"); goto non_fatal_error; } } } team->a2a = ucc_calloc(1, sizeof(*team->a2a), "mlx5_a2a"); if (!team->a2a) { return UCC_ERR_NO_MEMORY; } a2a = team->a2a; a2a->node_size = node_size; a2a->pd = ctx->shared_pd; a2a->ctx = ctx->shared_ctx; a2a->ib_port = ctx->ib_port; a2a->node.sbgp = node; a2a->net.sbgp = net; a2a->node.asr_rank = MLX5_ASR_RANK; a2a->num_dci_qps = UCC_TL_MLX5_TEAM_LIB(team)->cfg.num_dci_qps; a2a->sequence_number = 1; a2a->net.atomic.counters = NULL; a2a->net.ctrl_mr = NULL; a2a->net.remote_ctrl = NULL; a2a->net.rank_map = NULL; a2a->max_msg_size = MAX_MSG_SIZE; a2a->max_num_of_columns = ucc_div_round_up(node->group_size, 2 /* todo: there can be an estimation of minimal possible block size */); if (a2a->node.asr_rank == node->group_rank) { team->a2a_status.local = ucc_tl_mlx5_dm_init(team); if (UCC_OK != team->a2a_status.local) { tl_debug(UCC_TL_TEAM_LIB(team), "failed to init device memory"); } } return UCC_OK; non_fatal_error: team->a2a_status.local = UCC_ERR_NOT_SUPPORTED; return UCC_OK; } ucc_status_t ucc_tl_mlx5_team_test_alltoall_start(ucc_tl_mlx5_team_t *team) { ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(team); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; size_t storage_size; if (team->a2a_status.global != UCC_OK) { tl_debug(ctx->super.super.lib, "global status in error state: %s", ucc_status_string(team->a2a_status.global)); ucc_tl_mlx5_dm_cleanup(team); if (a2a) { ucc_free(a2a); team->a2a = NULL; } ucc_tl_mlx5_topo_cleanup(team); return team->a2a_status.global; } if (a2a->node.asr_rank == a2a->node.sbgp->group_rank) { a2a->net.net_size = a2a->net.sbgp->group_size; storage_size = OP_SEGMENT_SIZE(a2a) * MAX_OUTSTANDING_OPS; a2a->bcast_data.shmid = shmget(IPC_PRIVATE, storage_size, IPC_CREAT | 0600); if (a2a->bcast_data.shmid == -1) { tl_debug(ctx->super.super.lib, "failed to allocate sysv shm segment for %zd bytes", storage_size); } else { a2a->node.storage = shmat(a2a->bcast_data.shmid, NULL, 0); memset(a2a->node.storage, 0, storage_size); shmctl(a2a->bcast_data.shmid, IPC_RMID, NULL); } a2a->bcast_data.net_size = a2a->net.sbgp->group_size; } a2a->state = TL_MLX5_ALLTOALL_STATE_SHMID; team->a2a = a2a; return ucc_service_bcast( UCC_TL_CORE_TEAM(team), &a2a->bcast_data, sizeof(ucc_tl_mlx5_a2a_bcast_data_t), a2a->node.asr_rank, ucc_sbgp_to_subset(a2a->node.sbgp), &team->scoll_req); } static void ucc_tl_mlx5_alltoall_barrier_free(ucc_tl_mlx5_alltoall_t *a2a) { ibv_dereg_mr(a2a->net.barrier.mr); ucc_free(a2a->net.barrier.flags); } static ucc_status_t ucc_tl_mlx5_alltoall_barrier_alloc(ucc_tl_mlx5_team_t *team) { ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(team); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; size_t size; /* allocating net_size + 1 flags. Last one is used as local buf for barrier RDMA */ size = (a2a->net.net_size + 1) * sizeof(*a2a->net.barrier.flags) * MAX_OUTSTANDING_OPS; a2a->net.barrier.flags = ucc_calloc(1, size, "barrier"); if (!a2a->net.barrier.flags) { tl_error(UCC_TL_TEAM_LIB(team), "failed to allocate %zd bytes for barrier flags array", size); return UCC_ERR_NO_MEMORY; } a2a->net.barrier.mr = ibv_reg_mr(ctx->shared_pd, a2a->net.barrier.flags, size, IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE); if (!a2a->net.barrier.mr) { tl_error(UCC_TL_TEAM_LIB(team), "failed to register barrier flags array"); ucc_free(a2a->net.barrier.flags); return UCC_ERR_NO_MESSAGE; } return UCC_OK; } ucc_status_t ucc_tl_mlx5_team_test_alltoall_progress(ucc_tl_mlx5_team_t *team) { ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(team); ucc_tl_mlx5_alltoall_t *a2a = team->a2a; ucc_base_lib_t *lib = UCC_TL_TEAM_LIB(team); int i = 0; net_exchange_t *local_data = NULL; ucc_rank_t node_size, node_rank; ucc_status_t status; ucc_tl_mlx5_alltoall_op_t *op; int j, asr_cq_size, net_size, ret; struct ibv_port_attr port_attr; size_t op_seg_size, local_data_size, umr_buf_size; net_exchange_t *global_data, *remote_data; if (team->a2a_status.local < 0) { return team->a2a_status.local; } node_size = a2a->node.sbgp->group_size; node_rank = a2a->node.sbgp->group_rank; op_seg_size = OP_SEGMENT_SIZE(a2a); switch (a2a->state) { case TL_MLX5_ALLTOALL_STATE_SHMID: status = ucc_service_coll_test(team->scoll_req); if (status < 0) { tl_error(lib, "failure during service coll exchange: %s", ucc_status_string(status)); ucc_service_coll_finalize(team->scoll_req); return status; } if (UCC_INPROGRESS == status) { return status; } ucc_service_coll_finalize(team->scoll_req); if (a2a->bcast_data.shmid == -1) { tl_error(lib, "failed to allocate sysv shm segment"); return UCC_ERR_NO_MEMORY; } a2a->net.net_size = a2a->bcast_data.net_size; if (a2a->node.asr_rank != node_rank) { // shmat already performed for asr above a2a->node.storage = shmat(a2a->bcast_data.shmid, NULL, 0); } if (a2a->node.storage == (void *)(-1)) { tl_error(lib, "failed to shmat seg %d", a2a->bcast_data.shmid); return UCC_ERR_NO_MEMORY; } for (i = 0; i < MAX_OUTSTANDING_OPS; i++) { op = &a2a->node.ops[i]; op->ctrl = PTR_OFFSET(a2a->node.storage, op_seg_size * i); op->my_ctrl = PTR_OFFSET( op->ctrl, node_rank * sizeof(ucc_tl_mlx5_alltoall_ctrl_t)); } if (UCC_TL_MLX5_TEAM_LIB(team)->cfg.block_size > MAX_BLOCK_SIZE) { tl_error(lib, "max block_size is %d", MAX_BLOCK_SIZE); return UCC_ERR_NO_MESSAGE; } a2a->requested_block_size = UCC_TL_MLX5_TEAM_LIB(team)->cfg.block_size; // Non-ASR ranks exit here if (a2a->node.asr_rank != node_rank) { return UCC_OK; } status = ucc_tl_mlx5_alltoall_barrier_alloc(team); if (UCC_OK != status) { goto err_barrier; } a2a->net.blocks_sent = ucc_malloc(sizeof(*a2a->net.blocks_sent) * a2a->net.net_size * MAX_OUTSTANDING_OPS, "blocks_sent"); if (!a2a->net.blocks_sent) { tl_error(lib, "failed to allocated %zd bytes for blocks_sent array", sizeof(*a2a->net.blocks_sent) * a2a->net.net_size * MAX_OUTSTANDING_OPS); status = UCC_ERR_NO_MEMORY; goto err_blocks_sent; } for (i = 0; i < MAX_OUTSTANDING_OPS; i++) { op = &a2a->node.ops[i]; op->blocks_sent = PTR_OFFSET(a2a->net.blocks_sent, sizeof(*a2a->net.blocks_sent) * a2a->net.net_size * i); } memset(a2a->previous_msg_size, 0, sizeof(a2a->previous_msg_size)); status = ucc_tl_mlx5_alltoall_init_umr(team->a2a, lib); if (status != UCC_OK) { tl_error(lib, "failed to init UMR"); goto err_umr; } net_size = a2a->net.net_size; asr_cq_size = net_size * (SQUARED(a2a->node.sbgp->group_size / 2 + 1) + 1) * MAX_OUTSTANDING_OPS; a2a->net.cq = ibv_create_cq(ctx->shared_ctx, asr_cq_size, NULL, NULL, 0); if (!a2a->net.cq) { tl_error(lib, "failed to allocate ASR CQ"); status = UCC_ERR_NO_MESSAGE; goto err_cq; } a2a->is_dc = (net_size >= UCC_TL_MLX5_TEAM_LIB(team)->cfg.dc_threshold); ibv_query_port(ctx->shared_ctx, ctx->ib_port, &port_attr); a2a->net.ctrl_mr = ibv_reg_mr( ctx->shared_pd, a2a->node.storage, op_seg_size * MAX_OUTSTANDING_OPS, IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC | IBV_ACCESS_LOCAL_WRITE); if (!a2a->net.ctrl_mr) { tl_error(lib, "failed to register control data, errno %d", errno); status = UCC_ERR_NO_MESSAGE; goto err_ctrl_mr; } a2a->net.remote_ctrl = ucc_calloc(sizeof(*a2a->net.remote_ctrl), net_size, "remote_ctrl"); if (!a2a->net.remote_ctrl) { tl_error(lib, "failed to allocate %zd bytes for remote_ctrl", sizeof(*a2a->net.remote_ctrl) * net_size); status = UCC_ERR_NO_MESSAGE; goto err_remote_ctrl; } status = ucc_tl_mlx5_init_mkeys(team, lib); if (status != UCC_OK) { tl_error(lib, "failed to init mkeys"); goto err_mkeys; } // for each ASR - qp num, in addition to port lid, // ctrl segment rkey and address, receive mkey rkey local_data_size = sizeof(net_exchange_t); if (!a2a->is_dc) { /* need more space for net_size - 1 qpns */ local_data_size += sizeof(uint32_t) * (net_size - 1); } local_data = ucc_malloc(local_data_size * (net_size + 1), "exchange_data"); if (!local_data) { tl_error(lib, "failed to allocate %zd bytes for exchange data", local_data_size * (net_size + 1)); status = UCC_ERR_NO_MEMORY; goto err_local_data; } global_data = PTR_OFFSET(local_data, local_data_size); if (a2a->is_dc) { //SRQ struct ibv_srq_init_attr srq_attr; memset(&srq_attr, 0, sizeof(struct ibv_srq_init_attr)); srq_attr.attr.max_wr = 1; srq_attr.attr.max_sge = 1; a2a->net.srq = ibv_create_srq(ctx->shared_pd, &srq_attr); if (a2a->net.srq == NULL) { tl_error(lib, "failed to create SRQ"); status = UCC_ERR_NO_MESSAGE; goto err_srq; } //DCI int tx_depth = (SQUARED(a2a->node.sbgp->group_size / 2 + 1) * 2 + 2) * MAX_OUTSTANDING_OPS * ucc_div_round_up(a2a->net.net_size, a2a->num_dci_qps); a2a->net.dcis = ucc_malloc(sizeof(ucc_tl_mlx5_dci_t) * a2a->num_dci_qps); if (!a2a->net.dcis) { tl_error(ctx->super.super.lib, "failed to allocate mem"); status = UCC_ERR_NO_MEMORY; goto err_dcis; } for (i = 0; i < a2a->num_dci_qps; i++) { status = ucc_tl_mlx5_init_dci( &a2a->net.dcis[i], ctx->shared_pd, ctx->shared_ctx, a2a->net.cq, ctx->ib_port, tx_depth, &UCC_TL_MLX5_TEAM_LIB(team)->cfg.qp_conf, lib); if (UCC_OK != status) { goto err_init_dci; } } //DCT status = ucc_tl_mlx5_init_dct( ctx->shared_pd, ctx->shared_ctx, a2a->net.cq, a2a->net.srq, ctx->ib_port, &a2a->net.dct_qp, local_data->qpn, &UCC_TL_MLX5_TEAM_LIB(team)->cfg.qp_conf, lib); if (UCC_OK != status) { goto err_init_dct; } } else { a2a->net.rc_qps = ucc_malloc(sizeof(ucc_tl_mlx5_qp_t) * a2a->net.net_size); if (!a2a->net.rc_qps) { tl_error(lib, "failed to allocate asr qps array"); status = UCC_ERR_NO_MEMORY; goto err_alloc_rc_qps; } int tx_depth = (SQUARED(a2a->node.sbgp->group_size / 2 + 1) * 2 + 2) * MAX_OUTSTANDING_OPS; for (i = 0; i < a2a->net.net_size; i++) { status = ucc_tl_mlx5_create_rc_qp( ctx->shared_ctx, ctx->shared_pd, a2a->net.cq, tx_depth, &a2a->net.rc_qps[i], &local_data->qpn[i], lib); if (UCC_OK != status) { goto err_create_rc_qps; } } } local_data->port_lid = port_attr.lid; local_data->recv_mkey_rkey = a2a->node.team_recv_mkey->rkey; local_data->rd.team_rank = UCC_TL_TEAM_RANK(team); local_data->rd.sbgp_rank = a2a->net.sbgp->group_rank; local_data->net_ctrl.atomic.addr = a2a->net.atomic.counters; local_data->net_ctrl.atomic.rkey = a2a->net.atomic.mr->rkey; local_data->net_ctrl.barrier.addr = a2a->net.barrier.flags; local_data->net_ctrl.barrier.rkey = a2a->net.barrier.mr->rkey; status = ucc_service_allgather(UCC_TL_CORE_TEAM(team), local_data, global_data, local_data_size, ucc_sbgp_to_subset(a2a->net.sbgp), &team->scoll_req); if (UCC_OK != status) { tl_error(lib, "failed start service allgather"); goto err_service_allgather_post; } team->scoll_req->data = local_data; a2a->state = TL_MLX5_ALLTOALL_STATE_EXCHANGE_PROGRESS; case TL_MLX5_ALLTOALL_STATE_EXCHANGE_PROGRESS: status = ucc_service_coll_test(team->scoll_req); if (status < 0) { tl_error(UCC_TL_TEAM_LIB(team), "failure during service coll exchange: %s", ucc_status_string(status)); ucc_service_coll_finalize(team->scoll_req); goto err_service_allgather_progress; } if (UCC_INPROGRESS == status) { return status; } ucc_assert(status == UCC_OK); a2a->state = TL_MLX5_ALLTOALL_STATE_EXCHANGE_DONE; case TL_MLX5_ALLTOALL_STATE_EXCHANGE_DONE: local_data = team->scoll_req->data; ucc_service_coll_finalize(team->scoll_req); net_size = a2a->net.net_size; local_data_size = sizeof(net_exchange_t); if (!a2a->is_dc) { /* need more space for net_size - 1 qpns */ local_data_size += sizeof(uint32_t) * (net_size - 1); } global_data = PTR_OFFSET(local_data, local_data_size); status = build_rank_map(a2a, global_data, local_data_size); if (status != UCC_OK) { tl_error(lib, "failed to build rank map"); goto err_rank_map; } a2a->net.rkeys = ucc_malloc(sizeof(uint32_t) * net_size); if (!a2a->net.rkeys) { tl_error(lib, "failed to allocate %zd bytes for net rkeys", sizeof(uint32_t) * net_size); status = UCC_ERR_NO_MEMORY; goto err_rkeys; } if (a2a->is_dc) { a2a->net.remote_dctns = ucc_malloc(sizeof(uint32_t) * net_size); if (!a2a->net.remote_dctns) { tl_error(lib, "failed to allocate %zd bytes for remote_dctns", sizeof(uint32_t) * net_size); status = UCC_ERR_NO_MEMORY; goto err_remote_dctns; } a2a->net.ahs = ucc_malloc(sizeof(struct ibv_ah *) * net_size); if (!a2a->net.ahs) { tl_error(lib, "failed to allocate %zd bytes for net ahs", sizeof(struct ibv_ah *) * net_size); status = UCC_ERR_NO_MEMORY; goto err_ahs; } } for (i = 0; i < net_size; i++) { remote_data = PTR_OFFSET(global_data, i * local_data_size); if (a2a->is_dc) { a2a->net.remote_dctns[i] = remote_data->qpn[0]; status = ucc_tl_mlx5_create_ah(ctx->shared_pd, remote_data->port_lid, ctx->ib_port, &a2a->net.ahs[i], lib); if (UCC_OK != status) { tl_error(lib, "failed to create ah, %s", ucc_status_string(status)); goto err_create_ah; } } else { status = ucc_tl_mlx5_qp_connect( a2a->net.rc_qps[i].qp, remote_data->qpn[a2a->net.sbgp->group_rank], remote_data->port_lid, ctx->ib_port, &UCC_TL_MLX5_TEAM_LIB(team)->cfg.qp_conf, lib); if (UCC_OK != status) { tl_error(lib, "failed to connect rc qps, %s", ucc_status_string(status)); goto err_qp_connect; } } a2a->net.remote_ctrl[i] = remote_data->net_ctrl; a2a->net.rkeys[i] = remote_data->recv_mkey_rkey; } a2a->atomic_scratch_bf_mr = ibv_reg_mr(ctx->shared_pd, (void *)&a2a->atomic_scratch_bf, sizeof(a2a->atomic_scratch_bf), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); if (!a2a->atomic_scratch_bf_mr) { tl_error(lib, "failed to register atomic scratch buff (errno=%d)", errno); status = UCC_ERR_NO_MESSAGE; goto err_atomic_atomic_scratch_bf_mr; } /* allocate buffer for noninline UMR registration, has to be 2KB aligned */ umr_buf_size = ucc_align_up( sizeof(struct mlx5_wqe_umr_repeat_ent_seg) * (node_size + 1), 64); ret = ucc_posix_memalign(&a2a->node.umr_entries_buf, 2048, umr_buf_size); if (ret) { tl_error(lib, "failed to allocate %zd bytes for noninline UMR buffer", umr_buf_size); return UCC_ERR_NO_MEMORY; } a2a->node.umr_entries_mr = ibv_reg_mr( ctx->shared_pd, (void *)a2a->node.umr_entries_buf, umr_buf_size, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); if (!a2a->node.umr_entries_mr) { tl_error(lib, "failed to register umr buff (errno=%d)", errno); status = UCC_ERR_NO_MESSAGE; goto err_umr_entries_mr; } break; } return UCC_OK; err_umr_entries_mr: ibv_dereg_mr(a2a->atomic_scratch_bf_mr); err_atomic_atomic_scratch_bf_mr: if (a2a->is_dc) { err_create_ah: for (j = 0; j < i ; j++) { ibv_destroy_ah(a2a->net.ahs[j]); } ucc_free(a2a->net.ahs); err_ahs: ucc_free(a2a->net.remote_dctns); } err_qp_connect: err_remote_dctns: ucc_free(a2a->net.rkeys); err_rkeys: ucc_free(a2a->net.rank_map); err_rank_map: err_service_allgather_progress: err_service_allgather_post: if (!a2a->is_dc) { err_create_rc_qps: for (j = 0; j < i ; j++) { ibv_destroy_qp(a2a->net.rc_qps[j].qp); } ucc_free(a2a->net.rc_qps); } else { ibv_destroy_qp(a2a->net.dct_qp); err_init_dct: err_init_dci: for (j = 0; j < i; j++) { ibv_destroy_qp(a2a->net.dcis[j].dci_qp); } ucc_free(a2a->net.dcis); err_dcis: ibv_destroy_srq(a2a->net.srq); } err_alloc_rc_qps: err_srq: if (local_data) { ucc_free(local_data); } err_local_data: ucc_tl_mlx5_destroy_mkeys(a2a, 0, lib); err_mkeys: ucc_free(a2a->net.remote_ctrl); err_remote_ctrl: ibv_dereg_mr(a2a->net.ctrl_mr); err_ctrl_mr: ibv_destroy_cq(a2a->net.cq); err_cq: ucc_tl_mlx5_destroy_umr(a2a, lib); err_umr: ucc_free(a2a->net.blocks_sent); err_blocks_sent: ucc_tl_mlx5_alltoall_barrier_free(a2a); err_barrier: return status; } void ucc_tl_mlx5_alltoall_cleanup(ucc_tl_mlx5_team_t *team) { ucc_tl_mlx5_alltoall_t *a2a = team->a2a; ucc_base_lib_t * lib = UCC_TL_TEAM_LIB(team); ucc_status_t status; int i; if (!a2a) { return; } if (-1 == shmdt(a2a->node.storage)) { tl_error(lib, "failed to shmdt %p, errno %d", a2a->node.storage, errno); } if (a2a->node.asr_rank == a2a->node.sbgp->group_rank) { status = ucc_tl_mlx5_destroy_umr(a2a, lib); if (status != UCC_OK) { tl_error(lib, "failed to destroy UMR"); } ibv_dereg_mr(a2a->net.ctrl_mr); ucc_free(a2a->net.remote_ctrl); if (a2a->is_dc) { for (i = 0; i < a2a->num_dci_qps; i++) { ibv_destroy_qp(a2a->net.dcis[i].dci_qp); } ucc_free(a2a->net.dcis); ibv_destroy_qp(a2a->net.dct_qp); ibv_destroy_srq(a2a->net.srq); for (i = 0; i < a2a->net.net_size; i++) { ibv_destroy_ah(a2a->net.ahs[i]); } } else { for (i = 0; i < a2a->net.net_size; i++) { ibv_destroy_qp(a2a->net.rc_qps[i].qp); } } if (a2a->is_dc) { ucc_free(a2a->net.remote_dctns); ucc_free(a2a->net.ahs); } else { ucc_free(a2a->net.rc_qps); } if (ibv_destroy_cq(a2a->net.cq)) { tl_error(lib, "net cq destroy failed (errno=%d)", errno); } status = ucc_tl_mlx5_destroy_mkeys(a2a, 0, lib); if (status != UCC_OK) { tl_error(lib, "failed to destroy Mkeys"); } ucc_free(a2a->net.rkeys); ibv_dereg_mr(a2a->atomic_scratch_bf_mr); ucc_free(a2a->net.rank_map); ibv_dereg_mr(a2a->node.umr_entries_mr); ucc_free(a2a->node.umr_entries_buf); ucc_free(a2a->net.blocks_sent); ucc_tl_mlx5_alltoall_barrier_free(a2a); } ucc_free(a2a); } ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_rcache.c0000664000175000017500000000553515211535620022003 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5.h" static ucs_status_t rcache_reg_mr(void *context, ucc_rcache_t *rcache, //NOLINT: rcache is unused void *arg, ucc_rcache_region_t *rregion, uint16_t flags) //NOLINT: flags is unused { ucc_tl_mlx5_context_t *ctx = (ucc_tl_mlx5_context_t *)context; void *addr = (void *)rregion->super.start; size_t length = (size_t)(rregion->super.end - rregion->super.start); int *change_flag = (int *)arg; ucc_tl_mlx5_rcache_region_t *mlx5_rregion = ucc_derived_of(rregion, ucc_tl_mlx5_rcache_region_t); *change_flag = 1; mlx5_rregion->reg.mr = ibv_reg_mr(ctx->shared_pd, addr, length, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); if (!mlx5_rregion->reg.mr) { tl_error(ctx->super.super.lib, "failed to register memory"); return UCS_ERR_NO_MESSAGE; } return UCS_OK; } static void rcache_dereg_mr(void *context, //NOLINT: context is unused ucc_rcache_t *rcache, //NOLINT: rcache is unused ucc_rcache_region_t *rregion) { ucc_tl_mlx5_rcache_region_t *mlx5_rregion = ucc_derived_of(rregion, ucc_tl_mlx5_rcache_region_t); ibv_dereg_mr(mlx5_rregion->reg.mr); } static void ucc_tl_mlx5_rcache_dump_region_cb(void *context, //NOLINT ucc_rcache_t *rcache, //NOLINT ucs_rcache_region_t *rregion, char *buf, size_t max) { ucc_tl_mlx5_rcache_region_t *mlx5_rregion = ucc_derived_of(rregion, ucc_tl_mlx5_rcache_region_t); snprintf(buf, max, "bar ptr:%p", mlx5_rregion->reg.mr); } static ucc_rcache_ops_t ucc_tl_mlx5_rcache_ops = { .mem_reg = rcache_reg_mr, .mem_dereg = rcache_dereg_mr, .dump_region = ucc_tl_mlx5_rcache_dump_region_cb, #ifdef UCS_HAVE_RCACHE_MERGE_CB .merge = ucc_rcache_merge_cb_empty #endif }; ucc_status_t tl_mlx5_rcache_create(ucc_tl_mlx5_context_t *ctx) { ucc_rcache_params_t rcache_params; ucc_rcache_set_default_params(&rcache_params); rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_rcache_region_t); rcache_params.context = ctx; rcache_params.ops = &ucc_tl_mlx5_rcache_ops; rcache_params.ucm_events = UCM_EVENT_VM_UNMAPPED | UCM_EVENT_MEM_TYPE_FREE; return ucc_rcache_create(&rcache_params, "MLX5_A2A", &ctx->rcache); } ucc-1.8.0/src/components/tl/mlx5/Makefile.am0000664000175000017500000000427515211535620021002 0ustar alastairalastair# # Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # if TL_MLX5_ENABLED alltoall = \ alltoall/alltoall.h \ alltoall/alltoall.c \ alltoall/alltoall_mkeys.h \ alltoall/alltoall_mkeys.c \ alltoall/alltoall_inline.h \ alltoall/alltoall_coll.c mcast = \ mcast/tl_mlx5_mcast_context.c \ mcast/tl_mlx5_mcast.h \ mcast/tl_mlx5_mcast_coll.c \ mcast/tl_mlx5_mcast_coll.h \ mcast/tl_mlx5_mcast_rcache.h \ mcast/tl_mlx5_mcast_rcache.c \ mcast/p2p/ucc_tl_mlx5_mcast_p2p.h \ mcast/p2p/ucc_tl_mlx5_mcast_p2p.c \ mcast/tl_mlx5_mcast_progress.h \ mcast/tl_mlx5_mcast_progress.c \ mcast/tl_mlx5_mcast_helper.h \ mcast/tl_mlx5_mcast_helper.c \ mcast/tl_mlx5_mcast_service_coll.h \ mcast/tl_mlx5_mcast_service_coll.c \ mcast/tl_mlx5_mcast_one_sided_reliability.h \ mcast/tl_mlx5_mcast_one_sided_reliability.c \ mcast/tl_mlx5_mcast_one_sided_progress.h \ mcast/tl_mlx5_mcast_one_sided_progress.c \ mcast/tl_mlx5_mcast_allgather.h \ mcast/tl_mlx5_mcast_allgather.c \ mcast/tl_mlx5_mcast_hca_copy.h \ mcast/tl_mlx5_mcast_hca_copy.c \ mcast/tl_mlx5_mcast_team.c sources = \ tl_mlx5.h \ tl_mlx5.c \ tl_mlx5_lib.c \ tl_mlx5_context.c \ tl_mlx5_team.c \ tl_mlx5_coll.h \ tl_mlx5_coll.c \ tl_mlx5_ib.h \ tl_mlx5_ib.c \ tl_mlx5_wqe.h \ tl_mlx5_wqe.c \ tl_mlx5_pd.h \ tl_mlx5_pd.c \ tl_mlx5_rcache.c \ tl_mlx5_dm.c \ tl_mlx5_dm.h \ $(mcast) \ $(alltoall) module_LTLIBRARIES = libucc_tl_mlx5.la libucc_tl_mlx5_la_SOURCES = $(sources) libucc_tl_mlx5_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) libucc_tl_mlx5_la_CFLAGS = $(BASE_CFLAGS) libucc_tl_mlx5_la_LDFLAGS = -version-info $(SOVERSION) --as-needed libucc_tl_mlx5_la_LIBADD = $(UCC_TOP_BUILDDIR)/src/libucc.la $(IBVERBS_LIBADD) $(MLX5DV_LIBADD) $(RDMACM_LIBADD) include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_dm.h0000664000175000017500000000114515211535620021154 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5.h" ucc_status_t ucc_tl_mlx5_dm_alloc_reg(struct ibv_context *ib_ctx, struct ibv_pd *pd, int dm_host, size_t buf_size, size_t *buf_num_p, struct ibv_dm **ptr, struct ibv_mr **mr, ucc_base_lib_t *lib); void ucc_tl_mlx5_dm_cleanup(ucc_tl_mlx5_team_t *team); ucc_status_t ucc_tl_mlx5_dm_init(ucc_tl_mlx5_team_t *team); ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_context.c0000664000175000017500000003142415211535620022236 0ustar alastairalastair/** * Copyright (c) 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5.h" #include "utils/ucc_math.h" #include "schedule/ucc_schedule.h" #include #include "tl_mlx5_coll.h" #include "utils/arch/cpu.h" #include "tl_mlx5_pd.h" #include "tl_mlx5_ib.h" #define PD_OWNER_RANK 0 #define TL_MLX5_IB_PORT_INVALID -1 static ucc_status_t ucc_tl_mlx5_check_gpudirect_driver(ucc_base_lib_t *lib, const char *file) { ucc_status_t status = UCC_ERR_NO_RESOURCE; if (!access(file, F_OK)) { status = UCC_OK; } tl_debug(lib, "checking gpudirect driver: %s, status: %d %s", file, status, ucc_status_string(status)); return status; } static ucc_status_t ucc_tl_mlx5_check_gpudirect_driver_cuda(ucc_base_lib_t *lib) { /* Check peer memory driver is loaded, different driver versions use * different paths */ if (UCC_OK == ucc_tl_mlx5_check_gpudirect_driver(lib, "/sys/kernel/mm/memory_peers/nv_mem/version")) { return UCC_OK; } else if (UCC_OK == ucc_tl_mlx5_check_gpudirect_driver(lib, "/sys/module/nvidia_peermem/version")) { return UCC_OK; } else if (UCC_OK == ucc_tl_mlx5_check_gpudirect_driver(lib, "/sys/module/nv_peer_mem/version")) { return UCC_OK; } tl_debug(lib, "no gpudirect driver found, cuda memory is not supported"); return UCC_ERR_NOT_SUPPORTED; } UCC_CLASS_INIT_FUNC(ucc_tl_mlx5_context_t, const ucc_base_context_params_t *params, const ucc_base_config_t * config) { ucc_tl_mlx5_context_config_t *tl_mlx5_config = ucc_derived_of(config, ucc_tl_mlx5_context_config_t); ucc_status_t status; UCC_CLASS_CALL_SUPER_INIT(ucc_tl_context_t, &tl_mlx5_config->super, params->context); memcpy(&self->cfg, tl_mlx5_config, sizeof(*tl_mlx5_config)); self->sock = 0; self->rcache = NULL; self->shared_pd = NULL; self->shared_ctx = NULL; self->supported_mem_types = UCC_BIT(UCC_MEMORY_TYPE_HOST); if (UCC_OK == ucc_tl_mlx5_check_gpudirect_driver_cuda(self->super.super.lib)) { self->supported_mem_types |= UCC_BIT(UCC_MEMORY_TYPE_CUDA); } status = ucc_mpool_init( &self->req_mp, 0, ucc_max(sizeof(ucc_tl_mlx5_task_t), sizeof(ucc_tl_mlx5_schedule_t)), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_coll_task_mpool_ops, params->thread_mode, "tl_mlx5_req_mp"); if (UCC_OK != status) { tl_debug(self->super.super.lib, "failed to initialize tl_mlx5_req mpool"); return status; } if (!self->cfg.enable_alltoall) { tl_debug(self->super.super.lib, "alltoall is disabled by the env variable " "`UCC_TL_MLX5_ALLTOALL_ENABLE`"); } self->mcast.mcast_ctx_ready = 0; if (params->thread_mode == UCC_THREAD_SINGLE) { status = ucc_tl_mlx5_mcast_context_init(&(self->mcast), &(self->cfg.mcast_ctx_conf)); if (UCC_OK != status) { tl_debug(self->super.super.lib, "failed to initialize mcast context"); } else { self->mcast.mcast_ctx_ready = 1; } } return UCC_OK; } UCC_CLASS_CLEANUP_FUNC(ucc_tl_mlx5_context_t) { tl_debug(self->super.super.lib, "finalizing tl context: %p", self); if (self->rcache) { ucc_rcache_destroy(self->rcache); } if (UCC_OK != ucc_tl_mlx5_remove_shared_ctx_pd(self)) { tl_debug(self->super.super.lib, "failed to free ib ctx and pd"); }; if (self->sock) { close(self->sock); } ucc_mpool_cleanup(&self->req_mp, 1); if (self->mcast.mcast_ctx_ready) { ucc_tl_mlx5_mcast_clean_ctx(&self->mcast.mcast_context); } } UCC_CLASS_DEFINE(ucc_tl_mlx5_context_t, ucc_tl_context_t); ucc_status_t ucc_tl_mlx5_get_context_attr(const ucc_base_context_t *context, /* NOLINT */ ucc_base_ctx_attr_t *attr) { ucc_base_ctx_attr_clear(attr); attr->topo_required = 1; return UCC_OK; } ucc_status_t ucc_tl_mlx5_ib_ctx_pd_init(ucc_tl_mlx5_context_t *ctx) { int port = -1; char * ib_devname = NULL; int devname_len; char tmp[128], *pos, *end_pos; ucc_status_t status; if (ctx->cfg.devices.count > 0) { ib_devname = ctx->cfg.devices.names[0]; pos = strstr(ib_devname, ":"); end_pos = ib_devname + strlen(ib_devname); if (!pos) { devname_len = sizeof(tmp) - 1; } else { devname_len = (int)(pos - ib_devname); pos++; errno = 0; port = (int)strtol(pos, &end_pos, 10); if (errno != 0 || pos == end_pos) { tl_debug(ctx->super.super.lib, "wrong device's port number"); return UCC_ERR_INVALID_PARAM; } } strncpy(tmp, ib_devname, devname_len); tmp[devname_len] = '\0'; ib_devname = tmp; } status = ucc_tl_mlx5_create_ibv_ctx(&ib_devname, &ctx->shared_ctx, ctx->super.super.lib); if (UCC_OK != status) { tl_debug(ctx->super.super.lib, "failed to allocate ibv_context status %d", status); return status; } if (port == -1) { port = ucc_tl_mlx5_get_active_port(ctx->shared_ctx); } ctx->ib_port = port; if (-1 == port || !ucc_tl_mlx5_check_port_active(ctx->shared_ctx, port)) { tl_debug(ctx->super.super.lib, "no active ports found on %s", ib_devname); goto destroy_context; } tl_debug(ctx->super.super.lib, "using %s:%d", ib_devname, port); ctx->shared_pd = ibv_alloc_pd(ctx->shared_ctx); if (!ctx->shared_pd) { tl_debug(ctx->super.super.lib, "failed to allocate ib_pd"); goto destroy_context; } return UCC_OK; destroy_context: ibv_close_device(ctx->shared_ctx); return UCC_ERR_NO_RESOURCE; } typedef struct ucc_tl_mlx5_context_create_sbcast_data { int ib_port; char sock_path[]; } ucc_tl_mlx5_context_create_sbcast_data_t; ucc_status_t ucc_tl_mlx5_context_ib_ctx_pd_setup(ucc_base_context_t *context) { ucc_tl_mlx5_context_t *ctx = ucc_derived_of(context, ucc_tl_mlx5_context_t); ucc_context_t * core_ctx = context->ucc_context; const char * template = "/tmp/ucc.mlx5.XXXXXX"; const char * sockname = "/sock"; size_t sock_dir_len = strlen(template) + 1; size_t sock_path_len = sock_dir_len + strlen(sockname); size_t sbcast_data_length = sizeof(int) + sock_path_len; ucc_topo_t * topo = NULL; ucc_tl_team_t * steam = core_ctx->service_team; char sock_path[sock_path_len]; ucc_subset_t s; ucc_status_t status, global_status, local_status; ucc_sbgp_t * sbgp; ucc_coll_task_t *req; ucc_tl_mlx5_context_create_sbcast_data_t *sbcast_data; if (!ctx->cfg.enable_alltoall) { return UCC_OK; } if (!core_ctx->service_team) { tl_debug(context->lib, "failed to init ctx: need service team"); return UCC_ERR_NO_MESSAGE; } ucc_assert(core_ctx->params.mask & UCC_CONTEXT_PARAM_FIELD_OOB); sbcast_data = (ucc_tl_mlx5_context_create_sbcast_data_t *)ucc_malloc( sbcast_data_length); if (!sbcast_data) { tl_debug(context->lib, "failed to allocate buffer for sharing ib_ctx info"); return UCC_ERR_NO_MEMORY; } memset(&s.map, 0, sizeof(ucc_ep_map_t)); s.map.type = UCC_EP_MAP_FULL; s.map.ep_num = core_ctx->params.oob.n_oob_eps; s.myrank = core_ctx->rank; status = tl_mlx5_rcache_create(ctx); if (UCC_OK != status) { tl_debug(context->lib, "failed to create rcache status %d", status); goto start_allreduce; } status = ucc_topo_init(s, core_ctx->topo, &topo); if (UCC_OK != status) { tl_debug(context->lib, "failed to init mlx5 ctx topo %d", status); goto start_allreduce; } start_allreduce: local_status = status; status = UCC_TL_TEAM_IFACE(steam)->scoll.allreduce( &steam->super, &local_status, &global_status, UCC_DT_INT32, 1, UCC_OP_LOR, s, &req); if (UCC_OK != status) { tl_debug(context->lib, "failed to start mlx5 ctx allreduce"); goto err_global_status; } while (UCC_INPROGRESS == (status = ucc_collective_test(&req->super))) { ucc_context_progress(core_ctx); } ucc_collective_finalize_internal(req); if (UCC_OK != status) { tl_debug(context->lib, "failure during mlx5 ctx allreduce"); goto err_global_status; } if (global_status != UCC_OK || local_status != UCC_OK) { tl_debug(context->lib, "context creation failed - error detected in one " "or more ranks"); status = global_status; goto err_global_status; } sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); ctx->shared_ctx = NULL; ctx->shared_pd = NULL; ctx->is_imported = sbgp->group_rank != PD_OWNER_RANK; if (!ctx->is_imported) { status = ucc_tl_mlx5_ib_ctx_pd_init(ctx); if (UCC_OK != status) { ctx->ib_port = TL_MLX5_IB_PORT_INVALID; goto start_bcast; } if (UCC_SBGP_NOT_EXISTS == sbgp->status) { goto topo_ppn_1; } ucc_strncpy_safe(sock_path, template, sock_dir_len); if (mkdtemp(sock_path) != NULL) { strncat(sock_path, sockname, sizeof(sock_path) - strlen(sock_path) - 1); status = ucc_tl_mlx5_socket_init(ctx, sbgp->group_size, &ctx->sock, sock_path); if (UCC_OK != status) { sock_path[0] = '\0'; tl_debug(context->lib, "failed to init socket to share ib_ctx"); } } else { tl_debug(context->lib, "failed to create tmp file for socket path"); sock_path[0] = '\0'; } memcpy(sbcast_data->sock_path, sock_path, sizeof(sock_path)); } start_bcast: sbcast_data->ib_port = ctx->ib_port; s.map = sbgp->map; s.myrank = sbgp->group_rank; status = UCC_TL_TEAM_IFACE(steam)->scoll.bcast( &steam->super, sbcast_data, sbcast_data_length, PD_OWNER_RANK, s, &req); if (UCC_OK != status) { tl_debug(context->lib, "failed to start mlx5 ctx bcast"); goto err; } while (UCC_INPROGRESS == (status = ucc_collective_test(&req->super))) { ucc_context_progress(core_ctx); } ucc_collective_finalize_internal(req); if (UCC_OK != status) { tl_debug(context->lib, "failure during mlx5 ctx bcast"); goto err; } ctx->ib_port = sbcast_data->ib_port; memcpy(sock_path, sbcast_data->sock_path, sizeof(sock_path)); if (ctx->ib_port == TL_MLX5_IB_PORT_INVALID) { tl_debug(context->lib, "invalid ib port received"); status = UCC_ERR_NO_RESOURCE; goto err_ib_ctx_pd_init; } if (strlen(sock_path) == 0) { tl_debug(context->lib, "failed to share ctx and pd"); status = UCC_ERR_NO_RESOURCE; goto err; } status = ucc_tl_mlx5_share_ctx_pd(ctx, sock_path, sbgp->group_size, !ctx->is_imported, ctx->sock); if (UCC_OK != status) { goto err; } rmdir(sock_path); topo_ppn_1: ucc_free(sbcast_data); ucc_topo_cleanup(topo); tl_debug(ctx->super.super.lib, "initialized tl context: %p", ctx); return UCC_OK; err: ucc_tl_mlx5_remove_shared_ctx_pd(ctx); rmdir(sock_path); close(ctx->sock); err_ib_ctx_pd_init: err_global_status: if (topo) { ucc_topo_cleanup(topo); } if (ctx->rcache) { ucc_rcache_destroy(ctx->rcache); ctx->rcache = NULL; } ucc_free(sbcast_data); tl_debug(ctx->super.super.lib, "failed initialize tl context: %p", ctx); return status; } ucc_status_t ucc_tl_mlx5_context_create_epilog(ucc_base_context_t *context) { return ucc_tl_mlx5_context_ib_ctx_pd_setup(context); } ucc_status_t ucc_tl_mlx5_mem_map(const ucc_base_context_t *context, int type, /* NOLINT */ void *memh, void *tl_h) /* NOLINT */ { return UCC_ERR_NOT_IMPLEMENTED; } ucc_status_t ucc_tl_mlx5_mem_unmap(const ucc_base_context_t *context, int type, /* NOLINT */ void *memh) /* NOLINT */ { return UCC_ERR_NOT_IMPLEMENTED; } ucc_status_t ucc_tl_mlx5_memh_pack(const ucc_base_context_t *context, /* NOLINT */ int type, void *memh, void **pack_buffer) /* NOLINT */ { return UCC_ERR_NOT_IMPLEMENTED; } ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_wqe.c0000664000175000017500000002716015211535620021350 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "tl_mlx5_wqe.h" #include "tl_mlx5_ib.h" #include "utils/arch/cpu.h" #include "utils/ucc_math.h" #define SQ_WQE_SHIFT 6 #define DS_SIZE 16 // size of a single Data Segment in the WQE /* UMR pointer to KLMs/MTTs/RepeatBlock and BSFs location (when inline = 0) */ struct mlx5_wqe_umr_pointer_seg { __be32 reserved; __be32 mkey; __be64 address; }; static inline uint8_t get_umr_mr_flags(uint32_t acc) { return ((acc & IBV_ACCESS_REMOTE_ATOMIC ? MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_ATOMIC : 0) | (acc & IBV_ACCESS_REMOTE_WRITE ? MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_REMOTE_WRITE : 0) | (acc & IBV_ACCESS_REMOTE_READ ? MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_REMOTE_READ : 0) | (acc & IBV_ACCESS_LOCAL_WRITE ? MLX5_WQE_MKEY_CONTEXT_ACCESS_FLAGS_LOCAL_WRITE : 0)); } #define MLX5_OPCODE_LOCAL_MMO 0x32 typedef struct transpose_seg { __be32 element_size; /* 8 bit value */ //From PRM we should have the rows first and then the colls. This is probably a naming error __be16 num_cols; /* 7 bit value */ __be16 num_rows; /* 7 bit value */ __be64 padding; } transpose_seg_t; /* non-inline UMR registration */ ucc_status_t ucc_tl_mlx5_post_transpose(struct ibv_qp *qp, uint32_t src_mr_lkey, uint32_t dst_mr_key, uintptr_t src_mkey_addr, uintptr_t dst_addr, uint32_t element_size, uint16_t ncols, uint16_t nrows, int send_flags) { uint32_t opcode = MLX5_OPCODE_LOCAL_MMO; uint32_t opmode = 0x0; //TRanspose uint32_t n_ds = 4; struct ibv_qp_ex * qp_ex = ibv_qp_to_qp_ex(qp); struct mlx5dv_qp_ex * mqp = mlx5dv_qp_ex_from_ibv_qp_ex(qp_ex); int fm_ce_se = 0; char wqe_desc[n_ds * DS_SIZE]; struct mlx5_wqe_ctrl_seg *ctrl; struct mlx5_wqe_data_seg *data; transpose_seg_t * tseg; if (send_flags & IBV_SEND_SIGNALED) { qp_ex->wr_id = 0; fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE; } memset(wqe_desc, 0, n_ds * DS_SIZE); /* SET CTRL SEG */ ctrl = (void *)wqe_desc; mlx5dv_set_ctrl_seg(ctrl, /* pi */ 0x0, opcode, opmode, qp->qp_num, fm_ce_se, n_ds, 0x0, 0x0); /* SET TRANSPOSE SEG */ tseg = PTR_OFFSET(ctrl, DS_SIZE); tseg->element_size = htobe32(element_size); tseg->num_rows = htobe16(nrows); tseg->num_cols = htobe16(ncols); /* SET SRC DATA SEG */ data = PTR_OFFSET(tseg, DS_SIZE); mlx5dv_set_data_seg(data, ncols * nrows * element_size, src_mr_lkey, src_mkey_addr); /* SET DST DATA SEG */ data = PTR_OFFSET(data, DS_SIZE); mlx5dv_set_data_seg(data, ncols * nrows * element_size, dst_mr_key, dst_addr); mlx5dv_wr_raw_wqe(mqp, wqe_desc); return UCC_OK; } /* The strided block format is as the following: * | repeat_block | entry_block | entry_block |...| entry_block | * While the repeat entry contains details on the list of the block_entries. */ static void umr_pointer_seg_init(uint32_t repeat_count, uint16_t num_interleaved, struct mlx5dv_mr_interleaved *data, struct mlx5_wqe_umr_pointer_seg *pseg, uint32_t ptr_mkey, void *ptr_address, int *xlat_size, uint64_t *reglen) { uint64_t byte_count = 0; struct mlx5_wqe_umr_repeat_block_seg *rb; struct mlx5_wqe_umr_repeat_ent_seg * eb; int i; /* set pointer segment */ pseg->mkey = htobe32(ptr_mkey); pseg->address = htobe64((uint64_t)ptr_address); /* set actual repeated and entry blocks segments */ rb = ptr_address; rb->op = htobe32(0x400); // PRM header entry - repeated blocks rb->reserved = 0; rb->num_ent = htobe16(num_interleaved); rb->repeat_count = htobe32(repeat_count); eb = rb->entries; /* * ------------------------------------------------------------ * | repeat_block | entry_block | entry_block |...| entry_block * ------------------------------------------------------------ */ for (i = 0; i < num_interleaved; i++, eb++) { byte_count += data[i].bytes_count; eb->va = htobe64(data[i].addr); eb->byte_count = htobe16(data[i].bytes_count); eb->stride = htobe16(data[i].bytes_count + data[i].bytes_skip); eb->memkey = htobe32(data[i].lkey); } rb->byte_count = htobe32(byte_count); *reglen = byte_count * repeat_count; *xlat_size = (num_interleaved + 1) * sizeof(*eb); } ucc_status_t ucc_tl_mlx5_post_umr(struct ibv_qp * qp, struct mlx5dv_mkey *dv_mkey, uint32_t access_flags, uint32_t repeat_count, uint16_t num_entries, struct mlx5dv_mr_interleaved *data, uint32_t ptr_mkey, void *ptr_address) { uint32_t opcode = MLX5_OPCODE_UMR; uint64_t reglen = 0; uint32_t opmode = 0x0; uint32_t n_ds = (sizeof(struct mlx5_wqe_ctrl_seg) + sizeof(struct mlx5_wqe_umr_ctrl_seg) + sizeof(struct mlx5_wqe_mkey_context_seg) + sizeof(struct mlx5_wqe_umr_pointer_seg)) / DS_SIZE; uint8_t fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; struct ibv_qp_ex *qp_ex = ibv_qp_to_qp_ex(qp); struct mlx5dv_qp_ex *mqp = mlx5dv_qp_ex_from_ibv_qp_ex(qp_ex); struct mlx5_wqe_ctrl_seg *ctrl; struct mlx5_wqe_umr_ctrl_seg *umr_ctrl_seg; struct mlx5_wqe_mkey_context_seg *mk_seg; struct mlx5_wqe_umr_pointer_seg *pseg; char wqe_desc[n_ds * DS_SIZE]; int xlat_size; memset(wqe_desc, 0, n_ds * DS_SIZE); ctrl = (void *)wqe_desc; mlx5dv_set_ctrl_seg(ctrl, /* pi */ 0x0, opcode, opmode, qp->qp_num, fm_ce_se, n_ds, 0x0, htobe32(dv_mkey->lkey)); umr_ctrl_seg = PTR_OFFSET(ctrl, sizeof(*ctrl)); umr_ctrl_seg->mkey_mask = htobe64(MLX5_WQE_UMR_CTRL_MKEY_MASK_LEN | MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_LOCAL_WRITE | MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_REMOTE_READ | MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_REMOTE_WRITE | MLX5_WQE_UMR_CTRL_MKEY_MASK_ACCESS_ATOMIC | MLX5_WQE_UMR_CTRL_MKEY_MASK_FREE); mk_seg = PTR_OFFSET(umr_ctrl_seg, sizeof(*umr_ctrl_seg)); mk_seg->access_flags = get_umr_mr_flags(access_flags); mk_seg->qpn_mkey = htobe32(0xffffff00 | (dv_mkey->lkey & 0xff)); pseg = PTR_OFFSET(mk_seg, sizeof(*mk_seg)); umr_pointer_seg_init(repeat_count, num_entries, data, pseg, ptr_mkey, ptr_address, &xlat_size, ®len); mk_seg->len = htobe64(reglen); umr_ctrl_seg->klm_octowords = htobe16(ucc_align_up(xlat_size, 64) / DS_SIZE); ibv_wr_start(qp_ex); mlx5dv_wr_raw_wqe(mqp, wqe_desc); ibv_wr_complete(qp_ex); return UCC_OK; } ucc_status_t ucc_tl_mlx5_post_rdma(struct ibv_qp *qp, uint32_t qpn, struct ibv_ah *ah, uintptr_t src_mkey_addr, size_t len, uint32_t src_mr_lkey, uintptr_t dst_addr, uint32_t dst_mr_key, int send_flags, uint64_t wr_id) { uint32_t opcode = MLX5_OPCODE_RDMA_WRITE; uint32_t opmode = 0x0; struct ibv_qp_ex * qp_ex = ibv_qp_to_qp_ex(qp); struct mlx5dv_qp_ex * mqp = mlx5dv_qp_ex_from_ibv_qp_ex(qp_ex); uint8_t fm_ce_se = MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE; uint32_t n_ds = (sizeof(struct mlx5_wqe_ctrl_seg) + (ah ? sizeof(struct mlx5_wqe_datagram_seg) : 0) + sizeof(struct mlx5_wqe_raddr_seg) + sizeof(struct mlx5_wqe_data_seg)) / DS_SIZE; struct mlx5_wqe_ctrl_seg *ctrl; struct mlx5_wqe_data_seg *data; struct mlx5_wqe_datagram_seg *dseg; struct mlx5_wqe_raddr_seg * rseg; char wqe_desc[n_ds * DS_SIZE]; qp_ex->wr_id = wr_id; memset(wqe_desc, 0, n_ds * DS_SIZE); /* SET CTRL SEG */ ctrl = (void *)wqe_desc; if (send_flags & IBV_SEND_SIGNALED) { fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE; } mlx5dv_set_ctrl_seg(ctrl, /* pi */ 0x0, opcode, opmode, qp->qp_num, fm_ce_se, n_ds, 0x0, 0x0); if (ah) { dseg = PTR_OFFSET(ctrl, sizeof(*ctrl)); tl_mlx5_ah_to_av(ah, &dseg->av); dseg->av.dqp_dct |= htobe32(qpn | MLX5_EXTENDED_UD_AV); dseg->av.key.dc_key = htobe64(DC_KEY); rseg = PTR_OFFSET(dseg, sizeof(*dseg)); } else { rseg = PTR_OFFSET(ctrl, sizeof(*ctrl)); } rseg->raddr = htobe64(dst_addr); rseg->rkey = htobe32(dst_mr_key); /* SET SRC DATA SEG */ data = PTR_OFFSET(rseg, sizeof(*rseg)); mlx5dv_set_data_seg(data, len, src_mr_lkey, src_mkey_addr); mlx5dv_wr_raw_wqe(mqp, wqe_desc); return UCC_OK; } #define ACTION_RETRY 0x0ULL #define ACTION_SEND_ERR_CQE 0x1ULL #define ACTION ACTION_RETRY #define MLX5_OPCODE_WAIT 0xF typedef struct wait_on_data_seg { __be32 op; /* 4 bits op + 1 inv */ __be32 lkey; __be64 va_fail; // 3 bits action on fail + 61 bits of va into lkey __be64 data; // value to wait __be64 data_mask; // value to wait } wait_on_data_seg_t; ucc_status_t ucc_tl_mlx5_post_wait_on_data(struct ibv_qp *qp, uint64_t value, uint32_t lkey, uintptr_t addr, void *task_ptr) { uint32_t opcode = MLX5_OPCODE_WAIT; uint32_t opmode = 0x1; //wait on data uint32_t n_ds = 3; //CTRL + Wait on Data of Size 2 struct ibv_qp_ex *qp_ex = ibv_qp_to_qp_ex(qp); struct mlx5dv_qp_ex *mqp = mlx5dv_qp_ex_from_ibv_qp_ex(qp_ex); uint8_t fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; char wqe_desc[n_ds * DS_SIZE]; struct mlx5_wqe_ctrl_seg *ctrl; wait_on_data_seg_t * wseg; // required alignement on the buffer ucc_assert(addr % 8 == 0); memset(wqe_desc, 0, n_ds * DS_SIZE); /* SET CTRL SEG */ ibv_wr_start(qp_ex); qp_ex->wr_id = ((uint64_t)(uintptr_t)task_ptr) | 0x1; ctrl = (void *)wqe_desc; mlx5dv_set_ctrl_seg(ctrl, /* pi */ 0x0, opcode, opmode, qp->qp_num, fm_ce_se, n_ds, 0x0, 0x0); /* SET TRANSPOSE SEG */ wseg = PTR_OFFSET(ctrl, DS_SIZE); wseg->op = htobe32(0x1); //0x1 - OP_EQUAL wseg->lkey = htobe32(lkey); wseg->va_fail = htobe64((addr) | (ACTION)); wseg->data = value; wseg->data_mask = 0xFFFFFFFF; mlx5dv_wr_raw_wqe(mqp, wqe_desc); if (ibv_wr_complete(qp_ex)) { return UCC_ERR_NO_MESSAGE; } return UCC_OK; } ucc-1.8.0/src/components/tl/mlx5/tl_mlx5_coll.h0000664000175000017500000001142115211535620021503 0ustar alastairalastair/** * Copyright (c) 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_MLX5_COLL_H_ #define UCC_TL_MLX5_COLL_H_ #include "tl_mlx5.h" #include "schedule/ucc_schedule.h" #include "alltoall/alltoall.h" typedef struct ucc_tl_mlx5_task { ucc_coll_task_t super; union { struct { ucc_tl_mlx5_mcast_coll_req_t *req_handle; } coll_mcast; }; } ucc_tl_mlx5_task_t; typedef struct ucc_tl_mlx5_schedule { ucc_schedule_t super; union { struct { int seq_num; int seq_index; int num_of_blocks_columns; int block_height; int block_width; int started; int send_blocks_enqueued; int blocks_sent; int blocks_completed; ucc_tl_mlx5_alltoall_op_t *op; ucc_tl_mlx5_rcache_region_t *send_rcache_region_p; ucc_tl_mlx5_rcache_region_t *recv_rcache_region_p; size_t msg_size; ucc_service_coll_req_t *barrier_req; int barrier_scratch[2]; int wait_wc; } alltoall; }; } ucc_tl_mlx5_schedule_t; #define TASK_TEAM(_task) \ (ucc_derived_of((_task)->super.team, ucc_tl_mlx5_team_t)) #define TASK_CTX(_task) \ (ucc_derived_of((_task)->super.team->context, ucc_tl_mlx5_context_t)) #define TASK_LIB(_task) \ (ucc_derived_of((_task)->super.team->context->lib, ucc_tl_mlx5_lib_t)) #define TASK_ARGS(_task) (_task)->super.bargs.args #define TASK_SCHEDULE(_task) \ (ucc_derived_of((_task)->schedule, ucc_tl_mlx5_schedule_t)) #define SCHEDULE_TEAM(_schedule) \ (ucc_derived_of((_schedule)->super.super.team, ucc_tl_mlx5_team_t)) #define SCHEDULE_CTX(_schedule) \ (ucc_derived_of((_schedule)->super.super.team->context, \ ucc_tl_mlx5_context_t)) #define SCHEDULE_LIB(_schedule) \ (ucc_derived_of((_schedule)->super.super.team->context->lib, \ ucc_tl_mlx5_lib_t)) #define SCHEDULE_ARGS(_schedule) (_schedule)->super.super.bargs.args static inline ucc_tl_mlx5_task_t* ucc_tl_mlx5_get_task(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team) { ucc_tl_mlx5_team_t * tl_team = ucc_derived_of(team, ucc_tl_mlx5_team_t); ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(tl_team); ucc_tl_mlx5_task_t * task = ucc_mpool_get(&ctx->req_mp); UCC_TL_MLX5_PROFILE_REQUEST_NEW(task, "tl_mlx5_task", 0); ucc_coll_task_init(&task->super, coll_args, team); task->coll_mcast.req_handle = NULL; return task; } static inline void ucc_tl_mlx5_put_task(ucc_tl_mlx5_task_t *task) { UCC_TL_MLX5_PROFILE_REQUEST_FREE(task); ucc_mpool_put(task); } static inline ucc_status_t ucc_tl_mlx5_get_schedule(ucc_tl_mlx5_team_t *team, ucc_base_coll_args_t *coll_args, ucc_tl_mlx5_schedule_t **schedule) { ucc_tl_mlx5_context_t *ctx = UCC_TL_MLX5_TEAM_CTX(team); *schedule = ucc_mpool_get(&ctx->req_mp); if (ucc_unlikely(!(*schedule))) { return UCC_ERR_NO_MEMORY; } UCC_TL_MLX5_PROFILE_REQUEST_NEW(schedule, "tl_mlx5_sched", 0); return ucc_schedule_init(&((*schedule)->super), coll_args, &team->super.super); } static inline void ucc_tl_mlx5_put_schedule(ucc_tl_mlx5_schedule_t *schedule) { UCC_TL_MLX5_PROFILE_REQUEST_FREE(schedule); ucc_mpool_put(schedule); } ucc_status_t ucc_tl_mlx5_coll_mcast_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); ucc_status_t ucc_tl_mlx5_task_finalize(ucc_coll_task_t *coll_task); ucc_tl_mlx5_task_t* ucc_tl_mlx5_init_task(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_schedule_t *schedule); ucc_status_t ucc_tl_mlx5_coll_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_coll_task_t **task_h); #endif ucc-1.8.0/src/components/tl/ucc_tl_log.h0000664000175000017500000000130115211535620020327 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TL_LOG_H_ #define UCC_TL_LOG_H_ #include "components/base/ucc_base_iface.h" #define tl_error(_tl_lib, _fmt, ...) base_error((ucc_base_lib_t*)(_tl_lib), _fmt, ## __VA_ARGS__) #define tl_warn(_tl_lib, _fmt, ...) base_warn((ucc_base_lib_t*)(_tl_lib), _fmt, ## __VA_ARGS__) #define tl_info(_tl_lib, _fmt, ...) base_info((ucc_base_lib_t*)(_tl_lib), _fmt, ## __VA_ARGS__) #define tl_debug(_tl_lib, _fmt, ...) base_debug((ucc_base_lib_t*)(_tl_lib), _fmt, ## __VA_ARGS__) #define tl_trace(_tl_lib, _fmt, ...) base_trace((ucc_base_lib_t*)(_tl_lib), _fmt, ## __VA_ARGS__) #endif ucc-1.8.0/src/components/mc/0000775000175000017500000000000015211535620016031 5ustar alastairalastairucc-1.8.0/src/components/mc/cuda/0000775000175000017500000000000015211535620016745 5ustar alastairalastairucc-1.8.0/src/components/mc/cuda/mc_cuda.h0000664000175000017500000000166515211535620020521 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_MC_CUDA_H_ #define UCC_MC_CUDA_H_ #include #include "components/mc/base/ucc_mc_base.h" #include "components/mc/ucc_mc_log.h" #include "utils/ucc_mpool.h" #include "utils/arch/cuda_def.h" #include "mc_cuda_resources.h" typedef struct ucc_mc_cuda { ucc_mc_base_t super; ucc_spinlock_t init_spinlock; ucc_thread_mode_t thread_mode; ucc_mc_cuda_resources_hash_t *resources_hash; } ucc_mc_cuda_t; extern ucc_mc_cuda_t ucc_mc_cuda; #define MC_CUDA_CONFIG \ (ucc_derived_of(ucc_mc_cuda.super.config, ucc_mc_cuda_config_t)) ucc_status_t ucc_mc_cuda_get_resources(ucc_mc_cuda_resources_t **resources); ucc_status_t ucc_mc_cuda_memset(void *ptr, int val, size_t len); #endif ucc-1.8.0/src/components/mc/cuda/mc_cuda.c0000664000175000017500000003672515211535620020521 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "mc_cuda.h" #include "utils/ucc_malloc.h" #include "utils/arch/cpu.h" #include #include static ucc_config_field_t ucc_mc_cuda_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_mc_cuda_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_mc_config_table)}, {"MPOOL_ELEM_SIZE", "1Mb", "The size of each element in mc cuda mpool", ucc_offsetof(ucc_mc_cuda_config_t, mpool_elem_size), UCC_CONFIG_TYPE_MEMUNITS}, {"MPOOL_MAX_ELEMS", "8", "The max amount of elements in mc cuda mpool", ucc_offsetof(ucc_mc_cuda_config_t, mpool_max_elems), UCC_CONFIG_TYPE_UINT}, {NULL} }; static ucc_status_t ucc_mc_cuda_flush_not_supported() { mc_error(&ucc_mc_cuda.super, "consistency api is not supported"); return UCC_ERR_NOT_SUPPORTED; } #if CUDA_VERSION >= 11030 static ucc_status_t ucc_mc_cuda_flush_no_op() { return UCC_OK; } static ucc_status_t ucc_mc_cuda_flush_to_owner() { CUDADRV_FUNC(cuFlushGPUDirectRDMAWrites(CU_FLUSH_GPU_DIRECT_RDMA_WRITES_TARGET_CURRENT_CTX, CU_FLUSH_GPU_DIRECT_RDMA_WRITES_TO_OWNER)); return UCC_OK; } #endif static ucc_status_t ucc_mc_cuda_init(const ucc_mc_params_t *mc_params) { int num_devices, driver_ver; cudaError_t cuda_st; ucc_mc_cuda_config = ucc_derived_of(ucc_mc_cuda.super.config, ucc_mc_cuda_config_t); ucc_strncpy_safe(ucc_mc_cuda.super.config->log_component.name, ucc_mc_cuda.super.super.name, sizeof(ucc_mc_cuda.super.config->log_component.name)); ucc_mc_cuda.thread_mode = mc_params->thread_mode; cuda_st = cudaGetDeviceCount(&num_devices); if ((cuda_st != cudaSuccess) || (num_devices == 0)) { mc_debug(&ucc_mc_cuda.super, "cuda devices are not found"); return UCC_ERR_NO_RESOURCE; } CUDADRV_FUNC(cuDriverGetVersion(&driver_ver)); mc_debug(&ucc_mc_cuda.super, "driver version %d", driver_ver); #if CUDA_VERSION >= 11030 if (driver_ver >= 11030) { CUdevice cu_dev; CUresult cu_st; int attr; const char *cu_err_st_str; cu_st = cuCtxGetDevice(&cu_dev); if (cu_st != CUDA_SUCCESS){ cuGetErrorString(cu_st, &cu_err_st_str); mc_debug(&ucc_mc_cuda.super, "cuCtxGetDevice() failed: %s", cu_err_st_str); } else { attr = 0; CUDADRV_FUNC(cuDeviceGetAttribute(&attr, CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_FLUSH_WRITES_OPTIONS, cu_dev)); if (attr & CU_FLUSH_GPU_DIRECT_RDMA_WRITES_OPTION_HOST) { CUDADRV_FUNC(cuDeviceGetAttribute(&attr, CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WRITES_ORDERING, cu_dev)); if (CU_FLUSH_GPU_DIRECT_RDMA_WRITES_TO_OWNER > attr) { ucc_mc_cuda.super.ops.flush = ucc_mc_cuda_flush_to_owner; } else { ucc_mc_cuda.super.ops.flush = ucc_mc_cuda_flush_no_op; } } else { mc_debug(&ucc_mc_cuda.super, "consistency api is not supported"); } } } else { mc_debug(&ucc_mc_cuda.super, "cuFlushGPUDirectRDMAWrites is not supported " "with driver version %d", driver_ver); } #endif ucc_mc_cuda.resources_hash = kh_init(ucc_mc_cuda_resources_hash); // lock assures single mpool initiation when multiple threads concurrently execute // different collective operations thus concurrently entering init function. ucc_spinlock_init(&ucc_mc_cuda.init_spinlock, 0); return UCC_OK; } static ucc_status_t ucc_mc_cuda_get_attr(ucc_mc_attr_t *mc_attr) { if (mc_attr->field_mask & UCC_MC_ATTR_FIELD_THREAD_MODE) { mc_attr->thread_mode = ucc_mc_cuda.thread_mode; } if (mc_attr->field_mask & UCC_MC_ATTR_FIELD_FAST_ALLOC_SIZE) { if (MC_CUDA_CONFIG->mpool_max_elems > 0) { mc_attr->fast_alloc_size = MC_CUDA_CONFIG->mpool_elem_size; } else { mc_attr->fast_alloc_size = 0; } } return UCC_OK; } static ucc_status_t ucc_mc_cuda_mem_alloc(ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mt) { cudaError_t st; ucc_mc_buffer_header_t *h; h = ucc_malloc(sizeof(ucc_mc_buffer_header_t), "mc cuda"); if (ucc_unlikely(!h)) { mc_error(&ucc_mc_cuda.super, "failed to allocate %zd bytes", sizeof(ucc_mc_buffer_header_t)); return UCC_ERR_NO_MEMORY; } st = (mt == UCC_MEMORY_TYPE_CUDA) ? cudaMalloc(&h->addr, size) : cudaMallocManaged(&h->addr, size, cudaMemAttachGlobal); if (ucc_unlikely(st != cudaSuccess)) { cudaGetLastError(); mc_error(&ucc_mc_cuda.super, "failed to allocate %zd bytes, " "cuda error %d(%s)", size, st, cudaGetErrorString(st)); ucc_free(h); return UCC_ERR_NO_MEMORY; } h->from_pool = 0; h->mt = UCC_MEMORY_TYPE_CUDA; *h_ptr = h; mc_trace(&ucc_mc_cuda.super, "allocated %ld bytes with %s", size, ucc_memory_type_names[mt]); return UCC_OK; } static ucc_status_t ucc_mc_cuda_mem_pool_alloc(ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mt) { ucc_mc_buffer_header_t *h = NULL; ucc_mc_cuda_resources_t *resources; ucc_status_t status; if ((size <= MC_CUDA_CONFIG->mpool_elem_size) && (mt != UCC_MEMORY_TYPE_CUDA_MANAGED)) { status = ucc_mc_cuda_get_resources(&resources); if (ucc_unlikely(status != UCC_OK)) { return status; } h = (ucc_mc_buffer_header_t *)ucc_mpool_get(&resources->scratch_mpool); } if (!h) { // Slow path return ucc_mc_cuda_mem_alloc(h_ptr, size, mt); } if (ucc_unlikely(!h->addr)){ return UCC_ERR_NO_MEMORY; } *h_ptr = h; mc_trace(&ucc_mc_cuda.super, "allocated %ld bytes from cuda mpool", size); return UCC_OK; } static ucc_status_t ucc_mc_cuda_mem_free(ucc_mc_buffer_header_t *h_ptr) { cudaError_t st; st = cudaFree(h_ptr->addr); if (ucc_unlikely(st != cudaSuccess)) { cudaGetLastError(); mc_error(&ucc_mc_cuda.super, "failed to free mem at %p, " "cuda error %d(%s)", h_ptr, st, cudaGetErrorString(st)); return UCC_ERR_NO_MESSAGE; } ucc_free(h_ptr); return UCC_OK; } static ucc_status_t ucc_mc_cuda_mem_pool_free(ucc_mc_buffer_header_t *h_ptr) { if (!h_ptr->from_pool) { return ucc_mc_cuda_mem_free(h_ptr); } ucc_mpool_put(h_ptr); return UCC_OK; } static ucc_status_t ucc_mc_cuda_mem_pool_alloc_with_init(ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mt) { if (MC_CUDA_CONFIG->mpool_max_elems == 0) { ucc_mc_cuda.super.ops.mem_alloc = ucc_mc_cuda_mem_alloc; ucc_mc_cuda.super.ops.mem_free = ucc_mc_cuda_mem_free; return ucc_mc_cuda_mem_alloc(h_ptr, size, mt); } else { ucc_mc_cuda.super.ops.mem_alloc = ucc_mc_cuda_mem_pool_alloc; ucc_mc_cuda.super.ops.mem_free = ucc_mc_cuda_mem_pool_free; return ucc_mc_cuda_mem_pool_alloc(h_ptr, size, mt); } } static ucc_status_t ucc_mc_cuda_memcpy(void *dst, const void *src, size_t len, ucc_memory_type_t dst_mem, ucc_memory_type_t src_mem) { ucc_status_t status; ucc_mc_cuda_resources_t *resources; ucc_assert(dst_mem == UCC_MEMORY_TYPE_CUDA || src_mem == UCC_MEMORY_TYPE_CUDA || dst_mem == UCC_MEMORY_TYPE_CUDA_MANAGED || src_mem == UCC_MEMORY_TYPE_CUDA_MANAGED); status = ucc_mc_cuda_get_resources(&resources); if (ucc_unlikely(status) != UCC_OK) { return status; } status = CUDA_FUNC(cudaMemcpyAsync(dst, src, len, cudaMemcpyDefault, resources->stream)); if (ucc_unlikely(status != UCC_OK)) { mc_error(&ucc_mc_cuda.super, "failed to launch cudaMemcpyAsync, dst %p, src %p, len %zd", dst, src, len); return status; } status = CUDA_FUNC(cudaStreamSynchronize(resources->stream)); return status; } ucc_status_t ucc_mc_cuda_memset(void *ptr, int val, size_t len) { ucc_status_t status; ucc_mc_cuda_resources_t *resources; status = ucc_mc_cuda_get_resources(&resources); if (ucc_unlikely(status) != UCC_OK) { return status; } status = CUDA_FUNC(cudaMemsetAsync(ptr, val, len, resources->stream)); if (ucc_unlikely(status != UCC_OK)) { mc_error(&ucc_mc_cuda.super, "failed to launch cudaMemsetAsync, dst %p, len %zd", ptr, len); return status; } status = CUDA_FUNC(cudaStreamSynchronize(resources->stream)); return status; } static ucc_status_t ucc_mc_cuda_mem_query(const void *ptr, ucc_mem_attr_t *mem_attr) { struct cudaPointerAttributes attr; cudaError_t st; CUresult cu_err; ucc_memory_type_t mem_type; void *base_address; size_t alloc_length; if (!(mem_attr->field_mask & (UCC_MEM_ATTR_FIELD_MEM_TYPE | UCC_MEM_ATTR_FIELD_BASE_ADDRESS | UCC_MEM_ATTR_FIELD_ALLOC_LENGTH))) { return UCC_OK; } if (mem_attr->field_mask & UCC_MEM_ATTR_FIELD_MEM_TYPE) { st = cudaPointerGetAttributes(&attr, ptr); if (st != cudaSuccess) { cudaGetLastError(); return UCC_ERR_NOT_SUPPORTED; } #if CUDART_VERSION >= 10000 switch (attr.type) { case cudaMemoryTypeHost: mem_type = UCC_MEMORY_TYPE_HOST; break; case cudaMemoryTypeDevice: mem_type = UCC_MEMORY_TYPE_CUDA; break; case cudaMemoryTypeManaged: mem_type = UCC_MEMORY_TYPE_CUDA_MANAGED; break; default: return UCC_ERR_NOT_SUPPORTED; } #else if (attr.memoryType == cudaMemoryTypeDevice) { if (attr.isManaged) { mem_type = UCC_MEMORY_TYPE_CUDA_MANAGED; } else { mem_type = UCC_MEMORY_TYPE_CUDA; } } else if (attr.memoryType == cudaMemoryTypeHost) { mem_type = UCC_MEMORY_TYPE_HOST; } else { return UCC_ERR_NOT_SUPPORTED; } #endif mem_attr->mem_type = mem_type; } if (mem_attr->field_mask & (UCC_MEM_ATTR_FIELD_ALLOC_LENGTH | UCC_MEM_ATTR_FIELD_BASE_ADDRESS)) { cu_err = cuMemGetAddressRange((CUdeviceptr*)&base_address, &alloc_length, (CUdeviceptr)ptr); if (cu_err != CUDA_SUCCESS) { mc_debug(&ucc_mc_cuda.super, "cuMemGetAddressRange(%p) error: %d", ptr, cu_err); return UCC_ERR_NOT_SUPPORTED; } mem_attr->base_address = base_address; mem_attr->alloc_length = alloc_length; } return UCC_OK; } ucc_status_t ucc_mc_cuda_get_resources(ucc_mc_cuda_resources_t **resources) { CUcontext cu_ctx; unsigned long long int cu_ctx_id; ucc_status_t status; status = CUDADRV_FUNC(cuCtxGetCurrent(&cu_ctx)); if (ucc_unlikely(status != UCC_OK)) { mc_error(&ucc_mc_cuda.super, "failed to get current CUDA context"); return status; } #if CUDA_VERSION < 12000 cu_ctx_id = 1; #else CUresult cu_status; cu_status = cuCtxGetId(cu_ctx, &cu_ctx_id); if (ucc_unlikely(cu_status != CUDA_SUCCESS)) { /* worakround for pytorch, progress thread doesn't have cuda context for GPU 0*/ /* cuda context might be not initialized, try to proceed with default device*/ cu_ctx_id = 0x12345; mc_debug(&ucc_mc_cuda.super, "failed to get current CUDA context ID (%d)", cu_status); } #endif *resources = mc_cuda_resources_hash_get(ucc_mc_cuda.resources_hash, cu_ctx_id); if (ucc_unlikely(*resources == NULL)) { ucc_spin_lock(&ucc_mc_cuda.init_spinlock); *resources = mc_cuda_resources_hash_get(ucc_mc_cuda.resources_hash, cu_ctx_id); if (*resources == NULL) { *resources = ucc_malloc(sizeof(ucc_mc_cuda_resources_t), "mc cuda resources"); if (*resources == NULL) { mc_error(&ucc_mc_cuda.super, "failed to allocate %zd bytes for resources", sizeof(ucc_mc_cuda_resources_t)); ucc_spin_unlock(&ucc_mc_cuda.init_spinlock); return UCC_ERR_NO_MEMORY; } status = ucc_mc_cuda_resources_init(&ucc_mc_cuda.super, *resources); if (status != UCC_OK) { ucc_free(*resources); ucc_spin_unlock(&ucc_mc_cuda.init_spinlock); return status; } mc_cuda_resources_hash_put(ucc_mc_cuda.resources_hash, cu_ctx_id, *resources); } ucc_spin_unlock(&ucc_mc_cuda.init_spinlock); } return UCC_OK; } static ucc_status_t ucc_mc_cuda_finalize() { ucc_mc_cuda_resources_t *resources; resources = mc_cuda_resources_hash_pop(ucc_mc_cuda.resources_hash); while (resources) { ucc_mc_cuda_resources_cleanup(resources); resources = mc_cuda_resources_hash_pop(ucc_mc_cuda.resources_hash); } ucc_mc_cuda.super.ops.mem_alloc = ucc_mc_cuda_mem_pool_alloc_with_init; ucc_spinlock_destroy(&ucc_mc_cuda.init_spinlock); return UCC_OK; } ucc_mc_cuda_t ucc_mc_cuda = { .super.super.name = "cuda mc", .super.ref_cnt = 0, .super.ee_type = UCC_EE_CUDA_STREAM, .super.type = UCC_MEMORY_TYPE_CUDA, .super.init = ucc_mc_cuda_init, .super.get_attr = ucc_mc_cuda_get_attr, .super.finalize = ucc_mc_cuda_finalize, .super.ops.mem_query = ucc_mc_cuda_mem_query, .super.ops.mem_alloc = ucc_mc_cuda_mem_pool_alloc_with_init, .super.ops.mem_free = ucc_mc_cuda_mem_pool_free, .super.ops.memcpy = ucc_mc_cuda_memcpy, .super.ops.memset = ucc_mc_cuda_memset, .super.ops.flush = ucc_mc_cuda_flush_not_supported, .super.config_table = { .name = "CUDA memory component", .prefix = "MC_CUDA_", .table = ucc_mc_cuda_config_table, .size = sizeof(ucc_mc_cuda_config_t), }, }; ucc_mc_cuda_config_t *ucc_mc_cuda_config; UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_mc_cuda.super.config_table, &ucc_config_global_list); ucc-1.8.0/src/components/mc/cuda/mc_cuda_resources.h0000664000175000017500000000407015211535620022604 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_MC_CUDA_RESOURCES_H_ #define UCC_MC_CUDA_RESOURCES_H_ #include "components/mc/base/ucc_mc_base.h" #include "utils/arch/cuda_def.h" #include "utils/ucc_mpool.h" typedef struct ucc_mc_cuda_config { ucc_mc_config_t super; size_t mpool_elem_size; int mpool_max_elems; } ucc_mc_cuda_config_t; typedef struct ucc_mc_cuda_resources { CUcontext cu_ctx; cudaStream_t stream; ucc_mpool_t scratch_mpool; } ucc_mc_cuda_resources_t; extern ucc_mc_cuda_config_t *ucc_mc_cuda_config; ucc_status_t ucc_mc_cuda_resources_init(ucc_mc_base_t *mc, ucc_mc_cuda_resources_t *resources); void ucc_mc_cuda_resources_cleanup(ucc_mc_cuda_resources_t *resources); KHASH_INIT(ucc_mc_cuda_resources_hash, unsigned long long, void*, 1, \ kh_int64_hash_func, kh_int64_hash_equal); #define ucc_mc_cuda_resources_hash_t khash_t(ucc_mc_cuda_resources_hash) static inline void* mc_cuda_resources_hash_get(ucc_mc_cuda_resources_hash_t *h, unsigned long long key) { khiter_t k; void *value; k = kh_get(ucc_mc_cuda_resources_hash, h , key); if (k == kh_end(h)) { return NULL; } value = kh_value(h, k); return value; } static inline void mc_cuda_resources_hash_put(ucc_mc_cuda_resources_hash_t *h, unsigned long long key, void *value) { int ret; khiter_t k; k = kh_put(ucc_mc_cuda_resources_hash, h, key, &ret); kh_value(h, k) = value; } static inline void* mc_cuda_resources_hash_pop(ucc_mc_cuda_resources_hash_t *h) { void *resources = NULL; khiter_t k; k = kh_begin(h); while (k != kh_end(h)) { if (kh_exist(h, k)) { resources = kh_value(h, k); break; } k++; } if (resources) { kh_del(ucc_mc_cuda_resources_hash, h, k); } return resources; } #endif ucc-1.8.0/src/components/mc/cuda/mc_cuda_resources.c0000664000175000017500000000636315211535620022606 0ustar alastairalastair#include "mc_cuda_resources.h" #include "components/mc/ucc_mc_log.h" #include "utils/ucc_malloc.h" static ucc_status_t ucc_mc_cuda_chunk_alloc(ucc_mpool_t *mp, //NOLINT size_t *size_p, void **chunk_p) { *chunk_p = ucc_malloc(*size_p, "mc cuda"); if (!*chunk_p) { return UCC_ERR_NO_MEMORY; } return UCC_OK; } static void ucc_mc_cuda_chunk_init(ucc_mpool_t *mp, //NOLINT void *obj, void *chunk) //NOLINT { ucc_mc_buffer_header_t *h = (ucc_mc_buffer_header_t *)obj; cudaError_t st; st = cudaMalloc(&h->addr, ucc_mc_cuda_config->mpool_elem_size); if (st != cudaSuccess) { // h->addr will be 0 so ucc_mc_cuda_mem_alloc_pool function will // return UCC_ERR_NO_MEMORY. As such mc_error message is suffice. cudaGetLastError(); } h->from_pool = 1; h->mt = UCC_MEMORY_TYPE_CUDA; } static void ucc_mc_cuda_chunk_release(ucc_mpool_t *mp, void *chunk) //NOLINT: mp is unused { ucc_free(chunk); } static void ucc_mc_cuda_chunk_cleanup(ucc_mpool_t *mp, void *obj) //NOLINT: mp is unused { ucc_mc_buffer_header_t *h = (ucc_mc_buffer_header_t *)obj; cudaError_t st; st = cudaFree(h->addr); if (st != cudaSuccess) { cudaGetLastError(); } } static ucc_mpool_ops_t ucc_mc_ops = {.chunk_alloc = ucc_mc_cuda_chunk_alloc, .chunk_release = ucc_mc_cuda_chunk_release, .obj_init = ucc_mc_cuda_chunk_init, .obj_cleanup = ucc_mc_cuda_chunk_cleanup}; ucc_status_t ucc_mc_cuda_resources_init(ucc_mc_base_t *mc, ucc_mc_cuda_resources_t *resources) { ucc_status_t status; CUDADRV_CHECK(cuCtxGetCurrent(&resources->cu_ctx)); status = ucc_mpool_init(&resources->scratch_mpool, 0, sizeof(ucc_mc_buffer_header_t), 0, UCC_CACHE_LINE_SIZE, 1, ucc_mc_cuda_config->mpool_max_elems, &ucc_mc_ops, UCC_THREAD_MULTIPLE, "mc cuda mpool buffers"); if (status != UCC_OK) { mc_error(mc, "failed to create scratch buffers mpool"); return status; } status = CUDA_FUNC(cudaStreamCreateWithFlags(&resources->stream, cudaStreamNonBlocking)); if (status != UCC_OK) { mc_error(mc, "failed to create CUDA stream"); goto free_scratch_mpool; } return UCC_OK; free_scratch_mpool: ucc_mpool_cleanup(&resources->scratch_mpool, 0); return status; } void ucc_mc_cuda_resources_cleanup(ucc_mc_cuda_resources_t *resources) { CUcontext tmp_context; #if CUDA_VERSION >= 12000 CUresult status; unsigned long long int cu_ctx_id; status = cuCtxGetId(resources->cu_ctx, &cu_ctx_id); if (ucc_unlikely(status != CUDA_SUCCESS)) { // ctx is not available, can be due to cudaDeviceReset return; } #endif cuCtxPushCurrent(resources->cu_ctx); ucc_mpool_cleanup(&resources->scratch_mpool, 1); CUDA_FUNC(cudaStreamDestroy(resources->stream)); cuCtxPopCurrent(&tmp_context); } ucc-1.8.0/src/components/mc/cuda/Makefile.am0000664000175000017500000000123515211535620021002 0ustar alastairalastair# # Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # if HAVE_CUDA sources = \ mc_cuda.h \ mc_cuda.c \ mc_cuda_resources.c \ mc_cuda_resources.h module_LTLIBRARIES = libucc_mc_cuda.la libucc_mc_cuda_la_SOURCES = $(sources) libucc_mc_cuda_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) libucc_mc_cuda_la_CFLAGS = $(BASE_CFLAGS) libucc_mc_cuda_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(CUDA_LDFLAGS) libucc_mc_cuda_la_LIBADD = $(CUDA_LIBS) \ $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/mc/base/0000775000175000017500000000000015211535620016743 5ustar alastairalastairucc-1.8.0/src/components/mc/base/ucc_mc_base.h0000664000175000017500000001007115211535620021336 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_MC_BASE_H_ #define UCC_MC_BASE_H_ #include "config.h" #include "utils/ucc_component.h" #include "utils/ucc_class.h" #include "utils/ucc_parser.h" #include "utils/ucc_mpool.h" #include "core/ucc_global_opts.h" typedef struct ucc_mc_params { ucc_thread_mode_t thread_mode; } ucc_mc_params_t; typedef struct ucc_mc_buffer_header { ucc_memory_type_t mt; int from_pool; void *addr; } ucc_mc_buffer_header_t; /** * UCC memory attributes field mask */ typedef enum ucc_mem_attr_field { UCC_MEM_ATTR_FIELD_MEM_TYPE = UCC_BIT(0), /**< Indicate if memory type is populated. E.g. CPU/GPU */ UCC_MEM_ATTR_FIELD_BASE_ADDRESS = UCC_BIT(2), /**< Request base address of the allocation */ UCC_MEM_ATTR_FIELD_ALLOC_LENGTH = UCC_BIT(3) /**< Request the whole length of the allocation */ } ucc_mem_attr_field_t; typedef struct ucc_mem_attr { /** * Mask of valid fields in this structure, using bits from * @ref ucc_mem_attr_field_t. */ uint64_t field_mask; /** * The type of memory. E.g. CPU/GPU memory or some other valid type */ ucc_memory_type_t mem_type; /** * Base address of the allocation to which the provided buffer belongs to. * If the mc not support base address query, then the pointer passed to * ucc_mc_cuda_mem_query is returned as is. */ void *base_address; /** * If UCC_MEM_ATTR_FIELD_ALLOC_LENGTH is set this parameter should be equal * to known allocation length. On output length of the whole allocation * to which the provided buffer belongs to is returned. * If the md not support querying allocation length, then the length passed * to ucc_mc_cuda_mem_query is returned as is. */ size_t alloc_length; } ucc_mem_attr_t; /** * UCC memory component attributes field mask */ typedef enum ucc_mc_attr_field { UCC_MC_ATTR_FIELD_THREAD_MODE = UCC_BIT(0), /* size of memory pool chunk element */ UCC_MC_ATTR_FIELD_FAST_ALLOC_SIZE = UCC_BIT(1), } ucc_mc_attr_field_t; typedef struct ucc_mc_attr { /** * Mask of valid fields in this structure, using bits from * @ref ucc_mc_attr_field_t. */ uint64_t field_mask; ucc_thread_mode_t thread_mode; size_t fast_alloc_size; } ucc_mc_attr_t; /** * Array of string names for each memory type */ extern const char *ucc_memory_type_names[]; /** * Array of string names for each execution engine type */ extern const char *ucc_ee_type_names[]; typedef struct ucc_mc_config { ucc_log_component_config_t log_component; } ucc_mc_config_t; extern ucc_config_field_t ucc_mc_config_table[]; typedef struct ucc_mc_ops { ucc_status_t (*mem_query)(const void *ptr, ucc_mem_attr_t *mem_attr); ucc_status_t (*mem_alloc)(ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mt); ucc_status_t (*mem_free)(ucc_mc_buffer_header_t *h_ptr); ucc_status_t (*memcpy)(void *dst, const void *src, size_t len, ucc_memory_type_t dst_mem, ucc_memory_type_t src_mem); ucc_status_t (*memset)(void *dst, int value, size_t len); ucc_status_t (*flush)(); } ucc_mc_ops_t; typedef struct ucc_mc_base { ucc_component_iface_t super; uint32_t ref_cnt; ucc_ee_type_t ee_type; ucc_memory_type_t type; ucc_mc_config_t *config; ucc_config_global_list_entry_t config_table; ucc_status_t (*init)(const ucc_mc_params_t *mc_params); ucc_status_t (*get_attr)(ucc_mc_attr_t *mc_attr); ucc_status_t (*finalize)(); ucc_mc_ops_t ops; } ucc_mc_base_t; #endif ucc-1.8.0/src/components/mc/base/ucc_mc_base.c0000664000175000017500000000203315211535620021330 0ustar alastairalastair/** * Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_mc_base.h" #include "utils/ucc_mem_type.h" ucc_config_field_t ucc_mc_config_table[] = { {"LOG_LEVEL", "warn", "UCC logging level. Messages with a level higher or equal to the " "selected will be printed.\n" "Possible values are: fatal, error, warn, info, debug, trace, data, func, " "poll.", ucc_offsetof(ucc_mc_config_t, log_component), UCC_CONFIG_TYPE_LOG_COMP}, {NULL}}; const char *ucc_memory_type_names[] = { [UCC_MEMORY_TYPE_HOST] = "host", [UCC_MEMORY_TYPE_CUDA] = "cuda", [UCC_MEMORY_TYPE_CUDA_MANAGED] = "cuda-managed", [UCC_MEMORY_TYPE_ROCM] = "rocm", [UCC_MEMORY_TYPE_ROCM_MANAGED] = "rocm-managed", [UCC_MEMORY_TYPE_LAST] = "unknown", [UCC_MEMORY_TYPE_NOT_APPLY] = "not_apply"}; const char *ucc_ee_type_names[] = { [UCC_EE_CUDA_STREAM] = "cuda stream", [UCC_EE_CPU_THREAD] = "cpu thread"}; ucc-1.8.0/src/components/mc/rocm/0000775000175000017500000000000015211535620016771 5ustar alastairalastairucc-1.8.0/src/components/mc/rocm/mc_rocm.h0000664000175000017500000000574615211535620020575 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_MC_ROCM_H_ #define UCC_MC_ROCM_H_ #include "components/mc/base/ucc_mc_base.h" #include "components/mc/ucc_mc_log.h" #include "utils/ucc_mpool.h" #include "utils/arch/rocm_def.h" #include #include static inline ucc_status_t hip_error_to_ucc_status(hipError_t hip_err) { switch(hip_err) { case hipSuccess: return UCC_OK; case hipErrorNotReady: return UCC_INPROGRESS; default: break; } return UCC_ERR_NO_MESSAGE; } typedef ucc_status_t (*ucc_mc_rocm_task_post_fn) (uint32_t *dev_status, int blocking_wait, hipStream_t stream); typedef struct ucc_mc_rocm_config { ucc_mc_config_t super; size_t mpool_elem_size; int mpool_max_elems; } ucc_mc_rocm_config_t; typedef struct ucc_mc_rocm { ucc_mc_base_t super; int stream_initialized; hipStream_t stream; ucc_mpool_t events; ucc_mpool_t strm_reqs; ucc_mpool_t mpool; int mpool_init_flag; ucc_spinlock_t init_spinlock; ucc_thread_mode_t thread_mode; } ucc_mc_rocm_t; extern ucc_mc_rocm_t ucc_mc_rocm; #define MC_ROCM_CONFIG \ (ucc_derived_of(ucc_mc_rocm.super.config, ucc_mc_rocm_config_t)) #define UCC_MC_ROCM_INIT_STREAM() do { \ if (!ucc_mc_rocm.stream_initialized) { \ hipError_t hip_st = hipSuccess; \ ucc_spin_lock(&ucc_mc_rocm.init_spinlock); \ if (!ucc_mc_rocm.stream_initialized) { \ hip_st = hipStreamCreateWithFlags(&ucc_mc_rocm.stream, \ hipStreamNonBlocking); \ ucc_mc_rocm.stream_initialized = 1; \ } \ ucc_spin_unlock(&ucc_mc_rocm.init_spinlock); \ if(hip_st != hipSuccess) { \ mc_error(&ucc_mc_rocm.super, "rocm failed with ret:%d(%s)", \ hip_st, hipGetErrorString(hip_st)); \ return hip_error_to_ucc_status(hip_st); \ } \ } \ } while(0) #endif ucc-1.8.0/src/components/mc/rocm/Makefile.am0000664000175000017500000000127615211535620021033 0ustar alastairalastair# # Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. # if HAVE_ROCM sources = \ mc_rocm.h \ mc_rocm.c module_LTLIBRARIES = libucc_mc_rocm.la libucc_mc_rocm_la_SOURCES = $(sources) libucc_mc_rocm_la_CPPFLAGS = $(AM_CPPFLAGS) $(HIP_CPPFLAGS) $(ROCM_CPPFLAGS) $(BASE_CPPFLAGS) libucc_mc_rocm_la_CFLAGS = $(BASE_CFLAGS) libucc_mc_rocm_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(HIP_LDFLAGS) $(ROCM_LDFLAGS) libucc_mc_rocm_la_LIBADD = $(HIP_LIBS) $(ROCM_LIBS) \ $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/mc/rocm/mc_rocm.c0000664000175000017500000003220015211535620020551 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "mc_rocm.h" #include "utils/ucc_malloc.h" #include "utils/arch/cpu.h" #include #include #include #include static ucc_config_field_t ucc_mc_rocm_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_mc_rocm_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_mc_config_table)}, {"MPOOL_ELEM_SIZE", "1Mb", "The size of each element in mc rocm mpool", ucc_offsetof(ucc_mc_rocm_config_t, mpool_elem_size), UCC_CONFIG_TYPE_MEMUNITS}, {"MPOOL_MAX_ELEMS", "8", "The max amount of elements in mc rocm mpool", ucc_offsetof(ucc_mc_rocm_config_t, mpool_max_elems), UCC_CONFIG_TYPE_UINT}, {NULL} }; static ucc_status_t ucc_mc_rocm_init(const ucc_mc_params_t *mc_params) { int num_devices; hipError_t rocm_st; ucc_mc_rocm.stream = NULL; ucc_mc_rocm.stream_initialized = 0; ucc_strncpy_safe(ucc_mc_rocm.super.config->log_component.name, ucc_mc_rocm.super.super.name, sizeof(ucc_mc_rocm.super.config->log_component.name)); ucc_mc_rocm.thread_mode = mc_params->thread_mode; rocm_st = hipGetDeviceCount(&num_devices); if ((rocm_st != hipSuccess) || (num_devices == 0)) { mc_debug(&ucc_mc_rocm.super, "rocm devices are not found"); return hip_error_to_ucc_status(rocm_st); } // lock assures single mpool initiation when multiple threads concurrently execute // different collective operations thus concurrently entering init function. ucc_spinlock_init(&ucc_mc_rocm.init_spinlock, 0); return UCC_OK; } static ucc_status_t ucc_mc_rocm_get_attr(ucc_mc_attr_t *mc_attr) { if (mc_attr->field_mask & UCC_MC_ATTR_FIELD_THREAD_MODE) { mc_attr->thread_mode = ucc_mc_rocm.thread_mode; } return UCC_OK; } static ucc_status_t ucc_mc_rocm_mem_alloc(ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mt) { hipError_t st; ucc_mc_buffer_header_t *h = ucc_malloc(sizeof(ucc_mc_buffer_header_t), "mc rocm"); if (ucc_unlikely(!h)) { mc_error(&ucc_mc_rocm.super, "failed to allocate %zd bytes", sizeof(ucc_mc_buffer_header_t)); } st = hipMalloc(&h->addr, size); if (ucc_unlikely(st != hipSuccess)) { hipGetLastError(); mc_error(&ucc_mc_rocm.super, "failed to allocate %zd bytes, " "rocm error %d(%s)", size, st, hipGetErrorString(st)); ucc_free(h); return hip_error_to_ucc_status(st); } h->from_pool = 0; h->mt = mt; *h_ptr = h; mc_trace(&ucc_mc_rocm.super, "allocated %ld bytes with hipMalloc", size); return UCC_OK; } static ucc_status_t ucc_mc_rocm_mem_pool_alloc(ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mt) { ucc_mc_buffer_header_t *h = NULL; if (size <= MC_ROCM_CONFIG->mpool_elem_size) { h = (ucc_mc_buffer_header_t *)ucc_mpool_get(&ucc_mc_rocm.mpool); } if (!h) { // Slow path return ucc_mc_rocm_mem_alloc(h_ptr, size, mt); } if (ucc_unlikely(!h->addr)){ return UCC_ERR_NO_MEMORY; } *h_ptr = h; mc_trace(&ucc_mc_rocm.super, "allocated %ld bytes from rocm mpool", size); return UCC_OK; } static ucc_status_t ucc_mc_rocm_chunk_alloc(ucc_mpool_t *mp, //NOLINT size_t *size_p, void **chunk_p) { *chunk_p = ucc_malloc(*size_p, "mc rocm"); if (!*chunk_p) { mc_error(&ucc_mc_rocm.super, "failed to allocate %zd bytes", *size_p); return UCC_ERR_NO_MEMORY; } return UCC_OK; } static void ucc_mc_rocm_chunk_init(ucc_mpool_t *mp, //NOLINT void *obj, void *chunk) //NOLINT { ucc_mc_buffer_header_t *h = (ucc_mc_buffer_header_t *)obj; hipError_t st = hipMalloc(&h->addr, MC_ROCM_CONFIG->mpool_elem_size); if (st != hipSuccess) { // h->addr will be 0 so ucc_mc_rocm_mem_alloc_pool function will // return UCC_ERR_NO_MEMORY. As such mc_error message is suffice. hipGetLastError(); mc_error(&ucc_mc_rocm.super, "failed to allocate %zd bytes, " "rocm error %d(%s)", MC_ROCM_CONFIG->mpool_elem_size, st, hipGetErrorString(st)); } h->from_pool = 1; h->mt = UCC_MEMORY_TYPE_ROCM; } static void ucc_mc_rocm_chunk_release(ucc_mpool_t *mp, void *chunk) //NOLINT: mp is unused { ucc_free(chunk); } static void ucc_mc_rocm_chunk_cleanup(ucc_mpool_t *mp, void *obj) //NOLINT: mp is unused { ucc_mc_buffer_header_t *h = (ucc_mc_buffer_header_t *)obj; hipError_t st; st = hipFree(h->addr); if (st != hipSuccess) { hipGetLastError(); mc_error(&ucc_mc_rocm.super, "failed to free mem at %p, " "rocm error %d(%s)", obj, st, hipGetErrorString(st)); } } static ucc_mpool_ops_t ucc_mc_ops = {.chunk_alloc = ucc_mc_rocm_chunk_alloc, .chunk_release = ucc_mc_rocm_chunk_release, .obj_init = ucc_mc_rocm_chunk_init, .obj_cleanup = ucc_mc_rocm_chunk_cleanup}; static ucc_status_t ucc_mc_rocm_mem_free(ucc_mc_buffer_header_t *h_ptr) { hipError_t st; st = hipFree(h_ptr->addr); if (ucc_unlikely(st != hipSuccess)) { hipGetLastError(); mc_error(&ucc_mc_rocm.super, "failed to free mem at %p, " "hip error %d(%s)", h_ptr->addr, st, hipGetErrorString(st)); return hip_error_to_ucc_status(st); } ucc_free(h_ptr); return UCC_OK; } static ucc_status_t ucc_mc_rocm_mem_pool_free(ucc_mc_buffer_header_t *h_ptr) { if (!h_ptr->from_pool) { return ucc_mc_rocm_mem_free(h_ptr); } ucc_mpool_put(h_ptr); return UCC_OK; } static ucc_status_t ucc_mc_rocm_mem_pool_alloc_with_init(ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mt) { // lock assures single mpool initiation when multiple threads concurrently execute // different collective operations thus concurrently entering init function. ucc_spin_lock(&ucc_mc_rocm.init_spinlock); if (MC_ROCM_CONFIG->mpool_max_elems == 0) { ucc_mc_rocm.super.ops.mem_alloc = ucc_mc_rocm_mem_alloc; ucc_mc_rocm.super.ops.mem_free = ucc_mc_rocm_mem_free; ucc_spin_unlock(&ucc_mc_rocm.init_spinlock); return ucc_mc_rocm_mem_alloc(h_ptr, size, mt); } if (!ucc_mc_rocm.mpool_init_flag) { ucc_status_t status = ucc_mpool_init( &ucc_mc_rocm.mpool, 0, sizeof(ucc_mc_buffer_header_t), 0, UCC_CACHE_LINE_SIZE, 1, MC_ROCM_CONFIG->mpool_max_elems, &ucc_mc_ops, ucc_mc_rocm.thread_mode, "mc rocm mpool buffers"); if (status != UCC_OK) { ucc_spin_unlock(&ucc_mc_rocm.init_spinlock); return status; } ucc_mc_rocm.super.ops.mem_alloc = ucc_mc_rocm_mem_pool_alloc; ucc_mc_rocm.mpool_init_flag = 1; } ucc_spin_unlock(&ucc_mc_rocm.init_spinlock); return ucc_mc_rocm_mem_pool_alloc(h_ptr, size, mt); } static ucc_status_t ucc_mc_rocm_memcpy(void *dst, const void *src, size_t len, ucc_memory_type_t dst_mem, ucc_memory_type_t src_mem) { hipError_t st; ucc_assert(dst_mem == UCC_MEMORY_TYPE_ROCM || src_mem == UCC_MEMORY_TYPE_ROCM); UCC_MC_ROCM_INIT_STREAM(); st = hipMemcpyAsync(dst, src, len, hipMemcpyDefault, ucc_mc_rocm.stream); if (ucc_unlikely(st != hipSuccess)) { hipGetLastError(); mc_error(&ucc_mc_rocm.super, "failed to launch hipMemcpyAsync, dst %p, src %p, len %zd " "hip error %d(%s)", dst, src, len, st, hipGetErrorString(st)); return hip_error_to_ucc_status(st); } st = hipStreamSynchronize(ucc_mc_rocm.stream); if (ucc_unlikely(st != hipSuccess)) { hipGetLastError(); mc_error(&ucc_mc_rocm.super, "failed to synchronize mc_rocm.stream " "hip error %d(%s)", st, hipGetErrorString(st)); return hip_error_to_ucc_status(st); } return UCC_OK; } static ucc_status_t ucc_mc_rocm_memset(void *ptr, int val, size_t len) { hipError_t st; UCC_MC_ROCM_INIT_STREAM(); st = hipMemsetAsync(ptr, val, len, ucc_mc_rocm.stream); if (ucc_unlikely(st != hipSuccess)) { hipGetLastError(); mc_error(&ucc_mc_rocm.super, "failed to launch hipMemsetAsync, dst %p, len %zd " "hip error %d(%s)", ptr, len, st, hipGetErrorString(st)); return hip_error_to_ucc_status(st); } st = hipStreamSynchronize(ucc_mc_rocm.stream); if (ucc_unlikely(st != hipSuccess)) { hipGetLastError(); mc_error(&ucc_mc_rocm.super, "failed to synchronize mc_rocm.stream " "hip error %d(%s)", st, hipGetErrorString(st)); return hip_error_to_ucc_status(st); } return UCC_OK; } static ucc_status_t ucc_mc_rocm_mem_query(const void *ptr, ucc_mem_attr_t *mem_attr) { struct hipPointerAttribute_t attr; hipError_t st; ucc_memory_type_t mem_type; void *base_address; size_t alloc_length; if (!(mem_attr->field_mask & (UCC_MEM_ATTR_FIELD_MEM_TYPE | UCC_MEM_ATTR_FIELD_BASE_ADDRESS | UCC_MEM_ATTR_FIELD_ALLOC_LENGTH))) { return UCC_OK; } if (mem_attr->field_mask & UCC_MEM_ATTR_FIELD_MEM_TYPE) { st = hipPointerGetAttributes(&attr, ptr); if (st != hipSuccess) { hipGetLastError(); return UCC_ERR_NOT_SUPPORTED; } #if HIP_VERSION >= 50500000 switch (attr.type) { #else switch (attr.memoryType) { #endif case hipMemoryTypeHost: mem_type = (attr.isManaged ? UCC_MEMORY_TYPE_ROCM_MANAGED : UCC_MEMORY_TYPE_HOST); break; case hipMemoryTypeDevice: mem_type = UCC_MEMORY_TYPE_ROCM; break; #if HIP_VERSION >= 50300000 case hipMemoryTypeManaged: mem_type = UCC_MEMORY_TYPE_ROCM_MANAGED; break; #endif default: return UCC_ERR_NOT_SUPPORTED; } mem_attr->mem_type = mem_type; } if (mem_attr->field_mask & (UCC_MEM_ATTR_FIELD_ALLOC_LENGTH | UCC_MEM_ATTR_FIELD_BASE_ADDRESS)) { st = hipMemGetAddressRange((hipDeviceptr_t*)&base_address, &alloc_length, (hipDeviceptr_t)ptr); if (st != hipSuccess) { mc_error(&ucc_mc_rocm.super, "hipMemGetAddressRange(%p) error: %d(%s)", ptr, st, hipGetErrorString(st)); return hip_error_to_ucc_status(st); } mem_attr->base_address = base_address; mem_attr->alloc_length = alloc_length; } return UCC_OK; } static ucc_status_t ucc_mc_rocm_finalize() { if (ucc_mc_rocm.stream != NULL) { ROCMCHECK(hipStreamDestroy(ucc_mc_rocm.stream)); ucc_mc_rocm.stream = NULL; } if (ucc_mc_rocm.mpool_init_flag) { ucc_mpool_cleanup(&ucc_mc_rocm.mpool, 1); ucc_mc_rocm.mpool_init_flag = 0; ucc_mc_rocm.super.ops.mem_alloc = ucc_mc_rocm_mem_pool_alloc_with_init; } ucc_spinlock_destroy(&ucc_mc_rocm.init_spinlock); return UCC_OK; } ucc_mc_rocm_t ucc_mc_rocm = { .super.super.name = "rocm mc", .super.ref_cnt = 0, .super.ee_type = UCC_EE_ROCM_STREAM, .super.type = UCC_MEMORY_TYPE_ROCM, .super.init = ucc_mc_rocm_init, .super.get_attr = ucc_mc_rocm_get_attr, .super.finalize = ucc_mc_rocm_finalize, .super.ops.mem_query = ucc_mc_rocm_mem_query, .super.ops.mem_alloc = ucc_mc_rocm_mem_pool_alloc_with_init, .super.ops.mem_free = ucc_mc_rocm_mem_pool_free, .super.ops.memcpy = ucc_mc_rocm_memcpy, .super.ops.memset = ucc_mc_rocm_memset, .super.config_table = { .name = "ROCM memory component", .prefix = "MC_ROCM_", .table = ucc_mc_rocm_config_table, .size = sizeof(ucc_mc_rocm_config_t), }, .mpool_init_flag = 0, }; UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_mc_rocm.super.config_table, &ucc_config_global_list); ucc-1.8.0/src/components/mc/ucc_mc_log.h0000664000175000017500000000215415211535620020276 0ustar alastairalastair/** * Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_MC_LOG_H_ #define UCC_MC_LOG_H_ #include "utils/ucc_log.h" #define ucc_log_component_mc(_mc, _level, fmt, ...) \ ucc_log_component(_level, &((_mc)->config)->log_component, \ fmt, ##__VA_ARGS__) #define mc_error(_mc, _fmt, ...) \ ucc_log_component_mc(_mc, UCC_LOG_LEVEL_ERROR, _fmt, ##__VA_ARGS__) #define mc_warn(_mc, _fmt, ...) \ ucc_log_component_mc(_mc, UCC_LOG_LEVEL_WARN, _fmt, ##__VA_ARGS__) #define mc_info(_mc, _fmt, ...) \ ucc_log_component_mc(_mc, UCC_LOG_LEVEL_INFO, _fmt, ##__VA_ARGS__) #define mc_debug(_mc, _fmt, ...) \ ucc_log_component_mc(_mc, UCC_LOG_LEVEL_DEBUG, _fmt, ##__VA_ARGS__) #define mc_trace(_mc, _fmt, ...) \ ucc_log_component_mc(_mc, UCC_LOG_LEVEL_TRACE, _fmt, ##__VA_ARGS__) #endif ucc-1.8.0/src/components/mc/ucc_mc.c0000664000175000017500000001663615211535620017442 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "components/mc/base/ucc_mc_base.h" #include "ucc_mc.h" #include "utils/ucc_malloc.h" #include "utils/ucc_log.h" #ifdef HAVE_PROFILING_MC #include "utils/profile/ucc_profile.h" #else #include "utils/profile/ucc_profile_off.h" #endif #define UCC_MC_PROFILE_FUNC UCC_PROFILE_FUNC static const ucc_mc_ops_t *mc_ops[UCC_MEMORY_TYPE_LAST]; #define UCC_CHECK_MC_AVAILABLE(mc) \ do { \ if (ucc_unlikely(NULL == mc_ops[mc])) { \ ucc_error("no components supported memory type %s available", \ ucc_memory_type_names[mc]); \ return UCC_ERR_NOT_SUPPORTED; \ } \ } while (0) ucc_status_t ucc_mc_init(const ucc_mc_params_t *mc_params) { int i, n_mcs; ucc_mc_base_t *mc; ucc_status_t status; ucc_mc_attr_t attr; memset(mc_ops, 0, UCC_MEMORY_TYPE_LAST * sizeof(ucc_mc_ops_t *)); n_mcs = ucc_global_config.mc_framework.n_components; for (i = 0; i < n_mcs; i++) { mc = ucc_derived_of(ucc_global_config.mc_framework.components[i], ucc_mc_base_t); if (mc->ref_cnt == 0) { mc->config = ucc_malloc(mc->config_table.size); if (!mc->config) { ucc_error("failed to allocate %zd bytes for mc config", mc->config_table.size); continue; } status = ucc_config_parser_fill_opts( mc->config, &mc->config_table, "UCC_", 1); if (UCC_OK != status) { ucc_debug("failed to parse config for mc: %s (%d)", mc->super.name, status); ucc_free(mc->config); continue; } status = mc->init(mc_params); if (UCC_OK != status) { ucc_debug("mc_init failed for component: %s, skipping (%d)", mc->super.name, status); ucc_config_parser_release_opts(mc->config, mc->config_table.table); ucc_free(mc->config); continue; } ucc_debug("mc %s initialized", mc->super.name); } else { attr.field_mask = UCC_MC_ATTR_FIELD_THREAD_MODE; status = mc->get_attr(&attr); if (status != UCC_OK) { return status; } if (attr.thread_mode < mc_params->thread_mode) { ucc_info("mc %s was allready initilized with " "different thread mode: current tm %d, provided tm %d", mc->super.name, attr.thread_mode, mc_params->thread_mode); } } mc->ref_cnt++; mc_ops[mc->type] = &mc->ops; } return UCC_OK; } ucc_status_t ucc_mc_available(ucc_memory_type_t mem_type) { mem_type = (mem_type == UCC_MEMORY_TYPE_CUDA_MANAGED) ? UCC_MEMORY_TYPE_CUDA : mem_type; if (NULL == mc_ops[mem_type]) { return UCC_ERR_NOT_FOUND; } return UCC_OK; } ucc_status_t ucc_mc_get_mem_attr(const void *ptr, ucc_mem_attr_t *mem_attr) { ucc_status_t status; ucc_memory_type_t mt; mem_attr->mem_type = UCC_MEMORY_TYPE_HOST; mem_attr->base_address = (void *)ptr; if (ptr == NULL) { mem_attr->alloc_length = 0; return UCC_OK; } mt = (ucc_memory_type_t)(UCC_MEMORY_TYPE_HOST + 1); for (; mt < UCC_MEMORY_TYPE_LAST; mt++) { if (NULL != mc_ops[mt]) { status = mc_ops[mt]->mem_query(ptr, mem_attr); if (UCC_OK == status) { return UCC_OK; } } } return UCC_OK; } ucc_status_t ucc_mc_get_attr(ucc_mc_attr_t *attr, ucc_memory_type_t mem_type) { ucc_memory_type_t mt = (mem_type == UCC_MEMORY_TYPE_CUDA_MANAGED) ? UCC_MEMORY_TYPE_CUDA : mem_type; ucc_mc_base_t *mc; UCC_CHECK_MC_AVAILABLE(mt); mc = ucc_container_of(mc_ops[mt], ucc_mc_base_t, ops); return mc->get_attr(attr); } /* TODO: add the flexbility to bypass the mpool if the user asks for it */ UCC_MC_PROFILE_FUNC(ucc_status_t, ucc_mc_alloc, (h_ptr, size, mem_type), ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mem_type) { ucc_memory_type_t mt = (mem_type == UCC_MEMORY_TYPE_CUDA_MANAGED) ? UCC_MEMORY_TYPE_CUDA : mem_type; UCC_CHECK_MC_AVAILABLE(mt); return mc_ops[mt]->mem_alloc(h_ptr, size, mem_type); } ucc_status_t ucc_mc_free(ucc_mc_buffer_header_t *h_ptr) { ucc_memory_type_t mt = (h_ptr->mt == UCC_MEMORY_TYPE_CUDA_MANAGED) ? UCC_MEMORY_TYPE_CUDA : h_ptr->mt; UCC_CHECK_MC_AVAILABLE(mt); return mc_ops[mt]->mem_free(h_ptr); } UCC_MC_PROFILE_FUNC(ucc_status_t, ucc_mc_memcpy, (dst, src, len, dst_mem, src_mem), void *dst, const void *src, size_t len, ucc_memory_type_t dst_mem, ucc_memory_type_t src_mem) { ucc_memory_type_t mt; if (src_mem == UCC_MEMORY_TYPE_UNKNOWN || dst_mem == UCC_MEMORY_TYPE_UNKNOWN) { return UCC_ERR_INVALID_PARAM; } else if (src_mem == UCC_MEMORY_TYPE_HOST && dst_mem == UCC_MEMORY_TYPE_HOST) { UCC_CHECK_MC_AVAILABLE(UCC_MEMORY_TYPE_HOST); return mc_ops[UCC_MEMORY_TYPE_HOST]->memcpy(dst, src, len, UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_HOST); } /* take any non host MC component */ mt = (dst_mem == UCC_MEMORY_TYPE_HOST) ? src_mem : dst_mem; mt = (mt == UCC_MEMORY_TYPE_CUDA_MANAGED) ? UCC_MEMORY_TYPE_CUDA : mt; UCC_CHECK_MC_AVAILABLE(mt); return mc_ops[mt]->memcpy(dst, src, len, dst_mem, src_mem); } ucc_status_t ucc_mc_memset(void *ptr, int value, size_t size, ucc_memory_type_t mem_type) { mem_type = (mem_type == UCC_MEMORY_TYPE_CUDA_MANAGED) ? UCC_MEMORY_TYPE_CUDA : mem_type; UCC_CHECK_MC_AVAILABLE(mem_type); return mc_ops[mem_type]->memset(ptr, value, size); } ucc_status_t ucc_mc_flush(ucc_memory_type_t mem_type) { mem_type = (mem_type == UCC_MEMORY_TYPE_CUDA_MANAGED) ? UCC_MEMORY_TYPE_CUDA : mem_type; UCC_CHECK_MC_AVAILABLE(mem_type); if (mc_ops[mem_type]->flush) { return mc_ops[mem_type]->flush(); } return UCC_OK; } ucc_status_t ucc_mc_finalize() { ucc_memory_type_t mt; ucc_mc_base_t *mc; for (mt = UCC_MEMORY_TYPE_HOST; mt < UCC_MEMORY_TYPE_LAST; mt++) { if (NULL != mc_ops[mt]) { mc = ucc_container_of(mc_ops[mt], ucc_mc_base_t, ops); mc->ref_cnt--; if (mc->ref_cnt == 0) { mc->finalize(); ucc_config_parser_release_opts(mc->config, mc->config_table.table); ucc_free(mc->config); mc_ops[mt] = NULL; } } } return UCC_OK; } ucc-1.8.0/src/components/mc/cpu/0000775000175000017500000000000015211535620016620 5ustar alastairalastairucc-1.8.0/src/components/mc/cpu/mc_cpu.c0000664000175000017500000001775315211535620020247 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "mc_cpu.h" #include "utils/ucc_malloc.h" #include "utils/ucc_math.h" #include "utils/arch/cpu.h" #include static ucc_config_field_t ucc_mc_cpu_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_mc_cpu_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_mc_config_table)}, {"MPOOL_ELEM_SIZE", "1Mb", "The size of each element in mc cpu mpool", ucc_offsetof(ucc_mc_cpu_config_t, mpool_elem_size), UCC_CONFIG_TYPE_MEMUNITS}, {"MPOOL_MAX_ELEMS", "8", "The max amount of elements in mc cpu mpool", ucc_offsetof(ucc_mc_cpu_config_t, mpool_max_elems), UCC_CONFIG_TYPE_UINT}, {NULL} }; static ucc_status_t ucc_mc_cpu_init(const ucc_mc_params_t *mc_params) { ucc_strncpy_safe(ucc_mc_cpu.super.config->log_component.name, ucc_mc_cpu.super.super.name, sizeof(ucc_mc_cpu.super.config->log_component.name)); ucc_mc_cpu.thread_mode = mc_params->thread_mode; // lock assures single mpool initiation when multiple threads concurrently execute // different collective operations thus concurrently entering init function. ucc_spinlock_init(&ucc_mc_cpu.mpool_init_spinlock, 0); return UCC_OK; } static ucc_status_t ucc_mc_cpu_get_attr(ucc_mc_attr_t *mc_attr) { if (mc_attr->field_mask & UCC_MC_ATTR_FIELD_THREAD_MODE) { mc_attr->thread_mode = ucc_mc_cpu.thread_mode; } return UCC_OK; } static ucc_status_t ucc_mc_cpu_mem_alloc(ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mt) { size_t size_with_h = size + sizeof(ucc_mc_buffer_header_t); ucc_mc_buffer_header_t *h = (ucc_mc_buffer_header_t *)ucc_malloc(size_with_h, "mc cpu"); if (ucc_unlikely(!h)) { mc_error(&ucc_mc_cpu.super, "failed to allocate %zd bytes", size_with_h); return UCC_ERR_NO_MEMORY; } h->from_pool = 0; h->addr = PTR_OFFSET(h, sizeof(ucc_mc_buffer_header_t)); h->mt = mt; *h_ptr = h; mc_trace(&ucc_mc_cpu.super, "allocated %ld bytes with ucc_malloc", size); return UCC_OK; } static ucc_status_t ucc_mc_cpu_mem_pool_alloc(ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mt) { ucc_mc_buffer_header_t *h = NULL; if (size <= MC_CPU_CONFIG->mpool_elem_size) { h = (ucc_mc_buffer_header_t *)ucc_mpool_get(&ucc_mc_cpu.mpool); } if (!h) { // Slow path return ucc_mc_cpu_mem_alloc(h_ptr, size, mt); } mc_trace(&ucc_mc_cpu.super, "allocated %ld bytes from cpu mpool", size); *h_ptr = h; return UCC_OK; } static ucc_status_t ucc_mc_cpu_chunk_alloc(ucc_mpool_t *mp, //NOLINT size_t *size_p, void **chunk_p) { *chunk_p = ucc_malloc(*size_p, "mc cpu"); if (!*chunk_p) { mc_error(&ucc_mc_cpu.super, "failed to allocate %zd bytes", *size_p); return UCC_ERR_NO_MEMORY; } return UCC_OK; } static void ucc_mc_cpu_chunk_init(ucc_mpool_t *mp, //NOLINT void *obj, void *chunk) //NOLINT { ucc_mc_buffer_header_t *h = (ucc_mc_buffer_header_t *)obj; h->from_pool = 1; h->addr = PTR_OFFSET(h, sizeof(ucc_mc_buffer_header_t)); h->mt = UCC_MEMORY_TYPE_HOST; } static void ucc_mc_cpu_chunk_release(ucc_mpool_t *mp, void *chunk) //NOLINT { ucc_free(chunk); } static ucc_mpool_ops_t ucc_mc_ops = {.chunk_alloc = ucc_mc_cpu_chunk_alloc, .chunk_release = ucc_mc_cpu_chunk_release, .obj_init = ucc_mc_cpu_chunk_init, .obj_cleanup = NULL}; static ucc_status_t ucc_mc_cpu_mem_free(ucc_mc_buffer_header_t *h_ptr) { ucc_free(h_ptr); return UCC_OK; } static ucc_status_t ucc_mc_cpu_mem_pool_free(ucc_mc_buffer_header_t *h_ptr) { if (!h_ptr->from_pool) { return ucc_mc_cpu_mem_free(h_ptr); } ucc_mpool_put(h_ptr); return UCC_OK; } static ucc_status_t ucc_mc_cpu_mem_pool_alloc_with_init(ucc_mc_buffer_header_t **h_ptr, size_t size, ucc_memory_type_t mt) { // lock assures single mpool initiation when multiple threads concurrently // execute different collective operations each entering init function. ucc_spin_lock(&ucc_mc_cpu.mpool_init_spinlock); if (MC_CPU_CONFIG->mpool_max_elems == 0) { ucc_mc_cpu.super.ops.mem_alloc = ucc_mc_cpu_mem_alloc; ucc_mc_cpu.super.ops.mem_free = ucc_mc_cpu_mem_free; ucc_spin_unlock(&ucc_mc_cpu.mpool_init_spinlock); return ucc_mc_cpu_mem_alloc(h_ptr, size, mt); } if (!ucc_mc_cpu.mpool_init_flag) { ucc_status_t status = ucc_mpool_init( &ucc_mc_cpu.mpool, 0, sizeof(ucc_mc_buffer_header_t) + MC_CPU_CONFIG->mpool_elem_size, 0, UCC_CACHE_LINE_SIZE, 1, MC_CPU_CONFIG->mpool_max_elems, &ucc_mc_ops, ucc_mc_cpu.thread_mode, "mc cpu mpool buffers"); if (ucc_unlikely(status != UCC_OK)) { ucc_spin_unlock(&ucc_mc_cpu.mpool_init_spinlock); return status; } ucc_mc_cpu.super.ops.mem_alloc = ucc_mc_cpu_mem_pool_alloc; ucc_mc_cpu.mpool_init_flag = 1; } ucc_spin_unlock(&ucc_mc_cpu.mpool_init_spinlock); return ucc_mc_cpu_mem_pool_alloc(h_ptr, size, mt); } static ucc_status_t ucc_mc_cpu_memcpy(void *dst, const void *src, size_t len, ucc_memory_type_t dst_mem, //NOLINT ucc_memory_type_t src_mem) //NOLINT { ucc_assert((dst_mem == UCC_MEMORY_TYPE_HOST) && (src_mem == UCC_MEMORY_TYPE_HOST)); memcpy(dst, src, len); return UCC_OK; } static ucc_status_t ucc_mc_cpu_memset(void *ptr, int value, size_t len) { memset(ptr, value, len); return UCC_OK; } static ucc_status_t ucc_mc_cpu_mem_query(const void *ptr, //NOLINT ucc_mem_attr_t *mem_attr) //NOLINT { /* not supposed to be used */ mc_error(&ucc_mc_cpu.super, "host memory component shouldn't be used for" "mem type detection"); return UCC_ERR_NOT_SUPPORTED; } static ucc_status_t ucc_mc_cpu_finalize() { if (ucc_mc_cpu.mpool_init_flag) { ucc_mpool_cleanup(&ucc_mc_cpu.mpool, 1); ucc_mc_cpu.mpool_init_flag = 0; ucc_mc_cpu.super.ops.mem_alloc = ucc_mc_cpu_mem_pool_alloc_with_init; } ucc_spinlock_destroy(&ucc_mc_cpu.mpool_init_spinlock); return UCC_OK; } ucc_mc_cpu_t ucc_mc_cpu = { .super.super.name = "cpu mc", .super.ref_cnt = 0, .super.type = UCC_MEMORY_TYPE_HOST, .super.ee_type = UCC_EE_CPU_THREAD, .super.init = ucc_mc_cpu_init, .super.get_attr = ucc_mc_cpu_get_attr, .super.finalize = ucc_mc_cpu_finalize, .super.ops.mem_query = ucc_mc_cpu_mem_query, .super.ops.mem_alloc = ucc_mc_cpu_mem_pool_alloc_with_init, .super.ops.mem_free = ucc_mc_cpu_mem_pool_free, .super.ops.memcpy = ucc_mc_cpu_memcpy, .super.ops.memset = ucc_mc_cpu_memset, .super.ops.flush = NULL, .super.config_table = { .name = "CPU memory component", .prefix = "MC_CPU_", .table = ucc_mc_cpu_config_table, .size = sizeof(ucc_mc_cpu_config_t), }, .mpool_init_flag = 0, }; UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_mc_cpu.super.config_table, &ucc_config_global_list); ucc-1.8.0/src/components/mc/cpu/Makefile.am0000664000175000017500000000074015211535620020655 0ustar alastairalastair# # Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # sources = \ mc_cpu.h \ mc_cpu.c module_LTLIBRARIES = libucc_mc_cpu.la libucc_mc_cpu_la_SOURCES = $(sources) libucc_mc_cpu_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) libucc_mc_cpu_la_CFLAGS = $(BASE_CFLAGS) libucc_mc_cpu_la_LDFLAGS = -version-info $(SOVERSION) --as-needed libucc_mc_cpu_la_LIBADD = $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am ucc-1.8.0/src/components/mc/cpu/mc_cpu.h0000664000175000017500000000146415211535620020244 0ustar alastairalastair/** * Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_MC_CPU_H_ #define UCC_MC_CPU_H_ #include "components/mc/base/ucc_mc_base.h" #include "components/mc/ucc_mc_log.h" typedef struct ucc_mc_cpu_config { ucc_mc_config_t super; size_t mpool_elem_size; int mpool_max_elems; } ucc_mc_cpu_config_t; typedef struct ucc_mc_cpu { ucc_mc_base_t super; ucc_mpool_t mpool; int mpool_init_flag; ucc_spinlock_t mpool_init_spinlock; ucc_thread_mode_t thread_mode; } ucc_mc_cpu_t; extern ucc_mc_cpu_t ucc_mc_cpu; #define MC_CPU_CONFIG \ (ucc_derived_of(ucc_mc_cpu.super.config, ucc_mc_cpu_config_t)) #endif ucc-1.8.0/src/components/mc/ucc_mc.h0000664000175000017500000000240515211535620017434 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_MC_H_ #define UCC_MC_H_ #include "ucc/api/ucc.h" #include "components/mc/base/ucc_mc_base.h" #include "core/ucc_dt.h" #include "utils/ucc_math.h" ucc_status_t ucc_mc_init(const ucc_mc_params_t *mc_params); ucc_status_t ucc_mc_finalize(); ucc_status_t ucc_mc_available(ucc_memory_type_t mem_type); /** * Query for memory attributes. * @param [in] ptr Memory pointer to query. * @param [in,out] mem_attr Memory attributes. */ ucc_status_t ucc_mc_get_mem_attr(const void *ptr, ucc_mem_attr_t *mem_attr); ucc_status_t ucc_mc_get_attr(ucc_mc_attr_t *attr, ucc_memory_type_t mem_type); ucc_status_t ucc_mc_alloc(ucc_mc_buffer_header_t **h_ptr, size_t len, ucc_memory_type_t mem_type); ucc_status_t ucc_mc_free(ucc_mc_buffer_header_t *h_ptr); ucc_status_t ucc_mc_flush(ucc_memory_type_t mem_type); ucc_status_t ucc_mc_memcpy(void *dst, const void *src, size_t len, ucc_memory_type_t dst_mem, ucc_memory_type_t src_mem); ucc_status_t ucc_mc_memset(void *ptr, int value, size_t size, ucc_memory_type_t mem_type); #endif ucc-1.8.0/src/components/topo/0000775000175000017500000000000015211535620016413 5ustar alastairalastairucc-1.8.0/src/components/topo/ucc_sbgp.h0000664000175000017500000001152415211535620020354 0ustar alastairalastair/** * Copyright (c) 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_SBGP_H_ #define UCC_SBGP_H_ #include "ucc/api/ucc.h" #include "utils/ucc_datastruct.h" typedef enum ucc_sbgp_type_t { UCC_SBGP_NUMA, /* Group of ranks on the same NUMA domain. This group does not exist if processes are not bound to a single NUMA node. */ UCC_SBGP_SOCKET, /* Group of ranks on the same SOCKET. This group does not exist if processes are not bound to a single SOCKET node. */ UCC_SBGP_NODE, /* Group of ranks on the same NODE. */ UCC_SBGP_NODE_NVLINK, /* Group of ranks on the same NODE that share the same GPU or NVLink-connected GPUs. */ UCC_SBGP_NODE_LEADERS, /* Group of ranks with local_node_rank = 0. This group EXISTS when team spans at least 2 nodes. This group is ENABLED for procs with local_node_rank = 0. This group is DISABLED but EXISTS for procs with local_node_rank != 0*/ UCC_SBGP_NET, /* Group of ranks with the same local_node_rank. This group EXISTS when team spans at least 2 nodes AND the team has equal PPN across all the nodes. If EXISTS this group is ENABLED for all procs. */ UCC_SBGP_SOCKET_LEADERS, /* Group of ranks with local_socket_rank = 0. This group EXISTS when team spans at least 2 sockets. This group is ENABLED for procs with local_socket_rank = 0. This group is DISABLED but EXISTS for procs with local_socket_rank != 0 */ UCC_SBGP_NUMA_LEADERS, /* Same as SOCKET_LEADERS but for NUMA grouping */ UCC_SBGP_FULL, /* Group contains ALL the ranks of the team */ UCC_SBGP_FULL_HOST_ORDERED, /* Group contains ALL the ranks of the team ordered by host, socket, numa */ UCC_SBGP_LAST } ucc_sbgp_type_t; typedef enum ucc_sbgp_status_t { UCC_SBGP_NOT_INIT, UCC_SBGP_DISABLED, UCC_SBGP_ENABLED, UCC_SBGP_NOT_EXISTS, } ucc_sbgp_status_t; typedef struct ucc_topo ucc_topo_t; typedef struct ucc_sbgp_t { ucc_sbgp_type_t type; ucc_sbgp_status_t status; ucc_rank_t group_size; ucc_rank_t group_rank; ucc_rank_t *rank_map; ucc_ep_map_t map; } ucc_sbgp_t; const char* ucc_sbgp_str(ucc_sbgp_type_t type); /* The call creates a required subgroup specified by @in type in the topo->sbgps[type]. The created sbgp can be in either of 3 states: - NOT_EXISTS: means for a given topo (ucc team layout) there is no such grouping or the result group is of size 1. Example: 1) type == SBGP_SOCKET but processes are not bound to sockets at all; 2) type == SBGP_NODE_LEADERS but team is entirely on single node. - ENABLED: means the sbgp size >= 2 and calling process is part of that subgroup - DISABLED: means the subgrouping exists for the given ucc team (topo) but the calling process is NOT part of it. Note: this function returns subgroup LOCAL to calling process when multiple groups of the same time exist for a given topo. E.g., when team spans several sockets there exist several socket subgroups but the function below will initialize the sbgp which belongs to the calling process. */ ucc_status_t ucc_sbgp_create(ucc_topo_t *topo, ucc_sbgp_type_t type); ucc_status_t ucc_sbgp_cleanup(ucc_sbgp_t *sbgp); /* Returns ALL existing socket subgroups on the node of the calling process. If processes are not bound UCC_ERR_NOT_FOUND is returned. Also returns subgroups of size 1 in contrast to ucc_sbgp_create. */ ucc_status_t ucc_sbgp_create_all_sockets(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps); ucc_status_t ucc_sbgp_create_all_numas(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps); ucc_status_t ucc_sbgp_create_all_node_nvlinks(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps); ucc_status_t ucc_sbgp_create_node(ucc_topo_t *topo, ucc_sbgp_t *sbgp); static inline ucc_subset_t ucc_sbgp_to_subset(ucc_sbgp_t *sbgp) { ucc_subset_t s = { .map = sbgp->map, .myrank = sbgp->group_rank }; return s; } void ucc_sbgp_print(ucc_sbgp_t *sbgp); #endif ucc-1.8.0/src/components/topo/cuda/0000775000175000017500000000000015211535620017327 5ustar alastairalastairucc-1.8.0/src/components/topo/cuda/ucc_sysinfo_cuda.c0000664000175000017500000004437615211535620023031 0ustar alastairalastair#include "config.h" #include "ucc_sysinfo_cuda.h" #include "components/topo/ucc_sysinfo.h" #include "core/ucc_global_opts.h" #include "utils/debug/log_def.h" #include #include #ifdef HAVE_NVML_H #include "utils/ucc_malloc.h" #include "utils/ucc_math.h" #include #include #include #endif static ucc_config_field_t ucc_sysinfo_cuda_config_table[] = { {NULL} }; #ifdef HAVE_NVML_H static pthread_mutex_t ucc_sysinfo_cuda_nvml_lock = PTHREAD_MUTEX_INITIALIZER; #endif static ucc_status_t ucc_sysinfo_cuda_init(const ucc_sysinfo_params_t *params) { (void)params; return UCC_OK; } static ucc_status_t ucc_sysinfo_cuda_finalize() { return UCC_OK; } static ucc_status_t ucc_sysinfo_cuda_set_visible_devices( void *info, int n_info, const ucc_config_names_array_t *devices, uint32_t *visible_devices) { const ucc_gpu_info_t *gpus; CUcontext ctx = NULL; CUdevice cu_dev; CUresult cu_st; char pci_bus_id[32]; unsigned int domain; unsigned int bus; unsigned int pci_device; unsigned int function; int n; int i; if (!visible_devices || (n_info > 0 && !info)) { return UCC_ERR_INVALID_PARAM; } if (devices) { return UCC_ERR_NOT_SUPPORTED; } *visible_devices = 0; if (n_info <= 0) { return UCC_OK; } cu_st = cuCtxGetCurrent(&ctx); if (cu_st != CUDA_SUCCESS || ctx == NULL) { if (cu_st != CUDA_SUCCESS) { const char *cu_err_str = NULL; cuGetErrorString(cu_st, &cu_err_str); ucc_debug("cuCtxGetCurrent failed: %d (%s)", cu_st, cu_err_str ? cu_err_str : "unknown"); } return UCC_OK; } cu_st = cuCtxGetDevice(&cu_dev); if (cu_st != CUDA_SUCCESS) { const char *cu_err_str = NULL; cuGetErrorString(cu_st, &cu_err_str); ucc_debug("cuCtxGetDevice failed: %d (%s)", cu_st, cu_err_str ? cu_err_str : "unknown"); return UCC_ERR_NO_MESSAGE; } cu_st = cuDeviceGetPCIBusId(pci_bus_id, sizeof(pci_bus_id), cu_dev); if (cu_st != CUDA_SUCCESS) { const char *cu_err_str = NULL; cuGetErrorString(cu_st, &cu_err_str); ucc_debug("cuDeviceGetPCIBusId failed: %d (%s)", cu_st, cu_err_str ? cu_err_str : "unknown"); return UCC_ERR_NO_MESSAGE; } n = sscanf(pci_bus_id, "%x:%x:%x.%x", &domain, &bus, &pci_device, &function); if (n < 3) { return UCC_ERR_INVALID_PARAM; } if (n < 4) { function = 0; } gpus = (const ucc_gpu_info_t *)info; for (i = 0; i < n_info && i < (int)(sizeof(*visible_devices) * 8); i++) { if (gpus[i].pci.domain == (uint16_t)domain && gpus[i].pci.bus == (uint8_t)bus && gpus[i].pci.device == (uint8_t)pci_device && gpus[i].pci.function == (uint8_t)function) { *visible_devices = (1u << i); break; } } return UCC_OK; } #ifdef HAVE_NVML_H static void ucc_sysinfo_cuda_fill_pci_info(const nvmlPciInfo_t *nvml_pci, ucc_gpu_info_t *gpu) { unsigned int domain; unsigned int bus; unsigned int device; unsigned int function; int n; n = sscanf(nvml_pci->busId, "%x:%x:%x.%x", &domain, &bus, &device, &function); if (n == 4) { gpu->pci.domain = (uint16_t)domain; gpu->pci.bus = (uint8_t)bus; gpu->pci.device = (uint8_t)device; gpu->pci.function = (uint8_t)function; } else { gpu->pci.domain = (uint16_t)nvml_pci->domain; gpu->pci.bus = (uint8_t)nvml_pci->bus; gpu->pci.device = (uint8_t)nvml_pci->device; gpu->pci.function = 0; } gpu->pci.host_id = ucc_local_proc.host_id; } typedef struct ucc_sysinfo_pci_id { uint16_t domain; uint8_t bus; uint8_t device; uint8_t function; } ucc_sysinfo_pci_id_t; typedef struct ucc_sysinfo_nvlink_switch { ucc_sysinfo_pci_id_t pci; int gpu_links[UCC_MAX_HOST_GPUS]; } ucc_sysinfo_nvlink_switch_t; static ucc_sysinfo_pci_id_t ucc_sysinfo_cuda_pci_id_from_nvml(const nvmlPciInfo_t *nvml_pci) { ucc_sysinfo_pci_id_t pci; unsigned int domain; unsigned int bus; unsigned int device; unsigned int function; int n; n = sscanf(nvml_pci->busId, "%x:%x:%x.%x", &domain, &bus, &device, &function); if (n == 4) { pci.domain = (uint16_t)domain; pci.bus = (uint8_t)bus; pci.device = (uint8_t)device; pci.function = (uint8_t)function; } else { pci.domain = (uint16_t)nvml_pci->domain; pci.bus = (uint8_t)nvml_pci->bus; pci.device = (uint8_t)nvml_pci->device; pci.function = 0; } return pci; } static int ucc_sysinfo_cuda_pci_id_equal(const ucc_sysinfo_pci_id_t *a, const ucc_sysinfo_pci_id_t *b) { return (a->domain == b->domain) && (a->bus == b->bus) && (a->device == b->device) && (a->function == b->function); } static int ucc_sysinfo_cuda_find_gpu_by_pci(const nvmlPciInfo_t *nvml_pci, const ucc_gpu_info_t *gpus, int n_gpus) { unsigned int domain; unsigned int bus; unsigned int device; unsigned int function; int n; int i; n = sscanf(nvml_pci->busId, "%x:%x:%x.%x", &domain, &bus, &device, &function); if (n != 4) { domain = nvml_pci->domain; bus = nvml_pci->bus; device = nvml_pci->device; function = 0; } for (i = 0; i < n_gpus; i++) { if (gpus[i].pci.domain == (uint16_t)domain && gpus[i].pci.bus == (uint8_t)bus && gpus[i].pci.device == (uint8_t)device && gpus[i].pci.function == (uint8_t)function) { return i; } } return -1; } static ucc_status_t ucc_sysinfo_cuda_build_nvlink_matrix( ucc_gpu_info_t *gpus, int n_gpus, uint8_t nvlink_matrix[UCC_MAX_HOST_GPUS][UCC_MAX_HOST_GPUS]) { nvmlDevice_t nvml_dev; nvmlFieldValue_t nvml_value; nvmlPciInfo_t nvml_pci; nvmlReturn_t nvml_st; unsigned int num_nvlinks; ucc_sysinfo_nvlink_switch_t *switches; int switches_cap = 8; int n_switches = 0; int i, link, peer_gpu; memset(nvlink_matrix, 0, UCC_MAX_HOST_GPUS * UCC_MAX_HOST_GPUS); switches = ucc_calloc(switches_cap, sizeof(*switches), "sysinfo_cuda_nvlink_switches"); if (!switches) { return UCC_ERR_NO_MEMORY; } nvml_value.fieldId = NVML_FI_DEV_NVLINK_LINK_COUNT; for (i = 0; i < n_gpus; i++) { nvml_st = nvmlDeviceGetHandleByIndex(i, &nvml_dev); if (nvml_st != NVML_SUCCESS) { continue; } nvml_st = nvmlDeviceGetFieldValues(nvml_dev, 1, &nvml_value); num_nvlinks = ((nvml_st == NVML_SUCCESS) && (nvml_value.nvmlReturn == NVML_SUCCESS) && (nvml_value.valueType == NVML_VALUE_TYPE_UNSIGNED_INT)) ? nvml_value.value.uiVal : 0; for (link = 0; link < (int)num_nvlinks; link++) { int is_switch = 0; nvml_st = nvmlDeviceGetNvLinkRemotePciInfo_v2(nvml_dev, link, &nvml_pci); if (nvml_st != NVML_SUCCESS) { continue; } #if HAVE_NVML_REMOTE_DEVICE_TYPE nvmlIntNvLinkDeviceType_t nvml_dt; nvml_st = nvmlDeviceGetNvLinkRemoteDeviceType(nvml_dev, link, &nvml_dt); if (nvml_st == NVML_SUCCESS && nvml_dt == NVML_NVLINK_DEVICE_TYPE_SWITCH) { is_switch = 1; } #else nvmlDevice_t remote_dev; nvml_st = nvmlDeviceGetHandleByPciBusId_v2(nvml_pci.busId, &remote_dev); if (nvml_st == NVML_ERROR_NOT_FOUND) { is_switch = 1; } else if (nvml_st != NVML_SUCCESS) { continue; } #endif if (is_switch) { ucc_sysinfo_pci_id_t pci = ucc_sysinfo_cuda_pci_id_from_nvml( &nvml_pci); int idx = -1; int s; for (s = 0; s < n_switches; s++) { if (ucc_sysinfo_cuda_pci_id_equal(&switches[s].pci, &pci)) { idx = s; break; } } if (idx == -1) { if (n_switches == switches_cap) { int new_cap = switches_cap * 2; ucc_sysinfo_nvlink_switch_t *tmp; tmp = ucc_realloc(switches, new_cap * sizeof(*switches), "sysinfo_cuda_nvlink_switches"); if (!tmp) { ucc_free(switches); return UCC_ERR_NO_MEMORY; } memset(tmp + switches_cap, 0, (new_cap - switches_cap) * sizeof(*switches)); switches = tmp; switches_cap = new_cap; } idx = n_switches++; switches[idx].pci = pci; } switches[idx].gpu_links[i]++; continue; } peer_gpu = ucc_sysinfo_cuda_find_gpu_by_pci(&nvml_pci, gpus, n_gpus); if (peer_gpu >= 0 && peer_gpu < n_gpus) { if (peer_gpu != i && i < peer_gpu) { nvlink_matrix[i][peer_gpu]++; nvlink_matrix[peer_gpu][i]++; } } } } for (i = 0; i < n_switches; i++) { int g1, g2; for (g1 = 0; g1 < n_gpus; g1++) { if (switches[i].gpu_links[g1] == 0) { continue; } for (g2 = 0; g2 < n_gpus; g2++) { if (g1 == g2 || switches[i].gpu_links[g2] == 0) { continue; } nvlink_matrix[g1][g2] += ucc_min(switches[i].gpu_links[g1], switches[i].gpu_links[g2]); } } } ucc_free(switches); return UCC_OK; } static unsigned int ucc_sysinfo_cuda_get_nvlink_count(nvmlDevice_t dev) { nvmlFieldValue_t nvml_value; nvmlReturn_t nvml_st; nvml_value.fieldId = NVML_FI_DEV_NVLINK_LINK_COUNT; nvml_st = nvmlDeviceGetFieldValues(dev, 1, &nvml_value); if ((nvml_st == NVML_SUCCESS) && (nvml_value.nvmlReturn == NVML_SUCCESS) && (nvml_value.valueType == NVML_VALUE_TYPE_UNSIGNED_INT)) { return nvml_value.value.uiVal; } return 0; } static ucc_status_t ucc_sysinfo_cuda_get_nvswitch_connected(nvmlDevice_t dev, unsigned int num_nvlinks, int *connected) { nvmlReturn_t nvml_st; int link; *connected = 0; for (link = 0; link < (int)num_nvlinks; link++) { #if HAVE_NVML_REMOTE_DEVICE_TYPE nvmlIntNvLinkDeviceType_t nvml_dt; nvml_st = nvmlDeviceGetNvLinkRemoteDeviceType(dev, link, &nvml_dt); if (nvml_st != NVML_SUCCESS) { continue; } if (nvml_dt == NVML_NVLINK_DEVICE_TYPE_SWITCH) { *connected = 1; break; } #else nvmlPciInfo_t nvml_pci; nvmlDevice_t nvml_dev; nvml_st = nvmlDeviceGetNvLinkRemotePciInfo_v2(dev, link, &nvml_pci); if (nvml_st != NVML_SUCCESS) { continue; } nvml_st = nvmlDeviceGetHandleByPciBusId_v2(nvml_pci.busId, &nvml_dev); if (nvml_st == NVML_ERROR_NOT_FOUND) { *connected = 1; break; } #endif } return UCC_OK; } static ucc_status_t ucc_sysinfo_cuda_get_info(void **info, int *n_info) { ucc_sysinfo_gpu_info_t *gpu_info; unsigned int num_gpus; nvmlDevice_t nvml_dev; nvmlPciInfo_t nvml_pci; nvmlReturn_t nvml_st; ucc_status_t status; int n_gpus; int i; *info = NULL; *n_info = 0; pthread_mutex_lock(&ucc_sysinfo_cuda_nvml_lock); nvml_st = nvmlInit_v2(); if (nvml_st != NVML_SUCCESS) { ucc_debug("failed to init NVML: %s", nvmlErrorString(nvml_st)); pthread_mutex_unlock(&ucc_sysinfo_cuda_nvml_lock); return UCC_ERR_NO_RESOURCE; } nvml_st = nvmlDeviceGetCount(&num_gpus); if (nvml_st != NVML_SUCCESS) { ucc_debug("nvmlDeviceGetCount failed: %s", nvmlErrorString(nvml_st)); status = UCC_ERR_NO_RESOURCE; goto exit_nvml_shutdown; } if (num_gpus == 0) { status = UCC_OK; goto exit_nvml_shutdown; } n_gpus = ucc_min((int)num_gpus, UCC_MAX_HOST_GPUS); gpu_info = (ucc_sysinfo_gpu_info_t *)ucc_malloc(sizeof(*gpu_info), "sysinfo_cuda_gpu_info"); if (!gpu_info) { ucc_error("failed to allocate %zd bytes for gpu info", sizeof(*gpu_info)); status = UCC_ERR_NO_MEMORY; goto exit_nvml_shutdown; } memset(gpu_info, 0, sizeof(*gpu_info)); gpu_info->n_gpus = n_gpus; for (i = 0; i < n_gpus; i++) { char uuid_str[NVML_DEVICE_UUID_BUFFER_SIZE]; unsigned int num_nvlinks; nvml_st = nvmlDeviceGetHandleByIndex(i, &nvml_dev); if (nvml_st != NVML_SUCCESS) { ucc_debug("nvmlDeviceGetHandleByIndex failed: %s", nvmlErrorString(nvml_st)); status = UCC_ERR_NO_MESSAGE; goto free_gpu_info; } nvml_st = nvmlDeviceGetPciInfo(nvml_dev, &nvml_pci); if (nvml_st != NVML_SUCCESS) { ucc_debug("nvmlDeviceGetPciInfo failed: %s", nvmlErrorString(nvml_st)); status = UCC_ERR_NO_MESSAGE; goto free_gpu_info; } ucc_sysinfo_cuda_fill_pci_info(&nvml_pci, &gpu_info->gpus[i]); nvml_st = nvmlDeviceGetUUID(nvml_dev, uuid_str, sizeof(uuid_str)); if (nvml_st == NVML_SUCCESS) { gpu_info->gpus[i].uuid = (uint64_t)ucc_str_hash_djb2(uuid_str); } else { gpu_info->gpus[i].uuid = 0; } gpu_info->gpus[i].caps = 0; gpu_info->gpus[i].fabric_clique_id = UCC_GPU_FABRIC_CLIQUE_ID_INVALID; gpu_info->gpus[i].fabric_partition_id = UCC_GPU_FABRIC_PARTITION_ID_INVALID; num_nvlinks = ucc_sysinfo_cuda_get_nvlink_count(nvml_dev); if (num_nvlinks > 0) { int nvswitch = 0; gpu_info->gpus[i].caps |= UCC_GPU_CAP_NVLINK; status = ucc_sysinfo_cuda_get_nvswitch_connected( nvml_dev, num_nvlinks, &nvswitch); if (status != UCC_OK) { goto free_gpu_info; } if (nvswitch) { gpu_info->gpus[i].caps |= UCC_GPU_CAP_NVSWITCH; } } #if defined(HAVE_NVML_GPU_FABRIC_INFO_V) || defined(HAVE_NVML_GPU_FABRIC_INFO) { #ifdef HAVE_NVML_GPU_FABRIC_INFO_V nvmlGpuFabricInfoV_t fabric_info; fabric_info.version = nvmlGpuFabricInfo_v2; nvml_st = nvmlDeviceGetGpuFabricInfoV(nvml_dev, &fabric_info); #else nvmlGpuFabricInfo_t fabric_info; nvml_st = nvmlDeviceGetGpuFabricInfo(nvml_dev, &fabric_info); #endif if (nvml_st == NVML_SUCCESS && fabric_info.state == NVML_GPU_FABRIC_STATE_COMPLETED) { gpu_info->gpus[i].caps |= UCC_GPU_CAP_FABRIC; gpu_info->gpus[i].fabric_clique_id = fabric_info.cliqueId; #if defined(HAVE_NVML_GPU_FABRIC_INFO_V) && defined(HAVE_NVML_FABRIC_PARTITION_ID) gpu_info->gpus[i].fabric_partition_id = fabric_info.partitionId; #endif } } #endif ucc_debug("GPU %d: caps=0x%x clique=%llu partition=%u", i, gpu_info->gpus[i].caps, (unsigned long long)gpu_info->gpus[i].fabric_clique_id, (unsigned)gpu_info->gpus[i].fabric_partition_id); } if (num_gpus > UCC_MAX_HOST_GPUS) { ucc_debug("truncating gpu info from %u to %u entries", num_gpus, (unsigned int)UCC_MAX_HOST_GPUS); } status = ucc_sysinfo_cuda_build_nvlink_matrix(gpu_info->gpus, n_gpus, gpu_info->nvlink_matrix); if (status != UCC_OK) { goto free_gpu_info; } nvmlShutdown(); pthread_mutex_unlock(&ucc_sysinfo_cuda_nvml_lock); *info = gpu_info; *n_info = n_gpus; return UCC_OK; free_gpu_info: ucc_free(gpu_info); exit_nvml_shutdown: nvmlShutdown(); pthread_mutex_unlock(&ucc_sysinfo_cuda_nvml_lock); return status; } #else static ucc_status_t ucc_sysinfo_cuda_get_info(void **info, int *n_info) { *info = NULL; *n_info = 0; return UCC_ERR_NOT_SUPPORTED; } #endif ucc_sysinfo_cuda_t ucc_sysinfo_cuda = { .super.super.name = "cuda sysinfo", .super.ref_cnt = 0, .super.type = UCC_SYSINFO_TYPE_NVLINK, .super.init = ucc_sysinfo_cuda_init, .super.finalize = ucc_sysinfo_cuda_finalize, .super.ops.get_info = ucc_sysinfo_cuda_get_info, .super.ops.set_visible_devices = ucc_sysinfo_cuda_set_visible_devices, .super.config_table = { .name = "CUDA sysinfo component", .prefix = "SYSINFO_CUDA_", .table = ucc_sysinfo_cuda_config_table, .size = sizeof(ucc_sysinfo_config_t), }, }; UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_sysinfo_cuda.super.config_table, &ucc_config_global_list);ucc-1.8.0/src/components/topo/cuda/Makefile.am0000664000175000017500000000123415211535620021363 0ustar alastairalastair# # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # if HAVE_CUDA sources = \ ucc_sysinfo_cuda.h \ ucc_sysinfo_cuda.c module_LTLIBRARIES = libucc_sysinfo_cuda.la libucc_sysinfo_cuda_la_SOURCES = $(sources) libucc_sysinfo_cuda_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) libucc_sysinfo_cuda_la_CFLAGS = $(BASE_CFLAGS) libucc_sysinfo_cuda_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(CUDA_LDFLAGS) libucc_sysinfo_cuda_la_LIBADD = $(CUDA_LIBS) $(NVML_LIBS) \ $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/topo/cuda/ucc_sysinfo_cuda.h0000664000175000017500000000055615211535620023026 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_SYSINFO_CUDA_H_ #define UCC_SYSINFO_CUDA_H_ #include "components/topo/base/ucc_sysinfo_base.h" typedef struct ucc_sysinfo_cuda { ucc_sysinfo_base_t super; } ucc_sysinfo_cuda_t; extern ucc_sysinfo_cuda_t ucc_sysinfo_cuda; #endifucc-1.8.0/src/components/topo/ucc_topo.h0000664000175000017500000003071415211535620020404 0ustar alastairalastair/* * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_TOPO_H_ #define UCC_TOPO_H_ #include "ucc_sbgp.h" #include "utils/ucc_proc_info.h" #include "utils/ucc_coll_utils.h" /* Topo data structure initialized per UCC context. key elements of this struct are the arrays of ucc_proc_info_t and ucc_host_info_t that are constructed during ucc_context_topo_init using ucc_addr_storage_t. In other words the topo struct can be initialized after the exchange of addresses is performed. */ typedef struct ucc_context_topo { ucc_proc_info_t *procs; ucc_host_info_t *hosts; ucc_rank_t n_procs; ucc_rank_t nnodes; ucc_rank_t min_ppn; /*< smallest ppn value across the nodes spanned by the group of processes defined by ucc_addr_storage_t */ ucc_rank_t max_ppn; /*< biggest ppn across the nodes */ ucc_rank_t max_n_sockets; /*< max number of different sockets on a node */ uint32_t sock_bound; /*< global flag, 1 if processes are bound to sockets */ ucc_rank_t max_n_numas; /*< max number of different numa domains on a node */ uint32_t numa_bound; /*< global flag, 1 if processes are bound to numa nodes */ } ucc_context_topo_t; typedef struct ucc_addr_storage ucc_addr_storage_t; /* Device map for a team topo. Each entry corresponds to a TEAM RANK and stores an index into ucc_host_info_t::gpus for that rank's host. Use UCC_DEVICE_ID_INVALID when the rank has no device mapping. */ typedef struct ucc_device_map { ucc_device_id_t *device_ids; ucc_rank_t n_ranks; } ucc_device_map_t; /* This topo structure is initialized over a SUBSET of processes from ucc_context_topo_t. For example, if ucc_context_t is global then address exchange is performed during ucc_context_create and we have ctx wide ucc_addr_storage_t. So, we init ucc_context_topo_t on ucc_context. Then, ucc_team is a subset of ucc_context mapped via team->ctx_map. It represents a subset of ranks and we can initialize ucc_topo_t for that subset, ie for a team. */ typedef struct ucc_topo { ucc_context_topo_t *topo; /*< Cached pointer of the ctx topo */ ucc_sbgp_t sbgps[UCC_SBGP_LAST]; /*< LOCAL sbgps initialized on demand */ ucc_sbgp_t *all_sockets; /*< array of socket sbgps, init on demand */ int n_sockets; ucc_sbgp_t *all_numas; /*< array of numa sbgps, init on demand */ int n_numas; ucc_sbgp_t *all_nodes; /*< array of node sbgps, init on demand */ int n_nodes; ucc_rank_t node_leader_rank_id; /*< defines which rank on a node will be node leader. Similar to local node rank. currently set to 0, can be selected differently in the future */ ucc_rank_t node_leader_rank; /*< actual rank of the node leader in the original (ucc_team) ranking */ ucc_rank_t *node_leaders; /*< array mapping each rank to its node leader in the original (ucc_team) ranking, initialized on demand */ ucc_subset_t set; /*< subset of procs from the ucc_context_topo. for ucc_team topo it is team->ctx_map */ ucc_rank_t min_ppn; /*< min ppn across the nodes for a team */ ucc_rank_t max_ppn; /*< max ppn across the nodes for a team */ ucc_rank_t min_socket_size; /*< min number of processes on a socket, across all nodes of a team */ ucc_rank_t max_socket_size; /*< max number of processes on a socket, across all nodes of a team */ ucc_rank_t min_numa_size; /*< min number of processes on a numa, across all nodes of a team */ ucc_rank_t max_numa_size; /*< max number of processes on a numa, across all nodes of a team */ ucc_device_map_t device_map; /*< map of team ranks to device indices */ } ucc_topo_t; /* Initializes ctx level topo structure using addr_storage. Each address contains ucc_proc_info_t which is extracted and placed into array for each participating proc. The array is then sorted (see ucc_compare_proc_info in ucc_topo.c) that allows O(N) local subgroup discoveries */ ucc_status_t ucc_context_topo_init( ucc_addr_storage_t *storage, ucc_context_topo_t **topo); void ucc_context_topo_cleanup(ucc_context_topo_t *topo); /* Initializes topo structure for a subset, e.g. for a team */ ucc_status_t ucc_topo_init( ucc_subset_t set, ucc_context_topo_t *topo, ucc_topo_t **subset_topo); void ucc_topo_cleanup(ucc_topo_t *subset_topo); ucc_sbgp_t *ucc_topo_get_sbgp(ucc_topo_t *topo, ucc_sbgp_type_t type); int ucc_topo_is_single_node(ucc_topo_t *topo); /* Returns the array of ALL existing socket subgroups of given topo */ ucc_status_t ucc_topo_get_all_sockets(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps); /* Returns the array of ALL existing numa subgroups of given topo */ ucc_status_t ucc_topo_get_all_numas(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps); /* Returns the array of ALL existing node subgroups of given topo */ ucc_status_t ucc_topo_get_all_nodes(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps); static inline int ucc_rank_on_local_node(ucc_rank_t team_rank, ucc_topo_t *topo) { ucc_proc_info_t *procs = topo->topo->procs; ucc_rank_t ctx_rank = ucc_ep_map_eval(topo->set.map, team_rank); ucc_rank_t my_ctx_rank = ucc_ep_map_eval(topo->set.map, topo->set.myrank); return procs[ctx_rank].host_hash == procs[my_ctx_rank].host_hash; } static inline ucc_host_id_t ucc_topo_get_node_host_id(ucc_topo_t *topo, ucc_rank_t node_index) { ucc_sbgp_t *sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); ucc_rank_t leader_rank; ucc_rank_t ctx_rank; if (sbgp->status == UCC_SBGP_NOT_EXISTS) { ucc_assert(ucc_topo_is_single_node(topo)); leader_rank = 0; } else { leader_rank = ucc_ep_map_eval(sbgp->map, node_index); } ctx_rank = ucc_ep_map_eval(topo->set.map, leader_rank); ucc_assert(ctx_rank < topo->topo->n_procs); return topo->topo->procs[ctx_rank].host_id; } /* Returns min ppn value across the nodes */ static inline ucc_rank_t ucc_topo_min_ppn(ucc_topo_t *topo) { ucc_sbgp_t *sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); if (sbgp->status == UCC_SBGP_NOT_EXISTS) { ucc_assert(ucc_topo_is_single_node(topo)); return ucc_subset_size(&topo->set); } return topo->min_ppn; } /* Returns max ppn value across the nodes */ static inline ucc_rank_t ucc_topo_max_ppn(ucc_topo_t *topo) { ucc_sbgp_t *sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); if (sbgp->status == UCC_SBGP_NOT_EXISTS) { ucc_assert(ucc_topo_is_single_node(topo)); return ucc_subset_size(&topo->set); } return topo->max_ppn; } static inline int ucc_topo_is_single_ppn(ucc_topo_t *topo) { return ucc_topo_max_ppn(topo) == 1; } /* Returns true if PPN is the same across all the nodes */ static inline int ucc_topo_isoppn(ucc_topo_t *topo) { return ucc_topo_max_ppn(topo) == ucc_topo_min_ppn(topo); } /* Returns min socket size across the nodes. * If not set will return UCC_RANK_MAX, * in case of error will return UCC_RANK_INVALID. */ static inline ucc_rank_t ucc_topo_min_socket_size(ucc_topo_t *topo) { ucc_sbgp_t *sbgp; ucc_assert(topo->topo->sock_bound); sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); if (sbgp->status == UCC_SBGP_NOT_INIT) { return UCC_RANK_INVALID; } return topo->min_socket_size; } /* Returns max socket size across the nodes. * If not set will return 0, * in case of error will return UCC_RANK_INVALID. */ static inline ucc_rank_t ucc_topo_max_socket_size(ucc_topo_t *topo) { ucc_sbgp_t *sbgp; ucc_assert(topo->topo->sock_bound); sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); if (sbgp->status == UCC_SBGP_NOT_INIT) { return UCC_RANK_INVALID; } return topo->max_socket_size; } /* Returns min numa size across the nodes. * If not set will return UCC_RANK_MAX, * in case of error will return UCC_RANK_INVALID. */ static inline ucc_rank_t ucc_topo_min_numa_size(ucc_topo_t *topo) { ucc_sbgp_t *sbgp; ucc_assert(topo->topo->numa_bound); sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); if (sbgp->status == UCC_SBGP_NOT_INIT) { return UCC_RANK_INVALID; } return topo->min_numa_size; } /* Returns max numa size across the nodes. * If not set will return 0, * in case of error will return UCC_RANK_INVALID. */ static inline ucc_rank_t ucc_topo_max_numa_size(ucc_topo_t *topo) { ucc_sbgp_t *sbgp; ucc_assert(topo->topo->numa_bound); sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); if (sbgp->status == UCC_SBGP_NOT_INIT) { return UCC_RANK_INVALID; } return topo->max_numa_size; } static inline int ucc_topo_n_sockets(ucc_topo_t *topo) { ucc_sbgp_t *sbgp; if (!topo->topo->sock_bound) { return 0; } sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET_LEADERS); if (sbgp->status == UCC_SBGP_NOT_EXISTS) { return 1; } return sbgp->group_size; } static inline int ucc_topo_n_numas(ucc_topo_t *topo) { ucc_sbgp_t *sbgp; if (!topo->topo->numa_bound) { return 0; } sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NUMA_LEADERS); if (sbgp->status == UCC_SBGP_NOT_EXISTS) { return 1; } return sbgp->group_size; } static inline ucc_rank_t ucc_topo_nnodes(ucc_topo_t *topo) { ucc_sbgp_t *sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); if (sbgp->status == UCC_SBGP_NOT_EXISTS) { ucc_assert(ucc_topo_is_single_node(topo)); return 1; } return sbgp->group_size; } /* Returns node leaders array - array that maps each rank to the TEAM RANK that is the leader of that rank's node. Also returns per-node leaders array - array mapping node_id to the TEAM RANK of that node's leader */ ucc_status_t ucc_topo_get_node_leaders(ucc_topo_t *topo, ucc_rank_t **node_leaders_out); /** * @brief Checks if all ranks in the topology have valid GPU device information. * * This function iterates through all hosts in the provided topology subset * and verifies that each host has visible GPUs. It is typically used to * determine whether further device-based operations can be performed across * the topology. * * @param [in] topo Pointer to the topology structure. * * @return 1 if all ranks have device information, 0 otherwise. */ int ucc_topo_has_device_info(const ucc_topo_t *topo); /** * @brief Checks if all ranks in a sub-group are fully NVLink-connected * within a single node. * * This function determines whether all ranks within the provided sub-group * (sbgp) are fully connected via NVLink within the given topology. * It only considers intra-node NVLink links; cross-node pairs always * return not-connected. Use ucc_topo_is_single_nvlink_domain() for * multinode NVLink fabric (NVLS) eligibility checks. * * @param [in] topo Pointer to the topology structure. * @param [in] sbgp Pointer to the sub-group structure. * * @return 1 if all ranks are NVLink-connected, 0 otherwise. */ int ucc_topo_is_nvlink_fully_connected( const ucc_topo_t *topo, const ucc_sbgp_t *sbgp); /** * @brief Checks if all ranks in the topology subset share the same * NVLink fabric domain (required for multinode NVLS/NVLink-SHARP). * * Iterates all ranks in topo->set and verifies that every rank has * UCC_GPU_CAP_FABRIC set and an identical valid fabric_clique_id. * A fabric_clique_id of UCC_GPU_FABRIC_CLIQUE_ID_INVALID is treated as * "unknown / not populated" and * causes the function to return 0 immediately. * * @param [in] topo Pointer to the topology structure. * * @return 1 if all ranks are in the same NVLink fabric domain, 0 otherwise. */ int ucc_topo_is_single_nvlink_domain(const ucc_topo_t *topo); #endif ucc-1.8.0/src/components/topo/ucc_topo.c0000664000175000017500000005153715211535620020405 0ustar alastairalastair/* * Copyright (c) 2021-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "ucc_topo.h" #include "core/ucc_context.h" #include "ucc/api/ucc_status.h" #include "utils/ucc_malloc.h" #include "utils/ucc_proc_info.h" #include #include #include static int ucc_compare_proc_info(const void *a, const void *b) { const ucc_proc_info_t *d1 = (const ucc_proc_info_t *)a; const ucc_proc_info_t *d2 = (const ucc_proc_info_t *)b; if (d1->host_hash != d2->host_hash) { return d1->host_hash > d2->host_hash ? 1 : -1; } else if (d1->socket_id != d2->socket_id) { return d1->socket_id - d2->socket_id; } else if (d1->numa_id != d2->numa_id) { return d1->numa_id - d2->numa_id; } else { return d1->pid - d2->pid; } } static ucc_status_t ucc_context_topo_compute_layout( ucc_context_topo_t *topo, ucc_rank_t size) { ucc_rank_t current_ppn = 1; ucc_rank_t min_ppn = UCC_RANK_MAX; ucc_rank_t max_ppn = 0; ucc_rank_t nnodes = 1; int max_sockid = 0; int max_numaid = 0; ucc_proc_info_t *sorted; ucc_host_id_t current_hash, hash; int i, j; sorted = (ucc_proc_info_t *)ucc_malloc(size * sizeof(ucc_proc_info_t), "proc_sorted"); if (!sorted) { ucc_error("failed to allocate %zd bytes for proc sorted", size * sizeof(ucc_proc_info_t)); return UCC_ERR_NO_MEMORY; } memcpy(sorted, topo->procs, size * sizeof(ucc_proc_info_t)); qsort(sorted, size, sizeof(ucc_proc_info_t), ucc_compare_proc_info); current_hash = sorted[0].host_hash; for (i = 1; i < size; i++) { hash = sorted[i].host_hash; if (hash != current_hash) { for (j = 0; j < size; j++) { if (topo->procs[j].host_hash == current_hash) { topo->procs[j].host_id = nnodes - 1; } } if (current_ppn > max_ppn) max_ppn = current_ppn; if (current_ppn < min_ppn) min_ppn = current_ppn; nnodes++; current_hash = hash; current_ppn = 1; } else { current_ppn++; } } for (j = 0; j < size; j++) { if (topo->procs[j].socket_id > max_sockid) { max_sockid = topo->procs[j].socket_id; } if (topo->procs[j].numa_id > max_numaid) { max_numaid = topo->procs[j].numa_id; } if (topo->procs[j].host_hash == current_hash) { topo->procs[j].host_id = nnodes - 1; } } if (current_ppn > max_ppn) { max_ppn = current_ppn; } if (current_ppn < min_ppn) { min_ppn = current_ppn; } ucc_free(sorted); topo->nnodes = nnodes; topo->min_ppn = min_ppn; topo->max_ppn = max_ppn; topo->max_n_sockets = max_sockid + 1; topo->max_n_numas = max_numaid + 1; return UCC_OK; } ucc_status_t ucc_context_topo_init( ucc_addr_storage_t *storage, ucc_context_topo_t **_topo) { ucc_context_addr_header_t *h; ucc_context_topo_t *topo; int i; ucc_status_t status; if (storage->size < 2) { /* We should always expect at least 2 ranks data in the storage */ return UCC_ERR_NO_MESSAGE; } topo = ucc_malloc(sizeof(*topo), "topo"); if (!topo) { ucc_error("failed to allocate %zd bytes for topo", sizeof(*topo)); return UCC_ERR_NO_MEMORY; } topo->sock_bound = 1; topo->numa_bound = 1; topo->n_procs = storage->size; topo->procs = (ucc_proc_info_t *)ucc_malloc( storage->size * sizeof(ucc_proc_info_t), "topo_procs"); if (!topo->procs) { ucc_error("failed to allocate %zd bytes for topo_procs", storage->size * sizeof(ucc_proc_info_t)); ucc_free(topo); return UCC_ERR_NO_MEMORY; } topo->hosts = (ucc_host_info_t *)ucc_malloc( storage->size * sizeof(ucc_host_info_t), "topo_hosts"); if (!topo->hosts) { ucc_error( "failed to allocate %zd bytes for topo_hosts", storage->size * sizeof(ucc_host_info_t)); ucc_free(topo->procs); ucc_free(topo); return UCC_ERR_NO_MEMORY; } for (i = 0; i < storage->size; i++) { h = (ucc_context_addr_header_t *)PTR_OFFSET( storage->storage, storage->addr_len * i); topo->procs[i] = h->ctx_id.pi; topo->hosts[i] = h->host_info; if (h->ctx_id.pi.socket_id == UCC_SOCKET_ID_INVALID) { topo->sock_bound = 0; } if (h->ctx_id.pi.numa_id == UCC_NUMA_ID_INVALID) { topo->numa_bound = 0; } } status = ucc_context_topo_compute_layout(topo, storage->size); if (UCC_OK != status) { ucc_free(topo->procs); ucc_free(topo->hosts); ucc_free(topo); return status; } *_topo = topo; return UCC_OK; } void ucc_context_topo_cleanup(ucc_context_topo_t *topo) { if (topo) { ucc_free(topo->procs); ucc_free(topo->hosts); ucc_free(topo); } } ucc_status_t ucc_topo_init( ucc_subset_t set, ucc_context_topo_t *ctx_topo, ucc_topo_t **_topo) { ucc_topo_t *topo = ucc_malloc(sizeof(*topo), "topo"); ucc_rank_t size = ucc_subset_size(&set); ucc_rank_t ctx_rank; int i, all_valid = 1; if (!topo) { return UCC_ERR_NO_MEMORY; } if (!ctx_topo) { ucc_free(topo); return UCC_ERR_INVALID_PARAM; } topo->topo = ctx_topo; for (i = 0; i < UCC_SBGP_LAST; i++) { topo->sbgps[i].status = UCC_SBGP_NOT_INIT; } topo->n_sockets = -1; topo->node_leader_rank = UCC_RANK_INVALID; topo->node_leader_rank_id = 0; topo->set = set; topo->min_ppn = UCC_RANK_MAX; topo->max_ppn = 0; topo->min_socket_size = UCC_RANK_MAX; topo->max_socket_size = 0; topo->min_numa_size = UCC_RANK_MAX; topo->max_numa_size = 0; topo->all_sockets = NULL; topo->all_numas = NULL; topo->all_nodes = NULL; topo->node_leaders = NULL; topo->device_map.device_ids = NULL; topo->device_map.n_ranks = 0; topo->n_nodes = 0; for (i = 0; i < size; i++) { ctx_rank = ucc_ep_map_eval(set.map, i); if (!ctx_topo->hosts[ctx_rank].visible_gpus) { all_valid = 0; break; } } if (all_valid) { topo->device_map.device_ids = ucc_malloc(size * sizeof(ucc_device_id_t), "topo_device_map"); if (!topo->device_map.device_ids) { ucc_free(topo); return UCC_ERR_NO_MEMORY; } topo->device_map.n_ranks = size; for (i = 0; i < size; i++) { ctx_rank = ucc_ep_map_eval(set.map, i); topo->device_map.device_ids[i] = ucc_ilog2( ctx_topo->hosts[ctx_rank].visible_gpus); } } *_topo = topo; return UCC_OK; } void ucc_topo_cleanup(ucc_topo_t *topo) { int i; if (topo) { for (i = 0; i < UCC_SBGP_LAST; i++) { if (topo->sbgps[i].status == UCC_SBGP_ENABLED || topo->sbgps[i].status == UCC_SBGP_DISABLED) { ucc_sbgp_cleanup(&topo->sbgps[i]); } } if (topo->all_sockets) { for (i = 0; i < topo->n_sockets; i++) { if (topo->all_sockets[i].status == UCC_SBGP_ENABLED) { ucc_sbgp_cleanup(&topo->all_sockets[i]); } } ucc_free(topo->all_sockets); } if (topo->all_numas) { for (i = 0; i < topo->n_numas; i++) { if (topo->all_numas[i].status == UCC_SBGP_ENABLED) { ucc_sbgp_cleanup(&topo->all_numas[i]); } } ucc_free(topo->all_numas); } if (topo->all_nodes) { for (i = 0; i < topo->n_nodes; i++) { if (topo->all_nodes[i].status == UCC_SBGP_ENABLED) { ucc_sbgp_cleanup(&topo->all_nodes[i]); } } ucc_free(topo->all_nodes); } if (topo->node_leaders) { ucc_free(topo->node_leaders); } if (topo->device_map.device_ids) { ucc_free(topo->device_map.device_ids); } ucc_free(topo); } } ucc_sbgp_t *ucc_topo_get_sbgp(ucc_topo_t *topo, ucc_sbgp_type_t type) { if (topo->sbgps[type].status == UCC_SBGP_NOT_INIT) { if (UCC_OK != ucc_sbgp_create(topo, type)) { ucc_error("failed to create sbgp %s", ucc_sbgp_str(type)); /* sbgps[type]->status is set accordingly */ } } return &topo->sbgps[type]; } int ucc_topo_is_single_node(ucc_topo_t *topo) { ucc_sbgp_t *sbgp; sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); if (UCC_SBGP_ENABLED == sbgp->status && sbgp->group_size == ucc_subset_size(&topo->set)) { return 1; } return 0; } ucc_status_t ucc_topo_get_all_sockets(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps) { ucc_status_t status = UCC_OK; if (!topo->all_sockets) { status = ucc_sbgp_create_all_sockets(topo, &topo->all_sockets, n_sbgps); } *sbgps = topo->all_sockets; *n_sbgps = topo->n_sockets; return status; } ucc_status_t ucc_topo_get_all_numas(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps) { ucc_status_t status = UCC_OK; if (!topo->all_numas) { status = ucc_sbgp_create_all_numas(topo, &topo->all_numas, n_sbgps); } *sbgps = topo->all_numas; *n_sbgps = topo->n_numas; return status; } /* Returns invalid param if there's only one node in the team (leader sbgp does not exist). Otherwise, creates a node sbgp for every node. One or more sbgps may be UCC_SBGP_NOT_EXISTS if they have only one rank */ ucc_status_t ucc_sbgp_create_all_nodes(ucc_topo_t *topo, ucc_sbgp_t **_sbgps, int *n_sbgps) { ucc_rank_t myrank = topo->set.myrank; ucc_rank_t leader_rank = UCC_RANK_INVALID; ucc_sbgp_t *sbgps, *leader_sbgp; ucc_rank_t i; ucc_status_t status; ucc_rank_t nnodes; leader_sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); if (leader_sbgp->status == UCC_SBGP_NOT_INIT || leader_sbgp->status == UCC_SBGP_NOT_EXISTS) { ucc_debug("could not create all_nodes subgroups, leader subgroup " "does not exist for topo"); return UCC_ERR_INVALID_PARAM; } nnodes = leader_sbgp->group_size; sbgps = ucc_calloc(nnodes, sizeof(ucc_sbgp_t), "sbgps"); if (!sbgps) { ucc_error("failed to allocate %zd bytes for sbgps array", nnodes * sizeof(ucc_sbgp_t)); return UCC_ERR_NO_MEMORY; } for (i = 0; i < nnodes; i++) { ucc_rank_t ldr_team_rank = ucc_ep_map_eval(leader_sbgp->map, i); sbgps[i].type = UCC_SBGP_NODE; topo->set.myrank = ldr_team_rank; status = ucc_sbgp_create_node(topo, &sbgps[i]); if (status == UCC_ERR_NOT_FOUND) { /* ucc_sbgp_create_node returned because 0 == node_size */ sbgps[i].status = UCC_SBGP_NOT_EXISTS; continue; } else if (status != UCC_OK) { ucc_error("failed to create all_node subgroup %d", i); goto error; } if (ucc_rank_on_local_node(myrank, topo)) { leader_rank = topo->node_leader_rank; } if (sbgps[i].status == UCC_SBGP_NOT_EXISTS) { /* Single-member subgroup: free rank_map and skip map creation to avoid a dangling pointer in map.array.map */ if (sbgps[i].rank_map) { ucc_free(sbgps[i].rank_map); sbgps[i].rank_map = NULL; } } else if (sbgps[i].rank_map && sbgps[i].type != UCC_SBGP_FULL) { sbgps[i].map = ucc_ep_map_from_array( &sbgps[i].rank_map, sbgps[i].group_size, ucc_subset_size(&topo->set), 1); } } /* Reset myrank and node_leader_rank because the calls to ucc_sbgp_create_node above will have changed them */ topo->set.myrank = myrank; ucc_assert(leader_rank != UCC_RANK_INVALID); topo->node_leader_rank = leader_rank; *_sbgps = sbgps; *n_sbgps = nnodes; return UCC_OK; error: topo->set.myrank = myrank; for (i = 0; i < nnodes; i++) { if (sbgps[i].rank_map) { ucc_free(sbgps[i].rank_map); } } ucc_free(sbgps); return status; } ucc_status_t ucc_topo_get_all_nodes(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps) { ucc_status_t status = UCC_OK; if (!topo->all_nodes) { status = ucc_sbgp_create_all_nodes(topo, &topo->all_nodes, &topo->n_nodes); } *sbgps = topo->all_nodes; *n_sbgps = topo->n_nodes; return status; } static int ucc_topo_rank_device_info(const ucc_topo_t *topo, ucc_rank_t rank, const ucc_host_info_t **host, ucc_device_id_t *device_id) { ucc_rank_t ctx_rank = ucc_ep_map_eval(topo->set.map, rank); *host = &topo->topo->hosts[ctx_rank]; if (!(*host)->visible_gpus) { return 0; } *device_id = ucc_ilog2((*host)->visible_gpus); return 1; } int ucc_topo_has_device_info(const ucc_topo_t *topo) { ucc_host_info_t *host; ucc_rank_t i; if (!topo->topo) { return 0; } for (i = 0; i < ucc_subset_size(&topo->set); i++) { host = &topo->topo->hosts[ucc_ep_map_eval(topo->set.map, i)]; if (host->visible_gpus == 0) { return 0; } } return 1; } static int ucc_topo_ranks_on_node_nvlink(const ucc_topo_t *topo, ucc_rank_t rank1, ucc_rank_t rank2) { ucc_rank_t ctx_rank1 = ucc_ep_map_eval(topo->set.map, rank1); ucc_rank_t ctx_rank2 = ucc_ep_map_eval(topo->set.map, rank2); ucc_device_id_t dev1; ucc_device_id_t dev2; const ucc_host_info_t *host1; const ucc_host_info_t *host2; const ucc_gpu_info_t *gpu1; const ucc_gpu_info_t *gpu2; if (!ucc_topo_rank_device_info(topo, rank1, &host1, &dev1) || !ucc_topo_rank_device_info(topo, rank2, &host2, &dev2)) { return 0; } if (topo->topo->procs[ctx_rank1].host_hash != topo->topo->procs[ctx_rank2].host_hash) { return 0; } if (dev1 == dev2) { return 1; } gpu1 = &host1->gpus[dev1]; gpu2 = &host2->gpus[dev2]; if (gpu1->fabric_clique_id != UCC_GPU_FABRIC_CLIQUE_ID_INVALID && gpu1->fabric_clique_id == gpu2->fabric_clique_id) { return 1; } if (UCC_GPU_HAS_CAP(gpu1, UCC_GPU_CAP_NVSWITCH) && UCC_GPU_HAS_CAP(gpu2, UCC_GPU_CAP_NVSWITCH)) { return 1; } return host1->nvlink_matrix[dev1][dev2] > 0; } int ucc_topo_is_nvlink_fully_connected( const ucc_topo_t *topo, const ucc_sbgp_t *sbgp) { ucc_rank_t size; ucc_rank_t i, j; ucc_rank_t rank_i; if (!sbgp) { return 0; } size = sbgp->group_size; if (size <= 1) { return 1; } for (i = 0; i < size; i++) { rank_i = ucc_ep_map_eval(sbgp->map, i); for (j = i + 1; j < size; j++) { ucc_rank_t rank_j = ucc_ep_map_eval(sbgp->map, j); if (!ucc_topo_ranks_on_node_nvlink(topo, rank_i, rank_j)) { return 0; } } } return 1; } int ucc_topo_is_single_nvlink_domain(const ucc_topo_t *topo) { ucc_rank_t size = ucc_subset_size(&topo->set); const ucc_host_info_t *host; ucc_device_id_t dev; uint64_t ref_clique_id; uint32_t ref_partition_id; ucc_rank_t i; if (size == 0) { return 0; } if (!ucc_topo_rank_device_info(topo, 0, &host, &dev)) { ucc_debug("nvlink domain check: rank 0 has no device info"); return 0; } if (!UCC_GPU_HAS_CAP(&host->gpus[dev], UCC_GPU_CAP_FABRIC) || host->gpus[dev].fabric_clique_id == UCC_GPU_FABRIC_CLIQUE_ID_INVALID) { ucc_debug("nvlink domain check: rank 0 GPU %u not fabric-capable " "or clique_id invalid (caps=0x%x clique_id=%llu)", (unsigned)dev, host->gpus[dev].caps, (unsigned long long)host->gpus[dev].fabric_clique_id); return 0; } ref_clique_id = host->gpus[dev].fabric_clique_id; ref_partition_id = host->gpus[dev].fabric_partition_id; for (i = 1; i < size; i++) { if (!ucc_topo_rank_device_info(topo, i, &host, &dev)) { ucc_debug("nvlink domain check: rank %u has no device info", (unsigned)i); return 0; } if (!UCC_GPU_HAS_CAP(&host->gpus[dev], UCC_GPU_CAP_FABRIC)) { ucc_debug("nvlink domain check: rank %u GPU %u not fabric-capable", (unsigned)i, (unsigned)dev); return 0; } if (host->gpus[dev].fabric_clique_id != ref_clique_id) { ucc_debug("nvlink domain check: rank %u clique_id=%llu differs " "from rank 0 clique_id=%llu", (unsigned)i, (unsigned long long)host->gpus[dev].fabric_clique_id, (unsigned long long)ref_clique_id); return 0; } /* If partition IDs are populated (GB200+ NVL), require same partition. * UCC_GPU_FABRIC_PARTITION_ID_INVALID means single-partition or * unpopulated (pre-r525 NVML). */ if (ref_partition_id != UCC_GPU_FABRIC_PARTITION_ID_INVALID && host->gpus[dev].fabric_partition_id != ref_partition_id) { ucc_debug("nvlink domain check: rank %u partition_id=%u differs " "from rank 0 partition_id=%u; " "ranks span different NVL partitions", (unsigned)i, (unsigned)host->gpus[dev].fabric_partition_id, (unsigned)ref_partition_id); return 0; } } return 1; } ucc_status_t ucc_topo_get_node_leaders(ucc_topo_t *topo, ucc_rank_t **node_leaders_out) { ucc_subset_t *set = &topo->set; ucc_rank_t size = ucc_subset_size(set); ucc_rank_t nnodes = topo->topo->nnodes; ucc_rank_t i; ucc_rank_t *ranks_seen_per_node; ucc_rank_t *per_node_leaders; ucc_rank_t *node_leaders; if (topo->node_leaders) { *node_leaders_out = topo->node_leaders; return UCC_OK; } ucc_assert(nnodes > 1); /* Allocate arrays */ node_leaders = ucc_malloc(sizeof(ucc_rank_t) * size, "node_leaders"); if (!node_leaders) { ucc_error("failed to allocate %zd bytes for node_leaders array", size * sizeof(ucc_rank_t)); return UCC_ERR_NO_MEMORY; } ranks_seen_per_node = ucc_calloc(nnodes, sizeof(ucc_rank_t), "ranks_seen_per_node"); if (!ranks_seen_per_node) { ucc_error("failed to allocate %zd bytes for ranks_seen_per_node array", nnodes * sizeof(ucc_rank_t)); ucc_free(node_leaders); return UCC_ERR_NO_MEMORY; } per_node_leaders = ucc_calloc(nnodes, sizeof(ucc_rank_t), "per_node_leaders"); if (!per_node_leaders) { ucc_error("failed to allocate %zd bytes for per_node_leaders array", nnodes * sizeof(ucc_rank_t)); ucc_free(node_leaders); ucc_free(ranks_seen_per_node); return UCC_ERR_NO_MEMORY; } /* First pass: identify node leaders */ for (i = 0; i < size; i++) { ucc_rank_t ctx_rank = ucc_ep_map_eval(set->map, i); ucc_host_id_t current_host = topo->topo->procs[ctx_rank].host_id; /* Count ranks on this node */ ranks_seen_per_node[current_host]++; /* If this is the rank we want as leader for this node, mark it */ if (ranks_seen_per_node[current_host] == topo->node_leader_rank_id + 1) { per_node_leaders[current_host] = i; } } /* Second pass: propagate node leaders to all ranks */ for (i = 0; i < size; i++) { ucc_rank_t ctx_rank = ucc_ep_map_eval(set->map, i); ucc_host_id_t current_host = topo->topo->procs[ctx_rank].host_id; node_leaders[i] = per_node_leaders[current_host]; } topo->node_leaders = node_leaders; //NOLINTNEXTLINE *node_leaders_out = node_leaders; ucc_free(ranks_seen_per_node); ucc_free(per_node_leaders); return UCC_OK; } ucc-1.8.0/src/components/topo/ib/0000775000175000017500000000000015211535620017005 5ustar alastairalastairucc-1.8.0/src/components/topo/ib/ucc_sysinfo_ib.h0000664000175000017500000000054415211535620022157 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_SYSINFO_IB_H_ #define UCC_SYSINFO_IB_H_ #include "components/topo/base/ucc_sysinfo_base.h" typedef struct ucc_sysinfo_ib { ucc_sysinfo_base_t super; } ucc_sysinfo_ib_t; extern ucc_sysinfo_ib_t ucc_sysinfo_ib; #endif ucc-1.8.0/src/components/topo/ib/Makefile.am0000664000175000017500000000132715211535620021044 0ustar alastairalastair# # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # if HAVE_IBVERBS sources = \ ucc_sysinfo_ib.h \ ucc_sysinfo_ib.c module_LTLIBRARIES = libucc_sysinfo_ib.la libucc_sysinfo_ib_la_SOURCES = $(sources) libucc_sysinfo_ib_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) \ $(IBVERBS_CPPFLAGS) libucc_sysinfo_ib_la_CFLAGS = $(BASE_CFLAGS) libucc_sysinfo_ib_la_LDFLAGS = -version-info $(SOVERSION) --as-needed \ $(IBVERBS_LDFLAGS) libucc_sysinfo_ib_la_LIBADD = $(IBVERBS_LIBADD) \ $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/topo/ib/ucc_sysinfo_ib.c0000664000175000017500000001757115211535620022162 0ustar alastairalastair#include "config.h" #include "ucc_sysinfo_ib.h" #include "core/ucc_global_opts.h" #include "utils/ucc_malloc.h" #include "utils/ucc_proc_info.h" #include "utils/debug/log_def.h" #include "utils/ucc_string.h" #include #include #include #include #include static ucc_status_t ucc_sysinfo_ib_init(const ucc_sysinfo_params_t *params) { (void)params; return UCC_OK; } static ucc_status_t ucc_sysinfo_ib_finalize(void) { return UCC_OK; } static int ucc_sysinfo_ib_net_device_matches(const char *entry, const ucc_nic_info_t *nic) { const char *port_str; size_t name_len; unsigned long port; if (!entry || !nic || nic->name[0] == '\0') { return 0; } port_str = strchr(entry, ':'); if (!port_str) { return (strcmp(entry, nic->name) == 0); } name_len = (size_t)(port_str - entry); if (name_len == 0 || name_len >= sizeof(nic->name)) { return 0; } if ((strncmp(entry, nic->name, name_len) != 0) || (nic->name[name_len] != '\0')) { return 0; } port_str++; if (*port_str == '\0' || ucc_str_is_number(port_str) != UCC_OK) { return 0; } port = strtoul(port_str, NULL, 10); if (port > 0xff) { return 0; } return nic->port == (uint8_t)port; } static ucc_status_t ucc_sysinfo_ib_set_visible_devices( void *info, int n_info, const ucc_config_names_array_t *devices, uint32_t *visible_devices) { ucc_nic_info_t *nics; uint32_t mask; int i, j; if (!visible_devices || (n_info > 0 && !info)) { return UCC_ERR_INVALID_PARAM; } if (!devices) { return UCC_OK; } *visible_devices = 0; if (n_info <= 0) { return UCC_OK; } for (i = 0; i < devices->count; i++) { if (strcmp(devices->names[i], "all") == 0) { for (j = 0; j < n_info && j < (int)(sizeof(mask) * 8); j++) { *visible_devices |= (1u << j); } return UCC_OK; } } if (devices->count == 0) { for (j = 0; j < n_info && j < (int)(sizeof(mask) * 8); j++) { *visible_devices |= (1u << j); } return UCC_OK; } nics = (ucc_nic_info_t *)info; mask = 0; for (i = 0; i < n_info && i < (int)(sizeof(mask) * 8); i++) { for (j = 0; j < devices->count; j++) { if (ucc_sysinfo_ib_net_device_matches(devices->names[j], &nics[i])) { mask |= (1u << i); break; } } } *visible_devices = mask; return UCC_OK; } static void ucc_sysinfo_ib_fill_pci_info(const char *dev_name, ucc_nic_info_t *nic) { char path[PATH_MAX]; char line[256]; unsigned int domain; unsigned int bus; unsigned int device; unsigned int function; FILE *fp; nic->pci.host_id = ucc_local_proc.host_id; nic->pci.domain = 0; nic->pci.bus = 0; nic->pci.device = 0; nic->pci.function = 0; if (!dev_name) { return; } snprintf(path, sizeof(path), "/sys/class/infiniband/%s/device/uevent", dev_name); fp = fopen(path, "r"); if (!fp) { return; } while (fgets(line, sizeof(line), fp)) { if (strncmp(line, "PCI_SLOT_NAME=", 14) == 0) { if (sscanf(line + 14, "%x:%x:%x.%x", &domain, &bus, &device, &function) == 4) { nic->pci.domain = (uint16_t)domain; nic->pci.bus = (uint8_t)bus; nic->pci.device = (uint8_t)device; nic->pci.function = (uint8_t)function; } break; } } fclose(fp); } static ucc_status_t ucc_sysinfo_ib_get_info(void **info, int *n_info) { struct ibv_device **dev_list; struct ibv_context *ctx; struct ibv_device_attr dev_attr; struct ibv_port_attr port_attr; ucc_nic_info_t *nic_info; ucc_status_t status; const char *dev_name; int num_devs; int total_ports; int i; int port; int idx; if (info) { *info = NULL; } if (n_info) { *n_info = 0; } dev_list = ibv_get_device_list(&num_devs); if (!dev_list) { ucc_debug("ibv_get_device_list failed"); return UCC_ERR_NO_RESOURCE; } total_ports = 0; for (i = 0; i < num_devs; i++) { ctx = ibv_open_device(dev_list[i]); if (!ctx) { ucc_debug("ibv_open_device failed"); continue; } if (ibv_query_device(ctx, &dev_attr)) { ucc_debug("ibv_query_device failed"); ibv_close_device(ctx); continue; } for (port = 1; port <= dev_attr.phys_port_cnt; port++) { if (ibv_query_port(ctx, port, &port_attr)) { continue; } if (port_attr.state == IBV_PORT_ACTIVE) { total_ports++; } } ibv_close_device(ctx); } if (total_ports == 0) { ibv_free_device_list(dev_list); return UCC_OK; } nic_info = (ucc_nic_info_t *)ucc_malloc( total_ports * sizeof(*nic_info), "sysinfo_ib_nic_info"); if (!nic_info) { ucc_error("failed to allocate %d bytes for nic info", total_ports * (int)sizeof(*nic_info)); ibv_free_device_list(dev_list); return UCC_ERR_NO_MEMORY; } memset(nic_info, 0, total_ports * sizeof(*nic_info)); idx = 0; status = UCC_OK; for (i = 0; i < num_devs && idx < total_ports; i++) { dev_name = ibv_get_device_name(dev_list[i]); ctx = ibv_open_device(dev_list[i]); if (!ctx) { ucc_debug("ibv_open_device failed"); continue; } if (ibv_query_device(ctx, &dev_attr)) { ucc_debug("ibv_query_device failed"); ibv_close_device(ctx); continue; } for (port = 1; port <= dev_attr.phys_port_cnt; port++) { if (ibv_query_port(ctx, port, &port_attr)) { continue; } if (port_attr.state != IBV_PORT_ACTIVE) { continue; } ucc_sysinfo_ib_fill_pci_info(dev_name, &nic_info[idx]); nic_info[idx].port = (uint8_t)port; nic_info[idx].guid = (uint64_t)dev_attr.node_guid; if (dev_name) { snprintf(nic_info[idx].name, sizeof(nic_info[idx].name), "%s", dev_name); } idx++; } ibv_close_device(ctx); } ibv_free_device_list(dev_list); if (idx == 0) { ucc_free(nic_info); return UCC_OK; } if (info) { *info = nic_info; } else { ucc_free(nic_info); } if (n_info) { *n_info = idx; } return status; } static ucc_config_field_t ucc_sysinfo_ib_config_table[] = { {NULL} }; ucc_sysinfo_ib_t ucc_sysinfo_ib = { .super.super.name = "ib sysinfo", .super.type = UCC_SYSINFO_TYPE_IB, .super.ref_cnt = 0, .super.init = ucc_sysinfo_ib_init, .super.finalize = ucc_sysinfo_ib_finalize, .super.ops.get_info = ucc_sysinfo_ib_get_info, .super.ops.set_visible_devices = ucc_sysinfo_ib_set_visible_devices, .super.config_table = { .name = "IB sysinfo", .prefix = "SYSINFO_IB_", .table = ucc_sysinfo_ib_config_table, .size = sizeof(ucc_sysinfo_config_t), }, }; UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_sysinfo_ib.super.config_table, &ucc_config_global_list); ucc-1.8.0/src/components/topo/ucc_sysinfo.c0000664000175000017500000001352015211535620021104 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "ucc_sysinfo.h" #include "core/ucc_global_opts.h" #include "components/topo/base/ucc_sysinfo_base.h" #include "utils/debug/log_def.h" #include "utils/ucc_malloc.h" #include "utils/ucc_math.h" #include static const ucc_sysinfo_ops_t *sysinfo_ops[UCC_SYSINFO_TYPE_LAST]; static int sysinfo_initialized; ucc_status_t ucc_sysinfo_init(void) { ucc_sysinfo_base_t *sysinfo_base; ucc_sysinfo_params_t sysinfo_params; ucc_status_t status; int i; if (sysinfo_initialized) { return UCC_OK; } memset(&sysinfo_params, 0, sizeof(sysinfo_params)); for (i = 0; i < ucc_global_config.sysinfo_framework.n_components; i++) { sysinfo_base = ucc_derived_of( ucc_global_config.sysinfo_framework.components[i], ucc_sysinfo_base_t); if (sysinfo_base->ref_cnt == 0) { sysinfo_base->config = ucc_malloc(sysinfo_base->config_table.size); if (!sysinfo_base->config) { ucc_error("failed to allocate %zd bytes for sysinfo config", sysinfo_base->config_table.size); continue; } status = ucc_config_parser_fill_opts( sysinfo_base->config, &sysinfo_base->config_table, "UCC_", 1); if (UCC_OK != status) { ucc_error("failed to parse config for sysinfo: %s (%d)", sysinfo_base->super.name, status); ucc_free(sysinfo_base->config); continue; } status = sysinfo_base->init(&sysinfo_params); if (UCC_OK != status) { ucc_error("sysinfo_init failed for component: %s, skipping (%d)", sysinfo_base->super.name, status); ucc_config_parser_release_opts(sysinfo_base->config, sysinfo_base->config_table.table); ucc_free(sysinfo_base->config); continue; } ucc_debug("sysinfo %s initialized", sysinfo_base->super.name); } sysinfo_base->ref_cnt++; sysinfo_ops[sysinfo_base->type] = &sysinfo_base->ops; } sysinfo_initialized = 1; return UCC_OK; } ucc_status_t ucc_sysinfo_get_host_info(ucc_host_info_t *info) { ucc_sysinfo_gpu_info_t *gpu_info; ucc_nic_info_t *nic_info; ucc_status_t status; int n_info; int copy_n; if (!info) { return UCC_ERR_INVALID_PARAM; } memset(info, 0, sizeof(*info)); info->host_id = ucc_local_proc.host_id; if (sysinfo_ops[UCC_SYSINFO_TYPE_NVLINK]) { gpu_info = NULL; n_info = 0; status = sysinfo_ops[UCC_SYSINFO_TYPE_NVLINK]->get_info( (void **)&gpu_info, &n_info); if (status == UCC_OK && gpu_info && n_info > 0) { copy_n = ucc_min(n_info, UCC_MAX_HOST_GPUS); info->n_gpus = (uint8_t)copy_n; memcpy(info->gpus, gpu_info->gpus, copy_n * sizeof(*info->gpus)); memcpy(info->nvlink_matrix, gpu_info->nvlink_matrix, sizeof(info->nvlink_matrix)); if (n_info > UCC_MAX_HOST_GPUS) { ucc_debug("truncating gpu info from %d to %d entries", n_info, UCC_MAX_HOST_GPUS); } } else if (status == UCC_ERR_NOT_SUPPORTED || status == UCC_ERR_NO_RESOURCE || status == UCC_ERR_NO_MESSAGE) { status = UCC_OK; } if (gpu_info) { ucc_free(gpu_info); } if (status != UCC_OK) { return status; } } if (sysinfo_ops[UCC_SYSINFO_TYPE_IB]) { nic_info = NULL; n_info = 0; status = sysinfo_ops[UCC_SYSINFO_TYPE_IB]->get_info( (void **)&nic_info, &n_info); if (status == UCC_OK && nic_info && n_info > 0) { copy_n = ucc_min(n_info, UCC_MAX_HOST_NICS); info->n_nics = (uint8_t)copy_n; memcpy(info->nics, nic_info, copy_n * sizeof(*nic_info)); if (n_info > UCC_MAX_HOST_NICS) { ucc_debug("truncating nic info from %d to %d entries", n_info, UCC_MAX_HOST_NICS); } } else if (status == UCC_ERR_NOT_SUPPORTED || status == UCC_ERR_NO_RESOURCE || status == UCC_ERR_NO_MESSAGE) { status = UCC_OK; } if (nic_info) { ucc_free(nic_info); } if (status != UCC_OK) { return status; } } ucc_host_info_print(info); return UCC_OK; } ucc_status_t ucc_sysinfo_set_visible_devices(ucc_host_info_t *info, ucc_sysinfo_type_t device_type, const ucc_config_names_array_t *devices) { if (!info) { return UCC_ERR_INVALID_PARAM; } if (device_type == UCC_SYSINFO_TYPE_NVLINK) { if (!sysinfo_ops[UCC_SYSINFO_TYPE_NVLINK] || !sysinfo_ops[UCC_SYSINFO_TYPE_NVLINK]->set_visible_devices) { return UCC_ERR_NOT_SUPPORTED; } return sysinfo_ops[UCC_SYSINFO_TYPE_NVLINK]->set_visible_devices( info->gpus, info->n_gpus, devices, &info->visible_gpus); } else if (device_type == UCC_SYSINFO_TYPE_IB) { if (!sysinfo_ops[UCC_SYSINFO_TYPE_IB] || !sysinfo_ops[UCC_SYSINFO_TYPE_IB]->set_visible_devices) { return UCC_ERR_NOT_SUPPORTED; } return sysinfo_ops[UCC_SYSINFO_TYPE_IB]->set_visible_devices( info->nics, info->n_nics, devices, &info->visible_nics); } return UCC_ERR_NOT_SUPPORTED; } ucc-1.8.0/src/components/topo/base/0000775000175000017500000000000015211535620017325 5ustar alastairalastairucc-1.8.0/src/components/topo/base/ucc_topo_base.c0000664000175000017500000000000015211535620022264 0ustar alastairalastairucc-1.8.0/src/components/topo/base/ucc_sysinfo_base.h0000664000175000017500000000236315211535620023020 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_SYSINFO_BASE_H_ #define UCC_SYSINFO_BASE_H_ #include "config.h" #include "utils/ucc_component.h" #include "utils/ucc_parser.h" typedef enum ucc_sysinfo_type { UCC_SYSINFO_TYPE_NVLINK, UCC_SYSINFO_TYPE_IB, UCC_SYSINFO_TYPE_LAST } ucc_sysinfo_type_t; typedef struct ucc_sysinfo_params { char dummy; } ucc_sysinfo_params_t; typedef struct ucc_sysinfo_ops { ucc_status_t (*get_info)(void **info, int *n_info); ucc_status_t (*set_visible_devices)( void *info, int n_info, const ucc_config_names_array_t *devices, uint32_t *visible_devices); } ucc_sysinfo_ops_t; typedef struct ucc_sysinfo_config { char dummy; } ucc_sysinfo_config_t; extern ucc_config_field_t ucc_sysinfo_config_table[]; typedef struct ucc_sysinfo_base { ucc_component_iface_t super; ucc_sysinfo_type_t type; uint32_t ref_cnt; ucc_sysinfo_config_t *config; ucc_config_global_list_entry_t config_table; ucc_status_t (*init)(const ucc_sysinfo_params_t *params); ucc_status_t (*finalize)(); ucc_sysinfo_ops_t ops; } ucc_sysinfo_base_t; #endif ucc-1.8.0/src/components/topo/ucc_sysinfo.h0000664000175000017500000000131615211535620021111 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_SYSINFO_H_ #define UCC_SYSINFO_H_ #include "components/topo/base/ucc_sysinfo_base.h" #include "utils/ucc_proc_info.h" typedef struct ucc_sysinfo_gpu_info { int n_gpus; ucc_gpu_info_t gpus[UCC_MAX_HOST_GPUS]; uint8_t nvlink_matrix[UCC_MAX_HOST_GPUS][UCC_MAX_HOST_GPUS]; } ucc_sysinfo_gpu_info_t; ucc_status_t ucc_sysinfo_init(void); ucc_status_t ucc_sysinfo_get_host_info(ucc_host_info_t *info); ucc_status_t ucc_sysinfo_set_visible_devices( ucc_host_info_t *info, ucc_sysinfo_type_t device_type, const ucc_config_names_array_t *devices); #endif ucc-1.8.0/src/components/topo/ucc_sbgp.c0000664000175000017500000007736215211535620020363 0ustar alastairalastair/* * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_sbgp.h" #include "ucc_topo.h" #include "utils/debug/log_def.h" #include "utils/ucc_malloc.h" #include "utils/ucc_math.h" #include "utils/ucc_compiler_def.h" #include static char *ucc_sbgp_type_str[UCC_SBGP_LAST] = { "numa", "socket", "node", "node_nvlink", "node_leaders", "net", "socket_leaders", "numa_leaders", "flat", "flat_host_ordered"}; const char* ucc_sbgp_str(ucc_sbgp_type_t type) { return ucc_sbgp_type_str[type]; } #define UCC_TOPO_IS_BOUND(_topo, _sbgp_type) \ (UCC_SBGP_SOCKET == (_sbgp_type) || \ UCC_SBGP_SOCKET_LEADERS == (_sbgp_type)) ? \ (_topo)->topo->sock_bound : (_topo)->topo->numa_bound static inline int ucc_ranks_on_local_sn(ucc_rank_t rank1, ucc_rank_t rank2, ucc_topo_t *topo, ucc_sbgp_type_t type) { ucc_rank_t ctx_rank1 = ucc_ep_map_eval(topo->set.map, rank1); ucc_rank_t ctx_rank2 = ucc_ep_map_eval(topo->set.map, rank2); ucc_proc_info_t *proc1 = &topo->topo->procs[ctx_rank1]; ucc_proc_info_t *proc2 = &topo->topo->procs[ctx_rank2]; int bound = UCC_TOPO_IS_BOUND(topo, type); if (!bound) { return 0; } return proc1->host_hash == proc2->host_hash && ((UCC_SBGP_SOCKET == type) ? proc1->socket_id == proc2->socket_id : proc1->numa_id == proc2->numa_id); } static int ucc_topo_rank_device_info(ucc_topo_t *topo, ucc_rank_t rank, const ucc_host_info_t **host, ucc_device_id_t *device_id) { ucc_rank_t ctx_rank; ctx_rank = ucc_ep_map_eval(topo->set.map, rank); if (ctx_rank >= topo->topo->n_procs) { return 0; } *host = &topo->topo->hosts[ctx_rank]; if (!(*host)->visible_gpus) { return 0; } *device_id = ucc_ilog2((*host)->visible_gpus); return 1; } static int ucc_sbgp_has_device_info(ucc_topo_t *topo) { ucc_rank_t size = ucc_subset_size(&topo->set); ucc_rank_t i; const ucc_host_info_t *host; ucc_device_id_t dev; for (i = 0; i < size; i++) { if (ucc_topo_rank_device_info(topo, i, &host, &dev)) { return 1; } } return 0; } static int ucc_topo_ranks_on_node_nvlink(ucc_rank_t rank1, ucc_rank_t rank2, ucc_topo_t *topo) { ucc_rank_t ctx_rank1; ucc_rank_t ctx_rank2; ucc_device_id_t dev1; ucc_device_id_t dev2; const ucc_host_info_t *host1; const ucc_host_info_t *host2; const ucc_gpu_info_t *gpu1; const ucc_gpu_info_t *gpu2; if (!ucc_topo_rank_device_info(topo, rank1, &host1, &dev1) || !ucc_topo_rank_device_info(topo, rank2, &host2, &dev2)) { return 0; } ctx_rank1 = ucc_ep_map_eval(topo->set.map, rank1); ctx_rank2 = ucc_ep_map_eval(topo->set.map, rank2); if (topo->topo->procs[ctx_rank1].host_hash != topo->topo->procs[ctx_rank2].host_hash) { return 0; } if (dev1 == dev2) { return 1; } gpu1 = &host1->gpus[dev1]; gpu2 = &host2->gpus[dev2]; if (gpu1->fabric_clique_id != UCC_GPU_FABRIC_CLIQUE_ID_INVALID && gpu1->fabric_clique_id == gpu2->fabric_clique_id) { return 1; } if (UCC_GPU_HAS_CAP(gpu1, UCC_GPU_CAP_NVSWITCH) && UCC_GPU_HAS_CAP(gpu2, UCC_GPU_CAP_NVSWITCH)) { return 1; } if (host1->nvlink_matrix[dev1][dev2] > 0) { return 1; } return 0; } static ucc_status_t sbgp_create_node_nvlink(ucc_topo_t *topo, ucc_sbgp_t *sbgp) { ucc_sbgp_t *node_sbgp = &topo->sbgps[UCC_SBGP_NODE]; ucc_rank_t group_rank = topo->set.myrank; ucc_rank_t nv_rank = 0, nv_size = 0; ucc_rank_t *local_ranks; const ucc_host_info_t *host; ucc_device_id_t dev; int i; if (node_sbgp->status == UCC_SBGP_NOT_INIT) { ucc_sbgp_create(topo, UCC_SBGP_NODE); } if (node_sbgp->status != UCC_SBGP_ENABLED) { sbgp->status = UCC_SBGP_NOT_EXISTS; return UCC_OK; } if (!ucc_topo_rank_device_info(topo, group_rank, &host, &dev)) { sbgp->status = UCC_SBGP_NOT_EXISTS; return UCC_OK; } local_ranks = ucc_malloc(node_sbgp->group_size * sizeof(ucc_rank_t), "local_ranks"); if (!local_ranks) { ucc_error("failed to allocate %zd bytes for local_ranks array", node_sbgp->group_size * sizeof(ucc_rank_t)); return UCC_ERR_NO_MEMORY; } for (i = 0; i < node_sbgp->group_size; i++) { ucc_rank_t r = ucc_ep_map_eval(node_sbgp->map, i); if (ucc_topo_ranks_on_node_nvlink(r, group_rank, topo)) { local_ranks[nv_size] = r; if (r == group_rank) { nv_rank = nv_size; } nv_size++; } } sbgp->group_size = nv_size; sbgp->group_rank = nv_rank; if (nv_size > 1) { sbgp->status = UCC_SBGP_ENABLED; sbgp->rank_map = local_ranks; } else { sbgp->status = UCC_SBGP_NOT_EXISTS; ucc_free(local_ranks); } return UCC_OK; } static inline ucc_status_t sbgp_create_sn(ucc_topo_t *topo, ucc_sbgp_t *sbgp, ucc_rank_t group_rank, int allow_size_1) { ucc_sbgp_t *node_sbgp = &topo->sbgps[UCC_SBGP_NODE]; ucc_rank_t nlr = topo->node_leader_rank; ucc_rank_t sn_rank = 0, sn_size = 0; int i, r, nlr_pos; ucc_rank_t *local_ranks; ucc_assert(node_sbgp->status == UCC_SBGP_ENABLED); local_ranks = ucc_malloc(node_sbgp->group_size * sizeof(ucc_rank_t), "local_ranks"); if (!local_ranks) { ucc_error("failed to allocate %zd bytes for local_ranks array", node_sbgp->group_size * sizeof(ucc_rank_t)); return UCC_ERR_NO_MEMORY; } for (i = 0; i < node_sbgp->group_size; i++) { r = ucc_ep_map_eval(node_sbgp->map, i); if (ucc_ranks_on_local_sn(r, group_rank, topo, sbgp->type)) { local_ranks[sn_size] = r; if (r == group_rank) { sn_rank = sn_size; } sn_size++; } } sbgp->group_size = sn_size; sbgp->group_rank = sn_rank; nlr_pos = -1; for (i = 0; i < sn_size; i++) { if (nlr == local_ranks[i]) { nlr_pos = i; break; } } if (nlr_pos > 0) { if (sn_rank == 0) sbgp->group_rank = nlr_pos; if (sn_rank == nlr_pos) sbgp->group_rank = 0; SWAP(local_ranks[nlr_pos], local_ranks[0], int); } if (sn_size > 1 || allow_size_1) { sbgp->status = UCC_SBGP_ENABLED; sbgp->rank_map = local_ranks; } else { sbgp->status = UCC_SBGP_NOT_EXISTS; ucc_free(local_ranks); } return UCC_OK; } ucc_status_t ucc_sbgp_create_node(ucc_topo_t *topo, ucc_sbgp_t *sbgp) { ucc_subset_t *set = &topo->set; ucc_rank_t group_size = ucc_subset_size(set); ucc_rank_t group_rank = set->myrank; ucc_rank_t max_local_size = 256; ucc_rank_t ctx_nlr = topo->node_leader_rank_id; ucc_rank_t node_rank = 0, node_size = 0; int i; ucc_rank_t *local_ranks, *tmp; local_ranks = ucc_malloc(max_local_size * sizeof(ucc_rank_t), "local_ranks"); if (!local_ranks) { ucc_error("failed to allocate %zd bytes for local_ranks array", max_local_size * sizeof(ucc_rank_t)); return UCC_ERR_NO_MEMORY; } for (i = 0; i < group_size; i++) { if (ucc_rank_on_local_node(i, topo)) { if (node_size == max_local_size) { max_local_size *= 2; tmp = ucc_realloc(local_ranks, max_local_size * sizeof(ucc_rank_t)); if (!tmp) { ucc_error( "failed to allocate %zd bytes for local_ranks array", max_local_size * sizeof(ucc_rank_t)); ucc_free(local_ranks); return UCC_ERR_NO_MEMORY; } local_ranks = tmp; } local_ranks[node_size] = i; if (i == group_rank) { node_rank = node_size; } node_size++; } } if (0 == node_size) { /* We should always have at least 1 local rank */ ucc_free(local_ranks); return UCC_ERR_NOT_FOUND; } sbgp->group_size = node_size; sbgp->group_rank = node_rank; sbgp->rank_map = local_ranks; if (0 < ctx_nlr && ctx_nlr < node_size) { /* Rotate local_ranks array so that node_leader_rank_id becomes first in that array */ sbgp->rank_map = ucc_malloc(node_size * sizeof(ucc_rank_t), "rank_map"); if (!sbgp->rank_map) { ucc_error("failed to allocate %zd bytes for rank_map array", node_size * sizeof(ucc_rank_t)); ucc_free(local_ranks); return UCC_ERR_NO_MEMORY; } for (i = ctx_nlr; i < node_size; i++) { sbgp->rank_map[i - ctx_nlr] = local_ranks[i]; } for (i = 0; i < ctx_nlr; i++) { sbgp->rank_map[node_size - ctx_nlr + i] = local_ranks[i]; } sbgp->group_rank = (node_rank + node_size - ctx_nlr) % node_size; ucc_free(local_ranks); } topo->node_leader_rank = sbgp->rank_map[0]; if (node_size > 1) { sbgp->status = UCC_SBGP_ENABLED; } else { sbgp->status = UCC_SBGP_NOT_EXISTS; } return UCC_OK; } static int ucc_compare_nl_ranks(const void *a, const void *b) { ucc_rank_t *r1 = (ucc_rank_t *)a; ucc_rank_t *r2 = (ucc_rank_t *)b; if(*r1 > *r2) { return 1; } else if(*r1 == *r2) { return 0; } else { return -1; } } static ucc_status_t sbgp_create_node_leaders(ucc_topo_t *topo, ucc_sbgp_t *sbgp, int ctx_nlr) { ucc_subset_t *set = &topo->set; ucc_rank_t comm_size = ucc_subset_size(set); ucc_rank_t comm_rank = set->myrank; ucc_rank_t min_sbgp_size = UCC_RANK_MAX; ucc_rank_t max_sbgp_size = 0; ucc_rank_t max_ctx_sbgp_size = 0; ucc_rank_t *nl_array_3 = NULL; int i_am_node_leader = 0; int socket_bound = topo->topo->sock_bound; int numa_bound = topo->topo->numa_bound; int bound = socket_bound || numa_bound; ucc_rank_t nnodes = topo->topo->nnodes; ucc_rank_t n_node_leaders, ctx_rank, i; ucc_rank_t *nl_array_1, *nl_array_2; ucc_host_id_t host_id; uint8_t sbgp_id; ucc_assert(comm_size != 0 && nnodes != 0); if (topo->min_ppn != UCC_RANK_MAX && ctx_nlr >= topo->min_ppn) { sbgp->status = UCC_SBGP_NOT_EXISTS; return UCC_OK; } nl_array_1 = ucc_malloc(nnodes * sizeof(ucc_rank_t), "nl_array_1"); if (!nl_array_1) { ucc_error("failed to allocate %zd bytes for nl_array_1", nnodes * sizeof(ucc_rank_t)); return UCC_ERR_NO_MEMORY; } nl_array_2 = ucc_malloc(nnodes * sizeof(ucc_rank_t), "nl_array_2"); if (!nl_array_2) { ucc_error("failed to allocate %zd bytes for nl_array_2", nnodes * sizeof(ucc_rank_t)); ucc_free(nl_array_1); return UCC_ERR_NO_MEMORY; } if (bound) { max_ctx_sbgp_size = socket_bound ? topo->topo->max_n_sockets : topo->topo->max_n_numas; nl_array_3 = ucc_malloc(max_ctx_sbgp_size * nnodes * sizeof(ucc_rank_t), "nl_array_3"); if (!nl_array_3) { ucc_error("failed to allocate %zd bytes for nl_array_3", max_ctx_sbgp_size * nnodes * sizeof(ucc_rank_t)); ucc_free(nl_array_1); ucc_free(nl_array_2); return UCC_ERR_NO_MEMORY; } memset(nl_array_3, 0, max_ctx_sbgp_size * nnodes * sizeof(ucc_rank_t)); } for (i = 0; i < nnodes; i++) { nl_array_1[i] = 0; nl_array_2[i] = UCC_RANK_MAX; } for (i = 0; i < comm_size; i++) { ctx_rank = ucc_ep_map_eval(set->map, i); host_id = topo->topo->procs[ctx_rank].host_id; if (bound) { sbgp_id = socket_bound ? topo->topo->procs[ctx_rank].socket_id : topo->topo->procs[ctx_rank].numa_id; nl_array_3[sbgp_id + host_id * max_ctx_sbgp_size]++; } /* Find the first rank that maps to this node, store in nl_array_2 */ if (nl_array_1[host_id] == 0 || nl_array_1[host_id] == ctx_nlr) { nl_array_2[host_id] = i; } nl_array_1[host_id]++; } for (i = 0; i < nnodes; i++) { if (nl_array_1[i] > topo->max_ppn) { topo->max_ppn = nl_array_1[i]; } if (nl_array_1[i] != 0 && nl_array_1[i] < topo->min_ppn) { topo->min_ppn = nl_array_1[i]; } } if (bound) { for (i = 0; i < nnodes * max_ctx_sbgp_size; i++) { if (nl_array_3[i] == 0) { continue; } min_sbgp_size = ucc_min(min_sbgp_size, nl_array_3[i]); max_sbgp_size = ucc_max(max_sbgp_size, nl_array_3[i]); } if (socket_bound) { topo->min_socket_size = min_sbgp_size; topo->max_socket_size = max_sbgp_size; } else { topo->min_numa_size = min_sbgp_size; topo->max_numa_size = max_sbgp_size; } ucc_free(nl_array_3); } n_node_leaders = 0; if (ctx_nlr >= topo->min_ppn) { /* at least one node has less number of local ranks than ctx_nlr - can't build NET sbgp */ goto skip; } for (i = 0; i < nnodes; i++) { if (nl_array_2[i] != UCC_RANK_MAX) { if (comm_rank == nl_array_2[i]) { i_am_node_leader = 1; sbgp->group_rank = n_node_leaders; } nl_array_1[n_node_leaders++] = nl_array_2[i]; } } /* order the node leader sbgp by team rank instead of host id */ qsort(nl_array_1, n_node_leaders, sizeof(ucc_rank_t), ucc_compare_nl_ranks); if (i_am_node_leader) { for (i = 0; i < n_node_leaders; i++) { // my index in nl_array_1 was swapped, find where it was swapped to if (nl_array_1[i] == comm_rank) { sbgp->group_rank = i; break; } } } skip: ucc_free(nl_array_2); if (n_node_leaders > 1) { sbgp->group_size = n_node_leaders; if (i_am_node_leader) { sbgp->status = UCC_SBGP_ENABLED; } else { sbgp->status = UCC_SBGP_DISABLED; } sbgp->rank_map = nl_array_1; } else { ucc_free(nl_array_1); sbgp->status = UCC_SBGP_NOT_EXISTS; } return UCC_OK; } #define GET_SN_ID(_topo, _proc, _type) \ (((_type) == UCC_SBGP_SOCKET_LEADERS) \ ? (_topo)->topo->procs[(_proc)].socket_id \ : (_topo)->topo->procs[(_proc)].numa_id) static ucc_status_t sbgp_create_sn_leaders(ucc_topo_t *topo, ucc_sbgp_t *sbgp) { ucc_subset_t * set = &topo->set; ucc_sbgp_t * node_sbgp = &topo->sbgps[UCC_SBGP_NODE]; ucc_rank_t comm_rank = set->myrank; ucc_rank_t nlr = topo->node_leader_rank; int i_am_sn_leader = (nlr == comm_rank); ucc_rank_t n_sn_leaders = 1; int max_n_sns = (sbgp->type == UCC_SBGP_SOCKET_LEADERS) ? topo->topo->max_n_sockets : topo->topo->max_n_numas; ucc_rank_t *sl_array; ucc_socket_id_t nlr_sock_id; int i; sl_array = ucc_malloc(max_n_sns * sizeof(ucc_rank_t), "sl_array"); if (!sl_array) { ucc_error("failed to allocate %zd bytes for sl_array", max_n_sns * sizeof(ucc_rank_t)); return UCC_ERR_NO_MEMORY; } for (i = 0; i < max_n_sns; i++) { sl_array[i] = UCC_RANK_MAX; } nlr_sock_id = GET_SN_ID(topo, ucc_ep_map_eval(set->map, nlr), sbgp->type); sl_array[nlr_sock_id] = nlr; for (i = 0; i < node_sbgp->group_size; i++) { ucc_rank_t r = ucc_ep_map_eval(node_sbgp->map, i); ucc_rank_t ctx_rank = ucc_ep_map_eval(set->map, r); ucc_socket_id_t sn_id = GET_SN_ID(topo, ctx_rank, sbgp->type); if (sl_array[sn_id] == UCC_RANK_MAX) { n_sn_leaders++; sl_array[sn_id] = r; if (r == comm_rank) { i_am_sn_leader = 1; } } } if (UCC_SBGP_SOCKET_LEADERS == sbgp->type) { topo->n_sockets = n_sn_leaders; } else { ucc_assert(UCC_SBGP_NUMA_LEADERS == sbgp->type); topo->n_numas = n_sn_leaders; } if (n_sn_leaders > 1) { ucc_rank_t sl_rank = UCC_RANK_INVALID; sbgp->rank_map = ucc_malloc(sizeof(ucc_rank_t) * n_sn_leaders, "rank_map"); if (!sbgp->rank_map) { ucc_error("failed to allocate %zd bytes for rank_map", n_sn_leaders * sizeof(ucc_rank_t)); ucc_free(sl_array); return UCC_ERR_NO_MEMORY; } n_sn_leaders = 0; for (i = 0; i < max_n_sns; i++) { if (sl_array[i] != UCC_RANK_MAX) { sbgp->rank_map[n_sn_leaders] = sl_array[i]; if (comm_rank == sl_array[i]) { sl_rank = n_sn_leaders; } n_sn_leaders++; } } int nlr_pos = -1; for (i = 0; i < n_sn_leaders; i++) { if (sbgp->rank_map[i] == nlr) { nlr_pos = i; break; } } ucc_assert(nlr_pos >= 0); sbgp->group_rank = sl_rank; if (nlr_pos > 0) { if (sl_rank == 0) sbgp->group_rank = nlr_pos; if (sl_rank == nlr_pos) sbgp->group_rank = 0; SWAP(sbgp->rank_map[nlr_pos], sbgp->rank_map[0], int); } sbgp->group_size = n_sn_leaders; if (i_am_sn_leader) { sbgp->status = UCC_SBGP_ENABLED; } else { sbgp->status = UCC_SBGP_DISABLED; } } else { sbgp->status = UCC_SBGP_NOT_EXISTS; } ucc_free(sl_array); return UCC_OK; } static inline ucc_status_t sbgp_create_full(ucc_topo_t *topo, ucc_sbgp_t *sbgp) { sbgp->status = UCC_SBGP_ENABLED; sbgp->group_size = ucc_subset_size(&topo->set); sbgp->group_rank = topo->set.myrank; sbgp->map.type = UCC_EP_MAP_FULL; sbgp->map.ep_num = ucc_subset_size(&topo->set); return UCC_OK; } typedef struct proc_info_id { ucc_proc_info_t info; ucc_rank_t id; } proc_info_id_t; static int ucc_compare_proc_info_id(const void *a, const void *b) { const ucc_proc_info_t *d1 = &((const proc_info_id_t *)a)->info; const ucc_proc_info_t *d2 = &((const proc_info_id_t *)b)->info; if (d1->host_hash != d2->host_hash) { return d1->host_hash > d2->host_hash ? 1 : -1; } else if (d1->socket_id != d2->socket_id) { return d1->socket_id - d2->socket_id; } else if (d1->numa_id != d2->numa_id) { return d1->numa_id - d2->numa_id; } else { return 0; } } static ucc_status_t sbgp_create_full_ordered(ucc_topo_t *topo, ucc_sbgp_t *sbgp) { ucc_rank_t gsize = ucc_subset_size(&topo->set); ucc_proc_info_t *pinfo; ucc_host_id_t *visited; proc_info_id_t *sorted; ucc_rank_t i, j, num_visited; int is_sorted, d; if (!topo->topo || !topo->topo->procs) { ucc_error("sbgp_create_full_ordered: invalid topo, topo->topo=%p, procs=%p", topo->topo, topo->topo ? topo->topo->procs : NULL); return UCC_ERR_INVALID_PARAM; } pinfo = topo->topo->procs; ucc_assert(gsize > 0); sbgp->status = UCC_SBGP_ENABLED; sbgp->group_size = gsize; sbgp->group_rank = topo->set.myrank; sbgp->rank_map = ucc_malloc(sizeof(ucc_rank_t) * gsize, "rank_map"); if (ucc_unlikely(!sbgp->rank_map)) { ucc_error("failed to allocate %zd bytes for rank_map", gsize * sizeof(ucc_rank_t)); return UCC_ERR_NO_MEMORY; } visited = (ucc_host_id_t *)ucc_malloc(gsize * sizeof(ucc_host_id_t), "visited host"); if (ucc_unlikely(!visited)) { ucc_error("failed to allocate %zd bytes for list of visited nodes", gsize * sizeof(ucc_host_id_t)); ucc_free(sbgp->rank_map); return UCC_ERR_NO_MEMORY; } is_sorted = 1; num_visited = 1; visited[0] = pinfo[0].host_hash; for (i = 1; i < gsize; i++) { if (pinfo[i].host_hash != pinfo[i-1].host_hash) { /* check if we saw that host_hash before */ for (j = 0; j < num_visited; j++) { if (visited[j] == pinfo[i].host_hash) { break; } } if (j < num_visited) { /* this host was present already, ranks are not ordered */ is_sorted = 0; break; } /* add new host to the list of visited */ visited[num_visited++] = pinfo[i].host_hash; } else { d = ucc_compare_proc_info_id(&pinfo[i - 1], &pinfo[i]); if (d > 0) { is_sorted = 0; break; } } } ucc_free(visited); if (is_sorted) { for (i = 0; i < gsize; i++) { sbgp->rank_map[i] = i; } return UCC_OK; } sorted = (proc_info_id_t *)ucc_malloc(gsize * sizeof(proc_info_id_t), "proc_sorted"); if (ucc_unlikely(!sorted)) { ucc_error("failed to allocate %zd bytes for sorted proc info", gsize * sizeof(proc_info_id_t)); ucc_free(sbgp->rank_map); return UCC_ERR_NO_MEMORY; } for (i = 0; i < gsize; i++) { sorted[i].info = topo->topo->procs[i]; sorted[i].id = i; } qsort(sorted, gsize, sizeof(proc_info_id_t), ucc_compare_proc_info_id); for (i = 0; i < gsize; i++) { if (sorted[i].id == topo->set.myrank) { sbgp->group_rank = i; } sbgp->rank_map[i] = sorted[i].id; } /*TODO: try to detect map by numa,socket,node and use UCC_EP_MAP_CB to save * memory */ ucc_free(sorted); return UCC_OK; } ucc_status_t ucc_sbgp_create(ucc_topo_t *topo, ucc_sbgp_type_t type) { ucc_status_t status = UCC_OK; ucc_sbgp_t * sbgp = &topo->sbgps[type]; int sn_bound = UCC_TOPO_IS_BOUND(topo, type); sbgp->type = type; sbgp->status = UCC_SBGP_NOT_EXISTS; sbgp->rank_map = NULL; switch (type) { case UCC_SBGP_NODE: status = ucc_sbgp_create_node(topo, sbgp); break; case UCC_SBGP_NODE_NVLINK: status = sbgp_create_node_nvlink(topo, sbgp); break; case UCC_SBGP_FULL: status = sbgp_create_full(topo, sbgp); break; case UCC_SBGP_FULL_HOST_ORDERED: status = sbgp_create_full_ordered(topo, sbgp); break; case UCC_SBGP_SOCKET: case UCC_SBGP_NUMA: if (!sn_bound) { break; } if (topo->sbgps[UCC_SBGP_NODE].status == UCC_SBGP_NOT_INIT) { ucc_sbgp_create(topo, UCC_SBGP_NODE); } if (topo->sbgps[UCC_SBGP_NODE].status == UCC_SBGP_ENABLED) { status = sbgp_create_sn(topo, sbgp, topo->set.myrank, 0); } break; case UCC_SBGP_NODE_LEADERS: ucc_assert(UCC_SBGP_DISABLED != topo->sbgps[UCC_SBGP_NODE].status); status = sbgp_create_node_leaders(topo, sbgp, topo->node_leader_rank_id); break; case UCC_SBGP_NET: if (topo->sbgps[UCC_SBGP_NODE].status == UCC_SBGP_NOT_INIT) { ucc_sbgp_create(topo, UCC_SBGP_NODE); } ucc_assert(UCC_SBGP_DISABLED != topo->sbgps[UCC_SBGP_NODE].status); status = sbgp_create_node_leaders( topo, sbgp, topo->sbgps[UCC_SBGP_NODE].group_rank); break; case UCC_SBGP_SOCKET_LEADERS: case UCC_SBGP_NUMA_LEADERS: if (!sn_bound) { break; } if (topo->sbgps[UCC_SBGP_NODE].status == UCC_SBGP_NOT_INIT) { ucc_sbgp_create(topo, UCC_SBGP_NODE); } if (topo->sbgps[UCC_SBGP_NODE].status == UCC_SBGP_ENABLED) { status = sbgp_create_sn_leaders(topo, sbgp); } break; default: status = UCC_ERR_NOT_IMPLEMENTED; break; }; if (sbgp->rank_map && sbgp->type != UCC_SBGP_FULL) { sbgp->map = ucc_ep_map_from_array(&sbgp->rank_map, sbgp->group_size, ucc_subset_size(&topo->set), 1); } if (sbgp->rank_map && sbgp->status == UCC_SBGP_NOT_EXISTS) { ucc_free(sbgp->rank_map); } return status; } ucc_status_t ucc_sbgp_cleanup(ucc_sbgp_t *sbgp) { if (sbgp->rank_map) { ucc_free(sbgp->rank_map); sbgp->rank_map = NULL; } return UCC_OK; } void ucc_sbgp_print(ucc_sbgp_t *sbgp) { int i; if (sbgp->group_rank == 0 && sbgp->status == UCC_SBGP_ENABLED) { printf("sbgp: %15s: group_size %4d, team_ranks=[ ", ucc_sbgp_str(sbgp->type), sbgp->group_size); for (i = 0; i < sbgp->group_size; i++) { printf("%d ", sbgp->rank_map[i]); } printf("]"); printf("\n"); } } ucc_status_t ucc_sbgp_create_all_sns(ucc_topo_t *topo, ucc_sbgp_t **_sbgps, int *n_sbgps, ucc_sbgp_type_t type) { int sn_bound = UCC_TOPO_IS_BOUND(topo, type); ucc_sbgp_t *sbgps; ucc_sbgp_t * sn_leaders_sbgp; int n_sn_groups, i; ucc_rank_t sl_rank; ucc_status_t status; if (!sn_bound) { return UCC_ERR_NOT_FOUND; } sn_leaders_sbgp = ucc_topo_get_sbgp(topo, (UCC_SBGP_SOCKET == type) ? UCC_SBGP_SOCKET_LEADERS : UCC_SBGP_NUMA_LEADERS); n_sn_groups = (UCC_SBGP_SOCKET == type) ? topo->n_sockets : topo->n_numas; ucc_assert(n_sn_groups >= 1); if (topo->sbgps[UCC_SBGP_NODE].status != UCC_SBGP_ENABLED || topo->sbgps[UCC_SBGP_NODE].group_size < 1) { /* second conditional is to suppress LINTER */ return UCC_ERR_NOT_FOUND; } sbgps = ucc_calloc(n_sn_groups, sizeof(ucc_sbgp_t), "sn_sbgps"); if (!sbgps) { return UCC_ERR_NO_MEMORY; } for (i = 0; i < n_sn_groups; i++) { sbgps[i].type = type; sl_rank = (n_sn_groups > 1) ? ucc_ep_map_eval(sn_leaders_sbgp->map, i) : ucc_ep_map_eval(topo->sbgps[UCC_SBGP_NODE].map, 0); status = sbgp_create_sn(topo, &sbgps[i], sl_rank, 1); if (UCC_OK != status) { ucc_error("failed to create socket sbgp for sl_rank %d:%u", i, sl_rank); goto error; } if (sbgps[i].rank_map) { sbgps[i].map = ucc_ep_map_from_array(&sbgps[i].rank_map, sbgps[i].group_size, ucc_subset_size(&topo->set), 1); } } *_sbgps = sbgps; *n_sbgps = n_sn_groups; return UCC_OK; error: ucc_free(sbgps); return status; } ucc_status_t ucc_sbgp_create_all_sockets(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps) { return ucc_sbgp_create_all_sns(topo, sbgps, n_sbgps, UCC_SBGP_SOCKET); } ucc_status_t ucc_sbgp_create_all_numas(ucc_topo_t *topo, ucc_sbgp_t **sbgps, int *n_sbgps) { return ucc_sbgp_create_all_sns(topo, sbgps, n_sbgps, UCC_SBGP_NUMA); } ucc_status_t ucc_sbgp_create_all_node_nvlinks(ucc_topo_t *topo, ucc_sbgp_t **_sbgps, int *n_sbgps) { ucc_sbgp_t *node_sbgp = &topo->sbgps[UCC_SBGP_NODE]; ucc_sbgp_t *sbgps; ucc_rank_t local_size; ucc_rank_t *local_ranks; int *comp_id; int i, j; int n_comps = 0; ucc_status_t status = UCC_OK; if (node_sbgp->status == UCC_SBGP_NOT_INIT) { ucc_sbgp_create(topo, UCC_SBGP_NODE); } if (node_sbgp->status != UCC_SBGP_ENABLED || node_sbgp->group_size < 1 || !ucc_sbgp_has_device_info(topo)) { return UCC_ERR_NOT_FOUND; } local_size = node_sbgp->group_size; local_ranks = node_sbgp->rank_map; comp_id = ucc_malloc(local_size * sizeof(*comp_id), "nvlink_comp_id"); if (!comp_id) { return UCC_ERR_NO_MEMORY; } for (i = 0; i < (int)local_size; i++) { comp_id[i] = -1; } for (i = 0; i < (int)local_size; i++) { int queue_len = 0; int qidx = 0; int *queue; if (comp_id[i] != -1) { continue; } queue = ucc_malloc(local_size * sizeof(*queue), "nvlink_queue"); if (!queue) { status = UCC_ERR_NO_MEMORY; goto error; } comp_id[i] = n_comps; queue[queue_len++] = i; while (qidx < queue_len) { int cur = queue[qidx++]; for (j = 0; j < (int)local_size; j++) { if (comp_id[j] != -1) { continue; } if (ucc_topo_ranks_on_node_nvlink(local_ranks[cur], local_ranks[j], topo)) { comp_id[j] = n_comps; queue[queue_len++] = j; } } } ucc_free(queue); n_comps++; } sbgps = ucc_calloc(n_comps, sizeof(*sbgps), "nvlink_sbgps"); if (!sbgps) { status = UCC_ERR_NO_MEMORY; goto error; } for (i = 0; i < n_comps; i++) { ucc_rank_t count = 0; ucc_rank_t myrank = topo->set.myrank; int myrank_pos = -1; for (j = 0; j < (int)local_size; j++) { if (comp_id[j] == i) { count++; } } sbgps[i].type = UCC_SBGP_NODE_NVLINK; sbgps[i].status = UCC_SBGP_ENABLED; sbgps[i].group_size = count; sbgps[i].rank_map = ucc_malloc(count * sizeof(ucc_rank_t), "nvlink_rank_map"); if (!sbgps[i].rank_map) { status = UCC_ERR_NO_MEMORY; goto error_alloc; } count = 0; for (j = 0; j < (int)local_size; j++) { if (comp_id[j] == i) { sbgps[i].rank_map[count] = local_ranks[j]; if (local_ranks[j] == myrank) { myrank_pos = count; } count++; } } sbgps[i].group_rank = (myrank_pos >= 0) ? myrank_pos : 0; sbgps[i].map = ucc_ep_map_from_array(&sbgps[i].rank_map, sbgps[i].group_size, ucc_subset_size(&topo->set), 1); } ucc_free(comp_id); *_sbgps = sbgps; *n_sbgps = n_comps; return UCC_OK; error_alloc: for (i = 0; i < n_comps; i++) { if (sbgps[i].rank_map) { ucc_free(sbgps[i].rank_map); } } ucc_free(sbgps); error: ucc_free(comp_id); return status; } ucc-1.8.0/src/components/ec/0000775000175000017500000000000015211535620016021 5ustar alastairalastairucc-1.8.0/src/components/ec/cuda/0000775000175000017500000000000015211535620016735 5ustar alastairalastairucc-1.8.0/src/components/ec/cuda/kernel/0000775000175000017500000000000015211535620020215 5ustar alastairalastairucc-1.8.0/src/components/ec/cuda/kernel/ec_cuda_executor_reduce_complex.cu0000664000175000017500000000214115211535620027123 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UINT32_MAX #define __STDC_LIMIT_MACROS #include #endif #include "ec_cuda_executor_reduce_dev.h" #define UCC_EC_CUDA_REDUCE_OPS_DEVICE_ONLY #include "ec_cuda_reduce_ops.h" #undef UCC_EC_CUDA_REDUCE_OPS_DEVICE_ONLY __device__ ucc_status_t executor_reduce_complex( ucc_ee_executor_task_args_t *task, ucc_reduction_op_t op, ucc_datatype_t dt) { switch (dt) { case UCC_DT_FLOAT32_COMPLEX: #if SIZEOF_CUFLOATCOMPLEX == 8 DT_REDUCE_FLOAT_COMPLEX( cuFloatComplex, float, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_FOUR); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT64_COMPLEX: #if SIZEOF_CUDOUBLECOMPLEX == 16 DT_REDUCE_FLOAT_COMPLEX( cuDoubleComplex, double, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_TWO); break; #else return UCC_ERR_NOT_SUPPORTED; #endif default: return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } ucc-1.8.0/src/components/ec/cuda/kernel/ec_cuda_half_sm52.h0000664000175000017500000001073115211535620023613 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms of ucc. * * This file is copy-pasted from cuda_fp16.hpp in the CUDA toolkit and modified. * See the original cuda_fp16.hpp for terms of cuda_fp16.hpp. */ /** * We copy-pasted and modify cuda_fp16.hpp because half operators are only available * for SM>=5.3 but we need to support earlier architectures. On earlier architectures, * we emulate the operators by converting half to float, do the operation, then convert * the result back to half. * Since CUDA 12.2 similar functionality was added to cuda_fp16.hpp */ #pragma once #include /* Global-space operator functions are only available to nvcc compilation */ #if defined(__CUDACC__) /* Arithmetic FP16 operations in cuda_fp16.hpp only supported on arch >= 5.3, * however, we support early architectures*/ #if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 530) && (CUDA_VERSION < 12020) #if !defined(__CUDA_NO_HALF_OPERATORS__) /* Some basic arithmetic operations expected of a builtin */ __device__ __forceinline__ __half operator+(const __half &lh, const __half &rh) { return __float2half(__half2float(lh) + __half2float(rh)); } __device__ __forceinline__ __half operator-(const __half &lh, const __half &rh) { return __float2half(__half2float(lh) - __half2float(rh)); } __device__ __forceinline__ __half operator*(const __half &lh, const __half &rh) { return __float2half(__half2float(lh) * __half2float(rh)); } __device__ __forceinline__ __half operator/(const __half &lh, const __half &rh) { return __float2half(__half2float(lh) / __half2float(rh)); } __device__ __forceinline__ __half &operator+=(__half &lh, const __half &rh) { lh = __float2half(__half2float(lh) + __half2float(rh)); return lh; } __device__ __forceinline__ __half &operator-=(__half &lh, const __half &rh) { lh = __float2half(__half2float(lh) - __half2float(rh)); return lh; } __device__ __forceinline__ __half &operator*=(__half &lh, const __half &rh) { lh = __float2half(__half2float(lh) * __half2float(rh)); return lh; } __device__ __forceinline__ __half &operator/=(__half &lh, const __half &rh) { lh = __float2half(__half2float(lh) / __half2float(rh)); return lh; } /* Note for increment and decrement we use the raw value 0x3C00U equating to half(1.0F), to avoid the extra conversion */ __device__ __forceinline__ __half &operator++(__half &h) { __half_raw one; one.x = 0x3C00U; h += one; return h; } __device__ __forceinline__ __half &operator--(__half &h) { __half_raw one; one.x = 0x3C00U; h -= one; return h; } __device__ __forceinline__ __half operator++(__half &h, const int ignored) { // ignored on purpose. Parameter only needed to distinguish the function declaration from other types of operators. static_cast(ignored); const __half ret = h; __half_raw one; one.x = 0x3C00U; h += one; return ret; } __device__ __forceinline__ __half operator--(__half &h, const int ignored) { // ignored on purpose. Parameter only needed to distinguish the function declaration from other types of operators. static_cast(ignored); const __half ret = h; __half_raw one; one.x = 0x3C00U; h -= one; return ret; } /* Unary plus and inverse operators */ __device__ __forceinline__ __half operator+(const __half &h) { return h; } __device__ __forceinline__ __half operator-(const __half &h) { return __float2half(-__half2float(h)); } /* Some basic comparison operations to make it look like a builtin */ __device__ __forceinline__ bool operator==(const __half &lh, const __half &rh) { return __half2float(lh) == __half2float(rh); } __device__ __forceinline__ bool operator!=(const __half &lh, const __half &rh) { return __half2float(lh) != __half2float(rh); } __device__ __forceinline__ bool operator>(const __half &lh, const __half &rh) { return __half2float(lh) > __half2float(rh); } __device__ __forceinline__ bool operator<(const __half &lh, const __half &rh) { return __half2float(lh) < __half2float(rh); } __device__ __forceinline__ bool operator>=(const __half &lh, const __half &rh) { return __half2float(lh) >= __half2float(rh); } __device__ __forceinline__ bool operator<=(const __half &lh, const __half &rh) { return __half2float(lh) <= __half2float(rh); } #endif /* !defined(__CUDA_NO_HALF_OPERATORS__) */ #endif /* defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 530) */ #endif /* defined(__CUDACC__) */ ucc-1.8.0/src/components/ec/cuda/kernel/ec_cuda_executor_reduce_dev.h0000664000175000017500000000471515211535620026063 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_EC_CUDA_EXECUTOR_REDUCE_DEV_H_ #define UCC_EC_CUDA_EXECUTOR_REDUCE_DEV_H_ extern "C" { #include "../ec_cuda.h" } #define LAUNCH_REDUCE_A(NAME, _Type, _AlphaType, _task, _unroll, ...) \ do { \ if (_task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE) { \ ucc_reduce_cuda_##NAME< \ _Type, \ _AlphaType, \ true, \ false, \ _unroll, \ ucc_eee_task_reduce_t>(_task->reduce, _task->flags); \ } else { \ ucc_reduce_cuda_##NAME< \ _Type, \ _AlphaType, \ true, \ true, \ _unroll, \ ucc_eee_task_reduce_strided_t>( \ _task->reduce_strided, _task->flags); \ } \ return UCC_OK; \ } while (0) #define LAUNCH_REDUCE(NAME, _Type, _task, _unroll, ...) \ LAUNCH_REDUCE_A(NAME, _Type, _Type, _task, _unroll) __device__ ucc_status_t executor_reduce_int( ucc_ee_executor_task_args_t *task, ucc_reduction_op_t op, ucc_datatype_t dt); __device__ ucc_status_t executor_reduce_fp( ucc_ee_executor_task_args_t *task, ucc_reduction_op_t op, ucc_datatype_t dt); __device__ ucc_status_t executor_reduce_complex( ucc_ee_executor_task_args_t *task, ucc_reduction_op_t op, ucc_datatype_t dt); #endif ucc-1.8.0/src/components/ec/cuda/kernel/ec_cuda_wait_kernel.cu0000664000175000017500000000161515211535620024520 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UINT32_MAX #define __STDC_LIMIT_MACROS #include #endif #ifdef __cplusplus extern "C" { #endif #include "../ec_cuda.h" #ifdef __cplusplus } #endif __global__ void wait_kernel(volatile ucc_ec_cuda_executor_state_t *state) { ucc_ec_cuda_executor_state_t st; *state = UCC_EC_CUDA_EXECUTOR_STARTED; do { st = *state; } while (st != UCC_EC_CUDA_EXECUTOR_SHUTDOWN); *state = UCC_EC_CUDA_EXECUTOR_SHUTDOWN_ACK; return; } #ifdef __cplusplus extern "C" { #endif ucc_status_t ucc_ec_cuda_post_kernel_stream_task(ucc_ec_cuda_executor_state_t *state, cudaStream_t stream) { wait_kernel<<<1, 1, 0, stream>>>(state); CUDA_CHECK(cudaGetLastError()); return UCC_OK; } #ifdef __cplusplus } #endif ucc-1.8.0/src/components/ec/cuda/kernel/ec_cuda_executor_reduce_int.cu0000664000175000017500000000273015211535620026252 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UINT32_MAX #define __STDC_LIMIT_MACROS #include #endif #include "ec_cuda_executor_reduce_dev.h" #define UCC_EC_CUDA_REDUCE_OPS_DEVICE_ONLY #include "ec_cuda_reduce_ops.h" #undef UCC_EC_CUDA_REDUCE_OPS_DEVICE_ONLY __device__ ucc_status_t executor_reduce_int( ucc_ee_executor_task_args_t *task, ucc_reduction_op_t op, ucc_datatype_t dt) { switch (dt) { case UCC_DT_INT8: DT_REDUCE_INT(int8_t, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_TWO); break; case UCC_DT_INT16: DT_REDUCE_INT(int16_t, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_TWO); break; case UCC_DT_INT32: DT_REDUCE_INT(int32_t, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_FOUR); break; case UCC_DT_INT64: DT_REDUCE_INT(int64_t, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_FOUR); break; case UCC_DT_UINT8: DT_REDUCE_INT(uint8_t, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_TWO); break; case UCC_DT_UINT16: DT_REDUCE_INT(uint16_t, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_TWO); break; case UCC_DT_UINT32: DT_REDUCE_INT(uint32_t, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_TWO); break; case UCC_DT_UINT64: DT_REDUCE_INT(uint64_t, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_TWO); break; default: return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } ucc-1.8.0/src/components/ec/cuda/kernel/ec_cuda_reduce_ops.h0000664000175000017500000005405315211535620024170 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_EC_CUDA_REDUCE_OPS_H_ #define UCC_EC_CUDA_REDUCE_OPS_H_ extern "C" { #include "utils/ucc_math_op.h" #include "../ec_cuda.h" } #include "ec_cuda_half_sm52.h" #include #include #define COPY_LOOP_UNROLL 8 #define REDUCE_LOOP_UNROLL_TRIGGERED_FOUR 4 #define REDUCE_LOOP_UNROLL_TRIGGERED_TWO 2 #define REDUCE_LOOP_UNROLL_TRIGGERED_ONE 1 #define REDUCE_LOOP_UNROLL_INTERRUPTIBLE 1 typedef int4 vectype; __device__ inline cuDoubleComplex operator+ (const cuDoubleComplex & first, const cuDoubleComplex & second) { return cuCadd(first, second); } __device__ inline cuDoubleComplex operator* (const cuDoubleComplex & first, const cuDoubleComplex & second) { return cuCmul(first, second); } __device__ inline cuDoubleComplex operator* (const cuDoubleComplex & first, const double & second) { return make_cuDoubleComplex(cuCreal(first) * second, cuCimag(first) * second); } __device__ inline cuFloatComplex operator+ (const cuFloatComplex & first, const cuFloatComplex & second) { return cuCaddf(first, second); } __device__ inline cuFloatComplex operator* (const cuFloatComplex & first, const cuFloatComplex & second) { return cuCmulf(first, second); } __device__ inline cuFloatComplex operator* (const cuFloatComplex & first, const float & second) { return make_cuFloatComplex(cuCrealf(first) * second, cuCimagf(first) * second); } template __device__ int ptrAlignVec(T *ptr) { return (uint64_t)ptr % sizeof(VecType); } template __forceinline__ __device__ void LoadVec(T *d, T *s) { *(reinterpret_cast(d)) = *(reinterpret_cast(s)); } #define CUDA_REDUCE_WITH_OP_CHUNK(unroll, unroll_group_size, _OP, VecType) \ do { \ const int vectorize = sizeof(VecType) / sizeof(Type); \ const int group = \ triggered \ ? threadIdx.x / unroll_group_size \ : (threadIdx.x + blockIdx.x * blockDim.x) / unroll_group_size; \ const int num_groups = \ triggered ? blockDim.x / unroll_group_size \ : (blockDim.x * gridDim.x) / unroll_group_size; \ const int idx = threadIdx.x % unroll_group_size; \ const int factor = unroll_group_size * unroll * vectorize; \ const size_t start = \ offset + (group * unroll_group_size * unroll + idx) * vectorize; \ const size_t step = num_groups * factor; \ const size_t end = offset + ((count - offset) / factor) * factor; \ Type tmp1[unroll][vectorize]; \ Type tmp2[unroll][vectorize]; \ ucc_assert_system(blockDim.x % unroll_group_size == 0); \ for (line = start; line < end; line += step) { \ _Pragma("unroll") for (i = 0; i < unroll; i++) \ { \ LoadVec( \ tmp1[i], &s1[line + vectorize * unroll_group_size * i]); \ } \ for (j = 0; j < MAXSRCS && j < n_src2; j++) { \ _Pragma("unroll") for (i = 0; i < unroll; i++) \ { \ if (strided) { \ LoadVec( \ tmp2[i], \ &s2[line + vectorize * unroll_group_size * i + \ j * ld]); \ } else { \ LoadVec( \ tmp2[i], \ &s[1 + j] \ [line + vectorize * unroll_group_size * i]); \ } \ } \ _Pragma("unroll") for (i = 0; i < unroll; i++) \ { \ _Pragma("unroll") for (k = 0; k < vectorize; k++) \ { \ tmp1[i][k] = _OP(tmp1[i][k], tmp2[i][k]); \ } \ } \ } \ if (flags & UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA) { \ _Pragma("unroll") for (i = 0; i < unroll; i++) \ { \ _Pragma("unroll") for (k = 0; k < vectorize; k++) \ { \ tmp1[i][k] = tmp1[i][k] * (AlphaType)task.alpha; \ } \ } \ } \ _Pragma("unroll") for (i = 0; i < unroll; i++) \ { \ LoadVec( \ &d[line + vectorize * unroll_group_size * i], tmp1[i]); \ } \ } \ offset = max(offset, end); \ if (offset == count) { \ return; \ } \ } while (0) #define CUDA_REDUCE_WITH_OP_DEVICE(NAME, _OP) \ template < \ typename Type, \ typename AlphaType, \ bool triggered, \ bool strided, \ int UNROLL, \ typename TaskType> \ __device__ void ucc_reduce_cuda_##NAME(TaskType task, uint16_t flags) \ { \ Type *d = (Type *)task.dst; \ bool alignedVec = 0; \ size_t offset = 0; \ const size_t count = task.count; \ const int MAXSRCS = strided ? USHRT_MAX : UCC_EE_EXECUTOR_NUM_BUFS; \ const int ALLOC_SIZE = strided ? 1 : UCC_EE_EXECUTOR_NUM_BUFS; \ Type *s[ALLOC_SIZE]; \ Type *s1; \ Type *s2; \ __shared__ uint16_t n_src2; \ size_t ld; \ size_t i, j, k, line; \ if (strided) { \ ucc_eee_task_reduce_strided_t \ *task_strided_p = (ucc_eee_task_reduce_strided_t *)&task; \ n_src2 = task_strided_p->n_src2; \ s1 = (Type *)task_strided_p->src1; \ s2 = (Type *)task_strided_p->src2; \ ld = task_strided_p->stride / sizeof(Type); \ ucc_assert_system(task_strided_p->stride % sizeof(Type) == 0); \ alignedVec |= ptrAlignVec(s1); \ alignedVec |= ptrAlignVec(s2); \ alignedVec |= ((task_strided_p->stride % sizeof(vectype)) != 0); \ } else { \ ucc_eee_task_reduce_t \ *task_default_p = (ucc_eee_task_reduce_t *)&task; \ memcpy( \ s, \ task_default_p->srcs, \ UCC_EE_EXECUTOR_NUM_BUFS * sizeof(Type *)); \ n_src2 = task_default_p->n_srcs - 1; \ s1 = s[0]; \ for (int i = 0; i < MAXSRCS && i <= n_src2; i++) { \ alignedVec |= ptrAlignVec(s[i]); \ } \ } \ alignedVec |= ptrAlignVec(d); \ ucc_assert_system(sizeof(vectype) % sizeof(Type) == 0); \ /* Successive calls to CUDA_REDUCE_WITH_OP_CHUNK to reduce the buffer.*/ \ /* Each call enables or disables vectorization and/or loop unrolling */ \ /* optimizations. Each one of the four calls except the last one may */ \ /* only reduce the buffer partially and leave data remainder to be */ \ /* treated by the subsequent calls. */ \ if (triggered && alignedVec == 0) { \ CUDA_REDUCE_WITH_OP_CHUNK(UNROLL, WARP_SIZE, _OP, vectype); \ CUDA_REDUCE_WITH_OP_CHUNK(1, 1, _OP, vectype); \ } \ CUDA_REDUCE_WITH_OP_CHUNK(UNROLL, WARP_SIZE, _OP, Type); \ CUDA_REDUCE_WITH_OP_CHUNK(1, 1, _OP, Type); \ } #ifndef UCC_EC_CUDA_REDUCE_OPS_DEVICE_ONLY #define CUDA_REDUCE_WITH_OP(NAME, _OP) \ CUDA_REDUCE_WITH_OP_DEVICE(NAME, _OP) \ template \ __global__ void UCC_REDUCE_CUDA_DEFAULT_##NAME( \ ucc_eee_task_reduce_t task, uint16_t flags) \ { \ ucc_reduce_cuda_##NAME< \ Type, \ AlphaType, \ triggered, \ false, \ UNROLL, \ ucc_eee_task_reduce_t>(task, flags); \ } \ template \ __global__ void UCC_REDUCE_CUDA_STRIDED_##NAME( \ ucc_eee_task_reduce_strided_t task, uint16_t flags) \ { \ ucc_reduce_cuda_##NAME< \ Type, \ AlphaType, \ triggered, \ true, \ UNROLL, \ ucc_eee_task_reduce_strided_t>(task, flags); \ } #else #define CUDA_REDUCE_WITH_OP(NAME, _OP) CUDA_REDUCE_WITH_OP_DEVICE(NAME, _OP) #endif #ifndef UCC_EC_CUDA_REDUCE_OPS_DEVICE_ONLY #define CUDA_REDUCE_WITH_OP_MULTI_DST(NAME, _OP) \ template \ __global__ void UCC_REDUCE_CUDA_MULTI_DST_##NAME( \ ucc_eee_task_reduce_multi_dst_t arg) \ { \ size_t start = \ triggered ? threadIdx.x : threadIdx.x + blockIdx.x * blockDim.x; \ size_t step = triggered ? blockDim.x : blockDim.x * gridDim.x; \ for (int j = 0; j < arg.n_bufs; j++) { \ size_t count = arg.counts[j]; \ _Type *s2 = (_Type *)arg.src2[j]; \ _Type *s1 = (_Type *)arg.src1[j]; \ _Type *d = (_Type *)arg.dst[j]; \ for (size_t i = start; i < count; i += step) { \ d[i] = _OP##_2(s1[i], s2[i]); \ } \ } \ } #endif CUDA_REDUCE_WITH_OP(SUM, DO_OP_SUM); CUDA_REDUCE_WITH_OP(PROD, DO_OP_PROD); CUDA_REDUCE_WITH_OP(MIN, DO_OP_MIN); CUDA_REDUCE_WITH_OP(MAX, DO_OP_MAX); CUDA_REDUCE_WITH_OP(LAND, DO_OP_LAND); CUDA_REDUCE_WITH_OP(LOR, DO_OP_LOR); CUDA_REDUCE_WITH_OP(LXOR, DO_OP_LXOR); CUDA_REDUCE_WITH_OP(BAND, DO_OP_BAND); CUDA_REDUCE_WITH_OP(BOR, DO_OP_BOR); CUDA_REDUCE_WITH_OP(BXOR, DO_OP_BXOR); #ifndef UCC_EC_CUDA_REDUCE_OPS_DEVICE_ONLY CUDA_REDUCE_WITH_OP_MULTI_DST(SUM, DO_OP_SUM); CUDA_REDUCE_WITH_OP_MULTI_DST(PROD, DO_OP_PROD); CUDA_REDUCE_WITH_OP_MULTI_DST(MIN, DO_OP_MIN); CUDA_REDUCE_WITH_OP_MULTI_DST(MAX, DO_OP_MAX); CUDA_REDUCE_WITH_OP_MULTI_DST(LAND, DO_OP_LAND); CUDA_REDUCE_WITH_OP_MULTI_DST(LOR, DO_OP_LOR); CUDA_REDUCE_WITH_OP_MULTI_DST(LXOR, DO_OP_LXOR); CUDA_REDUCE_WITH_OP_MULTI_DST(BAND, DO_OP_BAND); CUDA_REDUCE_WITH_OP_MULTI_DST(BOR, DO_OP_BOR); CUDA_REDUCE_WITH_OP_MULTI_DST(BXOR, DO_OP_BXOR); #endif #define DT_REDUCE_INT(_Type, _task, _op, ...) \ do { \ switch (_op) { \ case UCC_OP_AVG: \ case UCC_OP_SUM: \ LAUNCH_REDUCE(SUM, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_PROD: \ LAUNCH_REDUCE(PROD, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_MIN: \ LAUNCH_REDUCE(MIN, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_MAX: \ LAUNCH_REDUCE(MAX, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_LAND: \ LAUNCH_REDUCE(LAND, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_BAND: \ LAUNCH_REDUCE(BAND, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_LOR: \ LAUNCH_REDUCE(LOR, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_BOR: \ LAUNCH_REDUCE(BOR, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_LXOR: \ LAUNCH_REDUCE(LXOR, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_BXOR: \ LAUNCH_REDUCE(BXOR, _Type, _task, __VA_ARGS__); \ break; \ default: \ return UCC_ERR_NOT_SUPPORTED; \ } \ } while (0) #define DT_REDUCE_FLOAT_COMPLEX(_Type, _alphaType, _task, _op, ...) \ do { \ switch (_op) { \ case UCC_OP_AVG: \ case UCC_OP_SUM: \ LAUNCH_REDUCE_A(SUM, _Type, _alphaType, _task, __VA_ARGS__); \ break; \ case UCC_OP_PROD: \ LAUNCH_REDUCE_A(PROD, _Type, _alphaType, _task, __VA_ARGS__); \ break; \ default: \ return UCC_ERR_NOT_SUPPORTED; \ } \ } while (0) #define DT_REDUCE_FLOAT(_Type, _task, _op, ...) \ do { \ switch (_op) { \ case UCC_OP_AVG: \ case UCC_OP_SUM: \ LAUNCH_REDUCE(SUM, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_PROD: \ LAUNCH_REDUCE(PROD, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_MIN: \ LAUNCH_REDUCE(MIN, _Type, _task, __VA_ARGS__); \ break; \ case UCC_OP_MAX: \ LAUNCH_REDUCE(MAX, _Type, _task, __VA_ARGS__); \ break; \ default: \ return UCC_ERR_NOT_SUPPORTED; \ } \ } while (0) #endif ucc-1.8.0/src/components/ec/cuda/kernel/ec_cuda_executor_reduce_fp.cu0000664000175000017500000000236515211535620026071 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UINT32_MAX #define __STDC_LIMIT_MACROS #include #endif #include "ec_cuda_executor_reduce_dev.h" #define UCC_EC_CUDA_REDUCE_OPS_DEVICE_ONLY #include "ec_cuda_reduce_ops.h" #undef UCC_EC_CUDA_REDUCE_OPS_DEVICE_ONLY __device__ ucc_status_t executor_reduce_fp( ucc_ee_executor_task_args_t *task, ucc_reduction_op_t op, ucc_datatype_t dt) { switch (dt) { case UCC_DT_FLOAT16: DT_REDUCE_FLOAT(__half, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_TWO); break; case UCC_DT_FLOAT32: #if SIZEOF_FLOAT == 4 DT_REDUCE_FLOAT(float, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_FOUR); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT64: #if SIZEOF_DOUBLE == 8 DT_REDUCE_FLOAT(double, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_FOUR); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_BFLOAT16: ucc_assert_system(2 == sizeof(__nv_bfloat16)); DT_REDUCE_FLOAT( __nv_bfloat16, task, op, REDUCE_LOOP_UNROLL_TRIGGERED_TWO); break; default: return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } ucc-1.8.0/src/components/ec/cuda/kernel/ec_cuda_reduce.cu0000664000175000017500000001463215211535620023466 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UINT32_MAX #define __STDC_LIMIT_MACROS #include #endif extern "C" { #include "../ec_cuda.h" } #include "ec_cuda_reduce_ops.h" #define align_pow2(_n, _p) ((_n) & ((_p) - 1)) __device__ inline void add_float4(float4 &d, const float4 &x, const float4 &y) { d.x = x.x + y.x; d.y = x.y + y.y; d.z = x.z + y.z; d.w = x.w + y.w; } template <> __global__ void UCC_REDUCE_CUDA_MULTI_DST_SUM(ucc_eee_task_reduce_multi_dst_t arg) { int blocks_per_buf = gridDim.x / arg.n_bufs; int buf_id = blockIdx.x / blocks_per_buf; size_t step = blockDim.x * blocks_per_buf; int idx = threadIdx.x + (blockIdx.x % blocks_per_buf) * blockDim.x; int align; align = align_pow2((intptr_t)arg.src1[buf_id], 16) | align_pow2((intptr_t)arg.src2[buf_id], 16) | align_pow2((intptr_t)arg.dst[buf_id], 16); if (align == 0) { /* aligned */ size_t count = arg.counts[buf_id] / 4; const float4 *s14 = (float4*)arg.src1[buf_id]; const float4 *s24 = (float4*)arg.src2[buf_id]; float4 *d4 = (float4*)arg.dst[buf_id]; for (size_t i = idx; i < count; i += step) { add_float4(d4[i], s14[i], s24[i]); } if (idx < arg.counts[buf_id] % 4) { size_t lidx = arg.counts[buf_id] - idx - 1; ((float*)arg.dst[buf_id])[lidx] = ((float*)arg.src1[buf_id])[lidx] + ((float*)arg.src2[buf_id])[lidx]; } } else { size_t count = arg.counts[buf_id]; const float *s1 = (float*)arg.src1[buf_id]; const float *s2 = (float*)arg.src2[buf_id]; float *d = (float*)arg.dst[buf_id]; for (size_t i = idx; i < count; i += step) { d[i] = s1[i] + s2[i]; } } } #define LAUNCH_REDUCE_A(NAME, type, _AlphaType, _task, s, b, t) \ do { \ if (_task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE) { \ UCC_REDUCE_CUDA_DEFAULT_##NAME \ <<>>(_task->reduce, _task->flags); \ } else if (_task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED) { \ UCC_REDUCE_CUDA_STRIDED_##NAME \ <<>>(_task->reduce_strided, _task->flags); \ } else { \ UCC_REDUCE_CUDA_MULTI_DST_##NAME \ <<>>(_task->reduce_multi_dst); \ } \ } while (0) #define LAUNCH_REDUCE(NAME, type, _task, s, b, t) \ LAUNCH_REDUCE_A(NAME, type, type, _task, s, b, t) extern "C" { ucc_status_t ucc_ec_cuda_reduce( ucc_ee_executor_task_args_t *task, unsigned num_threads, unsigned num_blocks, cudaStream_t stream) { int th = num_threads; unsigned long bk = num_blocks; ucc_reduction_op_t op; ucc_datatype_t dt; size_t count; int i; if (task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE) { count = task->reduce.count; dt = task->reduce.dt; op = task->reduce.op; bk = ucc_min((count + th - 1) / th, bk); } else if (task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED) { count = task->reduce_strided.count; dt = task->reduce_strided.dt; op = task->reduce_strided.op; bk = ucc_min((count + th - 1) / th, bk); } else { if (task->reduce_multi_dst.n_bufs == 0) { return UCC_OK; } count = 0; for (i = 0 ; i < task->reduce_multi_dst.n_bufs; i++) { count += task->reduce_multi_dst.counts[i]; } dt = task->reduce_multi_dst.dt; op = task->reduce_multi_dst.op; bk = 4 * task->reduce_multi_dst.n_bufs; } if (count == 0) { return UCC_OK; } switch (dt) { case UCC_DT_INT8: DT_REDUCE_INT(int8_t, task, op, stream, bk, th); break; case UCC_DT_INT16: DT_REDUCE_INT(int16_t, task, op, stream, bk, th); break; case UCC_DT_INT32: DT_REDUCE_INT(int32_t, task, op, stream, bk, th); break; case UCC_DT_INT64: DT_REDUCE_INT(int64_t, task, op, stream, bk, th); break; case UCC_DT_UINT8: DT_REDUCE_INT(uint8_t, task, op, stream, bk, th); break; case UCC_DT_UINT16: DT_REDUCE_INT(uint16_t, task, op, stream, bk, th); break; case UCC_DT_UINT32: DT_REDUCE_INT(uint32_t, task, op, stream, bk, th); break; case UCC_DT_UINT64: DT_REDUCE_INT(uint64_t, task, op, stream, bk, th); break; case UCC_DT_FLOAT16: DT_REDUCE_FLOAT(__half, task, op, stream, bk, th); break; case UCC_DT_FLOAT32: #if SIZEOF_FLOAT == 4 DT_REDUCE_FLOAT(float, task, op, stream, bk, th); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT64: #if SIZEOF_DOUBLE == 8 DT_REDUCE_FLOAT(double, task, op, stream, bk, th); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT32_COMPLEX: #if SIZEOF_CUFLOATCOMPLEX == 8 DT_REDUCE_FLOAT_COMPLEX(cuFloatComplex, float, task, op, stream, bk, th); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT64_COMPLEX: #if SIZEOF_CUDOUBLECOMPLEX == 16 DT_REDUCE_FLOAT_COMPLEX(cuDoubleComplex, double, task, op, stream, bk, th); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_BFLOAT16: ucc_assert_system(2 == sizeof(__nv_bfloat16)); DT_REDUCE_FLOAT(__nv_bfloat16, task, op, stream, bk, th); break; default: ec_error(&ucc_ec_cuda.super, "unsupported reduction type (%s)", ucc_datatype_str(dt)); return UCC_ERR_NOT_SUPPORTED; } CUDA_CHECK(cudaGetLastError()); return UCC_OK; } } ucc-1.8.0/src/components/ec/cuda/kernel/Makefile.am0000664000175000017500000000677315211535620022266 0ustar alastairalastair# # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # NVCCFLAGS = \ ${AM_CPPFLAGS} \ ${UCS_CPPFLAGS} \ ${NVCC_CFLAGS} \ -I${UCC_TOP_BUILDDIR} \ -I${UCC_TOP_SRCDIR}/src \ -I${UCC_TOP_BUILDDIR}/src \ --compiler-options -fno-rtti,-fno-exceptions NVCCFLAGS_RDC = $(NVCCFLAGS) -rdc=true LINK = $(LIBTOOL) --tag=CXX --mode=link $(NVCC) -o $@ CCLD = $(NVCC) .cu.o: $(NVCC) -c $< -o $@ $(NVCCFLAGS) $(NVCC_ARCH) .cu.lo: /bin/bash $(top_srcdir)/cuda_lt.sh "$(LIBTOOL)" $@ $(NVCC) -c $< $(NVCCFLAGS) $(NVCC_ARCH) ec_cuda_executor.lo: ec_cuda_executor.cu /bin/bash $(top_srcdir)/cuda_lt.sh "$(LIBTOOL)" $@ $(NVCC) -c $< $(NVCCFLAGS_RDC) $(NVCC_ARCH) ec_cuda_executor_reduce_int.lo: ec_cuda_executor_reduce_int.cu /bin/bash $(top_srcdir)/cuda_lt.sh "$(LIBTOOL)" $@ $(NVCC) -c $< $(NVCCFLAGS_RDC) $(NVCC_ARCH) ec_cuda_executor_reduce_fp.lo: ec_cuda_executor_reduce_fp.cu /bin/bash $(top_srcdir)/cuda_lt.sh "$(LIBTOOL)" $@ $(NVCC) -c $< $(NVCCFLAGS_RDC) $(NVCC_ARCH) ec_cuda_executor_reduce_complex.lo: ec_cuda_executor_reduce_complex.cu /bin/bash $(top_srcdir)/cuda_lt.sh "$(LIBTOOL)" $@ $(NVCC) -c $< $(NVCCFLAGS_RDC) $(NVCC_ARCH) # Device link step for RDC objects: resolves cross-TU __device__ calls. # Produces a proper libtool .lo object so that libtool reliably includes it # when the convenience library is absorbed into the parent shared library. ec_cuda_executor_dlink.lo: ec_cuda_executor.lo ec_cuda_executor_reduce_int.lo ec_cuda_executor_reduce_fp.lo ec_cuda_executor_reduce_complex.lo $(NVCC) -dlink \ .libs/ec_cuda_executor.o \ .libs/ec_cuda_executor_reduce_int.o \ .libs/ec_cuda_executor_reduce_fp.o \ .libs/ec_cuda_executor_reduce_complex.o \ -o .libs/ec_cuda_executor_dlink.o $(NVCC_ARCH) -Xcompiler -fPIC @cp -f .libs/ec_cuda_executor_dlink.o ec_cuda_executor_dlink.o @echo "# ec_cuda_executor_dlink.lo - a libtool object file" > $@ @echo "# Generated by libtool (device link step)" >> $@ @echo "" >> $@ @echo "# Please DO NOT delete this file!" >> $@ @echo "# It is necessary for linking the library." >> $@ @echo "" >> $@ @echo "# Name of the PIC object." >> $@ @echo "pic_object='.libs/ec_cuda_executor_dlink.o'" >> $@ @echo "" >> $@ @echo "# Name of the non-PIC object." >> $@ @echo "non_pic_object='ec_cuda_executor_dlink.o'" >> $@ CLEANFILES = ec_cuda_executor_dlink.lo ec_cuda_executor_dlink.o comp_noinst = libucc_ec_cuda_kernels.la libucc_ec_cuda_kernels_la_SOURCES = ec_cuda_wait_kernel.cu \ ec_cuda_executor.cu \ ec_cuda_executor_reduce_int.cu \ ec_cuda_executor_reduce_fp.cu \ ec_cuda_executor_reduce_complex.cu \ ec_cuda_reduce.cu libucc_ec_cuda_kernels_la_CPPFLAGS = libucc_ec_cuda_kernels_la_LDFLAGS = -prefer-pic libucc_ec_cuda_kernels_la_LIBADD = ec_cuda_executor_dlink.lo libucc_ec_cuda_kernels_la_DEPENDENCIES = ec_cuda_executor_dlink.lo noinst_LTLIBRARIES = $(comp_noinst) ucc-1.8.0/src/components/ec/cuda/kernel/ec_cuda_executor.cu0000664000175000017500000002171415211535620024054 0ustar alastairalastair/** * Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UINT32_MAX #define __STDC_LIMIT_MACROS #include "utils/arch/cuda_def.h" #include #endif extern "C" { #include "../ec_cuda.h" } #include "ec_cuda_executor_reduce_dev.h" #include using namespace cooperative_groups; #include "ec_cuda_reduce_ops.h" #define align_pow2(_n, _p) ((_n) & ((_p) - 1)) __global__ void executor_start(ucc_ec_cuda_executor_state_t *state, int *cidx) { *cidx = 0; *state = UCC_EC_CUDA_EXECUTOR_STARTED; } __global__ void executor_shutdown_ack(ucc_ec_cuda_executor_state_t *state) { *state = UCC_EC_CUDA_EXECUTOR_SHUTDOWN_ACK; } template __device__ void executor_copy_task(ucc_eee_task_copy_t &task) { size_t count = task.len; const char *s1 = reinterpret_cast(task.src); char *d1 = reinterpret_cast(task.dst); if (!(align_pow2((intptr_t)s1, sizeof(vectype)) || align_pow2((intptr_t)d1, sizeof(vectype)))) { int warp = threadIdx.x / WARP_SIZE; int num_warps = blockDim.x / WARP_SIZE; int idx = threadIdx.x % WARP_SIZE; const vectype *s4 = reinterpret_cast(s1); vectype *d4 = reinterpret_cast(d1); size_t num_lines = (count / (WARP_SIZE * UNROLL * sizeof(vectype))) * (WARP_SIZE * UNROLL); vectype tmp[UNROLL]; for (size_t line = warp * WARP_SIZE * UNROLL + idx; line < num_lines; line += num_warps * WARP_SIZE * UNROLL) { #pragma unroll for (int i = 0; i < UNROLL; i++) { tmp[i] = s4[line + WARP_SIZE * i]; } #pragma unroll for (int i = 0; i < UNROLL; i++) { d4[line + WARP_SIZE * i] = tmp[i]; } } count = count - num_lines * sizeof(vectype); if (count == 0) { return; } s4 = s4 + num_lines; d4 = d4 + num_lines; num_lines = count / sizeof(vectype); for (int line = threadIdx.x; line < num_lines; line += blockDim.x) { d4[line] = s4[line]; } count = count - num_lines * sizeof(vectype); if (count == 0) { return; } s1 = reinterpret_cast(s4 + num_lines); d1 = reinterpret_cast(d4 + num_lines); } for (size_t line = threadIdx.x; line < count; line += blockDim.x) { d1[line] = s1[line]; } } __device__ ucc_status_t executor_reduce(ucc_ee_executor_task_args_t *task) { ucc_reduction_op_t op; ucc_datatype_t dt; size_t count; if (task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE) { dt = task->reduce.dt; count = task->reduce.count; op = task->reduce.op; } else { ucc_assert_system(task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED); dt = task->reduce_strided.dt; count = task->reduce_strided.count; op = task->reduce_strided.op; } if (count == 0) { return UCC_OK; } if (executor_reduce_int(task, op, dt) == UCC_OK) { return UCC_OK; } if (executor_reduce_fp(task, op, dt) == UCC_OK) { return UCC_OK; } return executor_reduce_complex(task, op, dt); } __global__ void executor_kernel( volatile ucc_ec_cuda_executor_t *eee, int q_size, int useCoopLaunch) { const uint32_t worker_id = blockIdx.x; const uint32_t num_workers = gridDim.x; bool is_master = (threadIdx.x == 0) ? true: false; grid_group grid = this_grid(); int cidx_local, pidx_local; volatile int *pidx, *cidx; ucc_ee_executor_task_args_t *tasks; __shared__ ucc_ee_executor_task_args_t args; __shared__ bool worker_done; if (is_master) { if (useCoopLaunch) { *eee->dev_cidx = 0; *eee->dev_state = UCC_EC_CUDA_EXECUTOR_STARTED; } cidx_local = worker_id; pidx = eee->dev_pidx; cidx = eee->dev_cidx; tasks = eee->dev_tasks; } worker_done = false; if (useCoopLaunch) { grid.sync(); } else { __syncthreads(); } while (1) { if (is_master) { while ((*cidx % num_workers) != worker_id); do { pidx_local = *pidx; } while (*cidx == pidx_local); (*cidx)++; worker_done = (pidx_local == -1); if (!worker_done) { args = tasks[cidx_local]; } } __syncthreads(); if (worker_done) { if (useCoopLaunch) { grid.sync(); if (is_master && (worker_id == 0)) { *eee->dev_state = UCC_EC_CUDA_EXECUTOR_SHUTDOWN_ACK; } } return; } switch (args.task_type) { case UCC_EE_EXECUTOR_TASK_COPY: executor_copy_task(args.copy); break; case UCC_EE_EXECUTOR_TASK_REDUCE: case UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED: executor_reduce(&args); break; default: break; } __syncthreads(); __threadfence_system(); if (is_master) { tasks[cidx_local].task_type = UCC_EE_EXECUTOR_TASK_LAST; cidx_local = (cidx_local + num_workers) % q_size; } } } extern "C" { ucc_status_t ucc_ec_cuda_executor_kernel_calc_max_threads(int *max) { cudaFuncAttributes attr; CUDA_CHECK(cudaFuncGetAttributes(&attr, executor_kernel)); *max = (attr.maxThreadsPerBlock / WARP_SIZE) * WARP_SIZE; return UCC_OK; } ucc_status_t ucc_ec_cuda_persistent_kernel_start( ucc_ec_cuda_executor_t *eee, unsigned num_threads, unsigned num_blocks) { cudaStream_t stream = (cudaStream_t)eee->super.ee_context; int q_size = EC_CUDA_CONFIG->exec_max_tasks; int useCoopLaunch = EC_CUDA_CONFIG->use_cooperative_launch; int kernelUseCoop = useCoopLaunch; if (useCoopLaunch) { void *kernelArgs[] = {&eee, &q_size, &kernelUseCoop}; dim3 dimBlock(num_threads, 1, 1); dim3 dimGrid(num_blocks, 1, 1); cudaLaunchCooperativeKernel( (void *)executor_kernel, dimGrid, dimBlock, kernelArgs, 0, stream); } else { executor_start<<<1, 1, 0, stream>>>(eee->dev_state, eee->dev_cidx); executor_kernel<<>>( eee, q_size, kernelUseCoop); executor_shutdown_ack<<<1, 1, 0, stream>>>(eee->dev_state); } CUDA_CHECK(cudaGetLastError()); return UCC_OK; } __global__ void kernel_copy_multi(ucc_eee_task_copy_multi_t args) { int blocks_per_buf = gridDim.x / args.num_vectors; int buf_id = blockIdx.x / blocks_per_buf; char1 *src = (char1*)args.src[buf_id]; char1 *dst = (char1*)args.dst[buf_id]; size_t cnt = args.counts[buf_id]; size_t start = threadIdx.x + (blockIdx.x % blocks_per_buf) * blockDim.x; size_t step = blockDim.x * blocks_per_buf; for (size_t i = start; i < cnt; i += step) { dst[i] = src[i]; } } __global__ void kernel_copy_multi_aligned(ucc_eee_task_copy_multi_t args) { int blocks_per_buf = gridDim.x / args.num_vectors; int buf_id = blockIdx.x / blocks_per_buf; int idx = threadIdx.x + (blockIdx.x % blocks_per_buf) * blockDim.x; int step = blockDim.x * blocks_per_buf; size_t n = args.counts[buf_id] / sizeof(uint4); size_t num_iter = n / step + ((idx < n % step) ? 1 : 0); uint4 *src = (uint4*)args.src[buf_id]; uint4 *dst = (uint4*)args.dst[buf_id]; for(size_t i = 0; i < num_iter; i++) { dst[i * step + idx] = src[i * step + idx]; } if (idx < (args.counts[buf_id] % sizeof(uint4))) { ((char*)args.dst[buf_id])[args.counts[buf_id] - idx - 1] = ((char*)args.src[buf_id])[args.counts[buf_id] - idx - 1]; } } ucc_status_t ucc_ec_cuda_copy_multi_kernel(const ucc_ee_executor_task_args_t *args, cudaStream_t stream) { int nt = 1024; int nb = args->copy_multi.num_vectors * 4; int aligned = 1; for (int i = 0; i < args->copy_multi.num_vectors; i++) { if (align_pow2((intptr_t)args->copy_multi.src[i], 16) || align_pow2((intptr_t)args->copy_multi.dst[i], 16)) { aligned = 0; break; } } if (aligned) { kernel_copy_multi_aligned<<>>(args->copy_multi); } else { kernel_copy_multi<<>>(args->copy_multi); } CUDA_CHECK(cudaGetLastError()); return UCC_OK; } } ucc-1.8.0/src/components/ec/cuda/ec_cuda.h0000664000175000017500000000311615211535620020472 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_EC_CUDA_H_ #define UCC_EC_CUDA_H_ #include "components/ec/base/ucc_ec_base.h" #include "components/ec/ucc_ec_log.h" #include "utils/arch/cuda_def.h" #include "utils/ucc_mpool.h" #include "ec_cuda_resources.h" #include typedef ucc_status_t (*ucc_ec_cuda_task_post_fn) (uint32_t *dev_status, int blocking_wait, cudaStream_t stream); typedef struct ucc_ec_cuda { ucc_ec_base_t super; int exec_streams_initialized; ucc_ec_cuda_resources_hash_t *resources_hash; ucc_thread_mode_t thread_mode; ucc_ec_cuda_strm_task_mode_t strm_task_mode; ucc_spinlock_t init_spinlock; } ucc_ec_cuda_t; typedef struct ucc_ec_cuda_stream_request { uint32_t status; uint32_t *dev_status; cudaStream_t stream; } ucc_ec_cuda_stream_request_t; ucc_status_t ucc_ec_cuda_event_create(void **event); ucc_status_t ucc_ec_cuda_event_destroy(void *event); ucc_status_t ucc_ec_cuda_event_post(void *ee_context, void *event); ucc_status_t ucc_ec_cuda_event_test(void *event); ucc_status_t ucc_ec_cuda_get_resources(ucc_ec_cuda_resources_t **resources); extern ucc_ec_cuda_t ucc_ec_cuda; #define EC_CUDA_CONFIG \ (ucc_derived_of(ucc_ec_cuda.super.config, ucc_ec_cuda_config_t)) #endif ucc-1.8.0/src/components/ec/cuda/ec_cuda_executor_persistent.c0000664000175000017500000001427715211535620024675 0ustar alastairalastair/** * Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ec_cuda_executor.h" #include "utils/arch/cpu.h" ucc_status_t ucc_cuda_executor_persistent_task_post(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); int max_tasks = EC_CUDA_CONFIG->exec_max_tasks; ucc_ee_executor_task_args_t *subtask_args; ucc_ec_cuda_executor_persistent_task_t *ee_task; int i; ucc_ec_cuda_resources_t *resources; ucc_status_t status; status = ucc_ec_cuda_get_resources(&resources); if (ucc_unlikely(status != UCC_OK)) { return status; } if (ucc_ec_cuda.thread_mode == UCC_THREAD_MULTIPLE) { ucc_spin_lock(&eee->tasks_lock); } ee_task = ucc_mpool_get(&resources->executor_persistent_tasks); if (ucc_unlikely(!ee_task)) { return UCC_ERR_NO_MEMORY; } ee_task->super.status = UCC_INPROGRESS; ee_task->super.eee = executor; if (task_args->task_type == UCC_EE_EXECUTOR_TASK_COPY_MULTI) { ee_task->num_subtasks = task_args->copy_multi.num_vectors; for (i = 0; i < ee_task->num_subtasks; i++) { subtask_args = &(eee->tasks[(eee->pidx + i) % max_tasks]); subtask_args->task_type = UCC_EE_EXECUTOR_TASK_COPY; subtask_args->copy.src = task_args->copy_multi.src[i]; subtask_args->copy.dst = task_args->copy_multi.dst[i]; subtask_args->copy.len = task_args->copy_multi.counts[i]; ee_task->subtasks[i] = subtask_args; } } else if (task_args->task_type == UCC_EE_EXECUTOR_TASK_REDUCE_MULTI_DST) { ee_task->num_subtasks = task_args->reduce_multi_dst.n_bufs; for (i = 0; i < ee_task->num_subtasks; i++) { subtask_args = &(eee->tasks[(eee->pidx + i) % max_tasks]); subtask_args->task_type = UCC_EE_EXECUTOR_TASK_REDUCE; subtask_args->reduce.srcs[0] = task_args->reduce_multi_dst.src1[i]; subtask_args->reduce.srcs[1] = task_args->reduce_multi_dst.src2[i]; subtask_args->reduce.dst = task_args->reduce_multi_dst.dst[i]; subtask_args->reduce.count = task_args->reduce_multi_dst.counts[i]; subtask_args->reduce.dt = task_args->reduce_multi_dst.dt; subtask_args->reduce.op = task_args->reduce_multi_dst.op; subtask_args->reduce.n_srcs = 2; ee_task->subtasks[i] = subtask_args; } } else { ee_task->num_subtasks = 1; ee_task->subtasks[0] = &(eee->tasks[eee->pidx % max_tasks]); memcpy(ee_task->subtasks[0], task_args, sizeof(ucc_ee_executor_task_args_t)); } ucc_memory_cpu_store_fence(); eee->pidx += ee_task->num_subtasks; if (ucc_ec_cuda.thread_mode == UCC_THREAD_MULTIPLE) { ucc_spin_unlock(&eee->tasks_lock); } ec_debug(&ucc_ec_cuda.super, "executor task post, eee: %p", eee); *task = &ee_task->super; return UCC_OK; } ucc_status_t ucc_cuda_executor_persistent_task_test(const ucc_ee_executor_task_t *task) { ucc_ec_cuda_executor_persistent_task_t *ee_task; ucc_status_t status; int i; ee_task = ucc_derived_of(task, ucc_ec_cuda_executor_persistent_task_t); if (ee_task->super.status != UCC_INPROGRESS) { goto exit; } status = CUDA_FUNC(cudaGetLastError()); if (ucc_unlikely(status != UCC_OK)) { ee_task->super.status = status; goto exit; } for (i = 0; i < ee_task->num_subtasks; i++) { if (ee_task->subtasks[i]->task_type != UCC_EE_EXECUTOR_TASK_LAST) { goto exit; } } ee_task->super.status = UCC_OK; exit: return ee_task->super.status; } ucc_status_t ucc_cuda_executor_persistent_task_finalize(ucc_ee_executor_task_t *task) { ucc_assert(task->status == UCC_OK); ucc_mpool_put(task); return UCC_OK; } ucc_status_t ucc_cuda_executor_persistent_start(ucc_ee_executor_t *executor, void *ee_context) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); ucc_ec_cuda_resources_t *resources; ucc_status_t status; ucc_assert(eee->state == UCC_EC_CUDA_EXECUTOR_INITIALIZED); ec_debug(&ucc_ec_cuda.super, "executor start, eee: %p", eee); eee->super.ee_context = ee_context; eee->state = UCC_EC_CUDA_EXECUTOR_POSTED; eee->pidx = 0; eee->mode = UCC_EC_CUDA_EXECUTOR_MODE_PERSISTENT; status = ucc_ec_cuda_get_resources(&resources); if (ucc_unlikely(status != UCC_OK)) { return status; } status = ucc_ec_cuda_persistent_kernel_start( eee, resources->num_threads_exec, resources->num_blocks_exec); if (status != UCC_OK) { ec_error(&ucc_ec_cuda.super, "failed to launch executor kernel"); return status; } eee->ops.task_post = ucc_cuda_executor_persistent_task_post; eee->ops.task_test = ucc_cuda_executor_persistent_task_test; eee->ops.task_finalize = ucc_cuda_executor_persistent_task_finalize; return UCC_OK; } ucc_status_t ucc_cuda_executor_persistent_stop(ucc_ee_executor_t *executor) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); volatile ucc_ec_cuda_executor_state_t *st = &eee->state; ec_debug(&ucc_ec_cuda.super, "executor stop, eee: %p", eee); /* can be safely ended only if it's in STARTED or COMPLETED_ACK state */ ucc_assert((*st != UCC_EC_CUDA_EXECUTOR_POSTED) && (*st != UCC_EC_CUDA_EXECUTOR_SHUTDOWN)); *st = UCC_EC_CUDA_EXECUTOR_SHUTDOWN; eee->pidx = -1; while(*st != UCC_EC_CUDA_EXECUTOR_SHUTDOWN_ACK) { } eee->super.ee_context = NULL; eee->state = UCC_EC_CUDA_EXECUTOR_INITIALIZED; return UCC_OK; } ucc-1.8.0/src/components/ec/cuda/ec_cuda_resources.h0000664000175000017500000001177015211535620022571 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_EC_CUDA_RESOURCES_H_ #define UCC_EC_CUDA_RESOURCES_H_ #include "components/ec/base/ucc_ec_base.h" #include "utils/arch/cuda_def.h" #include "utils/ucc_mpool.h" #define MAX_SUBTASKS 12 #define WARP_SIZE 32 typedef enum ucc_ec_cuda_executor_state { UCC_EC_CUDA_EXECUTOR_INITIALIZED, UCC_EC_CUDA_EXECUTOR_POSTED, UCC_EC_CUDA_EXECUTOR_STARTED, UCC_EC_CUDA_EXECUTOR_SHUTDOWN, UCC_EC_CUDA_EXECUTOR_SHUTDOWN_ACK } ucc_ec_cuda_executor_state_t; typedef enum ucc_ec_cuda_executor_mode { UCC_EC_CUDA_EXECUTOR_MODE_PERSISTENT, UCC_EC_CUDA_EXECUTOR_MODE_INTERRUPTIBLE } ucc_ec_cuda_executor_mode_t; typedef struct ucc_ec_cuda_event { cudaEvent_t event; } ucc_ec_cuda_event_t; typedef struct ucc_ec_cuda_executor_task_ops { ucc_status_t (*task_post)(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task); ucc_status_t (*task_test)(const ucc_ee_executor_task_t *task); ucc_status_t (*task_finalize)(ucc_ee_executor_task_t *task); } ucc_ec_cuda_executor_task_ops_t; typedef struct ucc_ec_cuda_executor { ucc_ee_executor_t super; ucc_ec_cuda_executor_mode_t mode; uint64_t requested_ops; ucc_ec_cuda_executor_task_ops_t ops; ucc_spinlock_t tasks_lock; ucc_ec_cuda_executor_state_t state; int pidx; ucc_ee_executor_task_args_t *tasks; ucc_ec_cuda_executor_state_t *dev_state; ucc_ee_executor_task_args_t *dev_tasks; int *dev_pidx; int *dev_cidx; } ucc_ec_cuda_executor_t; typedef struct ucc_ec_cuda_executor_interruptible_task { ucc_ee_executor_task_t super; void *event; cudaGraph_t graph; cudaGraphExec_t graph_exec; } ucc_ec_cuda_executor_interruptible_task_t; typedef struct ucc_ec_cuda_executor_persistent_task { ucc_ee_executor_task_t super; int num_subtasks; ucc_ee_executor_task_args_t *subtasks[MAX_SUBTASKS]; } ucc_ec_cuda_executor_persistent_task_t; typedef struct ucc_ec_cuda_resources { CUcontext cu_ctx; ucc_mpool_t events; ucc_mpool_t executors; ucc_mpool_t executor_interruptible_tasks; ucc_mpool_t executor_persistent_tasks; int streams_initialized; int num_streams; cudaStream_t *exec_streams; int num_threads_reduce; int num_blocks_reduce; int num_threads_exec; int num_blocks_exec; } ucc_ec_cuda_resources_t; typedef enum ucc_ec_cuda_strm_task_mode { UCC_EC_CUDA_TASK_KERNEL, UCC_EC_CUDA_TASK_MEM_OPS, UCC_EC_CUDA_TASK_AUTO, UCC_EC_CUDA_TASK_LAST, } ucc_ec_cuda_strm_task_mode_t; typedef struct ucc_ec_cuda_config { ucc_ec_config_t super; ucc_ec_cuda_strm_task_mode_t strm_task_mode; unsigned long exec_num_workers; unsigned long exec_num_threads; unsigned long exec_max_tasks; unsigned long exec_num_streams; unsigned long reduce_num_blocks; int reduce_num_threads; int use_cooperative_launch; unsigned long exec_copy_thresh; } ucc_ec_cuda_config_t; extern ucc_ec_cuda_config_t *ucc_ec_cuda_config; ucc_status_t ucc_ec_cuda_resources_init(ucc_ec_base_t *ec, ucc_ec_cuda_resources_t *resources); void ucc_ec_cuda_resources_cleanup(ucc_ec_cuda_resources_t *resources); KHASH_INIT(ucc_ec_cuda_resources_hash, unsigned long long, void*, 1, \ kh_int64_hash_func, kh_int64_hash_equal); #define ucc_ec_cuda_resources_hash_t khash_t(ucc_ec_cuda_resources_hash) static inline void* ec_cuda_resources_hash_get(ucc_ec_cuda_resources_hash_t *h, unsigned long long key) { khiter_t k; void *value; k = kh_get(ucc_ec_cuda_resources_hash, h , key); if (k == kh_end(h)) { return NULL; } value = kh_value(h, k); return value; } static inline void ec_cuda_resources_hash_put(ucc_ec_cuda_resources_hash_t *h, unsigned long long key, void *value) { int ret; khiter_t k; k = kh_put(ucc_ec_cuda_resources_hash, h, key, &ret); kh_value(h, k) = value; } static inline void* ec_cuda_resources_hash_pop(ucc_ec_cuda_resources_hash_t *h) { void *resources = NULL; khiter_t k; k = kh_begin(h); while (k != kh_end(h)) { if (kh_exist(h, k)) { resources = kh_value(h, k); break; } k++; } if (resources) { kh_del(ucc_ec_cuda_resources_hash, h, k); } return resources; } #endif ucc-1.8.0/src/components/ec/cuda/ec_cuda_executor.c0000664000175000017500000001154215211535620022405 0ustar alastairalastair/** * Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ec_cuda_executor.h" #include "components/ec/ucc_ec_log.h" ucc_status_t ucc_cuda_executor_interruptible_start(ucc_ee_executor_t *executor); ucc_status_t ucc_cuda_executor_interruptible_stop(ucc_ee_executor_t *executor); ucc_status_t ucc_cuda_executor_persistent_start(ucc_ee_executor_t *executor, void *ee_context); ucc_status_t ucc_cuda_executor_persistent_stop(ucc_ee_executor_t *executor); ucc_status_t ucc_cuda_executor_persistent_wait_start(ucc_ee_executor_t *executor, void *ee_context); ucc_status_t ucc_cuda_executor_persistent_wait_stop(ucc_ee_executor_t *executor); ucc_status_t ucc_cuda_executor_init(const ucc_ee_executor_params_t *params, ucc_ee_executor_t **executor) { ucc_ec_cuda_executor_t *eee; ucc_ec_cuda_resources_t *resources; ucc_status_t status; status = ucc_ec_cuda_get_resources(&resources); if (ucc_unlikely(status != UCC_OK)) { return status; } eee = ucc_mpool_get(&resources->executors); if (ucc_unlikely(!eee)) { ec_error(&ucc_ec_cuda.super, "failed to allocate executor"); return UCC_ERR_NO_MEMORY; } if (params->mask & UCC_EE_EXECUTOR_PARAM_FIELD_TASK_TYPES) { eee->requested_ops = params->task_types; } else { /* if no task types provided assume all tasks types required */ eee->requested_ops = 1; } ec_trace(&ucc_ec_cuda.super, "executor init, eee: %p", eee); eee->super.ee_type = params->ee_type; eee->state = UCC_EC_CUDA_EXECUTOR_INITIALIZED; *executor = &eee->super; return UCC_OK; } ucc_status_t ucc_cuda_executor_status(const ucc_ee_executor_t *executor) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); switch (eee->state) { case UCC_EC_CUDA_EXECUTOR_INITIALIZED: return UCC_OPERATION_INITIALIZED; case UCC_EC_CUDA_EXECUTOR_POSTED: return UCC_INPROGRESS; case UCC_EC_CUDA_EXECUTOR_STARTED: return UCC_OK; default: /* executor has been destroyed */ return UCC_ERR_NO_RESOURCE; } } ucc_status_t ucc_cuda_executor_finalize(ucc_ee_executor_t *executor) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); ec_trace(&ucc_ec_cuda.super, "executor free, eee: %p", eee); ucc_assert(eee->state == UCC_EC_CUDA_EXECUTOR_INITIALIZED); ucc_mpool_put(eee); return UCC_OK; } ucc_status_t ucc_cuda_executor_task_post(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); return eee->ops.task_post(executor, task_args, task); } ucc_status_t ucc_cuda_executor_task_test(const ucc_ee_executor_task_t *task) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(task->eee, ucc_ec_cuda_executor_t); return eee->ops.task_test(task); } ucc_status_t ucc_cuda_executor_task_finalize(ucc_ee_executor_task_t *task) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(task->eee, ucc_ec_cuda_executor_t); return eee->ops.task_finalize(task); } ucc_status_t ucc_cuda_executor_start(ucc_ee_executor_t *executor, void *ee_context) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); if (!ee_context) { return ucc_cuda_executor_interruptible_start(executor); } else { if (eee->requested_ops == 0) { /* no operations requested, just mark stream busy */ return ucc_cuda_executor_persistent_wait_start(executor, ee_context); } else { return ucc_cuda_executor_persistent_start(executor, ee_context); } } } ucc_status_t ucc_cuda_executor_stop(ucc_ee_executor_t *executor) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); if (eee->mode == UCC_EC_CUDA_EXECUTOR_MODE_INTERRUPTIBLE) { return ucc_cuda_executor_interruptible_stop(executor); } else { if (eee->requested_ops == 0) { return ucc_cuda_executor_persistent_wait_stop(executor); } else { return ucc_cuda_executor_persistent_stop(executor); } } } ucc-1.8.0/src/components/ec/cuda/ec_cuda_executor_interruptible.c0000664000175000017500000002043015211535620025351 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ec_cuda_executor.h" #include "utils/ucc_atomic.h" ucc_status_t ucc_cuda_executor_interruptible_get_stream(cudaStream_t *stream) { static uint32_t last_used = 0; int num_streams = EC_CUDA_CONFIG->exec_num_streams; ucc_ec_cuda_resources_t *resources; ucc_status_t st; int i, j; uint32_t id; ucc_assert(num_streams > 0); if (ucc_unlikely(num_streams <= 0)) { return UCC_ERR_INVALID_PARAM; } st = ucc_ec_cuda_get_resources(&resources); if (ucc_unlikely(st != UCC_OK)) { return st; } if (ucc_unlikely(!resources->streams_initialized)) { ucc_spin_lock(&ucc_ec_cuda.init_spinlock); if (resources->streams_initialized) { goto unlock; } for(i = 0; i < num_streams; i++) { st = CUDA_FUNC(cudaStreamCreateWithFlags(&resources->exec_streams[i], cudaStreamNonBlocking)); if (st != UCC_OK) { for (j = 0; j < i; j++) { CUDA_FUNC(cudaStreamDestroy(resources->exec_streams[j])); } ucc_spin_unlock(&ucc_ec_cuda.init_spinlock); return st; } } resources->streams_initialized = 1; unlock: ucc_spin_unlock(&ucc_ec_cuda.init_spinlock); } id = ucc_atomic_fadd32(&last_used, 1); *stream = resources->exec_streams[id % num_streams]; return UCC_OK; } ucc_status_t ucc_ec_cuda_copy_multi_kernel(const ucc_ee_executor_task_args_t *args, cudaStream_t stream); ucc_status_t ucc_cuda_executor_interruptible_task_post(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task) { cudaStream_t stream = NULL; size_t num_nodes = UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS; ucc_ec_cuda_executor_interruptible_task_t *ee_task; ucc_status_t status; cudaGraphNode_t nodes[UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS]; ucc_ec_cuda_resources_t *resources; int i; status = ucc_ec_cuda_get_resources(&resources); if (ucc_unlikely(status != UCC_OK)) { return status; } status = ucc_cuda_executor_interruptible_get_stream(&stream); if (ucc_unlikely(status != UCC_OK)) { return status; } ee_task = ucc_mpool_get(&resources->executor_interruptible_tasks); if (ucc_unlikely(!ee_task)) { return UCC_ERR_NO_MEMORY; } status = ucc_ec_cuda_event_create(&ee_task->event); if (ucc_unlikely(status != UCC_OK)) { ucc_mpool_put(ee_task); return status; } ee_task->super.status = UCC_INPROGRESS; ee_task->super.eee = executor; memcpy(&ee_task->super.args, task_args, sizeof(ucc_ee_executor_task_args_t)); switch (task_args->task_type) { case UCC_EE_EXECUTOR_TASK_COPY: status = CUDA_FUNC( cudaMemcpyAsync(task_args->copy.dst, task_args->copy.src, task_args->copy.len, cudaMemcpyDefault, stream)); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_cuda.super, "failed to start memcpy op"); goto free_task; } break; case UCC_EE_EXECUTOR_TASK_COPY_MULTI: if ((task_args->copy_multi.counts[0] > EC_CUDA_CONFIG->exec_copy_thresh) && (task_args->copy_multi.num_vectors > 2)) { status = CUDA_FUNC(cudaGraphGetNodes(ee_task->graph, nodes, &num_nodes)); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_cuda.super, "failed to get graph nodes"); goto free_task; } for (i = 0; i < task_args->copy_multi.num_vectors; i++) { status = CUDA_FUNC( cudaGraphExecMemcpyNodeSetParams1D(ee_task->graph_exec, nodes[i], task_args->copy_multi.dst[i], task_args->copy_multi.src[i], task_args->copy_multi.counts[i], cudaMemcpyDefault)); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_cuda.super, "failed to instantiate graph"); goto free_task; } } for (; i < UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS; i++) { status = CUDA_FUNC( cudaGraphExecMemcpyNodeSetParams1D(ee_task->graph_exec, nodes[i], task_args->copy_multi.dst[0], task_args->copy_multi.src[0], 1, cudaMemcpyDefault)); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_cuda.super, "failed to instantiate graph"); goto free_task; } } status = CUDA_FUNC(cudaGraphLaunch(ee_task->graph_exec, stream)); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_cuda.super, "failed to instantiate graph"); goto free_task; } } else { status = ucc_ec_cuda_copy_multi_kernel(task_args, stream); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_cuda.super, "failed to start copy multi op"); goto free_task; } } break; case UCC_EE_EXECUTOR_TASK_REDUCE: case UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED: case UCC_EE_EXECUTOR_TASK_REDUCE_MULTI_DST: status = ucc_ec_cuda_reduce( (ucc_ee_executor_task_args_t *)task_args, resources->num_threads_reduce, resources->num_blocks_reduce, stream); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_cuda.super, "failed to start reduce op"); goto free_task; } break; default: ec_error(&ucc_ec_cuda.super, "executor operation %d is not supported", task_args->task_type); status = UCC_ERR_INVALID_PARAM; goto free_task; } status = ucc_ec_cuda_event_post(stream, ee_task->event); if (ucc_unlikely(status != UCC_OK)) { goto free_task; } *task = &ee_task->super; return UCC_OK; free_task: ucc_ec_cuda_event_destroy(ee_task->event); ucc_mpool_put(ee_task); return status; } ucc_status_t ucc_cuda_executor_interruptible_task_test(const ucc_ee_executor_task_t *task) { ucc_ec_cuda_executor_interruptible_task_t *ee_task = ucc_derived_of(task, ucc_ec_cuda_executor_interruptible_task_t); ee_task->super.status = ucc_ec_cuda_event_test(ee_task->event); return ee_task->super.status; } ucc_status_t ucc_cuda_executor_interruptible_task_finalize(ucc_ee_executor_task_t *task) { ucc_ec_cuda_executor_interruptible_task_t *ee_task = ucc_derived_of(task, ucc_ec_cuda_executor_interruptible_task_t); ucc_status_t status; ucc_assert(task->status == UCC_OK); status = ucc_ec_cuda_event_destroy(ee_task->event); ucc_mpool_put(task); return status; } ucc_status_t ucc_cuda_executor_interruptible_start(ucc_ee_executor_t *executor) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); eee->mode = UCC_EC_CUDA_EXECUTOR_MODE_INTERRUPTIBLE; eee->state = UCC_EC_CUDA_EXECUTOR_STARTED; eee->ops.task_post = ucc_cuda_executor_interruptible_task_post; eee->ops.task_test = ucc_cuda_executor_interruptible_task_test; eee->ops.task_finalize = ucc_cuda_executor_interruptible_task_finalize; return UCC_OK; } ucc_status_t ucc_cuda_executor_interruptible_stop(ucc_ee_executor_t *executor) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); eee->state = UCC_EC_CUDA_EXECUTOR_INITIALIZED; return UCC_OK; } ucc-1.8.0/src/components/ec/cuda/ec_cuda_executor_persistent_wait.c0000664000175000017500000000545115211535620025713 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ec_cuda_executor.h" ucc_status_t ucc_ec_cuda_post_kernel_stream_task(ucc_ec_cuda_executor_state_t *state, cudaStream_t stream); static ucc_status_t ucc_ec_cuda_post_driver_stream_task(ucc_ec_cuda_executor_state_t *state, cudaStream_t stream) { CUdeviceptr state_ptr = (CUdeviceptr)state; CUstreamBatchMemOpParams batch_memops[3] = {}; batch_memops[0].operation = CU_STREAM_MEM_OP_WRITE_VALUE_32; batch_memops[0].writeValue.address = state_ptr; batch_memops[0].writeValue.value = UCC_EC_CUDA_EXECUTOR_STARTED; batch_memops[0].writeValue.flags = 0; batch_memops[1].operation = CU_STREAM_MEM_OP_WAIT_VALUE_32; batch_memops[1].waitValue.address = state_ptr; batch_memops[1].waitValue.value = UCC_EC_CUDA_EXECUTOR_SHUTDOWN; batch_memops[1].waitValue.flags = CU_STREAM_WAIT_VALUE_EQ; batch_memops[2].operation = CU_STREAM_MEM_OP_WRITE_VALUE_32; batch_memops[2].writeValue.address = state_ptr; batch_memops[2].writeValue.value = UCC_EC_CUDA_EXECUTOR_SHUTDOWN_ACK; batch_memops[2].writeValue.flags = 0; CUDADRV_FUNC(cuStreamBatchMemOp(stream, 3, batch_memops, 0)); return UCC_OK; } ucc_status_t ucc_cuda_executor_persistent_wait_start(ucc_ee_executor_t *executor, void *ee_context) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); cudaStream_t stream = (cudaStream_t)ee_context; eee->super.ee_context = ee_context; eee->state = UCC_EC_CUDA_EXECUTOR_POSTED; eee->mode = UCC_EC_CUDA_EXECUTOR_MODE_PERSISTENT; ucc_memory_cpu_store_fence(); if (ucc_ec_cuda.strm_task_mode == UCC_EC_CUDA_TASK_KERNEL) { return ucc_ec_cuda_post_kernel_stream_task(eee->dev_state, stream); } else { return ucc_ec_cuda_post_driver_stream_task(eee->dev_state, stream); } } ucc_status_t ucc_cuda_executor_persistent_wait_stop(ucc_ee_executor_t *executor) { ucc_ec_cuda_executor_t *eee = ucc_derived_of(executor, ucc_ec_cuda_executor_t); volatile ucc_ec_cuda_executor_state_t *st = &eee->state; ec_debug(&ucc_ec_cuda.super, "executor stop, eee: %p", eee); ucc_assert((*st != UCC_EC_CUDA_EXECUTOR_POSTED) && (*st != UCC_EC_CUDA_EXECUTOR_SHUTDOWN)); *st = UCC_EC_CUDA_EXECUTOR_SHUTDOWN; while(*st != UCC_EC_CUDA_EXECUTOR_SHUTDOWN_ACK) { } eee->super.ee_context = NULL; eee->state = UCC_EC_CUDA_EXECUTOR_INITIALIZED; return UCC_OK; } ucc-1.8.0/src/components/ec/cuda/ec_cuda.c0000664000175000017500000002655615211535620020502 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ec_cuda.h" #include "ec_cuda_executor.h" #include "utils/ucc_malloc.h" #include "utils/arch/cpu.h" #include #include static const char *stream_task_modes[] = { [UCC_EC_CUDA_TASK_KERNEL] = "kernel", [UCC_EC_CUDA_TASK_MEM_OPS] = "driver", [UCC_EC_CUDA_TASK_AUTO] = "auto", [UCC_EC_CUDA_TASK_LAST] = NULL }; static ucc_config_field_t ucc_ec_cuda_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_ec_cuda_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_ec_config_table)}, {"STREAM_TASK_MODE", "auto", "Mechanism to create stream dependency\n" "kernel - use waiting kernel\n" "driver - use driver MEM_OPS\n" "auto - runtime automatically chooses best one", ucc_offsetof(ucc_ec_cuda_config_t, strm_task_mode), UCC_CONFIG_TYPE_ENUM(stream_task_modes)}, {"EXEC_NUM_WORKERS", "1", "Number of thread blocks to use for cuda persistent executor", ucc_offsetof(ucc_ec_cuda_config_t, exec_num_workers), UCC_CONFIG_TYPE_ULUNITS}, {"EXEC_NUM_THREADS", "auto", "Number of threads per block to use for cuda persistent executor", ucc_offsetof(ucc_ec_cuda_config_t, exec_num_threads), UCC_CONFIG_TYPE_ULUNITS}, {"EXEC_MAX_TASKS", "128", "Maximum number of outstanding tasks per executor", ucc_offsetof(ucc_ec_cuda_config_t, exec_max_tasks), UCC_CONFIG_TYPE_ULUNITS}, {"EXEC_NUM_STREAMS", "16", "Number of streams used by interruptible executor", ucc_offsetof(ucc_ec_cuda_config_t, exec_num_streams), UCC_CONFIG_TYPE_ULUNITS}, {"EXEC_COPY_LARGE_THRESH", "1M", "Single memcopy size to switch from kernel copy to cudaMemcpy", ucc_offsetof(ucc_ec_cuda_config_t, exec_copy_thresh), UCC_CONFIG_TYPE_MEMUNITS}, {"REDUCE_NUM_BLOCKS", "auto", "Number of thread blocks to use for reduction in interruptible mode", ucc_offsetof(ucc_ec_cuda_config_t, reduce_num_blocks), UCC_CONFIG_TYPE_ULUNITS}, {"REDUCE_NUM_THREADS", "auto", "Number of threads per block to use for reduction in interruptible " "executor", ucc_offsetof(ucc_ec_cuda_config_t, reduce_num_threads), UCC_CONFIG_TYPE_ULUNITS}, {"USE_COOPERATIVE_LAUNCH", "0", "whether to use cooperative launch in persistent kernel executor", ucc_offsetof(ucc_ec_cuda_config_t, use_cooperative_launch), UCC_CONFIG_TYPE_BOOL}, {NULL} }; static ucc_status_t ucc_ec_cuda_init(const ucc_ec_params_t *ec_params) { ucc_ec_cuda_config_t *cfg = EC_CUDA_CONFIG; int supports_coop_launch = 0; int device, num_devices; cudaError_t cuda_st; struct cudaDeviceProp prop; ucc_ec_cuda_config = ucc_derived_of(ucc_ec_cuda.super.config, ucc_ec_cuda_config_t); ucc_ec_cuda.exec_streams_initialized = 0; ucc_strncpy_safe(ucc_ec_cuda.super.config->log_component.name, ucc_ec_cuda.super.super.name, sizeof(ucc_ec_cuda.super.config->log_component.name)); ucc_ec_cuda.thread_mode = ec_params->thread_mode; cuda_st = cudaGetDeviceCount(&num_devices); if ((cuda_st != cudaSuccess) || (num_devices == 0)) { ec_debug(&ucc_ec_cuda.super, "CUDA devices are not found"); return UCC_ERR_NO_RESOURCE; } CUDA_CHECK(cudaGetDevice(&device)); CUDA_CHECK(cudaGetDeviceProperties(&prop, device)); if (cfg->reduce_num_blocks != UCC_ULUNITS_AUTO) { if (prop.maxGridSize[0] < cfg->reduce_num_blocks) { ec_warn(&ucc_ec_cuda.super, "number of blocks is too large, max supported is %d", prop.maxGridSize[0]); cfg->reduce_num_blocks = prop.maxGridSize[0]; } } else { cfg->reduce_num_blocks = prop.maxGridSize[0]; } if (cfg->exec_num_streams < 1) { ec_warn(&ucc_ec_cuda.super, "number of streams is too small, min supported 1"); cfg->exec_num_streams = 1; } if (cfg->strm_task_mode == UCC_EC_CUDA_TASK_KERNEL) { ucc_ec_cuda.strm_task_mode = UCC_EC_CUDA_TASK_KERNEL; } else { ucc_ec_cuda.strm_task_mode = UCC_EC_CUDA_TASK_MEM_OPS; #if CUDA_VERSION < 12000 CUresult cu_st; CUdevice cu_dev; int attr; cu_st = cuCtxGetDevice(&cu_dev); if (cu_st != CUDA_SUCCESS){ const char *cu_err_st_str; cuGetErrorString(cu_st, &cu_err_st_str); ec_debug(&ucc_ec_cuda.super, "cuCtxGetDevice() failed: %s", cu_err_st_str); attr = 0; } else { CUDADRV_FUNC(cuDeviceGetAttribute(&attr, CU_DEVICE_ATTRIBUTE_CAN_USE_STREAM_MEM_OPS, cu_dev)); } if (cfg->strm_task_mode == UCC_EC_CUDA_TASK_AUTO) { if (attr == 0) { ec_debug(&ucc_ec_cuda.super, "CUDA MEM OPS are not supported or disabled"); ucc_ec_cuda.strm_task_mode = UCC_EC_CUDA_TASK_KERNEL; } } else if (attr == 0) { ec_error(&ucc_ec_cuda.super, "CUDA MEM OPS are not supported or disabled"); return UCC_ERR_NOT_SUPPORTED; } #endif } if (cfg->use_cooperative_launch == 1) { cudaDeviceGetAttribute(&supports_coop_launch, cudaDevAttrCooperativeLaunch, device); if (!supports_coop_launch) { cfg->use_cooperative_launch = 0; ec_warn(&ucc_ec_cuda.super, "CUDA cooperative groups are not supported. " "Fall back to non cooperative launch."); } } ucc_ec_cuda.resources_hash = kh_init(ucc_ec_cuda_resources_hash); ucc_spinlock_init(&ucc_ec_cuda.init_spinlock, 0); return UCC_OK; } static ucc_status_t ucc_ec_cuda_get_attr(ucc_ec_attr_t *ec_attr) { if (ec_attr->field_mask & UCC_EC_ATTR_FIELD_THREAD_MODE) { ec_attr->thread_mode = ucc_ec_cuda.thread_mode; } return UCC_OK; } ucc_status_t ucc_ec_cuda_event_create(void **event) { ucc_ec_cuda_event_t *cuda_event; ucc_ec_cuda_resources_t *resources; ucc_status_t status; status = ucc_ec_cuda_get_resources(&resources); if (ucc_unlikely(status != UCC_OK)) { return status; } cuda_event = ucc_mpool_get(&resources->events); if (ucc_unlikely(!cuda_event)) { ec_error(&ucc_ec_cuda.super, "failed to get event from mpool"); return UCC_ERR_NO_MEMORY; } *event = cuda_event; return UCC_OK; } ucc_status_t ucc_ec_cuda_event_destroy(void *event) { ucc_ec_cuda_event_t *cuda_event = event; ucc_mpool_put(cuda_event); return UCC_OK; } ucc_status_t ucc_ec_cuda_event_post(void *ee_context, void *event) { cudaStream_t stream = (cudaStream_t )ee_context; ucc_ec_cuda_event_t *cuda_event = event; CUDA_CHECK(cudaEventRecord(cuda_event->event, stream)); return UCC_OK; } ucc_status_t ucc_ec_cuda_event_test(void *event) { ucc_ec_cuda_event_t *cuda_event = event; cudaError_t cu_err; cu_err = cudaEventQuery(cuda_event->event); if (ucc_unlikely((cu_err != cudaSuccess) && (cu_err != cudaErrorNotReady))) { CUDA_CHECK(cu_err); } return cuda_error_to_ucc_status(cu_err); } static ucc_status_t ucc_ec_cuda_finalize() { ucc_ec_cuda_resources_t *resources; resources = ec_cuda_resources_hash_pop(ucc_ec_cuda.resources_hash); while (resources) { ucc_ec_cuda_resources_cleanup(resources); resources = ec_cuda_resources_hash_pop(ucc_ec_cuda.resources_hash); } ucc_spinlock_destroy(&ucc_ec_cuda.init_spinlock); return UCC_OK; } ucc_status_t ucc_ec_cuda_get_resources(ucc_ec_cuda_resources_t **resources) { CUcontext cu_ctx; unsigned long long int cu_ctx_id; ucc_status_t status; status = CUDADRV_FUNC(cuCtxGetCurrent(&cu_ctx)); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_cuda.super, "failed to get current CUDA context"); return status; } #if CUDA_VERSION < 12000 cu_ctx_id = 1; #else status = CUDADRV_FUNC(cuCtxGetId(cu_ctx, &cu_ctx_id)); if (ucc_unlikely(status != UCC_OK)) { /* worakround for pytorch, progress thread doesn't have cuda context for GPU 0*/ cu_ctx_id = 0x12345; ec_debug(&ucc_ec_cuda.super, "failed to get currect CUDA context ID"); } #endif *resources = ec_cuda_resources_hash_get(ucc_ec_cuda.resources_hash, cu_ctx_id); if (ucc_unlikely(*resources == NULL)) { ucc_spin_lock(&ucc_ec_cuda.init_spinlock); *resources = ec_cuda_resources_hash_get(ucc_ec_cuda.resources_hash, cu_ctx_id); if (*resources == NULL) { *resources = ucc_malloc(sizeof(ucc_ec_cuda_resources_t), "ec cuda resources"); if (*resources == NULL) { ec_error(&ucc_ec_cuda.super, "failed to allocate %zd bytes for resources", sizeof(ucc_ec_cuda_resources_t)); ucc_spin_unlock(&ucc_ec_cuda.init_spinlock); return UCC_ERR_NO_MEMORY; } status = ucc_ec_cuda_resources_init(&ucc_ec_cuda.super, *resources); if (status != UCC_OK) { ucc_free(*resources); ucc_spin_unlock(&ucc_ec_cuda.init_spinlock); return status; } ec_cuda_resources_hash_put(ucc_ec_cuda.resources_hash, cu_ctx_id, *resources); } ucc_spin_unlock(&ucc_ec_cuda.init_spinlock); } return UCC_OK; } ucc_ec_cuda_t ucc_ec_cuda = { .super.super.name = "cuda ec", .super.ref_cnt = 0, .super.type = UCC_EE_CUDA_STREAM, .super.init = ucc_ec_cuda_init, .super.get_attr = ucc_ec_cuda_get_attr, .super.finalize = ucc_ec_cuda_finalize, .super.config_table = { .name = "CUDA execution component", .prefix = "EC_CUDA_", .table = ucc_ec_cuda_config_table, .size = sizeof(ucc_ec_cuda_config_t), }, .super.ops.create_event = ucc_ec_cuda_event_create, .super.ops.destroy_event = ucc_ec_cuda_event_destroy, .super.ops.event_post = ucc_ec_cuda_event_post, .super.ops.event_test = ucc_ec_cuda_event_test, .super.executor_ops.init = ucc_cuda_executor_init, .super.executor_ops.start = ucc_cuda_executor_start, .super.executor_ops.status = ucc_cuda_executor_status, .super.executor_ops.stop = ucc_cuda_executor_stop, .super.executor_ops.task_post = ucc_cuda_executor_task_post, .super.executor_ops.task_test = ucc_cuda_executor_task_test, .super.executor_ops.task_finalize = ucc_cuda_executor_task_finalize, .super.executor_ops.finalize = ucc_cuda_executor_finalize, }; ucc_ec_cuda_config_t *ucc_ec_cuda_config; UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_ec_cuda.super.config_table, &ucc_config_global_list); ucc-1.8.0/src/components/ec/cuda/Makefile.am0000664000175000017500000000174615211535620021001 0ustar alastairalastair# # Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # if HAVE_CUDA SUBDIRS = kernel sources = \ ec_cuda.h \ ec_cuda.c \ ec_cuda_executor.h \ ec_cuda_executor.c \ ec_cuda_executor_interruptible.c \ ec_cuda_executor_persistent.c \ ec_cuda_executor_persistent_wait.c \ ec_cuda_resources.c \ ec_cuda_resources.h module_LTLIBRARIES = libucc_ec_cuda.la libucc_ec_cuda_la_SOURCES = $(sources) libucc_ec_cuda_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) $(CUDA_CPPFLAGS) libucc_ec_cuda_la_CFLAGS = $(BASE_CFLAGS) libucc_ec_cuda_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(CUDA_LDFLAGS) -lstdc++ libucc_ec_cuda_la_LIBADD = $(CUDA_LIBS) \ $(UCC_TOP_BUILDDIR)/src/libucc.la \ kernel/libucc_ec_cuda_kernels.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/ec/cuda/ec_cuda_resources.c0000664000175000017500000002404315211535620022561 0ustar alastairalastair#include "ec_cuda_resources.h" #include "components/ec/ucc_ec_log.h" #include "utils/ucc_malloc.h" static void ucc_ec_cuda_event_init(ucc_mpool_t *mp, void *obj, void *chunk) //NOLINT: mp is unused { ucc_ec_cuda_event_t *base = (ucc_ec_cuda_event_t *) obj; CUDA_FUNC(cudaEventCreateWithFlags(&base->event, cudaEventDisableTiming)); } static void ucc_ec_cuda_event_cleanup(ucc_mpool_t *mp, void *obj) //NOLINT: mp is unused { ucc_ec_cuda_event_t *base = (ucc_ec_cuda_event_t *) obj; CUDA_FUNC(cudaEventDestroy(base->event)); } static ucc_mpool_ops_t ucc_ec_cuda_event_mpool_ops = { .chunk_alloc = ucc_mpool_hugetlb_malloc, .chunk_release = ucc_mpool_hugetlb_free, .obj_init = ucc_ec_cuda_event_init, .obj_cleanup = ucc_ec_cuda_event_cleanup, }; static ucc_status_t ucc_ec_cuda_ee_executor_mpool_chunk_malloc(ucc_mpool_t *mp, //NOLINT: mp is unused size_t *size_p, void ** chunk_p) { return CUDA_FUNC(cudaHostAlloc((void**)chunk_p, *size_p, cudaHostAllocMapped)); } static void ucc_ec_cuda_ee_executor_mpool_chunk_free(ucc_mpool_t *mp, //NOLINT: mp is unused void *chunk) { CUDA_FUNC(cudaFreeHost(chunk)); } static void ucc_ec_cuda_executor_chunk_init(ucc_mpool_t *mp, void *obj, //NOLINT: mp is unused void *chunk) //NOLINT: chunk is unused { ucc_ec_cuda_executor_t *eee = (ucc_ec_cuda_executor_t*) obj; int max_tasks = ucc_ec_cuda_config->exec_max_tasks; CUDA_FUNC(cudaHostGetDevicePointer( (void**)(&eee->dev_state), (void *)&eee->state, 0)); CUDA_FUNC(cudaHostGetDevicePointer( (void**)(&eee->dev_pidx), (void *)&eee->pidx, 0)); CUDA_FUNC(cudaMalloc((void**)&eee->dev_cidx, sizeof(*eee->dev_cidx))); CUDA_FUNC(cudaHostAlloc((void**)&eee->tasks, max_tasks * MAX_SUBTASKS * sizeof(ucc_ee_executor_task_args_t), cudaHostAllocMapped)); CUDA_FUNC(cudaHostGetDevicePointer( (void**)(&eee->dev_tasks), (void *)eee->tasks, 0)); ucc_spinlock_init(&eee->tasks_lock, 0); } static void ucc_ec_cuda_executor_chunk_cleanup(ucc_mpool_t *mp, void *obj) //NOLINT: mp is unused { ucc_ec_cuda_executor_t *eee = (ucc_ec_cuda_executor_t*) obj; CUDA_FUNC(cudaFree((void*)eee->dev_cidx)); CUDA_FUNC(cudaFreeHost((void*)eee->tasks)); ucc_spinlock_destroy(&eee->tasks_lock); } static ucc_mpool_ops_t ucc_ec_cuda_ee_executor_mpool_ops = { .chunk_alloc = ucc_ec_cuda_ee_executor_mpool_chunk_malloc, .chunk_release = ucc_ec_cuda_ee_executor_mpool_chunk_free, .obj_init = ucc_ec_cuda_executor_chunk_init, .obj_cleanup = ucc_ec_cuda_executor_chunk_cleanup, }; static void ucc_ec_cuda_graph_init(ucc_mpool_t *mp, void *obj, void *chunk) //NOLINT: mp is unused { ucc_ec_cuda_executor_interruptible_task_t *task = (ucc_ec_cuda_executor_interruptible_task_t *) obj; cudaGraphNode_t memcpy_node; int i; CUDA_FUNC(cudaGraphCreate(&task->graph, 0)); for (i = 0; i < UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS; i++) { CUDA_FUNC( cudaGraphAddMemcpyNode1D(&memcpy_node, task->graph, NULL, 0, (void*)1, (void*)1, 1, cudaMemcpyDefault)); } CUDA_FUNC( cudaGraphInstantiateWithFlags(&task->graph_exec, task->graph, 0)); } static void ucc_ec_cuda_graph_cleanup(ucc_mpool_t *mp, void *obj) //NOLINT: mp is unused { ucc_ec_cuda_executor_interruptible_task_t *task = (ucc_ec_cuda_executor_interruptible_task_t *) obj; CUDA_FUNC(cudaGraphExecDestroy(task->graph_exec)); CUDA_FUNC(cudaGraphDestroy(task->graph)); } static ucc_mpool_ops_t ucc_ec_cuda_interruptible_task_mpool_ops = { .chunk_alloc = ucc_mpool_hugetlb_malloc, .chunk_release = ucc_mpool_hugetlb_free, .obj_init = ucc_ec_cuda_graph_init, .obj_cleanup = ucc_ec_cuda_graph_cleanup, }; ucc_status_t ucc_ec_cuda_executor_kernel_calc_max_threads(int *max); static void ucc_ec_cuda_set_threads_nbr( ucc_ec_base_t *ec, int *nt, int maxThreadsPerBlock, int is_reduce) { ucc_status_t status; if (*nt != UCC_ULUNITS_AUTO) { if (maxThreadsPerBlock < *nt) { ec_warn( ec, "number of threads per block is too large, max supported is %d", maxThreadsPerBlock); } else if ((*nt % WARP_SIZE) != 0) { ec_warn( ec, "number of threads per block must be divisible by " "WARP_SIZE(=%d)", WARP_SIZE); } } else { *nt = (maxThreadsPerBlock / WARP_SIZE) * WARP_SIZE; if (!is_reduce) { // Pass max threads per block, lowering it if necessary // based on kernel occupancy requirements status = ucc_ec_cuda_executor_kernel_calc_max_threads(nt); if (status != UCC_OK) { ec_error( ec, "Error while calculating max threads: %s", ucc_status_string(status)); } } } } ucc_status_t ucc_ec_cuda_resources_init(ucc_ec_base_t *ec, ucc_ec_cuda_resources_t *resources) { ucc_status_t status; int num_streams; int max_threads_per_block; CUdevice device; CUDADRV_CHECK(cuCtxGetCurrent(&resources->cu_ctx)); resources->num_threads_reduce = ucc_ec_cuda_config->reduce_num_threads; resources->num_blocks_reduce = ucc_ec_cuda_config->reduce_num_blocks; resources->num_threads_exec = ucc_ec_cuda_config->exec_num_threads; resources->num_blocks_exec = ucc_ec_cuda_config->exec_num_workers; CUDADRV_CHECK(cuCtxGetDevice(&device)); CUDADRV_CHECK(cuDeviceGetAttribute( &max_threads_per_block, CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK, device)); ucc_ec_cuda_set_threads_nbr( ec, &resources->num_threads_reduce, max_threads_per_block, 1); ucc_ec_cuda_set_threads_nbr( ec, &resources->num_threads_exec, max_threads_per_block, 0); status = ucc_mpool_init( &resources->events, 0, sizeof(ucc_ec_cuda_event_t), 0, UCC_CACHE_LINE_SIZE, 16, UINT_MAX, &ucc_ec_cuda_event_mpool_ops, UCC_THREAD_MULTIPLE, "CUDA Event Objects"); if (status != UCC_OK) { ec_error(ec, "failed to create CUDA events pool"); goto exit_err; } status = ucc_mpool_init(&resources->executors, 0, sizeof(ucc_ec_cuda_executor_t), 0, UCC_CACHE_LINE_SIZE, 16, UINT_MAX, &ucc_ec_cuda_ee_executor_mpool_ops, UCC_THREAD_MULTIPLE, "CUDA EE executor objects"); if (status != UCC_OK) { ec_error(ec, "failed to create executors pool"); goto free_events_mpool; } status = ucc_mpool_init(&resources->executor_interruptible_tasks, 0, sizeof(ucc_ec_cuda_executor_interruptible_task_t), 0, UCC_CACHE_LINE_SIZE, 16, UINT_MAX, &ucc_ec_cuda_interruptible_task_mpool_ops, UCC_THREAD_MULTIPLE, "interruptible executor tasks"); if (status != UCC_OK) { ec_error(ec, "failed to create interruptible tasks pool"); goto free_executors_mpool; } status = ucc_mpool_init(&resources->executor_persistent_tasks, 0, sizeof(ucc_ec_cuda_executor_persistent_task_t), 0, UCC_CACHE_LINE_SIZE, 16, UINT_MAX, NULL, UCC_THREAD_MULTIPLE, "persistent executor tasks"); if (status != UCC_OK) { ec_error(ec, "failed to create persistent tasks pool"); goto free_interruptible_tasks_mpool; } num_streams = ucc_ec_cuda_config->exec_num_streams; resources->exec_streams = ucc_calloc(num_streams, sizeof(cudaStream_t), "ec cuda streams"); if (!resources->exec_streams) { ec_error(ec, "failed to allocate %zd bytes for executor streams", sizeof(cudaStream_t) * num_streams); status = UCC_ERR_NO_MEMORY; goto free_persistent_tasks_mpool; } ec_debug( ec, "initialized cuda resources: cuCtx=%p, num_threads_reduce=%d, " "num_blocks_reduce=%d, num_threads_exec=%d, num_blocks_exec=%d", resources->cu_ctx, resources->num_threads_reduce, resources->num_blocks_reduce, resources->num_threads_exec, resources->num_blocks_exec); return UCC_OK; free_persistent_tasks_mpool: ucc_mpool_cleanup(&resources->executor_persistent_tasks, 0); free_interruptible_tasks_mpool: ucc_mpool_cleanup(&resources->executor_persistent_tasks, 0); free_executors_mpool: ucc_mpool_cleanup(&resources->executors, 0); free_events_mpool: ucc_mpool_cleanup(&resources->events, 0); exit_err: return status; } void ucc_ec_cuda_resources_cleanup(ucc_ec_cuda_resources_t *resources) { int i; CUcontext tmp_context; #if CUDA_VERSION >= 12000 CUresult status; unsigned long long int cu_ctx_id; status = cuCtxGetId(resources->cu_ctx, &cu_ctx_id); if (ucc_unlikely(status != CUDA_SUCCESS)) { // ctx is not available, can be due to cudaDeviceReset return; } #endif cuCtxPushCurrent(resources->cu_ctx); for (i = 0; i < ucc_ec_cuda_config->exec_num_streams; i++) { if (resources->exec_streams[i] != NULL) { CUDA_FUNC(cudaStreamDestroy(resources->exec_streams[i])); } } ucc_mpool_cleanup(&resources->events, 1); ucc_mpool_cleanup(&resources->executors, 1); ucc_mpool_cleanup(&resources->executor_interruptible_tasks, 1); ucc_mpool_cleanup(&resources->executor_persistent_tasks, 1); ucc_free(resources->exec_streams); cuCtxPopCurrent(&tmp_context); } ucc-1.8.0/src/components/ec/cuda/ec_cuda_executor.h0000664000175000017500000000235515211535620022414 0ustar alastairalastair/** * Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_EC_CUDA_EXECUTOR_H_ #define UCC_EC_CUDA_EXECUTOR_H_ #include "ec_cuda.h" ucc_status_t ucc_cuda_executor_init( const ucc_ee_executor_params_t *params, ucc_ee_executor_t **executor); ucc_status_t ucc_cuda_executor_status(const ucc_ee_executor_t *executor); ucc_status_t ucc_cuda_executor_finalize(ucc_ee_executor_t *executor); ucc_status_t ucc_cuda_executor_start( ucc_ee_executor_t *executor, void *ee_context); ucc_status_t ucc_cuda_executor_stop(ucc_ee_executor_t *executor); ucc_status_t ucc_cuda_executor_task_post( ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task); ucc_status_t ucc_cuda_executor_task_test(const ucc_ee_executor_task_t *task); ucc_status_t ucc_cuda_executor_task_finalize(ucc_ee_executor_task_t *task); /* implemented in ec_cuda_executor.cu */ ucc_status_t ucc_ec_cuda_persistent_kernel_start( ucc_ec_cuda_executor_t *eee, unsigned num_threads, unsigned num_blocks); ucc_status_t ucc_ec_cuda_reduce( ucc_ee_executor_task_args_t *task, unsigned num_threads, unsigned num_blocks, cudaStream_t stream); #endif ucc-1.8.0/src/components/ec/base/0000775000175000017500000000000015211535620016733 5ustar alastairalastairucc-1.8.0/src/components/ec/base/ucc_ec_base.h0000664000175000017500000001604515211535620021325 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_EC_BASE_H_ #define UCC_EC_BASE_H_ #include "config.h" #include "utils/ucc_component.h" #include "utils/ucc_class.h" #include "utils/ucc_parser.h" #include "utils/ucc_log.h" typedef struct ucc_ec_config { ucc_log_component_config_t log_component; } ucc_ec_config_t; extern ucc_config_field_t ucc_ec_config_table[]; typedef struct ucc_ec_params { ucc_thread_mode_t thread_mode; } ucc_ec_params_t; /** * UCC execution component attributes field mask */ typedef enum ucc_ec_attr_field { UCC_EC_ATTR_FIELD_THREAD_MODE = UCC_BIT(0), UCC_EC_ATTR_FILED_MAX_EXECUTORS_BUFS = UCC_BIT(1) } ucc_ec_attr_field_t; typedef struct ucc_ec_attr { /** * Mask of valid fields in this structure, using bits from * @ref ucc_ec_attr_field_t. */ uint64_t field_mask; ucc_thread_mode_t thread_mode; /** * Maximum number of buffers in ucc_ee_executor_task_args, * includes src buffer */ int max_ee_bufs; } ucc_ec_attr_t; typedef struct ucc_ec_ops { ucc_status_t (*create_event)(void **event); ucc_status_t (*destroy_event)(void *event); ucc_status_t (*event_post)(void *ee_context, void *event); ucc_status_t (*event_test)(void *event); } ucc_ec_ops_t; typedef struct ucc_ee_executor { ucc_ee_type_t ee_type; void *ee_context; } ucc_ee_executor_t; enum ucc_ee_executor_params_field { UCC_EE_EXECUTOR_PARAM_FIELD_TYPE = UCC_BIT(0), UCC_EE_EXECUTOR_PARAM_FIELD_TASK_TYPES = UCC_BIT(1), }; typedef enum ucc_ee_executor_task_type { UCC_EE_EXECUTOR_TASK_REDUCE = UCC_BIT(0), UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED = UCC_BIT(1), UCC_EE_EXECUTOR_TASK_REDUCE_MULTI_DST = UCC_BIT(2), UCC_EE_EXECUTOR_TASK_COPY = UCC_BIT(3), UCC_EE_EXECUTOR_TASK_COPY_MULTI = UCC_BIT(4), UCC_EE_EXECUTOR_TASK_LAST } ucc_ee_executor_task_type_t; typedef struct ucc_ee_executor_params { uint64_t mask; ucc_ee_type_t ee_type; uint64_t task_types; } ucc_ee_executor_params_t; #define UCC_EE_EXECUTOR_NUM_BUFS 9 /* Maximum number of buffers for UCC_EE_EXECUTOR_TASK_REDUCE_MULTI_DST and UCC_EE_EXECUTOR_TASK_COPY_MULTI operations */ #define UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS 7 /* Reduces "n_srcs" buffers (each contains "count" elements of type "dt") into "dst" buffer. If UCC_EEE_TASK_FLAG_REDUCE_SRCS_EXT flag is not set on task_args then the sources are taken from "srcs" and n_srcs can not exceed UCC_EE_EXECUTOR_NUM_BUFS. If UCC_EEE_TASK_FLAG_REDUCE_SRCS_EXT flag IS set on task_args then the sources are taken from "src_ext". In that case it is caller responsibility to make sure srcs_ext pointer is valid until task is complete. If UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA flag is set on task_args each element of the result of reduction is multiplied by "alpha" */ typedef struct ucc_eee_task_reduce { void * dst; union { void * srcs[UCC_EE_EXECUTOR_NUM_BUFS]; void **srcs_ext; }; size_t count; double alpha; ucc_datatype_t dt; ucc_reduction_op_t op; uint16_t n_srcs; } ucc_eee_task_reduce_t; /* Performs "n_bufs" independent reductions for corresponding buffers in src1 and src2 and saving result into "dst" buffer */ typedef struct ucc_eee_task_reduce_multi_dst { void *dst[UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS]; void *src1[UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS]; void *src2[UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS]; size_t counts[UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS]; ucc_datatype_t dt; ucc_reduction_op_t op; uint16_t n_bufs; } ucc_eee_task_reduce_multi_dst_t; /* Reduces "n_srcs2+1" buffers (each contains "count" elements of type "dt") into "dst" buffer. The first source buffer is "src1". Other n_src2 source buffers are defined as SRC[i] = src2 + stride * i, where stride is defined in bytes. If UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA flag is set on task_args each element of the result of reduction is multiplied by "alpha" */ typedef struct ucc_eee_task_reduce_strided { void * dst; void * src1; void * src2; size_t stride; size_t count; double alpha; ucc_datatype_t dt; ucc_reduction_op_t op; uint16_t n_src2; } ucc_eee_task_reduce_strided_t; /* Copies len bytes from "src" into "dst" */ typedef struct ucc_eee_task_copy { const void *src; void *dst; size_t len; } ucc_eee_task_copy_t; enum ucc_eee_task_flags { UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA = UCC_BIT(0), UCC_EEE_TASK_FLAG_REDUCE_SRCS_EXT = UCC_BIT(1) }; /* Performs "num_vectors" copies from SRC[i] to DST[i] */ typedef struct ucc_eee_task_copy_multi{ void *src[UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS]; void *dst[UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS]; size_t counts[UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS]; size_t num_vectors; } ucc_eee_task_copy_multi_t; typedef struct ucc_ee_executor_task_args { uint16_t task_type; uint16_t flags; union { ucc_eee_task_reduce_t reduce; ucc_eee_task_reduce_strided_t reduce_strided; ucc_eee_task_reduce_multi_dst_t reduce_multi_dst; ucc_eee_task_copy_t copy; ucc_eee_task_copy_multi_t copy_multi; }; } ucc_ee_executor_task_args_t; typedef struct ucc_ee_executor_task { ucc_ee_executor_t *eee; ucc_ee_executor_task_args_t args; ucc_status_t status; } ucc_ee_executor_task_t; typedef struct ucc_ee_executor_ops { ucc_status_t (*init)(const ucc_ee_executor_params_t *params, ucc_ee_executor_t **executor); ucc_status_t (*status)(const ucc_ee_executor_t *executor); ucc_status_t (*start)(ucc_ee_executor_t *executor, void *ee_context); ucc_status_t (*stop)(ucc_ee_executor_t *executor); ucc_status_t (*finalize)(ucc_ee_executor_t *executor); ucc_status_t (*task_post)(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task); ucc_status_t (*task_test)(const ucc_ee_executor_task_t *task); ucc_status_t (*task_finalize)(ucc_ee_executor_task_t *task); } ucc_ee_executor_ops_t; typedef struct ucc_ec_base { ucc_component_iface_t super; uint32_t ref_cnt; ucc_ee_type_t type; ucc_ec_config_t *config; ucc_config_global_list_entry_t config_table; ucc_status_t (*init)(const ucc_ec_params_t *ec_params); ucc_status_t (*get_attr)(ucc_ec_attr_t *ec_attr); ucc_status_t (*finalize)(); ucc_ec_ops_t ops; ucc_ee_executor_ops_t executor_ops; } ucc_ec_base_t; #endif ucc-1.8.0/src/components/ec/base/ucc_ec_base.c0000664000175000017500000000077715211535620021325 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_ec_base.h" ucc_config_field_t ucc_ec_config_table[] = { {"LOG_LEVEL", "warn", "UCC logging level. Messages with a level higher or equal to the " "selected will be printed.\n" "Possible values are: fatal, error, warn, info, debug, trace, data, func, " "poll.", ucc_offsetof(ucc_ec_config_t, log_component), UCC_CONFIG_TYPE_LOG_COMP}, {NULL}}; ucc-1.8.0/src/components/ec/rocm/0000775000175000017500000000000015211535620016761 5ustar alastairalastairucc-1.8.0/src/components/ec/rocm/kernel/0000775000175000017500000000000015211535620020241 5ustar alastairalastairucc-1.8.0/src/components/ec/rocm/kernel/ec_rocm_executor_kernel.cu0000664000175000017500000003106215211535620025461 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "ec_rocm.h" #include "utils/ucc_math.h" #include #define align_pow2(_n, _p) ((_n) & ((_p) - 1)) __global__ void executor_start(ucc_ec_rocm_executor_state_t *state, int *cidx) { *cidx = 0; *state = UCC_EC_ROCM_EXECUTOR_STARTED; } __global__ void executor_shutdown_ack(ucc_ec_rocm_executor_state_t *state) { *state = UCC_EC_ROCM_EXECUTOR_SHUTDOWN_ACK; } template __device__ void executor_copy(T* __restrict__ d, T* __restrict__ s, size_t count) { size_t start = threadIdx.x; const size_t step = blockDim.x; for (size_t i = start; i < count; i+=step) { d[i] = s[i]; } } template __device__ void executor_copy_aligned(T* __restrict__ d, T* __restrict__ s, size_t count) { size_t idx = threadIdx.x; const size_t step = blockDim.x; const int n = count / sizeof(T); const int num_iter = n / step + ((idx < n % step) ? 1 : 0); char1 *s1 = (char1*)s; char1 *d1 = (char1*)d; for(int i = 0; i < num_iter; i+=4) { d[i * step + idx] = s[i * step + idx]; d[(i+1) * step + idx] = s[(i+1) * step + idx]; d[(i+2) * step + idx] = s[(i+2) * step + idx]; d[(i+3) * step + idx] = s[(i+3) * step + idx]; } if (idx < count % sizeof(T)) { d1[count - idx - 1] = s1[count - idx - 1]; } } __device__ inline void add_float4(float4 &d, const float4 &x, const float4 &y) { d.x = x.x + y.x; d.y = x.y + y.y; d.z = x.z + y.z; d.w = x.w + y.w; } __device__ void executor_reduce_float_sum_aligned_2(const float *s1, const float *s2, float *d, size_t count) { const float4 *s14 = (const float4*)s1; const float4 *s24 = (const float4*)s2; float4 *d4 = (float4*)d; const size_t idx = threadIdx.x; const size_t step = blockDim.x; const int n = count / 4; const int num_iter = n / step + ((idx < n % step) ? 1 : 0); for(int i = 0; i < num_iter; i++) { add_float4(d4[i * step + idx], s14[i * step + idx], s24[i * step + idx]); } if (idx < count % 4) { d[count - idx - 1] = s1[count - idx - 1] + s2[count - idx - 1]; } } template __device__ void executor_reduce_sum(const T** __restrict__ srcs, uint16_t n_srcs, T* __restrict__ d, size_t count) { const size_t step = blockDim.x; const size_t start = threadIdx.x; for (size_t i = start; i < count; i+=step) { d[i] = srcs[0][i] + srcs[1][i]; for (size_t j = 2; j < n_srcs; j++) { d[i] = d[i] + srcs[j][i]; } } } template __device__ void executor_reduce_strided_sum(const T* __restrict__ s1, const T* __restrict__ s2, T* __restrict__ d, size_t count, size_t n_src2, size_t stride) { const size_t step = blockDim.x; const size_t start = threadIdx.x; const size_t ld = stride / sizeof(T); for (size_t i = start; i < count; i+=step) { d[i] = s1[i] + s2[i]; for (size_t j = 1; j < n_src2; j++) { d[i] = d[i] + s2[i + j*ld]; } } } __device__ void executor_copy_task(ucc_eee_task_copy_t *task) { bool aligned = !(align_pow2((intptr_t)task->dst, 16) || align_pow2((intptr_t)task->src, 16)); if (aligned) { executor_copy_aligned((uint4 *)task->dst, (uint4 *)task->src, task->len); } else { executor_copy((char *)task->dst, (char *)task->src, task->len); } } __device__ void executor_reduce_strided_task(ucc_eee_task_reduce_strided_t *task) { bool aligned = !(align_pow2((intptr_t)task->dst, 16) || align_pow2((intptr_t)task->src1, 16) || align_pow2((intptr_t)task->src2, 16)); switch (task->dt) { case UCC_DT_FLOAT32: if (task->n_src2 == 1 && aligned) { executor_reduce_float_sum_aligned_2((float *)task->src1, (float *)task->src2, (float *)task->dst, task->count); } else { executor_reduce_strided_sum((float *)task->src1, (float *)task->src2, (float *)task->dst, task->count, task->n_src2, task->stride); } break; case UCC_DT_FLOAT64: executor_reduce_strided_sum((double *)task->src1, (double *)task->src2, (double *)task->dst, task->count, task->n_src2, task->stride); break; case UCC_DT_INT32: executor_reduce_strided_sum((int32_t *)task->src1, (int32_t *)task->src2, (int32_t *)task->dst, task->count, task->n_src2, task->stride); break; default: break; } } __device__ void executor_reduce_task(ucc_eee_task_reduce_t *task) { bool aligned = !(align_pow2((intptr_t)task->dst, 16) || align_pow2((intptr_t)task->srcs[0], 16) || align_pow2((intptr_t)task->srcs[1], 16)); switch (task->dt) { case UCC_DT_FLOAT32: if (task->n_srcs == 2 && aligned) { executor_reduce_float_sum_aligned_2((float *)task->srcs[0], (float *)task->srcs[1], (float *)task->dst, task->count); } else { executor_reduce_sum((const float **)task->srcs, task->n_srcs, (float *)task->dst, task->count); } break; case UCC_DT_FLOAT64: executor_reduce_sum((const double **)task->srcs, task->n_srcs, (double *)task->dst, task->count); break; case UCC_DT_INT32: executor_reduce_sum((const int32_t **)task->srcs, task->n_srcs, (int32_t *)task->dst, task->count); break; default: break; } } __device__ void executor_copy_multi(ucc_eee_task_copy_multi_t *task) { const size_t step = blockDim.x; size_t min_size = task->counts[0]; size_t idx = threadIdx.x; __shared__ int4 *dsts[UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS]; __shared__ int4 *srcs[UCC_EE_EXECUTOR_MULTI_OP_NUM_BUFS]; bool aligned; for (int i = 0; i < task->num_vectors; i++) { dsts[i] = (int4*)task->dst[i]; srcs[i] = (int4*)task->src[i]; aligned = !(align_pow2((intptr_t)srcs[i], 16) || align_pow2((intptr_t)dsts[i], 16)); if (!aligned) { break; } if (task->counts[i] < min_size) { min_size = task->counts[i]; } } if (!aligned || min_size < 16) { for (int i = 0; i < task->num_vectors; i++) { executor_copy((char*)task->dst[i], (char*)task->src[i], task->counts[i]); } return; } const int n = min_size / sizeof(uint4); const int num_iter = n / step + ((threadIdx.x < n % step) ? 1 : 0); for (size_t i = 0; i < num_iter; i++) { for (int j = 0; j < task->num_vectors; j++) { dsts[j][idx] = srcs[j][idx]; } idx += step; } const size_t left = min_size + min_size % sizeof(uint4); for (int i = 0; i < task->num_vectors; i++) { executor_copy((char*)task->dst[i] + left, (char*)task->src[i] + left, task->counts[i] - left); } } __global__ void executor_kernel(volatile ucc_ec_rocm_executor_t *eee, int q_size) { const uint32_t worker_id = blockIdx.x; const uint32_t num_workers = gridDim.x; bool is_master = (threadIdx.x == 0) ? true: false; int cidx_local, pidx_local; volatile int *pidx, *cidx; ucc_ee_executor_task_t *tasks; __shared__ ucc_ee_executor_task_args_t args; __shared__ bool worker_done; if (is_master) { cidx_local = worker_id; pidx = eee->dev_pidx; cidx = eee->dev_cidx; tasks = eee->dev_tasks; } worker_done = false; __syncthreads(); while (1) { if (is_master) { while ((*cidx % num_workers) != worker_id); do { pidx_local = *pidx; } while (*cidx == pidx_local); (*cidx)++; worker_done = (pidx_local == -1); if (!worker_done) { args = tasks[cidx_local].args; } } __syncthreads(); if (worker_done) { return; } switch (args.task_type) { case UCC_EE_EXECUTOR_TASK_COPY: executor_copy_task(&args.copy); break; case UCC_EE_EXECUTOR_TASK_REDUCE: executor_reduce_task(&args.reduce); break; case UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED: executor_reduce_strided_task(&args.reduce_strided); break; case UCC_EE_EXECUTOR_TASK_COPY_MULTI: executor_copy_multi(&args.copy_multi); break; default: break; } __syncthreads(); __threadfence_system(); if (is_master) { tasks[cidx_local].status = UCC_OK; cidx_local = (cidx_local + num_workers) %q_size; } } } #ifdef __cplusplus extern "C" { #endif ucc_status_t ucc_ec_rocm_persistent_kernel_start(ucc_ec_rocm_executor_t *eee) { hipStream_t stream = (hipStream_t)eee->super.ee_context; int nb = EC_ROCM_CONFIG->exec_num_workers; int nt = EC_ROCM_CONFIG->exec_num_threads; int q_size = EC_ROCM_CONFIG->exec_max_tasks; executor_start<<<1, 1, 0, stream>>>(eee->dev_state, eee->dev_cidx); executor_kernel<<>>(eee, q_size); executor_shutdown_ack<<<1, 1, 0, stream>>>(eee->dev_state); ROCMCHECK(hipGetLastError()); return UCC_OK; } __global__ void kernel_copy_multi(ucc_eee_task_copy_multi_t args) { int blocks_per_buf = gridDim.x / args.num_vectors; int buf_id = blockIdx.x / blocks_per_buf; char1 *src = (char1*)args.src[buf_id]; char1 *dst = (char1*)args.dst[buf_id]; size_t cnt = args.counts[buf_id]; size_t start = threadIdx.x + (blockIdx.x % blocks_per_buf) * blockDim.x; size_t step = blockDim.x * blocks_per_buf; for (size_t i = start; i < cnt; i += step) { dst[i] = src[i]; } } __global__ void kernel_copy_multi_aligned(ucc_eee_task_copy_multi_t args) { int blocks_per_buf = gridDim.x / args.num_vectors; int buf_id = blockIdx.x / blocks_per_buf; int idx = threadIdx.x + (blockIdx.x % blocks_per_buf) * blockDim.x; int step = blockDim.x * blocks_per_buf; size_t n = args.counts[buf_id] / sizeof(uint4); size_t num_iter = n / step + ((idx < n % step) ? 1 : 0); uint4 *src = (uint4*)args.src[buf_id]; uint4 *dst = (uint4*)args.dst[buf_id]; for(size_t i = 0; i < num_iter; i++) { dst[i * step + idx] = src[i * step + idx]; } if (idx < (args.counts[buf_id] % sizeof(uint4))) { ((char*)args.dst[buf_id])[args.counts[buf_id] - idx - 1] = ((char*)args.src[buf_id])[args.counts[buf_id] - idx - 1]; } } ucc_status_t ucc_ec_rocm_copy_multi_kernel(const ucc_ee_executor_task_args_t *args, hipStream_t stream) { int nt = 1024; int nb = args->copy_multi.num_vectors * 4; int aligned = 1; for (int i = 0; i < args->copy_multi.num_vectors; i++) { if (align_pow2((intptr_t)args->copy_multi.src[i], 16) || align_pow2((intptr_t)args->copy_multi.dst[i], 16)) { aligned = 0; break; } } if (aligned) { kernel_copy_multi_aligned<<>>(args->copy_multi); } else { kernel_copy_multi<<>>(args->copy_multi); } ROCMCHECK(hipGetLastError()); return UCC_OK; } #ifdef __cplusplus } #endif ucc-1.8.0/src/components/ec/rocm/kernel/ec_rocm_reduce.cu0000664000175000017500000006175615211535620023547 0ustar alastairalastair/** * Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "ec_rocm.h" #include "utils/ucc_math_op.h" #include #include #define ROCM_REDUCE_WITH_OP_DEFAULT(NAME, _OP) \ template \ __global__ void UCC_REDUCE_ROCM_DEFAULT_##NAME(ucc_eee_task_reduce_t task, \ uint16_t flags) \ { \ size_t start = blockIdx.x * blockDim.x + threadIdx.x; \ size_t step = blockDim.x * gridDim.x; \ size_t count = task.count; \ int n_srcs = task.n_srcs; \ const _Type **s = (const _Type **)task.srcs; \ _Type * d = (_Type *)task.dst; \ size_t i; \ \ switch (n_srcs) { \ case 2: \ for (i = start; i < count; i += step) { \ d[i] = _OP##_2(s[0][i], s[1][i]); \ } \ break; \ case 3: \ for (i = start; i < count; i += step) { \ d[i] = _OP##_3(s[0][i], s[1][i], s[2][i]); \ } \ break; \ case 4: \ for (i = start; i < count; i += step) { \ d[i] = _OP##_4(s[0][i], s[1][i], s[2][i], s[3][i]); \ } \ break; \ default: \ for (i = start; i < count; i += step) { \ d[i] = _OP(s[0][i], s[1][i]); \ for (size_t j = 2; j < n_srcs; j++) { \ d[i] = _OP(d[i], s[j][i]); \ } \ } \ break; \ } \ if (flags & UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA) { \ for (i = start; i < count; i += step) { \ d[i] = d[i] * (_AlphaType)task.alpha; \ } \ } \ } #define ROCM_REDUCE_WITH_COMPLEX_PRODUCT_DEFAULT(NAME, _OP) \ template \ __global__ void UCC_REDUCE_ROCM_DEFAULT_COMPLEX_##NAME(ucc_eee_task_reduce_t task, \ uint16_t flags) \ { \ size_t start = blockIdx.x * blockDim.x + threadIdx.x; \ size_t step = blockDim.x * gridDim.x; \ size_t count = task.count; \ int n_srcs = task.n_srcs; \ const _Type **s = (const _Type **)task.srcs; \ _Type * d = (_Type *)task.dst; \ size_t i; \ \ switch (n_srcs) { \ case 2: \ for (i = start; i < count; i += step) { \ d[i] = _OP(s[0][i], s[1][i]); \ } \ break; \ default: \ for (i = start; i < count; i += step) { \ d[i] = _OP(s[0][i], s[1][i]); \ for (size_t j = 2; j < n_srcs; j++) { \ d[i] = _OP(d[i], s[j][i]); \ } \ } \ break; \ } \ if (flags & UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA) { \ for (i = start; i < count; i += step) { \ d[i] = d[i] * (_AlphaType)task.alpha; \ } \ } \ } #define ROCM_REDUCE_WITH_OP_STRIDED(NAME, _OP) \ template \ __global__ void UCC_REDUCE_ROCM_STRIDED_##NAME( \ const _Type *s1, const _Type *s2, _Type *d, size_t count, \ size_t stride, uint16_t n_src2, const bool with_alpha, \ const double alpha) \ { \ size_t start = blockIdx.x * blockDim.x + threadIdx.x; \ size_t step = blockDim.x * gridDim.x; \ size_t ld = stride / sizeof(_Type); \ size_t i; \ \ ucc_assert_system(stride % sizeof(_Type) == 0); \ switch (n_src2) { \ case 1: \ for (i = start; i < count; i += step) { \ d[i] = _OP##_2(s1[i], s2[i]); \ } \ break; \ case 2: \ for (i = start; i < count; i += step) { \ d[i] = _OP##_3(s1[i], s2[i], s2[i + ld]); \ } \ break; \ case 3: \ for (i = start; i < count; i += step) { \ d[i] = _OP##_4(s1[i], s2[i], s2[i + ld], s2[i + 2 * ld]); \ } \ break; \ default: \ for (i = start; i < count; i += step) { \ d[i] = _OP(s1[i], s2[i]); \ for (size_t j = 1; j < n_src2; j++) { \ d[i] = _OP(d[i], s2[i + j * ld]); \ } \ } \ break; \ } \ if (with_alpha) { \ for (i = start; i < count; i += step) { \ d[i] = d[i] * (_AlphaType)alpha; \ } \ } \ } #define ROCM_REDUCE_WITH_COMPLEX_PRODUCT_STRIDED(NAME, _OP) \ template \ __global__ void UCC_REDUCE_ROCM_STRIDED_COMPLEX_##NAME( \ const _Type *s1, const _Type *s2, _Type *d, size_t count, \ size_t stride, uint16_t n_src2, const bool with_alpha, \ const double alpha) \ { \ size_t start = blockIdx.x * blockDim.x + threadIdx.x; \ size_t step = blockDim.x * gridDim.x; \ size_t ld = stride / sizeof(_Type); \ size_t i; \ \ ucc_assert_system(stride % sizeof(_Type) == 0); \ switch (n_src2) { \ case 1: \ for (i = start; i < count; i += step) { \ d[i] = _OP(s1[i], s2[i]); \ } \ break; \ default: \ for (i = start; i < count; i += step) { \ d[i] = _OP(s1[i], s2[i]); \ for (size_t j = 1; j < n_src2; j++) { \ d[i] = _OP(d[i], s2[i + j * ld]); \ } \ } \ break; \ } \ if (with_alpha) { \ for (i = start; i < count; i += step) { \ d[i] = d[i] * (_AlphaType)alpha; \ } \ } \ } ROCM_REDUCE_WITH_OP_DEFAULT(SUM, DO_OP_SUM); ROCM_REDUCE_WITH_OP_DEFAULT(PROD, DO_OP_PROD); ROCM_REDUCE_WITH_COMPLEX_PRODUCT_DEFAULT(PROD_DOUBLE, hipCmul); ROCM_REDUCE_WITH_COMPLEX_PRODUCT_DEFAULT(PROD_FLOAT, hipCmulf); ROCM_REDUCE_WITH_OP_DEFAULT(MIN, DO_OP_MIN); ROCM_REDUCE_WITH_OP_DEFAULT(MAX, DO_OP_MAX); ROCM_REDUCE_WITH_OP_DEFAULT(LAND, DO_OP_LAND); ROCM_REDUCE_WITH_OP_DEFAULT(LOR, DO_OP_LOR); ROCM_REDUCE_WITH_OP_DEFAULT(LXOR, DO_OP_LXOR); ROCM_REDUCE_WITH_OP_DEFAULT(BAND, DO_OP_BAND); ROCM_REDUCE_WITH_OP_DEFAULT(BOR, DO_OP_BOR); ROCM_REDUCE_WITH_OP_DEFAULT(BXOR, DO_OP_BXOR); ROCM_REDUCE_WITH_OP_STRIDED(SUM, DO_OP_SUM); ROCM_REDUCE_WITH_OP_STRIDED(PROD, DO_OP_PROD); ROCM_REDUCE_WITH_COMPLEX_PRODUCT_STRIDED(PROD_DOUBLE, hipCmul); ROCM_REDUCE_WITH_COMPLEX_PRODUCT_STRIDED(PROD_FLOAT, hipCmulf); ROCM_REDUCE_WITH_OP_STRIDED(MIN, DO_OP_MIN); ROCM_REDUCE_WITH_OP_STRIDED(MAX, DO_OP_MAX); ROCM_REDUCE_WITH_OP_STRIDED(LAND, DO_OP_LAND); ROCM_REDUCE_WITH_OP_STRIDED(LOR, DO_OP_LOR); ROCM_REDUCE_WITH_OP_STRIDED(LXOR, DO_OP_LXOR); ROCM_REDUCE_WITH_OP_STRIDED(BAND, DO_OP_BAND); ROCM_REDUCE_WITH_OP_STRIDED(BOR, DO_OP_BOR); ROCM_REDUCE_WITH_OP_STRIDED(BXOR, DO_OP_BXOR); #define LAUNCH_KERNEL_A(NAME, type, _AlphaType, _task, s, b, t) \ do { \ if (_task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE) { \ UCC_REDUCE_ROCM_DEFAULT_##NAME \ <<>>(_task->reduce, _task->flags); \ } else { \ ucc_eee_task_reduce_strided_t *trs = &_task->reduce_strided; \ UCC_REDUCE_ROCM_STRIDED_##NAME<<>>( \ (type *)trs->src1, (type *)trs->src2, (type *)trs->dst, \ trs->count, trs->stride, trs->n_src2, \ (bool)(_task->flags & UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA), \ trs->alpha); \ } \ } while (0) #define LAUNCH_KERNEL_B(NAME, type, _AlphaType, _task, s, b, t) \ do { \ if (_task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE) { \ UCC_REDUCE_ROCM_DEFAULT_COMPLEX_##NAME \ <<>>(_task->reduce, _task->flags); \ } else { \ ucc_eee_task_reduce_strided_t *trs = &_task->reduce_strided; \ UCC_REDUCE_ROCM_STRIDED_COMPLEX_##NAME<<>>( \ (type *)trs->src1, (type *)trs->src2, (type *)trs->dst, \ trs->count, trs->stride, trs->n_src2, \ (bool)(_task->flags & UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA), \ trs->alpha); \ } \ } while (0) #define LAUNCH_KERNEL(NAME, type, _task, s, b, t) \ LAUNCH_KERNEL_A(NAME, type, type, _task, s, b, t) #define DT_REDUCE_INT(type, _task, _op, s, b, t) \ do { \ switch (_op) { \ case UCC_OP_AVG: \ case UCC_OP_SUM: \ LAUNCH_KERNEL(SUM, type, _task, s, b, t); \ break; \ case UCC_OP_PROD: \ LAUNCH_KERNEL(PROD, type, _task, s, b, t); \ break; \ case UCC_OP_MIN: \ LAUNCH_KERNEL(MIN, type, _task, s, b, t); \ break; \ case UCC_OP_MAX: \ LAUNCH_KERNEL(MAX, type, _task, s, b, t); \ break; \ case UCC_OP_LAND: \ LAUNCH_KERNEL(LAND, type, _task, s, b, t); \ break; \ case UCC_OP_BAND: \ LAUNCH_KERNEL(BAND, type, _task, s, b, t); \ break; \ case UCC_OP_LOR: \ LAUNCH_KERNEL(LOR, type, _task, s, b, t); \ break; \ case UCC_OP_BOR: \ LAUNCH_KERNEL(BOR, type, _task, s, b, t); \ break; \ case UCC_OP_LXOR: \ LAUNCH_KERNEL(LXOR, type, _task, s, b, t); \ break; \ case UCC_OP_BXOR: \ LAUNCH_KERNEL(BXOR, type, _task, s, b, t); \ break; \ default: \ ec_error(&ucc_ec_rocm.super, \ "int dtype does not support " \ "requested reduce op: %s", \ ucc_reduction_op_str(_op)); \ return UCC_ERR_NOT_SUPPORTED; \ } \ } while (0) #define DT_REDUCE_FLOAT(type, _task, _op, s, b, t) \ do { \ switch (_op) { \ case UCC_OP_AVG: \ case UCC_OP_SUM: \ LAUNCH_KERNEL(SUM, type, _task, s, b, t); \ break; \ case UCC_OP_PROD: \ LAUNCH_KERNEL(PROD, type, _task, s, b, t); \ break; \ case UCC_OP_MIN: \ LAUNCH_KERNEL(MIN, type, _task, s, b, t); \ break; \ case UCC_OP_MAX: \ LAUNCH_KERNEL(MAX, type, _task, s, b, t); \ break; \ default: \ ec_error(&ucc_ec_rocm.super, \ "float dtype does not support " \ "requested reduce op: %s", \ ucc_reduction_op_str(_op)); \ return UCC_ERR_NOT_SUPPORTED; \ } \ } while (0) #define DT_REDUCE_FLOAT_COMPLEX(NAME, type, _alphaType, _task, _op, s, b, t) \ do { \ switch (_op) { \ case UCC_OP_AVG: \ case UCC_OP_SUM: \ LAUNCH_KERNEL_A(SUM, type , _alphaType, _task, s, b, t); \ break; \ case UCC_OP_PROD: \ LAUNCH_KERNEL_B(NAME, type, _alphaType, _task, s, b, t); \ break; \ default: \ ec_error(&ucc_ec_rocm.super, \ "float complex dtype does not support " \ "requested reduce op: %s", \ ucc_reduction_op_str(_op)); \ return UCC_ERR_NOT_SUPPORTED; \ } \ } while (0) #ifdef __cplusplus extern "C" { #endif ucc_status_t ucc_ec_rocm_reduce(ucc_ee_executor_task_args_t *task, hipStream_t stream) { int th = EC_ROCM_CONFIG->reduce_num_threads; unsigned long bk = EC_ROCM_CONFIG->reduce_num_blocks; ucc_reduction_op_t op; ucc_datatype_t dt; size_t count; if (task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE) { dt = task->reduce.dt; count = task->reduce.count; op = task->reduce.op; } else { ucc_assert(task->task_type == UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED); dt = task->reduce_strided.dt; count = task->reduce_strided.count; op = task->reduce_strided.op; } if (count == 0) { return UCC_OK; } bk = ucc_min((count + th - 1) / th, bk); switch (dt) { case UCC_DT_INT8: DT_REDUCE_INT(int8_t, task, op, stream, bk, th); break; case UCC_DT_INT16: DT_REDUCE_INT(int16_t, task, op, stream, bk, th); break; case UCC_DT_INT32: DT_REDUCE_INT(int32_t, task, op, stream, bk, th); break; case UCC_DT_INT64: DT_REDUCE_INT(int64_t, task, op, stream, bk, th); break; case UCC_DT_UINT8: DT_REDUCE_INT(uint8_t, task, op, stream, bk, th); break; case UCC_DT_UINT16: DT_REDUCE_INT(uint16_t, task, op, stream, bk, th); break; case UCC_DT_UINT32: DT_REDUCE_INT(uint32_t, task, op, stream, bk, th); break; case UCC_DT_UINT64: DT_REDUCE_INT(uint64_t, task, op, stream, bk, th); break; case UCC_DT_FLOAT16: DT_REDUCE_FLOAT(__half, task, op, stream, bk, th); break; case UCC_DT_FLOAT32: #if SIZEOF_FLOAT == 4 DT_REDUCE_FLOAT(float, task, op, stream, bk, th); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT64: #if SIZEOF_DOUBLE == 8 DT_REDUCE_FLOAT(double, task, op, stream, bk, th); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT32_COMPLEX: DT_REDUCE_FLOAT_COMPLEX(PROD_FLOAT, hipFloatComplex, float, task, op, stream, bk, th); break; case UCC_DT_FLOAT64_COMPLEX: DT_REDUCE_FLOAT_COMPLEX(PROD_DOUBLE, hipDoubleComplex, double, task, op, stream, bk, th); break; case UCC_DT_BFLOAT16: ucc_assert(2 == sizeof(hip_bfloat16)); DT_REDUCE_FLOAT(hip_bfloat16, task, op, stream, bk, th); break; default: ec_error(&ucc_ec_rocm.super, "unsupported reduction type (%s)", ucc_datatype_str(dt)); return UCC_ERR_NOT_SUPPORTED; } ROCMCHECK(hipGetLastError()); return UCC_OK; } #ifdef __cplusplus } #endif ucc-1.8.0/src/components/ec/rocm/kernel/Makefile.am0000664000175000017500000000200615211535620022273 0ustar alastairalastair# # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # HIPCCFLAGS = \ ${AM_CPPFLAGS} \ ${UCS_CPPFLAGS} \ ${HIP_CPPFLAGS} \ ${ROCM_CPPFLAGS} \ -I${UCC_TOP_BUILDDIR} \ -I${UCC_TOP_SRCDIR} \ -I${UCC_TOP_SRCDIR}/src \ -I${UCC_TOP_BUILDDIR}/src \ -I${UCC_TOP_SRCDIR}/src/components/ec/rocm LINK = $(LIBTOOL) --mode=link $(CC) -o $@ .cu.o: $(HIPCC) -c $< -o $@ $(ROCM_ARCH) $(HIPCCFLAGS) .cu.lo: /bin/bash $(top_srcdir)/cuda_lt.sh "$(LIBTOOL)" $@ $(HIPCC) -c $< $(ROCM_ARCH) $(HIPCCFLAGS) comp_noinst = libucc_ec_rocm_kernels.la libucc_ec_rocm_kernels_la_SOURCES = ec_rocm_executor_kernel.cu \ ec_rocm_reduce.cu libucc_ec_rocm_kernels_la_CPPFLAGS = noinst_LTLIBRARIES = $(comp_noinst) ucc-1.8.0/src/components/ec/rocm/ec_rocm_executor.h0000664000175000017500000000261115211535620022457 0ustar alastairalastair/** * Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_EC_ROCM_EXECUTOR_H_ #define UCC_EC_ROCM_EXECUTOR_H_ #include "ec_rocm.h" ucc_status_t ucc_rocm_executor_init(const ucc_ee_executor_params_t *params, ucc_ee_executor_t **executor); ucc_status_t ucc_rocm_executor_status(const ucc_ee_executor_t *executor); ucc_status_t ucc_rocm_executor_finalize(ucc_ee_executor_t *executor); ucc_status_t ucc_rocm_executor_start(ucc_ee_executor_t *executor, void *ee_context); ucc_status_t ucc_rocm_executor_stop(ucc_ee_executor_t *executor); ucc_status_t ucc_rocm_executor_task_post(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task); ucc_status_t ucc_rocm_executor_task_test(const ucc_ee_executor_task_t *task); ucc_status_t ucc_rocm_executor_task_finalize(ucc_ee_executor_task_t *task); /* implemented in ec_rocm_executor.cu */ ucc_status_t ucc_ec_rocm_persistent_kernel_start(ucc_ec_rocm_executor_t *eee); ucc_status_t ucc_ec_rocm_reduce(ucc_ee_executor_task_args_t *task, hipStream_t stream); #endif ucc-1.8.0/src/components/ec/rocm/ec_rocm_executor_persistent.c0000664000175000017500000001037015211535620024733 0ustar alastairalastair/** * Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "ec_rocm_executor.h" #include "utils/arch/cpu.h" ucc_status_t ucc_rocm_executor_persistent_task_post(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(executor, ucc_ec_rocm_executor_t); int max_tasks = EC_ROCM_CONFIG->exec_max_tasks; ucc_ee_executor_task_t *ee_task; ucc_datatype_t dt; ucc_reduction_op_t op; if (task_args->task_type != UCC_EE_EXECUTOR_TASK_COPY && task_args->task_type != UCC_EE_EXECUTOR_TASK_COPY_MULTI) { if (task_args->task_type == UCC_EE_EXECUTOR_TASK_REDUCE) { dt = task_args->reduce.dt; op = task_args->reduce.op; } else { dt = task_args->reduce_strided.dt; op = task_args->reduce_strided.op; } if (op != UCC_OP_SUM) { ec_error(&ucc_ec_rocm.super, "not supported reduction op: %s", ucc_reduction_op_str(op)); return UCC_ERR_NOT_SUPPORTED; } if ((dt != UCC_DT_FLOAT32) && (dt != UCC_DT_FLOAT64) && (dt != UCC_DT_INT32)) { ec_error(&ucc_ec_rocm.super, "not supported reduction dtype: %s", ucc_datatype_str(dt)); return UCC_ERR_NOT_SUPPORTED; } } if (ucc_ec_rocm.thread_mode == UCC_THREAD_MULTIPLE) { ucc_spin_lock(&eee->tasks_lock); } ee_task = &(eee->tasks[eee->pidx % max_tasks]); ee_task->eee = executor; ee_task->status = UCC_OPERATION_INITIALIZED; memcpy(&ee_task->args, task_args, sizeof(ucc_ee_executor_task_args_t)); ucc_memory_cpu_store_fence(); eee->pidx += 1; if (ucc_ec_rocm.thread_mode == UCC_THREAD_MULTIPLE) { ucc_spin_unlock(&eee->tasks_lock); } ec_debug(&ucc_ec_rocm.super, "executor task post, eee: %p", eee); *task = ee_task; return UCC_OK; } ucc_status_t ucc_rocm_executor_persistent_task_test(const ucc_ee_executor_task_t *task) { ROCMCHECK(hipGetLastError()); return task->status; } ucc_status_t ucc_rocm_executor_persistent_task_finalize(ucc_ee_executor_task_t *task) //NOLINT: task is unused { return UCC_OK; } ucc_status_t ucc_rocm_executor_persistent_start(ucc_ee_executor_t *executor, void *ee_context) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(executor, ucc_ec_rocm_executor_t); ucc_status_t status; ucc_assert(eee->state == UCC_EC_ROCM_EXECUTOR_INITIALIZED); ec_debug(&ucc_ec_rocm.super, "executor start, eee: %p", eee); eee->super.ee_context = ee_context; eee->state = UCC_EC_ROCM_EXECUTOR_POSTED; eee->pidx = 0; eee->mode = UCC_EC_ROCM_EXECUTOR_MODE_PERSISTENT; status = ucc_ec_rocm_persistent_kernel_start(eee); if (status != UCC_OK) { ec_error(&ucc_ec_rocm.super, "failed to launch executor kernel"); return status; } eee->ops.task_post = ucc_rocm_executor_persistent_task_post; eee->ops.task_test = ucc_rocm_executor_persistent_task_test; eee->ops.task_finalize = ucc_rocm_executor_persistent_task_finalize; return UCC_OK; } ucc_status_t ucc_rocm_executor_persistent_stop(ucc_ee_executor_t *executor) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(executor, ucc_ec_rocm_executor_t); volatile ucc_ec_rocm_executor_state_t *st = &eee->state; ec_debug(&ucc_ec_rocm.super, "executor stop, eee: %p", eee); /* can be safely ended only if it's in STARTED or COMPLETED_ACK state */ ucc_assert((*st != UCC_EC_ROCM_EXECUTOR_POSTED) && (*st != UCC_EC_ROCM_EXECUTOR_SHUTDOWN)); *st = UCC_EC_ROCM_EXECUTOR_SHUTDOWN; eee->pidx = -1; while(*st != UCC_EC_ROCM_EXECUTOR_SHUTDOWN_ACK) { } eee->super.ee_context = NULL; eee->state = UCC_EC_ROCM_EXECUTOR_INITIALIZED; return UCC_OK; } ucc-1.8.0/src/components/ec/rocm/ec_rocm_executor.c0000664000175000017500000001156015211535620022455 0ustar alastairalastair/** * Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "ec_rocm_executor.h" #include "components/ec/ucc_ec.h" ucc_status_t ucc_rocm_executor_persistent_start(ucc_ee_executor_t *executor, void *ee_context); ucc_status_t ucc_rocm_executor_persistent_stop(ucc_ee_executor_t *executor); ucc_status_t ucc_rocm_executor_interruptible_start(ucc_ee_executor_t *executor); ucc_status_t ucc_rocm_executor_interruptible_stop(ucc_ee_executor_t *executor); ucc_status_t ucc_rocm_executor_init(const ucc_ee_executor_params_t *params, ucc_ee_executor_t **executor) { ucc_ec_rocm_executor_t *eee = ucc_mpool_get(&ucc_ec_rocm.executors); ucc_status_t status; ucc_ee_executor_params_t cpu_params = { .mask = UCC_EE_EXECUTOR_PARAM_FIELD_TYPE, .ee_type = UCC_EE_CPU_THREAD }; if (ucc_unlikely(!eee)) { ec_error(&ucc_ec_rocm.super, "failed to allocate executor"); return UCC_ERR_NO_MEMORY; } ec_debug(&ucc_ec_rocm.super, "executor init, eee: %p", eee); eee->super.ee_type = params->ee_type; eee->state = UCC_EC_ROCM_EXECUTOR_INITIALIZED; status = ucc_ee_executor_init(&cpu_params, &ucc_ec_rocm.cpu_executor); if (status != UCC_OK) { ec_error(&ucc_ec_rocm.super, "Error initializing CPU executor from ROCm component"); } *executor = &eee->super; return UCC_OK; } ucc_status_t ucc_rocm_executor_status(const ucc_ee_executor_t *executor) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(executor, ucc_ec_rocm_executor_t); switch (eee->state) { case UCC_EC_ROCM_EXECUTOR_INITIALIZED: return UCC_OPERATION_INITIALIZED; case UCC_EC_ROCM_EXECUTOR_POSTED: return UCC_INPROGRESS; case UCC_EC_ROCM_EXECUTOR_STARTED: return UCC_OK; default: /* executor has been destroyed */ return UCC_ERR_NO_RESOURCE; } } ucc_status_t ucc_rocm_executor_finalize(ucc_ee_executor_t *executor) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(executor, ucc_ec_rocm_executor_t); ucc_status_t status; ec_debug(&ucc_ec_rocm.super, "executor free, eee: %p", eee); ucc_assert(eee->state == UCC_EC_ROCM_EXECUTOR_INITIALIZED); ucc_mpool_put(eee); status = ucc_ee_executor_finalize(ucc_ec_rocm.cpu_executor); if (status != UCC_OK) { ec_error(&ucc_ec_rocm.super, "Error finalizing CPU executor from ROCm component"); } return status; } ucc_status_t ucc_rocm_executor_task_post(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(executor, ucc_ec_rocm_executor_t); return eee->ops.task_post(executor, task_args, task); } ucc_status_t ucc_rocm_executor_task_test(const ucc_ee_executor_task_t *task) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(task->eee, ucc_ec_rocm_executor_t); return eee->ops.task_test(task); } ucc_status_t ucc_rocm_executor_task_finalize(ucc_ee_executor_task_t *task) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(task->eee, ucc_ec_rocm_executor_t); return eee->ops.task_finalize(task); } ucc_status_t ucc_rocm_executor_start(ucc_ee_executor_t *executor, void *ee_context) { ucc_status_t status; status = ucc_ee_executor_start(ucc_ec_rocm.cpu_executor, ee_context); if (status != UCC_OK) { ec_error(&ucc_ec_rocm.super, "Error starting CPU executor from ROCm component"); return status; } if (!ee_context) { return ucc_rocm_executor_interruptible_start(executor); } else { return ucc_rocm_executor_persistent_start(executor, ee_context); } } ucc_status_t ucc_rocm_executor_stop(ucc_ee_executor_t *executor) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(executor, ucc_ec_rocm_executor_t); ucc_status_t status; status = ucc_ee_executor_stop(ucc_ec_rocm.cpu_executor); if (status != UCC_OK) { ec_error(&ucc_ec_rocm.super, "Error stopping CPU executor from ROCm component"); return status; } if (eee->mode == UCC_EC_ROCM_EXECUTOR_MODE_INTERRUPTIBLE) { return ucc_rocm_executor_interruptible_stop(executor); } else { return ucc_rocm_executor_persistent_stop(executor); } } ucc-1.8.0/src/components/ec/rocm/Makefile.am0000664000175000017500000000170015211535620021013 0ustar alastairalastair# # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. # if HAVE_ROCM SUBDIRS = kernel sources = \ ec_rocm.h \ ec_rocm.c \ ec_rocm_executor.h \ ec_rocm_executor.c \ ec_rocm_executor_interruptible.c \ ec_rocm_executor_persistent.c module_LTLIBRARIES = libucc_ec_rocm.la libucc_ec_rocm_la_SOURCES = $(sources) libucc_ec_rocm_la_CPPFLAGS = $(AM_CPPFLAGS) $(HIP_CPPFLAGS) $(ROCM_CPPFLAGS) $(BASE_CPPFLAGS) libucc_ec_rocm_la_CFLAGS = $(BASE_CFLAGS) libucc_ec_rocm_la_LDFLAGS = -version-info $(SOVERSION) --as-needed $(HIP_LDFLAGS) $(ROCM_LDFLAGS) libucc_ec_rocm_la_LIBADD = $(HIP_LIBS) $(ROCM_LIBS) \ kernel/libucc_ec_rocm_kernels.la \ $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am endif ucc-1.8.0/src/components/ec/rocm/ec_rocm.h0000664000175000017500000001362715211535620020552 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022-2023. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_EC_ROCM_H_ #define UCC_EC_ROCM_H_ #include "components/ec/base/ucc_ec_base.h" #include "components/ec/ucc_ec_log.h" #include "core/ucc_ee.h" #include "utils/ucc_mpool.h" #include "utils/arch/rocm_def.h" #include #include #include #include #pragma GCC diagnostic push #if defined(__clang__) #pragma GCC diagnostic ignored "-Wunknown-warning-option" #endif #include #pragma GCC diagnostic pop typedef enum ucc_ec_task_status { UCC_EC_ROCM_TASK_COMPLETED, UCC_EC_ROCM_TASK_POSTED, UCC_EC_ROCM_TASK_STARTED, UCC_EC_ROCM_TASK_COMPLETED_ACK } ucc_ec_task_status_t; typedef enum ucc_ec_rocm_executor_state { UCC_EC_ROCM_EXECUTOR_INITIALIZED, UCC_EC_ROCM_EXECUTOR_POSTED, UCC_EC_ROCM_EXECUTOR_STARTED, UCC_EC_ROCM_EXECUTOR_SHUTDOWN, UCC_EC_ROCM_EXECUTOR_SHUTDOWN_ACK } ucc_ec_rocm_executor_state_t; typedef enum ucc_ec_rocm_executor_mode { UCC_EC_ROCM_EXECUTOR_MODE_PERSISTENT, UCC_EC_ROCM_EXECUTOR_MODE_INTERRUPTIBLE } ucc_ec_rocm_executor_mode_t; static inline ucc_status_t hip_error_to_ucc_status(hipError_t hip_err) { switch(hip_err) { case hipSuccess: return UCC_OK; case hipErrorNotReady: return UCC_INPROGRESS; default: break; } return UCC_ERR_NO_MESSAGE; } typedef ucc_status_t (*ucc_ec_rocm_task_post_fn) (uint32_t *dev_status, int blocking_wait, hipStream_t stream); typedef struct ucc_ec_rocm_config { ucc_ec_config_t super; unsigned long exec_num_workers; unsigned long exec_num_threads; unsigned long exec_max_tasks; unsigned long exec_num_streams; unsigned long reduce_num_blocks; int reduce_num_threads; int reduce_host_limit; int copy_host_limit; } ucc_ec_rocm_config_t; typedef struct ucc_ec_rocm { ucc_ec_base_t super; int stream_initialized; hipStream_t stream; int exec_streams_initialized; hipStream_t *exec_streams; ucc_mpool_t events; ucc_mpool_t strm_reqs; ucc_mpool_t executors; ucc_mpool_t executor_interruptible_tasks; ucc_thread_mode_t thread_mode; ucc_spinlock_t init_spinlock; ucc_ee_executor_t *cpu_executor; } ucc_ec_rocm_t; typedef struct ucc_rocm_ec_event { hipEvent_t event; } ucc_ec_rocm_event_t; typedef struct ucc_ec_rocm_stream_request { uint32_t status; uint32_t *dev_status; hipStream_t stream; } ucc_ec_rocm_stream_request_t; typedef struct ucc_ec_rocm_executor_interruptible_task { ucc_ee_executor_task_t super; void *event; } ucc_ec_rocm_executor_interruptible_task_t; typedef struct ucc_ec_rocm_executor_task_ops { ucc_status_t (*task_post)(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task); ucc_status_t (*task_test)(const ucc_ee_executor_task_t *task); ucc_status_t (*task_finalize)(ucc_ee_executor_task_t *task); } ucc_ec_rocm_executor_task_ops_t; typedef struct ucc_ec_rocm_executor { ucc_ee_executor_t super; ucc_ec_rocm_executor_mode_t mode; ucc_ec_rocm_executor_task_ops_t ops; ucc_spinlock_t tasks_lock; ucc_ec_rocm_executor_state_t state; int pidx; ucc_ee_executor_task_t *tasks; ucc_ec_rocm_executor_state_t *dev_state; ucc_ee_executor_task_t *dev_tasks; int *dev_pidx; int *dev_cidx; } ucc_ec_rocm_executor_t; ucc_status_t ucc_ec_rocm_event_create(void **event); ucc_status_t ucc_ec_rocm_event_destroy(void *event); ucc_status_t ucc_ec_rocm_event_post(void *ee_context, void *event); ucc_status_t ucc_ec_rocm_event_test(void *event); extern ucc_ec_rocm_t ucc_ec_rocm; #define EC_ROCM_CONFIG \ (ucc_derived_of(ucc_ec_rocm.super.config, ucc_ec_rocm_config_t)) #define UCC_EC_ROCM_INIT_STREAM() do { \ if (!ucc_ec_rocm.stream_initialized) { \ hipError_t hip_st = hipSuccess; \ ucc_spin_lock(&ucc_ec_rocm.init_spinlock); \ if (!ucc_ec_rocm.stream_initialized) { \ hip_st = hipStreamCreateWithFlags(&ucc_ec_rocm.stream, \ hipStreamNonBlocking); \ ucc_ec_rocm.stream_initialized = 1; \ } \ ucc_spin_unlock(&ucc_ec_rocm.init_spinlock); \ if(hip_st != hipSuccess) { \ ec_error(&ucc_ec_rocm.super, "rocm failed with ret:%d(%s)", \ hip_st, hipGetErrorString(hip_st)); \ return UCC_ERR_NO_MESSAGE; \ } \ } \ } while(0) #endif ucc-1.8.0/src/components/ec/rocm/ec_rocm.c0000664000175000017500000002702115211535620020536 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "ec_rocm.h" #include "ec_rocm_executor.h" #include "utils/ucc_malloc.h" #include "utils/arch/cpu.h" #include #include #include static ucc_config_field_t ucc_ec_rocm_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_ec_rocm_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_ec_config_table)}, {"EXEC_NUM_WORKERS", "1", "Number of thread blocks to use for rocm executor", ucc_offsetof(ucc_ec_rocm_config_t, exec_num_workers), UCC_CONFIG_TYPE_ULUNITS}, {"EXEC_NUM_THREADS", "512", "Number of thread per block to use for rocm executor", ucc_offsetof(ucc_ec_rocm_config_t, exec_num_threads), UCC_CONFIG_TYPE_ULUNITS}, {"EXEC_MAX_TASKS", "128", "Maximum number of outstanding tasks per executor", ucc_offsetof(ucc_ec_rocm_config_t, exec_max_tasks), UCC_CONFIG_TYPE_ULUNITS}, {"EXEC_NUM_STREAMS", "8", "Number of streams used by interruptible executor", ucc_offsetof(ucc_ec_rocm_config_t, exec_num_streams), UCC_CONFIG_TYPE_ULUNITS}, {"REDUCE_NUM_BLOCKS", "auto", "Number of thread blocks to use for reduction in interruptible mode", ucc_offsetof(ucc_ec_rocm_config_t, reduce_num_blocks), UCC_CONFIG_TYPE_ULUNITS}, {"REDUCE_HOST_LIMIT", "256", "Maximum data size for which to use host-based reduction operations", ucc_offsetof(ucc_ec_rocm_config_t, reduce_host_limit), UCC_CONFIG_TYPE_MEMUNITS}, /* Disabled by default. * Recommended settings: MI100: 64 bytes, MI200: 4kbytes */ {"COPY_HOST_LIMIT", "0", "Maximum data size for which to use host-based copy operations", ucc_offsetof(ucc_ec_rocm_config_t, copy_host_limit), UCC_CONFIG_TYPE_MEMUNITS}, {NULL} }; static ucc_status_t ucc_ec_rocm_ee_executor_mpool_chunk_malloc(ucc_mpool_t *mp, //NOLINT: mp is unused size_t *size_p, void ** chunk_p) { return ROCM_FUNC(hipHostMalloc((void**)chunk_p, *size_p, hipHostMallocMapped)); } static void ucc_ec_rocm_ee_executor_mpool_chunk_free(ucc_mpool_t *mp, //NOLINT: mp is unused void *chunk) { ROCM_FUNC(hipHostFree(chunk)); } static void ucc_ec_rocm_executor_chunk_init(ucc_mpool_t *mp, void *obj, //NOLINT: mp is unused void *chunk) //NOLINT: chunk is unused { ucc_ec_rocm_executor_t *eee = (ucc_ec_rocm_executor_t*) obj; int max_tasks = EC_ROCM_CONFIG->exec_max_tasks; ROCM_FUNC(hipHostGetDevicePointer( (void**)(&eee->dev_state), (void *)&eee->state, 0)); ROCM_FUNC(hipHostGetDevicePointer( (void**)(&eee->dev_pidx), (void *)&eee->pidx, 0)); ROCM_FUNC(hipMalloc((void**)&eee->dev_cidx, sizeof(*eee->dev_cidx))); ROCM_FUNC(hipHostMalloc((void**)&eee->tasks, max_tasks * sizeof(ucc_ee_executor_task_t), hipHostMallocMapped)); ROCM_FUNC(hipHostGetDevicePointer( (void**)(&eee->dev_tasks), (void *)eee->tasks, 0)); if (ucc_ec_rocm.thread_mode == UCC_THREAD_MULTIPLE) { ucc_spinlock_init(&eee->tasks_lock, 0); } } static void ucc_ec_rocm_executor_chunk_cleanup(ucc_mpool_t *mp, void *obj) //NOLINT: mp is unused { ucc_ec_rocm_executor_t *eee = (ucc_ec_rocm_executor_t*) obj; ROCM_FUNC(hipFree((void*)eee->dev_cidx)); ROCM_FUNC(hipHostFree((void*)eee->tasks)); if (ucc_ec_rocm.thread_mode == UCC_THREAD_MULTIPLE) { ucc_spinlock_destroy(&eee->tasks_lock); } } static ucc_mpool_ops_t ucc_ec_rocm_ee_executor_mpool_ops = { .chunk_alloc = ucc_ec_rocm_ee_executor_mpool_chunk_malloc, .chunk_release = ucc_ec_rocm_ee_executor_mpool_chunk_free, .obj_init = ucc_ec_rocm_executor_chunk_init, .obj_cleanup = ucc_ec_rocm_executor_chunk_cleanup, }; static void ucc_ec_rocm_event_init(ucc_mpool_t *mp, void *obj, void *chunk) //NOLINT: mp is unused { ucc_ec_rocm_event_t *base = (ucc_ec_rocm_event_t *) obj; if (ucc_unlikely( hipSuccess != hipEventCreateWithFlags(&base->event, hipEventDisableTiming))) { ec_error(&ucc_ec_rocm.super, "hipEventCreateWithFlags Failed"); } } static void ucc_ec_rocm_event_cleanup(ucc_mpool_t *mp, void *obj) //NOLINT: mp is unused { ucc_ec_rocm_event_t *base = (ucc_ec_rocm_event_t *) obj; if (ucc_unlikely(hipSuccess != hipEventDestroy(base->event))) { ec_error(&ucc_ec_rocm.super, "hipEventDestroy Failed"); } } static ucc_mpool_ops_t ucc_ec_rocm_event_mpool_ops = { .chunk_alloc = ucc_mpool_hugetlb_malloc, .chunk_release = ucc_mpool_hugetlb_free, .obj_init = ucc_ec_rocm_event_init, .obj_cleanup = ucc_ec_rocm_event_cleanup, }; static ucc_status_t ucc_ec_rocm_init(const ucc_ec_params_t *ec_params) { ucc_ec_rocm_config_t *cfg = EC_ROCM_CONFIG; ucc_status_t status; int device, num_devices; hipError_t rocm_st; hipDeviceProp_t prop; ucc_ec_rocm.stream = NULL; ucc_ec_rocm.stream_initialized = 0; ucc_ec_rocm.exec_streams_initialized = 0; ucc_strncpy_safe(ucc_ec_rocm.super.config->log_component.name, ucc_ec_rocm.super.super.name, sizeof(ucc_ec_rocm.super.config->log_component.name)); ucc_ec_rocm.thread_mode = ec_params->thread_mode; rocm_st = hipGetDeviceCount(&num_devices); if ((rocm_st != hipSuccess) || (num_devices == 0)) { ec_debug(&ucc_ec_rocm.super, "rocm devices are not found"); return UCC_ERR_NO_RESOURCE; } ROCMCHECK(hipGetDevice(&device)); ROCMCHECK(hipGetDeviceProperties(&prop, device)); cfg->reduce_num_threads = prop.maxThreadsPerBlock; if (cfg->reduce_num_blocks != UCC_ULUNITS_AUTO) { if (prop.maxGridSize[0] < cfg->reduce_num_blocks) { ec_warn(&ucc_ec_rocm.super, "number of blocks is too large, max supported %d", prop.maxGridSize[0]); cfg->reduce_num_blocks = prop.maxGridSize[0]; } } else { cfg->reduce_num_blocks = prop.maxGridSize[0]; } if (cfg->exec_num_streams < 1) { ec_warn(&ucc_ec_rocm.super, "number of streams is too small, min supported 1"); cfg->exec_num_streams = 1; } /*create event pool */ ucc_ec_rocm.exec_streams = ucc_calloc(cfg->exec_num_streams, sizeof(hipStream_t), "ec rocm streams"); if (!ucc_ec_rocm.exec_streams) { ec_error(&ucc_ec_rocm.super, "failed to allocate streams array"); return UCC_ERR_NO_MEMORY; } status = ucc_mpool_init(&ucc_ec_rocm.events, 0, sizeof(ucc_ec_rocm_event_t), 0, UCC_CACHE_LINE_SIZE, 16, UINT_MAX, &ucc_ec_rocm_event_mpool_ops, UCC_THREAD_MULTIPLE, "ROCM Event Objects"); if (status != UCC_OK) { ec_error(&ucc_ec_rocm.super, "failed to create event pool"); return status; } status = ucc_mpool_init( &ucc_ec_rocm.executors, 0, sizeof(ucc_ec_rocm_executor_t), 0, UCC_CACHE_LINE_SIZE, 16, UINT_MAX, &ucc_ec_rocm_ee_executor_mpool_ops, UCC_THREAD_MULTIPLE, "EE executor Objects"); if (status != UCC_OK) { ec_error(&ucc_ec_rocm.super, "failed to create executors pool"); return status; } status = ucc_mpool_init( &ucc_ec_rocm.executor_interruptible_tasks, 0, sizeof(ucc_ec_rocm_executor_interruptible_task_t), 0, UCC_CACHE_LINE_SIZE, 16, UINT_MAX, NULL, UCC_THREAD_MULTIPLE, "interruptible executor tasks"); if (status != UCC_OK) { ec_error(&ucc_ec_rocm.super, "failed to create interruptible tasks pool"); return status; } ucc_spinlock_init(&ucc_ec_rocm.init_spinlock, 0); return UCC_OK; } static ucc_status_t ucc_ec_rocm_get_attr(ucc_ec_attr_t *ec_attr) { if (ec_attr->field_mask & UCC_EC_ATTR_FIELD_THREAD_MODE) { ec_attr->thread_mode = ucc_ec_rocm.thread_mode; } return UCC_OK; } ucc_status_t ucc_ec_rocm_event_create(void **event) { ucc_ec_rocm_event_t *rocm_event; rocm_event = ucc_mpool_get(&ucc_ec_rocm.events); if (ucc_unlikely(!rocm_event)) { ec_error(&ucc_ec_rocm.super, "Failed to allocate rocm event"); return UCC_ERR_NO_MEMORY; } *event = rocm_event; return UCC_OK; } ucc_status_t ucc_ec_rocm_event_destroy(void *event) { ucc_ec_rocm_event_t *rocm_event = event; ucc_mpool_put(rocm_event); return UCC_OK; } ucc_status_t ucc_ec_rocm_event_post(void *ee_context, void *event) { hipStream_t stream = (hipStream_t) ee_context; ucc_ec_rocm_event_t *rocm_event = event; ROCMCHECK(hipEventRecord(rocm_event->event, stream)); return UCC_OK; } ucc_status_t ucc_ec_rocm_event_test(void *event) { ucc_ec_rocm_event_t *rocm_event = event; hipError_t hip_err; hip_err = hipEventQuery(rocm_event->event); if (ucc_unlikely((hip_err != hipSuccess) && (hip_err != hipErrorNotReady))) { ROCMCHECK(hip_err); } return hip_error_to_ucc_status(hip_err); } static ucc_status_t ucc_ec_rocm_finalize() { int i; if (ucc_ec_rocm.stream != NULL) { ROCMCHECK(hipStreamDestroy(ucc_ec_rocm.stream)); ucc_ec_rocm.stream = NULL; } if (ucc_ec_rocm.exec_streams_initialized) { for (i = 0; i < EC_ROCM_CONFIG->exec_num_streams; i++) { ROCM_FUNC(hipStreamDestroy(ucc_ec_rocm.exec_streams[i])); } ucc_ec_rocm.exec_streams_initialized = 0; } ucc_mpool_cleanup(&ucc_ec_rocm.events, 1); ucc_mpool_cleanup(&ucc_ec_rocm.executors, 1); ucc_free(ucc_ec_rocm.exec_streams); return UCC_OK; } ucc_ec_rocm_t ucc_ec_rocm = { .super.super.name = "rocm ec", .super.ref_cnt = 0, .super.type = UCC_EE_ROCM_STREAM, .super.init = ucc_ec_rocm_init, .super.get_attr = ucc_ec_rocm_get_attr, .super.finalize = ucc_ec_rocm_finalize, .super.config_table = { .name = "ROCM execution component", .prefix = "EC_ROCM_", .table = ucc_ec_rocm_config_table, .size = sizeof(ucc_ec_rocm_config_t), }, .super.ops.create_event = ucc_ec_rocm_event_create, .super.ops.destroy_event = ucc_ec_rocm_event_destroy, .super.ops.event_post = ucc_ec_rocm_event_post, .super.ops.event_test = ucc_ec_rocm_event_test, .super.executor_ops.init = ucc_rocm_executor_init, .super.executor_ops.start = ucc_rocm_executor_start, .super.executor_ops.status = ucc_rocm_executor_status, .super.executor_ops.stop = ucc_rocm_executor_stop, .super.executor_ops.task_post = ucc_rocm_executor_task_post, .super.executor_ops.task_test = ucc_rocm_executor_task_test, .super.executor_ops.task_finalize = ucc_rocm_executor_task_finalize, .super.executor_ops.finalize = ucc_rocm_executor_finalize, }; UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_ec_rocm.super.config_table, &ucc_config_global_list); ucc-1.8.0/src/components/ec/rocm/ec_rocm_executor_interruptible.c0000664000175000017500000002060315211535620025423 0ustar alastairalastair/** * Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include #include "ec_rocm_executor.h" #include "components/mc/ucc_mc.h" #include "components/ec/ucc_ec.h" #include "utils/ucc_atomic.h" static bool ucc_ec_rocm_copy_multi_use_host (const ucc_ee_executor_task_args_t* task_args) { bool result = true; for (int i = 0; i < task_args->copy_multi.num_vectors; i++) { if (task_args->copy_multi.counts[i] > EC_ROCM_CONFIG->copy_host_limit) { result = false; break; } } return result; } static int ucc_ec_rocm_total_reduce_len(const ucc_ee_executor_task_args_t* task_args) { int total_len = 0; ucc_datatype_t dt; size_t count; if (task_args->task_type == UCC_EE_EXECUTOR_TASK_REDUCE) { dt = task_args->reduce.dt; count = task_args->reduce.count; } else { ucc_assert(task_args->task_type == UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED); dt = task_args->reduce_strided.dt; count = task_args->reduce_strided.count; } total_len += count * ucc_dt_size(dt); return total_len; } static bool ucc_ec_rocm_host_dt_supported(const ucc_ee_executor_task_args_t* task_args) { bool result = false; ucc_datatype_t dt; if (task_args->task_type == UCC_EE_EXECUTOR_TASK_REDUCE) { dt = task_args->reduce.dt; } else { ucc_assert(task_args->task_type == UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED); dt = task_args->reduce_strided.dt; } if (dt != UCC_DT_BFLOAT16 && dt != UCC_DT_FLOAT16 && dt != UCC_DT_FLOAT32_COMPLEX && dt != UCC_DT_FLOAT64_COMPLEX) { result = true; } return result; } static inline bool ec_rocm_use_host_ops(const ucc_ee_executor_task_args_t *_task_args) { if ( (_task_args->task_type == UCC_EE_EXECUTOR_TASK_COPY && _task_args->copy.len <= EC_ROCM_CONFIG->copy_host_limit) || (_task_args->task_type == UCC_EE_EXECUTOR_TASK_COPY_MULTI && ucc_ec_rocm_copy_multi_use_host(_task_args)) || ((_task_args->task_type == UCC_EE_EXECUTOR_TASK_REDUCE || _task_args->task_type == UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED) && ucc_ec_rocm_total_reduce_len(_task_args) <= EC_ROCM_CONFIG->reduce_host_limit && ucc_ec_rocm_host_dt_supported(_task_args) )) { return true; } return false; } ucc_status_t ucc_rocm_executor_interruptible_get_stream(hipStream_t *stream) { static uint32_t last_used = 0; int num_streams = EC_ROCM_CONFIG->exec_num_streams; ucc_status_t st; int i, j; uint32_t id; ucc_assert(num_streams > 0); if (ucc_unlikely(!ucc_ec_rocm.exec_streams_initialized)) { ucc_spin_lock(&ucc_ec_rocm.init_spinlock); if (ucc_ec_rocm.exec_streams_initialized) { goto unlock; } for(i = 0; i < num_streams; i++) { st = ROCM_FUNC(hipStreamCreateWithFlags(&ucc_ec_rocm.exec_streams[i], hipStreamNonBlocking)); if (st != UCC_OK) { for (j = 0; j < i; j++) { ROCM_FUNC(hipStreamDestroy(ucc_ec_rocm.exec_streams[j])); } ucc_spin_unlock(&ucc_ec_rocm.init_spinlock); return st; } } ucc_ec_rocm.exec_streams_initialized = 1; unlock: ucc_spin_unlock(&ucc_ec_rocm.init_spinlock); } id = ucc_atomic_fadd32(&last_used, 1); *stream = ucc_ec_rocm.exec_streams[id % num_streams]; return UCC_OK; } ucc_status_t ucc_ec_rocm_copy_multi_kernel(const ucc_ee_executor_task_args_t *args, hipStream_t stream); ucc_status_t ucc_rocm_executor_interruptible_task_post(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task) { ucc_ec_rocm_executor_interruptible_task_t *ee_task; hipStream_t stream; ucc_status_t status; if (ec_rocm_use_host_ops(task_args)) { status = ucc_ee_executor_task_post(ucc_ec_rocm.cpu_executor, task_args, task); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_rocm.super, "failed to execute host ops from ROCm component"); } return status; } status = ucc_rocm_executor_interruptible_get_stream(&stream); if (ucc_unlikely(status != UCC_OK)) { return status; } ee_task = ucc_mpool_get(&ucc_ec_rocm.executor_interruptible_tasks); if (ucc_unlikely(!ee_task)) { return UCC_ERR_NO_MEMORY; } status = ucc_ec_rocm_event_create(&ee_task->event); if (ucc_unlikely(status != UCC_OK)) { ucc_mpool_put(ee_task); return status; } ee_task->super.status = UCC_INPROGRESS; ee_task->super.eee = executor; memcpy(&ee_task->super.args, task_args, sizeof(ucc_ee_executor_task_args_t)); switch (task_args->task_type) { case UCC_EE_EXECUTOR_TASK_COPY: status = ROCM_FUNC(hipMemcpyAsync(task_args->copy.dst, task_args->copy.src, task_args->copy.len, hipMemcpyDefault, stream)); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_rocm.super, "failed to start memcpy op"); goto free_task; } break; case UCC_EE_EXECUTOR_TASK_COPY_MULTI: status = ucc_ec_rocm_copy_multi_kernel(task_args, stream); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_rocm.super, "failed to start copy multi op"); goto free_task; } break; case UCC_EE_EXECUTOR_TASK_REDUCE: case UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED: status = ucc_ec_rocm_reduce((ucc_ee_executor_task_args_t *)task_args, stream); if (ucc_unlikely(status != UCC_OK)) { ec_error(&ucc_ec_rocm.super, "failed to start reduce op"); goto free_task; } break; default: ec_error(&ucc_ec_rocm.super, "executor operation is not supported task_type %d", task_args->task_type); status = UCC_ERR_INVALID_PARAM; goto free_task; } status = ucc_ec_rocm_event_post(stream, ee_task->event); if (ucc_unlikely(status != UCC_OK)) { goto free_task; } *task = &ee_task->super; return UCC_OK; free_task: ucc_ec_rocm_event_destroy(ee_task->event); ucc_mpool_put(ee_task); return status; } ucc_status_t ucc_rocm_executor_interruptible_task_test(const ucc_ee_executor_task_t *task) { ucc_ec_rocm_executor_interruptible_task_t *ee_task = ucc_derived_of(task, ucc_ec_rocm_executor_interruptible_task_t); ee_task->super.status = ucc_ec_rocm_event_test(ee_task->event); return ee_task->super.status; } ucc_status_t ucc_rocm_executor_interruptible_task_finalize(ucc_ee_executor_task_t *task) { ucc_ec_rocm_executor_interruptible_task_t *ee_task = ucc_derived_of(task, ucc_ec_rocm_executor_interruptible_task_t); ucc_status_t status; status = ucc_ec_rocm_event_destroy(ee_task->event); ucc_mpool_put(task); return status; } ucc_status_t ucc_rocm_executor_interruptible_start(ucc_ee_executor_t *executor) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(executor, ucc_ec_rocm_executor_t); eee->mode = UCC_EC_ROCM_EXECUTOR_MODE_INTERRUPTIBLE; eee->state = UCC_EC_ROCM_EXECUTOR_STARTED; eee->ops.task_post = ucc_rocm_executor_interruptible_task_post; eee->ops.task_test = ucc_rocm_executor_interruptible_task_test; eee->ops.task_finalize = ucc_rocm_executor_interruptible_task_finalize; return UCC_OK; } ucc_status_t ucc_rocm_executor_interruptible_stop(ucc_ee_executor_t *executor) { ucc_ec_rocm_executor_t *eee = ucc_derived_of(executor, ucc_ec_rocm_executor_t); eee->state = UCC_EC_ROCM_EXECUTOR_INITIALIZED; return UCC_OK; } ucc-1.8.0/src/components/ec/ucc_ec.c0000664000175000017500000001516415211535620017415 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include #include "config.h" #include "base/ucc_ec_base.h" #include "ucc_ec.h" #include "core/ucc_global_opts.h" #include "utils/ucc_malloc.h" #include "utils/ucc_log.h" static const ucc_ec_ops_t *ec_ops[UCC_EE_LAST]; static const ucc_ee_executor_ops_t *executor_ops[UCC_EE_LAST]; static pthread_mutex_t ucc_ec_mutex = PTHREAD_MUTEX_INITIALIZER; #define UCC_CHECK_EC_AVAILABLE(ee) \ do { \ if (NULL == ec_ops[ee]) { \ return UCC_ERR_NOT_SUPPORTED; \ } \ } while (0) ucc_status_t ucc_ec_init(const ucc_ec_params_t *ec_params) { int i, n_ecs; ucc_ec_base_t *ec; ucc_status_t status; ucc_ec_attr_t attr; pthread_mutex_lock(&ucc_ec_mutex); memset(ec_ops, 0, UCC_EE_LAST * sizeof(ucc_ec_ops_t *)); n_ecs = ucc_global_config.ec_framework.n_components; for (i = 0; i < n_ecs; i++) { ec = ucc_derived_of(ucc_global_config.ec_framework.components[i], ucc_ec_base_t); if (ec->ref_cnt == 0) { ec->config = ucc_malloc(ec->config_table.size); if (!ec->config) { ucc_error("failed to allocate %zd bytes for ec config", ec->config_table.size); continue; } status = ucc_config_parser_fill_opts( ec->config, &ec->config_table, "UCC_", 1); if (UCC_OK != status) { ucc_debug("failed to parse config for EC component: %s (%d)", ec->super.name, status); ucc_free(ec->config); continue; } status = ec->init(ec_params); if (UCC_OK != status) { ucc_debug("ec_init failed for component: %s, skipping (%d)", ec->super.name, status); ucc_config_parser_release_opts(ec->config, ec->config_table.table); ucc_free(ec->config); continue; } ucc_debug("ec %s initialized", ec->super.name); } else { attr.field_mask = UCC_EC_ATTR_FIELD_THREAD_MODE; status = ec->get_attr(&attr); if (status != UCC_OK) { pthread_mutex_unlock(&ucc_ec_mutex); return status; } if (attr.thread_mode < ec_params->thread_mode) { ucc_info("ec %s was allready initilized with " "different thread mode: current tm %d, provided tm %d", ec->super.name, attr.thread_mode, ec_params->thread_mode); } } ec->ref_cnt++; ec_ops[ec->type] = &ec->ops; executor_ops[ec->type] = &ec->executor_ops; } pthread_mutex_unlock(&ucc_ec_mutex); return UCC_OK; } ucc_status_t ucc_ec_available(ucc_ee_type_t ee_type) { if (NULL == ec_ops[ee_type]) { return UCC_ERR_NOT_FOUND; } return UCC_OK; } ucc_status_t ucc_ec_get_attr(ucc_ec_attr_t *attr) { if (attr->field_mask & UCC_EC_ATTR_FILED_MAX_EXECUTORS_BUFS) { attr->max_ee_bufs = UCC_EE_EXECUTOR_NUM_BUFS; } return UCC_OK; } ucc_status_t ucc_ec_finalize() { ucc_ee_type_t et; ucc_ec_base_t *ec; pthread_mutex_lock(&ucc_ec_mutex); for (et = UCC_EE_FIRST; et < UCC_EE_LAST; et++) { if (NULL != ec_ops[et]) { ec = ucc_container_of(ec_ops[et], ucc_ec_base_t, ops); ec->ref_cnt--; if (ec->ref_cnt == 0) { ec->finalize(); ucc_config_parser_release_opts(ec->config, ec->config_table.table); ucc_free(ec->config); ec_ops[et] = NULL; } } } pthread_mutex_unlock(&ucc_ec_mutex); return UCC_OK; } ucc_status_t ucc_ec_create_event(void **event, ucc_ee_type_t ee_type) { UCC_CHECK_EC_AVAILABLE(ee_type); return ec_ops[ee_type]->create_event(event); } ucc_status_t ucc_ec_destroy_event(void *event, ucc_ee_type_t ee_type) { UCC_CHECK_EC_AVAILABLE(ee_type); return ec_ops[ee_type]->destroy_event(event); } ucc_status_t ucc_ec_event_post(void *ee_context, void *event, ucc_ee_type_t ee_type) { UCC_CHECK_EC_AVAILABLE(ee_type); return ec_ops[ee_type]->event_post(ee_context, event); } ucc_status_t ucc_ec_event_test(void *event, ucc_ee_type_t ee_type) { UCC_CHECK_EC_AVAILABLE(ee_type); return ec_ops[ee_type]->event_test(event); } ucc_status_t ucc_ee_executor_init(const ucc_ee_executor_params_t *params, ucc_ee_executor_t **executor) { UCC_CHECK_EC_AVAILABLE(params->ee_type); return executor_ops[params->ee_type]->init(params, executor); } ucc_status_t ucc_ee_executor_status(const ucc_ee_executor_t *executor) { UCC_CHECK_EC_AVAILABLE(executor->ee_type); return executor_ops[executor->ee_type]->status(executor); } ucc_status_t ucc_ee_executor_start(ucc_ee_executor_t *executor, void *ee_context) { UCC_CHECK_EC_AVAILABLE(executor->ee_type); return executor_ops[executor->ee_type]->start(executor, ee_context); } ucc_status_t ucc_ee_executor_stop(ucc_ee_executor_t *executor) { UCC_CHECK_EC_AVAILABLE(executor->ee_type); return executor_ops[executor->ee_type]->stop(executor); } ucc_status_t ucc_ee_executor_finalize(ucc_ee_executor_t *executor) { UCC_CHECK_EC_AVAILABLE(executor->ee_type); return executor_ops[executor->ee_type]->finalize(executor); } ucc_status_t ucc_ee_executor_task_post(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task) { UCC_CHECK_EC_AVAILABLE(executor->ee_type); return executor_ops[executor->ee_type]->task_post(executor, task_args, task); } ucc_status_t ucc_ee_executor_task_test(const ucc_ee_executor_task_t *task) { UCC_CHECK_EC_AVAILABLE(task->eee->ee_type); return executor_ops[task->eee->ee_type]->task_test(task); } ucc_status_t ucc_ee_executor_task_finalize(ucc_ee_executor_task_t *task) { UCC_CHECK_EC_AVAILABLE(task->eee->ee_type); return executor_ops[task->eee->ee_type]->task_finalize(task); } ucc-1.8.0/src/components/ec/cpu/0000775000175000017500000000000015211535620016610 5ustar alastairalastairucc-1.8.0/src/components/ec/cpu/ec_cpu.c0000664000175000017500000001622015211535620020213 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ec_cpu.h" #include "utils/arch/cpu.h" #include "components/mc/ucc_mc.h" #include static ucc_config_field_t ucc_ec_cpu_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_ec_cpu_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_ec_config_table)}, {NULL} }; static ucc_status_t ucc_ec_cpu_init(const ucc_ec_params_t *ec_params) { ucc_status_t status; ucc_strncpy_safe(ucc_ec_cpu.super.config->log_component.name, ucc_ec_cpu.super.super.name, sizeof(ucc_ec_cpu.super.config->log_component.name)); ucc_ec_cpu.thread_mode = ec_params->thread_mode; status = ucc_mpool_init(&ucc_ec_cpu.executors, 0, sizeof(ucc_ee_executor_t), 0, UCC_CACHE_LINE_SIZE, 16, UINT_MAX, NULL, ec_params->thread_mode, "ec cpu executors"); if (status != UCC_OK) { ec_error(&ucc_ec_cpu.super, "failed to created ec cpu executors mpool"); return status; } status = ucc_mpool_init(&ucc_ec_cpu.executor_tasks, 0, sizeof(ucc_ee_executor_task_t), 0, UCC_CACHE_LINE_SIZE, 16, UINT_MAX, NULL, ec_params->thread_mode, "ec cpu executor tasks"); if (status != UCC_OK) { ec_error(&ucc_ec_cpu.super, "failed to created ec cpu executor tasks mpool"); ucc_mpool_cleanup(&ucc_ec_cpu.executors, 1); return status; } return UCC_OK; } static ucc_status_t ucc_ec_cpu_get_attr(ucc_ec_attr_t *ec_attr) { if (ec_attr->field_mask & UCC_EC_ATTR_FIELD_THREAD_MODE) { ec_attr->thread_mode = ucc_ec_cpu.thread_mode; } return UCC_OK; } static ucc_status_t ucc_ec_cpu_finalize() { ucc_mpool_cleanup(&ucc_ec_cpu.executors, 1); ucc_mpool_cleanup(&ucc_ec_cpu.executor_tasks, 1); return UCC_OK; } ucc_status_t ucc_cpu_executor_init(const ucc_ee_executor_params_t *params, ucc_ee_executor_t **executor) { ucc_ee_executor_t *eee = ucc_mpool_get(&ucc_ec_cpu.executors); ec_trace(&ucc_ec_cpu.super, "executor init, eee: %p", eee); if (ucc_unlikely(!eee)) { ec_error(&ucc_ec_cpu.super, "failed to allocate executor"); return UCC_ERR_NO_MEMORY; } eee->ee_type = params->ee_type; *executor = eee; return UCC_OK; } ucc_status_t ucc_cpu_executor_start(ucc_ee_executor_t *executor, //NOLINT void *ee_context) //NOLINT { return UCC_OK; } ucc_status_t ucc_cpu_executor_status(const ucc_ee_executor_t *executor) //NOLINT { return UCC_OK; } ucc_status_t ucc_cpu_executor_stop(ucc_ee_executor_t *executor) //NOLINT { return UCC_OK; } ucc_status_t ucc_cpu_executor_task_post(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task) { ucc_status_t status = UCC_OK; ucc_ee_executor_task_t *eee_task; eee_task = ucc_mpool_get(&ucc_ec_cpu.executor_tasks); if (ucc_unlikely(!eee_task)) { return UCC_ERR_NO_MEMORY; } eee_task->eee = executor; switch (task_args->task_type) { case UCC_EE_EXECUTOR_TASK_REDUCE: status = ucc_ec_cpu_reduce((ucc_eee_task_reduce_t *)&task_args->reduce, task_args->reduce.dst, (task_args->flags & UCC_EEE_TASK_FLAG_REDUCE_SRCS_EXT) ? task_args->reduce.srcs_ext : task_args->reduce.srcs, task_args->flags); if (ucc_unlikely(UCC_OK != status)) { goto free_task; } break; case UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED: { ucc_eee_task_reduce_strided_t *trs = (ucc_eee_task_reduce_strided_t *)&task_args->reduce_strided; size_t n_srcs = trs->n_src2 + 1; uint16_t flags = task_args->flags; void ** srcs; ucc_eee_task_reduce_t tr; int i; if (n_srcs <= UCC_EE_EXECUTOR_NUM_BUFS) { srcs = &tr.srcs[0]; } else { srcs = alloca(n_srcs * sizeof(void *)); flags |= UCC_EEE_TASK_FLAG_REDUCE_SRCS_EXT; tr.srcs_ext = srcs; } srcs[0] = trs->src1; for (i = 0; i < n_srcs - 1; i++) { srcs[i + 1] = PTR_OFFSET(trs->src2, trs->stride * i); } tr.count = trs->count; tr.dt = trs->dt; tr.op = trs->op; tr.n_srcs = n_srcs; tr.dst = trs->dst; tr.alpha = trs->alpha; status = ucc_ec_cpu_reduce(&tr, tr.dst, srcs, flags); if (ucc_unlikely(UCC_OK != status)) { goto free_task; } } break; case UCC_EE_EXECUTOR_TASK_COPY: memcpy(task_args->copy.dst, task_args->copy.src, task_args->copy.len); break; case UCC_EE_EXECUTOR_TASK_COPY_MULTI: default: status = UCC_ERR_NOT_SUPPORTED; goto free_task; } eee_task->status = status; *task = eee_task; return status; free_task: ucc_mpool_put(eee_task); return status; } ucc_status_t ucc_cpu_executor_task_test(const ucc_ee_executor_task_t *task) { return task->status; } ucc_status_t ucc_cpu_executor_task_finalize(ucc_ee_executor_task_t *task) { ucc_mpool_put(task); return UCC_OK; } ucc_status_t ucc_cpu_executor_finalize(ucc_ee_executor_t *executor) { ec_trace(&ucc_ec_cpu.super, "executor finalize, eee: %p", executor); ucc_mpool_put(executor); return UCC_OK; } ucc_ec_cpu_t ucc_ec_cpu = { .super.super.name = "cpu ec", .super.ref_cnt = 0, .super.type = UCC_EE_CPU_THREAD, .super.init = ucc_ec_cpu_init, .super.get_attr = ucc_ec_cpu_get_attr, .super.finalize = ucc_ec_cpu_finalize, .super.config_table = { .name = "CPU execution component", .prefix = "EC_CPU_", .table = ucc_ec_cpu_config_table, .size = sizeof(ucc_ec_cpu_config_t), }, .super.ops.create_event = NULL, .super.ops.destroy_event = NULL, .super.ops.event_post = NULL, .super.ops.event_test = NULL, .super.executor_ops.init = ucc_cpu_executor_init, .super.executor_ops.start = ucc_cpu_executor_start, .super.executor_ops.status = ucc_cpu_executor_status, .super.executor_ops.stop = ucc_cpu_executor_stop, .super.executor_ops.task_post = ucc_cpu_executor_task_post, .super.executor_ops.task_test = ucc_cpu_executor_task_test, .super.executor_ops.task_finalize = ucc_cpu_executor_task_finalize, .super.executor_ops.finalize = ucc_cpu_executor_finalize, }; UCC_CONFIG_REGISTER_TABLE_ENTRY(&ucc_ec_cpu.super.config_table, &ucc_config_global_list); ucc-1.8.0/src/components/ec/cpu/ec_cpu_reduce.c0000664000175000017500000004661115211535620021551 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "utils/ucc_math_op.h" #include "ec_cpu.h" #include #define DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, OP) \ do { \ size_t _i, _j; \ type _tmp; \ size_t __count = _count; \ switch (_n_srcs) { \ case 2: \ for (_i = 0; _i < __count; _i++) { \ d[_i] = OP##_2(s[0][_i], s[1][_i]); \ } \ break; \ case 3: \ for (_i = 0; _i < __count; _i++) { \ d[_i] = OP##_3(s[0][_i], s[1][_i], s[2][_i]); \ } \ break; \ case 4: \ for (_i = 0; _i < __count; _i++) { \ d[_i] = OP##_4(s[0][_i], s[1][_i], s[2][_i], s[3][_i]); \ } \ break; \ case 5: \ for (_i = 0; _i < __count; _i++) { \ d[_i] = \ OP##_5(s[0][_i], s[1][_i], s[2][_i], s[3][_i], s[4][_i]); \ } \ break; \ case 6: \ for (_i = 0; _i < __count; _i++) { \ d[_i] = OP##_6(s[0][_i], s[1][_i], s[2][_i], s[3][_i], \ s[4][_i], s[5][_i]); \ } \ break; \ case 7: \ for (_i = 0; _i < __count; _i++) { \ d[_i] = OP##_7(s[0][_i], s[1][_i], s[2][_i], s[3][_i], \ s[4][_i], s[5][_i], s[6][_i]); \ } \ break; \ case 8: \ for (_i = 0; _i < __count; _i++) { \ d[_i] = OP##_8(s[0][_i], s[1][_i], s[2][_i], s[3][_i], \ s[4][_i], s[5][_i], s[6][_i], s[7][_i]); \ } \ break; \ default: \ for (_i = 0; _i < __count; _i++) { \ _tmp = OP##_8(s[0][_i], s[1][_i], s[2][_i], s[3][_i], \ s[4][_i], s[5][_i], s[6][_i], s[7][_i]); \ for (_j = 8; _j < _n_srcs; _j++) { \ _tmp = OP##_2(_tmp, s[_j][_i]); \ } \ d[_i] = _tmp; \ } \ break; \ } \ } while (0) #define VEC_OP(_d, _count, _alpha) \ do { \ size_t _i; \ for (_i = 0; _i < _count; _i++) { \ _d[_i] = _d[_i] * _alpha; \ } \ } while (0) #define DO_DT_REDUCE_INT(type, _srcs, _dst, _op, _count, _n_srcs) \ do { \ const type **restrict s = (const type **)_srcs; \ type *restrict d = (type * ) _dst; \ switch (_op) { \ case UCC_OP_AVG: \ case UCC_OP_SUM: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_SUM); \ if (flags & UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA) { \ VEC_OP(d, _count, task->alpha); \ } \ break; \ case UCC_OP_MIN: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_MIN); \ break; \ case UCC_OP_MAX: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_MAX); \ break; \ case UCC_OP_PROD: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_PROD); \ break; \ case UCC_OP_LAND: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_LAND); \ break; \ case UCC_OP_BAND: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_BAND); \ break; \ case UCC_OP_LOR: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_LOR); \ break; \ case UCC_OP_BOR: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_BOR); \ break; \ case UCC_OP_LXOR: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_LXOR); \ break; \ case UCC_OP_BXOR: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_BXOR); \ break; \ default: \ ec_error(&ucc_ec_cpu.super, \ "int dtype does not support " \ "requested reduce op: %s", \ ucc_reduction_op_str(_op)); \ return UCC_ERR_NOT_SUPPORTED; \ } \ } while (0) #define DO_DT_REDUCE_WITH_OP_BFLOAT16(_srcs, _dst, _count, _n_srcs, _OP, \ _alpha) \ do { \ float _tmp; \ size_t _i, _j; \ int16_t **_s = (int16_t **)_srcs; \ int16_t * _d = (int16_t *)_dst; \ for (_i = 0; _i < _count; _i++) { \ _tmp = _OP(bfloat16tofloat32(&_s[0][_i]), \ bfloat16tofloat32(&_s[1][_i])); \ for (_j = 2; _j < _n_srcs; _j++) { \ _tmp = _OP(_tmp, bfloat16tofloat32(&_s[_j][_i])); \ } \ float32tobfloat16(_tmp *_alpha, &_d[_i]); \ } \ } while (0) #define DO_DT_REDUCE_BFLOAT16(_srcs, _dst, _op, _count, _n_srcs) \ do { \ float _a = (flags & UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA) ? task->alpha \ : 1.0f; \ switch (_op) { \ case UCC_OP_AVG: \ case UCC_OP_SUM: \ DO_DT_REDUCE_WITH_OP_BFLOAT16(_srcs, _dst, _count, _n_srcs, \ DO_OP_SUM, _a); \ break; \ case UCC_OP_PROD: \ DO_DT_REDUCE_WITH_OP_BFLOAT16(_srcs, _dst, _count, _n_srcs, \ DO_OP_PROD, _a); \ break; \ case UCC_OP_MIN: \ DO_DT_REDUCE_WITH_OP_BFLOAT16(_srcs, _dst, _count, _n_srcs, \ DO_OP_MIN, _a); \ break; \ case UCC_OP_MAX: \ DO_DT_REDUCE_WITH_OP_BFLOAT16(_srcs, _dst, _count, _n_srcs, \ DO_OP_MAX, _a); \ break; \ default: \ ec_error(&ucc_ec_cpu.super, \ "bfloat16 dtype does not support " \ "requested reduce op: %s", \ ucc_reduction_op_str(_op)); \ return UCC_ERR_NOT_SUPPORTED; \ } \ } while (0) #define DO_DT_REDUCE_FLOAT(type, _srcs, _dst, _op, _count, _n_srcs) \ do { \ const type **restrict s = (const type **)_srcs; \ type *restrict d = (type *) _dst; \ switch (_op) { \ case UCC_OP_AVG: \ case UCC_OP_SUM: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_SUM); \ break; \ case UCC_OP_PROD: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_PROD); \ break; \ case UCC_OP_MIN: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_MIN); \ break; \ case UCC_OP_MAX: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_MAX); \ break; \ default: \ ec_error(&ucc_ec_cpu.super, \ "float dtype does not support " \ "requested reduce op: %s", \ ucc_reduction_op_str(_op)); \ return UCC_ERR_NOT_SUPPORTED; \ } \ if (flags & UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA) { \ VEC_OP(d, _count, task->alpha); \ } \ } while (0) #define DO_DT_REDUCE_FLOAT_COMPLEX(type, _srcs, _dst, _op, _count, _n_srcs) \ do { \ const type **restrict s = (const type **)_srcs; \ type *restrict d = (type *) _dst; \ switch (_op) { \ case UCC_OP_AVG: \ case UCC_OP_SUM: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_SUM); \ break; \ case UCC_OP_PROD: \ DO_DT_REDUCE_WITH_OP(type, s, d, _count, _n_srcs, DO_OP_PROD); \ break; \ default: \ ec_error(&ucc_ec_cpu.super, \ "float complex dtype does not support " \ "requested reduce op: %s", \ ucc_reduction_op_str(_op)); \ return UCC_ERR_NOT_SUPPORTED; \ } \ if (flags & UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA) { \ VEC_OP(d, _count, task->alpha); \ } \ } while (0) ucc_status_t ucc_ec_cpu_reduce(ucc_eee_task_reduce_t *task, void * restrict dst, void * const * restrict srcs, uint16_t flags) { switch (task->dt) { case UCC_DT_INT8: DO_DT_REDUCE_INT(int8_t, srcs, dst, task->op, task->count, task->n_srcs); break; case UCC_DT_INT16: DO_DT_REDUCE_INT(int16_t, srcs, dst, task->op, task->count, task->n_srcs); break; case UCC_DT_INT32: DO_DT_REDUCE_INT(int32_t, srcs, dst, task->op, task->count, task->n_srcs); break; case UCC_DT_INT64: DO_DT_REDUCE_INT(int64_t, srcs, dst, task->op, task->count, task->n_srcs); break; case UCC_DT_UINT8: DO_DT_REDUCE_INT(uint8_t, srcs, dst, task->op, task->count, task->n_srcs); break; case UCC_DT_UINT16: DO_DT_REDUCE_INT(uint16_t, srcs, dst, task->op, task->count, task->n_srcs); break; case UCC_DT_UINT32: DO_DT_REDUCE_INT(uint32_t, srcs, dst, task->op, task->count, task->n_srcs); break; case UCC_DT_UINT64: DO_DT_REDUCE_INT(uint64_t, srcs, dst, task->op, task->count, task->n_srcs); break; case UCC_DT_FLOAT32: #if SIZEOF_FLOAT == 4 DO_DT_REDUCE_FLOAT(float, srcs, dst, task->op, task->count, task->n_srcs); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT64: #if SIZEOF_DOUBLE == 8 DO_DT_REDUCE_FLOAT(double, srcs, dst, task->op, task->count, task->n_srcs); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT128: #if SIZEOF_LONG_DOUBLE == 16 DO_DT_REDUCE_FLOAT(long double, srcs, dst, task->op, task->count, task->n_srcs); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_BFLOAT16: DO_DT_REDUCE_BFLOAT16(srcs, dst, task->op, task->count, task->n_srcs); break; case UCC_DT_FLOAT32_COMPLEX: #if SIZEOF_FLOAT__COMPLEX == 8 DO_DT_REDUCE_FLOAT_COMPLEX(float complex, srcs, dst, task->op, task->count, task->n_srcs); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT64_COMPLEX: #if SIZEOF_DOUBLE__COMPLEX == 16 DO_DT_REDUCE_FLOAT_COMPLEX(double complex, srcs, dst, task->op, task->count, task->n_srcs); break; #else return UCC_ERR_NOT_SUPPORTED; #endif case UCC_DT_FLOAT128_COMPLEX: #if SIZEOF_LONG_DOUBLE__COMPLEX == 32 DO_DT_REDUCE_FLOAT_COMPLEX(long double complex, srcs, dst, task->op, task->count, task->n_srcs); break; #else return UCC_ERR_NOT_SUPPORTED; #endif default: ec_error(&ucc_ec_cpu.super, "unsupported reduction type (%s)", ucc_datatype_str(task->dt)); return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } ucc-1.8.0/src/components/ec/cpu/ec_cpu.h0000664000175000017500000000135415211535620020222 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_EC_CPU_H_ #define UCC_EC_CPU_H_ #include "components/ec/base/ucc_ec_base.h" #include "components/ec/ucc_ec_log.h" #include "utils/ucc_mpool.h" typedef struct ucc_ec_cpu_config { ucc_ec_config_t super; } ucc_ec_cpu_config_t; typedef struct ucc_ec_cpu { ucc_ec_base_t super; ucc_thread_mode_t thread_mode; ucc_mpool_t executors; ucc_mpool_t executor_tasks; ucc_spinlock_t init_spinlock; } ucc_ec_cpu_t; extern ucc_ec_cpu_t ucc_ec_cpu; ucc_status_t ucc_ec_cpu_reduce(ucc_eee_task_reduce_t *task, void * restrict dst, void * const * restrict srcs, uint16_t flags); #endif ucc-1.8.0/src/components/ec/cpu/Makefile.am0000664000175000017500000000075615211535620020654 0ustar alastairalastair# # Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # sources = \ ec_cpu.h \ ec_cpu.c \ ec_cpu_reduce.c module_LTLIBRARIES = libucc_ec_cpu.la libucc_ec_cpu_la_SOURCES = $(sources) libucc_ec_cpu_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) libucc_ec_cpu_la_CFLAGS = $(BASE_CFLAGS) libucc_ec_cpu_la_LDFLAGS = -version-info $(SOVERSION) --as-needed libucc_ec_cpu_la_LIBADD = $(UCC_TOP_BUILDDIR)/src/libucc.la include $(top_srcdir)/config/module.am ucc-1.8.0/src/components/ec/ucc_ec.h0000664000175000017500000000310615211535620017413 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_EC_H_ #define UCC_EC_H_ #include "ucc/api/ucc.h" #include "components/ec/base/ucc_ec_base.h" ucc_status_t ucc_ec_init(const ucc_ec_params_t *ec_params); ucc_status_t ucc_ec_available(ucc_ee_type_t ee_type); ucc_status_t ucc_ec_get_attr(ucc_ec_attr_t *attr); ucc_status_t ucc_ec_finalize(); ucc_status_t ucc_ec_create_event(void **event, ucc_ee_type_t ee_type); ucc_status_t ucc_ec_destroy_event(void *event, ucc_ee_type_t ee_type); ucc_status_t ucc_ec_event_post(void *ee_context, void *event, ucc_ee_type_t ee_type); ucc_status_t ucc_ec_event_test(void *event, ucc_ee_type_t ee_type); ucc_status_t ucc_ee_executor_init(const ucc_ee_executor_params_t *params, ucc_ee_executor_t **executor); ucc_status_t ucc_ee_executor_status(const ucc_ee_executor_t *executor); ucc_status_t ucc_ee_executor_start(ucc_ee_executor_t *executor, void *ee_context); ucc_status_t ucc_ee_executor_stop(ucc_ee_executor_t *executor); ucc_status_t ucc_ee_executor_finalize(ucc_ee_executor_t *executor); ucc_status_t ucc_ee_executor_task_post(ucc_ee_executor_t *executor, const ucc_ee_executor_task_args_t *task_args, ucc_ee_executor_task_t **task); ucc_status_t ucc_ee_executor_task_test(const ucc_ee_executor_task_t *task); ucc_status_t ucc_ee_executor_task_finalize(ucc_ee_executor_task_t *task); #endif ucc-1.8.0/src/components/ec/ucc_ec_log.h0000664000175000017500000000214715211535620020260 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_EC_LOG_H_ #define UCC_EC_LOG_H_ #include "utils/ucc_log.h" #define ucc_log_component_ec(_ec, _level, fmt, ...) \ ucc_log_component(_level, &((_ec)->config)->log_component, \ fmt, ##__VA_ARGS__) #define ec_error(_ec, _fmt, ...) \ ucc_log_component_ec(_ec, UCC_LOG_LEVEL_ERROR, _fmt, ##__VA_ARGS__) #define ec_warn(_ec, _fmt, ...) \ ucc_log_component_ec(_ec, UCC_LOG_LEVEL_WARN, _fmt, ##__VA_ARGS__) #define ec_info(_ec, _fmt, ...) \ ucc_log_component_ec(_ec, UCC_LOG_LEVEL_INFO, _fmt, ##__VA_ARGS__) #define ec_debug(_ec, _fmt, ...) \ ucc_log_component_ec(_ec, UCC_LOG_LEVEL_DEBUG, _fmt, ##__VA_ARGS__) #define ec_trace(_ec, _fmt, ...) \ ucc_log_component_ec(_ec, UCC_LOG_LEVEL_TRACE, _fmt, ##__VA_ARGS__) #endif ucc-1.8.0/src/schedule/0000775000175000017500000000000015211535620015041 5ustar alastairalastairucc-1.8.0/src/schedule/ucc_schedule_pipelined.c0000664000175000017500000002663215211535620021675 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_schedule.h" #include "ucc_schedule_pipelined.h" #include "coll_score/ucc_coll_score.h" #include "core/ucc_context.h" const char* ucc_pipeline_order_names[] = { [UCC_PIPELINE_PARALLEL] = "parallel", [UCC_PIPELINE_ORDERED] = "ordered", [UCC_PIPELINE_SEQUENTIAL] = "sequential", [UCC_PIPELINE_LAST] = NULL }; static ucc_status_t ucc_frag_start_handler(ucc_coll_task_t *parent, ucc_coll_task_t *task) { ucc_schedule_pipelined_t *schedule = ucc_derived_of(parent, ucc_schedule_pipelined_t); ucc_schedule_t *frag = ucc_derived_of(task, ucc_schedule_t); ucc_status_t st; task->start_time = parent->start_time; if (schedule->frag_setup) { st = schedule->frag_setup(schedule, frag, schedule->n_frags_started); if (ucc_unlikely(UCC_OK != st)) { ucc_error("failed to setup fragment %d of pipelined schedule", schedule->n_frags_started); return st; } } schedule->next_frag_to_post = (schedule->next_frag_to_post + 1) % schedule->n_frags; ucc_trace_req("sched %p started frag %p frag_num %d next_to_post %d", schedule, frag, schedule->n_frags_started, schedule->next_frag_to_post); schedule->n_frags_started++; schedule->n_frags_in_pipeline++; st = task->post(task); if (ucc_unlikely(st < 0)) { ucc_error("failed to post fragment %d of pipelined schedule", schedule->n_frags_started - 1); } return st; } static ucc_status_t ucc_schedule_pipelined_completed_handler(ucc_coll_task_t *parent_task, ucc_coll_task_t *task) { ucc_schedule_pipelined_t *schedule = ucc_container_of(task, ucc_schedule_pipelined_t, super); ucc_schedule_t *frag = ucc_derived_of(parent_task, ucc_schedule_t); int i; if (UCC_TASK_THREAD_MODE(task) == UCC_THREAD_MULTIPLE) { ucc_recursive_spin_lock(&schedule->lock); } schedule->super.n_completed_tasks += 1; schedule->n_frags_in_pipeline--; /* Mark this fragment as needing restart. Using this flag instead of checking frag->super.status avoids a race condition where another handler could see status==OK before ucc_task_complete reads it. */ frag->super.flags |= UCC_COLL_TASK_FLAG_RESTART_PENDING; ucc_trace_req( "sched %p completed frag %p, n_completed %d, n_started %d, n_total %d", schedule, frag, schedule->super.n_completed_tasks, schedule->n_frags_started, schedule->super.n_tasks); if (schedule->super.n_completed_tasks == schedule->super.n_tasks) { schedule->super.super.status = UCC_OK; if (UCC_TASK_THREAD_MODE(task) == UCC_THREAD_MULTIPLE) { ucc_recursive_spin_unlock(&schedule->lock); } ucc_task_complete(task); return UCC_OK; } /* Restart fragments in order, starting from next_frag_to_post. Only restart fragments that have the restart_pending flag set. */ while (schedule->super.n_completed_tasks + schedule->n_frags_in_pipeline < schedule->super.n_tasks) { ucc_status_t st; frag = schedule->frags[schedule->next_frag_to_post]; if (!(frag->super.flags & UCC_COLL_TASK_FLAG_RESTART_PENDING)) { /* The next fragment hasn't completed yet, stop here. It will trigger its own handler when it completes. */ break; } ucc_trace_req("sched %p restarting frag %d %p", schedule, schedule->next_frag_to_post, frag); frag->super.flags &= ~UCC_COLL_TASK_FLAG_RESTART_PENDING; frag->super.status = UCC_OPERATION_INITIALIZED; frag->n_completed_tasks = 0; for (i = 0; i < frag->n_tasks; i++) { frag->tasks[i]->n_deps += frag->tasks[i]->n_deps_base; frag->tasks[i]->status = UCC_OPERATION_INITIALIZED; } st = ucc_frag_start_handler(&schedule->super.super, &frag->super); if (ucc_unlikely(st < 0)) { schedule->super.super.status = st; if (UCC_TASK_THREAD_MODE(task) == UCC_THREAD_MULTIPLE) { ucc_recursive_spin_unlock(&schedule->lock); } ucc_task_complete(task); return st; } } if (UCC_TASK_THREAD_MODE(task) == UCC_THREAD_MULTIPLE) { ucc_recursive_spin_unlock(&schedule->lock); } return UCC_OK; } ucc_status_t ucc_schedule_pipelined_finalize(ucc_coll_task_t *task) { ucc_schedule_pipelined_t *schedule_p = ucc_derived_of(task, ucc_schedule_pipelined_t); ucc_schedule_t **frags = schedule_p->frags; int i; ucc_trace_req("schedule pipelined %p is complete", schedule_p); for (i = 0; i < schedule_p->n_frags; i++) { schedule_p->frags[i]->super.finalize(&frags[i]->super); } if (UCC_TASK_THREAD_MODE(task) == UCC_THREAD_MULTIPLE) { ucc_recursive_spinlock_destroy(&schedule_p->lock); } return UCC_OK; } ucc_status_t ucc_schedule_pipelined_post(ucc_coll_task_t *task) { ucc_schedule_pipelined_t *schedule_p = ucc_derived_of(task, ucc_schedule_pipelined_t); ucc_schedule_t **frags = schedule_p->frags; int i, j; schedule_p->super.super.super.status = UCC_OPERATION_INITIALIZED; schedule_p->super.n_completed_tasks = 0; schedule_p->n_frags_started = 0; schedule_p->next_frag_to_post = 0; schedule_p->n_frags_in_pipeline = 0; for (i = 0; i < schedule_p->n_frags; i++) { frags[i]->n_completed_tasks = 0; frags[i]->super.flags &= ~UCC_COLL_TASK_FLAG_RESTART_PENDING; frags[i]->super.super.status = UCC_OPERATION_INITIALIZED; for (j = 0; j < frags[i]->n_tasks; j++) { frags[i]->tasks[j]->n_deps = frags[i]->tasks[j]->n_deps_base; frags[i]->tasks[j]->n_deps_satisfied = 0; frags[i]->tasks[j]->super.status = UCC_OPERATION_INITIALIZED; if (i == 0 && schedule_p->n_frags > 1 && UCC_PIPELINE_PARALLEL != schedule_p->order) { frags[0]->tasks[j]->n_deps_satisfied++; } } } return ucc_schedule_start(task); } ucc_status_t ucc_schedule_pipelined_init(ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_schedule_frag_init_fn_t frag_init, ucc_schedule_frag_setup_fn_t frag_setup, int n_frags, int n_frags_total, ucc_pipeline_order_t order, ucc_schedule_pipelined_t *schedule) { ucc_event_t task_dependency_event = UCC_EVENT_LAST; int i, j; ucc_status_t status; ucc_schedule_t **frags; if (ucc_unlikely(n_frags > UCC_SCHEDULE_PIPELINED_MAX_FRAGS)) { ucc_error("n_frags %d exceeds max limit of %d", n_frags, UCC_SCHEDULE_PIPELINED_MAX_FRAGS); return UCC_ERR_INVALID_PARAM; } if (n_frags > 1) { /* determine dependency between frags */ switch (order) { case UCC_PIPELINE_PARALLEL: /* no dependency between tasks of different fragments */ task_dependency_event = UCC_EVENT_LAST; break; case UCC_PIPELINE_ORDERED: /* next fragment starts when previous has started */ task_dependency_event = UCC_EVENT_TASK_STARTED; break; case UCC_PIPELINE_SEQUENTIAL: /* next fragment starts when previous has completed */ task_dependency_event = UCC_EVENT_COMPLETED; break; default: return UCC_ERR_INVALID_PARAM; } } status = ucc_schedule_init(&schedule->super, coll_args, team); if (ucc_unlikely(status != UCC_OK)) { ucc_error("failed to init pipelined schedule"); return status; } if (UCC_TASK_THREAD_MODE(&schedule->super.super) == UCC_THREAD_MULTIPLE) { ucc_recursive_spinlock_init(&schedule->lock, 0); } schedule->super.super.flags |= UCC_COLL_TASK_FLAG_IS_PIPELINED_SCHEDULE; schedule->super.n_tasks = n_frags_total; schedule->n_frags = n_frags; schedule->order = order; schedule->frag_setup = frag_setup; schedule->next_frag_to_post = 0; schedule->n_frags_in_pipeline = 0; schedule->super.super.finalize = ucc_schedule_pipelined_finalize; schedule->super.super.post = ucc_schedule_pipelined_post; frags = schedule->frags; for (i = 0; i < n_frags; i++) { status = frag_init(coll_args, schedule, team, &frags[i]); if (ucc_unlikely(UCC_OK != status)) { ucc_error("failed to initialize fragment for pipeline"); goto err; } frags[i]->super.schedule = &schedule->super; if (frags[i]->super.flags & UCC_COLL_TASK_FLAG_EXECUTOR) { schedule->super.super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; } frags[i]->super.status = UCC_OPERATION_INITIALIZED; frags[i]->super.super.status = UCC_OPERATION_INITIALIZED; } for (i = 0; i < n_frags; i++) { for (j = 0; j < frags[i]->n_tasks; j++) { frags[i]->tasks[j]->n_deps_base = frags[i]->tasks[j]->n_deps; if (task_dependency_event != UCC_EVENT_LAST) { UCC_CHECK_GOTO( ucc_event_manager_subscribe( frags[(i + n_frags - 1) % n_frags]->tasks[j], task_dependency_event, frags[i]->tasks[j], ucc_dependency_handler), err, status); frags[i]->tasks[j]->n_deps_base++; } } UCC_CHECK_GOTO(ucc_event_manager_subscribe( &schedule->super.super, UCC_EVENT_SCHEDULE_STARTED, &frags[i]->super, ucc_frag_start_handler), err, status); UCC_CHECK_GOTO(ucc_event_manager_subscribe( &frags[i]->super, UCC_EVENT_COMPLETED_SCHEDULE, &schedule->super.super, ucc_schedule_pipelined_completed_handler), err, status); } return UCC_OK; err: for (i = i - 1; i >= 0; i--) { frags[i]->super.finalize(&frags[i]->super); } return status; } ucc_status_t ucc_dependency_handler(ucc_coll_task_t *parent, ucc_coll_task_t *task) { ucc_status_t status; uint32_t n_deps_satisfied; n_deps_satisfied = ucc_atomic_fadd32(&task->n_deps_satisfied, 1); ucc_assert(task->n_deps_satisfied > n_deps_satisfied); ucc_trace_req("task %p, n_deps %d, satisfied %d", task, task->n_deps, n_deps_satisfied); if (task->n_deps == n_deps_satisfied + 1) { task->start_time = parent->start_time; status = task->post(task); if (status >= 0) { ucc_event_manager_notify(task, UCC_EVENT_TASK_STARTED); } return status; } return UCC_OK; } ucc-1.8.0/src/schedule/ucc_schedule.c0000664000175000017500000002062315211535620017636 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "ucc_schedule.h" #include "utils/ucc_compiler_def.h" #include "utils/ucc_mpool.h" #include "components/base/ucc_base_iface.h" #include "coll_score/ucc_coll_score.h" #include "core/ucc_context.h" static void ucc_coll_task_mpool_obj_init(ucc_mpool_t *mp, void *obj, //NOLINT void *chunk) //NOLINT { ucc_coll_task_t *task = obj; ucc_coll_task_construct(task); } static void ucc_coll_task_mpool_obj_cleanup(ucc_mpool_t *mp, void *obj) //NOLINT { ucc_coll_task_t *task = obj; ucc_coll_task_destruct(task); } struct ucc_mpool_ops ucc_coll_task_mpool_ops = { .chunk_alloc = ucc_mpool_hugetlb_malloc, .chunk_release = ucc_mpool_hugetlb_free, .obj_init = ucc_coll_task_mpool_obj_init, .obj_cleanup = ucc_coll_task_mpool_obj_cleanup }; static ucc_status_t ucc_event_manager_init(ucc_coll_task_t *task) { ucc_event_manager_t *em; ucc_list_for_each(em, &task->em_list, list_elem) { em->n_listeners = 0; } return UCC_OK; } ucc_status_t ucc_event_manager_subscribe(ucc_coll_task_t *parent_task, ucc_event_t event, ucc_coll_task_t *task, ucc_task_event_handler_p handler) { ucc_event_manager_t *em; ucc_list_for_each(em, &parent_task->em_list, list_elem) { if (em->n_listeners < MAX_LISTENERS) { set: em->listeners[em->n_listeners].task = task; em->listeners[em->n_listeners].event = event; em->listeners[em->n_listeners].handler = handler; em->n_listeners++; return UCC_OK; } } em = ucc_malloc(sizeof(*em), "em"); if (ucc_unlikely(!em)) { ucc_error("failed to allocate %zd bytes for event_manager", sizeof(*em)); return UCC_ERR_NO_MEMORY; } em->n_listeners = 0; ucc_list_add_tail(&parent_task->em_list, &em->list_elem); goto set; } void ucc_coll_task_construct(ucc_coll_task_t *task) { ucc_list_head_init(&task->em_list); } void ucc_coll_task_destruct(ucc_coll_task_t *task) { ucc_event_manager_t *em, *m; ucc_list_for_each_safe(em, m, &task->em_list, list_elem) { ucc_list_del(&em->list_elem); free(em); } } ucc_status_t ucc_dummy_post(ucc_coll_task_t *task) { task->status = UCC_OK; return ucc_task_complete(task); } ucc_status_t ucc_dummy_finalize(ucc_coll_task_t *task) { ucc_mpool_put(task); return UCC_OK; } /* NOLINTNEXTLINE task argument is not used*/ void ucc_dummy_progress(ucc_coll_task_t *task) { /* this function should never be called */ ucc_assert_always(0); } ucc_status_t ucc_coll_task_init(ucc_coll_task_t *task, ucc_base_coll_args_t *bargs, ucc_base_team_t *team) { task->flags = 0; task->ee = NULL; task->team = team; task->n_deps = 0; task->n_deps_satisfied = 0; task->bargs.args.mask = 0; task->schedule = NULL; task->executor = NULL; task->super.status = UCC_OPERATION_INITIALIZED; task->triggered_post_setup = NULL; task->triggered_post = ucc_triggered_post; task->post = ucc_dummy_post; task->finalize = ucc_dummy_finalize; task->progress = ucc_dummy_progress; // Prevent asymmetric memory copy-out of garbage address at task complete task->bargs.asymmetric_save_info.scratch = NULL; if (bargs) { memcpy(&task->bargs, bargs, sizeof(*bargs)); } ucc_lf_queue_init_elem(&task->lf_elem); return ucc_event_manager_init(task); } ucc_status_t ucc_coll_task_get_executor(ucc_coll_task_t *task, ucc_ee_executor_t **exec) { ucc_status_t st = UCC_OK; if (task->executor == NULL) { if (ucc_unlikely(!task->schedule)) { ucc_error("executor wasn't initialized for the collective"); return UCC_ERR_INVALID_PARAM; } st = ucc_coll_task_get_executor(&task->schedule->super, &task->executor); } *exec = task->executor; return st; } static ucc_status_t ucc_task_error_handler(ucc_coll_task_t *parent_task, ucc_coll_task_t *task) { ucc_event_manager_t *em; ucc_coll_task_t *listener; int i; task->super.status = parent_task->super.status; ucc_list_for_each(em, &parent_task->em_list, list_elem) { for (i = 0; i < em->n_listeners; i++) { listener = em->listeners[i].task; if (listener->super.status != parent_task->super.status) { /* status has not been propagated yet */ ucc_task_error_handler(task, listener); } } } return UCC_OK; } ucc_status_t ucc_event_manager_notify(ucc_coll_task_t *parent_task, ucc_event_t event) { ucc_event_manager_t *em; ucc_coll_task_t *task; ucc_status_t status; int i; ucc_list_for_each(em, &parent_task->em_list, list_elem) { for (i = 0; i < em->n_listeners; i++) { task = em->listeners[i].task; if (ucc_unlikely(event == UCC_EVENT_ERROR)) { ucc_task_error_handler(parent_task, task); continue; } if (em->listeners[i].event == event) { status = em->listeners[i].handler(parent_task, task); if (ucc_unlikely(status != UCC_OK)) { return status; } } } } return UCC_OK; } static ucc_status_t ucc_schedule_completed_handler(ucc_coll_task_t *parent_task, //NOLINT ucc_coll_task_t *task) { ucc_schedule_t *self = ucc_container_of(task, ucc_schedule_t, super); uint32_t n_completed_tasks; n_completed_tasks = ucc_atomic_fadd32(&self->n_completed_tasks, 1); if (n_completed_tasks + 1 == self->n_tasks) { self->super.status = UCC_OK; ucc_task_complete(&self->super); } return UCC_OK; } ucc_status_t ucc_schedule_init(ucc_schedule_t *schedule, ucc_base_coll_args_t *bargs, ucc_base_team_t *team) { ucc_status_t status; status = ucc_coll_task_init(&schedule->super, bargs, team); schedule->super.flags |= UCC_COLL_TASK_FLAG_IS_SCHEDULE; schedule->ctx = team->context->ucc_context; schedule->n_tasks = 0; return status; } ucc_status_t ucc_schedule_add_task(ucc_schedule_t *schedule, ucc_coll_task_t *task) { ucc_status_t status; status = ucc_event_manager_subscribe(task, UCC_EVENT_COMPLETED_SCHEDULE, &schedule->super, ucc_schedule_completed_handler); task->schedule = schedule; schedule->tasks[schedule->n_tasks++] = task; if (task->flags & UCC_COLL_TASK_FLAG_EXECUTOR) { schedule->super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; } return status; } ucc_status_t ucc_schedule_start(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); schedule->n_completed_tasks = 0; schedule->super.status = UCC_INPROGRESS; schedule->super.super.status = UCC_INPROGRESS; return ucc_event_manager_notify(&schedule->super, UCC_EVENT_SCHEDULE_STARTED); } ucc_status_t ucc_task_start_handler(ucc_coll_task_t *parent, ucc_coll_task_t *task) { task->start_time = parent->start_time; return task->post(task); } ucc_status_t ucc_schedule_finalize(ucc_coll_task_t *task) { ucc_schedule_t *schedule = ucc_derived_of(task, ucc_schedule_t); ucc_status_t status_overall = UCC_OK; ucc_status_t status; int i; for (i = 0; i < schedule->n_tasks; i++) { if (schedule->tasks[i]->finalize) { status = schedule->tasks[i]->finalize(schedule->tasks[i]); if (UCC_OK != status) { status_overall = status; } } } return status_overall; } ucc-1.8.0/src/schedule/ucc_schedule.h0000664000175000017500000002770515211535620017653 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_SCHEDULE_H_ #define UCC_SCHEDULE_H_ #include "ucc/api/ucc.h" #include "utils/ucc_list.h" #include "utils/ucc_log.h" #include "utils/ucc_lock_free_queue.h" #include "utils/ucc_coll_utils.h" #include "components/base/ucc_base_iface.h" #include "components/ec/ucc_ec.h" #include "components/mc/ucc_mc.h" #include "components/cl/ucc_cl_type.h" #define MAX_LISTENERS 4 typedef enum { UCC_EVENT_COMPLETED = 0, UCC_EVENT_SCHEDULE_STARTED, UCC_EVENT_TASK_STARTED, UCC_EVENT_COMPLETED_SCHEDULE, /*< Event is used to notify SCHEDULE that one of its task has completed */ UCC_EVENT_ERROR, UCC_EVENT_LAST } ucc_event_t; typedef struct ucc_coll_task ucc_coll_task_t; typedef struct ucc_schedule ucc_schedule_t; typedef struct ucc_base_team ucc_base_team_t; typedef ucc_status_t (*ucc_coll_post_fn_t)(ucc_coll_task_t *task); typedef void (*ucc_coll_progress_fn_t)(ucc_coll_task_t *task); typedef ucc_status_t (*ucc_coll_finalize_fn_t)(ucc_coll_task_t *task); typedef ucc_status_t (*ucc_task_event_handler_p)(ucc_coll_task_t *parent, ucc_coll_task_t *task); /* triggered post setup function will be launched before starting executor */ typedef ucc_status_t (*ucc_coll_triggered_post_setup_fn_t)(ucc_coll_task_t *task); typedef ucc_status_t (*ucc_coll_triggered_post_fn_t)(ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *task); typedef struct ucc_em_listener { ucc_coll_task_t *task; ucc_task_event_handler_p handler; ucc_event_t event; } ucc_em_listener_t; typedef struct ucc_event_manager { ucc_list_link_t list_elem; unsigned n_listeners; ucc_em_listener_t listeners[MAX_LISTENERS]; } ucc_event_manager_t; /* Forward declaration for service collective request */ typedef struct ucc_service_coll_req ucc_service_coll_req_t; /** * @brief Datatype validation state for rooted collectives * * This structure holds the state for transparent datatype validation * before executing the actual collective operation. The validation uses a service * allreduce with MIN operation to efficiently detect mismatches. * * Design: Uses a schedule with two tasks: * 1. Validation task: performs service allreduce (MIN) and validates results * 2. Actual collective task: depends on validation completing successfully * * Validation algorithm uses min/max trick with single in-place allreduce: * - Send/receive [dt, -dt, mem, -mem] with MIN reduction * - After reduction: if min(dt) == -min(-dt), all ranks have same dt * - Same principle for memory type * - Message size: 8 bytes (4 × int16, doesn't scale with number of ranks) * - Optimized for predefined datatypes only (max value 136 fits in int16) * * If validation fails, the dependency mechanism prevents the actual task from posting. */ typedef struct ucc_dt_check_state { ucc_service_coll_req_t *check_req; /* Service allreduce request */ int16_t values[4]; /* In-place: [dt, -dt, mem, -mem] */ ucc_subset_t subset; /* Subset for service allreduce */ int validated; /* 1 if validation passed, 0 if failed */ struct ucc_coll_task *actual_task; /* Pointer to actual collective task */ } ucc_dt_check_state_t; enum { UCC_COLL_TASK_FLAG_CB = UCC_BIT(0), /* executor is required for collective*/ UCC_COLL_TASK_FLAG_EXECUTOR = UCC_BIT(1), /* user visible task */ UCC_COLL_TASK_FLAG_TOP_LEVEL = UCC_BIT(2), /* stop executor in task complete*/ UCC_COLL_TASK_FLAG_EXECUTOR_STOP = UCC_BIT(3), /* destroy executor in task complete */ UCC_COLL_TASK_FLAG_EXECUTOR_DESTROY = UCC_BIT(4), /* if set task can be casted to scheulde */ UCC_COLL_TASK_FLAG_IS_SCHEDULE = UCC_BIT(5), /* if set task can be casted to scheulde */ UCC_COLL_TASK_FLAG_IS_PIPELINED_SCHEDULE = UCC_BIT(6), /* fragment completed and is waiting to be restarted (pipelined schedules) */ UCC_COLL_TASK_FLAG_RESTART_PENDING = UCC_BIT(7), }; typedef struct ucc_coll_task { ucc_coll_req_t super; /** * Task internal status, TLs and CLs should use it to track collective * state. super.status is visible to user and should be updated only * by core level to avoid potential races */ ucc_status_t status; ucc_list_link_t em_list; ucc_base_coll_args_t bargs; ucc_base_team_t *team; /* CL/TL team pointer */ ucc_schedule_t *schedule; uint32_t flags; ucc_coll_post_fn_t post; ucc_coll_triggered_post_setup_fn_t triggered_post_setup; ucc_coll_triggered_post_fn_t triggered_post; ucc_coll_progress_fn_t progress; ucc_coll_finalize_fn_t finalize; ucc_coll_callback_t cb; ucc_ee_h ee; ucc_ev_t *ev; ucc_coll_task_t *triggered_task; ucc_ee_executor_t *executor; union { /* used for st & locked mt progress queue */ ucc_list_link_t list_elem; /* used for lf mt progress queue */ ucc_lf_queue_elem_t lf_elem; }; uint32_t n_deps; uint32_t n_deps_satisfied; uint32_t n_deps_base; /* timestamp of the start time: either post or triggered_post */ double start_time; uint32_t seq_num; } ucc_coll_task_t; extern struct ucc_mpool_ops ucc_coll_task_mpool_ops; typedef struct ucc_context ucc_context_t; #define UCC_SCHEDULE_MAX_TASKS 8 typedef struct ucc_schedule { ucc_coll_task_t super; uint32_t n_completed_tasks; uint32_t n_tasks; ucc_context_t *ctx; ucc_coll_task_t *tasks[UCC_SCHEDULE_MAX_TASKS]; } ucc_schedule_t; /** * @brief Extended schedule for datatype validation * * This schedule type includes embedded validation state to avoid * increasing the size of ucc_coll_task_t. The validation state is * accessed through the schedule pointer. */ typedef struct ucc_dt_check_schedule { ucc_schedule_t super; ucc_dt_check_state_t dt_check; } ucc_dt_check_schedule_t; void ucc_coll_task_construct(ucc_coll_task_t *task); void ucc_coll_task_destruct(ucc_coll_task_t *task); ucc_status_t ucc_coll_task_init(ucc_coll_task_t *task, ucc_base_coll_args_t *args, ucc_base_team_t *team); ucc_status_t ucc_coll_task_get_executor(ucc_coll_task_t *task, ucc_ee_executor_t **exec); ucc_status_t ucc_event_manager_subscribe(ucc_coll_task_t *parent_task, ucc_event_t event, ucc_coll_task_t *task, ucc_task_event_handler_p handler); ucc_status_t ucc_event_manager_notify(ucc_coll_task_t *parent_task, ucc_event_t event); ucc_status_t ucc_schedule_init(ucc_schedule_t *schedule, ucc_base_coll_args_t *bargs, ucc_base_team_t *team); ucc_status_t ucc_schedule_add_task(ucc_schedule_t *schedule, ucc_coll_task_t *task); ucc_status_t ucc_schedule_start(ucc_coll_task_t *task); ucc_status_t ucc_task_start_handler(ucc_coll_task_t *parent, ucc_coll_task_t *task); ucc_status_t ucc_schedule_finalize(ucc_coll_task_t *task); ucc_status_t ucc_dependency_handler(ucc_coll_task_t *parent, ucc_coll_task_t *task); ucc_status_t ucc_triggered_post(ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *task); static inline ucc_status_t ucc_task_complete(ucc_coll_task_t *task) { ucc_status_t status = task->status; ucc_coll_callback_t cb = task->cb; int has_cb = task->flags & UCC_COLL_TASK_FLAG_CB; int has_sched = task->schedule != NULL; ucc_assert((status == UCC_OK) || (status < 0)); /* If task is part of a schedule then it can be released during ucc_event_manager_notify(EVENT_COMPLETED_SCHEDULE) below. Sequence: notify => schedule->n_completed_tasks++ => schedule->super.status = UCC_OK => user releases schedule from another thread => schedule_finalize => schedule finalizes all the tasks. After that the task ptr should not be accessed. This is why notification of schedule is done separately in the end of this function. Internal implementation must make sure that tasks with schedules are not released during a callback (if set). */ if (ucc_likely(status == UCC_OK)) { ucc_buffer_info_asymmetric_memtype_t *save = &task->bargs.asymmetric_save_info; if (save->scratch && task->bargs.args.coll_type != UCC_COLL_TYPE_SCATTERV && task->bargs.args.coll_type != UCC_COLL_TYPE_SCATTER) { status = ucc_copy_asymmetric_buffer(task); if (status != UCC_OK) { ucc_error("failure copying out asymmetric buffer: %s", ucc_status_string(status)); } } status = ucc_event_manager_notify(task, UCC_EVENT_COMPLETED); } else { /* error in task status */ if (UCC_ERR_TIMED_OUT == status) { char coll_str[256]; ucc_coll_str(task, coll_str, sizeof(coll_str), UCC_LOG_LEVEL_DEBUG); ucc_warn("timeout %g sec. has expired on %s", task->bargs.args.timeout, coll_str); } else { ucc_error("failure in task %p, %s", task, ucc_status_string(task->status)); } ucc_assert_always(status < 0); ucc_event_manager_notify(task, UCC_EVENT_ERROR); } if ((task->executor) && (task->flags & UCC_COLL_TASK_FLAG_EXECUTOR_STOP)) { status = ucc_ee_executor_stop(task->executor); if (ucc_unlikely(status != UCC_OK)) { ucc_error("failed to stop executor %s", ucc_status_string(status)); } } if ((task->executor) && (task->flags & UCC_COLL_TASK_FLAG_EXECUTOR_DESTROY)) { status = ucc_ee_executor_finalize(task->executor); if (ucc_unlikely(status != UCC_OK)) { ucc_error("failed to finalize executor %s", ucc_status_string(status)); } task->executor = NULL; } task->super.status = status; if (has_cb) { cb.cb(cb.data, status); } if (has_sched && status == UCC_OK) { status = ucc_event_manager_notify(task, UCC_EVENT_COMPLETED_SCHEDULE); } return status; } static inline ucc_status_t ucc_task_subscribe_dep(ucc_coll_task_t *target, ucc_coll_task_t *subscriber, ucc_event_t event) { ucc_status_t status = ucc_event_manager_subscribe(target, event, subscriber, ucc_dependency_handler); subscriber->n_deps++; return status; } #define UCC_TASK_LIB(_task) (((ucc_coll_task_t *)_task)->team->context->lib) #define UCC_TASK_CORE_CTX(_task) \ (((ucc_coll_task_t *)_task)->team->context->ucc_context) #define UCC_TASK_THREAD_MODE(_task) (UCC_TASK_CORE_CTX(_task)->thread_mode) /* Check if the task is part of a CL hier team */ static inline int ucc_coll_task_is_cl_hier(const ucc_coll_task_t *task) { return task && task->team && (task->team->params.scope == UCC_CL_HIER); } #endif ucc-1.8.0/src/schedule/ucc_schedule_pipelined.h0000664000175000017500000001053315211535620021673 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_SCHEDULE_PIPELINED_H_ #define UCC_SCHEDULE_PIPELINED_H_ #include "components/base/ucc_base_iface.h" #define UCC_SCHEDULE_FRAG_MAX_TASKS 8 #define UCC_SCHEDULE_PIPELINED_MAX_FRAGS 4 typedef struct ucc_schedule_pipelined ucc_schedule_pipelined_t; /* frag_init is the callback provided by the user of pipelined framework (e.g., TL that needs to build a pipeline) that is reponsible for allocation of a single fragment schedule */ typedef ucc_status_t (*ucc_schedule_frag_init_fn_t)( ucc_base_coll_args_t *coll_args, ucc_schedule_pipelined_t *schedule_p, ucc_base_team_t *team, ucc_schedule_t **frag); /* frag setup is the callback which is triggered whenever a fragment is being re-launched (or launched for the first time ). This callback is used to update coll_args values (ptr offsets, counts) depending on the frag num. frag_num - is the sequencial number of currently launched fragment. */ typedef ucc_status_t (*ucc_schedule_frag_setup_fn_t)( ucc_schedule_pipelined_t *schedule_p, ucc_schedule_t *frag, int frag_num); typedef enum { UCC_PIPELINE_PARALLEL, /*< Tasks of different frags can start concurrently in parallel. Out-of-order launch of frags is possible */ UCC_PIPELINE_ORDERED, /*< Tasks of different frags are ordered on EVENT_STARTED - no out-of-order */ UCC_PIPELINE_SEQUENTIAL, /*< Tasks of different frags are ordered on EVENT_COMPLETED - no out-of-order. Less parallelism compared to ORDERED, but does not oversubscribe resources */ UCC_PIPELINE_LAST } ucc_pipeline_order_t; typedef struct ucc_pipeline_params { size_t threshold; size_t frag_size; unsigned n_frags; unsigned pdepth; ucc_pipeline_order_t order; } ucc_pipeline_params_t; static inline void ucc_pipeline_nfrags_pdepth(ucc_pipeline_params_t *p, size_t msgsize, int *n_frags, int *pipeline_depth) { int min_num_frags; *n_frags = 1; if (msgsize > p->threshold) { min_num_frags = ucc_div_round_up(msgsize, p->frag_size); *n_frags = ucc_max(min_num_frags, p->n_frags); } *pipeline_depth = ucc_min(*n_frags, p->pdepth); } extern const char* ucc_pipeline_order_names[]; typedef struct ucc_schedule_pipelined { ucc_schedule_t super; /* Array of the frag schedules - 1 schedule per pipeline entry */ ucc_schedule_t * frags[UCC_SCHEDULE_PIPELINED_MAX_FRAGS]; /* n_frags - is the depth of the pipeline, ie how many fragments can be outstanding at a time */ int n_frags; /* total number of fragments started so far. Note, total number of frags to be executed is stored in super.n_tasks */ int n_frags_started; /* number of frags active in the pipeline */ int n_frags_in_pipeline; /* sequential flag. if set to 1 the pipeline sets additional deps between the tasks in different frags. This prevents out-of-order task launch in different frags of a pipeline */ ucc_pipeline_order_t order; int next_frag_to_post; ucc_schedule_frag_setup_fn_t frag_setup; ucc_recursive_spinlock_t lock; } ucc_schedule_pipelined_t; /* Creates a pipelined schedule for the algorithm defined by "frag_init". frag_init, frag_setup - client callbacks used to init the pipeline. n_frags - pipeline depth n_frags_total - total number of fragments to be launched. If n_frags_total > n_frags, then some frag schedules will be re-launched multiple times. */ ucc_status_t ucc_schedule_pipelined_init( ucc_base_coll_args_t *coll_args, ucc_base_team_t *team, ucc_schedule_frag_init_fn_t frag_init, ucc_schedule_frag_setup_fn_t frag_setup, int n_frags, int n_frags_total, ucc_pipeline_order_t order, ucc_schedule_pipelined_t *schedule_p); ucc_status_t ucc_schedule_pipelined_post(ucc_coll_task_t *task); ucc_status_t ucc_schedule_pipelined_finalize(ucc_coll_task_t *task); #endif ucc-1.8.0/src/utils/0000775000175000017500000000000015211535620014405 5ustar alastairalastairucc-1.8.0/src/utils/ucc_status.c0000664000175000017500000000221315211535620016724 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "config.h" #include "ucc/api/ucc_status.h" #include const char *ucc_status_string(ucc_status_t status) { static char error_str[128] = {0}; switch (status) { case UCC_OK: return "Success"; case UCC_INPROGRESS: return "Operation in progress"; case UCC_OPERATION_INITIALIZED: return "Operation initialized"; case UCC_ERR_NOT_SUPPORTED: return "Operation is not supported"; case UCC_ERR_NOT_IMPLEMENTED: return "Not implemented"; case UCC_ERR_INVALID_PARAM: return "Invalid parameter"; case UCC_ERR_NO_MEMORY: return "Out of memory"; case UCC_ERR_NO_RESOURCE: return "Resources are not available for the operation"; case UCC_ERR_NO_MESSAGE: return "Unhandled error"; case UCC_ERR_NOT_FOUND: return "Not found"; case UCC_ERR_TIMED_OUT: return "Timeout expired"; default: snprintf(error_str, sizeof(error_str) - 1, "Unknown error %d", status); return error_str; }; } ucc-1.8.0/src/utils/ucc_coll_utils.c0000664000175000017500000010134515211535620017560 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_coll_utils.h" #include "components/base/ucc_base_iface.h" #include "core/ucc_team.h" #include "schedule/ucc_schedule_pipelined.h" #define STR_TYPE_CHECK(_str, _p, _prefix) \ do { \ if ((0 == strcasecmp(_UCC_PP_MAKE_STRING(_p), _str))) { \ return _prefix##_p; \ } \ } while (0) #define STR_COLL_TYPE_CHECK(_str, _p) STR_TYPE_CHECK(_str, _p, UCC_COLL_TYPE_) ucc_coll_type_t ucc_coll_type_from_str(const char *str) { STR_COLL_TYPE_CHECK(str, ALLGATHER); STR_COLL_TYPE_CHECK(str, ALLGATHERV); STR_COLL_TYPE_CHECK(str, ALLREDUCE); STR_COLL_TYPE_CHECK(str, ALLTOALL); STR_COLL_TYPE_CHECK(str, ALLTOALLV); STR_COLL_TYPE_CHECK(str, BARRIER); STR_COLL_TYPE_CHECK(str, BCAST); STR_COLL_TYPE_CHECK(str, FANIN); STR_COLL_TYPE_CHECK(str, FANOUT); STR_COLL_TYPE_CHECK(str, GATHER); STR_COLL_TYPE_CHECK(str, GATHERV); STR_COLL_TYPE_CHECK(str, REDUCE); STR_COLL_TYPE_CHECK(str, REDUCE_SCATTER); STR_COLL_TYPE_CHECK(str, REDUCE_SCATTERV); STR_COLL_TYPE_CHECK(str, SCATTER); STR_COLL_TYPE_CHECK(str, SCATTERV); return UCC_COLL_TYPE_LAST; } #define STR_MEM_TYPE_CHECK(_str, _p) \ do { \ if (0 == strcasecmp("CudaManaged", _str)) { \ STR_TYPE_CHECK("cuda_managed", _p, UCC_MEMORY_TYPE_); \ } else if (0 == strcasecmp("RocmManaged", _str)) { \ STR_TYPE_CHECK("rocm_managed", _p, UCC_MEMORY_TYPE_); \ } \ STR_TYPE_CHECK(_str, _p, UCC_MEMORY_TYPE_); \ } while (0) ucc_memory_type_t ucc_mem_type_from_str(const char *str) { STR_MEM_TYPE_CHECK(str, HOST); STR_MEM_TYPE_CHECK(str, CUDA); STR_MEM_TYPE_CHECK(str, CUDA_MANAGED); STR_MEM_TYPE_CHECK(str, ROCM); STR_MEM_TYPE_CHECK(str, ROCM_MANAGED); return UCC_MEMORY_TYPE_LAST; } int ucc_coll_args_is_mem_symmetric(const ucc_coll_args_t *args, ucc_rank_t rank) { ucc_rank_t root = args->root; if (UCC_IS_INPLACE(*args)) { return 1; } switch (args->coll_type) { case UCC_COLL_TYPE_BARRIER: case UCC_COLL_TYPE_BCAST: case UCC_COLL_TYPE_FANIN: case UCC_COLL_TYPE_FANOUT: return 1; case UCC_COLL_TYPE_ALLTOALL: case UCC_COLL_TYPE_ALLREDUCE: case UCC_COLL_TYPE_ALLGATHER: case UCC_COLL_TYPE_REDUCE_SCATTER: return args->dst.info.mem_type == args->src.info.mem_type; case UCC_COLL_TYPE_ALLGATHERV: case UCC_COLL_TYPE_REDUCE_SCATTERV: return args->dst.info_v.mem_type == args->src.info.mem_type; case UCC_COLL_TYPE_ALLTOALLV: return args->dst.info_v.mem_type == args->src.info_v.mem_type; case UCC_COLL_TYPE_REDUCE: case UCC_COLL_TYPE_GATHER: case UCC_COLL_TYPE_SCATTER: return root != rank || (args->dst.info.mem_type == args->src.info.mem_type); case UCC_COLL_TYPE_GATHERV: return root != rank || (args->dst.info_v.mem_type == args->src.info.mem_type); case UCC_COLL_TYPE_SCATTERV: return root != rank || (args->dst.info.mem_type == args->src.info_v.mem_type); default: break; } return 0; } /* If this is the root and the src/dst buffers are asymmetric, one buffer needs to have a new allocation to make the mem types match. If that buffer was the dst buffer, copy the result back into the old dst on task completion */ ucc_status_t ucc_coll_args_init_asymmetric_buffer(ucc_coll_args_t *args, ucc_team_h team, ucc_buffer_info_asymmetric_memtype_t *save_info) { ucc_status_t status = UCC_OK; if (UCC_IS_INPLACE(*args)) { return UCC_ERR_INVALID_PARAM; } switch (args->coll_type) { case UCC_COLL_TYPE_REDUCE: case UCC_COLL_TYPE_GATHER: { ucc_memory_type_t mem_type = args->src.info.mem_type; if (args->coll_type == UCC_COLL_TYPE_SCATTERV) { mem_type = args->src.info_v.mem_type; } memcpy(&save_info->old_asymmetric_buffer.info, &args->dst.info, sizeof(ucc_coll_buffer_info_t)); status = ucc_mc_alloc(&save_info->scratch, ucc_dt_size(args->dst.info.datatype) * args->dst.info.count, mem_type); if (ucc_unlikely(UCC_OK != status)) { ucc_error("failed to allocate replacement " "memory for asymmetric buffer"); return status; } args->dst.info.buffer = save_info->scratch->addr; args->dst.info.mem_type = mem_type; return UCC_OK; } case UCC_COLL_TYPE_GATHERV: { memcpy(&save_info->old_asymmetric_buffer.info_v, &args->dst.info_v, sizeof(ucc_coll_buffer_info_v_t)); status = ucc_mc_alloc(&save_info->scratch, ucc_coll_args_get_v_buffer_size(args, args->dst.info_v.counts, args->dst.info_v.displacements, team->size), args->src.info.mem_type); if (ucc_unlikely(UCC_OK != status)) { ucc_error("failed to allocate replacement " "memory for asymmetric buffer"); return status; } args->dst.info_v.buffer = save_info->scratch->addr; args->dst.info_v.mem_type = args->src.info.mem_type; return UCC_OK; } case UCC_COLL_TYPE_SCATTER: { ucc_memory_type_t mem_type = args->dst.info.mem_type; memcpy(&save_info->old_asymmetric_buffer.info, &args->src.info, sizeof(ucc_coll_buffer_info_t)); status = ucc_mc_alloc(&save_info->scratch, ucc_dt_size(args->src.info.datatype) * args->src.info.count, mem_type); if (ucc_unlikely(UCC_OK != status)) { ucc_error("failed to allocate replacement " "memory for asymmetric buffer"); return status; } args->src.info.buffer = save_info->scratch->addr; args->src.info.mem_type = mem_type; return UCC_OK; } case UCC_COLL_TYPE_SCATTERV: { ucc_memory_type_t mem_type = args->dst.info.mem_type; memcpy(&save_info->old_asymmetric_buffer.info_v, &args->src.info_v, sizeof(ucc_coll_buffer_info_v_t)); status = ucc_mc_alloc(&save_info->scratch, ucc_coll_args_get_v_buffer_size(args, args->src.info_v.counts, args->src.info_v.displacements, team->size), mem_type); if (ucc_unlikely(UCC_OK != status)) { ucc_error("failed to allocate replacement " "memory for asymmetric buffer"); return status; } args->src.info_v.buffer = save_info->scratch->addr; args->src.info_v.mem_type = mem_type; return UCC_OK; } default: break; } return UCC_ERR_INVALID_PARAM; } ucc_status_t ucc_coll_args_free_asymmetric_buffer(ucc_coll_task_t *task) { ucc_status_t status = UCC_OK; ucc_buffer_info_asymmetric_memtype_t *save = &task->bargs.asymmetric_save_info; if (UCC_IS_INPLACE(task->bargs.args)) { return UCC_ERR_INVALID_PARAM; } if (save->scratch == NULL) { ucc_error("failure trying to free NULL asymmetric buffer"); return UCC_ERR_INVALID_PARAM; } status = ucc_mc_free(save->scratch); if (ucc_unlikely(status != UCC_OK)) { ucc_error("error freeing scratch asymmetric buffer: %s", ucc_status_string(status)); } save->scratch = NULL; return status; } ucc_status_t ucc_copy_asymmetric_buffer(ucc_coll_task_t *task) { ucc_status_t status = UCC_OK; ucc_coll_args_t *coll_args = &task->bargs.args; ucc_buffer_info_asymmetric_memtype_t *save = &task->bargs.asymmetric_save_info; ucc_rank_t size = task->team->params.size; if(task->bargs.args.coll_type == UCC_COLL_TYPE_SCATTERV) { // copy in status = ucc_mc_memcpy(save->scratch->addr, save->old_asymmetric_buffer.info_v.buffer, ucc_coll_args_get_v_buffer_size(coll_args, save->old_asymmetric_buffer.info_v.counts, save->old_asymmetric_buffer.info_v.displacements, size), save->scratch->mt, save->old_asymmetric_buffer.info_v.mem_type); } else if(task->bargs.args.coll_type == UCC_COLL_TYPE_SCATTER) { // copy in status = ucc_mc_memcpy(save->scratch->addr, save->old_asymmetric_buffer.info.buffer, ucc_dt_size(save->old_asymmetric_buffer.info.datatype) * save->old_asymmetric_buffer.info.count, save->scratch->mt, save->old_asymmetric_buffer.info.mem_type); } else if(task->bargs.args.coll_type == UCC_COLL_TYPE_GATHERV) { // copy out status = ucc_mc_memcpy(save->old_asymmetric_buffer.info_v.buffer, save->scratch->addr, ucc_coll_args_get_v_buffer_size(coll_args, save->old_asymmetric_buffer.info_v.counts, save->old_asymmetric_buffer.info_v.displacements, size), save->old_asymmetric_buffer.info_v.mem_type, save->scratch->mt); } else { // copy out status = ucc_mc_memcpy(save->old_asymmetric_buffer.info.buffer, save->scratch->addr, ucc_dt_size(save->old_asymmetric_buffer.info.datatype) * save->old_asymmetric_buffer.info.count, save->old_asymmetric_buffer.info.mem_type, save->scratch->mt); } if (ucc_unlikely(status != UCC_OK)) { ucc_error("error copying back to old asymmetric buffer: %s", ucc_status_string(status)); } return status; } int ucc_coll_args_is_predefined_dt(const ucc_coll_args_t *args, ucc_rank_t rank) { switch (args->coll_type) { case UCC_COLL_TYPE_BARRIER: case UCC_COLL_TYPE_FANIN: case UCC_COLL_TYPE_FANOUT: return 1; case UCC_COLL_TYPE_ALLREDUCE: case UCC_COLL_TYPE_REDUCE_SCATTER: case UCC_COLL_TYPE_ALLGATHER: case UCC_COLL_TYPE_ALLTOALL: return UCC_DT_IS_PREDEFINED(args->dst.info.datatype) && (UCC_IS_INPLACE(*args) || UCC_DT_IS_PREDEFINED(args->src.info.datatype)); case UCC_COLL_TYPE_ALLGATHERV: case UCC_COLL_TYPE_REDUCE_SCATTERV: return UCC_DT_IS_PREDEFINED(args->dst.info_v.datatype) && (UCC_IS_INPLACE(*args) || UCC_DT_IS_PREDEFINED(args->src.info.datatype)); case UCC_COLL_TYPE_ALLTOALLV: return UCC_DT_IS_PREDEFINED(args->dst.info_v.datatype) && (UCC_IS_INPLACE(*args) || UCC_DT_IS_PREDEFINED(args->src.info_v.datatype)); case UCC_COLL_TYPE_BCAST: return UCC_DT_IS_PREDEFINED(args->src.info.datatype); case UCC_COLL_TYPE_GATHER: case UCC_COLL_TYPE_REDUCE: if (UCC_IS_ROOT(*args, rank)) { return UCC_DT_IS_PREDEFINED(args->dst.info.datatype) && (UCC_IS_INPLACE(*args) || UCC_DT_IS_PREDEFINED(args->src.info.datatype)); } else { return UCC_DT_IS_PREDEFINED(args->src.info.datatype); } case UCC_COLL_TYPE_GATHERV: if (UCC_IS_ROOT(*args, rank)) { return UCC_DT_IS_PREDEFINED(args->dst.info_v.datatype) && (UCC_IS_INPLACE(*args) || UCC_DT_IS_PREDEFINED(args->src.info.datatype)); } else { return UCC_DT_IS_PREDEFINED(args->src.info.datatype); } case UCC_COLL_TYPE_SCATTER: if (UCC_IS_ROOT(*args, rank)) { return UCC_DT_IS_PREDEFINED(args->src.info.datatype) && (UCC_IS_INPLACE(*args) || UCC_DT_IS_PREDEFINED(args->dst.info.datatype)); } else { return UCC_DT_IS_PREDEFINED(args->dst.info.datatype); } case UCC_COLL_TYPE_SCATTERV: if (UCC_IS_ROOT(*args, rank)) { return UCC_DT_IS_PREDEFINED(args->src.info_v.datatype) && (UCC_IS_INPLACE(*args) || UCC_DT_IS_PREDEFINED(args->dst.info.datatype)); } else { return UCC_DT_IS_PREDEFINED(args->dst.info.datatype); } default: ucc_error("invalid collective type %d", args->coll_type); return -1; } } ucc_memory_type_t ucc_coll_args_mem_type(const ucc_coll_args_t *args, ucc_rank_t rank) { ucc_rank_t root = args->root; switch (args->coll_type) { case UCC_COLL_TYPE_BARRIER: case UCC_COLL_TYPE_FANIN: case UCC_COLL_TYPE_FANOUT: return UCC_MEMORY_TYPE_NOT_APPLY; case UCC_COLL_TYPE_BCAST: return args->src.info.mem_type; case UCC_COLL_TYPE_ALLTOALL: case UCC_COLL_TYPE_ALLREDUCE: case UCC_COLL_TYPE_ALLGATHER: case UCC_COLL_TYPE_REDUCE_SCATTER: return args->dst.info.mem_type; case UCC_COLL_TYPE_ALLGATHERV: case UCC_COLL_TYPE_REDUCE_SCATTERV: case UCC_COLL_TYPE_ALLTOALLV: return args->dst.info_v.mem_type; case UCC_COLL_TYPE_REDUCE: case UCC_COLL_TYPE_GATHER: return (root == rank) ? args->dst.info.mem_type : args->src.info.mem_type; case UCC_COLL_TYPE_SCATTER: return (root == rank) ? args->src.info.mem_type : args->dst.info.mem_type; case UCC_COLL_TYPE_GATHERV: return (root == rank) ? args->dst.info_v.mem_type : args->src.info.mem_type; case UCC_COLL_TYPE_SCATTERV: return (root == rank) ? args->src.info_v.mem_type : args->dst.info.mem_type; default: break; } return UCC_MEMORY_TYPE_UNKNOWN; } size_t ucc_coll_args_msgsize(const ucc_coll_args_t *args, ucc_rank_t rank, ucc_rank_t size) { ucc_rank_t root = args->root; switch (args->coll_type) { case UCC_COLL_TYPE_BARRIER: case UCC_COLL_TYPE_FANIN: case UCC_COLL_TYPE_FANOUT: return 0; case UCC_COLL_TYPE_BCAST: return args->src.info.count * ucc_dt_size(args->src.info.datatype); case UCC_COLL_TYPE_ALLREDUCE: case UCC_COLL_TYPE_ALLTOALL: case UCC_COLL_TYPE_ALLGATHER: case UCC_COLL_TYPE_REDUCE_SCATTER: return args->dst.info.count * ucc_dt_size(args->dst.info.datatype); case UCC_COLL_TYPE_ALLGATHERV: case UCC_COLL_TYPE_REDUCE_SCATTERV: return ucc_coll_args_get_total_count(args, args->dst.info_v.counts, size) * ucc_dt_size(args->dst.info_v.datatype); case UCC_COLL_TYPE_ALLTOALLV: case UCC_COLL_TYPE_GATHERV: case UCC_COLL_TYPE_SCATTERV: /* This means all team members can not know the msg size estimate w/o communication. Local args information is not enough. This prohibits algorithm selection based on msg size thresholds w/o additinoal exchange. */ return UCC_MSG_SIZE_ASYMMETRIC; case UCC_COLL_TYPE_REDUCE: return (root == rank) ? args->dst.info.count * ucc_dt_size(args->dst.info.datatype) : args->src.info.count * ucc_dt_size(args->src.info.datatype); case UCC_COLL_TYPE_GATHER: return (root == rank) ? args->dst.info.count * ucc_dt_size(args->dst.info.datatype) : args->src.info.count * ucc_dt_size(args->src.info.datatype) * size; case UCC_COLL_TYPE_SCATTER: return (root == rank) ? args->src.info.count * ucc_dt_size(args->src.info.datatype) : args->dst.info.count * ucc_dt_size(args->dst.info.datatype) * size; default: ucc_assert(args->coll_type == UCC_COLL_TYPE_LAST); } return 0; } static inline int64_t ucc_ep_map_get_elem(void **array, int i, int is64) { if (is64) { return (int64_t) (*(uint64_t **)(array))[i]; } else { return (int64_t) (*(ucc_rank_t **)(array))[i]; } } static inline ucc_ep_map_t ucc_ep_map_from_array_generic(void **array, ucc_rank_t size, ucc_rank_t full_size, int need_free, int is64) { int is_const_stride = 0; ucc_ep_map_t map; int64_t stride; ucc_rank_t i; map.type = (ucc_ep_map_type_t)0; map.ep_num = size; if (size > 1) { /* try to detect strided pattern */ stride = ucc_ep_map_get_elem(array, 1, is64) - ucc_ep_map_get_elem(array, 0, is64); is_const_stride = 1; for (i = 2; i < size; i++) { if ((ucc_ep_map_get_elem(array, i, is64) - ucc_ep_map_get_elem(array, i - 1, is64)) != stride) { is_const_stride = 0; break; } } } if (is_const_stride) { if ((stride == 1) && (size == full_size)) { map.type = UCC_EP_MAP_FULL; /* coverity[uninit_use] - union fields not used for UCC_EP_MAP_FULL */ } else { map.type = UCC_EP_MAP_STRIDED; map.strided.start = (uint64_t) ucc_ep_map_get_elem(array, 0, is64); map.strided.stride = stride; } if (need_free) { ucc_free(*array); *array = NULL; } } else { map.type = UCC_EP_MAP_ARRAY; map.array.map = (void *)(*array); map.array.elem_size = is64 ? sizeof(uint64_t) : sizeof(ucc_rank_t); } return map; } ucc_ep_map_t ucc_ep_map_from_array(ucc_rank_t **array, ucc_rank_t size, ucc_rank_t full_size, int need_free) { return ucc_ep_map_from_array_generic((void **) array, size, full_size, need_free, 0); } ucc_ep_map_t ucc_ep_map_from_array_64(uint64_t **array, ucc_rank_t size, ucc_rank_t full_size, int need_free) { return ucc_ep_map_from_array_generic((void **) array, size, full_size, need_free, 1); } int ucc_coll_args_is_rooted(ucc_coll_type_t ct) { if (ct == UCC_COLL_TYPE_REDUCE || ct == UCC_COLL_TYPE_BCAST || ct == UCC_COLL_TYPE_GATHER || ct == UCC_COLL_TYPE_SCATTER || ct == UCC_COLL_TYPE_FANIN || ct == UCC_COLL_TYPE_FANOUT || ct == UCC_COLL_TYPE_GATHERV || ct == UCC_COLL_TYPE_SCATTERV) { return 1; } return 0; } #define COLL_ARGS_HEADER_STR_MAX_SIZE 32 void ucc_coll_args_str(const ucc_coll_args_t *args, ucc_rank_t trank, ucc_rank_t tsize, char *str, size_t len) { ucc_coll_type_t ct = args->coll_type; ucc_rank_t root = args->root; ucc_coll_buffer_info_t src_info = {0}; ucc_coll_buffer_info_t dst_info = {0}; char hdr[COLL_ARGS_HEADER_STR_MAX_SIZE] = ""; char tmp[32]; size_t count; int left, has_src, has_dst; ucc_snprintf_safe(hdr, COLL_ARGS_HEADER_STR_MAX_SIZE, "%s", ucc_coll_type_str(ct)); if (ucc_coll_args_is_reduction(ct)) { ucc_snprintf_safe(tmp, sizeof(tmp), " %s", ucc_reduction_op_str(args->op)); left = COLL_ARGS_HEADER_STR_MAX_SIZE - strlen(hdr); strncat(hdr, tmp, left); } if (UCC_IS_INPLACE(*args)) { ucc_snprintf_safe(tmp, sizeof(tmp), " inplace"); left = COLL_ARGS_HEADER_STR_MAX_SIZE - strlen(hdr); strncat(hdr, tmp, left); } if (UCC_IS_PERSISTENT(*args)) { ucc_snprintf_safe(tmp, sizeof(tmp), " persistent"); left = COLL_ARGS_HEADER_STR_MAX_SIZE - strlen(hdr); strncat(hdr, tmp, left); } if (ucc_coll_args_is_rooted(ct)) { ucc_snprintf_safe(tmp, sizeof(tmp), " root %u", root); left = COLL_ARGS_HEADER_STR_MAX_SIZE - strlen(hdr); strncat(hdr, tmp, left); } has_src = has_dst = 0; switch (ct) { case UCC_COLL_TYPE_ALLGATHER: case UCC_COLL_TYPE_ALLREDUCE: case UCC_COLL_TYPE_ALLTOALL: case UCC_COLL_TYPE_REDUCE_SCATTER: dst_info = args->dst.info; has_dst = 1; if (!UCC_IS_INPLACE(*args)) { src_info = args->src.info; has_src = 1; } break; case UCC_COLL_TYPE_ALLGATHERV: case UCC_COLL_TYPE_REDUCE_SCATTERV: count = ucc_coll_args_get_total_count(args, args->dst.info_v.counts, tsize); dst_info.buffer = args->dst.info_v.buffer; dst_info.count = count; dst_info.datatype = args->dst.info_v.datatype; dst_info.mem_type = args->dst.info_v.mem_type; has_dst = 1; if (!UCC_IS_INPLACE(*args)) { src_info = args->src.info; has_src = 1; } break; case UCC_COLL_TYPE_ALLTOALLV: count = ucc_coll_args_get_total_count(args, args->dst.info_v.counts, tsize); dst_info.buffer = args->dst.info_v.buffer; dst_info.count = count; dst_info.datatype = args->dst.info_v.datatype; dst_info.mem_type = args->dst.info_v.mem_type; has_dst = 1; if (!UCC_IS_INPLACE(*args)) { count = ucc_coll_args_get_total_count(args, args->src.info_v.counts, tsize); src_info.buffer = args->src.info_v.buffer; src_info.count = count; src_info.datatype = args->src.info_v.datatype; src_info.mem_type = args->src.info_v.mem_type; has_src = 1; } case UCC_COLL_TYPE_BARRIER: case UCC_COLL_TYPE_FANIN: case UCC_COLL_TYPE_FANOUT: break; case UCC_COLL_TYPE_BCAST: src_info = args->src.info; has_src = 1; break; case UCC_COLL_TYPE_GATHER: case UCC_COLL_TYPE_REDUCE: if (UCC_IS_ROOT(*args, trank)) { dst_info = args->dst.info; has_dst = 1; } if (!UCC_IS_ROOT(*args, trank) || !UCC_IS_INPLACE(*args)) { src_info = args->src.info; has_src = 1; } break; case UCC_COLL_TYPE_GATHERV: if (UCC_IS_ROOT(*args, trank)) { count = ucc_coll_args_get_total_count(args, args->dst.info_v.counts, tsize); dst_info.buffer = args->dst.info_v.buffer; dst_info.count = count; dst_info.datatype = args->dst.info_v.datatype; dst_info.mem_type = args->dst.info_v.mem_type; has_dst = 1; } if (!UCC_IS_ROOT(*args, trank) || !UCC_IS_INPLACE(*args)) { src_info = args->src.info; has_src = 1; } break; case UCC_COLL_TYPE_SCATTER: if (UCC_IS_ROOT(*args, trank)) { src_info = args->src.info; has_src = 1; } if (!UCC_IS_ROOT(*args, trank) || !UCC_IS_INPLACE(*args)) { dst_info = args->dst.info; has_dst = 1; } break; case UCC_COLL_TYPE_SCATTERV: if (UCC_IS_ROOT(*args, trank)) { count = ucc_coll_args_get_total_count(args, args->src.info_v.counts, tsize); src_info.buffer = args->src.info_v.buffer; src_info.count = count; src_info.datatype = args->src.info_v.datatype; src_info.mem_type = args->src.info_v.mem_type; has_src = 1; } if (!UCC_IS_ROOT(*args, trank) || !UCC_IS_INPLACE(*args)) { dst_info = args->dst.info; has_dst = 1; } break; default: ucc_assert(args->coll_type == UCC_COLL_TYPE_LAST); return; } if (has_src && has_dst) { ucc_snprintf_safe(str, len, "%s: src={%p, %zd, %s, %s}, dst={%p, %zd, %s, %s}", hdr, src_info.buffer, src_info.count, ucc_datatype_str(src_info.datatype), ucc_mem_type_str(src_info.mem_type), dst_info.buffer, dst_info.count, ucc_datatype_str(dst_info.datatype), ucc_mem_type_str(dst_info.mem_type)); } else if (has_src && !has_dst) { ucc_snprintf_safe(str, len, "%s: src={%p, %zd, %s, %s}", hdr, src_info.buffer, src_info.count, ucc_datatype_str(src_info.datatype), ucc_mem_type_str(src_info.mem_type)); } else if (!has_src && has_dst) { ucc_snprintf_safe(str, len, "%s: dst={%p, %zd, %s, %s}", hdr, dst_info.buffer, dst_info.count, ucc_datatype_str(dst_info.datatype), ucc_mem_type_str(dst_info.mem_type)); } else { ucc_snprintf_safe(str, len, "%s", hdr); } } void ucc_coll_task_components_str(const ucc_coll_task_t *task, char *str, size_t *len) { ucc_schedule_t *schedule; ucc_schedule_pipelined_t *schedule_pipelined; int i; if (task->flags & UCC_COLL_TASK_FLAG_IS_PIPELINED_SCHEDULE) { schedule_pipelined = ucc_derived_of(task, ucc_schedule_pipelined_t); for (i = 0; i < schedule_pipelined->n_frags; i++) { ucc_coll_task_components_str(&schedule_pipelined->frags[i]->super, str, len); } } else if (task->flags & UCC_COLL_TASK_FLAG_IS_SCHEDULE) { schedule = ucc_derived_of(task, ucc_schedule_t); for (i = 0; i < schedule->n_tasks; i++) { ucc_coll_task_components_str(schedule->tasks[i], str, len); } } else { if (!strstr(str, task->team->context->lib->log_component.name)) { if (*len == 0) { sprintf(str + *len, "%s", task->team->context->lib->log_component.name); *len = strlen(task->team->context->lib->log_component.name) + *len; } else { sprintf(str + *len, ", %s", task->team->context->lib->log_component.name); *len = strlen(task->team->context->lib->log_component.name) + *len + 2; } } } } void ucc_coll_str(const ucc_coll_task_t *task, char *str, size_t len, int verbosity) { ucc_team_t *team = task->bargs.team; int rc; if (verbosity >= UCC_LOG_LEVEL_DIAG) { ucc_coll_args_str(&task->bargs.args, team->rank, team->size, str, len); } if (verbosity >= UCC_LOG_LEVEL_INFO) { size_t tl_info_len = 0; char task_info[64], cl_info[16], tl_info[32]; if (!task->team) { /* zero size collective, no CL or TL */ strncpy(cl_info, "NoOp", sizeof(cl_info)); strncpy(tl_info, "NoOp", sizeof(tl_info)); } else if (task->team->context->lib->log_component.name[0] == 'C') { /* it's not CL BASIC task */ ucc_strncpy_safe(cl_info, task->team->context->lib->log_component.name, sizeof(cl_info)); ucc_coll_task_components_str(task, tl_info, &tl_info_len); } else { ucc_strncpy_safe(cl_info, "CL_BASIC", sizeof(cl_info)); ucc_strncpy_safe(tl_info, task->team->context->lib->log_component.name, sizeof(tl_info)); } ucc_coll_args_str(&task->bargs.args, team->rank, team->size, str, len); rc = ucc_snprintf_safe(task_info, sizeof(task_info), "; %s {%s}, team_id %d", cl_info, tl_info, team->id); if (rc < 0) { return; } strncat(str, task_info, len - strlen(str)); } if (verbosity >= UCC_LOG_LEVEL_DEBUG) { char task_info[64]; ucc_snprintf_safe(task_info, sizeof(task_info), " rank %u, ctx_rank %u, seq_num %d, req %p", team->rank, ucc_ep_map_eval(team->ctx_map, team->rank), task->seq_num, task); strncat(str, task_info, len - strlen(str)); } } typedef struct ucc_ep_map_nested { ucc_ep_map_t *base_map; ucc_ep_map_t *sub_map; } ucc_ep_map_nested_t; uint64_t ucc_ep_map_nested_cb(uint64_t ep, void *cb_ctx) { ucc_ep_map_nested_t *nested = cb_ctx; ucc_rank_t r; r = ucc_ep_map_eval(*nested->sub_map, (ucc_rank_t)ep); return (uint64_t)ucc_ep_map_eval(*nested->base_map, r); } ucc_status_t ucc_ep_map_create_nested(ucc_ep_map_t *base_map, ucc_ep_map_t *sub_map, ucc_ep_map_t *out) { ucc_ep_map_nested_t *nested; nested = ucc_malloc(sizeof(*nested), "nested_map"); if (ucc_unlikely(!nested)) { ucc_error("failed to allocate %zd bytes for nested map", sizeof(*nested)); return UCC_ERR_NO_MEMORY; } nested->base_map = base_map; nested->sub_map = sub_map; out->type = UCC_EP_MAP_CB; out->ep_num = sub_map->ep_num; out->cb.cb = ucc_ep_map_nested_cb; out->cb.cb_ctx = nested; return UCC_OK; } void ucc_ep_map_destroy_nested(ucc_ep_map_t *out) { ucc_free(out->cb.cb_ctx); } ucc_ep_map_t ucc_ep_map_create_reverse(ucc_rank_t size) { ucc_ep_map_t map = {.type = UCC_EP_MAP_STRIDED, .ep_num = size, .strided.start = size - 1, .strided.stride = -1}; return map; } int ucc_ep_map_is_identity(const ucc_ep_map_t *map) { if ((map->type == UCC_EP_MAP_FULL) || ((map->type == UCC_EP_MAP_STRIDED) && (map->strided.start == 0) && (map->strided.stride == 1))) { return 1; } else { return 0; } } static inline int ucc_ep_map_is_reverse(ucc_ep_map_t *map, int reversed_reordered_flag) { return (((map->type == UCC_EP_MAP_STRIDED) && (map->strided.start == map->ep_num - 1) && (map->strided.stride == -1)) || reversed_reordered_flag); } ucc_status_t ucc_ep_map_create_inverse(ucc_ep_map_t map, ucc_ep_map_t *inv_map, int reversed_reordered_flag) { ucc_ep_map_t inv; ucc_rank_t i, r; ucc_rank_t max_rank; if (ucc_ep_map_is_reverse(&map, reversed_reordered_flag)) { inv = map; } else { inv.type = UCC_EP_MAP_ARRAY; inv.ep_num = map.ep_num; inv.array.elem_size = sizeof(ucc_rank_t); max_rank = 0; for (i = 0; i < map.ep_num; i++) { r = (ucc_rank_t)ucc_ep_map_eval(map, i); if (r > max_rank) { max_rank = r; } } inv.array.map = ucc_malloc(sizeof(ucc_rank_t) * (max_rank + 1), "inv_map"); if (!inv.array.map) { ucc_error("failed to allocate %zd bytes for inv map\n", sizeof(ucc_rank_t) * map.ep_num); return UCC_ERR_NO_MEMORY; } for (i = 0; i < map.ep_num; i++) { r = (ucc_rank_t)ucc_ep_map_eval(map, i); *((ucc_rank_t *)PTR_OFFSET(inv.array.map, sizeof(ucc_rank_t) * r)) = i; } } *inv_map = inv; return UCC_OK; } void ucc_ep_map_destroy(ucc_ep_map_t *map) { if (map->type == UCC_EP_MAP_ARRAY) { ucc_free(map->array.map); } } ucc-1.8.0/src/utils/ucc_assert.c0000664000175000017500000000224015211535620016702 0ustar alastairalastair/** * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #include "ucc_assert.h" #include "ucc_log.h" #include "ucc_string.h" #include void ucc_fatal_error_message(const char *file, unsigned line, const char *function, char *message_buf) //NOLINT: function is unused { char *message_line, *save_ptr = NULL; ucc_log_flush(); message_line = (message_buf == NULL) ? NULL : strtok_r(message_buf, "\n", &save_ptr); while (message_line != NULL) { ucc_log_fatal_error("%13s:%-4u %s", ucc_basename(file), line, message_line); message_line = strtok_r(NULL, "\n", &save_ptr); } abort(); } void ucc_fatal_error_format(const char *file, unsigned line, const char *function, const char *format, ...) { const size_t buffer_size = ucc_log_get_buffer_size(); char *buffer; va_list ap; buffer = alloca(buffer_size); va_start(ap, format); //NOLINTNEXTLINE vsnprintf(buffer, buffer_size, format, ap); va_end(ap); ucc_fatal_error_message(file, line, function, buffer); } ucc-1.8.0/src/utils/ucc_mpool.h0000664000175000017500000000376015211535620016544 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_MPOOL_H_ #define UCC_MPOOL_H_ #include "config.h" #include "ucc/api/ucc.h" #include #include "ucc_compiler_def.h" #include "ucc_spinlock.h" typedef struct ucc_mpool ucc_mpool_t; typedef struct ucc_mpool_ops { ucc_status_t (*chunk_alloc)(ucc_mpool_t *mp, size_t *size_p, void **chunk_p); void (*chunk_release)(ucc_mpool_t *mp, void *chunk); void (*obj_init)(ucc_mpool_t *mp, void *obj, void *chunk); void (*obj_cleanup)(ucc_mpool_t *mp, void *obj); } ucc_mpool_ops_t; struct ucc_mpool { ucs_mpool_t super; ucc_mpool_ops_t * ucc_ops; ucc_thread_mode_t tm; ucc_spinlock_t lock; }; ucc_status_t ucc_mpool_init(ucc_mpool_t *mp, size_t priv_size, size_t elem_size, size_t align_offset, size_t alignment, unsigned elems_per_chunk, unsigned max_elems, ucc_mpool_ops_t *ops, ucc_thread_mode_t tm, const char *name); void ucc_mpool_cleanup(ucc_mpool_t *mp, int leak_check); ucc_status_t ucc_mpool_hugetlb_malloc(ucc_mpool_t *mp, size_t *size_p, void **chunk_p); void ucc_mpool_hugetlb_free(ucc_mpool_t *mp, void *chunk); static inline void *ucc_mpool_get(ucc_mpool_t *mp) { void *ret; if (UCC_THREAD_SINGLE == mp->tm) { return ucs_mpool_get(&mp->super); } ucc_spin_lock(&mp->lock); ret = ucs_mpool_get(&mp->super); ucc_spin_unlock(&mp->lock); return ret; } static inline void ucc_mpool_put(void *obj) { ucs_mpool_elem_t *elem = (ucs_mpool_elem_t *)obj - 1; ucc_mpool_t * mp = ucc_derived_of(elem->mpool, ucc_mpool_t); if (UCC_THREAD_SINGLE == mp->tm) { ucs_mpool_put(obj); return; } ucc_spin_lock(&mp->lock); ucs_mpool_put(obj); ucc_spin_unlock(&mp->lock); } #endif ucc-1.8.0/src/utils/ucc_math_op.h0000664000175000017500000001245015211535620017041 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_MATH_OP_H_ #define UCC_MATH_OP_H_ #include "ucc_math.h" #define DO_OP_2(_op, _v1, _v2) (_v1 _op _v2) #define DO_OP_3(_op, _v1, _v2, _v3) (_v1 _op _v2 _op _v3) #define DO_OP_4(_op, _v1, _v2, _v3, _v4) (_v1 _op _v2 _op _v3 _op _v4) #define DO_OP_5(_op, _v1, _v2, _v3, _v4, _v5) \ (_v1 _op _v2 _op _v3 _op _v4 _op _v5) #define DO_OP_6(_op, _v1, _v2, _v3, _v4, _v5, _v6) \ (_v1 _op _v2 _op _v3 _op _v4 _op _v5 _op _v6) #define DO_OP_7(_op, _v1, _v2, _v3, _v4, _v5, _v6, _v7) \ (_v1 _op _v2 _op _v3 _op _v4 _op _v5 _op _v6 _op _v7) #define DO_OP_8(_op, _v1, _v2, _v3, _v4, _v5, _v6, _v7, _v8) \ (_v1 _op _v2 _op _v3 _op _v4 _op _v5 _op _v6 _op _v7 _op _v8) #define DO_OP_SUM_2(...) DO_OP_2(+, __VA_ARGS__) #define DO_OP_SUM_3(...) DO_OP_3(+, __VA_ARGS__) #define DO_OP_SUM_4(...) DO_OP_4(+, __VA_ARGS__) #define DO_OP_SUM_5(...) DO_OP_5(+, __VA_ARGS__) #define DO_OP_SUM_6(...) DO_OP_6(+, __VA_ARGS__) #define DO_OP_SUM_7(...) DO_OP_7(+, __VA_ARGS__) #define DO_OP_SUM_8(...) DO_OP_8(+, __VA_ARGS__) #define DO_OP_PROD_2(...) DO_OP_2(*, __VA_ARGS__) #define DO_OP_PROD_3(...) DO_OP_3(*, __VA_ARGS__) #define DO_OP_PROD_4(...) DO_OP_4(*, __VA_ARGS__) #define DO_OP_PROD_5(...) DO_OP_5(*, __VA_ARGS__) #define DO_OP_PROD_6(...) DO_OP_6(*, __VA_ARGS__) #define DO_OP_PROD_7(...) DO_OP_7(*, __VA_ARGS__) #define DO_OP_PROD_8(...) DO_OP_8(*, __VA_ARGS__) #define DO_OP_LAND_2(...) DO_OP_2(&&, __VA_ARGS__) #define DO_OP_LAND_3(...) DO_OP_3(&&, __VA_ARGS__) #define DO_OP_LAND_4(...) DO_OP_4(&&, __VA_ARGS__) #define DO_OP_LAND_5(...) DO_OP_5(&&, __VA_ARGS__) #define DO_OP_LAND_6(...) DO_OP_6(&&, __VA_ARGS__) #define DO_OP_LAND_7(...) DO_OP_7(&&, __VA_ARGS__) #define DO_OP_LAND_8(...) DO_OP_8(&&, __VA_ARGS__) #define DO_OP_BAND_2(...) DO_OP_2(&, __VA_ARGS__) #define DO_OP_BAND_3(...) DO_OP_3(&, __VA_ARGS__) #define DO_OP_BAND_4(...) DO_OP_4(&, __VA_ARGS__) #define DO_OP_BAND_5(...) DO_OP_5(&, __VA_ARGS__) #define DO_OP_BAND_6(...) DO_OP_6(&, __VA_ARGS__) #define DO_OP_BAND_7(...) DO_OP_7(&, __VA_ARGS__) #define DO_OP_BAND_8(...) DO_OP_8(&, __VA_ARGS__) #define DO_OP_LOR_2(...) DO_OP_2(||, __VA_ARGS__) #define DO_OP_LOR_3(...) DO_OP_3(||, __VA_ARGS__) #define DO_OP_LOR_4(...) DO_OP_4(||, __VA_ARGS__) #define DO_OP_LOR_5(...) DO_OP_5(||, __VA_ARGS__) #define DO_OP_LOR_6(...) DO_OP_6(||, __VA_ARGS__) #define DO_OP_LOR_7(...) DO_OP_7(||, __VA_ARGS__) #define DO_OP_LOR_8(...) DO_OP_8(||, __VA_ARGS__) #define DO_OP_BOR_2(...) DO_OP_2(|, __VA_ARGS__) #define DO_OP_BOR_3(...) DO_OP_3(|, __VA_ARGS__) #define DO_OP_BOR_4(...) DO_OP_4(|, __VA_ARGS__) #define DO_OP_BOR_5(...) DO_OP_5(|, __VA_ARGS__) #define DO_OP_BOR_6(...) DO_OP_6(|, __VA_ARGS__) #define DO_OP_BOR_7(...) DO_OP_7(|, __VA_ARGS__) #define DO_OP_BOR_8(...) DO_OP_8(|, __VA_ARGS__) #define DO_OP_BXOR_2(...) DO_OP_2(^, __VA_ARGS__) #define DO_OP_BXOR_3(...) DO_OP_3(^, __VA_ARGS__) #define DO_OP_BXOR_4(...) DO_OP_4(^, __VA_ARGS__) #define DO_OP_BXOR_5(...) DO_OP_5(^, __VA_ARGS__) #define DO_OP_BXOR_6(...) DO_OP_6(^, __VA_ARGS__) #define DO_OP_BXOR_7(...) DO_OP_7(^, __VA_ARGS__) #define DO_OP_BXOR_8(...) DO_OP_8(^, __VA_ARGS__) #define DO_OP__3(_OP, _v1, _v2, _v3) _OP(_OP(_v1, _v2), _v3) #define DO_OP__4(_OP, _v1, _v2, _v3, _v4) _OP(_OP(_v1, _v2), _OP(_v3, _v4)) #define DO_OP__5(_OP, _v1, _v2, _v3, _v4, _v5) \ _OP(_OP(_v1, _v2), DO_OP__3(_OP, _v3, _v4, _v5)) #define DO_OP__6(_OP, _v1, _v2, _v3, _v4, _v5, _v6) \ _OP(DO_OP__3(_OP, _v1, _v2, _v3), DO_OP__3(_OP, _v4, _v5, _v6)) #define DO_OP__7(_OP, _v1, _v2, _v3, _v4, _v5, _v6, _v7) \ _OP(DO_OP__3(_OP, _v1, _v2, _v3), DO_OP__4(_OP, _v4, _v5, _v6, _v7)) #define DO_OP__8(_OP, _v1, _v2, _v3, _v4, _v5, _v6, _v7, _v8) \ _OP(DO_OP__4(_OP, _v1, _v2, _v3, _v4), DO_OP__4(_OP, _v5, _v6, _v7, _v8)) #define DO_OP_MAX_2(_v1, _v2) DO_OP_MAX(_v1, _v2) #define DO_OP_MAX_3(...) DO_OP__3(DO_OP_MAX, __VA_ARGS__) #define DO_OP_MAX_4(...) DO_OP__4(DO_OP_MAX, __VA_ARGS__) #define DO_OP_MAX_5(...) DO_OP__5(DO_OP_MAX, __VA_ARGS__) #define DO_OP_MAX_6(...) DO_OP__6(DO_OP_MAX, __VA_ARGS__) #define DO_OP_MAX_7(...) DO_OP__7(DO_OP_MAX, __VA_ARGS__) #define DO_OP_MAX_8(...) DO_OP__8(DO_OP_MAX, __VA_ARGS__) #define DO_OP_MIN_2(_v1, _v2) DO_OP_MIN(_v1, _v2) #define DO_OP_MIN_3(...) DO_OP__3(DO_OP_MIN, __VA_ARGS__) #define DO_OP_MIN_4(...) DO_OP__4(DO_OP_MIN, __VA_ARGS__) #define DO_OP_MIN_5(...) DO_OP__5(DO_OP_MIN, __VA_ARGS__) #define DO_OP_MIN_6(...) DO_OP__6(DO_OP_MIN, __VA_ARGS__) #define DO_OP_MIN_7(...) DO_OP__7(DO_OP_MIN, __VA_ARGS__) #define DO_OP_MIN_8(...) DO_OP__8(DO_OP_MIN, __VA_ARGS__) #define DO_OP_LXOR_2(_v1, _v2) DO_OP_LXOR(_v1, _v2) #define DO_OP_LXOR_3(...) DO_OP__3(DO_OP_LXOR, __VA_ARGS__) #define DO_OP_LXOR_4(...) DO_OP__4(DO_OP_LXOR, __VA_ARGS__) #define DO_OP_LXOR_5(...) DO_OP__5(DO_OP_LXOR, __VA_ARGS__) #define DO_OP_LXOR_6(...) DO_OP__6(DO_OP_LXOR, __VA_ARGS__) #define DO_OP_LXOR_7(...) DO_OP__7(DO_OP_LXOR, __VA_ARGS__) #define DO_OP_LXOR_8(...) DO_OP__8(DO_OP_LXOR, __VA_ARGS__) #endif ucc-1.8.0/src/utils/ucc_coll_utils.h0000664000175000017500000003733015211535620017567 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_COLL_UTILS_H_ #define UCC_COLL_UTILS_H_ #include "config.h" #include "ucc_datastruct.h" #include "ucc_math.h" #include "utils/ucc_time.h" #include "utils/ucc_assert.h" #include #define UCC_COLL_TYPE_NUM (ucc_ilog2(UCC_COLL_TYPE_LAST - 1) + 1) #define UCC_COLL_TYPE_ALL ((UCC_COLL_TYPE_LAST << 1) - 3) #define UCC_MSG_SIZE_INVALID SIZE_MAX #define UCC_MSG_SIZE_ASYMMETRIC (UCC_MSG_SIZE_INVALID - 1) #define UCC_IS_INPLACE(_args) \ (((_args).mask & UCC_COLL_ARGS_FIELD_FLAGS) && \ ((_args).flags & UCC_COLL_ARGS_FLAG_IN_PLACE)) #define UCC_IS_PERSISTENT(_args) \ (((_args).mask & UCC_COLL_ARGS_FIELD_FLAGS) && \ ((_args).flags & UCC_COLL_ARGS_FLAG_PERSISTENT)) #define UCC_IS_ROOT(_args, _myrank) ((_args).root == (_myrank)) #define UCC_COLL_TIMEOUT_REQUIRED(_task) \ (((_task)->bargs.args.mask & UCC_COLL_ARGS_FIELD_FLAGS) && \ ((_task)->bargs.args.flags & UCC_COLL_ARGS_FLAG_TIMEOUT)) #define UCC_COLL_SET_TIMEOUT(_task, _timeout) do { \ (_task)->bargs.args.mask |= UCC_COLL_ARGS_FIELD_FLAGS; \ (_task)->bargs.args.flags |= UCC_COLL_ARGS_FLAG_TIMEOUT; \ (_task)->bargs.args.timeout = _timeout; \ (_task)->start_time = ucc_get_time(); \ } while(0) #define UCC_COLL_ARGS_COUNT64(_args) \ (((_args)->mask & UCC_COLL_ARGS_FIELD_FLAGS) && \ ((_args)->flags & UCC_COLL_ARGS_FLAG_COUNT_64BIT)) #define UCC_COLL_ARGS_DISPL64(_args) \ (((_args)->mask & UCC_COLL_ARGS_FIELD_FLAGS) && \ ((_args)->flags & UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT)) #define UCC_COLL_IS_SRC_CONTIG(_args) \ (((_args)->mask & UCC_COLL_ARGS_FIELD_FLAGS) && \ ((_args)->flags & UCC_COLL_ARGS_FLAG_CONTIG_SRC_BUFFER)) #define UCC_COLL_IS_DST_CONTIG(_args) \ (((_args)->mask & UCC_COLL_ARGS_FIELD_FLAGS) && \ ((_args)->flags & UCC_COLL_ARGS_FLAG_CONTIG_DST_BUFFER)) #define UCC_COLL_ARGS_CONTIG_BUFFER(_args) \ (UCC_COLL_IS_SRC_CONTIG(_args) && UCC_COLL_IS_DST_CONTIG(_args)) #define UCC_COLL_ARGS_ACTIVE_SET(_args) \ ((_args)->mask & UCC_COLL_ARGS_FIELD_ACTIVE_SET) static inline int ucc_coll_args_is_reduction(ucc_coll_type_t ct) { if (ct == UCC_COLL_TYPE_ALLREDUCE || ct == UCC_COLL_TYPE_REDUCE || ct == UCC_COLL_TYPE_REDUCE_SCATTER || ct == UCC_COLL_TYPE_REDUCE_SCATTERV) { return 1; } return 0; } static inline size_t ucc_coll_args_get_count(const ucc_coll_args_t *args, const ucc_count_t *counts, ucc_rank_t idx) { if (UCC_COLL_ARGS_COUNT64(args)) { return ((uint64_t *)counts)[idx]; } return ((uint32_t *)counts)[idx]; } static inline void ucc_coll_args_set_count(const ucc_coll_args_t *args, const ucc_count_t *counts, ucc_rank_t idx, size_t val) { if (UCC_COLL_ARGS_COUNT64(args)) { ((uint64_t *)counts)[idx] = (uint64_t)val; } else { ((uint32_t *)counts)[idx] = (uint32_t)val; } } static inline size_t ucc_coll_args_get_max_count(const ucc_coll_args_t *args, const ucc_count_t * counts, ucc_rank_t size) { size_t max_count = 0, c; int i; for (i = 0; i < size; i++) { c = ucc_coll_args_get_count(args, counts, i); if (c > max_count) { max_count = c; } } return max_count; } static inline size_t ucc_coll_args_get_displacement(const ucc_coll_args_t *args, const ucc_aint_t *displacements, ucc_rank_t idx) { if (UCC_COLL_ARGS_DISPL64(args)) { return ((uint64_t *)displacements)[idx]; } return ((uint32_t *)displacements)[idx]; } static inline void ucc_coll_args_set_displacement(const ucc_coll_args_t *args, const ucc_aint_t *displacements, ucc_rank_t idx, size_t val) { if (UCC_COLL_ARGS_DISPL64(args)) { ((uint64_t *)displacements)[idx] = (uint64_t)val; } else { ((uint32_t *)displacements)[idx] = (uint32_t)val; } } static inline size_t ucc_coll_args_get_total_count(const ucc_coll_args_t *args, const ucc_count_t *counts, ucc_rank_t size) { size_t count = 0; ucc_rank_t i; // TODO switch to base args and cache total count there - can we do it ? if ((args->mask & UCC_COLL_ARGS_FIELD_FLAGS) && (args->flags & UCC_COLL_ARGS_FLAG_COUNT_64BIT)) { for (i = 0; i < size; i++) { count += ((uint64_t *)counts)[i]; } } else { for (i = 0; i < size; i++) { count += ((uint32_t *)counts)[i]; } } return count; } /* Check if the displacements are contig, whether or not the user passed UCC_COLL_ARGS_FLAG_CONTIG_DST_BUFFER in coll args */ static inline int ucc_coll_args_is_disp_contig(const ucc_coll_args_t *args, ucc_rank_t size) { size_t count_accumulator = 0; size_t disps; ucc_rank_t i; if (!UCC_COLL_IS_DST_CONTIG(args)) { for (i = 0; i < size; i++) { disps = ucc_coll_args_get_displacement( args, args->dst.info_v.displacements, i); if (disps != count_accumulator) { return 0; } count_accumulator += ucc_coll_args_get_count( args, args->dst.info_v.counts, i); } } return 1; } static inline size_t ucc_coll_args_get_v_buffer_size(const ucc_coll_args_t *args, const ucc_count_t *counts, const ucc_aint_t *displacements, ucc_rank_t size) { ucc_rank_t i; size_t max_disp, idx_count; max_disp = ucc_coll_args_get_displacement(args, displacements, 0); idx_count = ucc_coll_args_get_count(args, counts, 0); for (i = 1; i < size; i++) { size_t disp_i = ucc_coll_args_get_displacement(args, displacements, i); if (disp_i > max_disp) { max_disp = disp_i; idx_count = ucc_coll_args_get_count(args, counts, i); } } return max_disp + idx_count; } typedef struct ucc_base_coll_args ucc_base_coll_args_t; ucc_coll_type_t ucc_coll_type_from_str(const char *str); ucc_memory_type_t ucc_mem_type_from_str(const char *str); size_t ucc_coll_args_msgsize(const ucc_coll_args_t *args, ucc_rank_t rank, ucc_rank_t size); ucc_memory_type_t ucc_coll_args_mem_type(const ucc_coll_args_t *args, ucc_rank_t rank); /* Convert rank from subset space to rank space (UCC team space) */ static inline ucc_rank_t ucc_ep_map_eval(ucc_ep_map_t map, ucc_rank_t rank) { ucc_rank_t r; switch(map.type) { case UCC_EP_MAP_FULL: r = rank; break; case UCC_EP_MAP_STRIDED: r = map.strided.start + rank*map.strided.stride; break; case UCC_EP_MAP_ARRAY: r = *((ucc_rank_t*)((ptrdiff_t)map.array.map + rank*map.array.elem_size)); break; case UCC_EP_MAP_CB: r = (ucc_rank_t)map.cb.cb(rank, map.cb.cb_ctx); break; default: r = UCC_RANK_INVALID; } return r; } /* Builds ucc_ep_map_t from the array of ucc_rank_t. The routine tries to search for a strided pattern to optimize storage and map lookup. @param [in] array pointer to the array to build the map from @param [in] size size of the array @param [in] full_size if size == full_size and stride=1 is detected the map can be optimized to be FULL @param [in] need_free if set to 1 the input @array is freed and set to NULL in the case of strided pattern. User must check and free the array otherwise. */ ucc_ep_map_t ucc_ep_map_from_array(ucc_rank_t **array, ucc_rank_t size, ucc_rank_t full_size, int need_free); /* Builds ucc_ep_map_t from the array of uint64_t. The routine tries to search for a strided pattern to optimize storage and map lookup. @param [in] array pointer to the array to build the map from @param [in] size size of the array @param [in] full_size if size == full_size and stride=1 is detected the map can be optimized to be FULL @param [in] need_free if set to 1 the input @array is freed and set to NULL in the case of strided pattern. User must check and free the array otherwise. */ ucc_ep_map_t ucc_ep_map_from_array_64(uint64_t **array, ucc_rank_t size, ucc_rank_t full_size, int need_free); typedef struct ucc_coll_task ucc_coll_task_t; void ucc_coll_str(const ucc_coll_task_t *task, char *str, size_t len, int verbosity); void ucc_coll_args_str(const ucc_coll_args_t *args, ucc_rank_t trank, ucc_rank_t tsize, char *str, size_t len); /* Creates a rank map that reverses rank order, ie rank r -> size - 1 - r */ ucc_ep_map_t ucc_ep_map_create_reverse(ucc_rank_t size); /* Creates an inverse mapping for a given map, only if given map is not a reverse map or a reordered map within the reverse direction task. @param [in] map given map @param [in] inv_map output map @param [in] reversed_reordered_flag 1 if is reverse direction task and reorder ranks is configured as yes, 0 otherwise. */ ucc_status_t ucc_ep_map_create_inverse(ucc_ep_map_t map, ucc_ep_map_t *inv_map, int reversed_reordered_flag); ucc_status_t ucc_ep_map_create_nested(ucc_ep_map_t *base_map, ucc_ep_map_t *sub_map, ucc_ep_map_t *out); int ucc_ep_map_is_identity(const ucc_ep_map_t *map); void ucc_ep_map_destroy_nested(ucc_ep_map_t *out); void ucc_ep_map_destroy(ucc_ep_map_t *map); /* The two helper routines below are used to partition a buffer consisiting of total_count elements into blocks. This is used, e.g., in ReduceScatter or during fragmentation process. First total_count is devided into n_blocks. If the devision has remainder then it is evenly distributed among first blocks. */ static inline size_t ucc_buffer_block_count(size_t total_count, ucc_rank_t n_blocks, ucc_rank_t block) { size_t block_count = total_count / n_blocks; size_t left = total_count % n_blocks; return (block < left) ? block_count + 1 : block_count; } static inline size_t ucc_buffer_block_offset(size_t total_count, ucc_rank_t n_blocks, ucc_rank_t block) { size_t block_count = total_count / n_blocks; size_t left = total_count % n_blocks; size_t offset = block * block_count + left; return (block < left) ? offset - (left - block) : offset; } static inline size_t ucc_buffer_vector_block_offset(ucc_count_t *counts, int is64, ucc_rank_t rank) { size_t offset = 0; ucc_rank_t i; if (is64) { for (i = 0; i < rank; i++) { offset += ((uint64_t *)counts)[i]; } } else { for (i = 0; i < rank; i++) { offset += ((uint32_t *)counts)[i]; } } return offset; } /* Given the rank space A (e.g. core ucc team), a subset B (e.g. active set within the core team), the ep_map that maps ranks from the subset B to A, and the rank of a process within A. The function below computes the local rank of the process within subset B. i.e., convert from rank space (UCC team) to subset space */ static inline ucc_rank_t ucc_ep_map_local_rank(ucc_ep_map_t map, ucc_rank_t rank) { ucc_rank_t i, local_rank = UCC_RANK_INVALID; int64_t vrank; switch(map.type) { case UCC_EP_MAP_FULL: local_rank = rank; break; case UCC_EP_MAP_STRIDED: vrank = (int64_t)rank - (int64_t)map.strided.start; vrank /= map.strided.stride; ucc_assert(vrank >= 0 && vrank < map.ep_num); local_rank = (ucc_rank_t)vrank; break; case UCC_EP_MAP_ARRAY: case UCC_EP_MAP_CB: for (i = 0; i < map.ep_num; i++) { if (rank == ucc_ep_map_eval(map, i)) { local_rank = i; break; } } default: break; } return local_rank; } static inline ucc_ep_map_t ucc_active_set_to_ep_map(ucc_coll_args_t *args) { ucc_ep_map_t map; map.type = UCC_EP_MAP_STRIDED; map.ep_num = args->active_set.size; map.strided.start = args->active_set.start; map.strided.stride = args->active_set.stride; return map; } static inline size_t ucc_buffer_block_count_aligned(size_t total_count, ucc_rank_t n_blocks, ucc_rank_t block, int alignment) { size_t block_count = ucc_align_up_pow2(ucc_max(total_count / n_blocks, 1), alignment); size_t offset = block_count * block; return (total_count < offset) ? 0: ucc_min(total_count - offset, block_count); } static inline size_t ucc_buffer_block_offset_aligned(size_t total_count, ucc_rank_t n_blocks, ucc_rank_t block, int alignment) { size_t block_count = ucc_align_up_pow2(ucc_max(total_count / n_blocks, 1), alignment); size_t offset = block_count * block; return ucc_min(offset, total_count); } /* Returns non-zero if collective defined by args operates on predefined dt. @param [in] args pointer to the collective args. @param [in] rank rank to check, used only for rooted collective operations. */ int ucc_coll_args_is_predefined_dt(const ucc_coll_args_t *args, ucc_rank_t rank); int ucc_coll_args_is_mem_symmetric(const ucc_coll_args_t *args, ucc_rank_t rank); int ucc_coll_args_is_rooted(ucc_coll_type_t ct); typedef struct ucc_buffer_info_asymmetric_memtype ucc_buffer_info_asymmetric_memtype_t; typedef struct ucc_mc_buffer_header ucc_mc_buffer_header_t; ucc_status_t ucc_coll_args_init_asymmetric_buffer(ucc_coll_args_t *args, ucc_team_h team, ucc_buffer_info_asymmetric_memtype_t *save_info); ucc_status_t ucc_coll_args_free_asymmetric_buffer(ucc_coll_task_t *task); ucc_status_t ucc_copy_asymmetric_buffer(ucc_coll_task_t *task); #endif ucc-1.8.0/src/utils/ini.h0000664000175000017500000001260115211535620015335 0ustar alastairalastair/* inih -- simple .INI file parser SPDX-License-Identifier: BSD-3-Clause Copyright (C) 2009-2020, Ben Hoyt inih is released under the New BSD license (see LICENSE.txt). Go to the project home page for more info: https://github.com/benhoyt/inih */ #ifndef UCC_CONFIG_INI_H #define UCC_CONFIG_INI_H /* Make this header file easier to include in C++ code */ #ifdef __cplusplus extern "C" { #endif #include /* Nonzero if ini_handler callback should accept lineno parameter. */ #ifndef UCC_INI_HANDLER_LINENO #define UCC_INI_HANDLER_LINENO 0 #endif /* Typedef for prototype of handler function. */ #if UCC_INI_HANDLER_LINENO typedef int (*ini_handler)(void* user, const char* section, const char* name, const char* value, int lineno); #else typedef int (*ini_handler)(void* user, const char* section, const char* name, const char* value); #endif /* Typedef for prototype of fgets-style reader function. */ typedef char* (*ini_reader)(char* str, int num, void* stream); /* Parse given INI-style file. May have [section]s, name=value pairs (whitespace stripped), and comments starting with ';' (semicolon). Section is "" if name=value pair parsed before any section heading. name:value pairs are also supported as a concession to Python's configparser. For each name=value pair parsed, call handler function with given user pointer as well as section, name, and value (data only valid for duration of handler call). Handler should return nonzero on success, zero on error. Returns 0 on success, line number of first error on parse error (doesn't stop on first error), -1 on file open error, or -2 on memory allocation error (only when INI_USE_STACK is zero). */ int ucc_ini_parse(const char* filename, ini_handler handler, void* user); /* Same as ini_parse(), but takes a FILE* instead of filename. This doesn't close the file when it's finished -- the caller must do that. */ int ucc_ini_parse_file(FILE* file, ini_handler handler, void* user); /* Same as ini_parse(), but takes an ini_reader function pointer instead of filename. Used for implementing custom or string-based I/O (see also ini_parse_string). */ int ucc_ini_parse_stream(ini_reader reader, void* stream, ini_handler handler, void* user); /* Same as ini_parse(), but takes a zero-terminated string with the INI data instead of a file. Useful for parsing INI data from a network socket or already in memory. */ int ucc_ini_parse_string(const char* string, ini_handler handler, void* user); /* Nonzero to allow multi-line value parsing, in the style of Python's configparser. If allowed, ini_parse() will call the handler with the same name for each subsequent line parsed. */ #ifndef UCC_INI_ALLOW_MULTILINE #define UCC_INI_ALLOW_MULTILINE 1 #endif /* Nonzero to allow a UTF-8 BOM sequence (0xEF 0xBB 0xBF) at the start of the file. See https://github.com/benhoyt/inih/issues/21 */ #ifndef UCC_INI_ALLOW_BOM #define UCC_INI_ALLOW_BOM 1 #endif /* Chars that begin a start-of-line comment. Per Python configparser, allow both ; and # comments at the start of a line by default. */ #ifndef UCC_INI_START_COMMENT_PREFIXES #define UCC_INI_START_COMMENT_PREFIXES ";#" #endif /* Nonzero to allow inline comments (with valid inline comment characters specified by INI_INLINE_COMMENT_PREFIXES). Set to 0 to turn off and match Python 3.2+ configparser behaviour. */ #ifndef UCC_INI_ALLOW_INLINE_COMMENTS #define UCC_INI_ALLOW_INLINE_COMMENTS 1 #endif #ifndef UCC_INI_INLINE_COMMENT_PREFIXES #define UCC_INI_INLINE_COMMENT_PREFIXES ";" #endif /* Nonzero to use stack for line buffer, zero to use heap (malloc/free). */ #ifndef UCC_INI_USE_STACK #define UCC_INI_USE_STACK 1 #endif /* Maximum line length for any line in INI file (stack or heap). Note that this must be 3 more than the longest line (due to '\r', '\n', and '\0'). */ #ifndef UCC_INI_MAX_LINE #define UCC_INI_MAX_LINE 500 #endif /* Nonzero to allow heap line buffer to grow via realloc(), zero for a fixed-size buffer of INI_MAX_LINE bytes. Only applies if INI_USE_STACK is zero. */ #ifndef UCC_INI_ALLOW_REALLOC #define UCC_INI_ALLOW_REALLOC 0 #endif /* Initial size in bytes for heap line buffer. Only applies if INI_USE_STACK is zero. */ #ifndef UCC_INI_INITIAL_ALLOC #define UCC_INI_INITIAL_ALLOC 200 #endif /* Stop parsing on first error (default is to keep parsing). */ #ifndef UCC_INI_STOP_ON_FIRST_ERROR #define UCC_INI_STOP_ON_FIRST_ERROR 0 #endif /* Nonzero to call the handler at the start of each new section (with name and value NULL). Default is to only call the handler on each name=value pair. */ #ifndef UCC_INI_CALL_HANDLER_ON_NEW_SECTION #define UCC_INI_CALL_HANDLER_ON_NEW_SECTION 0 #endif /* Nonzero to allow a name without a value (no '=' or ':' on the line) and call the handler with value NULL in this case. Default is to treat no-value lines as an error. */ #ifndef UCC_INI_ALLOW_NO_VALUE #define UCC_INI_ALLOW_NO_VALUE 0 #endif /* Nonzero to use custom ini_malloc, ini_free, and ini_realloc memory allocation functions (INI_USE_STACK must also be 0). These functions must have the same signatures as malloc/free/realloc and behave in a similar way. ini_realloc is only needed if INI_ALLOW_REALLOC is set. */ #ifndef UCC_INI_CUSTOM_ALLOCATOR #define UCC_INI_CUSTOM_ALLOCATOR 0 #endif #ifdef __cplusplus } #endif #endif /* UCC_CONFIG_INI_H */ ucc-1.8.0/src/utils/profile/0000775000175000017500000000000015211535620016045 5ustar alastairalastairucc-1.8.0/src/utils/profile/ucc_profile.c0000664000175000017500000000115315211535620020503 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_profile.h" #include "utils/ucc_compiler_def.h" ucs_profile_context_t *ucc_profile_ctx; ucc_status_t ucc_profile_init(unsigned profile_mode, const char *file_name, size_t max_file_size) { ucs_status_t status; status = ucs_profile_init(profile_mode, file_name, max_file_size, &ucc_profile_ctx); return ucs_status_to_ucc_status(status); } void ucc_profile_cleanup() { ucs_profile_cleanup(ucc_profile_ctx); } ucc-1.8.0/src/utils/profile/ucc_profile.h0000664000175000017500000000137215211535620020513 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_PROFILE_H_ #define UCC_PROFILE_H_ #include #include "ucc/api/ucc_status.h" #ifdef HAVE_PROFILING # include "ucc_profile_on.h" #else # include "ucc_profile_off.h" #endif /** * Initialize profiling system. * * @param [in] profile_mode Profiling mode. * @param [in] file_name Profiling file. * @param [in] max_file_size Limit for profiling log size. * * @return Status code. */ ucc_status_t ucc_profile_init(unsigned profile_mode, const char *file_name, size_t max_file_size); /** * Save and cleanup profiling. */ void ucc_profile_cleanup(); #endif ucc-1.8.0/src/utils/profile/ucc_profile_core.h0000664000175000017500000000025115211535620021516 0ustar alastairalastair#ifdef HAVE_PROFILING_CORE #include "utils/profile/ucc_profile.h" #else #include "utils/profile/ucc_profile_off.h" #endif #define UCC_CORE_PROFILE_FUNC UCC_PROFILE_FUNC ucc-1.8.0/src/utils/profile/ucc_profile_off.h0000664000175000017500000000131415211535620021341 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_PROFILE_OFF_H_ #define UCC_PROFILE_OFF_H_ #undef UCC_PROFILE_FUNC #undef UCC_PROFILE_REQUEST_NEW #undef UCC_PROFILE_REQUEST_EVENT #undef UCC_PROFILE_REQUEST_FREE #define UCC_PROFILE_FUNC(_ret_type, _name, _arglist, ...) _ret_type _name(__VA_ARGS__) #define UCC_PROFILE_FUNC_VOID(_name, _arglist, ...) void _name(__VA_ARGS__) #define UCC_PROFILE_REQUEST_NEW(...) UCS_EMPTY_STATEMENT #define UCC_PROFILE_REQUEST_EVENT(...) UCS_EMPTY_STATEMENT #define UCC_PROFILE_REQUEST_FREE(...) UCS_EMPTY_STATEMENT #endif ucc-1.8.0/src/utils/profile/ucc_profile_on.h0000664000175000017500000000637715211535620021221 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_PROFILE_ON_H_ #define UCC_PROFILE_ON_H_ #include "core/ucc_global_opts.h" #include extern ucs_profile_context_t *ucc_profile_ctx; #undef UCC_PROFILE_FUNC #undef UCC_PROFILE_FUNC_VOID #undef UCC_PROFILE_REQUEST_NEW #undef UCC_PROFILE_REQUEST_EVENT #undef UCC_PROFILE_REQUEST_FREE #ifdef UCS_PROFILE_LOC_ID_DISABLED /** * Create a profiled function. Uses default profile context. * * Usage: * UCC_PROFILE_FUNC(, , (a, b), int a, char b) * * @param _ret_type Function return type. * @param _name Function name. * @param _arglist List of argument *names* only. * @param ... Argument declarations (with types). */ #define UCC_PROFILE_FUNC(_ret_type, _name, _arglist, ...) \ UCS_PROFILE_CTX_FUNC_ALWAYS(ucc_profile_ctx, _ret_type, _name, _arglist, ## __VA_ARGS__) /** * Create a profiled function whose return type is void. Uses default profile * context. * * Usage: * UCC_PROFILE_FUNC_VOID(, (a, b), int a, char b) * * @param _name Function name. * @param _arglist List of argument *names* only. * @param ... Argument declarations (with types). */ #define UCC_PROFILE_FUNC_VOID(_name, _arglist, ...) \ UCS_PROFILE_CTX_FUNC_VOID_ALWAYS(ucc_profile_ctx, _name, _arglist, ## __VA_ARGS__) /* * Profile a new request allocation. * * @param _req Request pointer. * @param _name Allocation site name. * @param _param32 Custom 32-bit parameter. */ #define UCC_PROFILE_REQUEST_NEW(_req, _name, _param32) \ UCS_PROFILE_CTX_RECORD_ALWAYS(ucc_profile_ctx, UCS_PROFILE_TYPE_REQUEST_NEW, \ (_name), (_param32), (uintptr_t)(_req)); /* * Profile a request progress event. * * @param _req Request pointer. * @param _name Event name. * @param _param32 Custom 32-bit parameter. */ #define UCC_PROFILE_REQUEST_EVENT(_req, _name, _param32) \ UCS_PROFILE_CTX_RECORD_ALWAYS(ucc_profile_ctx, UCS_PROFILE_TYPE_REQUEST_EVENT, \ (_name), (_param32), (uintptr_t)(_req)); /* * Profile a request release. * * @param _req Request pointer. */ #define UCC_PROFILE_REQUEST_FREE(_req) \ UCS_PROFILE_CTX_RECORD_ALWAYS(ucc_profile_ctx, UCS_PROFILE_TYPE_REQUEST_FREE, \ "", 0, (uintptr_t)(_req)); #else #define UCC_PROFILE_FUNC(_ret_type, _name, _arglist, ...) \ _UCS_PROFILE_CTX_FUNC(ucc_profile_ctx, _ret_type, _name, _arglist, ## __VA_ARGS__) #define UCC_PROFILE_FUNC_VOID(_name, _arglist, ...) \ _UCS_PROFILE_CTX_FUNC_VOID(ucc_profile_ctx, _name, _arglist, ## __VA_ARGS__) #define UCC_PROFILE_REQUEST_NEW(_req, _name, _param32) \ UCS_PROFILE_CTX_RECORD(ucc_profile_ctx, UCS_PROFILE_TYPE_REQUEST_NEW, \ (_name), (_param32), (uintptr_t)(_req)); #define UCC_PROFILE_REQUEST_EVENT(_req, _name, _param32) \ UCS_PROFILE_CTX_RECORD(ucc_profile_ctx, UCS_PROFILE_TYPE_REQUEST_EVENT, \ (_name), (_param32), (uintptr_t)(_req)); #define UCC_PROFILE_REQUEST_FREE(_req) \ UCS_PROFILE_CTX_RECORD(ucc_profile_ctx, UCS_PROFILE_TYPE_REQUEST_FREE, \ "", 0, (uintptr_t)(_req)); #endif #endif ucc-1.8.0/src/utils/ucc_time.h0000664000175000017500000000073415211535620016352 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_TIME_H_ #define UCC_TIME_H_ #include "config.h" #include #define UCC_USEC_PER_SEC 1000000ul /* Micro */ /** * @return The current accurate time, in seconds. */ static inline double ucc_get_time() { struct timeval tv; gettimeofday(&tv, NULL); return tv.tv_sec + (tv.tv_usec / (double)UCC_USEC_PER_SEC); } #endif ucc-1.8.0/src/utils/ucc_dt_reduce.h0000664000175000017500000001017415211535620017351 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_DT_REDUCE_H_ #define UCC_DT_REDUCE_H_ #include "components/ec/base/ucc_ec_base.h" #include "components/mc/ucc_mc.h" #include "components/ec/ucc_ec.h" static inline ucc_status_t ucc_dt_reduce_userdefined(void *src1, void *src2, void *dst, size_t n_vectors, size_t count, size_t stride, ucc_dt_generic_t *dt) { ucc_reduce_cb_params_t params = {.mask = 0, .src1 = src1, .src2 = src2, .dst = dst, .n_vectors = n_vectors, .count = count, .stride = stride, .dt = dt}; return dt->ops.reduce.cb(¶ms); } static inline ucc_status_t ucc_dt_reduce_strided(void *src1, void *src2, void *dst, size_t n_vectors, size_t count, size_t stride, ucc_datatype_t dt, ucc_coll_args_t *args, uint16_t flags, double alpha, ucc_ee_executor_t *exec, ucc_ee_executor_task_t **task) { ucc_ee_executor_task_args_t eargs; if (count == 0 || n_vectors == 0) { *task = NULL; return UCC_OK; } if (!UCC_DT_IS_PREDEFINED(dt)) { ucc_assert(UCC_DT_HAS_REDUCE(dt)); *task = NULL; return ucc_dt_reduce_userdefined(src1, src2, dst, n_vectors, count, stride, ucc_dt_to_generic(dt)); } else { eargs.flags = flags; eargs.task_type = UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED; eargs.reduce_strided.count = count; eargs.reduce_strided.dt = dt; eargs.reduce_strided.op = args->op; eargs.reduce_strided.n_src2 = n_vectors; eargs.reduce_strided.dst = dst; eargs.reduce_strided.src1 = src1; eargs.reduce_strided.src2 = src2; eargs.reduce_strided.stride = stride; eargs.reduce_strided.alpha = alpha; return ucc_ee_executor_task_post(exec, &eargs, task); } } static inline ucc_status_t ucc_dt_reduce(void *src1, void *src2, void *dst, size_t count, ucc_datatype_t dt, ucc_coll_args_t *args, uint16_t flags, double alpha, ucc_ee_executor_t *exec, ucc_ee_executor_task_t **task) { return ucc_dt_reduce_strided(src1, src2, dst, 1, count, 0, dt, args, flags, alpha, exec, task); } static inline ucc_status_t ucc_dt_reduce_multi(void **srcs, void *dst, size_t n_srcs, size_t count, ucc_datatype_t dt, ucc_coll_args_t *args, uint16_t flags, double alpha, ucc_ee_executor_t *exec, ucc_ee_executor_task_t **task) { ucc_ee_executor_task_args_t eargs; ucc_assert(n_srcs <= UCC_EE_EXECUTOR_NUM_BUFS); if (count == 0 || n_srcs == 0) { *task = NULL; return UCC_OK; } if (!UCC_DT_IS_PREDEFINED(dt)) { ucc_assert(UCC_DT_HAS_REDUCE(dt)); *task = NULL; return ucc_dt_reduce_userdefined(srcs, NULL, dst, n_srcs, count, 0, ucc_dt_to_generic(dt)); } else { eargs.task_type = UCC_EE_EXECUTOR_TASK_REDUCE; eargs.flags = flags; eargs.reduce.alpha = alpha; eargs.reduce.count = count; eargs.reduce.dt = dt; eargs.reduce.op = args->op; eargs.reduce.dst = dst; eargs.reduce.n_srcs = n_srcs; for (size_t i = 0; i < n_srcs; i++) { eargs.reduce.srcs[i] = srcs[i]; } return ucc_ee_executor_task_post(exec, &eargs, task); } } #endif ucc-1.8.0/src/utils/ucc_datastruct.h0000664000175000017500000000270415211535620017571 0ustar alastairalastair/** * Copyright (c) 2001-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_DATASTRUCT_H_ #define UCC_DATASTRUCT_H_ #include #include "ucc_list.h" #include #define UCC_LIST_HEAD UCS_LIST_HEAD typedef uint32_t ucc_rank_t; #define UCC_RANK_INVALID UINT32_MAX #define UCC_RANK_MAX UCC_RANK_INVALID - 1 typedef struct ucc_subset { ucc_ep_map_t map; ucc_rank_t myrank; } ucc_subset_t; static inline ucc_rank_t ucc_subset_size(const ucc_subset_t *set) { return (ucc_rank_t)set->map.ep_num; } typedef struct ucc_mrange { ucc_list_link_t list_elem; size_t start; size_t end; uint32_t mtypes; unsigned value; } ucc_mrange_t; typedef struct ucc_mrange_uint { ucc_list_link_t ranges; unsigned default_value; } ucc_mrange_uint_t; ucc_status_t ucc_mrange_uint_copy(ucc_mrange_uint_t *dst, const ucc_mrange_uint_t *src); void ucc_mrange_uint_destroy(ucc_mrange_uint_t *param); static inline unsigned ucc_mrange_uint_get( ucc_mrange_uint_t *param, size_t range_value, ucc_memory_type_t mem_type) { ucc_mrange_t *r; ucc_list_for_each(r, ¶m->ranges, list_elem) { if (r->start <= range_value && range_value <= r->end && (UCC_BIT(mem_type) & r->mtypes)) { return r->value; } } return param->default_value; } #endif ucc-1.8.0/src/utils/ucc_class.h0000664000175000017500000001377215211535620016527 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_CLASS_H_ #define UCC_CLASS_H_ #include "config.h" #include #include "utils/ucc_compiler_def.h" #define UCC_CLASS_DEFINE UCS_CLASS_DEFINE #define UCC_CLASS_DELETE UCS_CLASS_DELETE #define UCC_CLASS_CLEANUP_FUNC UCS_CLASS_CLEANUP_FUNC #define UCC_CLASS_CLEANUP UCS_CLASS_CLEANUP #define UCC_CLASS_NEW_FUNC_NAME UCS_CLASS_NEW_FUNC_NAME #define UCC_CLASS_INIT_FUNC(_type, ...) \ ucc_status_t _UCS_CLASS_INIT_NAME(_type)(_type *self, \ ucs_class_t *_myclass, \ int *_init_count, ## __VA_ARGS__) \ #define UCC_CLASS_DECLARE(_type, ...) \ extern ucs_class_t _UCS_CLASS_DECL_NAME(_type); \ UCC_CLASS_INIT_FUNC(_type, ## __VA_ARGS__); \ #define UCC_CLASS_INIT(_type, _obj, ...) \ ({ \ ucs_class_t *_cls = &_UCS_CLASS_DECL_NAME(_type); \ int _init_counter = 1; \ ucc_status_t __status; \ \ __status = _UCS_CLASS_INIT_NAME(_type)((_type*)(_obj), _cls, \ &_init_counter, ## __VA_ARGS__); \ if (__status != UCC_OK) { \ ucs_class_call_cleanup_chain(&_UCS_CLASS_DECL_NAME(_type), \ (_obj), _init_counter); \ } \ \ (__status); \ }) #define UCC_CLASS_NEW(_type, _obj, ...) \ _UCC_CLASS_NEW (_type, _obj, ## __VA_ARGS__) #define _UCC_CLASS_NEW(_type, _obj, ...) \ ({ \ ucs_class_t *cls = &_UCS_CLASS_DECL_NAME(_type); \ ucc_status_t _status; \ void *obj; \ \ obj = ucs_class_malloc(cls); \ if (obj != NULL) { \ _status = UCC_CLASS_INIT(_type, obj, ## __VA_ARGS__); \ if (_status == UCC_OK) { \ *(_obj) = (typeof(*(_obj)))obj; /* Success - assign pointer */ \ } else { \ ucs_class_free(obj); /* Initialization failure */ \ } \ } else { \ _status = UCC_ERR_NO_MEMORY; /* Allocation failure */ \ } \ \ (_status); \ }) #define UCC_CLASS_CALL_SUPER_INIT(_superclass, ...) \ { \ { \ ucc_status_t _status = _UCS_CLASS_INIT_NAME(_superclass)( \ &self->super, _myclass->superclass, _init_count, \ ##__VA_ARGS__); \ if (ucc_unlikely(_status != UCC_OK)) { \ return _status; \ } \ if (_myclass->superclass != &_UCS_CLASS_DECL_NAME(void)) { \ ++(*_init_count); \ } \ } \ } #define UCC_CLASS_CALL_BASE_INIT() \ { \ { \ if ((_init_count == NULL) || (_myclass == NULL)) { \ return UCC_ERR_INVALID_PARAM; \ } \ } \ } #define UCC_CLASS_DECLARE_NAMED_NEW_FUNC(_name, _argtype, ...) \ ucc_status_t _name(UCS_PP_FOREACH( \ _UCS_CLASS_INIT_ARG_DEFINE, _, \ UCS_PP_ZIP((UCS_PP_SEQ(UCS_PP_NUM_ARGS(__VA_ARGS__))), (__VA_ARGS__))) \ _argtype **obj_p) #define UCC_CLASS_DEFINE_NAMED_NEW_FUNC(_name, _type, _argtype, ...) \ UCC_CLASS_DECLARE_NAMED_NEW_FUNC(_name, _argtype, ##__VA_ARGS__) \ { \ ucc_status_t status; \ ucs_status_t ucs_status; \ *obj_p = NULL; \ status = UCC_CLASS_NEW( \ _type, \ obj_p UCS_PP_FOREACH(_UCS_CLASS_INIT_ARG_PASS, _, \ UCS_PP_SEQ(UCS_PP_NUM_ARGS(__VA_ARGS__)))); \ ucs_status = ucc_status_to_ucs_status(status); \ ucs_class_check_new_func_result(ucs_status, *obj_p); \ return status; \ } #define UCC_CLASS_DECLARE_NEW_FUNC(_type, _argtype, ...) \ UCC_CLASS_DECLARE_NAMED_NEW_FUNC(UCS_CLASS_NEW_FUNC_NAME(_type), _argtype, \ ##__VA_ARGS__) #define UCC_CLASS_DEFINE_NEW_FUNC(_type, _argtype, ...) \ UCC_CLASS_DEFINE_NAMED_NEW_FUNC(UCS_CLASS_NEW_FUNC_NAME(_type), _type, \ _argtype, ##__VA_ARGS__) #define UCC_CLASS_DECLARE_DELETE_FUNC UCS_CLASS_DECLARE_DELETE_FUNC #define UCC_CLASS_DEFINE_DELETE_FUNC UCS_CLASS_DEFINE_DELETE_FUNC #define UCC_CLASS_DELETE_FUNC_NAME UCS_CLASS_DELETE_FUNC_NAME #endif ucc-1.8.0/src/utils/ucc_sys.c0000664000175000017500000001314615211535620016226 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "ucc_sys.h" #include "ucc_math.h" #include "ucc_log.h" #include #include #include #include #include #include "ucc_string.h" ucc_status_t ucc_sysv_alloc(size_t *size, void **addr, int *shm_id) { size_t alloc_size; void *ptr; int ret; alloc_size = ucc_align_up(*size, getpagesize()); *shm_id = shmget(IPC_PRIVATE, alloc_size, IPC_CREAT | 0666); if (*shm_id < 0) { ucc_error("failed to shmget with IPC_PRIVATE, size %zd, IPC_CREAT " "errno: %d(%s)", alloc_size, errno, strerror(errno)); return UCC_ERR_NO_RESOURCE; } ptr = shmat(*shm_id, NULL, 0); /* Remove segment, the attachment keeps a reference to the mapping */ /* FIXME having additional attaches to a removed segment is not portable * behavior */ ret = shmctl(*shm_id, IPC_RMID, NULL); if (ret != 0) { ucc_warn("shmctl(IPC_RMID, shmid=%d) errno: %d(%s)", *shm_id, errno, strerror(errno)); } if (ptr == (void*)-1) { ucc_error("failed to shmat errno: %d(%s)", errno, strerror(errno)); if (errno == ENOMEM) { return UCC_ERR_NO_MEMORY; } else { return UCC_ERR_NO_MESSAGE; } } *size = alloc_size; *addr = ptr; return UCC_OK; } ucc_status_t ucc_sysv_free(void *addr) { int ret; ret = shmdt(addr); if (ret) { ucc_warn("failed to detach shm at %p, errno: %d(%s)", addr, errno, strerror(errno)); return UCC_ERR_INVALID_PARAM; } return UCC_OK; } /** * @return Regular page size on the system. */ size_t ucc_get_page_size() { static long page_size = 0; if (page_size == 0) { page_size = sysconf(_SC_PAGESIZE); if (page_size < 0) { page_size = 4096; ucc_debug("_SC_PAGESIZE undefined, setting default value to %ld", page_size); } } return page_size; } static ucc_status_t ucc_sys_get_lib_info(Dl_info *dl_info) { int ret; (void)dlerror(); ret = dladdr(ucc_sys_get_lib_info, dl_info); if (ret == 0) { return UCC_ERR_NO_MESSAGE; } return UCC_OK; } const char* ucc_sys_get_lib_path() { ucc_status_t status; Dl_info dl_info; status = ucc_sys_get_lib_info(&dl_info); if (status != UCC_OK) { return NULL; } return dl_info.dli_fname; } ucc_status_t ucc_sys_dirname(const char *path, char **out) { char *path_dup = strdup(path); char *dirname_path; if (!path_dup) { return UCC_ERR_NO_MEMORY; } dirname_path = strdup(dirname(path_dup)); free(path_dup); *out = dirname_path; return UCC_OK; } ucc_status_t ucc_sys_path_join(const char *path1, const char *path2, char **out) { const char *strs[3] = {path1, "/", path2}; return ucc_str_concat_n(strs, 3, out); } ucc_status_t ucc_open_output_stream(const char *config_str, ucc_log_level_t err_log_level, FILE **p_fstream, int *p_need_close, const char **p_next_token, char **p_filename) { FILE *output_stream; char filename[256]; char *template; const char *p; size_t len; *p_next_token = config_str; if (p_filename != NULL) { *p_filename = NULL; } len = strcspn(config_str, ":"); if (!strncmp(config_str, "stdout", len)) { *p_fstream = stdout; *p_need_close = 0; *p_next_token = config_str + len; } else if (!strncmp(config_str, "stderr", len)) { *p_fstream = stderr; *p_need_close = 0; *p_next_token = config_str + len; } else { if (!strncmp(config_str, "file:", 5)) { p = config_str + 5; } else { p = config_str; } len = strcspn(p, ":"); template = strndup(p, len); ucc_fill_filename_template(template, filename, sizeof(filename)); free(template); output_stream = fopen(filename, "w"); if (output_stream == NULL) { ucc_log(err_log_level, "failed to open '%s' for writing: %m", filename); return UCC_ERR_IO_ERROR; } if (p_filename != NULL) { *p_filename = ucc_strdup(filename, "filename"); if (*p_filename == NULL) { ucc_log(err_log_level, "failed to allocate filename for '%s'", filename); fclose(output_stream); return UCC_ERR_NO_MEMORY; } } *p_fstream = output_stream; *p_need_close = 1; *p_next_token = p + len; } return UCC_OK; } int ucc_get_first_cpu() { //TODO: fix, not implemented return 0; } const char *ucc_get_user_name() { static char username[256] = {0}; if (*username == 0) { getlogin_r(username, sizeof(username)); } return username; } const char *ucc_get_host_name() { static char hostname[HOST_NAME_MAX] = {0}; if (*hostname == 0) { gethostname(hostname, sizeof(hostname)); strtok(hostname, "."); } return hostname; } pid_t ucc_get_tid(void) { #ifdef SYS_gettid return syscall(SYS_gettid); #elif defined(HAVE_SYS_THR_H) long id; thr_self(&id); return (id); #else #error "Port me" #endif } const char *ucc_get_exe() { static char exe[1024]; int ret; ret = readlink("/proc/self/exe", exe, sizeof(exe) - 1); if (ret < 0) { exe[0] = '\0'; } else { exe[ret] = '\0'; } return exe; } ucc-1.8.0/src/utils/ucc_proc_info.h0000664000175000017500000001110015211535620017357 0ustar alastairalastair/** * Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_PROC_INFO_H_ #define UCC_PROC_INFO_H_ #include "config.h" #include #include typedef uint64_t ucc_host_id_t; typedef uint8_t ucc_socket_id_t; typedef uint8_t ucc_numa_id_t; typedef uint8_t ucc_device_id_t; #define UCC_SOCKET_ID_INVALID ((ucc_socket_id_t)-1) #define UCC_NUMA_ID_INVALID ((ucc_numa_id_t)-1) #define UCC_DEVICE_ID_INVALID ((ucc_device_id_t)-1) #define UCC_HOST_ID_INVALID ((ucc_host_id_t)-1) #define UCC_GPU_FABRIC_CLIQUE_ID_INVALID ((uint64_t)0) #define UCC_GPU_FABRIC_PARTITION_ID_INVALID ((uint32_t)0) #define UCC_MAX_SOCKET_ID (UCC_SOCKET_ID_INVALID - 1) #define UCC_MAX_NUMA_ID (UCC_NUMA_ID_INVALID - 1) #define UCC_MAX_HOST_GPUS 16 #define UCC_MAX_HOST_NICS 16 typedef struct ucc_proc_info { ucc_host_id_t host_hash; ucc_socket_id_t socket_id; ucc_numa_id_t numa_id; ucc_host_id_t host_id; pid_t pid; } ucc_proc_info_t; typedef struct ucc_pci_info { /**< Unique host identifier to which the PCI device belongs */ ucc_host_id_t host_id; /**< PCI domain */ uint16_t domain; /**< PCI bus */ uint8_t bus; /**< PCI device */ uint8_t device; /**< PCI function */ uint8_t function; } ucc_pci_info_t; typedef enum ucc_gpu_cap { UCC_GPU_CAP_NVLINK = UCC_BIT(0), UCC_GPU_CAP_NVSWITCH = UCC_BIT(1), UCC_GPU_CAP_FABRIC = UCC_BIT(2), } ucc_gpu_cap_t; #define UCC_GPU_HAS_CAP(_gpu, _cap) ((_gpu)->caps & (_cap)) typedef struct ucc_gpu_info { ucc_pci_info_t pci; /**< Bitmask of ucc_gpu_cap_t flags */ uint32_t caps; /**< NVLink partition ID for GB200+ NVL sub-fabric partitions. * UCC_GPU_FABRIC_PARTITION_ID_INVALID means single partition or * not populated (NVML < r525). */ uint32_t fabric_partition_id; /**< NVSwitch fabric clique ID (UCC_GPU_FABRIC_CLIQUE_ID_INVALID if unknown) */ uint64_t fabric_clique_id; /**< Hash of GPU UUID for unique identification */ uint64_t uuid; } ucc_gpu_info_t; typedef struct ucc_nic_info { ucc_pci_info_t pci; /**< IB port number */ uint8_t port; /**< Unique NIC identifier (GUID) */ uint64_t guid; /**< Device name, e.g., "mlx5_0" */ char name[16]; } ucc_nic_info_t; typedef struct ucc_host_info { /**< Unique host identifier */ ucc_host_id_t host_id; /**< Number of GPUs on host */ uint8_t n_gpus; /**host_id != pci2->host_id) { dist += 10000; } if (pci1->domain != pci2->domain) { dist += 1000; } diff = (pci1->bus > pci2->bus) ? (pci1->bus - pci2->bus) : (pci2->bus - pci1->bus); dist += diff * 10; diff = (pci1->device > pci2->device) ? (pci1->device - pci2->device) : (pci2->device - pci1->device); dist += diff * 2; diff = (pci1->function > pci2->function) ? (pci1->function - pci2->function) : (pci2->function - pci1->function); dist += diff; return dist; } static inline int ucc_compare_pci_info(const void *a, const void *b) { return ucc_pci_distance(a, b) == 0; } #define UCC_PROC_INFO_EQUAL(_pi1, _pi2) \ (((_pi1).host_hash == (_pi2).host_hash) && \ ((_pi1).pid == (_pi2).pid)) //TODO maybe need tid ? void ucc_proc_info_print(const ucc_proc_info_t *info); void ucc_host_info_print(const ucc_host_info_t *info); ucc_status_t ucc_local_proc_info_init(); uint64_t ucc_get_system_id(); #endif ucc-1.8.0/src/utils/ucc_parser.c0000664000175000017500000010235715211535620016707 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_parser.h" #include "ucc_malloc.h" #include "ucc_log.h" #include "ucc_mem_type.h" #include "ucc_string.h" #include "ini.h" #include "components/topo/ucc_topo.h" #include "schedule/ucc_schedule.h" #include "schedule/ucc_schedule_pipelined.h" #define UCC_ADD_KEY_VALUE_TO_HASH(_type, _h, _name, _val) \ do { \ khiter_t _iter; \ int _res; \ char *_dup; \ _iter = kh_get(_type, _h, _name); \ if (_iter != kh_end(_h)) { \ ucc_warn("found duplicate '%s' in config file", _name); \ return 0; \ } else { \ _dup = strdup(_name); \ if (!_dup) { \ ucc_error("failed to dup str for kh_put"); \ return 0; \ } \ _iter = kh_put(_type, _h, _dup, &_res); \ if (_res == UCS_KH_PUT_FAILED) { \ ucc_free(_dup); \ ucc_error("inserting '%s' to config map failed", _name); \ return 0; \ } \ } \ _dup = strdup(_val); \ if (!_dup) { \ ucc_error("failed to dup str for kh_val"); \ return 0; \ } \ kh_val(_h, _iter) = _dup; /* NOLINT */ \ return 1; \ } while (0) static int ucc_check_section(ucc_section_desc_t sec_desc, ucc_cpu_vendor_t vendor, ucc_cpu_model_t model, ucc_rank_t team_size, ucc_rank_t ppn_min, ucc_rank_t ppn_max, ucc_rank_t sock_min, ucc_rank_t sock_max, ucc_rank_t nnodes) { if (sec_desc.mask & UCC_TUNING_DESC_FIELD_VENDOR) { if (sec_desc.vendor != vendor) { return 0; } } if (sec_desc.mask & UCC_TUNING_DESC_FIELD_MODEL) { if (sec_desc.model != model) { return 0; } } if (sec_desc.mask & UCC_TUNING_DESC_FIELD_TEAM_SIZE) { if (team_size < sec_desc.min_team_size || team_size > sec_desc.max_team_size) { return 0; } } if (sec_desc.mask & UCC_TUNING_DESC_FIELD_PPN) { if (ppn_min < sec_desc.min_ppn || ppn_max > sec_desc.max_ppn) { return 0; } } if (sec_desc.mask & UCC_TUNING_DESC_FIELD_SOCK) { if (sock_min < sec_desc.min_sock || sock_max > sec_desc.max_sock) { return 0; } } if (sec_desc.mask & UCC_TUNING_DESC_FIELD_NNODES) { if (nnodes < sec_desc.min_nnodes || nnodes > sec_desc.max_nnodes) { return 0; } } return 1; } static inline int ucc_check_range(char *range_str, ucc_rank_t *begin, ucc_rank_t *end) { char **range = ucc_str_split(range_str, "-"); char *str_end; unsigned n_range; long pbegin, pend; if (!range) { goto split_err; } n_range = ucc_str_split_count(range); pbegin = strtol(range[0], &str_end, 10); pend = pbegin; if (n_range > 2 || *str_end != '\0' || pbegin < 0) { goto val_err; } if (n_range == 2) { pend = strtol(range[1], &str_end, 10); if (*str_end != '\0' || pend < 0) { goto val_err; } } *begin = (ucc_rank_t)pbegin; *end = (ucc_rank_t)pend; ucc_str_split_free(range); return 1; val_err: ucc_str_split_free(range); split_err: ucc_warn("invalid range defined in section name\n"); return 0; } static inline ucc_status_t ucc_parse_section_name_to_desc(const char *sec_name, ucc_section_desc_t *desc) { char **split, **cur_str; unsigned n_split, i; split = ucc_str_split(sec_name, " "); if (!split) { ucc_warn("invalid section name\n"); return UCC_ERR_INVALID_PARAM; } desc->mask = 0; n_split = ucc_str_split_count(split); for (i = 0; i < n_split; i++) { cur_str = ucc_str_split(split[i], "="); if (!cur_str) { ucc_warn("invalid section key=value definition\n"); goto err_cur_str; } if (strcasecmp(cur_str[0], "vendor") == 0) { desc->vendor = ucc_get_vendor_from_str(cur_str[1]); desc->mask |= UCC_TUNING_DESC_FIELD_VENDOR; } else if (strcmp(cur_str[0], "model") == 0) { desc->model = ucc_get_model_from_str(cur_str[1]); desc->mask |= UCC_TUNING_DESC_FIELD_MODEL; } else if (strcmp(cur_str[0], "team_size") == 0) { if (!ucc_check_range(cur_str[1], &desc->min_team_size, &desc->max_team_size)) { goto err_key; } desc->mask |= UCC_TUNING_DESC_FIELD_TEAM_SIZE; } else if (strcmp(cur_str[0], "ppn") == 0) { if (!ucc_check_range(cur_str[1], &desc->min_ppn, &desc->max_ppn)) { goto err_key; } desc->mask |= UCC_TUNING_DESC_FIELD_PPN; } else if (strcmp(cur_str[0], "sock") == 0) { if (!ucc_check_range(cur_str[1], &desc->min_sock, &desc->max_sock)) { goto err_key; } desc->mask |= UCC_TUNING_DESC_FIELD_SOCK; } else if (strcmp(cur_str[0], "nnodes") == 0) { if (!ucc_check_range(cur_str[1], &desc->min_nnodes, &desc->max_nnodes)) { goto err_key; } desc->mask |= UCC_TUNING_DESC_FIELD_NNODES; } else { ucc_warn("key %s not defined as part of tuning section params\n", cur_str[0]); goto err_key; } ucc_str_split_free(cur_str); } ucc_str_split_free(split); return UCC_OK; err_key: ucc_str_split_free(cur_str); err_cur_str: ucc_str_split_free(split); return UCC_ERR_INVALID_PARAM; } ucc_status_t ucc_config_names_array_merge(ucc_config_names_array_t *dst, const ucc_config_names_array_t *src) { int i, n_new; n_new = 0; if (dst->count == 0) { return ucc_config_names_array_dup(dst, src); } else { for (i = 0; i < src->count; i++) { if (ucc_config_names_search(dst, src->names[i]) < 0) { /* found new entry in src which is not part of dst */ n_new++; } } if (n_new) { dst->names = ucc_realloc(dst->names, (dst->count + n_new) * sizeof(char *), "ucc_config_names_array"); if (ucc_unlikely(!dst->names)) { return UCC_ERR_NO_MEMORY; } for (i = 0; i < src->count; i++) { if (ucc_config_names_search(dst, src->names[i]) < 0) { dst->names[dst->count++] = strdup(src->names[i]); if (ucc_unlikely(!dst->names[dst->count - 1])) { ucc_error("failed to dup config_names_array entry"); return UCC_ERR_NO_MEMORY; } } } } } return UCC_OK; } ucc_status_t ucc_config_names_array_dup(ucc_config_names_array_t *dst, const ucc_config_names_array_t *src) { int i; if (dst->count != 0) { ucc_config_names_array_free(dst); } dst->names = ucc_malloc(sizeof(char*) * src->count, "ucc_config_names_array"); if (!dst->names) { ucc_error("failed to allocate %zd bytes for ucc_config_names_array", sizeof(char *) * src->count); return UCC_ERR_NO_MEMORY; } dst->count = src->count; for (i = 0; i < src->count; i++) { dst->names[i] = strdup(src->names[i]); if (!dst->names[i]) { ucc_error("failed to dup config_names_array entry"); goto err; } } return UCC_OK; err: for (i = i - 1; i >= 0; i--) { free(dst->names[i]); } return UCC_ERR_NO_MEMORY; } ucc_status_t ucc_config_names_array_to_string(const ucc_config_names_array_t *array, char *str, size_t max) { size_t cur_len = 0; size_t i; int ret; if (!str || !array || max == 0) { return UCC_ERR_INVALID_PARAM; } for (i = 0; i < array->count; i++) { ret = snprintf(str + cur_len, max - cur_len, "%s%s", array->names[i], (i < array->count - 1) ? "," : ""); if (ret < 0) { return UCC_ERR_NO_MEMORY; } cur_len += ret; } return UCC_OK; } void ucc_config_names_array_free(ucc_config_names_array_t *array) { int i; if (array->names != NULL) { for (i = 0; i < array->count; i++) { free(array->names[i]); } ucc_free(array->names); } array->count = 0; } int ucc_config_names_search(const ucc_config_names_array_t *config_names, const char * str) { unsigned i; for (i = 0; i < config_names->count; ++i) { if (!strcmp(config_names->names[i], str)) { return i; } } return -1; } ucc_status_t ucc_config_allow_list_process(const ucc_config_allow_list_t * list, const ucc_config_names_array_t *all, ucc_config_names_list_t * out) { ucc_status_t status = UCC_OK; int i; out->array.names = NULL; out->requested = 0; switch (list->mode){ case UCC_CONFIG_ALLOW_LIST_ALLOW: out->requested = 1; status = ucc_config_names_array_dup(&out->array, &list->array); break; case UCC_CONFIG_ALLOW_LIST_ALLOW_ALL: status = ucc_config_names_array_dup(&out->array, all); break; case UCC_CONFIG_ALLOW_LIST_NEGATE: out->array.count = 0; out->array.names = ucc_malloc(sizeof(char *) * all->count, "names"); if (!out->array.names) { ucc_error("failed to allocate %zd bytes for names array", sizeof(char *) * all->count); status = UCC_ERR_NO_MEMORY; break; } for (i = 0; i < all->count; i++) { if (ucc_config_names_search(&list->array, all->names[i]) < 0) { out->array.names[out->array.count++] = strdup(all->names[i]); } } break; default: ucc_assert(0); } return status; } static int ucc_file_parse_handler(void *arg, const char *section, const char *name, const char *value) { ucc_file_config_t *cfg = arg; khash_t(ucc_cfg_file) *vars = &cfg->vars; khash_t(ucc_sections) *sections = &cfg->sections; ucc_section_wrap_t *sec_wrap; khiter_t iter; int result; char *dup; ucc_status_t status; if (!name) { return 1; } if (NULL != getenv(name)) { /* variable is set in env, skip it. Env gets precedence over file */ ; return 1; } if (strlen(name) < 4 || NULL == strstr(name, "UCC_")) { ucc_warn("incorrect parameter name %s " "(param name should start with UCC_ or _UCC_)", name); return 0; } if (strlen(section) > 0) { /* has section */ iter = kh_get(ucc_sections, sections, section); if (iter == kh_end(sections)) { /* section hash table doesn't exist */ sec_wrap = ucc_calloc(1, sizeof(*sec_wrap), strcat("section cfg: ", section)); if (!sec_wrap) { ucc_error("failed to allocate %zd bytes for section config", sizeof(*sec_wrap)); return 0; } dup = strdup(section); if (!dup) { ucc_free(sec_wrap); ucc_error("failed to dup str for kh_put"); return 0; } iter = kh_put(ucc_sections, sections, dup, &result); if (result == UCS_KH_PUT_FAILED) { ucc_free(sec_wrap); ucc_free(dup); ucc_error("inserting '%s' to config map failed", name); return 0; } status = ucc_parse_section_name_to_desc(section, &sec_wrap->desc); if (status != UCC_OK) { ucc_free(sec_wrap); ucc_free(dup); return 0; } /* new hash table for section */ kh_init_inplace(ucc_sec, &sec_wrap->vals_h); // NOLINT kh_val(sections, iter) = sec_wrap; // NOLINT } else { sec_wrap = kh_val(sections, iter); } UCC_ADD_KEY_VALUE_TO_HASH(ucc_sec, &sec_wrap->vals_h, name, value); } /* param not part of a section */ UCC_ADD_KEY_VALUE_TO_HASH(ucc_cfg_file, vars, name, value); } ucc_status_t ucc_parse_file_config(const char * filename, ucc_file_config_t **cfg_p) { ucc_file_config_t *cfg; int ret; ucc_status_t status; cfg = ucc_calloc(1, sizeof(*cfg), "file_cfg"); if (!cfg) { ucc_error("failed to allocate %zd bytes for file config", sizeof(*cfg)); return UCC_ERR_NO_MEMORY; } kh_init_inplace(ucc_cfg_file, &cfg->vars); kh_init_inplace(ucc_sections, &cfg->sections); ret = ucc_ini_parse(filename, ucc_file_parse_handler, cfg); if (-1 == ret) { /* according to ucs/ini.h -1 means error in file open */ status = UCC_ERR_NOT_FOUND; goto out; } else if (ret) { ucc_error("failed to parse config file %s", filename); status = UCC_ERR_INVALID_PARAM; goto out; } cfg->filename = strdup(filename); *cfg_p = cfg; return UCC_OK; out: ucc_free(cfg); return status; } void ucc_release_file_config(ucc_file_config_t *cfg) { const char *key, *section_key; char *value, *section_val; khash_t(ucc_sec) *section; ucc_section_wrap_t *sec_wrap; int j; ucc_free(cfg->filename); kh_foreach(&cfg->vars, key, value, { ucc_free((void *)key); ucc_free(value); }) kh_destroy_inplace(ucc_cfg_file, &cfg->vars); kh_foreach(&cfg->sections, key, sec_wrap, { section = &sec_wrap->vals_h; for (j = kh_begin(section); j != kh_end(section); ++j) { if (!kh_exist(section, j)) continue; section_key = kh_key(section, j); section_val = kh_val(section, j); ucc_free((void *)section_key); ucc_free(section_val); } ucc_free((void *)key); kh_destroy_inplace(ucc_sec, section); }) kh_destroy_inplace(ucc_sections, &cfg->sections); ucc_free(cfg); } static const char *ucc_file_config_get_by_section(ucc_file_config_t *cfg, const char *var_name, const char *section) { khash_t(ucc_sections) *sections = &cfg->sections; khash_t(ucc_sec) *sec; khiter_t iter; ucc_section_wrap_t *sec_wrap; iter = kh_get(ucc_sections, sections, section); if (iter == kh_end(sections)) { /* section not found*/ return NULL; } sec_wrap = kh_val(sections, iter); sec = &sec_wrap->vals_h; iter = kh_get(ucc_sec, sec, var_name); if (iter == kh_end(sec)) { /* variable not found in section*/ return NULL; } return kh_val(sec, iter); } static const char *ucc_file_config_get(ucc_file_config_t *cfg, const char *var_name, const char *section) { khash_t(ucc_cfg_file) *vars = &cfg->vars; khiter_t iter; if (strlen(section) > 0) { return (ucc_file_config_get_by_section(cfg, var_name, section)); } iter = kh_get(ucc_cfg_file, vars, var_name); if (iter == kh_end(vars)) { /* variable not found in prefix hash*/ return NULL; } return kh_val(vars, iter); } static ucc_status_t ucc_apply_file_cfg_value(void * opts, ucc_config_field_t *fields, const char * base_prefix, const char *component_prefix, const char *name, const char *section) { char var[512]; size_t left = sizeof(var); const char *base_prefix_var; const char *cfg_value; ucc_strncpy_safe(var, base_prefix, left); left -= strlen(base_prefix); strncat(var, component_prefix, left); left -= strlen(component_prefix); strncat(var, name, left); base_prefix_var = strstr(var, "UCC_"); cfg_value = ucc_file_config_get(ucc_global_config.file_cfg, base_prefix_var, section); if (cfg_value) { return ucc_config_parser_set_value(opts, fields, name, cfg_value); }; if (base_prefix_var != var) { cfg_value = ucc_file_config_get(ucc_global_config.file_cfg, var, section); if (cfg_value) { return ucc_config_parser_set_value(opts, fields, name, cfg_value); } } return UCC_ERR_NOT_FOUND; } static ucc_status_t ucc_apply_file_cfg(void *opts, ucc_config_field_t *fields, const char *env_prefix, const char *component_prefix, const char *section) { ucc_status_t status = UCC_OK; while (fields->name != NULL) { if (strlen(fields->name) == 0) { status = ucc_apply_file_cfg( opts, (ucc_config_field_t *)fields->parser.arg, env_prefix, component_prefix, section); if (UCC_OK != status) { return status; } fields++; continue; } status = ucc_apply_file_cfg_value( opts, fields, env_prefix, component_prefix ? component_prefix : "", fields->name, section); if (status == UCC_ERR_NOT_FOUND && component_prefix) { status = ucc_apply_file_cfg_value(opts, fields, env_prefix, "", fields->name, section); } if (status != UCC_OK && status != UCC_ERR_NOT_FOUND) { return status; } fields++; } return UCC_OK; } /* Team cfg table values have been previously copied from lib cfg. * Here, tuning values from cfg file are applied overwriting specific values * in team cfg table. * Special case needed for param tune key which is equivalent to UCC_TL_#_TUNE. * Returns: UCC_OK on success, * error status if values from cfg file cannot be applied. */ ucc_status_t ucc_add_team_sections(void *team_cfg, ucc_config_field_t *tl_fields, ucc_topo_t *team_topo, const char **tuning_str, const char *tune_key, const char *env_prefix, const char *component_prefix) { khash_t(ucc_sections) *sections = &ucc_global_config.file_cfg->sections; ucc_cpu_vendor_t vendor = ucc_arch_get_cpu_vendor(); ucc_cpu_model_t model = ucc_arch_get_cpu_model(); ucc_rank_t ppn_min = ucc_topo_min_ppn(team_topo); ucc_rank_t ppn_max = ucc_topo_max_ppn(team_topo); ucc_rank_t sock_min = ucc_topo_min_socket_size(team_topo); ucc_rank_t sock_max = ucc_topo_max_socket_size(team_topo); ucc_rank_t nnodes = ucc_topo_nnodes(team_topo); ucc_rank_t team_size = team_topo->set.map.ep_num; int found = 0; khash_t(ucc_sec) *sec_h; khiter_t i, j; const char *sec_name; ucc_section_wrap_t *sec; ucc_status_t status; for (i = kh_begin(sections); i != kh_end(sections); ++i) { if (!kh_exist(sections, i)) continue; sec_name = kh_key(sections, i); sec = kh_val(sections, i); if (ucc_check_section(sec->desc, vendor, model, team_size, ppn_min, ppn_max, sock_min, sock_max, nnodes)) { sec_h = &sec->vals_h; j = kh_get(ucc_sec, sec_h, tune_key); if (j != kh_end(sec_h)) { *tuning_str = kh_val(sec_h, j); } status = ucc_apply_file_cfg(team_cfg, tl_fields, env_prefix, component_prefix, sec_name); found = 1; } } return found ? status : UCC_ERR_NOT_FOUND; } ucc_status_t ucc_config_parser_fill_opts(void *opts, ucs_config_global_list_entry_t *entry, const char *env_prefix, int ignore_errors) { ucs_status_t ucs_status; ucc_status_t status; #if UCS_HAVE_CONFIG_GLOBAL_LIST_ENTRY_FLAGS ucs_status = ucs_config_parser_fill_opts(opts, entry, env_prefix, ignore_errors); #else ucs_status = ucs_config_parser_fill_opts(opts, entry->table, env_prefix, entry->prefix, 0); #endif status = ucs_status_to_ucc_status(ucs_status); if (UCC_OK == status && ucc_global_config.file_cfg) { status = ucc_apply_file_cfg(opts, entry->table, env_prefix, entry->prefix, ""); } return status; } void ucc_config_parser_print_all_opts(FILE *stream, const char *prefix, ucc_config_print_flags_t flags, ucc_list_link_t * config_list) { ucc_config_global_list_entry_t *entry; ucs_config_print_flags_t ucs_flags; ucc_status_t status; char title[64]; void * opts; ucs_flags = ucc_print_flags_to_ucs_print_flags(flags); ucc_list_for_each(entry, config_list, list) { if ((entry->table == NULL) || (entry->table[0].name == NULL)) { /* don't print title for an empty configuration table */ continue; } opts = ucc_malloc(entry->size, "tmp_opts"); if (opts == NULL) { ucc_error("could not allocate configuration of size %zu", entry->size); continue; } status = ucc_config_parser_fill_opts(opts, entry, prefix, 0); if (status != UCC_OK) { ucc_free(opts); continue; } snprintf(title, sizeof(title), "%s configuration", entry->name); UCS_CONFIG_PARSER_PRINT_OPTS(stream, title, opts, entry->table, entry->prefix, prefix, ucs_flags); ucs_config_parser_release_opts(opts, entry->table); ucc_free(opts); } } ucc_status_t ucc_config_clone_table(const void *src, void *dst, const void *arg) { return ucs_status_to_ucc_status(ucs_config_clone_table(src, dst, arg)); } static ucc_pipeline_params_t ucc_pipeline_params_auto = { .threshold = 0, .n_frags = 0, .frag_size = 0, .pdepth = 0, .order = UCC_PIPELINE_PARALLEL, }; static ucc_pipeline_params_t ucc_pipeline_params_no = { .threshold = SIZE_MAX, .n_frags = 0, .frag_size = 0, .pdepth = 1, .order = UCC_PIPELINE_PARALLEL, }; static ucc_pipeline_params_t ucc_pipeline_params_default = { .threshold = SIZE_MAX, .n_frags = 2, .frag_size = SIZE_MAX, .pdepth = 2, .order = UCC_PIPELINE_SEQUENTIAL, }; int ucc_pipeline_params_is_auto(const ucc_pipeline_params_t *p) { if ((p->threshold == ucc_pipeline_params_auto.threshold) && (p->n_frags == ucc_pipeline_params_auto.n_frags) && (p->frag_size == ucc_pipeline_params_auto.frag_size) && (p->pdepth == ucc_pipeline_params_auto.pdepth) && (p->order == ucc_pipeline_params_auto.order)) { return 1; } return 0; } int ucc_config_sscanf_pipeline_params(const char *buf, void *dest, const void *arg) //NOLINT { ucc_pipeline_params_t *p = dest; ucc_status_t status; int i, n_tokens, order; char ** tokens, **t2; if (strlen(buf) == 0) { return 0; } /* Special value: auto */ if (!strcasecmp(buf, UCS_VALUE_AUTO_STR)) { *p = ucc_pipeline_params_auto; return 1; } if (!strcasecmp(buf, "n")) { *p = ucc_pipeline_params_no; return 1; } *p = ucc_pipeline_params_default; tokens = ucc_str_split(buf, ":"); if (!tokens) { return 0; } n_tokens = ucc_str_split_count(tokens); for (i = 0; i < n_tokens; i++) { if ((order = ucc_string_find_in_list( tokens[i], ucc_pipeline_order_names, 0)) >= 0) { p->order = (ucc_pipeline_order_t)order; continue; } t2 = ucc_str_split(tokens[i], "="); if (!t2) { goto out; } if (ucc_str_split_count(t2) != 2) { goto out; } if (0 == strcmp(t2[0], "thresh")) { status = ucc_str_to_memunits(t2[1], &p->threshold); if (UCC_OK != status) { goto out; } } else if (0 == strcmp(t2[0], "fragsize")) { status = ucc_str_to_memunits(t2[1], &p->frag_size); if (UCC_OK != status) { goto out; } } else if (0 == strcmp(t2[0], "nfrags")) { status = ucc_str_is_number(t2[1]); if (UCC_OK != status) { goto out; } p->n_frags = atoi(t2[1]); } else if (0 == strcmp(t2[0], "pdepth")) { status = ucc_str_is_number(t2[1]); if (UCC_OK != status) { goto out; } p->pdepth = atoi(t2[1]); } ucc_str_split_free(t2); } ucc_str_split_free(tokens); return 1; out: if (t2) { ucc_str_split_free(t2); } ucc_str_split_free(tokens); return 0; } int ucc_config_sprintf_pipeline_params(char *buf, size_t max, const void *src, const void *arg) //NOLINT { const ucc_pipeline_params_t *p = src; char thresh[32], frag_size[32]; if (ucc_pipeline_params_is_auto(p)) { return snprintf(buf, max, "auto"); } if (!memcmp(p, &ucc_pipeline_params_no, sizeof(*p))) { return snprintf(buf, max, "n"); } return snprintf( buf, max, "thresh=%s:nfrags=%d:fragsize=%s:pdepth=%d:order=%s", ucs_memunits_to_str(p->threshold, thresh, sizeof(thresh)), p->n_frags, ucs_memunits_to_str(p->frag_size, frag_size, sizeof(frag_size)), p->pdepth, ucc_pipeline_order_names[p->order]); } ucs_status_t ucc_config_clone_pipeline_params(const void *src, void *dest, const void *arg) //NOLINT { memcpy(dest, src, sizeof(ucc_pipeline_params_t)); return UCS_OK; } void ucc_config_release_pipeline_params(void *ptr, const void *arg) { } int ucc_config_sscanf_uint_ranged(const char *buf, void *dest, const void *arg) //NOLINT { ucc_mrange_uint_t *p = dest; char **ranges, **tokens; unsigned n_ranges, i, j, n_tokens; size_t start, end; ucc_mrange_t *r; uint32_t mt_map; ucc_list_head_init(&p->ranges); /* Special value: auto */ if (!strcasecmp(buf, UCS_VALUE_AUTO_STR)) { p->default_value = UCC_UUNITS_AUTO; return 1; } ranges = ucc_str_split(buf, ","); if (!ranges) { return 0; } n_ranges = ucc_str_split_count(ranges); for (i = 0; i < n_ranges; i++) { tokens = ucc_str_split(ranges[i], ":"); if (!tokens) { goto err_ranges; } n_tokens = ucc_str_split_count(tokens); if (n_tokens > 3 || (UCC_OK != ucc_str_is_number(tokens[n_tokens - 1]) && strcasecmp(tokens[n_tokens - 1], UCS_VALUE_AUTO_STR) != 0)) { goto err_tokens; } if (n_tokens == 1) { if (!strcasecmp(tokens[n_tokens - 1], UCS_VALUE_AUTO_STR)) { /* Special value: auto */ p->default_value = UCC_UUNITS_AUTO; } else { /* default value without range */ p->default_value = atoi(tokens[0]); } } else { r = ucc_malloc(sizeof(*r), "mrange"); if (!r) { goto err_tokens; } r->mtypes = UCC_MEM_TYPE_MASK_FULL; r->start = 0; r->end = SIZE_MAX; for (j = 0; j < n_tokens; j++) { if (UCC_OK == ucc_str_is_number(tokens[j])) { /* value */ r->value = atoi(tokens[j]); continue; } if (UCC_OK == ucc_str_to_mtype_map(tokens[j], "^", &mt_map)) { r->mtypes = mt_map; continue; } if (UCC_OK == ucc_str_to_memunits_range(tokens[j], &start, &end)) { r->start = start; r->end = end; continue; } ucc_free(r); goto err_tokens; } ucc_list_add_tail(&p->ranges, &r->list_elem); } ucc_str_split_free(tokens); } ucc_str_split_free(ranges); return 1; err_tokens: ucc_str_split_free(tokens); err_ranges: ucc_str_split_free(ranges); return 0; } #define MAX_TMP_BUF_LENGTH 128 int ucc_config_sprintf_uint_ranged(char *buf, size_t max, const void *src, const void *arg) // NOLINT { const ucc_mrange_uint_t *s = src; ucc_mrange_t *r; char tmp_start[MAX_TMP_BUF_LENGTH]; char tmp_end[MAX_TMP_BUF_LENGTH]; char tmp_mtypes[MAX_TMP_BUF_LENGTH]; size_t last; ucc_list_for_each(r, &s->ranges, list_elem) { ucs_memunits_to_str(r->start, tmp_start, MAX_TMP_BUF_LENGTH); ucs_memunits_to_str(r->end, tmp_end, MAX_TMP_BUF_LENGTH); if (r->mtypes == UCC_MEM_TYPE_MASK_FULL) { ucc_snprintf_safe(buf, max, "%s-%s:%u", tmp_start, tmp_end, r->value); } else { ucc_mtype_map_to_str(r->mtypes, "^", tmp_mtypes, MAX_TMP_BUF_LENGTH); ucc_snprintf_safe(buf, max, "%s-%s:%s:%u", tmp_start, tmp_end, tmp_mtypes, r->value); } last = strlen(buf); if (max - last - 1 <= 0) { /* no more space in buf for next range*/ return 1; } buf[last] = ','; buf[last + 1] = '\0'; max -= last + 1; buf += last + 1; } if (s->default_value == UCC_UUNITS_AUTO) { ucc_snprintf_safe(buf, max, "%s", "auto"); } else { ucc_snprintf_safe(buf, max, "%u", s->default_value); } return 1; } ucs_status_t ucc_config_clone_uint_ranged(const void *src, void *dest, const void *arg) //NOLINT { return ucc_status_to_ucs_status(ucc_mrange_uint_copy(dest, src)); } void ucc_config_release_uint_ranged(void *ptr, const void *arg) //NOLINT { ucc_mrange_uint_destroy(ptr); } size_t ucc_config_memunits_get(size_t config_size, size_t auto_size, size_t max_size) { if (config_size == UCC_MEMUNITS_AUTO) { return auto_size; } else { return ucs_min(config_size, max_size); } } ucc-1.8.0/src/utils/ucc_parser.h0000664000175000017500000003407015211535620016710 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_PARSER_H_ #define UCC_PARSER_H_ #include "config.h" #include "ucc/api/ucc_status.h" #include "ucc/api/ucc_def.h" #include "utils/ucc_datastruct.h" #include "utils/ucc_compiler_def.h" #include "utils/ucc_list.h" #include "utils/arch/cpu.h" #include "khash.h" #include #include #include #include #include typedef ucs_config_field_t ucc_config_field_t; typedef ucs_config_names_array_t ucc_config_names_array_t; typedef ucs_config_global_list_entry_t ucc_config_global_list_entry_t; typedef ucs_config_allow_list_t ucc_config_allow_list_t; typedef struct ucc_topo ucc_topo_t; typedef struct ucc_file_config ucc_file_config_t; #if UCS_HAVE_CONFIG_GLOBAL_LIST_ENTRY_FLAGS #define UCC_CONFIG_DECLARE_TABLE(_table, _name, _prefix, _type) \ static ucc_config_global_list_entry_t _table##_config_entry = { \ .name = _name, \ .prefix = _prefix, \ .table = _table, \ .size = sizeof(_type), \ .list = {NULL, NULL}, \ .flags = 0 \ }; #else #define UCC_CONFIG_DECLARE_TABLE(_table, _name, _prefix, _type) \ static ucc_config_global_list_entry_t _table##_config_entry = { \ .name = _name, \ .prefix = _prefix, \ .table = _table, \ .size = sizeof(_type), \ .list = {NULL, NULL}, \ }; #endif #ifdef UCS_HAVE_PARSER_PRINT_FILTER_ARG #define UCS_CONFIG_PARSER_PRINT_OPTS(_stream, _title, _opts, _fields, _tprefix, _prefix, _flags) \ ucs_config_parser_print_opts((_stream), (_title), (_opts), (_fields), (_tprefix), (_prefix), (_flags), NULL) #else #define UCS_CONFIG_PARSER_PRINT_OPTS(_stream, _title, _opts, _fields, _tprefix, _prefix, _flags) \ ucs_config_parser_print_opts((_stream), (_title), (_opts), (_fields), (_tprefix), (_prefix), (_flags)) #endif #define UCC_MEMUNITS_AUTO ((size_t)-2) #define UCC_CONFIG_GET_TABLE(_table) &_table##_config_entry #define UCC_CONFIG_TYPE_LOG_COMP UCS_CONFIG_TYPE_LOG_COMP #define UCC_CONFIG_REGISTER_TABLE UCS_CONFIG_REGISTER_TABLE #define UCC_CONFIG_REGISTER_TABLE_ENTRY UCS_CONFIG_REGISTER_TABLE_ENTRY #define UCC_CONFIG_TYPE_LOG_COMP UCS_CONFIG_TYPE_LOG_COMP #define UCC_CONFIG_TYPE_STRING UCS_CONFIG_TYPE_STRING #define UCC_CONFIG_TYPE_INT UCS_CONFIG_TYPE_INT #define UCC_CONFIG_TYPE_UINT UCS_CONFIG_TYPE_UINT #define UCC_CONFIG_TYPE_STRING_ARRAY UCS_CONFIG_TYPE_STRING_ARRAY #define UCC_CONFIG_TYPE_ALLOW_LIST UCS_CONFIG_TYPE_ALLOW_LIST #define UCC_CONFIG_TYPE_ARRAY UCS_CONFIG_TYPE_ARRAY #define UCC_CONFIG_TYPE_TABLE UCS_CONFIG_TYPE_TABLE #define UCC_CONFIG_TYPE_ULUNITS UCS_CONFIG_TYPE_ULUNITS #define UCC_CONFIG_TYPE_ULONG UCS_CONFIG_TYPE_ULONG #define UCC_CONFIG_TYPE_ENUM UCS_CONFIG_TYPE_ENUM #define UCC_CONFIG_TYPE_MEMUNITS UCS_CONFIG_TYPE_MEMUNITS #define UCC_ULUNITS_AUTO UCS_ULUNITS_AUTO #define UCC_CONFIG_TYPE_BITMAP UCS_CONFIG_TYPE_BITMAP #define UCC_CONFIG_TYPE_BOOL UCS_CONFIG_TYPE_BOOL #define UCC_CONFIG_ALLOW_LIST_NEGATE UCS_CONFIG_ALLOW_LIST_NEGATE #define UCC_CONFIG_ALLOW_LIST_ALLOW_ALL UCS_CONFIG_ALLOW_LIST_ALLOW_ALL #define UCC_CONFIG_ALLOW_LIST_ALLOW UCS_CONFIG_ALLOW_LIST_ALLOW #define UCC_CONFIG_TYPE_TERNARY UCS_CONFIG_TYPE_TERNARY #define UCC_CONFIG_TYPE_ON_OFF_AUTO UCS_CONFIG_TYPE_ON_OFF_AUTO #define UCC_UUNITS_AUTO ((unsigned)-2) typedef enum ucc_ternary_auto_value { UCC_NO = UCS_NO, UCC_YES = UCS_YES, UCC_TRY = UCS_TRY, UCC_AUTO = UCS_AUTO, UCC_TERNARY_LAST } ucc_ternary_auto_value_t; typedef enum ucc_on_off_auto_value { UCC_CONFIG_OFF = UCS_CONFIG_OFF, UCC_CONFIG_ON = UCS_CONFIG_ON, UCC_CONFIG_AUTO = UCS_CONFIG_AUTO, UCC_CONFIG_ON_OFF_LAST } ucc_on_off_auto_value_t; enum tuning_mask { UCC_TUNING_DESC_FIELD_VENDOR = UCC_BIT(0), UCC_TUNING_DESC_FIELD_MODEL = UCC_BIT(1), UCC_TUNING_DESC_FIELD_TEAM_SIZE = UCC_BIT(2), UCC_TUNING_DESC_FIELD_PPN = UCC_BIT(3), UCC_TUNING_DESC_FIELD_NNODES = UCC_BIT(4), UCC_TUNING_DESC_FIELD_SOCK = UCC_BIT(5) }; typedef struct ucc_section_desc { uint64_t mask; ucc_cpu_vendor_t vendor; ucc_cpu_model_t model; ucc_rank_t min_team_size; ucc_rank_t max_team_size; ucc_rank_t min_ppn; ucc_rank_t max_ppn; ucc_rank_t min_sock; ucc_rank_t max_sock; ucc_rank_t min_nnodes; ucc_rank_t max_nnodes; } ucc_section_desc_t; KHASH_MAP_INIT_STR(ucc_sec, char *); typedef struct ucc_section_wrap { ucc_section_desc_t desc; khash_t(ucc_sec) vals_h; } ucc_section_wrap_t; KHASH_MAP_INIT_STR(ucc_cfg_file, char *); KHASH_MAP_INIT_STR(ucc_sections, ucc_section_wrap_t *); typedef struct ucc_file_config { char *filename; khash_t(ucc_cfg_file) vars; khash_t(ucc_sections) sections; } ucc_file_config_t; /* Convenience structure used, for example, to represent TLS list. "requested" field is set to 1 if the list of entries was explicitly requested by user. Union with allow_list is used in order to use this structure directly as config_field. */ typedef struct ucc_config_names_list { union { struct { ucc_config_names_array_t array; int requested; }; ucc_config_allow_list_t list; }; } ucc_config_names_list_t; ucc_status_t ucc_config_parser_fill_opts(void *opts, ucs_config_global_list_entry_t *entry, const char *env_prefix, int ignore_errors); ucc_status_t ucc_add_team_sections(void *team_cfg, ucc_config_field_t *tl_field, ucc_topo_t *team_topo, const char **tuning_str, const char *tune_key, const char *env_prefix, const char *component_prefix); static inline void ucc_config_parser_release_opts(void *opts, ucc_config_field_t *fields) { ucs_config_parser_release_opts(opts, fields); } static inline ucc_status_t ucc_config_parser_set_value(void *opts, ucc_config_field_t *fields, const char *name, const char *value) { ucs_status_t status; #if UCS_HAVE_PARSER_SET_VALUE_TABLE_PREFIX status = ucs_config_parser_set_value(opts, fields, NULL, name, value); #else status = ucs_config_parser_set_value(opts, fields, name, value); #endif return ucs_status_to_ucc_status(status); } static inline ucs_config_print_flags_t ucc_print_flags_to_ucs_print_flags(ucc_config_print_flags_t flags) { int ucs_flags = 0; if (flags & UCC_CONFIG_PRINT_CONFIG) { ucs_flags |= UCS_CONFIG_PRINT_CONFIG; } if (flags & UCC_CONFIG_PRINT_HEADER) { ucs_flags |= UCS_CONFIG_PRINT_HEADER; } if (flags & UCC_CONFIG_PRINT_DOC) { ucs_flags |= UCS_CONFIG_PRINT_DOC; } if (flags & UCC_CONFIG_PRINT_HIDDEN) { ucs_flags |= UCS_CONFIG_PRINT_HIDDEN; } return (ucs_config_print_flags_t)ucs_flags; } static inline void ucc_config_parser_print_opts(FILE *stream, const char *title, const void *opts, ucc_config_field_t *fields, const char *table_prefix, const char *prefix, ucc_config_print_flags_t flags) { ucs_config_print_flags_t ucs_flags; ucs_flags = ucc_print_flags_to_ucs_print_flags(flags); UCS_CONFIG_PARSER_PRINT_OPTS(stream, title, opts, fields, table_prefix, prefix, ucs_flags); } void ucc_config_parser_print_all_opts(FILE *stream, const char *prefix, ucc_config_print_flags_t flags, ucc_list_link_t * config_list); ucc_status_t ucc_config_names_array_dup(ucc_config_names_array_t *dst, const ucc_config_names_array_t *src); ucc_status_t ucc_config_names_array_merge(ucc_config_names_array_t *dst, const ucc_config_names_array_t *src); ucc_status_t ucc_config_names_array_to_string(const ucc_config_names_array_t *array, char *str, size_t max); void ucc_config_names_array_free(ucc_config_names_array_t *array); int ucc_config_names_search(const ucc_config_names_array_t *config_names, const char * str); static inline int ucc_config_names_array_is_all(const ucc_config_names_array_t *array) { return (array->count == 1) && (0 == strcmp(array->names[0], "all")); } ucc_status_t ucc_config_allow_list_process(const ucc_config_allow_list_t * list, const ucc_config_names_array_t *all, ucc_config_names_list_t * out); ucc_status_t ucc_parse_file_config(const char * filename, ucc_file_config_t **cfg); void ucc_release_file_config(ucc_file_config_t *cfg); typedef struct ucc_pipeline_params ucc_pipeline_params_t; ucc_status_t ucc_config_clone_table(const void *src, void *dst, const void *arg); int ucc_pipeline_params_is_auto(const ucc_pipeline_params_t *p); int ucc_config_sscanf_pipeline_params(const char *buf, void *dest, const void *arg); int ucc_config_sprintf_pipeline_params(char *buf, size_t max, const void *src, const void *arg); ucs_status_t ucc_config_clone_pipeline_params(const void *src, void *dest, const void *arg); void ucc_config_release_pipeline_params(void *ptr, const void *arg); int ucc_config_sscanf_uint_ranged(const char *buf, void *dest, const void *arg); int ucc_config_sprintf_uint_ranged(char *buf, size_t max, const void *src, const void *arg); ucs_status_t ucc_config_clone_uint_ranged(const void *src, void *dest, const void *arg); void ucc_config_release_uint_ranged(void *ptr, const void *arg); /** * Translate configuration value of "MEMUNITS" type to actual value. * * @param config_size Size specified by configuration. * @param auto_size Default size when configured to 'auto'. * @param max_size Maximal size to trim "inf". */ size_t ucc_config_memunits_get(size_t config_size, size_t auto_size, size_t max_size); #ifdef UCS_HAVE_PARSER_CONFIG_DOC #define UCC_CONFIG_TYPE_UINT_RANGED \ { \ ucc_config_sscanf_uint_ranged, ucc_config_sprintf_uint_ranged, \ ucc_config_clone_uint_ranged, ucc_config_release_uint_ranged, \ ucs_config_help_generic, ucs_config_doc_nop, \ "[-:[mtype]:value," \ "-:[mtype]:value,...,]default_value\n" \ "# value and default_value can be \"auto\"" \ } #define UCC_CONFIG_TYPE_PIPELINE_PARAMS \ { \ ucc_config_sscanf_pipeline_params, ucc_config_sprintf_pipeline_params, \ ucc_config_clone_pipeline_params, \ ucc_config_release_pipeline_params, ucs_config_help_generic, \ ucs_config_doc_nop, \ "thresh=:fragsize=:nfrags=" \ ":pdepth=:" \ } #else #define UCC_CONFIG_TYPE_UINT_RANGED \ { \ ucc_config_sscanf_uint_ranged, ucc_config_sprintf_uint_ranged, \ ucc_config_clone_uint_ranged, ucc_config_release_uint_ranged, \ ucs_config_help_generic, "[-:[mtype]:value," \ "-:[mtype]:value,...,]default_value\n" \ "# value and default_value can be \"auto\"" \ } #define UCC_CONFIG_TYPE_PIPELINE_PARAMS \ { \ ucc_config_sscanf_pipeline_params, ucc_config_sprintf_pipeline_params, \ ucc_config_clone_pipeline_params, \ ucc_config_release_pipeline_params, ucs_config_help_generic, \ "thresh=:fragsize=:nfrags=" \ ":pdepth=:" \ } #endif #endif ucc-1.8.0/src/utils/ucc_string.h0000664000175000017500000000667715211535620016736 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_STRING_H_ #define UCC_STRING_H_ #include "config.h" #include "ucc/api/ucc_def.h" #include #include #define ucc_memunits_range_str ucs_memunits_range_str char** ucc_str_split(const char *str, const char *delim); unsigned ucc_str_split_count(char **split); void ucc_str_split_free(char **split); ucc_status_t ucc_str_is_number(const char *str); ucc_status_t ucc_str_to_memunits(const char *buf, void *dest); /* Finds last occurence of pattern in string */ const char* ucc_strstr_last(const char* string, const char* pattern); /* Concatenates 2 strings. The space allocated for "out" must be released with ucc_free. */ ucc_status_t ucc_str_concat(const char *str1, const char *str2, char **out); ucc_status_t ucc_str_concat_n(const char *strs[], int n, char **out); ucc_status_t ucc_str_to_mtype_map(const char *str, const char* delim, uint32_t *mt_map); void ucc_mtype_map_to_str(uint32_t mt_map, const char *delim, char *buf, size_t max); ucc_status_t ucc_str_to_memunits_range(const char *str, size_t *start, size_t *end); /** * Find a string in a NULL-terminated array of strings. * * @param str String to search for. * @param string_list NULL-terminated array of strings. * @param case_sensitive Whether to perform case sensitive search. * * @return Index of the string in the array, or -1 if not found. */ ssize_t ucc_string_find_in_list(const char *str, const char **string_list, int case_sensitive); /** * Same as strncpy(), but guarantee that the last char in the buffer is '\0'. */ void ucc_strncpy_zero(char *dest, const char *src, size_t max); /** * Format a string to a buffer of given size, and fill the rest of the buffer * with '\0'. Also, guarantee that the last char in the buffer is '\0'. * * @param buf Buffer to format the string to. * @param size Buffer size. * @param fmt Format string. */ void ucc_snprintf_zero(char *buf, size_t size, const char *fmt, ...) UCC_F_PRINTF(3, 4); /** * Allocates a path buffer of size PATH_MAX. * * @param buffer_p Pointer to the buffer. * The buffer is allocated and should be released by the caller. * @param name Name of the buffer for logging. * * @return UCC_OK on success, UCC_ERR_NO_MEMORY on failure. */ ucc_status_t ucc_string_alloc_path_buffer(char **buffer_p, const char *name); /** * Fill a filename template. The following values in the string are replaced: * %p - replaced by process id * %h - replaced by host name * %c - replaced by the first CPU we are bound to * %t - replaced by local time * %u - replaced by user name * %e - replaced by executable basename * %i - replaced by user id * * @param tmpl File name template (possibly containing formatting sequences) * @param buf Filled with resulting file name * @param max Maximal size of destination buffer. */ void ucc_fill_filename_template(const char *tmpl, char *buf, size_t max); /** * Get pointer to file name in path, same as basename but do not * modify source string. * * @param path Path to parse. * * @return file name */ const char* ucc_basename(const char *path); char *ucc_strdup(const char *src, const char *name); #endif ucc-1.8.0/src/utils/khash.h0000664000175000017500000005765515211535620015676 0ustar alastairalastair/* The MIT License Copyright (c) 2008, 2009, 2011 by Attractive Chaos Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /* An example: #include "khash.h" KHASH_MAP_INIT_INT(32, char) int main() { int ret, is_missing; khiter_t k; khash_t(32) *h = kh_init(32); k = kh_put(32, h, 5, &ret); kh_value(h, k) = 10; k = kh_get(32, h, 10); is_missing = (k == kh_end(h)); k = kh_get(32, h, 5); kh_del(32, h, k); for (k = kh_begin(h); k != kh_end(h); ++k) if (kh_exist(h, k)) kh_value(h, k) = 1; kh_destroy(32, h); return 0; } */ /* 2013-05-02 (0.2.8): * Use quadratic probing. When the capacity is power of 2, stepping function i*(i+1)/2 guarantees to traverse each bucket. It is better than double hashing on cache performance and is more robust than linear probing. In theory, double hashing should be more robust than quadratic probing. However, my implementation is probably not for large hash tables, because the second hash function is closely tied to the first hash function, which reduce the effectiveness of double hashing. Reference: http://research.cs.vt.edu/AVresearch/hashing/quadratic.php 2011-12-29 (0.2.7): * Minor code clean up; no actual effect. 2011-09-16 (0.2.6): * The capacity is a power of 2. This seems to dramatically improve the speed for simple keys. Thank Zilong Tan for the suggestion. Reference: - http://code.google.com/p/ulib/ - http://nothings.org/computer/judy/ * Allow to optionally use linear probing which usually has better performance for random input. Double hashing is still the default as it is more robust to certain non-random input. * Added Wang's integer hash function (not used by default). This hash function is more robust to certain non-random input. 2011-02-14 (0.2.5): * Allow to declare global functions. 2009-09-26 (0.2.4): * Improve portability 2008-09-19 (0.2.3): * Corrected the example * Improved interfaces 2008-09-11 (0.2.2): * Improved speed a little in kh_put() 2008-09-10 (0.2.1): * Added kh_clear() * Fixed a compiling error 2008-09-02 (0.2.0): * Changed to token concatenation which increases flexibility. 2008-08-31 (0.1.2): * Fixed a bug in kh_get(), which has not been tested previously. 2008-08-31 (0.1.1): * Added destructor */ #ifndef __AC_KHASH_H #define __AC_KHASH_H /*! @header Generic hash table library. */ #define AC_VERSION_KHASH_H "0.2.8" #include #include #include /* Clang analyzer thinks that `h->flags` can be NULL, but this is * the wrong assumption - add `kassert()` to suppress the warning */ #ifdef __clang_analyzer__ #include #define kassert(...) assert(__VA_ARGS__) #define kmemset_analyzer(P, Z, N) kmemset(P, Z, N) #else #define kassert(...) #define kmemset_analyzer(P, Z, N) #endif /* compiler specific configuration */ #if UINT_MAX == 0xffffffffu typedef unsigned int khint32_t; #elif ULONG_MAX == 0xffffffffu typedef unsigned long khint32_t; #endif #if ULONG_MAX == ULLONG_MAX typedef unsigned long khint64_t; #else typedef unsigned long long khint64_t; #endif #ifndef kh_inline #ifdef _MSC_VER #define kh_inline __inline #else #define kh_inline inline #endif #endif /* kh_inline */ #ifndef klib_unused #if (defined __clang__ && __clang_major__ >= 3) || (defined __GNUC__ && __GNUC__ >= 3) #define klib_unused __attribute__ ((__unused__)) #else #define klib_unused #endif #endif /* klib_unused */ typedef khint32_t khint_t; typedef khint_t khiter_t; #define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2) #define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1) #define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3) #define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1))) #define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1))) #define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1))) #define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1)) #define __ac_fsize(m) ((m) < 16? 1 : (m)>>4) #ifndef kroundup32 #define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) #endif #ifndef kcalloc #define kcalloc(N,Z) calloc(N,Z) #endif #ifndef kmalloc #define kmalloc(Z) malloc(Z) #endif #ifndef krealloc #define krealloc(P,Z) realloc(P,Z) #endif #ifndef kfree #define kfree(P) free(P) #endif #ifndef kmemset #define kmemset(P,Z,N) memset(P,Z,N) #endif static const double __ac_HASH_UPPER = 0.77; #define __KHASH_TYPE(name, khkey_t, khval_t) \ typedef struct kh_##name##_s { \ khint_t n_buckets, size, n_occupied, upper_bound; \ khint32_t *flags; \ khkey_t *keys; \ khval_t *vals; \ } kh_##name##_t; #define __KHASH_PROTOTYPES(name, khkey_t, khval_t) \ extern kh_##name##_t *kh_init_##name(void); \ extern kh_##name##_t *kh_init_##name##_inplace(kh_##name##_t *h); \ extern void kh_destroy_##name(kh_##name##_t *h); \ extern void kh_destroy_##name##_inplace(kh_##name##_t *h); \ extern void kh_clear_##name(kh_##name##_t *h); \ extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \ extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \ extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \ extern void kh_del_##name(kh_##name##_t *h, khint_t x); #define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ SCOPE kh_##name##_t *kh_init_##name(void) { \ return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t)); \ } \ SCOPE kh_##name##_t *kh_init_##name##_inplace(kh_##name##_t *h) { \ return (kh_##name##_t*)kmemset(h, 0, sizeof(kh_##name##_t)); \ } \ SCOPE void kh_destroy_##name(kh_##name##_t *h) \ { \ if (h) { \ kfree((void *)h->keys); kfree(h->flags); \ kfree((void *)h->vals); \ kfree(h); \ } \ } \ SCOPE void kh_destroy_##name##_inplace(kh_##name##_t *h) \ { \ kfree((void *)h->keys); \ kfree((void *)h->flags); \ kfree((void *)h->vals); \ } \ SCOPE void kh_clear_##name(kh_##name##_t *h) \ { \ if (h && h->flags) { \ memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \ h->size = h->n_occupied = 0; \ } \ } \ SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \ { \ if (h->n_buckets) { \ khint_t k, i, last, mask, step = 0; \ mask = h->n_buckets - 1; \ \ kassert(h->flags != NULL); \ \ k = __hash_func(key); i = k & mask; \ last = i; \ while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ i = (i + (++step)) & mask; \ if (i == last) return h->n_buckets; \ } \ return __ac_iseither(h->flags, i)? h->n_buckets : i; \ } else return 0; \ } \ SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \ { /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \ khint32_t *new_flags = 0; \ khint_t j = 1; \ { \ kroundup32(new_n_buckets); \ if (new_n_buckets < 4) new_n_buckets = 4; \ if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; /* requested size is too small */ \ else { /* hash table size to be changed (shrink or expand); rehash */ \ new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ if (!new_flags) return -1; \ memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ if (h->n_buckets < new_n_buckets) { /* expand */ \ khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ if (!new_keys) { kfree(new_flags); return -1; } \ h->keys = new_keys; \ kmemset_analyzer(h->keys + (h->n_buckets * sizeof(khkey_t)), 0, \ (new_n_buckets - h->n_buckets) * sizeof(khkey_t)); \ if (kh_is_map) { \ khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ if (!new_vals) { kfree(new_flags); return -1; } \ h->vals = new_vals; \ } \ } /* otherwise shrink */ \ } \ } \ if (j) { /* rehashing is needed */ \ for (j = 0; j != h->n_buckets; ++j) { \ if (__ac_iseither(h->flags, j) == 0) { \ khkey_t key = h->keys[j]; \ khval_t val; \ khint_t new_mask; \ new_mask = new_n_buckets - 1; \ if (kh_is_map) val = h->vals[j]; \ __ac_set_isdel_true(h->flags, j); \ while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \ khint_t k, i, step = 0; \ k = __hash_func(key); \ i = k & new_mask; \ while (!__ac_isempty(new_flags, i)) i = (i + (++step)) & new_mask; \ __ac_set_isempty_false(new_flags, i); \ if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \ { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \ if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \ __ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \ } else { /* write the element and jump out of the loop */ \ h->keys[i] = key; \ if (kh_is_map) h->vals[i] = val; \ break; \ } \ } \ } \ } \ if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \ h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ } \ kfree(h->flags); /* free the working space */ \ h->flags = new_flags; \ h->n_buckets = new_n_buckets; \ h->n_occupied = h->size; \ h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \ } \ return 0; \ } \ SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \ { \ khint_t x; \ if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \ if (h->n_buckets > (h->size<<1)) { \ if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \ *ret = -1; return h->n_buckets; \ } \ } else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \ *ret = -1; return h->n_buckets; \ } \ } /* TODO: to implement automatically shrinking; resize() already support shrinking */ \ \ kassert(h->flags != NULL); \ \ { \ khint_t k, i, site, last, mask = h->n_buckets - 1, step = 0; \ x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \ if (__ac_isempty(h->flags, i)) x = i; /* for speed up */ \ else { \ last = i; \ while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ if (__ac_isdel(h->flags, i)) site = i; \ i = (i + (++step)) & mask; \ if (i == last) { x = site; break; } \ } \ if (x == h->n_buckets) { \ if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \ else x = i; \ } \ } \ } \ if (__ac_isempty(h->flags, x)) { /* not present at all */ \ h->keys[x] = key; \ __ac_set_isboth_false(h->flags, x); \ ++h->size; ++h->n_occupied; \ *ret = 1; \ } else if (__ac_isdel(h->flags, x)) { /* deleted */ \ h->keys[x] = key; \ __ac_set_isboth_false(h->flags, x); \ ++h->size; \ *ret = 2; \ } else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \ return x; \ } \ SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \ { \ if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \ __ac_set_isdel_true(h->flags, x); \ --h->size; \ } \ } #define KHASH_DECLARE(name, khkey_t, khval_t) \ __KHASH_TYPE(name, khkey_t, khval_t) \ __KHASH_PROTOTYPES(name, khkey_t, khval_t) #define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ __KHASH_TYPE(name, khkey_t, khval_t) \ __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) #define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ KHASH_INIT2(name, static kh_inline klib_unused, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) #define KHASH_TYPE(name, khkey_t, khval_t) \ __KHASH_TYPE(name, khkey_t, khval_t) #define KHASH_IMPL(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ __KHASH_IMPL(name, static kh_inline klib_unused, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) /* --- BEGIN OF HASH FUNCTIONS --- */ /*! @function @abstract Integer hash function @param key The integer [khint32_t] @return The hash value [khint_t] */ #define kh_int_hash_func(key) (khint32_t)(key) /*! @function @abstract Integer comparison function */ #define kh_int_hash_equal(a, b) ((a) == (b)) /*! @function @abstract 64-bit integer hash function @param key The integer [khint64_t] @return The hash value [khint_t] */ #define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11) /*! @function @abstract 64-bit integer comparison function */ #define kh_int64_hash_equal(a, b) ((a) == (b)) /*! @function @abstract const char* hash function @param s Pointer to a null terminated string @return The hash value */ static kh_inline khint_t __ac_X31_hash_string(const char *s) { khint_t h = (khint_t)*s; if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s; return h; } /*! @function @abstract Another interface to const char* hash function @param key Pointer to a null terminated string [const char*] @return The hash value [khint_t] */ #define kh_str_hash_func(key) __ac_X31_hash_string(key) /*! @function @abstract Const char* comparison function */ #define kh_str_hash_equal(a, b) (strcmp(a, b) == 0) /* NOLINT */ static kh_inline khint_t __ac_Wang_hash(khint_t key) { key += ~(key << 15); key ^= (key >> 10); key += (key << 3); key ^= (key >> 6); key += ~(key << 11); key ^= (key >> 16); return key; } #define kh_int_hash_func2(key) __ac_Wang_hash((khint_t)key) /* --- END OF HASH FUNCTIONS --- */ /* Other convenient macros... */ /*! @abstract Type of the hash table. @param name Name of the hash table [symbol] */ #define khash_t(name) kh_##name##_t /*! @function @abstract Initiate a hash table. @param name Name of the hash table [symbol] @return Pointer to the hash table [khash_t(name)*] */ #define kh_init(name) kh_init_##name() /*! @function @abstract Initiate a hash table if the in-place case. @param name Name of the hash table [symbol] @param h Pointer to the hash table [khash_t(name)*] */ #define kh_init_inplace(name, h) kh_init_##name##_inplace(h) /*! @function @abstract Destroy a hash table. @param name Name of the hash table [symbol] @param h Pointer to the hash table [khash_t(name)*] */ #define kh_destroy(name, h) kh_destroy_##name(h) /*! @function @abstract Destroy a hash table if the in-place case. @param name Name of the hash table [symbol] @param h Pointer to the hash table [khash_t(name)*] */ #define kh_destroy_inplace(name, h) kh_destroy_##name##_inplace(h) /*! @function @abstract Reset a hash table without deallocating memory. @param name Name of the hash table [symbol] @param h Pointer to the hash table [khash_t(name)*] */ #define kh_clear(name, h) kh_clear_##name(h) /*! @function @abstract Resize a hash table. @param name Name of the hash table [symbol] @param h Pointer to the hash table [khash_t(name)*] @param s New size [khint_t] */ #define kh_resize(name, h, s) kh_resize_##name(h, s) /*! @function @abstract Insert a key to the hash table. @param name Name of the hash table [symbol] @param h Pointer to the hash table [khash_t(name)*] @param k Key [type of keys] @param r Extra return code: -1 if the operation failed; 0 if the key is present in the hash table; 1 if the bucket is empty (never used); 2 if the element in the bucket has been deleted [int*] @return Iterator to the inserted element [khint_t] */ #define kh_put(name, h, k, r) kh_put_##name(h, k, r) typedef enum ucs_kh_put { UCS_KH_PUT_FAILED = -1, UCS_KH_PUT_KEY_PRESENT = 0, UCS_KH_PUT_BUCKET_EMPTY = 1, UCS_KH_PUT_BUCKET_CLEAR = 2 } ucs_kh_put_t; /*! @function @abstract Retrieve a key from the hash table. @param name Name of the hash table [symbol] @param h Pointer to the hash table [khash_t(name)*] @param k Key [type of keys] @return Iterator to the found element, or kh_end(h) if the element is absent [khint_t] */ #define kh_get(name, h, k) kh_get_##name(h, k) /*! @function @abstract Remove a key from the hash table. @param name Name of the hash table [symbol] @param h Pointer to the hash table [khash_t(name)*] @param k Iterator to the element to be deleted [khint_t] */ #define kh_del(name, h, k) kh_del_##name(h, k) /*! @function @abstract Test whether a bucket contains data. @param h Pointer to the hash table [khash_t(name)*] @param x Iterator to the bucket [khint_t] @return 1 if containing data; 0 otherwise [int] */ #define kh_exist(h, x) (!__ac_iseither((h)->flags, (x))) /*! @function @abstract Get key given an iterator @param h Pointer to the hash table [khash_t(name)*] @param x Iterator to the bucket [khint_t] @return Key [type of keys] */ #define kh_key(h, x) ((h)->keys[x]) /*! @function @abstract Get value given an iterator @param h Pointer to the hash table [khash_t(name)*] @param x Iterator to the bucket [khint_t] @return Value [type of values] @discussion For hash sets, calling this results in segfault. */ #define kh_val(h, x) ((h)->vals[x]) /*! @function @abstract Alias of kh_val() */ #define kh_value(h, x) ((h)->vals[x]) /*! @function @abstract Get the start iterator @param h Pointer to the hash table [khash_t(name)*] @return The start iterator [khint_t] */ #define kh_begin(h) (khint_t)(0) /*! @function @abstract Get the end iterator @param h Pointer to the hash table [khash_t(name)*] @return The end iterator [khint_t] */ #define kh_end(h) ((h)->n_buckets) /*! @function @abstract Get the number of elements in the hash table @param h Pointer to the hash table [khash_t(name)*] @return Number of elements in the hash table [khint_t] */ #define kh_size(h) ((h)->size) /*! @function @abstract Get the number of buckets in the hash table @param h Pointer to the hash table [khash_t(name)*] @return Number of buckets in the hash table [khint_t] */ #define kh_n_buckets(h) ((h)->n_buckets) /*! @function @abstract Iterate over the entries in the hash table @param h Pointer to the hash table [khash_t(name)*] @param kvar Variable to which key will be assigned @param vvar Variable to which value will be assigned @param code Block of code to execute */ #define kh_foreach(h, kvar, vvar, code) { khint_t __i; \ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \ if (!kh_exist(h,__i)) continue; \ (kvar) = kh_key(h,__i); \ (vvar) = kh_val(h,__i); \ code; \ } } /*! @function @abstract Iterate over the keys in the hash table @param h Pointer to the hash table [khash_t(name)*] @param kvar Variable to which key will be assigned @param code Block of code to execute */ #define kh_foreach_key(h, kvar, code) { khint_t __i; \ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \ if (!kh_exist(h,__i)) continue; \ (kvar) = kh_key(h,__i); \ code; \ } } /*! @function @abstract Iterate over the values in the hash table @param h Pointer to the hash table [khash_t(name)*] @param vvar Variable to which value will be assigned @param code Block of code to execute */ #define kh_foreach_value(h, vvar, code) { khint_t __i; \ for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \ if (!kh_exist(h,__i)) continue; \ (vvar) = kh_val(h,__i); \ code; \ } } /* More conenient interfaces */ /*! @function @abstract Instantiate a hash set containing integer keys @param name Name of the hash table [symbol] */ #define KHASH_SET_INIT_INT(name) \ KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal) /*! @function @abstract Instantiate a hash map containing integer keys @param name Name of the hash table [symbol] @param khval_t Type of values [type] */ #define KHASH_MAP_INIT_INT(name, khval_t) \ KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal) /*! @function @abstract Instantiate a hash map containing 64-bit integer keys @param name Name of the hash table [symbol] */ #define KHASH_SET_INIT_INT64(name) \ KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal) /*! @function @abstract Instantiate a hash map containing 64-bit integer keys @param name Name of the hash table [symbol] @param khval_t Type of values [type] */ #define KHASH_MAP_INIT_INT64(name, khval_t) \ KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal) typedef const char *kh_cstr_t; /*! @function @abstract Instantiate a hash map containing const char* keys @param name Name of the hash table [symbol] */ #define KHASH_SET_INIT_STR(name) \ KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal) /*! @function @abstract Instantiate a hash map containing const char* keys @param name Name of the hash table [symbol] @param khval_t Type of values [type] */ #define KHASH_MAP_INIT_STR(name, khval_t) \ KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal) #endif /* __AC_KHASH_H */ ucc-1.8.0/src/utils/ucc_lock_free_queue.h0000664000175000017500000000673315211535620020556 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_LOCKFREE_QUEUE_H_ #define UCC_LOCKFREE_QUEUE_H_ #include "utils/ucc_spinlock.h" #include "utils/ucc_atomic.h" #include "utils/ucc_list.h" #include /* This data structure is thread safe */ // Number of elements in a single lock free pool - could be changed, but in tests performed great #define LINE_SIZE 8 #define NUM_POOLS 2 typedef struct ucc_lf_queue_elem { uint8_t was_queued; ucc_list_link_t locked_list_elem; } ucc_lf_queue_elem_t; typedef struct ucc_lf_queue { ucc_spinlock_t locked_queue_lock[NUM_POOLS]; ucc_lf_queue_elem_t *elements[NUM_POOLS][LINE_SIZE]; uint8_t which_pool; ucc_list_link_t locked_queue[NUM_POOLS]; } ucc_lf_queue_t; static inline void ucc_lf_queue_init_elem(ucc_lf_queue_elem_t *elem){ elem->was_queued = 0; } static inline void ucc_lf_queue_enqueue(ucc_lf_queue_t * queue, ucc_lf_queue_elem_t *elem) { uint8_t which_pool = elem->was_queued ^ (queue->which_pool & 1); int i; for (i = 0; i < LINE_SIZE; i++) { if (ucc_atomic_bool_cswap64( (uint64_t *)&(queue->elements[which_pool][i]), 0LL, (uint64_t)elem)) { return; } } ucc_spin_lock(&queue->locked_queue_lock[which_pool]); ucc_list_add_tail(&queue->locked_queue[which_pool], &elem->locked_list_elem); ucc_spin_unlock(&queue->locked_queue_lock[which_pool]); } static inline ucc_lf_queue_elem_t *ucc_lf_queue_dequeue(ucc_lf_queue_t *queue, int is_first_call) { // Save value in the beginning of the function uint8_t curr_which_pool = queue->which_pool; uint8_t which_pool = curr_which_pool & 1; // turn from even/odd -> bool int i; ucc_lf_queue_elem_t *elem; for (i = 0; i < LINE_SIZE; i++) { elem = queue->elements[which_pool][i]; if (elem) { if (ucc_atomic_bool_cswap64( (uint64_t *)&(queue->elements[which_pool][i]), (uint64_t)elem, 0LL)) { elem->was_queued = 1; return elem; } } } elem = NULL; ucc_spin_lock(&queue->locked_queue_lock[which_pool]); if (!ucc_list_is_empty(&queue->locked_queue[which_pool])) { elem = ucc_list_extract_head(&queue->locked_queue[which_pool], ucc_lf_queue_elem_t, locked_list_elem); elem->was_queued = 1; } ucc_spin_unlock(&queue->locked_queue_lock[which_pool]); if (!elem) { if (ucc_atomic_bool_cswap8(&queue->which_pool, curr_which_pool, curr_which_pool + 1) && is_first_call) { return ucc_lf_queue_dequeue(queue, 0); } } return elem; } static inline void ucc_lf_queue_destroy(ucc_lf_queue_t *queue) { ucc_spinlock_destroy(&queue->locked_queue_lock[0]); ucc_spinlock_destroy(&queue->locked_queue_lock[1]); } static inline void ucc_lf_queue_init(ucc_lf_queue_t *queue) { memset(&queue->elements, 0, NUM_POOLS * LINE_SIZE * sizeof(ucc_lf_queue_elem_t *)); ucc_spinlock_init(&queue->locked_queue_lock[0], 0); ucc_spinlock_init(&queue->locked_queue_lock[1], 0); ucc_list_head_init(&queue->locked_queue[0]); ucc_list_head_init(&queue->locked_queue[1]); queue->which_pool = 0; } #endif ucc-1.8.0/src/utils/ucc_math.c0000664000175000017500000000146715211535620016344 0ustar alastairalastair/** * Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "ucc/api/ucc.h" #include "ucc_math.h" static int _compare(const void *a, const void *b) { return (*(int *)a - *(int *)b); } static int _compare_inv(const void *a, const void *b) { return (*(int *)b - *(int *)a); } static inline int _unique(int *first, int *last) { int *begin = first; int *result = first; if (first == last) { return 1; } while (++first != last) { if (!(*result == *first)) { *(++result) = *first; } } return (++result - begin); } int ucc_sort_uniq(int *array, int len, int inverse) { qsort(array, len, sizeof(int), inverse ? _compare_inv : _compare); return _unique(array, array + len); } ucc-1.8.0/src/utils/ucc_mpool.c0000664000175000017500000000676715211535620016551 0ustar alastairalastair#include "ucc_mpool.h" #include "ucc_malloc.h" #include "ucc_log.h" static ucc_mpool_ops_t ucc_default_mpool_ops = { .chunk_alloc = ucc_mpool_hugetlb_malloc, .chunk_release = ucc_mpool_hugetlb_free, .obj_init = NULL, .obj_cleanup = NULL }; static ucs_status_t ucc_mpool_chunk_alloc_wrapper(ucs_mpool_t *mp, size_t *size_p, void **chunk_p) { ucc_mpool_t *mpool = ucc_derived_of(mp, ucc_mpool_t); ucc_status_t st; st = mpool->ucc_ops->chunk_alloc(mpool, size_p, chunk_p); return ucc_status_to_ucs_status(st); } static void ucc_mpool_chunk_release_wrapper(ucs_mpool_t *mp, void *chunk) { ucc_mpool_t *mpool = ucc_derived_of(mp, ucc_mpool_t); mpool->ucc_ops->chunk_release(mpool, chunk); } static void ucc_mpool_obj_init_wrapper(ucs_mpool_t *mp, void *obj, void *chunk) { ucc_mpool_t *mpool = ucc_derived_of(mp, ucc_mpool_t); mpool->ucc_ops->obj_init(mpool, obj, chunk); } static void ucc_mpool_obj_cleanup_wrapper(ucs_mpool_t *mp, void *obj) { ucc_mpool_t *mpool = ucc_derived_of(mp, ucc_mpool_t); mpool->ucc_ops->obj_cleanup(mpool, obj); } ucc_status_t ucc_mpool_init(ucc_mpool_t *mp, size_t priv_size, size_t elem_size, size_t align_offset, size_t alignment, unsigned elems_per_chunk, unsigned max_elems, ucc_mpool_ops_t *ops, ucc_thread_mode_t tm, const char *name) { ucs_mpool_ops_t *ucs_ops = ucc_calloc(1, sizeof(*ucs_ops), "mpool_ops"); #if UCS_HAVE_MPOOL_PARAMS ucs_mpool_params_t params; #endif if (!ucs_ops) { ucc_error("failed to allocate %zd bytes for mpool ucs ops", sizeof(*ucs_ops)); return UCC_ERR_NO_MEMORY; } ucc_spinlock_init(&mp->lock, 0); mp->tm = tm; mp->ucc_ops = ops ? ops : &ucc_default_mpool_ops; ucs_ops->chunk_alloc = ucc_mpool_chunk_alloc_wrapper; ucs_ops->chunk_release = ucc_mpool_chunk_release_wrapper; if (mp->ucc_ops->obj_init != NULL) { ucs_ops->obj_init = ucc_mpool_obj_init_wrapper; } if (mp->ucc_ops->obj_cleanup != NULL) { ucs_ops->obj_cleanup = ucc_mpool_obj_cleanup_wrapper; } #if UCS_HAVE_MPOOL_PARAMS ucs_mpool_params_reset(¶ms); params.priv_size = priv_size; params.elem_size = elem_size; params.align_offset = align_offset; params.alignment = alignment; params.malloc_safe = 0; params.elems_per_chunk = elems_per_chunk; params.max_chunk_size = SIZE_MAX; params.max_elems = max_elems; params.grow_factor = 1.0; params.ops = ucs_ops; params.name = name; return ucs_status_to_ucc_status(ucs_mpool_init(¶ms, &mp->super)); #else return ucs_status_to_ucc_status( ucs_mpool_init(&mp->super, priv_size, elem_size, align_offset, alignment, elems_per_chunk, max_elems, ucs_ops, name)); #endif } void ucc_mpool_cleanup(ucc_mpool_t *mp, int leak_check) { void *ops = (void*)mp->super.data->ops; ucs_mpool_cleanup(&mp->super, leak_check); ucc_free(ops); ucc_spinlock_destroy(&mp->lock); } ucc_status_t ucc_mpool_hugetlb_malloc(ucc_mpool_t *mp, size_t *size_p, void **chunk_p) { ucs_status_t st; st = ucs_mpool_hugetlb_malloc(&mp->super, size_p, chunk_p); return ucs_status_to_ucc_status(st); } void ucc_mpool_hugetlb_free(ucc_mpool_t *mp, void *chunk) { ucs_mpool_hugetlb_free(&mp->super, chunk); } ucc-1.8.0/src/utils/ucc_component.c0000664000175000017500000002137215211535620017412 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "config.h" #include "ucc_malloc.h" #include "ucc_component.h" #include "ucc_log.h" #include "ucc_math.h" #include "core/ucc_global_opts.h" #include "utils/ucc_string.h" #include #include #include #include #include #include #include #include #define IFACE_NAME_LEN_MAX \ (UCC_MAX_FRAMEWORK_NAME_LEN + UCC_MAX_COMPONENT_NAME_LEN + 32) static ucc_status_t ucc_component_load_one(const char *so_path, const char *framework_name, ucc_component_iface_t **c_iface) { char framework_pattern[UCC_MAX_FRAMEWORK_NAME_LEN + 16]; char *error, iface_struct[IFACE_NAME_LEN_MAX]; void *handle; ucc_component_iface_t *iface; ptrdiff_t basename_start; size_t iface_struct_name_len; ucc_snprintf_safe(framework_pattern, sizeof(framework_pattern), "ucc_%s_", framework_name); basename_start = ((ptrdiff_t)ucc_strstr_last(so_path, framework_pattern) - (ptrdiff_t)so_path); if (basename_start < 0) { return UCC_ERR_NO_MESSAGE; } /* The name of the iface stract matches the basename of .so component object. basename_start - the starting position of the component name in the full .so path. The name_len is also decreased by 3 to remove ".so" extension from the name; */ iface_struct_name_len = strlen(so_path) - basename_start - 3; ucc_strncpy_safe(iface_struct, so_path + basename_start, iface_struct_name_len + 1); handle = dlopen(so_path, RTLD_LAZY); if (!handle) { error = dlerror(); ucc_debug("failed to load UCC component library: %s (%s)", so_path, error); goto error; } iface = (ucc_component_iface_t *)dlsym(handle, iface_struct); if ((error = dlerror()) != NULL) { ucc_error("failed to get iface %s from %s object (%s)", iface_struct, so_path, error); goto iface_error; } if (!iface) { ucc_error("iface %s is NULL in %s object", iface_struct, so_path); goto iface_error; } iface->dl_handle = handle; iface->id = ucc_str_hash_djb2(iface->name); (*c_iface) = iface; return UCC_OK; iface_error: dlclose(handle); error: *c_iface = NULL; return UCC_ERR_NO_MESSAGE; } #define CHECK_COMPONENT_UNIQ(_framework, _field) \ do { \ ucc_component_iface_t **c = _framework->components; \ int i, j; \ for (i = 0; i < framework->n_components; i++) { \ for (j = i + 1; j < framework->n_components; j++) { \ if (c[i]->_field == c[j]->_field) { \ ucc_error("components %s and %s have the same " \ "default " UCC_PP_MAKE_STRING(_field) " %lu", \ c[i]->name, c[j]->name, \ (unsigned long)c[i]->_field); \ return UCC_ERR_INVALID_PARAM; \ } \ } \ } \ } while (0) ucc_status_t ucc_component_check_scores_uniq(ucc_component_framework_t *framework) { CHECK_COMPONENT_UNIQ(framework, score); return UCC_OK; } static ucc_status_t ucc_component_check_ids_uniq(ucc_component_framework_t *framework) { CHECK_COMPONENT_UNIQ(framework, id); return UCC_OK; } ucc_status_t ucc_components_load(const char *framework_name, ucc_component_framework_t *framework) { glob_t globbuf; int i, n_loaded; char *full_pattern; ucc_status_t status; size_t pattern_size; ucc_component_iface_t **ifaces = NULL; framework->n_components = 0; framework->components = NULL; if (strlen(framework_name) == 0 || strlen(framework_name) > UCC_MAX_FRAMEWORK_NAME_LEN) { ucc_error("unsupported framework_name length: %s, len %zd", framework_name, strlen(framework_name)); return UCC_ERR_INVALID_PARAM; } pattern_size = strlen(ucc_global_config.component_path) + strlen(framework_name) + 16; full_pattern = (char *)ucc_malloc(pattern_size, "full_pattern"); if (!full_pattern) { ucc_error("failed to allocate %zd bytes for full_pattern", pattern_size); return UCC_ERR_NO_MEMORY; } ucc_snprintf_safe(full_pattern, pattern_size, "%s/libucc_%s_*.so", ucc_global_config.component_path, framework_name); glob(full_pattern, 0, NULL, &globbuf); ucc_free(full_pattern); n_loaded = 0; dlerror(); /* Clear any existing error */ ifaces = (ucc_component_iface_t **)ucc_malloc( globbuf.gl_pathc * sizeof(ucc_component_iface_t *), "ifaces"); if (!ifaces) { ucc_error("failed to allocate %zd bytes for ifaces", globbuf.gl_pathc * sizeof(ucc_component_iface_t *)); return UCC_ERR_NO_MEMORY; } for (i = 0; i < globbuf.gl_pathc; i++) { status = ucc_component_load_one(globbuf.gl_pathv[i], framework_name, &ifaces[n_loaded]); if (status != UCC_OK) { continue; } n_loaded++; } assert(n_loaded <= globbuf.gl_pathc); if (globbuf.gl_pathc > 0) { globfree(&globbuf); } if (!n_loaded) { if (ifaces) { ucc_free(ifaces); } return UCC_ERR_NOT_FOUND; } ifaces = ucc_realloc(ifaces, n_loaded * sizeof(ucc_component_iface_t *), "ifaces"); framework->components = ifaces; framework->n_components = n_loaded; if (UCC_OK != ucc_component_check_ids_uniq(framework)) { /* This can only happen when the new component is added (potentially as a plugin - black box) and its name hash will produce collision with some other component. This has nearly 0 probability and must be resolved by the component developer via just a rename of a component. */ ucc_error("all components of a framwork must have uniq name hash"); status = UCC_ERR_INVALID_PARAM; goto err; } framework->names.names = ucc_malloc(sizeof(char *) * n_loaded, "components_names"); if (!framework->names.names) { ucc_error("failed to allocate %zd bytes for components names", sizeof(char *) * n_loaded); status = UCC_ERR_NO_MEMORY; goto err; } framework->names.count = n_loaded; for (i = 0; i < n_loaded; i++) { framework->names.names[i] = strdup(framework->components[i]->name); } return UCC_OK; err: ucc_free(framework->components); return status; } ucc_component_iface_t* ucc_get_component(ucc_component_framework_t *framework, const char *component_name) { int i; for (i = 0; i < framework->n_components; i++) { if (0 == strcmp(framework->components[i]->name, component_name)) { return framework->components[i]; } } return NULL; } char* ucc_get_framework_components_list(ucc_component_framework_t *framework, const char* delimiter) { char *list = NULL; size_t len = 0; int i; for (i = 0; i < framework->n_components; i++) { /* component name + ',' delimiter */ len += strlen(framework->components[i]->name) + 1; } if (len) { list = ucc_malloc(len + 1, "components_list"); if (!list) { ucc_error("failed to allocate %zd bytes for components_list", len); return NULL; } list[0] = '\0'; for (i = 0; i < framework->n_components; i++) { strncat(list, framework->components[i]->name, len); len -= strlen(framework->components[i]->name); if (i < framework->n_components - 1) { strncat(list, delimiter, len); len -= strlen(delimiter); } } } return list; } ucc-1.8.0/src/utils/ucc_log.h0000664000175000017500000001136515211535620016177 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_LOG_H_ #define UCC_LOG_H_ #include "config.h" #include "core/ucc_global_opts.h" #include "core/ucc_dt.h" #include "utils/ucc_mem_type.h" #include "utils/debug/log_def.h" /* Collective trace logger */ #define ucc_log_component_collective_trace(_level, fmt, ...) \ ucc_log_component(_level, &ucc_global_config.coll_trace, fmt, \ ##__VA_ARGS__) #define ucc_coll_trace_info(_fmt, ...) \ ucc_log_component_collective_trace(UCC_LOG_LEVEL_INFO, _fmt, ##__VA_ARGS__) #define ucc_coll_trace_debug(_fmt, ...) \ ucc_log_component_collective_trace(UCC_LOG_LEVEL_DEBUG, _fmt, ##__VA_ARGS__) static inline const char* ucc_coll_type_str(ucc_coll_type_t ct) { switch(ct) { case UCC_COLL_TYPE_BARRIER: return "Barrier"; case UCC_COLL_TYPE_BCAST: return "Bcast"; case UCC_COLL_TYPE_ALLREDUCE: return "Allreduce"; case UCC_COLL_TYPE_REDUCE: return "Reduce"; case UCC_COLL_TYPE_ALLTOALL: return "Alltoall"; case UCC_COLL_TYPE_ALLTOALLV: return "Alltoallv"; case UCC_COLL_TYPE_ALLGATHER: return "Allgather"; case UCC_COLL_TYPE_ALLGATHERV: return "Allgatherv"; case UCC_COLL_TYPE_GATHER: return "Gather"; case UCC_COLL_TYPE_GATHERV: return "Gatherv"; case UCC_COLL_TYPE_SCATTER: return "Scatter"; case UCC_COLL_TYPE_SCATTERV: return "Scatterv"; case UCC_COLL_TYPE_FANIN: return "Fanin"; case UCC_COLL_TYPE_FANOUT: return "Fanout"; case UCC_COLL_TYPE_REDUCE_SCATTER: return "Reduce_scatter"; case UCC_COLL_TYPE_REDUCE_SCATTERV: return "Reduce_scatterv"; default: break; } return ""; } static inline const char* ucc_datatype_str(ucc_datatype_t dt) { switch (dt) { case UCC_DT_INT8: return "int8"; case UCC_DT_UINT8: return "uint8"; case UCC_DT_INT16: return "int16"; case UCC_DT_UINT16: return "uint16"; case UCC_DT_FLOAT16: return "float16"; case UCC_DT_BFLOAT16: return "bfloat16"; case UCC_DT_INT32: return "int32"; case UCC_DT_UINT32: return "uint32"; case UCC_DT_FLOAT32: return "float32"; case UCC_DT_INT64: return "int64"; case UCC_DT_UINT64: return "uint64"; case UCC_DT_FLOAT64: return "float64"; case UCC_DT_FLOAT128: return "float128"; case UCC_DT_INT128: return "int128"; case UCC_DT_UINT128: return "uint128"; case UCC_DT_FLOAT32_COMPLEX: return "float32_complex"; case UCC_DT_FLOAT64_COMPLEX: return "float64_complex"; case UCC_DT_FLOAT128_COMPLEX: return "float128_complex"; default: return "userdefined"; } } static inline const char* ucc_reduction_op_str(ucc_reduction_op_t op) { switch(op) { case UCC_OP_SUM: return "sum"; case UCC_OP_PROD: return "prod"; case UCC_OP_MAX: return "max"; case UCC_OP_MIN: return "min"; case UCC_OP_LAND: return "land"; case UCC_OP_LOR: return "lor"; case UCC_OP_LXOR: return "lxor"; case UCC_OP_BAND: return "band"; case UCC_OP_BOR: return "bor"; case UCC_OP_BXOR: return "bxor"; case UCC_OP_MAXLOC: return "maxloc"; case UCC_OP_MINLOC: return "minloc"; case UCC_OP_AVG: return "avg"; default: return NULL; } } static inline const char* ucc_mem_type_str(ucc_memory_type_t ct) { switch((int)ct) { case UCC_MEMORY_TYPE_HOST: return "Host"; case UCC_MEMORY_TYPE_CUDA: return "Cuda"; case UCC_MEMORY_TYPE_CUDA_MANAGED: return "CudaManaged"; case UCC_MEMORY_TYPE_ROCM: return "Rocm"; case UCC_MEMORY_TYPE_ROCM_MANAGED: return "RocmManaged"; case UCC_MEMORY_TYPE_ASYMMETRIC: return "asymmetric"; case UCC_MEMORY_TYPE_NOT_APPLY: return "n/a"; default: break; } return "invalid"; } static inline const char* ucc_thread_mode_str(ucc_thread_mode_t tm) { switch(tm) { case UCC_THREAD_SINGLE: return "single"; case UCC_THREAD_FUNNELED: return "funneled"; case UCC_THREAD_MULTIPLE: return "multiple"; } return "invalid"; } static inline const char* ucc_context_type_str(ucc_context_type_t ct) { switch(ct) { case UCC_CONTEXT_EXCLUSIVE: return "exclusive"; case UCC_CONTEXT_SHARED: return "shared"; } return "invalid"; } #endif ucc-1.8.0/src/utils/ucc_math.h0000664000175000017500000001067415211535620016351 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_MATH_H_ #define UCC_MATH_H_ #include "config.h" #include #include "ucc_datastruct.h" #include "ucc/api/ucc.h" #include "ucc_compiler_def.h" #define ucc_min(_a, _b) ucs_min((_a), (_b)) #define ucc_max(_a, _b) ucs_max((_a), (_b)) #define ucc_ilog2(_v) ucs_ilog2((_v)) #define ucc_ceil(_a, _b) (((_a)%(_b))?((_a)/(_b)+1)*(_b):(_a)) #define DO_OP_MAX(_v1, _v2) (_v1 > _v2 ? _v1 : _v2) #define DO_OP_MIN(_v1, _v2) (_v1 < _v2 ? _v1 : _v2) #define DO_OP_SUM(_v1, _v2) (_v1 + _v2) #define DO_OP_PROD(_v1, _v2) (_v1 * _v2) #define DO_OP_LAND(_v1, _v2) (_v1 && _v2) #define DO_OP_BAND(_v1, _v2) (_v1 & _v2) #define DO_OP_LOR(_v1, _v2) (_v1 || _v2) #define DO_OP_BOR(_v1, _v2) (_v1 | _v2) #define DO_OP_LXOR(_v1, _v2) ((!_v1) != (!_v2)) #define DO_OP_BXOR(_v1, _v2) (_v1 ^ _v2) #define PTR_OFFSET(_ptr, _offset) \ ((void *)((ptrdiff_t)(_ptr) + (size_t)(_offset))) /* Returns the number of trailing 0-bits in x, starting at the least * significant bit position. If x is 0, the result is undefined. */ #define ucc_count_trailing_zero_bits(_n) \ ((sizeof(_n) <= 4) ? __builtin_ctz((uint32_t)(_n)) : __builtin_ctzl(_n)) /* Returns the number of leading 0-bits in _n. * If _n is 0, the result is undefined */ #define ucc_count_leading_zero_bits(_n) \ ((sizeof(_n) <= 4) ? __builtin_clz((uint32_t)(_n)) : __builtin_clzl(_n)) /* http://www.cse.yorku.ca/~oz/hash.html - Dan Bernstein string hash function */ static inline unsigned long ucc_str_hash_djb2(const char *str) { unsigned long hash = 5381; int c; while ('\0' != (c = *str++)) { hash = ((hash << 5) + hash) + c; } return hash; } /* Sorts the input integer array in-place keeping only unique elements */ int ucc_sort_uniq(int *array, int len, int inverse); #define SWAP(_x, _y, _type) \ do { \ _type _tmp = (_x); \ (_x) = (_y); \ (_y) = _tmp; \ } while (0) #define ucc_div_round_up(_n, _d) (((_n) + (_d) - 1) / (_d)) // compute the greatest exponent v such that p**v divides n static inline uint32_t calc_valuation(uint32_t n, uint32_t p) { uint32_t v = 0; uint32_t q = n; while (q % p == 0) { v++; q /= p; } return v; } static inline float bfloat16tofloat32(const void *bfloat16_ptr) { float res = 0; #if UCC_BIG_ENDIAN ((uint16_t *)(&res))[0] = *((uint16_t *)bfloat16_ptr); #else ((uint16_t *)(&res))[1] = *((uint16_t *)bfloat16_ptr); #endif return res; } static inline void float32tobfloat16(float float_val, void *bfloat16_ptr) { #if UCC_BIG_ENDIAN *((uint16_t *)bfloat16_ptr) = ((uint16_t *)(&float_val))[0]; #else *((uint16_t *)bfloat16_ptr) = ((uint16_t *)(&float_val))[1]; #endif } #define ucc_padding(_n, _alignment) \ ( ((_alignment) - (_n) % (_alignment)) % (_alignment) ) #define ucc_align_down(_n, _alignment) \ ( (_n) - ((_n) % (_alignment)) ) #define ucc_align_up(_n, _alignment) \ ( (_n) + ucc_padding(_n, _alignment) ) #define ucc_align_down_pow2(_n, _alignment) \ ( (_n) & ~((_alignment) - 1) ) #define ucc_align_up_pow2(_n, _alignment) \ ucc_align_down_pow2((_n) + (_alignment) - 1, _alignment) /* compute the log2 of n, rounded up */ static inline int lognum(int n) { int count = 1; int lognum = 0; while (count < n) { count = count << 1; lognum++; } return lognum; } /* return log2 of nearest greater or equal power of 2 */ static inline uint32_t ucc_ilog2_ceil(uint32_t n) { int x = ucc_count_leading_zero_bits(n); // leading zeros if (!(n & (n - 1))) { return 31 - x; // pow2 case } return 31 - x + 1; } /* return the lowest greater or equal power of 2 */ static inline int ucc_round_up_power2(int value) { if (value <= 1) { return 1; } return 1 << (32 - ucc_count_leading_zero_bits(value - 1)); } #endif ucc-1.8.0/src/utils/ucc_malloc.h0000664000175000017500000000072115211535620016657 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_MALLOC_H_ #define UCC_MALLOC_H_ #include "config.h" #include #define ucc_malloc(_s, ...) malloc(_s) #define ucc_posix_memalign(_ptr, _align, _size, ...) posix_memalign(_ptr, _align, _size) #define ucc_calloc(_n, _s, ...) calloc(_n, _s) #define ucc_realloc(_p, _s, ...) realloc(_p, _s) #define ucc_free(_p) free(_p) #endif ucc-1.8.0/src/utils/debug/0000775000175000017500000000000015211535620015473 5ustar alastairalastairucc-1.8.0/src/utils/debug/log.h0000664000175000017500000000051015211535620016421 0ustar alastairalastair/** * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2014. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_LOG_H_ #define UCC_LOG_H_ #ifdef HAVE_CONFIG_H # include "config.h" /* Defines UCS_MAX_LOG_LEVEL */ #endif #include /* Contains actual logger implementation */ #endif ucc-1.8.0/src/utils/debug/log.c0000664000175000017500000004263415211535620016431 0ustar alastairalastair/** * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2014. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifdef HAVE_CONFIG_H # include "config.h" #endif #include "log.h" #include #include #include #include #include #include #include #include #define UCC_MAX_LOG_HANDLERS 32 #define UCC_LOG_TIME_FMT "[%lu.%06lu]" #define UCC_LOG_METADATA_FMT "%17s:%-4u %-4s %-5s %*s" #define UCC_LOG_PROC_DATA_FMT "[%s:%-5d:%s]" #define UCC_LOG_COMPACT_FMT UCC_LOG_TIME_FMT " " UCC_LOG_PROC_DATA_FMT " " #define UCC_LOG_SHORT_FMT UCC_LOG_TIME_FMT " [%s] " UCC_LOG_METADATA_FMT "%s\n" #define UCC_LOG_FMT UCC_LOG_TIME_FMT " " UCC_LOG_PROC_DATA_FMT " " \ UCC_LOG_METADATA_FMT "%s\n" #define UCC_LOG_TIME_ARG(_tv) (_tv)->tv_sec, (_tv)->tv_usec #define UCC_LOG_METADATA_ARG(_short_file, _line, _level, _comp_conf) \ (_short_file), (_line), (_comp_conf)->name, \ ucc_log_level_names[_level], (ucc_log_current_indent * 2), "" #define UCC_LOG_PROC_DATA_ARG() \ ucc_log_hostname, ucc_log_get_pid(), ucc_log_get_thread_name() #define UCC_LOG_COMPACT_ARG(_tv)\ UCC_LOG_TIME_ARG(_tv), UCC_LOG_PROC_DATA_ARG() #define UCC_LOG_SHORT_ARG(_short_file, _line, _level, _comp_conf, _tv, \ _message) \ UCC_LOG_TIME_ARG(_tv), ucc_log_get_thread_name(), \ UCC_LOG_METADATA_ARG(_short_file, _line, _level, _comp_conf), \ (_message) #define UCC_LOG_ARG(_short_file, _line, _level, _comp_conf, _tv, _message) \ UCC_LOG_TIME_ARG(_tv), UCC_LOG_PROC_DATA_ARG(), \ UCC_LOG_METADATA_ARG(_short_file, _line, _level, _comp_conf), (_message) KHASH_MAP_INIT_STR(ucc_log_filter, char); const char *ucc_log_level_names[] = { [UCC_LOG_LEVEL_FATAL] = "FATAL", [UCC_LOG_LEVEL_ERROR] = "ERROR", [UCC_LOG_LEVEL_WARN] = "WARN", [UCC_LOG_LEVEL_DIAG] = "DIAG", [UCC_LOG_LEVEL_INFO] = "INFO", [UCC_LOG_LEVEL_DEBUG] = "DEBUG", [UCC_LOG_LEVEL_TRACE] = "TRACE", [UCC_LOG_LEVEL_TRACE_REQ] = "REQ", [UCC_LOG_LEVEL_TRACE_DATA] = "DATA", [UCC_LOG_LEVEL_TRACE_ASYNC] = "ASYNC", [UCC_LOG_LEVEL_TRACE_FUNC] = "FUNC", [UCC_LOG_LEVEL_TRACE_POLL] = "POLL", [UCC_LOG_LEVEL_LAST] = NULL, [UCC_LOG_LEVEL_PRINT] = "PRINT" }; static unsigned ucc_log_handlers_count = 0; static int ucc_log_initialized = 0; static int __thread ucc_log_current_indent = 0; static char ucc_log_hostname[HOST_NAME_MAX] = {0}; static int ucc_log_pid = 0; static FILE *ucc_log_file = NULL; static char *ucc_log_file_base_name = NULL; static int ucc_log_file_close = 0; static int ucc_log_file_last_idx = 0; static uint32_t ucc_log_thread_count = 0; static char __thread ucc_log_thread_name[32] = {0}; static ucc_log_func_t ucc_log_handlers[UCC_MAX_LOG_HANDLERS]; static ucc_spinlock_t ucc_log_global_filter_lock; static khash_t(ucc_log_filter) ucc_log_global_filter; static inline int ucc_log_get_pid() { if (ucc_unlikely(ucc_log_pid == 0)) { return getpid(); } return ucc_log_pid; } static const char *ucc_log_get_thread_name() { char *name = ucc_log_thread_name; uint32_t thread_num; if (ucc_unlikely(name[0] == '\0')) { thread_num = ucc_atomic_fadd32(&ucc_log_thread_count, 1); ucc_snprintf_safe(ucc_log_thread_name, sizeof(ucc_log_thread_name), "%u", thread_num); } return name; } void ucc_log_flush() { if (ucc_log_file != NULL) { fflush(ucc_log_file); if (ucc_log_file_close) { /* non-stdout/stderr */ fsync(fileno(ucc_log_file)); } } } size_t ucc_log_get_buffer_size() { return ucc_config_memunits_get(ucc_global_config.log_buffer_size, 256, UCC_ALLOCA_MAX_SIZE); } static void ucc_log_get_file_name(char *log_file_name, size_t max, int idx) { ucc_assert(idx <= ucc_global_config.log_file_rotate); if (idx == 0) { ucc_strncpy_zero(log_file_name, ucc_log_file_base_name, max); return; } ucc_snprintf_zero(log_file_name, max, "%s.%d", ucc_log_file_base_name, idx); } static void ucc_log_file_rotate() { char *old_log_file_name, *new_log_file_name; int idx, ret; ucc_status_t status; status = ucc_string_alloc_path_buffer(&old_log_file_name, "old_log_file_name"); if (status != UCC_OK) { goto out; } if (ucc_log_file_last_idx == ucc_global_config.log_file_rotate) { /* remove the last file and log rotation from the * `log_file_rotate - 1` file */ ucc_log_get_file_name(old_log_file_name, PATH_MAX, ucc_log_file_last_idx); unlink(old_log_file_name); } else { ucc_log_file_last_idx++; } ucc_assert(ucc_log_file_last_idx <= ucc_global_config.log_file_rotate); status = ucc_string_alloc_path_buffer(&new_log_file_name, "new_log_file_name"); if (status != UCC_OK) { goto out_free_old_log_file_name; } for (idx = ucc_log_file_last_idx - 1; idx >= 0; --idx) { ucc_log_get_file_name(old_log_file_name, PATH_MAX, idx); ucc_log_get_file_name(new_log_file_name, PATH_MAX, idx + 1); if (access(old_log_file_name, W_OK) != 0) { ucc_fatal("unable to write to %s", old_log_file_name); } /* coverity[toctou] */ ret = rename(old_log_file_name, new_log_file_name); if (ret) { ucc_fatal("failed to rename %s to %s: %m", old_log_file_name, new_log_file_name); } if (access(old_log_file_name, F_OK) != -1) { ucc_fatal("%s must not exist on the filesystem", old_log_file_name); } if (access(new_log_file_name, W_OK) != 0) { ucc_fatal("unable to write to %s", new_log_file_name); } } ucc_free(new_log_file_name); out_free_old_log_file_name: ucc_free(old_log_file_name); out: return; } static void ucc_log_handle_file_max_size(int log_entry_len) { const char *next_token; /* check if it is necessary to find a new storage for logs */ if ((log_entry_len + ftell(ucc_log_file)) < ucc_global_config.log_file_size) { return; } fclose(ucc_log_file); if (ucc_global_config.log_file_rotate != 0) { ucc_log_file_rotate(); } else { unlink(ucc_log_file_base_name); } ucc_open_output_stream(ucc_log_file_base_name, UCC_LOG_LEVEL_FATAL, &ucc_log_file, &ucc_log_file_close, &next_token, NULL); } void ucc_log_print_compact(const char *str) { struct timeval tv; gettimeofday(&tv, NULL); if (ucc_log_initialized) { if (ucc_log_file_close) { /* non-stdout/stderr */ ucc_log_handle_file_max_size(strlen(str) + 1); } fprintf(ucc_log_file, UCC_LOG_COMPACT_FMT " %s\n", UCC_LOG_COMPACT_ARG(&tv), str); } else { fprintf(stdout, UCC_LOG_COMPACT_FMT " %s\n", UCC_LOG_COMPACT_ARG(&tv), str); } } static void ucc_log_print(const char *short_file, int line, ucc_log_level_t level, const ucc_log_component_config_t *comp_conf, const struct timeval *tv, const char *message) { int log_entry_len; if (ucc_log_initialized) { if (ucc_log_file_close) { /* non-stdout/stderr */ /* get log entry size */ log_entry_len = snprintf(NULL, 0, UCC_LOG_FMT, UCC_LOG_ARG(short_file, line, level, comp_conf, tv, message)); ucc_log_handle_file_max_size(log_entry_len); } fprintf(ucc_log_file, UCC_LOG_FMT, UCC_LOG_ARG(short_file, line, level, comp_conf, tv, message)); } else { fprintf(stdout, UCC_LOG_SHORT_FMT, UCC_LOG_SHORT_ARG(short_file, line, level, comp_conf, tv, message)); } } ucc_log_func_rc_t ucc_log_default_handler(const char *file, unsigned line, const char *function, ucc_log_level_t level, const ucc_log_component_config_t *comp_conf, const char *format, va_list ap) { size_t buffer_size = ucc_log_get_buffer_size(); char *saveptr = ""; const char *short_file; struct timeval tv; khiter_t khiter; char *log_line; char match; int khret; char *buf; const char *filename; if (!ucc_log_component_is_enabled(level, comp_conf) && (level != UCC_LOG_LEVEL_PRINT)) { return UCC_LOG_FUNC_RC_CONTINUE; } ucc_spin_lock(&ucc_log_global_filter_lock); khiter = kh_get(ucc_log_filter, &ucc_log_global_filter, file); if (ucc_unlikely(khiter == kh_end(&ucc_log_global_filter))) { /* Add source file name to the hash */ match = fnmatch(ucc_global_config.log_component.file_filter, file, 0) != FNM_NOMATCH; filename = ucc_strdup(file, "log filter filename"); if (filename == NULL) { ucc_fatal("cannot allocate log filtering entry for '%s'", file); } khiter = kh_put(ucc_log_filter, &ucc_log_global_filter, filename, &khret); ucc_assert((khret == UCS_KH_PUT_BUCKET_EMPTY) || (khret == UCS_KH_PUT_BUCKET_CLEAR)); kh_val(&ucc_log_global_filter, khiter) = match; } else { match = kh_val(&ucc_log_global_filter, khiter); } ucc_spin_unlock(&ucc_log_global_filter_lock); if (!match) { return UCC_LOG_FUNC_RC_CONTINUE; } buf = ucc_alloca(buffer_size + 1); buf[buffer_size] = 0; vsnprintf(buf, buffer_size, format, ap); if (level <= ucc_global_config.log_level_trigger) { ucc_fatal_error_message(file, line, function, buf); } else { short_file = ucc_basename(file); gettimeofday(&tv, NULL); log_line = strtok_r(buf, "\n", &saveptr); while (log_line != NULL) { ucc_log_print(short_file, line, level, comp_conf, &tv, log_line); log_line = strtok_r(NULL, "\n", &saveptr); } } /* flush the log file if the log_level of this message is fatal or error */ if (level <= UCC_LOG_LEVEL_ERROR) { ucc_log_flush(); } return UCC_LOG_FUNC_RC_CONTINUE; } void ucc_log_push_handler(ucc_log_func_t handler) { if (ucc_log_handlers_count < UCC_MAX_LOG_HANDLERS) { ucc_log_handlers[ucc_log_handlers_count++] = handler; } } void ucc_log_pop_handler() { if (ucc_log_handlers_count > 0) { --ucc_log_handlers_count; } } void ucc_log_indent(int delta) { ucc_log_current_indent += delta; ucc_assert(ucc_log_current_indent >= 0); } int ucc_log_get_current_indent() { return ucc_log_current_indent; } unsigned ucc_log_num_handlers() { return ucc_log_handlers_count; } void ucc_log_dispatch(const char *file, unsigned line, const char *function, ucc_log_level_t level, ucc_log_component_config_t *comp_conf, const char *format, ...) { ucc_log_func_rc_t rc; unsigned idx; va_list ap; /* Call handlers in reverse order */ rc = UCC_LOG_FUNC_RC_CONTINUE; idx = ucc_log_handlers_count; while ((idx > 0) && (rc == UCC_LOG_FUNC_RC_CONTINUE)) { --idx; va_start(ap, format); rc = ucc_log_handlers[idx](file, line, function, level, comp_conf, format, ap); va_end(ap); } } void ucc_log_fatal_error(const char *format, ...) { size_t buffer_size = ucc_log_get_buffer_size(); FILE *stream = stderr; char *buffer, *p; va_list ap; int ret; /* Initialize va_list before any potential analyzer-confusing macros */ va_start(ap, format); buffer = ucc_alloca(buffer_size + 1); p = buffer; /* Print hostname:pid */ snprintf(p, buffer_size, "[%s:%-5d:%s:%d] ", ucc_log_hostname, ucc_log_get_pid(), ucc_log_get_thread_name(), ucc_get_tid()); buffer_size -= strlen(p); p += strlen(p); /* Print rest of the message */ // NOLINTNEXTLINE(clang-analyzer-valist.Uninitialized) vsnprintf(p, buffer_size, format, ap); va_end(ap); buffer_size -= strlen(p); p += strlen(p); /* Newline */ snprintf(p, buffer_size, "\n"); /* Flush stderr, and write the message directly to the pipe */ fflush(stream); ret = write(fileno(stream), buffer, strlen(buffer)); (void)ret; } /** * Print a bitmap as a list of ranges. * * @param n Number equivalent to the first bit in the bitmap. * @param bitmap Compressed array of bits. * @param length Number of bits in the bitmap. */ const char *ucc_log_bitmap_to_str(unsigned n, uint8_t *bitmap, size_t length) { static char buf[512] = {0}; int first, in_range; unsigned prev = 0, end = 0; char *p, *endp; size_t i; p = buf; endp = buf + sizeof(buf) - 4; first = 1; in_range = 0; for (i = 0; i < length; ++i) { if (bitmap[i / 8] & UCC_BIT(i % 8)) { if (first) { p += snprintf(p, endp - p, "%d", n); if (p > endp) { goto overflow; } } else if (n == prev + 1) { in_range = 1; end = n; } else { if (in_range) { p += snprintf(p, endp - p, "-%d", end); if (p > endp) { goto overflow; } } in_range = 0; p += snprintf(p, endp - p, ",%d", n); if (p > endp) { goto overflow; } } first = 0; prev = n; } ++n; } if (in_range) { p += snprintf(p, endp - p, "-%d", end); if (p > endp) { goto overflow; } } return buf; overflow: { /* Ensure we do not overflow the buffer when indicating truncation */ size_t remaining = (size_t)((buf + sizeof(buf) - 1) - p); if (remaining > 0) { ucc_strncpy_zero(p, "...", remaining + 1); } } return buf; } void ucc_log_early_init() { ucc_log_initialized = 0; ucc_log_hostname[0] = 0; ucc_log_pid = getpid(); ucc_log_file = NULL; ucc_log_file_last_idx = 0; ucc_log_file_close = 0; ucc_log_thread_count = 0; } static void ucc_log_atfork_prepare() { ucc_log_pid = 0; ucc_log_flush(); } static void ucc_log_atfork_post() { ucc_log_pid = getpid(); } void ucc_log_init() { const char *next_token; if (ucc_log_initialized) { return; } ucc_log_initialized = 1; /* Set this to 1 immediately to avoid infinite recursion */ if (ucc_global_config.log_file_size < ucc_log_get_buffer_size()) { ucc_fatal("the maximal log file size (%zu) has to be >= %zu", ucc_global_config.log_file_size, ucc_log_get_buffer_size()); } if (ucc_global_config.log_file_rotate > INT_MAX) { ucc_fatal("the log file rotate (%u) has to be <= %d", ucc_global_config.log_file_rotate, INT_MAX); } ucc_spinlock_init(&ucc_log_global_filter_lock, 0); kh_init_inplace(ucc_log_filter, &ucc_log_global_filter); ucc_strncpy_zero(ucc_log_hostname, ucc_get_host_name(), sizeof(ucc_log_hostname)); ucc_log_file = stdout; ucc_log_file_base_name = NULL; ucc_log_file_close = 0; ucc_log_file_last_idx = 0; ucc_log_push_handler(ucc_log_default_handler); if (strlen(ucc_global_config.log_file) != 0) { ucc_open_output_stream(ucc_global_config.log_file, UCC_LOG_LEVEL_FATAL, &ucc_log_file, &ucc_log_file_close, &next_token, &ucc_log_file_base_name); } pthread_atfork(ucc_log_atfork_prepare, ucc_log_atfork_post, ucc_log_atfork_post); } void ucc_log_cleanup() { const char *filename; ucc_assert(ucc_log_initialized); ucc_log_flush(); if (ucc_log_file_close) { fclose(ucc_log_file); } ucc_spinlock_destroy(&ucc_log_global_filter_lock); kh_foreach_key(&ucc_log_global_filter, filename, { ucc_free((void*)filename); }) ; /* code format script wants it there */ kh_destroy_inplace(ucc_log_filter, &ucc_log_global_filter); ucc_free(ucc_log_file_base_name); ucc_log_file_base_name = NULL; ucc_log_file = NULL; ucc_log_file_last_idx = 0; ucc_log_initialized = 0; ucc_log_handlers_count = 0; } void ucc_log_print_backtrace(ucc_log_level_t level) { (void)level; /* suppress unused parameter warning */ //TODO: fix this ucc_assert_always(0); } void ucc_log_set_thread_name(const char *format, ...) { va_list ap; va_start(ap, format); memset(ucc_log_thread_name, 0, sizeof(ucc_log_thread_name)); // NOLINTNEXTLINE(clang-analyzer-valist.Uninitialized) vsnprintf(ucc_log_thread_name, sizeof(ucc_log_thread_name) - 1, format, ap); va_end(ap); } ucc-1.8.0/src/utils/debug/log_def.h0000664000175000017500000001511715211535620017250 0ustar alastairalastair/** * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2020. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_LOG_DEF_H_ #define UCC_LOG_DEF_H_ #ifndef UCC_MAX_LOG_LEVEL # define UCC_MAX_LOG_LEVEL UCC_LOG_LEVEL_LAST #endif #include #include #include #include #include BEGIN_C_DECLS /** @file log_def.h */ #define ucc_log_component_is_enabled(_level, _comp_log_config) \ ucc_unlikely(((_level) <= UCC_MAX_LOG_LEVEL) && \ ((_level) <= (((ucc_log_component_config_t*)(_comp_log_config))->log_level))) #define ucc_log_is_enabled(_level) \ ucc_log_component_is_enabled(_level, &ucc_global_config.log_component) #define ucc_log_component(_level, _comp_log_config, _fmt, ...) \ do { \ if (ucc_log_component_is_enabled(_level, _comp_log_config)) { \ ucc_log_dispatch(__FILE__, __LINE__, __func__, \ (ucc_log_level_t)(_level), _comp_log_config, _fmt, ## __VA_ARGS__); \ } \ } while (0) #define ucc_log(_level, _fmt, ...) \ do { \ ucc_log_component(_level, &ucc_global_config.log_component, _fmt, ## __VA_ARGS__); \ } while (0) #define ucc_error(_fmt, ...) ucc_log(UCC_LOG_LEVEL_ERROR, _fmt, ## __VA_ARGS__) #define ucc_warn(_fmt, ...) ucc_log(UCC_LOG_LEVEL_WARN, _fmt, ## __VA_ARGS__) #define ucc_diag(_fmt, ...) ucc_log(UCC_LOG_LEVEL_DIAG, _fmt, ## __VA_ARGS__) #define ucc_info(_fmt, ...) ucc_log(UCC_LOG_LEVEL_INFO, _fmt, ## __VA_ARGS__) #define ucc_debug(_fmt, ...) ucc_log(UCC_LOG_LEVEL_DEBUG, _fmt, ## __VA_ARGS__) #define ucc_trace(_fmt, ...) ucc_log(UCC_LOG_LEVEL_TRACE, _fmt, ## __VA_ARGS__) #define ucc_trace_req(_fmt, ...) ucc_log(UCC_LOG_LEVEL_TRACE_REQ, _fmt, ## __VA_ARGS__) #define ucc_trace_data(_fmt, ...) ucc_log(UCC_LOG_LEVEL_TRACE_DATA, _fmt, ## __VA_ARGS__) #define ucc_trace_async(_fmt, ...) ucc_log(UCC_LOG_LEVEL_TRACE_ASYNC, _fmt, ## __VA_ARGS__) #define ucc_trace_func(_fmt, ...) ucc_log(UCC_LOG_LEVEL_TRACE_FUNC, "%s(" _fmt ")", __func__, ## __VA_ARGS__) #define ucc_trace_poll(_fmt, ...) ucc_log(UCC_LOG_LEVEL_TRACE_POLL, _fmt, ## __VA_ARGS__) #define ucc_log_indent_level(_level, _delta) \ do { \ if (ucc_log_component_is_enabled(_level, \ &ucc_global_config.log_component)) { \ ucc_log_indent(_delta); \ } \ } while (0) /** * Print a message regardless of current log level. Output can be * enabled/disabled via environment variable/configuration settings. * * During debugging it can be useful to add a few prints to the code * without changing a current log level. Also it is useful to be able * to see messages only from specific processes. For example, one may * want to see prints only from rank 0 when debugging MPI. * * The function is intended for debugging only. It should not be used * in the real code. */ #define ucc_print(_fmt, ...) \ do { \ if (ucc_global_config.log_print_enable) { \ ucc_log_dispatch(__FILE__, __LINE__, __func__, \ UCC_LOG_LEVEL_PRINT, &ucc_global_config.log_component, _fmt, ## __VA_ARGS__); \ } \ } while(0) typedef enum { UCC_LOG_FUNC_RC_STOP, UCC_LOG_FUNC_RC_CONTINUE } ucc_log_func_rc_t; /** * Function type for handling log messages. * * @param file Source file name. * @param line Source line number. * @param function Function name. * @param level Log level. * @param comp_conf Component specific log config. * @param message Log message - format string. * @param ap Log message format parameters. * * @return UCC_LOG_FUNC_RC_CONTINUE - continue to next log handler. * UCC_LOG_FUNC_RC_STOP - don't continue. */ typedef ucc_log_func_rc_t (*ucc_log_func_t)(const char *file, unsigned line, const char *function, ucc_log_level_t level, const ucc_log_component_config_t *comp_conf, const char *message, va_list ap); extern const char *ucc_log_level_names[]; /** * Dispatch a logging message. * * @param [in] file Source file name. * @param [in] line Source line number. * @param [in] function Function name which generated the log. * @param [in] level Log level of the message. * @param [in] comp_conf Component log config. * @param [in] message Log format. */ void ucc_log_dispatch(const char *file, unsigned line, const char *function, ucc_log_level_t level, ucc_log_component_config_t *comp_conf, const char *format, ...) UCC_F_PRINTF(6, 7); /** * Flush logging output. */ void ucc_log_flush(void); /** * @return Configured log buffer size */ size_t ucc_log_get_buffer_size(void); /** * Print a compact log line (without file/line prefixes) to the log stream. * * @param [in] str Log line to print. */ void ucc_log_print_compact(const char *str); /** * Default log handler, which prints the message to the output configured in * UCC global options. See @ref ucc_log_func_t. */ ucc_log_func_rc_t ucc_log_default_handler(const char *file, unsigned line, const char *function, ucc_log_level_t level, const ucc_log_component_config_t *comp_conf, const char *format, va_list ap); /** * Show a fatal error */ void ucc_log_fatal_error(const char *format, ...) UCC_F_PRINTF(1, 2); /** * Initialize/cleanup logging subsystem. */ void ucc_log_early_init(void); void ucc_log_init(void); void ucc_log_cleanup(void); const char *ucc_log_bitmap_to_str(unsigned n, uint8_t *bitmap, size_t length); /** * Add/remove logging handlers */ void ucc_log_push_handler(ucc_log_func_t handler); void ucc_log_pop_handler(void); unsigned ucc_log_num_handlers(void); /** * Add indentation to all subsequent log messages. * * @param [in] delta How much indentation to add, on top of the current * indentation level. * A negative number will reduce the indentation level. */ void ucc_log_indent(int delta); /** * @return Current log indent level. */ int ucc_log_get_current_indent(void); /** * Log backtrace. * * @param level Log level. */ void ucc_log_print_backtrace(ucc_log_level_t level); /** * Set the name for current thread, to appear in log messages * * @param name Thread name to set */ void ucc_log_set_thread_name(const char *format, ...) UCC_F_PRINTF(1, 2); END_C_DECLS #endif ucc-1.8.0/src/utils/debug/types.h0000664000175000017500000000247115211535620017014 0ustar alastairalastair/** * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2019. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_LOG_TYPES_H_ #define UCC_LOG_TYPES_H_ /** * Logging levels. */ typedef enum { UCC_LOG_LEVEL_FATAL, /* Immediate termination */ UCC_LOG_LEVEL_ERROR, /* Error is returned to the user */ UCC_LOG_LEVEL_WARN, /* Something's wrong, but we continue */ UCC_LOG_LEVEL_DIAG, /* Diagnostics, silent adjustments or internal error handling */ UCC_LOG_LEVEL_INFO, /* Information */ UCC_LOG_LEVEL_DEBUG, /* Low-volume debugging */ UCC_LOG_LEVEL_TRACE, /* High-volume debugging */ UCC_LOG_LEVEL_TRACE_REQ, /* Every send/receive request */ UCC_LOG_LEVEL_TRACE_DATA, /* Data sent/received on the transport */ UCC_LOG_LEVEL_TRACE_ASYNC, /* Asynchronous progress engine */ UCC_LOG_LEVEL_TRACE_FUNC, /* Function calls */ UCC_LOG_LEVEL_TRACE_POLL, /* Polling functions */ UCC_LOG_LEVEL_LAST, UCC_LOG_LEVEL_PRINT /* Temporary output */ } ucc_log_level_t; /** * Logging component. */ typedef struct ucc_log_component_config { ucc_log_level_t log_level; char name[16]; const char *file_filter; /* glob pattern of source files */ } ucc_log_component_config_t; #endif ucc-1.8.0/src/utils/ucc_list.h0000664000175000017500000000304115211535620016361 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_LIST_H_ #define UCC_LIST_H_ #include "config.h" #include #define ucc_list_link_t ucs_list_link_t #define ucc_list_head_init ucs_list_head_init #define ucc_list_add_tail ucs_list_add_tail #define ucc_list_del ucs_list_del #define ucc_list_for_each_safe ucs_list_for_each_safe #define ucc_list_for_each ucs_list_for_each #define ucc_list_is_empty ucs_list_is_empty #define ucc_list_extract_head ucs_list_extract_head #define ucc_list_length ucs_list_length #define ucc_list_head ucs_list_head #define ucc_list_tail ucs_list_tail #define ucc_list_next ucs_list_next #define ucc_list_insert_after ucs_list_insert_after #define ucc_list_insert_before ucs_list_insert_before #define ucc_list_destruct(_list, _elem_type, _elem_destruct, _member) \ do { \ _elem_type *_elem, *_tmp; \ ucc_list_for_each_safe(_elem, _tmp, _list, _member) \ { \ ucc_list_del(&_elem->_member); \ _elem_destruct(_elem); \ } \ } while (0) #endif ucc-1.8.0/src/utils/ucc_debug.h0000664000175000017500000000133415211535620016477 0ustar alastairalastair/** * Copyright (c) 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_DEBUG_H_ #define UCC_DEBUG_H_ #include "config.h" #include "ucc/api/ucc.h" static inline void ucc_check_wait_for_debugger(ucc_rank_t ctx_rank) { const char *wait_for_rank; if (NULL != (wait_for_rank = getenv("UCC_DEBUGGER_WAIT"))) { volatile int waiting = 1; if (atoi(wait_for_rank) == ctx_rank) { char hostname[256]; gethostname(hostname, sizeof(hostname)); printf("PID %d (ctx rank %d) waiting for attach on %s\n" "Set var waiting = 0 in debugger to continue\n", getpid(), ctx_rank, hostname); while (waiting) { sleep(1); } } } } #endif ucc-1.8.0/src/utils/ucc_assert.h0000664000175000017500000000707415211535620016721 0ustar alastairalastair/** * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_ASSERT_H #define UCC_ASSERT_H #include "ucc_compiler_def.h" #if ENABLE_DEBUG == 1 || UCC_ENABLE_ASSERT == 1 #include #define ucc_assert(_cond) ucc_assert_always(_cond) #define ucc_assertv(_cond) ucc_assertv_always(_cond) #define ucc_assert_system(_cond) assert(_cond) #else #define ucc_assert(_cond) do {} while(0) #define ucc_assert_system(_cond) do {} while(0) #define ucc_assertv(_cond) do {} while(0) #endif BEGIN_C_DECLS /** * Fail if _expression evaluates to 0 */ #define ucc_assert_always(_expression) \ do { \ if (!ucc_likely(_expression)) { \ ucc_fatal_error_format( \ __FILE__, \ __LINE__, \ __FUNCTION__, \ "Assertion `%s' failed", \ #_expression); \ } \ } while (0) /** * Fail if _expression evaluates to 0 and print a formatted error message */ #define ucc_assertv_always(_expression, _fmt, ...) \ do { \ if (!ucc_likely(_expression)) { \ ucc_fatal_error_format( \ __FILE__, \ __LINE__, \ __func__, \ "Assertion `%s' failed: " _fmt, \ #_expression, \ ##__VA_ARGS__); \ } \ } while (0) /** * Generate a fatal error and stop the program. * * @param [in] file Source file name * @param [in] line Source line number * @param [in] function Calling function name * @param [in] message_buf Error message buffer. Multi-line message is * supported. * * IMPORTANT NOTE: message_buf could be overridden by this function */ void ucc_fatal_error_message(const char *file, unsigned line, const char *function, char *message_buf) UCS_F_NORETURN; /** * Generate a fatal error and stop the program. * * @param [in] file Source file name * @param [in] line Source line number * @param [in] function Calling function name * @param [in] format Error message format string. Multi-line message is * supported. */ void ucc_fatal_error_format(const char *file, unsigned line, const char *function, const char *format, ...) UCC_F_NORETURN; /** * Generate a fatal error */ #define ucc_fatal(_fmt, ...) \ ucc_fatal_error_format(__FILE__, __LINE__, __func__, \ "Fatal: " _fmt, ## __VA_ARGS__) END_C_DECLS #endif ucc-1.8.0/src/utils/ucc_sys.h0000664000175000017500000000456415211535620016237 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_SYS_H_ #define UCC_SYS_H_ #include "ucc/api/ucc_status.h" #include "utils/ucc_compiler_def.h" #include "utils/ucc_log.h" #include #include #include #include ucc_status_t ucc_sysv_alloc(size_t *size, void **addr, int *shm_id); ucc_status_t ucc_sysv_free(void *addr); const char* ucc_sys_get_lib_path(); ucc_status_t ucc_sys_dirname(const char *path, char **out); ucc_status_t ucc_sys_path_join(const char *path1, const char *path2, char **out); size_t ucc_get_page_size(); /** * Open an output stream according to user configuration: * - file: - file name, %p, %h, %c are substituted. * - stdout * - stderr * * @param [in] config_str The file name or name of the output stream * (stdout/stderr). * @param [in] err_log_level Logging level that should be used for printing * errors. * @param [out] p_fstream Pointer that is filled with the stream handle. * User is responsible to close the stream handle then. * @param [out] p_need_close Pointer to the variable that is set to whether * fclose() should be called to release resources (1) * or not (0). * @param [out] p_next_token Pointer that is set to remainder of @config_str. * @param [out] p_filename Pointer to the variable that is filled with the * resulted name of the log file (if it is not NULL). * Caller is responsible to release memory then. * * @return UCC_OK if successful, or error code otherwise. */ ucc_status_t ucc_open_output_stream(const char *config_str, ucc_log_level_t err_log_level, FILE **p_fstream, int *p_need_close, const char **p_next_token, char **p_filename); /** * @return Path to the main executable. */ const char *ucc_get_exe(); /** * @return Host name. */ const char *ucc_get_host_name(); /** * @return user name. */ const char *ucc_get_user_name(); /** * Get the first processor number we are bound to. */ int ucc_get_first_cpu(); /** * Get current thread (LWP) id. */ pid_t ucc_get_tid(void); #endif ucc-1.8.0/src/utils/ucc_atomic.h0000664000175000017500000000135215211535620016665 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_ATOMIC_H_ #define UCC_ATOMIC_H_ #include "config.h" #include #define ucc_atomic_add32 ucs_atomic_add32 #define ucc_atomic_fadd32 ucs_atomic_fadd32 #define ucc_atomic_fadd8 ucs_atomic_fadd8 #define ucc_atomic_sub32 ucs_atomic_sub32 #define ucc_atomic_add64 ucs_atomic_add64 #define ucc_atomic_sub64 ucs_atomic_sub64 #define ucc_atomic_cswap8 ucs_atomic_cswap8 #define ucc_atomic_cswap64 ucs_atomic_cswap64 #define ucc_atomic_bool_cswap8 ucs_atomic_bool_cswap8 #define ucc_atomic_bool_cswap64 ucs_atomic_bool_cswap64 #endif ucc-1.8.0/src/utils/ucc_datastruct.c0000664000175000017500000000235415211535620017565 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_datastruct.h" #include "ucc_malloc.h" #include "ucc_compiler_def.h" #include "ucc_log.h" ucc_status_t ucc_mrange_uint_copy(ucc_mrange_uint_t *dst, const ucc_mrange_uint_t *src) { ucc_mrange_t *r, *r_dup; dst->default_value = src->default_value; ucc_list_head_init(&dst->ranges); ucc_list_for_each(r, &src->ranges, list_elem) { r_dup = ucc_malloc(sizeof(*r_dup), "range_dup"); if (ucc_unlikely(!r_dup)) { ucc_error("failed to allocate %zd bytes for mrange", sizeof(*r_dup)); goto err; } r_dup->start = r->start; r_dup->end = r->end; r_dup->value = r->value; r_dup->mtypes = r->mtypes; ucc_list_add_tail(&dst->ranges, &r_dup->list_elem); } return UCC_OK; err: ucc_mrange_uint_destroy(dst); return UCC_ERR_NO_MEMORY; } void ucc_mrange_uint_destroy(ucc_mrange_uint_t *param) { ucc_mrange_t *r, *r_tmp; ucc_list_for_each_safe(r, r_tmp, ¶m->ranges, list_elem) { ucc_list_del(&r->list_elem); ucc_free(r); } } ucc-1.8.0/src/utils/ucc_mem_type.h0000664000175000017500000000201615211535620017226 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_MEM_TYPE_H_ #define UCC_MEM_TYPE_H_ #include "ucc/api/ucc.h" /* * Internal extensions of ucc_memory_type_t used to represent special cases. * These are intentionally *not* part of the public ucc_memory_type_t enum. */ #define UCC_MEMORY_TYPE_ASYMMETRIC \ ((ucc_memory_type_t)((int)UCC_MEMORY_TYPE_LAST + 1)) #define UCC_MEMORY_TYPE_NOT_APPLY \ ((ucc_memory_type_t)((int)UCC_MEMORY_TYPE_LAST + 2)) #define UCC_MEM_TYPE_MASK_FULL (UCC_BIT(UCC_MEMORY_TYPE_HOST) | \ UCC_BIT(UCC_MEMORY_TYPE_CUDA) | \ UCC_BIT(UCC_MEMORY_TYPE_CUDA_MANAGED) | \ UCC_BIT(UCC_MEMORY_TYPE_ROCM) | \ UCC_BIT(UCC_MEMORY_TYPE_ROCM_MANAGED)) #endif ucc-1.8.0/src/utils/ucc_compiler_def.h0000664000175000017500000001145015211535620020041 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_COMPILER_DEF_H_ #define UCC_COMPILER_DEF_H_ #include "config.h" #include "ucc/api/ucc_status.h" #include #include #include #include #include #ifndef SIZE_MAX #define SIZE_MAX ((size_t) -1) #endif #define ucc_offsetof ucs_offsetof #define ucc_container_of ucs_container_of #define ucc_derived_of ucs_derived_of #define ucc_strncpy_safe ucs_strncpy_safe #define ucc_snprintf_safe snprintf #define ucc_likely ucs_likely #define ucc_unlikely ucs_unlikely #define ucc_string_split ucs_string_split /* * Assertions which are checked in compile-time * In case of failure a compiler msg looks like this: * error: duplicate case value switch(0) {case 0:case (_cond):;} * * Usage: UCC_STATIC_ASSERT(condition) */ #define UCC_STATIC_ASSERT(_cond) \ switch(0) {case 0:case (_cond):;} /* Maximal allocation size for on-stack buffers */ #define UCC_ALLOCA_MAX_SIZE 1200 /** * alloca which makes sure the size is small enough. */ #define ucc_alloca(_size) \ ({ \ ucc_assert((_size) <= UCC_ALLOCA_MAX_SIZE); \ alloca(_size); \ }) /** * Prevent compiler from reordering instructions */ #define ucc_compiler_fence() asm volatile(""::: "memory") typedef int ucc_score_t; #define _UCC_PP_MAKE_STRING(x) #x #define UCC_PP_MAKE_STRING(x) _UCC_PP_MAKE_STRING(x) #define UCC_PP_QUOTE UCS_PP_QUOTE #define UCC_EMPTY_STATEMENT {} /* Packed structure */ #define UCC_S_PACKED __attribute__((packed)) /** * suppress unaligned pointer warning */ #define ucc_unaligned_ptr(_ptr) ({void *_p = (void*)(_ptr); _p;}) /* * Enable compiler checks for printf-like formatting. * * @param fmtargN number of formatting argument * @param vargN number of variadic argument */ #define UCC_F_PRINTF(fmtargN, vargN) __attribute__((format(printf, fmtargN, vargN))) /* A function which should not be optimized */ #if defined(HAVE_ATTRIBUTE_NOOPTIMIZE) && (HAVE_ATTRIBUTE_NOOPTIMIZE == 1) #define UCC_F_NOOPTIMIZE __attribute__((optimize("O0"))) #else #define UCC_F_NOOPTIMIZE #endif /* A function which does not return */ #define UCC_F_NORETURN __attribute__((noreturn)) #define UCC_COPY_PARAM_BY_FIELD(_dst, _src, _FIELD, _field) \ do { \ if ((_src)->mask & (_FIELD)) { \ (_dst)->_field = (_src)->_field; \ } \ } while (0) static inline ucc_status_t ucs_status_to_ucc_status(ucs_status_t status) { switch (status) { case UCS_OK: return UCC_OK; case UCS_INPROGRESS: return UCC_INPROGRESS; case UCS_ERR_NO_MEMORY: return UCC_ERR_NO_MEMORY; case UCS_ERR_INVALID_PARAM: return UCC_ERR_INVALID_PARAM; case UCS_ERR_NO_RESOURCE: return UCC_ERR_NO_RESOURCE; default: break; } return UCC_ERR_NO_MESSAGE; } static inline ucs_status_t ucc_status_to_ucs_status(ucc_status_t status) { switch (status) { case UCC_OK: return UCS_OK; case UCC_INPROGRESS: return UCS_INPROGRESS; case UCC_ERR_NO_MEMORY: return UCS_ERR_NO_MEMORY; case UCC_ERR_INVALID_PARAM: return UCS_ERR_INVALID_PARAM; case UCC_ERR_NO_RESOURCE: return UCS_ERR_NO_RESOURCE; default: break; } return UCS_ERR_NO_MESSAGE; } #define ucc_for_each_bit ucs_for_each_bit #define UCC_CHECK_GOTO(_cmd, _label, _status) \ do { \ _status = (_cmd); \ if (ucc_unlikely(_status != UCC_OK)) { \ goto _label; \ } \ } while (0) #if defined(__clang__) #define ucc_assume(x) __builtin_assume(x) #elif defined(__NVCOMPILER) #define ucc_assume(x) __builtin_assume(x) #elif defined(__GNUC__) #if (__GNUC__ >= 13) /* GCC 13+ has __attribute__((assume)) */ #define ucc_assume(x) __attribute__((assume(x))) #else /* For older GCC versions, we can use __builtin_unreachable() as a fallback */ #define ucc_assume(x) ((x) ? (void)0 : __builtin_unreachable()) #endif #else #define ucc_assume(x) do {} while (0) /* No-op for unsupported compilers */ #endif #endif ucc-1.8.0/src/utils/ucc_component.h0000664000175000017500000000354215211535620017416 0ustar alastairalastair/** * Copyright (c) 2001-2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_COMPONENT_H_ #define UCC_COMPONENT_H_ #include "config.h" #include "ucc/api/ucc.h" #include "utils/ucc_compiler_def.h" #include "utils/ucc_parser.h" #define UCC_MAX_FRAMEWORK_NAME_LEN 64 #define UCC_MAX_COMPONENT_NAME_LEN 64 typedef struct ucc_component_iface { const char *name; unsigned long id; void *dl_handle; ucc_score_t score; } ucc_component_iface_t; typedef struct ucc_component_framework { char *framework_name; int n_components; ucc_component_iface_t **components; ucc_config_names_array_t names; } ucc_component_framework_t; /* ucc_components_load searches for all available dynamic components with the name matching the pattern: libucc__*.so. The search is performed in the ucc_global_config.component_path. Each dynamic component must have a component interface structure defined. This structure must inherit from ucc_component_iface_t. The name of the structure must follow the pattern: ucc__. */ ucc_status_t ucc_components_load(const char *framework_name, ucc_component_framework_t *framework); /* get the component_iface_t from the initialized framework using the iface name. Returns NULL if the iface with the given name is not found in the framework. */ ucc_component_iface_t* ucc_get_component(ucc_component_framework_t *framework, const char *component_name); ucc_status_t ucc_component_check_scores_uniq(ucc_component_framework_t *framework); char* ucc_get_framework_components_list(ucc_component_framework_t *framework, const char* delimiter); #endif ucc-1.8.0/src/utils/ucc_proc_info.c0000664000175000017500000003277515211535620017377 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "config.h" #include #include #include #include #include #include #include #include #include "ucc_proc_info.h" #include "utils/ucc_malloc.h" #include "utils/ucc_math.h" #include "utils/ucc_sys.h" #ifdef HAVE_UCS_GET_SYSTEM_ID #include #endif ucc_proc_info_t ucc_local_proc; ucc_host_info_t ucc_local_host; uint64_t ucc_get_system_id() { #ifdef HAVE_UCS_GET_SYSTEM_ID return ucs_get_system_id(); #else return ucc_str_hash_djb2(ucc_get_host_name()); #endif } void ucc_proc_info_print(const ucc_proc_info_t *info) { char socket_str[16]; char numa_str[16]; if (!info) { return; } if (info->socket_id == UCC_SOCKET_ID_INVALID) { ucc_snprintf_safe(socket_str, sizeof(socket_str), "n/a"); } else { ucc_snprintf_safe(socket_str, sizeof(socket_str), "%u", (unsigned)info->socket_id); } if (info->numa_id == UCC_NUMA_ID_INVALID) { ucc_snprintf_safe(numa_str, sizeof(numa_str), "n/a"); } else { ucc_snprintf_safe(numa_str, sizeof(numa_str), "%u", (unsigned)info->numa_id); } ucc_debug("proc_info: host_hash=%" PRIu64 " host_id=%" PRIu64 " pid=%d socket=%s numa=%s", info->host_hash, info->host_id, info->pid, socket_str, numa_str); } void ucc_host_info_print(const ucc_host_info_t *info) { int i; if (!info) { return; } ucc_debug("host_info: host_id=%" PRIu64 " n_gpus=%u n_nics=%u", info->host_id, (unsigned)info->n_gpus, (unsigned)info->n_nics); for (i = 0; i < info->n_gpus; i++) { const ucc_gpu_info_t *gpu = &info->gpus[i]; ucc_debug("gpu_info: pci=%04x:%02x:%02x.%u caps=0x%x " "clique=%" PRIu64 " uuid=0x%016" PRIx64, (unsigned)gpu->pci.domain, (unsigned)gpu->pci.bus, (unsigned)gpu->pci.device, (unsigned)gpu->pci.function, gpu->caps, gpu->fabric_clique_id, gpu->uuid); } if (info->n_gpus > 0) { size_t buf_size = 64 + (size_t)info->n_gpus * info->n_gpus * 4 + (size_t)info->n_gpus * 16; char *line = ucc_malloc(buf_size, "nvlink_matrix_line"); if (line) { int offset = 0; offset += snprintf(line + offset, buf_size - offset, "nvlink_matrix cols:"); for (i = 0; i < info->n_gpus && offset < (int)buf_size; i++) { offset += snprintf(line + offset, buf_size - offset, " %u", (unsigned)i); } offset += snprintf(line + offset, buf_size - offset, " rows:"); for (i = 0; i < info->n_gpus && offset < (int)buf_size; i++) { int j; offset += snprintf(line + offset, buf_size - offset, " gpu%u[", (unsigned)i); for (j = 0; j < info->n_gpus && offset < (int)buf_size; j++) { offset += snprintf( line + offset, buf_size - offset, "%u%s", (unsigned)info->nvlink_matrix[i][j], (j + 1 < info->n_gpus) ? "," : ""); } offset += snprintf(line + offset, buf_size - offset, "]"); } ucc_debug("%s", line); ucc_free(line); } } for (i = 0; i < info->n_nics; i++) { const ucc_nic_info_t *nic = &info->nics[i]; ucc_debug("nic_info: pci=%04x:%02x:%02x.%u port=%u guid=0x%016" PRIx64 " name=%s", (unsigned)nic->pci.domain, (unsigned)nic->pci.bus, (unsigned)nic->pci.device, (unsigned)nic->pci.function, (unsigned)nic->port, nic->guid, nic->name); } } typedef unsigned long int cpu_mask_t; #define NCPUBITS (8 * sizeof(cpu_mask_t)) #define CPUELT(cpu) ((cpu) / NCPUBITS) #define CPUMASK(cpu) ((cpu_mask_t)1 << ((cpu) % NCPUBITS)) #define SBGP_CPU_ISSET(cpu, setsize, cpusetp) \ ({ \ size_t __cpu = (cpu); \ __cpu < 8 * (setsize) \ ? ((((const cpu_mask_t *)((cpusetp)->__bits))[__CPUELT(__cpu)] & \ CPUMASK(__cpu))) != 0 \ : 0; \ }) static int parse_cpuset_file(FILE *file, int *nr_psbl_cpus) { unsigned long start, stop; while (fscanf(file, "%lu", &start) == 1) { int c = fgetc(file); stop = start; if (c == '-') { if (fscanf(file, "%lu", &stop) != 1) { /* Range is usually - */ errno = EINVAL; return -1; } c = fgetc(file); } if (c == EOF || c == '\n') { *nr_psbl_cpus = (int)stop + 1; break; } if (c != ',') { /* Wrong terminating char */ errno = EINVAL; return -1; } } return 0; } static ucc_status_t ucc_get_bound_socket_id(ucc_socket_id_t *socketid) { cpu_set_t *cpuset = NULL; int sockid = -1, sockid2 = -1; int try, i, n_sockets, cpu, nr_cpus, nr_psbl_cpus = 0; size_t setsize; FILE * fptr, *possible; char str[1024]; int * socket_ids, tmpid; /* Get the number of total procs and online procs */ nr_cpus = sysconf(_SC_NPROCESSORS_CONF); /* Need to make sure nr_cpus !< possible_cpus+1 */ possible = fopen("/sys/devices/system/cpu/possible", "r"); if (possible) { if (parse_cpuset_file(possible, &nr_psbl_cpus) == 0) { if (nr_cpus < nr_psbl_cpus + 1) nr_cpus = nr_psbl_cpus; } fclose(possible); } if (!nr_cpus) { return UCC_ERR_NO_MESSAGE; } /* The cpuset size on some kernels needs to be bigger than * the number of nr_cpus, hwloc gets around this * by blocking on a loop and increasing nr_cpus. * We will try 1000 (arbitrary) attempts, and revert to hwloc * if all fail */ setsize = ((nr_cpus + NCPUBITS - 1) / NCPUBITS) * sizeof(cpu_mask_t); cpuset = __sched_cpualloc(nr_cpus); if (NULL == cpuset) { return UCC_ERR_NO_MESSAGE; } try = 1000; while (0 < sched_getaffinity(0, setsize, cpuset) && try > 0) { __sched_cpufree(cpuset); try--; nr_cpus *= 2; cpuset = __sched_cpualloc(nr_cpus); if (NULL == cpuset) { try = 0; break; } setsize = ((nr_cpus + NCPUBITS - 1) / NCPUBITS) * sizeof(cpu_mask_t); } /* If after all tries we're still not getting it, error out * let hwloc take over */ if (try == 0) { ucc_warn("Error when manually trying to discover socket_id using " "sched_getaffinity()"); __sched_cpufree(cpuset); return UCC_ERR_NO_MESSAGE; } socket_ids = ucc_malloc(nr_cpus * sizeof(int), "socket_ids"); if (!socket_ids) { ucc_error("failed to allocate %zd bytes for socket_ids array", nr_cpus * sizeof(int)); __sched_cpufree(cpuset); return UCC_ERR_NO_MEMORY; } /* Loop through all cpus, and check if I'm bound to the socket */ for (cpu = 0; cpu < nr_cpus; cpu++) { socket_ids[cpu] = -1; sprintf(str, "/sys/bus/cpu/devices/cpu%d/topology/physical_package_id", cpu); fptr = fopen(str, "r"); if (!fptr) { /* Do nothing just skip */ continue; } /* Read socket id from file */ if ((1 == fscanf(fptr, "%d", &tmpid)) && (tmpid >= 0)) { socket_ids[cpu] = tmpid; if (SBGP_CPU_ISSET(cpu, setsize, cpuset)) { if (sockid == -1) { sockid = tmpid; } else if (tmpid != sockid && sockid2 == -1) { sockid2 = tmpid; } } } fclose(fptr); } /* Check that a process is bound to 1 and only 1 socket */ if ((sockid != -1) && (sockid2 == -1)) { /* Some archs (eg. POWER) seem to have non-linear socket_ids. * Convert to logical index by findig first occurence of tmpid in * the global socket_ids array. */ n_sockets = ucc_sort_uniq(socket_ids, nr_cpus, 0); for (i = 0; i < n_sockets; i++) { if (socket_ids[i] == sockid) { if (i > (int)UCC_MAX_SOCKET_ID) { ucc_debug("too large socket id %d", i); __sched_cpufree(cpuset); return UCC_ERR_NOT_SUPPORTED; } *socketid = i; break; } } ucc_assert(((*socketid) != UCC_SOCKET_ID_INVALID) && ((*socketid) < nr_cpus)); } __sched_cpufree(cpuset); ucc_free(socket_ids); return UCC_OK; } #define LOAD_NUMA_SYM(_sym) \ ({ \ void *h = dlsym(handle, _sym); \ if ((error = dlerror()) != NULL) { \ ucc_debug("%s", error); \ status = UCC_ERR_NOT_FOUND; \ goto error; \ } \ h; \ }) static ucc_status_t ucc_get_bound_numa_id(ucc_numa_id_t *numaid) { ucc_status_t status = UCC_OK; char * error; void * handle, *cpumask; int i, numa_node, n_cfg_cpus, nn; int (*ucc_numa_available)(void); int (*ucc_numa_num_configured_cpus)(void); void *(*ucc_numa_allocate_cpumask)(void); void *(*ucc_numa_sched_getaffinity)(int, void *); int (*ucc_numa_bitmask_isbitset)(void *, int); int (*ucc_numa_node_of_cpu)(int); int (*ucc_numa_bitmask_free)(void *); handle = dlopen("libnuma.so", RTLD_LAZY); if (!handle) { ucc_debug("%s", dlerror()); return UCC_ERR_NOT_FOUND; } ucc_numa_available = (int(*)(void))LOAD_NUMA_SYM("numa_available"); ucc_numa_num_configured_cpus = (int(*)(void))LOAD_NUMA_SYM("numa_num_configured_cpus"); ucc_numa_allocate_cpumask = (void*(*)(void))LOAD_NUMA_SYM("numa_allocate_cpumask"); ucc_numa_sched_getaffinity = (void*(*)(int, void*))LOAD_NUMA_SYM("numa_sched_getaffinity"); ucc_numa_bitmask_isbitset = (int(*)(void*, int))LOAD_NUMA_SYM("numa_bitmask_isbitset"); ucc_numa_node_of_cpu = (int(*)(int))LOAD_NUMA_SYM("numa_node_of_cpu"); ucc_numa_bitmask_free = (int(*)(void*))LOAD_NUMA_SYM("numa_bitmask_free"); if (-1 == ucc_numa_available()) { ucc_debug("libnuma is not available"); status = UCC_ERR_NO_MESSAGE; goto error; } /* Load the cpumask which a process is bound to, then loop through cpus from that mask and check the numa nodes those cpus belong to. If there are more than 1 numa nodes detected return -1, i.e. not bound to a numa. */ cpumask = ucc_numa_allocate_cpumask(); if (!cpumask) { ucc_error("numa_allocate_cpumask failed"); status = UCC_ERR_NO_MESSAGE; goto error; } ucc_numa_sched_getaffinity(getpid(), cpumask); numa_node = -1; n_cfg_cpus = ucc_numa_num_configured_cpus(); for (i = 0; i < n_cfg_cpus; i++) { if (ucc_numa_bitmask_isbitset(cpumask, i)) { nn = ucc_numa_node_of_cpu(i); if (numa_node == -1) { numa_node = nn; } else if (numa_node != nn && numa_node >= 0) { /* At least 2 different numa nodes detected for a given cpu set. set numa_node to -1, which means not bound to a numa. */ numa_node = -1; break; } } } ucc_numa_bitmask_free(cpumask); if (numa_node >= 0) { if (numa_node > (int)UCC_MAX_NUMA_ID) { ucc_debug("too large numa id %d", numa_node); status = UCC_ERR_NOT_SUPPORTED; goto error; } *numaid = numa_node; } error: dlclose(handle); return status; } ucc_status_t ucc_local_proc_info_init() { ucc_local_proc.host_hash = ucc_get_system_id(); ucc_local_proc.pid = getpid(); ucc_local_proc.socket_id = UCC_SOCKET_ID_INVALID; ucc_local_proc.numa_id = UCC_NUMA_ID_INVALID; if (UCC_OK != ucc_get_bound_socket_id(&ucc_local_proc.socket_id)) { ucc_debug("failed to get bound socket id"); } if (UCC_OK != ucc_get_bound_numa_id(&ucc_local_proc.numa_id)) { ucc_debug("failed to get bound numa id"); } ucc_debug( "proc pid %d, host %s, host_hash %lu, sockid %d, numaid %d", ucc_local_proc.pid, ucc_get_host_name(), ucc_local_proc.host_hash, (int)ucc_local_proc.socket_id, (int)ucc_local_proc.numa_id); return UCC_OK; } ucc-1.8.0/src/utils/ucc_string.c0000664000175000017500000002031415211535620016711 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_string.h" #include "ucc_malloc.h" #include "ucc_log.h" #include "ucc_coll_utils.h" #include #include #include #include #include char **ucc_str_split(const char *str, const char *delim) { unsigned alloc_size = 8; unsigned size = 0; char **out; char *str_copy, *token, *saveptr; int i; out = ucc_malloc(alloc_size * sizeof(char *), "str_split"); if (!out) { ucc_error("failed to allocate %zd bytes for str_split", alloc_size * sizeof(char *)); return NULL; } str_copy = strdup(str); if (!str_copy) { ucc_error("failed to duplicate string"); goto error; } token = strtok_r(str_copy, delim, &saveptr); while (NULL != token) { out[size] = strdup(token); if (!out[size]) { ucc_error("failed to duplicate string"); goto error; } size++; if (size == (alloc_size - 1)) { /* keep 1 for NULL mark */ alloc_size *= 2; out = ucc_realloc(out, alloc_size * sizeof(char *), "str_split"); if (!out) { ucc_error("failed to reallocate %zd bytes for str_split", alloc_size * sizeof(char *)); goto error; } } token = strtok_r(NULL, delim, &saveptr); } out[size] = NULL; ucc_free(str_copy); return out; error: if (out) { for (i = 0; i < size; i++) { ucc_free(out[i]); } ucc_free(out); } ucc_free(str_copy); return NULL; } unsigned ucc_str_split_count(char **split) { int count; if (NULL == split) { return 0; } count = 0; while (NULL != (*split)) { count++; split++; } return count; } void ucc_str_split_free(char **split) { char **iter = split; if (NULL == split) { return; } while (NULL != (*iter)) { ucc_free(*iter); iter++; } free(split); } ucc_status_t ucc_str_is_number(const char *str) { unsigned i, len; len = strlen(str); for (i = 0; i < len; i++) { if (!isdigit(str[i])) { return UCC_ERR_INVALID_PARAM; } } return UCC_OK; } ucc_status_t ucc_str_to_memunits(const char *buf, void *dest) { return ucs_status_to_ucc_status(ucs_str_to_memunits(buf, dest)); } const char* ucc_strstr_last(const char* string, const char* pattern) { const char *found = NULL; while ((string = strstr(string, pattern))) { found = string++; } return found; } ucc_status_t ucc_str_concat_n(const char *strs[], int n, char **out) { size_t len = 1; char *rst; int i; for (i = 0; i < n; i++) { len += strlen(strs[i]); } rst = ucc_malloc(len, "str_concat"); if (!rst) { ucc_error("failed to allocate %zd bytes for concatenated string", len); return UCC_ERR_NO_MEMORY; } ucc_strncpy_safe(rst, strs[0], len); for (i = 1; i < n; i++) { len -= strlen(strs[i - 1]); strncat(rst, strs[i], len); } *out = rst; return UCC_OK; } ucc_status_t ucc_str_concat(const char *str1, const char *str2, char **out) { const char *strs[2] = {str1, str2}; return ucc_str_concat_n(strs, 2, out); } ucc_status_t ucc_str_to_memunits_range(const char *str, size_t *start, size_t *end) { ucc_status_t status = UCC_OK; char **munits; unsigned n_munits; munits = ucc_str_split(str, "-"); if (!munits) { return UCC_ERR_NO_MEMORY; } n_munits = ucc_str_split_count(munits); if (n_munits != 2 || UCC_OK != ucc_str_to_memunits(munits[0], start) || UCC_OK != ucc_str_to_memunits(munits[1], end)) { status = UCC_ERR_INVALID_PARAM; } ucc_str_split_free(munits); return status; } ucc_status_t ucc_str_to_mtype_map(const char *str, const char *delim, uint32_t *mt_map) { ucc_status_t status = UCC_OK; char ** tokens; unsigned i, n_tokens; ucc_memory_type_t t; *mt_map = 0; tokens = ucc_str_split(str, delim); if (!tokens) { return UCC_ERR_NO_MEMORY; } n_tokens = ucc_str_split_count(tokens); for (i = 0; i < n_tokens; i++) { t = ucc_mem_type_from_str(tokens[i]); if (t == UCC_MEMORY_TYPE_LAST) { /* entry does not match any memory type name */ status = UCC_ERR_INVALID_PARAM; goto out; } *mt_map |= UCC_BIT(t); } out: ucc_str_split_free(tokens); return status; } void ucc_mtype_map_to_str(uint32_t mt_map, const char *delim, char *buf, size_t max) { int i; size_t last; for (i = 0; i < UCC_MEMORY_TYPE_LAST; i++) { if (UCC_BIT(i) & mt_map) { ucc_snprintf_safe(buf, max, "%s%s", ucc_mem_type_str((ucc_memory_type_t)i), delim); last = strlen(buf); if (max - last -1 <= 0) { /* no more space in buf for next range*/ return; } max -= last; buf += last; } } /* remove last delimiter */ buf -= strlen(delim); *buf = '\0'; } ssize_t ucc_string_find_in_list(const char *str, const char **string_list, int case_sensitive) { size_t i; for (i = 0; string_list[i] != NULL; ++i) { if ((case_sensitive && (strcmp(string_list[i], str) == 0)) || (!case_sensitive && (strcasecmp(string_list[i], str) == 0))) { return i; } } return -1; } void ucc_strncpy_zero(char *dest, const char *src, size_t max) { if (max) { strncpy(dest, src, max - 1); dest[max - 1] = '\0'; } } void ucc_snprintf_zero(char *buf, size_t size, const char *fmt, ...) { va_list ap; memset(buf, 0, size); va_start(ap, fmt); // NOLINTNEXTLINE(clang-analyzer-valist.Uninitialized) vsnprintf(buf, size, fmt, ap); va_end(ap); } ucc_status_t ucc_string_alloc_path_buffer(char **buffer_p, const char *name) { char *temp_buffer = ucc_malloc(PATH_MAX, name); if (temp_buffer == NULL) { ucc_error("failed to allocate memory for %s", name); return UCC_ERR_NO_MEMORY; } *buffer_p = temp_buffer; return UCC_OK; } void ucc_fill_filename_template(const char *tmpl, char *buf, size_t max) { char *p, *end; const char *pf, *pp; size_t length; time_t t; p = buf; end = buf + max - 1; *end = 0; pf = tmpl; while (*pf != 0 && p < end) { pp = strchr(pf, '%'); if (pp == NULL) { strncpy(p, pf, end - p); p = end; break; } length = ucs_min(pp - pf, end - p); strncpy(p, pf, length); p += length; /* default length of the modifier (e.g. %p) */ length = 2; switch (*(pp + 1)) { case 'p': snprintf(p, end - p, "%d", getpid()); break; case 'h': snprintf(p, end - p, "%s", ucc_get_host_name()); break; case 'c': snprintf(p, end - p, "%02d", ucc_get_first_cpu()); break; case 't': t = time(NULL); strftime(p, end - p, "%Y-%m-%d-%H-%M-%S", localtime(&t)); break; case 'u': snprintf(p, end - p, "%s", ucc_basename(ucc_get_user_name())); break; case 'e': snprintf(p, end - p, "%s", ucc_basename(ucc_get_exe())); break; case 'i': snprintf(p, end - p, "%u", geteuid()); break; default: *(p++) = *pp; length = 1; break; } pf = pp + length; p += strlen(p); } *p = 0; } /* NOLINTNEXTLINE */ char *ucc_strdup(const char *src, const char *name) { char *str = strdup(src); return str; } const char* ucc_basename(const char *path) { const char *name = strrchr(path, '/'); return (name == NULL) ? path : name + 1; } ucc-1.8.0/src/utils/ucc_rcache.h0000664000175000017500000000546615211535620016650 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_RCACHE_H_ #define UCC_RCACHE_H_ #include #include #include //TODO: handle external events #define ucc_rcache_t ucs_rcache_t #define ucc_rcache_ops_t ucs_rcache_ops_t #define ucc_rcache_params_t ucs_rcache_params_t #define ucc_rcache_region_t ucs_rcache_region_t static inline void ucc_rcache_set_default_params(ucs_rcache_params_t *rcache_params) { rcache_params->region_struct_size = sizeof(ucs_rcache_region_t); rcache_params->ucm_events = 0; rcache_params->ucm_event_priority = 1000; rcache_params->ops = NULL; rcache_params->context = NULL; rcache_params->flags = 0; rcache_params->max_regions = UCS_MEMUNITS_INF; rcache_params->max_size = UCS_MEMUNITS_INF; rcache_params->max_unreleased = UCS_MEMUNITS_INF; } #define ucc_rcache_destroy ucs_rcache_destroy #define ucc_rcache_region_hold ucs_rcache_region_hold #define ucc_rcache_region_put ucs_rcache_region_put #define ucc_rcache_region_invalidate ucs_rcache_region_invalidate static inline void ucc_rcache_merge_cb_empty(void *context, ucs_rcache_t *rcache, void *arg, ucs_rcache_region_t *region) { return; } /* Wrapper functions for status conversion */ static inline ucc_status_t ucc_rcache_create(const ucc_rcache_params_t *params, const char *name, ucc_rcache_t **rcache_p) { #ifndef UCS_HAVE_RCACHE_REGION_ALIGNMENT ucc_rcache_params_t params_dup = *params; params_dup.alignment = UCS_PGT_ADDR_ALIGN; params_dup.max_alignment = ucc_get_page_size(); return ucs_status_to_ucc_status(ucs_rcache_create( ¶ms_dup, name, NULL, rcache_p)); #else return ucs_status_to_ucc_status(ucs_rcache_create( params, name, NULL, rcache_p)); #endif } /* [arg] parameter allows passing additional information from mem_reg callabck. For example, it can be used to indicate whether the entry was found in the cache or new registration happened. */ static inline ucc_status_t ucc_rcache_get(ucc_rcache_t *rcache, void *address, size_t length, void *arg, ucc_rcache_region_t **region_p) { ucs_status_t status; #ifdef UCS_HAVE_RCACHE_REGION_ALIGNMENT status = ucs_rcache_get(rcache, address, length, UCS_PGT_ADDR_ALIGN, PROT_READ | PROT_WRITE, arg, region_p); #else status = ucs_rcache_get(rcache, address, length, PROT_READ | PROT_WRITE, arg, region_p); #endif return ucs_status_to_ucc_status(status); } #endif ucc-1.8.0/src/utils/arch/0000775000175000017500000000000015211535620015322 5ustar alastairalastairucc-1.8.0/src/utils/arch/cuda_def.h0000664000175000017500000001066715211535620017237 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_CUDA_DEF_H #define UCC_CUDA_DEF_H #include "config.h" #if HAVE_CUDA #include "utils/ucc_log.h" #include #include static inline ucc_status_t cuda_error_to_ucc_status(cudaError_t cuda_status) { ucc_status_t ucc_status; switch(cuda_status) { case cudaSuccess: ucc_status = UCC_OK; break; case cudaErrorNotReady: ucc_status = UCC_INPROGRESS; break; case cudaErrorInvalidValue: ucc_status = UCC_ERR_INVALID_PARAM; break; default: ucc_status = UCC_ERR_NO_MESSAGE; } return ucc_status; } #define CUDA_FUNC(_func) \ ({ \ ucc_status_t _status; \ do { \ cudaError_t _result = (_func); \ if (ucc_unlikely(cudaSuccess != _result)) { \ ucc_error("%s() failed: %d(%s)", \ #_func, _result, cudaGetErrorString(_result)); \ } \ _status = cuda_error_to_ucc_status(_result); \ } while (0); \ _status; \ }) #define CUDADRV_FUNC(_func) \ ({ \ ucc_status_t _status = UCC_OK; \ do { \ CUresult _result = (_func); \ const char *cu_err_str; \ if (ucc_unlikely(CUDA_SUCCESS != _result)) { \ cuGetErrorString(_result, &cu_err_str); \ ucc_error("%s() failed: %d(%s)", \ #_func, _result, cu_err_str); \ _status = UCC_ERR_NO_MESSAGE; \ } \ } while (0); \ _status; \ }) #define CUDA_CHECK(_cmd) \ /* coverity[dead_error_line] */ \ do { \ ucc_status_t _cuda_status = CUDA_FUNC(_cmd); \ if (ucc_unlikely(_cuda_status != UCC_OK)) { \ return _cuda_status; \ } \ } while(0) #define CUDADRV_CHECK(_cmd) \ /* coverity[dead_error_line] */ \ do { \ ucc_status_t _cuda_status = CUDADRV_FUNC(_cmd); \ if (ucc_unlikely(_cuda_status != UCC_OK)) { \ return _cuda_status; \ } \ } while(0) #define CUDA_CHECK_GOTO(_cmd, _label, _cuda_status) \ do { \ _cuda_status = CUDA_FUNC(_cmd); \ if (ucc_unlikely(_cuda_status != UCC_OK)) { \ goto _label; \ } \ } while (0) #endif #endif ucc-1.8.0/src/utils/arch/riscv64/0000775000175000017500000000000015211535620016622 5ustar alastairalastairucc-1.8.0/src/utils/arch/riscv64/cpu.h0000664000175000017500000000200115211535620017553 0ustar alastairalastair/** * Copyright (c) 2001-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) ARM Ltd. 2016-2017. ALL RIGHTS RESERVED. * Copyright (C) Rivos Inc. 2023 * * See file LICENSE for terms. */ #ifndef UCC_UTILS_ARCH_RISCV64_CPU_H_ #define UCC_UTILS_ARCH_RISCV64_CPU_H_ #define UCC_ARCH_CACHE_LINE_SIZE 64 /* RVWMO rules */ #define ucc_memory_bus_fence() asm volatile("fence iorw, iorw" ::: "memory") #define ucc_memory_bus_store_fence() asm volatile("fence ow, ow" ::: "memory") #define ucc_memory_bus_load_fence() asm volatile("fence ir, ir" ::: "memory") #define ucc_memory_cpu_fence() asm volatile("fence rw, rw" ::: "memory") #define ucc_memory_cpu_store_fence() asm volatile("fence rw, w" ::: "memory") #define ucc_memory_cpu_load_fence() asm volatile("fence r, rw" ::: "memory") static inline ucc_cpu_model_t ucc_arch_get_cpu_model() { return UCC_CPU_MODEL_UNKNOWN; } static inline ucc_cpu_vendor_t ucc_arch_get_cpu_vendor() { return UCC_CPU_VENDOR_GENERIC_RISCV; } #endif ucc-1.8.0/src/utils/arch/ppc64/0000775000175000017500000000000015211535620016256 5ustar alastairalastairucc-1.8.0/src/utils/arch/ppc64/cpu.h0000664000175000017500000000212015211535620017211 0ustar alastairalastair/** * Copyright (c) 2001-2013, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) ARM Ltd. 2016-2017. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_PPC64_CPU_H_ #define UCC_PPC64_CPU_H_ #define UCC_ARCH_CACHE_LINE_SIZE 128 /* Assume the worst - weak memory ordering */ #define ucc_memory_bus_fence() asm volatile ("sync"::: "memory") #define ucc_memory_bus_store_fence() ucc_memory_bus_fence() #define ucc_memory_bus_load_fence() ucc_memory_bus_fence() #define ucc_memory_cpu_fence() ucc_memory_bus_fence() #define ucc_memory_cpu_store_fence() asm volatile ("lwsync \n" \ ::: "memory") #define ucc_memory_cpu_load_fence() asm volatile ("lwsync \n" \ "isync \n" \ ::: "memory") static inline ucc_cpu_model_t ucc_arch_get_cpu_model() { return UCC_CPU_MODEL_UNKNOWN; } static inline ucc_cpu_vendor_t ucc_arch_get_cpu_vendor() { return UCC_CPU_VENDOR_GENERIC_PPC; } #endif ucc-1.8.0/src/utils/arch/cpu.h0000664000175000017500000001046115211535620016264 0ustar alastairalastair/** * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2024. ALL RIGHTS RESERVED. * Copyright (C) ARM Ltd. 2016. ALL RIGHTS RESERVED. * Copyright (C) Shanghai Zhaoxin Semiconductor Co., Ltd. 2020. ALL RIGHTS RESERVED. * Copyright (C) Rivos Inc. 2023 * Copyright (C) 2026, Fujitsu Limited. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_ARCH_CPU_H #define UCC_ARCH_CPU_H #ifdef HAVE_CONFIG_H # include "config.h" #endif #include "utils/ucc_compiler_def.h" #include /* CPU models */ typedef enum ucc_cpu_model { UCC_CPU_MODEL_UNKNOWN, UCC_CPU_MODEL_INTEL_IVYBRIDGE, UCC_CPU_MODEL_INTEL_SANDYBRIDGE, UCC_CPU_MODEL_INTEL_NEHALEM, UCC_CPU_MODEL_INTEL_WESTMERE, UCC_CPU_MODEL_INTEL_HASWELL, UCC_CPU_MODEL_INTEL_BROADWELL, UCC_CPU_MODEL_INTEL_SKYLAKE, UCC_CPU_MODEL_ARM_AARCH64, UCC_CPU_MODEL_AMD_NAPLES, UCC_CPU_MODEL_AMD_ROME, UCC_CPU_MODEL_AMD_MILAN, UCC_CPU_MODEL_AMD_GENOA, UCC_CPU_MODEL_ZHAOXIN_ZHANGJIANG, UCC_CPU_MODEL_ZHAOXIN_WUDAOKOU, UCC_CPU_MODEL_ZHAOXIN_LUJIAZUI, UCC_CPU_MODEL_NVIDIA_GRACE, UCC_CPU_MODEL_FUJITSU_A64FX, UCC_CPU_MODEL_FUJITSU_MONAKA, UCC_CPU_MODEL_LAST } ucc_cpu_model_t; /* CPU vendors */ typedef enum ucc_cpu_vendor { UCC_CPU_VENDOR_UNKNOWN, UCC_CPU_VENDOR_INTEL, UCC_CPU_VENDOR_AMD, UCC_CPU_VENDOR_GENERIC_ARM, UCC_CPU_VENDOR_GENERIC_PPC, UCC_CPU_VENDOR_GENERIC_RISCV, UCC_CPU_VENDOR_FUJITSU_ARM, UCC_CPU_VENDOR_ZHAOXIN, UCC_CPU_VENDOR_NVIDIA, UCC_CPU_VENDOR_LAST } ucc_cpu_vendor_t; static inline ucc_cpu_vendor_t ucc_get_vendor_from_str(const char *v_name) { if (strcasecmp(v_name, "intel") == 0) return UCC_CPU_VENDOR_INTEL; if (strcasecmp(v_name, "amd") == 0) return UCC_CPU_VENDOR_AMD; if (strcasecmp(v_name, "arm") == 0) return UCC_CPU_VENDOR_GENERIC_ARM; if (strcasecmp(v_name, "ppc") == 0) return UCC_CPU_VENDOR_GENERIC_PPC; if (strcasecmp(v_name, "riscv") == 0) return UCC_CPU_VENDOR_GENERIC_RISCV; if (strcasecmp(v_name, "fujitsu") == 0) return UCC_CPU_VENDOR_FUJITSU_ARM; if (strcasecmp(v_name, "zhaoxin") == 0) return UCC_CPU_VENDOR_ZHAOXIN; if (strcasecmp(v_name, "nvidia") == 0) return UCC_CPU_VENDOR_NVIDIA; return UCC_CPU_VENDOR_UNKNOWN; } static inline ucc_cpu_model_t ucc_get_model_from_str(const char *m_name) { if (strcasecmp(m_name, "ivybridge") == 0) return UCC_CPU_MODEL_INTEL_IVYBRIDGE; if (strcasecmp(m_name, "sandybridge") == 0) return UCC_CPU_MODEL_INTEL_SANDYBRIDGE; if (strcasecmp(m_name, "nehalem") == 0) return UCC_CPU_MODEL_INTEL_NEHALEM; if (strcasecmp(m_name, "westmere") == 0) return UCC_CPU_MODEL_INTEL_WESTMERE; if (strcasecmp(m_name, "haswell") == 0) return UCC_CPU_MODEL_INTEL_HASWELL; if (strcasecmp(m_name, "broadwell") == 0) return UCC_CPU_MODEL_INTEL_BROADWELL; if (strcasecmp(m_name, "skylake") == 0) return UCC_CPU_MODEL_INTEL_SKYLAKE; if (strcasecmp(m_name, "aarch64") == 0) return UCC_CPU_MODEL_ARM_AARCH64; if (strcasecmp(m_name, "naples") == 0) return UCC_CPU_MODEL_AMD_NAPLES; if (strcasecmp(m_name, "rome") == 0) return UCC_CPU_MODEL_AMD_ROME; if (strcasecmp(m_name, "milan") == 0) return UCC_CPU_MODEL_AMD_MILAN; if (strcasecmp(m_name, "genoa") == 0) return UCC_CPU_MODEL_AMD_GENOA; if (strcasecmp(m_name, "zhangjiang") == 0) return UCC_CPU_MODEL_ZHAOXIN_ZHANGJIANG; if (strcasecmp(m_name, "wudaokou") == 0) return UCC_CPU_MODEL_ZHAOXIN_WUDAOKOU; if (strcasecmp(m_name, "lujiazui") == 0) return UCC_CPU_MODEL_ZHAOXIN_LUJIAZUI; if (strcasecmp(m_name, "grace") == 0) return UCC_CPU_MODEL_NVIDIA_GRACE; if (strcasecmp(m_name, "a64fx") == 0) return UCC_CPU_MODEL_FUJITSU_A64FX; if (strcasecmp(m_name, "monaka") == 0) return UCC_CPU_MODEL_FUJITSU_MONAKA; return UCC_CPU_MODEL_UNKNOWN; } #if defined(__x86_64__) # include "x86_64/cpu.h" #elif defined(__powerpc64__) # include "ppc64/cpu.h" #elif defined(__aarch64__) # include "aarch64/cpu.h" #elif defined(__riscv) && (__riscv_xlen == 64) # include "riscv64/cpu.h" #else # error "Unsupported architecture" #endif #define UCC_CACHE_LINE_SIZE UCC_ARCH_CACHE_LINE_SIZE #endif ucc-1.8.0/src/utils/arch/x86_64/0000775000175000017500000000000015211535620016260 5ustar alastairalastairucc-1.8.0/src/utils/arch/x86_64/cpu.h0000664000175000017500000000156515211535620017227 0ustar alastairalastair/** * Copyright (c) 2001-2013, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) ARM Ltd. 2016-2017. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_X86_64_H_ #define UCC_X86_64_H_ #include "utils/ucc_compiler_def.h" #define UCC_ARCH_CACHE_LINE_SIZE 64 /** * In x86_64, there is strong ordering of each processor with respect to another * processor, but weak ordering with respect to the bus. */ #define ucc_memory_bus_store_fence() asm volatile ("sfence" ::: "memory") #define ucc_memory_bus_load_fence() asm volatile ("lfence" ::: "memory") #define ucc_memory_cpu_fence() ucc_compiler_fence() #define ucc_memory_cpu_store_fence() ucc_compiler_fence() #define ucc_memory_cpu_load_fence() ucc_compiler_fence() ucc_cpu_model_t ucc_arch_get_cpu_model() UCC_F_NOOPTIMIZE; ucc_cpu_vendor_t ucc_arch_get_cpu_vendor(); #endif ucc-1.8.0/src/utils/arch/x86_64/cpu.c0000664000175000017500000001325615211535620017222 0ustar alastairalastair/** * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2023. ALL RIGHTS RESERVED. * Copyright (C) Advanced Micro Devices, Inc. 2019. ALL RIGHTS RESERVED. * Copyright (C) Shanghai Zhaoxin Semiconductor Co., Ltd. 2020. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #if defined(__x86_64__) #ifdef HAVE_CONFIG_H # include "config.h" #endif #include "utils/arch/cpu.h" #define X86_CPUID_GENUINEINTEL "GenuntelineI" /* GenuineIntel in magic notation */ #define X86_CPUID_AUTHENTICAMD "AuthcAMDenti" /* AuthenticAMD in magic notation */ #define X86_CPUID_CENTAURHAULS "CentaulsaurH" /* CentaurHauls in magic notation */ #define X86_CPUID_SHANGHAI " Shai angh" /* Shanghai in magic notation */ #define X86_CPUID_GET_MODEL 0x00000001u #define X86_CPUID_GET_BASE_VALUE 0x00000000u #define X86_CPUID_GET_EXTD_VALUE 0x00000007u #define X86_CPUID_GET_MAX_VALUE 0x80000000u #define X86_CPUID_INVARIANT_TSC 0x80000007u #define X86_CPUID_GET_CACHE_INFO 0x00000002u #define X86_CPUID_GET_LEAF4_INFO 0x00000004u typedef union ucc_x86_cpu_registers { struct { union { uint32_t eax; uint8_t max_iter; /* leaf 2 - max iterations */ }; union { struct { uint32_t ebx; uint32_t ecx; uint32_t edx; }; char id[sizeof(uint32_t) * 3]; /* leaf 0 - CPU ID */ }; }; union { uint32_t value; uint8_t tag[sizeof(uint32_t)]; } reg[4]; /* leaf 2 tags */ } UCC_S_PACKED ucc_x86_cpu_registers; /* CPU version */ typedef union ucc_x86_cpu_version { struct { unsigned stepping : 4; unsigned model : 4; unsigned family : 4; unsigned type : 2; unsigned unused : 2; unsigned ext_model : 4; unsigned ext_family : 8; }; uint32_t reg; } UCC_S_PACKED ucc_x86_cpu_version_t; static UCC_F_NOOPTIMIZE inline void ucc_x86_cpuid(uint32_t level, uint32_t *a, uint32_t *b, uint32_t *c, uint32_t *d) { asm volatile ("cpuid\n\t" : "=a"(*a), "=b"(*b), "=c"(*c), "=d"(*d) : "0"(level)); } ucc_cpu_vendor_t ucc_arch_get_cpu_vendor() { ucc_x86_cpu_registers reg = {}; /* Silence static checker */ ucc_x86_cpuid(X86_CPUID_GET_BASE_VALUE, ucc_unaligned_ptr(®.eax), ucc_unaligned_ptr(®.ebx), ucc_unaligned_ptr(®.ecx), ucc_unaligned_ptr(®.edx)); if (!memcmp(reg.id, X86_CPUID_GENUINEINTEL, sizeof(X86_CPUID_GENUINEINTEL) - 1)) { return UCC_CPU_VENDOR_INTEL; } else if (!memcmp(reg.id, X86_CPUID_AUTHENTICAMD, sizeof(X86_CPUID_AUTHENTICAMD) - 1)) { return UCC_CPU_VENDOR_AMD; } else if (!memcmp(reg.id, X86_CPUID_CENTAURHAULS, sizeof(X86_CPUID_CENTAURHAULS) - 1) || !memcmp(reg.id, X86_CPUID_SHANGHAI, sizeof(X86_CPUID_SHANGHAI) - 1)) { return UCC_CPU_VENDOR_ZHAOXIN; } return UCC_CPU_VENDOR_UNKNOWN; } ucc_cpu_model_t ucc_arch_get_cpu_model() { ucc_x86_cpu_version_t version = {}; /* Silence static checker */ uint32_t _ebx, _ecx, _edx; uint32_t model, family; /* Get CPU model/family */ ucc_x86_cpuid(X86_CPUID_GET_MODEL, ucc_unaligned_ptr(&version.reg), &_ebx, &_ecx, &_edx); model = version.model; family = version.family; /* Adjust family/model */ if (family == 0xf) { family += version.ext_family; } if ((family == 0x6) || (family == 0x7) || (family == 0xf) || (family == 0x17) || (family == 0x19)) { model = (version.ext_model << 4) | model; } if (ucc_arch_get_cpu_vendor() == UCC_CPU_VENDOR_ZHAOXIN) { if (family == 0x06) { switch (model) { case 0x0f: return UCC_CPU_MODEL_ZHAOXIN_ZHANGJIANG; } } if (family == 0x07) { switch (model) { case 0x1b: return UCC_CPU_MODEL_ZHAOXIN_WUDAOKOU; case 0x3b: return UCC_CPU_MODEL_ZHAOXIN_LUJIAZUI; } } } else { /* Check known CPUs */ if (family == 0x06) { switch (model) { case 0x3a: case 0x3e: return UCC_CPU_MODEL_INTEL_IVYBRIDGE; case 0x2a: case 0x2d: return UCC_CPU_MODEL_INTEL_SANDYBRIDGE; case 0x1a: case 0x1e: case 0x1f: case 0x2e: return UCC_CPU_MODEL_INTEL_NEHALEM; case 0x25: case 0x2c: case 0x2f: return UCC_CPU_MODEL_INTEL_WESTMERE; case 0x3c: case 0x3f: case 0x45: case 0x46: return UCC_CPU_MODEL_INTEL_HASWELL; case 0x3d: case 0x47: case 0x4f: case 0x56: return UCC_CPU_MODEL_INTEL_BROADWELL; case 0x5e: case 0x4e: case 0x55: return UCC_CPU_MODEL_INTEL_SKYLAKE; } } if (family == 0x17) { switch (model) { case 0x29: return UCC_CPU_MODEL_AMD_NAPLES; case 0x31: return UCC_CPU_MODEL_AMD_ROME; } } if (family == 0x19) { switch (model) { case 0x00: case 0x01: return UCC_CPU_MODEL_AMD_MILAN; case 0x11: return UCC_CPU_MODEL_AMD_GENOA; } } } return UCC_CPU_MODEL_UNKNOWN; } #endif ucc-1.8.0/src/utils/arch/aarch64/0000775000175000017500000000000015211535620016552 5ustar alastairalastairucc-1.8.0/src/utils/arch/aarch64/cpu.h0000664000175000017500000000526215211535620017517 0ustar alastairalastair/** * Copyright (c) 2001-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) ARM Ltd. 2016-2020. ALL RIGHTS RESERVED. * Copyright (C) Stony Brook University. 2016-2020. ALL RIGHTS RESERVED. * Copyright (C) 2026, Fujitsu Limited. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_AARCH64_CPU_H_ #define UCC_AARCH64_CPU_H_ #define UCC_ARCH_CACHE_LINE_SIZE 64 /** * Assume the worst - weak memory ordering. */ #define ucc_aarch64_dmb(_op) asm volatile ("dmb " #_op ::: "memory") #define ucc_aarch64_isb(_op) asm volatile ("isb " #_op ::: "memory") #define ucc_aarch64_dsb(_op) asm volatile ("dsb " #_op ::: "memory") /* The macro is used to serialize stores across Normal NC (or Device) and WB * memory, (see Arm Spec, B2.7.2). Based on recent changes in Linux kernel: * https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=22ec71615d824f4f11d38d0e55a88d8956b7e45f * * The underlying barrier code was changed to use lighter weight DMB instead * of DSB. The barrier used for synchronization of access between write back * and device mapped memory (PCIe BAR). */ #define ucc_memory_bus_store_fence() ucc_aarch64_dmb(oshst) #define ucc_memory_bus_load_fence() ucc_aarch64_dmb(oshld) #define ucc_memory_cpu_fence() ucc_aarch64_dmb(ish) #define ucc_memory_cpu_store_fence() ucc_aarch64_dsb(ishst) #define ucc_memory_cpu_load_fence() ucc_aarch64_dmb(ishld) typedef struct ucc_aarch64_cpuid { int implementer; int architecture; int variant; int part; int revision; } ucc_aarch64_cpuid_t; /** * Get ARM CPU identifier and version */ void ucc_aarch64_cpuid(ucc_aarch64_cpuid_t *cpuid); static inline ucc_cpu_vendor_t ucc_arch_get_cpu_vendor() { ucc_aarch64_cpuid_t cpuid; ucc_aarch64_cpuid(&cpuid); if ((cpuid.implementer == 0x46) && (cpuid.architecture == 8)) { return UCC_CPU_VENDOR_FUJITSU_ARM; } if ((cpuid.implementer == 0x41) && (cpuid.architecture == 8)) { return UCC_CPU_VENDOR_NVIDIA; } return UCC_CPU_VENDOR_GENERIC_ARM; } static inline ucc_cpu_model_t ucc_arch_get_cpu_model() { ucc_aarch64_cpuid_t cpuid; ucc_aarch64_cpuid(&cpuid); if ((ucc_arch_get_cpu_vendor() == UCC_CPU_VENDOR_NVIDIA) && (cpuid.part == 0xd4f)) { return UCC_CPU_MODEL_NVIDIA_GRACE; } if (ucc_arch_get_cpu_vendor() == UCC_CPU_VENDOR_FUJITSU_ARM) { if (cpuid.part == 0x001) { return UCC_CPU_MODEL_FUJITSU_A64FX; } if (cpuid.part == 0x003) { return UCC_CPU_MODEL_FUJITSU_MONAKA; } } return UCC_CPU_MODEL_ARM_AARCH64; } #endif ucc-1.8.0/src/utils/arch/aarch64/cpu.c0000664000175000017500000000341615211535620017511 0ustar alastairalastair/** * Copyright (c) 2001-2013, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) ARM Ltd. 2016-2017. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #if defined(__aarch64__) #ifdef HAVE_CONFIG_H # include "config.h" #endif #include "utils/arch/cpu.h" #include static void ucc_aarch64_cpuid_from_proc(ucc_aarch64_cpuid_t *cpuid) { char buf[256]; int value; FILE* f; cpuid->implementer = -1; cpuid->architecture = -1; cpuid->variant = -1; cpuid->part = -1; cpuid->revision = -1; f = fopen("/proc/cpuinfo","r"); if (!f) { return; } while (fgets(buf, sizeof(buf), f)) { if (sscanf(buf, "CPU implementer : 0x%x", &value) == 1) { cpuid->implementer = value; } else if (sscanf(buf, "CPU architecture : %d", &value) == 1) { cpuid->architecture = value; } else if (sscanf(buf, "CPU variant : 0x%x", &value) == 1) { cpuid->variant = value; } else if (sscanf(buf, "CPU part : 0x%x", &value) == 1) { cpuid->part = value; } else if (sscanf(buf, "CPU revision : %d", &value) == 1) { cpuid->revision = value; } if ((cpuid->implementer != -1) && (cpuid->architecture != -1) && (cpuid->variant != -1) && (cpuid->part != -1) && (cpuid->revision != -1)) { break; } } fclose(f); } void ucc_aarch64_cpuid(ucc_aarch64_cpuid_t *cpuid) { static ucc_aarch64_cpuid_t cached_cpuid; static int initialized = 0; if (!initialized) { ucc_aarch64_cpuid_from_proc(&cached_cpuid); ucc_memory_cpu_store_fence(); initialized = 1; } ucc_memory_cpu_load_fence(); *cpuid = cached_cpuid; } #endif ucc-1.8.0/src/utils/arch/rocm_def.h0000664000175000017500000000363115211535620017254 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_ROCM_DEF_H #define UCC_ROCM_DEF_H #include "config.h" #if HAVE_ROCM #include "utils/ucc_log.h" #include #define ROCMCHECK(cmd) do { \ hipError_t e = cmd; \ if(e != hipSuccess) { \ ucc_error("ROCm failed with ret:%d(%s)", e, \ hipGetErrorString(e)); \ return UCC_ERR_NO_MESSAGE; \ } \ } while(0) #define ROCM_FUNC(_func) \ ({ \ ucc_status_t _status = UCC_OK; \ do { \ hipError_t _result = (_func); \ if (hipSuccess != _result) { \ ucc_error("%s() failed: %d(%s)", \ #_func, _result, hipGetErrorString(_result)); \ _status = UCC_ERR_INVALID_PARAM; \ } \ } while (0); \ _status; \ }) #endif /* HAVE_ROCM */ #endif /* UCC_ROCM_DEF_H */ ucc-1.8.0/src/utils/ucc_queue.h0000664000175000017500000001734215211535620016543 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_QUEUE_H_ #define UCC_QUEUE_H_ #include #include typedef struct ucc_queue_elem ucc_queue_elem_t; typedef struct ucc_queue_head ucc_queue_head_t; typedef ucc_queue_elem_t** ucc_queue_iter_t; /** * Queue element type. */ struct ucc_queue_elem { ucc_queue_elem_t *next; }; /** * Queue type. */ struct ucc_queue_head { ucc_queue_elem_t *head; ucc_queue_elem_t **ptail; }; /** * Initialize a queue. * * @param queue Queue to initialize. */ static inline void ucc_queue_head_init(ucc_queue_head_t *queue) { #ifdef __clang_analyzer__ queue->head = (ucc_queue_elem_t*)(void*)queue; #endif queue->ptail = &queue->head; } /** * @return Queue length. */ static inline size_t ucc_queue_length(ucc_queue_head_t *queue) { ucc_queue_elem_t **pelem; size_t length; length = 0; for (pelem = &queue->head; pelem != queue->ptail; pelem = &(*pelem)->next) { ++length; } return length; } /** * @return Whether the queue is empty. */ static inline int ucc_queue_is_empty(ucc_queue_head_t *queue) { return queue->ptail == &queue->head; } /** * Enqueue an element to the tail of the queue. * * @param queue Queue to add to. * @param elem Element to add. */ static inline void ucc_queue_push(ucc_queue_head_t *queue, ucc_queue_elem_t *elem) { *queue->ptail = elem; queue->ptail = &elem->next; #if UCC_ENABLE_ASSERT elem->next = NULL; /* For sanity check below */ #endif } /** * Add an element to the head of the queue. * * @param queue Queue to add to. * @param elem Element to add. */ static inline void ucc_queue_push_head(ucc_queue_head_t *queue, ucc_queue_elem_t *elem) { elem->next = queue->head; queue->head = elem; if (queue->ptail == &queue->head) { queue->ptail = &elem->next; } } /** * Dequeue an element from the head of the queue, assuming the queue is not empty. * * @param queue Non-empty queue to pull from. * @return Element from the head of the queue. */ static inline ucc_queue_elem_t *ucc_queue_pull_non_empty(ucc_queue_head_t *queue) { ucc_queue_elem_t *elem; elem = queue->head; queue->head = elem->next; if (queue->ptail == &elem->next) { queue->ptail = &queue->head; } return elem; } /** * Delete an element. * The element must be valid when deleting it. * After the call, iter points to the next element, and the element may be released. */ static inline void ucc_queue_del_iter(ucc_queue_head_t *queue, ucc_queue_iter_t iter) { assert((iter != NULL) && (*iter != NULL)); if (queue->ptail == &(*iter)->next) { queue->ptail = iter; /* deleting the last element */ *iter = NULL; /* make *ptail point to NULL */ } else { *iter = (*iter)->next; } } /** * Dequeue an element from the head of the queue. * * @param queue Queue to pull from. * @return Element from the head of the queue, or NULL if the queue is empty. */ static inline ucc_queue_elem_t *ucc_queue_pull(ucc_queue_head_t *queue) { if (ucc_queue_is_empty(queue)) return NULL; return ucc_queue_pull_non_empty(queue); } /** * Insert all elements from one queue to another queue, leaving the first queue * empty. * * @param queue Queue to push elements to. * @param new_elems Queue of elements to add. */ static inline void ucc_queue_splice(ucc_queue_head_t *queue, ucc_queue_head_t *new_elems) { if (!ucc_queue_is_empty(new_elems)) { *queue->ptail = new_elems->head; queue->ptail = new_elems->ptail; new_elems->ptail = &new_elems->head; } } /** * Convenience macro to pull from a non-empty queue and return the containing element. * * @param queue Non-empty queue to pull from. * @param type Container element type. * @param member Queue element member inside the container. * * @return Pulled element. */ #define ucc_queue_pull_elem_non_empty(queue, type, member) \ ucc_container_of(ucc_queue_pull_non_empty(queue), type, member) /** * Convenience macro to get the head element of a non-empty queue. * * @param queue Non-empty queue whose head element to get. * @param type Container element type. * @param member Queue element member inside the container. * * @return Head element. */ #define ucc_queue_head_elem_non_empty(queue, type, member) \ ucc_container_of((queue)->head, type, member) /** * Convenience macro to get the tail element of a non-empty queue. * * @param queue Non-empty queue whose head element to get. * @param type Container element type. * @param member Queue element member inside the container. * * @return Head element. */ #define ucc_queue_tail_elem_non_empty(queue, type, member) \ ucc_container_of((queue)->ptail, type, member) /** * Iterate over queue elements. The queue must not be modified during the iteration. * * @param elem Variable which will hold point to the element in the queue. * @param queue Queue to iterate on. * @param member Member inside 'elem' which is the queue link. */ #define ucc_queue_for_each(elem, queue, member) \ /* we set `ptail` field to queue address to not subtract NULL pointer */ \ for (*(queue)->ptail = (ucc_queue_elem_t*)(void*)(queue), \ elem = ucc_container_of((queue)->head, typeof(*elem), member); \ (UCC_PTR_BYTE_OFFSET(elem, ucc_offsetof(typeof(*elem), member)) != \ (void*)(queue)); \ elem = ucc_container_of(elem->member.next, typeof(*elem), member)) /** * Iterate over queue elements. The current element may be safely removed from * the queue using ucc_queue_del_iter(). * * @param elem Variable which will hold point to the element in the queue. * @param iter Iterator variable. May be passed to ucc_queue_del_iter(). * @param queue Queue to iterate on. * @param member Member inside 'elem' which is the queue link. */ #define ucc_queue_for_each_safe(elem, iter, queue, member) \ for (iter = &(queue)->head, \ elem = ucc_container_of(*iter, typeof(*elem), member); \ iter != (queue)->ptail; \ iter = (*iter == &elem->member) ? &(*iter)->next : iter, \ elem = ucc_container_of(*iter, typeof(*elem), member)) /** * Iterate and extract elements from the queue while a condition is true. * * @param elem Variable which will hold point to the element in the queue. * @param queue Queue to iterate on. * @param member Member inside 'elem' which is the queue link. * @param cond Condition to continue iterating. * * TODO optimize */ #define ucc_queue_for_each_extract(elem, queue, member, cond) \ for (elem = ucc_container_of((queue)->head, typeof(*elem), member); \ \ !ucc_queue_is_empty(queue) && (cond) && ucc_queue_pull_non_empty(queue); \ \ elem = ucc_container_of((queue)->head, typeof(*elem), member)) /* * Queue iteration */ static inline ucc_queue_iter_t ucc_queue_iter_begin(ucc_queue_head_t *q) { return &q->head; } static inline ucc_queue_iter_t ucc_queue_iter_next(ucc_queue_iter_t i) { return &(*i)->next; } static inline int ucc_queue_iter_end(ucc_queue_head_t *q, ucc_queue_iter_t i) { return i == q->ptail; } static inline void ucc_queue_remove(ucc_queue_head_t *queue, ucc_queue_elem_t *elem) { ucc_queue_iter_t iter = ucc_queue_iter_begin(queue); while (!ucc_queue_iter_end(queue, iter)) { if (*iter == elem) { ucc_queue_del_iter(queue, iter); return; } iter = ucc_queue_iter_next(iter); } } #define ucc_queue_iter_elem(elem, iter, member) \ ucc_container_of(*iter, typeof(*elem), member) #endif ucc-1.8.0/src/utils/ini.c0000664000175000017500000002016115211535620015330 0ustar alastairalastair/* inih -- simple .INI file parser SPDX-License-Identifier: BSD-3-Clause Copyright (C) 2009-2020, Ben Hoyt inih is released under the New BSD license (see LICENSE.txt). Go to the project home page for more info: https://github.com/benhoyt/inih */ #if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_WARNINGS) #define _CRT_SECURE_NO_WARNINGS #endif #ifdef HAVE_CONFIG_H # include "config.h" #endif #include #include #include #include "ini.h" #if !UCC_INI_USE_STACK #if UCC_INI_CUSTOM_ALLOCATOR #include void* ini_malloc(size_t size); void ini_free(void* ptr); void* ini_realloc(void* ptr, size_t size); #else #include #define ini_malloc malloc #define ini_free free #define ini_realloc realloc #endif #endif #define MAX_SECTION 128 #define MAX_NAME 50 /* Used by ini_parse_string() to keep track of string parsing state. */ typedef struct { const char* ptr; size_t num_left; } ini_parse_string_ctx; /* Strip whitespace chars off end of given string, in place. Return s. */ static char* rstrip(char* s) { char* p = s + strlen(s); while (p > s && isspace((unsigned char)(*--p))) *p = '\0'; return s; } /* Return pointer to first non-whitespace char in given string. */ static char* lskip(const char* s) { while (*s && isspace((unsigned char)(*s))) s++; return (char*)s; } /* Return pointer to first char (of chars) or inline comment in given string, or pointer to NUL at end of string if neither found. Inline comment must be prefixed by a whitespace character to register as a comment. */ static char* find_chars_or_comment(const char* s, const char* chars) { #if UCC_INI_ALLOW_INLINE_COMMENTS int was_space = 0; while (*s && (!chars || !strchr(chars, *s)) && !(was_space && strchr(UCC_INI_INLINE_COMMENT_PREFIXES, *s))) { was_space = isspace((unsigned char)(*s)); s++; } #else while (*s && (!chars || !strchr(chars, *s))) { s++; } #endif return (char*)s; } /* Similar to strncpy, but ensures dest (size bytes) is NUL-terminated, and doesn't pad with NULs. */ static char* strncpy0(char* dest, const char* src, size_t size) { /* Could use strncpy internally, but it causes gcc warnings (see issue #91) */ size_t i; for (i = 0; i < size - 1 && src[i]; i++) dest[i] = src[i]; dest[i] = '\0'; return dest; } /* See documentation in header file. */ int ucc_ini_parse_stream(ini_reader reader, void* stream, ini_handler handler, void* user) { /* Uses a fair bit of stack (use heap instead if you need to) */ #if UCC_INI_USE_STACK char line[UCC_INI_MAX_LINE]; int max_line = UCC_INI_MAX_LINE; #else char* line; size_t max_line = UCC_INI_INITIAL_ALLOC; #endif #if UCC_INI_ALLOW_REALLOC && !UCC_INI_USE_STACK char* new_line; size_t offset; #endif char section[MAX_SECTION] = ""; char prev_name[MAX_NAME] = ""; char* start; char* end; char* name; char* value; int lineno = 0; int error = 0; #if !UCC_INI_USE_STACK line = (char*)ini_malloc(UCC_INI_INITIAL_ALLOC); if (!line) { return -2; } #endif #if UCC_INI_HANDLER_LINENO #define HANDLER(u, s, n, v) handler(u, s, n, v, lineno) #else #define HANDLER(u, s, n, v) handler(u, s, n, v) #endif /* Scan through stream line by line */ while (reader(line, (int)max_line, stream) != NULL) { #if UCC_INI_ALLOW_REALLOC && !INI_USE_STACK offset = strlen(line); while (offset == max_line - 1 && line[offset - 1] != '\n') { max_line *= 2; if (max_line > UCC_INI_MAX_LINE) max_line = UCC_INI_MAX_LINE; new_line = ini_realloc(line, max_line); if (!new_line) { ini_free(line); return -2; } line = new_line; if (reader(line + offset, (int)(max_line - offset), stream) == NULL) break; if (max_line >= UCC_INI_MAX_LINE) break; offset += strlen(line + offset); } #endif lineno++; start = line; #if UCC_INI_ALLOW_BOM if (lineno == 1 && (unsigned char)start[0] == 0xEF && (unsigned char)start[1] == 0xBB && (unsigned char)start[2] == 0xBF) { start += 3; } #endif start = lskip(rstrip(start)); if (strchr(UCC_INI_START_COMMENT_PREFIXES, *start)) { /* Start-of-line comment */ } #if UCC_INI_ALLOW_MULTILINE else if (*prev_name && *start && start > line) { /* Non-blank line with leading whitespace, treat as continuation of previous name's value (as per Python configparser). */ if (!HANDLER(user, section, prev_name, start) && !error) error = lineno; } #endif else if (*start == '[') { /* A "[section]" line */ end = find_chars_or_comment(start + 1, "]"); if (*end == ']') { *end = '\0'; strncpy0(section, start + 1, sizeof(section)); *prev_name = '\0'; #if UCC_INI_CALL_HANDLER_ON_NEW_SECTION if (!HANDLER(user, section, NULL, NULL) && !error) error = lineno; #endif } else if (!error) { /* No ']' found on section line */ error = lineno; } } else if (*start) { /* Not a comment, must be a name[=:]value pair */ end = find_chars_or_comment(start, "=:"); if (*end == '=' || *end == ':') { *end = '\0'; name = rstrip(start); value = end + 1; #if UCC_INI_ALLOW_INLINE_COMMENTS end = find_chars_or_comment(value, NULL); if (*end) *end = '\0'; #endif value = lskip(value); rstrip(value); /* Valid name[=:]value pair found, call handler */ strncpy0(prev_name, name, sizeof(prev_name)); if (!HANDLER(user, section, name, value) && !error) error = lineno; } else if (!error) { /* No '=' or ':' found on name[=:]value line */ #if UCC_INI_ALLOW_NO_VALUE *end = '\0'; name = rstrip(start); if (!HANDLER(user, section, name, NULL) && !error) error = lineno; #else error = lineno; #endif } } #if UCC_INI_STOP_ON_FIRST_ERROR if (error) break; #endif } #if !UCC_INI_USE_STACK ini_free(line); #endif return error; } /* See documentation in header file. */ int ucc_ini_parse_file(FILE* file, ini_handler handler, void* user) { return ucc_ini_parse_stream((ini_reader)fgets, file, handler, user); } /* See documentation in header file. */ int ucc_ini_parse(const char* filename, ini_handler handler, void* user) { FILE* file; int error; file = fopen(filename, "r"); if (!file) return -1; error = ucc_ini_parse_file(file, handler, user); fclose(file); return error; } /* An ini_reader function to read the next line from a string buffer. This is the fgets() equivalent used by ini_parse_string(). */ static char* ini_reader_string(char* str, int num, void* stream) { ini_parse_string_ctx* ctx = (ini_parse_string_ctx*)stream; const char* ctx_ptr = ctx->ptr; size_t ctx_num_left = ctx->num_left; char* strp = str; char c; if (ctx_num_left == 0 || num < 2) return NULL; while (num > 1 && ctx_num_left != 0) { c = *ctx_ptr++; ctx_num_left--; *strp++ = c; if (c == '\n') break; num--; } *strp = '\0'; ctx->ptr = ctx_ptr; ctx->num_left = ctx_num_left; return str; } /* See documentation in header file. */ int ucc_ini_parse_string(const char* string, ini_handler handler, void* user) { ini_parse_string_ctx ctx; ctx.ptr = string; ctx.num_left = strlen(string); return ucc_ini_parse_stream((ini_reader)ini_reader_string, &ctx, handler, user); } ucc-1.8.0/src/utils/ucc_spinlock.h0000664000175000017500000000171415211535620017235 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_SPINLOCK_H_ #define UCC_SPINLOCK_H_ #include "config.h" #include "ucs/type/spinlock.h" #define ucc_spinlock_t ucs_spinlock_t #define ucc_spinlock_init ucs_spinlock_init #define ucc_spinlock_destroy ucs_spinlock_destroy #define ucc_spin_lock ucs_spin_lock #define ucc_spin_try_lock ucs_spin_try_lock #define ucc_spin_unlock ucs_spin_unlock #define ucc_recursive_spinlock_t ucs_recursive_spinlock_t #define ucc_recursive_spinlock_init ucs_recursive_spinlock_init #define ucc_recursive_spinlock_destroy ucs_recursive_spinlock_destroy #define ucc_recursive_spin_is_owner ucs_recursive_spin_is_owner #define ucc_recursive_spin_lock ucs_recursive_spin_lock #define ucc_recursive_spin_trylock ucs_recursive_spin_trylock #define ucc_recursive_spin_unlock ucs_recursive_spin_unlock #endif ucc-1.8.0/src/coll_patterns/0000775000175000017500000000000015211535620016116 5ustar alastairalastairucc-1.8.0/src/coll_patterns/recursive_knomial.h0000664000175000017500000003005715211535620022015 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef RECURSIVE_KNOMIAL_H_ #define RECURSIVE_KNOMIAL_H_ #define UCC_KN_PEER_NULL ((ucc_rank_t)-1) typedef uint16_t ucc_kn_radix_t; enum { KN_NODE_BASE, /* Participates in the main loop of the recursive KN algorithm */ KN_NODE_PROXY, /* Participates in the main loop and receives/sends the data from/to EXTRA */ KN_NODE_EXTRA /* Just sends/receives the data to/from its PROXY */ }; enum { KN_PATTERN_REDUCE_SCATTER = 1, KN_PATTERN_REDUCE_SCATTERX, KN_PATTERN_REDUCE_SCATTERV, KN_PATTERN_ALLGATHER, KN_PATTERN_ALLGATHERV, KN_PATTERN_ALLGATHERX, KN_PATTERN_GATHER, KN_PATTERN_GATHERX, }; typedef struct ucc_knomial_pattern { ucc_kn_radix_t radix; /* knomial tree radix */ uint8_t type; /* pattern type */ uint8_t iteration; /* current iteration */ uint8_t n_iters; /* number of iterations in knomial algorithm */ uint8_t pow_radix_sup; /* smallest integer N such that (radix ** N) >= size */ uint8_t node_type; /* type of current rank: BASE, PROXY or EXTRA */ uint8_t backward; /* boolean, iteration direction */ ucc_rank_t radix_pow; /* power of radix for current algorithm iteration * forward: initial value is 1 * backward: initial valus is full_pow_size if have >1 full subtrees, OR * (full_pow_size / radix) otherwise */ ucc_rank_t full_pow_size; /* largest power of radix <= size. It is equal to * (radix ** pow_radix_sup) if (radix ** pow_radix_sup) == size, OR * (radix ** (_pow_radix_sup - 1)) otherwise */ ucc_rank_t size; /* total number of ranks */ ucc_rank_t rank; /* process rank */ ucc_rank_t n_extra; /* number of "extra" ranks to be served by "proxies" */ size_t block_size_counts; size_t count; /* collective buffer size */ ucc_count_t *counts; ucc_rank_t block_size; ptrdiff_t block_offset; int is64; } ucc_knomial_pattern_t; /** * Calculate number of full subtrees * @param [in] p ucc_knomial_pattern * @return number of full subtrees */ static inline ucc_rank_t ucc_kn_pattern_n_full(ucc_knomial_pattern_t *p) { return p->size / p->full_pow_size; } static inline ucc_rank_t ucc_kn_pattern_radix_pow_init(ucc_knomial_pattern_t *p, int backward) { ucc_rank_t n_full = ucc_kn_pattern_n_full(p); return backward ? ((n_full == 1) ? p->full_pow_size / p->radix : p->full_pow_size) : 1; } /** * Initializes recursive knomial tree attributes. * @param [in] radix Knomial radix * @param [in] rank Rank in a team * @param [in] size Team size * @param [out] p ucc_knomial_pattern */ static inline void ucc_knomial_pattern_init_impl(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, ucc_knomial_pattern_t *p, int backward, int has_extra) { ucc_rank_t fs = radix; ucc_rank_t n_full_subtrees; p->pow_radix_sup = 1; while (fs < size) { p->pow_radix_sup++; fs *= radix; } p->full_pow_size = (fs != size) ? fs / radix : fs; p->radix = radix; p->size = size; p->rank = rank; p->backward = backward; p->iteration = 0; n_full_subtrees = ucc_kn_pattern_n_full(p); p->n_extra = has_extra ? size - n_full_subtrees * p->full_pow_size : 0; p->n_iters = (p->n_extra && n_full_subtrees == 1) ? p->pow_radix_sup - 1 : p->pow_radix_sup; p->radix_pow = ucc_kn_pattern_radix_pow_init(p, backward); p->node_type = KN_NODE_BASE; if (rank < p->n_extra * 2) { p->node_type = (rank % 2) ? KN_NODE_EXTRA : KN_NODE_PROXY; } } static inline void ucc_knomial_pattern_init_backward(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, ucc_knomial_pattern_t *p) { ucc_knomial_pattern_init_impl(size, rank, radix, p, 1, 1); } static inline void ucc_knomial_pattern_init(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, ucc_knomial_pattern_t *p) { ucc_knomial_pattern_init_impl(size, rank, radix, p, 0, 1); } static inline void ucc_knomial_pattern_init_no_extra(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, ucc_knomial_pattern_t *p) { ucc_knomial_pattern_init_impl(size, rank, radix, p, 0, 0); } static inline ucc_rank_t ucc_knomial_pattern_get_proxy(ucc_knomial_pattern_t *p, ucc_rank_t rank) { return rank - 1; } static inline ucc_rank_t ucc_knomial_pattern_get_extra(ucc_knomial_pattern_t *p, ucc_rank_t rank) { return rank + 1; } static inline int ucc_knomial_pattern_loop_done(ucc_knomial_pattern_t *p) { return p->iteration == p->n_iters; } static inline int ucc_knomial_pattern_loop_first_iteration(ucc_knomial_pattern_t *p) { return p->iteration == 0; } static inline int ucc_knomial_pattern_loop_last_iteration(ucc_knomial_pattern_t *p) { return p->iteration == p->n_iters - 1; } /* returns new rank id by excluding all extra ranks */ static inline ucc_rank_t ucc_knomial_pattern_loop_rank(ucc_knomial_pattern_t *p, ucc_rank_t rank) { return (rank < p->n_extra * 2) ? rank / 2 : rank - p->n_extra; } /* returns original rank id */ static inline ucc_rank_t ucc_knomial_pattern_loop_rank_inv(ucc_knomial_pattern_t *p, ucc_rank_t rank) { return (rank < p->n_extra) ? rank * 2 : rank + p->n_extra; } static inline ucc_rank_t ucc_knomial_pattern_get_loop_peer(ucc_knomial_pattern_t *p, ucc_rank_t rank, ucc_kn_radix_t loop_step) { ucc_assert(p->node_type == KN_NODE_BASE || p->node_type == KN_NODE_PROXY); ucc_assert(loop_step >= 1 && loop_step < p->radix); ucc_assert((rank >= p->n_extra * 2) || ((rank % 2) == 0)); ucc_rank_t loop_rank = ucc_knomial_pattern_loop_rank(p, rank); ucc_rank_t step_size = p->radix_pow * p->radix; ucc_rank_t peer = (loop_rank + loop_step * p->radix_pow) % step_size + ucc_align_down(loop_rank, step_size); return (peer >= (p->size - p->n_extra)) ? UCC_KN_PEER_NULL: ucc_knomial_pattern_loop_rank_inv(p, peer); } static inline ucc_rank_t ucc_knomial_pattern_get_base_rank(ucc_knomial_pattern_t *p, ucc_rank_t rank) { ucc_rank_t step_size = p->radix_pow * p->radix; ucc_rank_t lrank; ucc_kn_radix_t s; lrank = ucc_knomial_pattern_loop_rank(p, rank); s = ucc_div_round_up(step_size - (lrank % step_size), p->radix_pow); if (s == p->radix) { return rank; } else { return ucc_knomial_pattern_get_loop_peer(p, rank, s); } } /* return the index of rank in the loop assuming smallest rank has index 0 */ static inline ucc_kn_radix_t ucc_knomial_pattern_get_loop_index(ucc_knomial_pattern_t *p, ucc_rank_t rank) { ucc_rank_t base_rank = ucc_knomial_pattern_get_base_rank(p, rank); ucc_rank_t rank0 = ucc_knomial_pattern_loop_rank(p, base_rank); ucc_rank_t cur_rank = ucc_knomial_pattern_loop_rank(p, rank); return (cur_rank - rank0) / p->radix_pow; } static inline void ucc_knomial_pattern_next_iteration(ucc_knomial_pattern_t *p) { p->iteration++; p->radix_pow *= p->radix; } static inline void ucc_knomial_pattern_prev_iteration(ucc_knomial_pattern_t *p) { p->iteration--; p->radix_pow /= p->radix; } static inline void ucc_knomial_pattern_next_iteration_backward(ucc_knomial_pattern_t *p) { p->iteration++; p->radix_pow /= p->radix; } static inline ucc_kn_radix_t ucc_knomial_pattern_get_min_radix(ucc_kn_radix_t cfg_radix, ucc_rank_t team_size, size_t count) { ucc_kn_radix_t radix = ucc_min(cfg_radix, team_size); if (((count + radix - 1) / radix * (radix - 1) > count) || ((radix - 1) > count)) { radix = 2; } return radix; } /* Calculates for each rank at which distance it should receive */ static inline ucc_rank_t ucc_knomial_calc_recv_dist(ucc_rank_t team_size, ucc_rank_t rank, ucc_rank_t radix, ucc_rank_t root) { ucc_rank_t root_base = 0; ucc_rank_t dist = 1; if (rank == root) { return 0; } while (dist <= team_size) { if (rank < root_base + radix * dist) { break; } dist *= radix; } return dist; } /* Calculates (sub) opt radix for Allreduce SRA and Bcast SAG, by minimizing n_extra ranks */ static inline ucc_rank_t ucc_kn_get_opt_radix(ucc_rank_t team_size, ucc_kn_radix_t min_radix, ucc_kn_radix_t max_radix) { ucc_rank_t n_extra = 0, min_val = team_size; ucc_kn_radix_t min_i = min_radix; ucc_kn_radix_t max_r = ucc_max(max_radix, min_radix); ucc_kn_radix_t r; ucc_rank_t fs; for (r = min_radix; r <= max_r; r++) { fs = r; while (fs < team_size) { fs = fs * r; } fs = (fs == team_size) ? fs : fs / r; n_extra = team_size - (team_size / fs) * fs; if (n_extra == 0) { return r; } if (n_extra < min_val) { min_val = n_extra; min_i = r; } } return min_i; } /* A set of convenience macros used to implement sw based progress of the algorithms that use kn pattern */ enum { UCC_KN_PHASE_INIT, UCC_KN_PHASE_LOOP, /* main loop of recursive k-ing */ UCC_KN_PHASE_REDUCE, /* reduce data received from peer */ UCC_KN_PHASE_EXTRA, /* recv from extra rank */ UCC_KN_PHASE_EXTRA_REDUCE, /* reduce data received from extra rank */ UCC_KN_PHASE_PROXY, /* recv from proxy rank */ UCC_KN_PHASE_COMPLETE, /* any work after main loop, e.g. memcpy */ }; #define UCC_KN_CHECK_PHASE(_p) \ case _p: \ goto _p; #define UCC_KN_REDUCE_GOTO_PHASE(_phase) \ do { \ switch (_phase) { \ UCC_KN_CHECK_PHASE(UCC_KN_PHASE_EXTRA); \ UCC_KN_CHECK_PHASE(UCC_KN_PHASE_EXTRA_REDUCE); \ UCC_KN_CHECK_PHASE(UCC_KN_PHASE_LOOP); \ UCC_KN_CHECK_PHASE(UCC_KN_PHASE_REDUCE); \ UCC_KN_CHECK_PHASE(UCC_KN_PHASE_PROXY); \ UCC_KN_CHECK_PHASE(UCC_KN_PHASE_COMPLETE); \ case UCC_KN_PHASE_INIT: \ break; \ }; \ } while (0) #define UCC_KN_GOTO_PHASE(_phase) \ do { \ switch (_phase) { \ UCC_KN_CHECK_PHASE(UCC_KN_PHASE_EXTRA); \ UCC_KN_CHECK_PHASE(UCC_KN_PHASE_LOOP); \ UCC_KN_CHECK_PHASE(UCC_KN_PHASE_PROXY); \ case UCC_KN_PHASE_INIT: \ break; \ }; \ } while (0) #endif ucc-1.8.0/src/coll_patterns/ring.h0000664000175000017500000001662215211535620017235 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_RING_PATTERN_H_ #define UCC_RING_PATTERN_H_ #include "utils/ucc_coll_utils.h" #include "utils/debug/log_def.h" #include "utils/ucc_malloc.h" typedef struct ucc_ring_pattern { ucc_rank_t size; unsigned num_rings; /* Optional array of maps from ring index to team rank. */ const ucc_ep_map_t *maps; /* Optional cached ring rank per ring (local index). */ ucc_rank_t *ring_ranks; } ucc_ring_pattern_t; /** * Initializes a ucc_ring_pattern_t structure with the given size and number of rings. * * @param size Number of ranks in the ring. * @param num_rings Number of rings in the pattern. * @param p Pointer to the ucc_ring_pattern_t structure to initialize. * * This function sets the size and num_rings fields, and * initializes maps and ring_ranks pointers to NULL. Rings * are not initialized. */ static inline void ucc_ring_pattern_init(ucc_rank_t size, unsigned num_rings, ucc_ring_pattern_t *p) { p->size = size; p->num_rings = num_rings; p->maps = NULL; p->ring_ranks = NULL; } /** * Initializes a ucc_ring_pattern_t structure with the provided * array of endpoint maps. * * @param maps Pointer to array of ucc_ep_map_t, one per ring. * @param num_rings Number of rings (number of maps). * @param p Pointer to the ucc_ring_pattern_t structure to initialize. * * The function sets the size field based on the ep_num of the first map if * num_rings is greater than zero, otherwise sets size to 0. It sets the * maps and num_rings fields, and initializes ring_ranks to NULL. */ static inline void ucc_ring_pattern_init_map(const ucc_ep_map_t *maps, unsigned num_rings, ucc_ring_pattern_t *p) { p->size = num_rings ? (ucc_rank_t)maps[0].ep_num : 0; p->num_rings = num_rings; p->maps = maps; p->ring_ranks = NULL; } /** * Initializes a ucc_ring_pattern_t structure with the given topo. * * @param topo Pointer to the ucc_topo_t structure to initialize. * @param memory_type Memory type to use for the rings. * @param num_rings Number of rings in the pattern. * @param p Pointer to the ucc_ring_pattern_t structure to initialize. * * This function initializes the rings using topology information. */ ucc_status_t ucc_ring_pattern_init_topo( ucc_topo_t *topo, ucc_memory_type_t memory_type, unsigned num_rings, ucc_ring_pattern_t *p); /** * Sets the local ring rank(s) for all rings in the given ucc_ring_pattern_t. * * @param p Pointer to the ucc_ring_pattern_t structure. * @param rank Global (logical) rank to be mapped to local ring rank(s). * * This function allocates and fills the ring_ranks array in the pattern * structure. For each ring, ring_ranks[ring_id] is set to the local rank * corresponding to the provided global rank according to the ring's ep_map, * or just to rank itself if no maps are present. */ static inline void ucc_ring_pattern_set_rank(ucc_ring_pattern_t *p, ucc_rank_t rank) { ucc_rank_t ring_id; ucc_assert(p != NULL); ucc_assert(p->num_rings > 0); if (p->ring_ranks == NULL) { p->ring_ranks = (ucc_rank_t *)ucc_malloc(p->num_rings * sizeof(ucc_rank_t), "ring_ranks"); ucc_assert(p->ring_ranks != NULL); } for (ring_id = 0; ring_id < p->num_rings; ring_id++) { p->ring_ranks[ring_id] = p->maps ? ucc_ep_map_local_rank(p->maps[ring_id], rank) : rank; ucc_assert(p->ring_ranks[ring_id] != UCC_RANK_INVALID); } } static inline ucc_rank_t ucc_ring_pattern_rank(ucc_ring_pattern_t *p, ucc_rank_t ring_id) { ucc_assert(p != NULL); ucc_assert(ring_id < p->num_rings); ucc_assert(p->ring_ranks != NULL); return p->ring_ranks[ring_id]; } static inline ucc_rank_t ucc_ring_pattern_size(ucc_ring_pattern_t *p, ucc_rank_t ring_id) { ucc_assert(ring_id < p->num_rings); return p->size; } static inline ucc_rank_t ucc_ring_pattern_eval(ucc_ring_pattern_t *p, ucc_rank_t ring_id, ucc_rank_t rank) { ucc_assert(ring_id < p->num_rings); ucc_assert(!p->maps || p->maps[ring_id].ep_num == p->size); return p->maps ? ucc_ep_map_eval(p->maps[ring_id], rank) : rank; } static inline ucc_rank_t ucc_ring_pattern_get_send_peer(ucc_ring_pattern_t *p, ucc_rank_t ring_id, ucc_rank_t rank) { ucc_rank_t size = ucc_ring_pattern_size(p, ring_id); return ucc_ring_pattern_eval(p, ring_id, (rank + 1) % size); } static inline ucc_rank_t ucc_ring_pattern_get_recv_peer(ucc_ring_pattern_t *p, ucc_rank_t ring_id, ucc_rank_t rank) { ucc_rank_t size = ucc_ring_pattern_size(p, ring_id); return ucc_ring_pattern_eval(p, ring_id, (rank - 1 + size) % size); } static inline ucc_rank_t ucc_ring_pattern_get_send_block(ucc_ring_pattern_t *p, ucc_rank_t ring_id, ucc_rank_t rank, ucc_rank_t step) { ucc_rank_t size = ucc_ring_pattern_size(p, ring_id); return ucc_ring_pattern_eval(p, ring_id, (rank - step + size) % size); } static inline ucc_rank_t ucc_ring_pattern_get_recv_block(ucc_ring_pattern_t *p, ucc_rank_t ring_id, ucc_rank_t rank, ucc_rank_t step) { ucc_rank_t size = ucc_ring_pattern_size(p, ring_id); return ucc_ring_pattern_eval(p, ring_id, (rank - step - 1 + size) % size); } static inline void ucc_ring_pattern_print(ucc_ring_pattern_t *p) { ucc_rank_t ring_id, i, rank; char line[256]; size_t left; int n; ucc_assert(p != NULL); for (ring_id = 0; ring_id < p->num_rings; ring_id++) { n = snprintf(line, sizeof(line), "ring %u: ", ring_id); if (n < 0) { continue; } if ((size_t)n >= sizeof(line)) { line[sizeof(line) - 1] = '\0'; ucc_debug("%s", line); continue; } left = sizeof(line) - (size_t)n; for (i = 0; i < p->size; i++) { rank = ucc_ring_pattern_eval(p, ring_id, i); n = snprintf(line + (sizeof(line) - left), left, "%u%s", rank, (i + 1 == p->size) ? "" : " -> "); if (n < 0) { break; } if ((size_t)n >= left) { line[sizeof(line) - left] = '\0'; ucc_debug("%s", line); n = snprintf(line, sizeof(line), " %u%s", rank, (i + 1 == p->size) ? "" : " -> "); if (n < 0) { break; } if ((size_t)n >= sizeof(line)) { line[sizeof(line) - 1] = '\0'; ucc_debug("%s", line); left = sizeof(line); continue; } left = sizeof(line) - (size_t)n; continue; } left -= (size_t)n; } if (left != sizeof(line)) { ucc_debug("%s", line); } } } static inline void ucc_ring_pattern_destroy(ucc_ring_pattern_t *p) { if (!p) { return; } if (p->maps) { ucc_free((void *)p->maps); p->maps = NULL; } if (p->ring_ranks) { ucc_free(p->ring_ranks); p->ring_ranks = NULL; } } #endif ucc-1.8.0/src/coll_patterns/sra_knomial.h0000664000175000017500000004027015211535620020571 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef SRA_KNOMIAL_H_ #define SRA_KNOMIAL_H_ #include "recursive_knomial.h" /** * Computes actual radix at current iteration * @param [in] p ucc_knomial_pattern * @return radix */ static inline ucc_rank_t ucc_kn_compute_step_radix(ucc_knomial_pattern_t *p) { int n_full = ucc_kn_pattern_n_full(p); return p->radix_pow * p->radix >= p->size ? (n_full > 1 ? n_full : p->radix) : p->radix; } /* segment index in exchange group of tree */ static inline ucc_rank_t ucc_kn_compute_seg_index(ucc_rank_t peer, ucc_rank_t kpow_num, ucc_knomial_pattern_t *p) { ucc_rank_t peer_position, peer_base_rank, peer_index; peer = ucc_knomial_pattern_loop_rank(p, peer); peer_base_rank = ucc_align_down(peer, kpow_num * p->radix); peer_position = peer_base_rank == 0 ? peer : peer % (peer_base_rank); peer_index = peer_position / kpow_num; return peer_index; } /** * Computes segment size * @param [in] block_count size of the block * @param [in] radix tree radix * @param [si] si segment index * @return segment size */ static inline size_t ucc_sra_kn_compute_seg_size(size_t block_count, ucc_kn_radix_t radix, ucc_rank_t si) { return ucc_buffer_block_count(block_count, radix, si); } /** * Computes segment offset * @param [in] block_count size of the block * @param [in] radix tree radix * @param [si] si segment index * @return segment offset */ static inline size_t ucc_sra_kn_compute_seg_offset(size_t block_count, ucc_kn_radix_t radix, ucc_rank_t si) { return ucc_buffer_block_offset(block_count, radix, si); } static inline size_t ucc_sra_kn_compute_block_count(size_t count, ucc_rank_t rank, ucc_knomial_pattern_t *p) { size_t block_count = count; ucc_rank_t k_pow = 1; ucc_rank_t i, my_si, my_seg_len, steps; steps = p->backward ? p->n_iters - p->iteration - 1 : p->iteration; for (i = 0; i < steps; i++) { my_si = ucc_kn_compute_seg_index(rank, k_pow, p); my_seg_len = ucc_sra_kn_compute_seg_size(block_count, p->radix, my_si); block_count = my_seg_len; k_pow *= p->radix; } return block_count; } static inline void ucc_sra_kn_get_offset_and_seglen(size_t count, size_t dt_size, ucc_rank_t rank, ucc_rank_t size, ucc_kn_radix_t radix, ptrdiff_t *offset, size_t *seglen) { ptrdiff_t _offset = 0; size_t my_seg_len = 0; ucc_rank_t my_si, step_radix; size_t my_seg_offset; ucc_knomial_pattern_t p; ucc_knomial_pattern_init(size, rank, radix, &p); if (KN_NODE_EXTRA == p.node_type) { goto out; } while (!ucc_knomial_pattern_loop_done(&p)) { step_radix = ucc_kn_compute_step_radix(&p); my_si = ucc_kn_compute_seg_index(rank, p.radix_pow, &p); my_seg_offset = ucc_sra_kn_compute_seg_offset(count, step_radix, my_si); count = ucc_sra_kn_compute_seg_size(count, step_radix, my_si); _offset += my_seg_offset * dt_size; ucc_knomial_pattern_next_iteration(&p); } my_seg_len = count; out: if (offset) *offset = _offset; if (seglen) *seglen = my_seg_len; } static inline ptrdiff_t ucc_sra_kn_get_offset(size_t count, size_t dt_size, ucc_rank_t rank, ucc_rank_t size, ucc_kn_radix_t radix) { ptrdiff_t offset; ucc_sra_kn_get_offset_and_seglen(count, dt_size, rank, size, radix, &offset, NULL); return offset; } typedef struct ucc_kn_seg_desc { ucc_rank_t seg_size; ucc_rank_t seg_offset; ucc_rank_t seg_start_rank_loop; ucc_rank_t seg_end_rank_loop; ucc_rank_t seg_start; ucc_rank_t seg_end; } ucc_kn_seg_desc_t; static inline void ucc_kn_seg_desc_compute(ucc_knomial_pattern_t *p, ucc_kn_seg_desc_t *seg, ucc_rank_t peer) { ucc_rank_t step_radix = ucc_kn_compute_step_radix(p); ucc_rank_t seg_index = ucc_kn_compute_seg_index(peer, p->radix_pow, p); /* size of the peer's segment in "loop ranks" - ie how many ranks are aggregated under this segment */ seg->seg_size = p->block_size / step_radix; /* offset of the peer's segment in "loop ranks" */ seg->seg_offset = seg->seg_size * seg_index; /* start and end position of the segment in "loop ranks" */ seg->seg_start_rank_loop = seg->seg_offset + p->block_offset; seg->seg_end_rank_loop = seg->seg_start_rank_loop + seg->seg_size; /* start and end position of the segment in the original "ranks" */ seg->seg_start = ucc_knomial_pattern_loop_rank_inv(p, seg->seg_start_rank_loop); seg->seg_end = ucc_knomial_pattern_loop_rank_inv(p, seg->seg_end_rank_loop); } static inline void ucc_knx_block(ucc_rank_t rank, ucc_rank_t size, ucc_kn_radix_t radix, size_t count, int iter, size_t *b_count, ptrdiff_t *b_offset) { ucc_rank_t offset = 0; ucc_rank_t block_count; ucc_kn_radix_t step_radix; ucc_rank_t my_si; ucc_knomial_pattern_t p; ucc_knomial_pattern_init(size, rank, radix, &p); if (KN_NODE_EXTRA == p.node_type) { *b_count = *b_offset = 0; return; } block_count = count; while (p.iteration < iter) { step_radix = ucc_kn_compute_step_radix(&p); my_si = ucc_kn_compute_seg_index(rank, p.radix_pow, &p); offset += ucc_buffer_block_offset(block_count, step_radix, my_si); block_count = ucc_buffer_block_count(block_count, step_radix, my_si); ucc_knomial_pattern_next_iteration(&p); } *b_count = block_count; *b_offset = offset; } static inline void ucc_kn_g_pattern_init(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, size_t count, ucc_knomial_pattern_t *p) { ucc_knomial_pattern_init_no_extra(size, rank, radix, p); p->type = KN_PATTERN_GATHER; p->count = count; p->block_size = p->radix_pow * radix; p->block_offset = ucc_knomial_pattern_loop_rank(p, rank) / p->block_size * p->block_size; } static inline void ucc_kn_gx_pattern_init(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, size_t count, ucc_knomial_pattern_t *p) { ucc_knomial_pattern_init_backward(size, rank, radix, p); p->type = KN_PATTERN_GATHERX; p->count = count; if (p->node_type != KN_NODE_EXTRA) { p->block_size = ucc_kn_compute_step_radix(p); ucc_knx_block(rank, size, radix, count, p->n_iters - 1, &p->block_size_counts, &p->block_offset); } } static inline void ucc_kn_g_pattern_peer_seg(ucc_rank_t peer, ucc_knomial_pattern_t *p, size_t *seg_count, ptrdiff_t *seg_offset) { ucc_rank_t step_radix, seg_index; *seg_count = 0; *seg_offset = 0; switch (p->type) { case KN_PATTERN_GATHER: *seg_count = ucc_min(p->radix_pow, p->size - peer) * (p->count / p->size); *seg_offset = peer * (p->count / p->size); return; case KN_PATTERN_GATHERX: step_radix = ucc_kn_compute_step_radix(p); seg_index = ucc_kn_compute_seg_index(peer, p->radix_pow, p); *seg_offset = ucc_buffer_block_offset(p->block_size_counts, step_radix, seg_index) + p->block_offset; *seg_count = ucc_buffer_block_count(p->block_size_counts, step_radix, seg_index); return; default: ucc_assert(0); } } static inline void ucc_kn_g_pattern_next_iter(ucc_knomial_pattern_t *p) { ucc_rank_t rank; if (p->type == KN_PATTERN_GATHERX) { ucc_knomial_pattern_next_iteration_backward(p); if (!ucc_knomial_pattern_loop_done(p)) { ucc_knx_block(p->rank, p->size, p->radix, p->count, p->n_iters - 1 - p->iteration, &p->block_size_counts, &p->block_offset); } } else { rank = ucc_knomial_pattern_loop_rank(p, p->rank); ucc_knomial_pattern_next_iteration(p); if (!ucc_knomial_pattern_loop_done(p)) { p->block_size *= ucc_kn_compute_step_radix(p); p->block_offset = rank / p->block_size * p->block_size; } } } static inline void ucc_kn_ag_pattern_init(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, size_t count, ucc_knomial_pattern_t *p) { ucc_knomial_pattern_init(size, rank, radix, p); p->type = KN_PATTERN_ALLGATHER; p->count = count; p->block_size = p->radix_pow * radix; p->block_offset = ucc_knomial_pattern_loop_rank(p, rank) / p->block_size * p->block_size; } static inline void ucc_kn_agx_pattern_init(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, size_t count, ucc_knomial_pattern_t *p) { ucc_knomial_pattern_init_backward(size, rank, radix, p); p->type = KN_PATTERN_ALLGATHERX; p->count = count; if (p->node_type != KN_NODE_EXTRA) { p->block_size = ucc_kn_compute_step_radix(p); ucc_knx_block(rank, size, radix, count, p->n_iters - 1, &p->block_size_counts, &p->block_offset); } } static inline void ucc_kn_agv_pattern_init(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, ucc_count_t *counts, int is64, ucc_knomial_pattern_t *p) { ucc_knomial_pattern_init(size, rank, radix, p); p->type = KN_PATTERN_ALLGATHERV; p->counts = counts; p->is64 = is64; p->block_size = p->radix_pow * radix; p->block_offset = ucc_knomial_pattern_loop_rank(p, rank) / p->block_size * p->block_size; } static inline void ucc_kn_ag_pattern_peer_seg(ucc_rank_t peer, ucc_knomial_pattern_t *p, size_t *seg_count, ptrdiff_t *seg_offset) { ucc_rank_t step_radix, seg_index; ucc_kn_seg_desc_t s; *seg_count = 0; *seg_offset = 0; switch (p->type) { case KN_PATTERN_ALLGATHERX: step_radix = ucc_kn_compute_step_radix(p); seg_index = ucc_kn_compute_seg_index(peer, p->radix_pow, p); *seg_offset = ucc_buffer_block_offset(p->block_size_counts, step_radix, seg_index) + p->block_offset; *seg_count = ucc_buffer_block_count(p->block_size_counts, step_radix, seg_index); return; case KN_PATTERN_ALLGATHER: ucc_kn_seg_desc_compute(p, &s, peer); *seg_offset = ucc_buffer_block_offset(p->count, p->size, s.seg_start); *seg_count = ucc_buffer_block_offset(p->count, p->size, s.seg_end) - *seg_offset; return; case KN_PATTERN_ALLGATHERV: ucc_kn_seg_desc_compute(p, &s, peer); *seg_offset = ucc_buffer_vector_block_offset(p->counts, p->is64, s.seg_start); *seg_count = ucc_buffer_vector_block_offset(p->counts, p->is64, s.seg_end) - *seg_offset; return; default: ucc_assert(0); } } static inline void ucc_kn_ag_pattern_next_iter(ucc_knomial_pattern_t *p) { ucc_rank_t rank; if (p->type == KN_PATTERN_ALLGATHERX) { ucc_knomial_pattern_next_iteration_backward(p); if (!ucc_knomial_pattern_loop_done(p)) { ucc_knx_block(p->rank, p->size, p->radix, p->count, p->n_iters - 1 - p->iteration, &p->block_size_counts, &p->block_offset); } } else { rank = ucc_knomial_pattern_loop_rank(p, p->rank); ucc_knomial_pattern_next_iteration(p); if (!ucc_knomial_pattern_loop_done(p)) { p->block_size *= ucc_kn_compute_step_radix(p); p->block_offset = rank / p->block_size * p->block_size; } } } static inline void ucc_kn_rs_pattern_init(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, size_t count, ucc_knomial_pattern_t *p) { ucc_knomial_pattern_init_backward(size, rank, radix, p); p->type = KN_PATTERN_REDUCE_SCATTER; p->count = count; p->block_size_counts = count; p->block_size = size - p->n_extra; p->block_offset = 0; } static inline void ucc_kn_rsx_pattern_init(ucc_rank_t size, ucc_rank_t rank, ucc_kn_radix_t radix, size_t count, ucc_knomial_pattern_t *p) { ucc_knomial_pattern_init(size, rank, radix, p); p->type = KN_PATTERN_REDUCE_SCATTERX; p->count = count; p->block_size_counts = count; p->block_size = size - p->n_extra; } static inline void ucc_kn_rs_pattern_peer_seg(ucc_rank_t peer, ucc_knomial_pattern_t *p, size_t *peer_seg_count, ptrdiff_t *peer_seg_offset) { ucc_rank_t step_radix, seg_index; ucc_kn_seg_desc_t s; ucc_rank_t block_offset_inv; /* offset of the segment in counts of datatypes from the start of the buffer */ size_t peer_seg_offset_base; /* offset of the current block in counts of datatypes from the start of the buffer */ size_t block_offset_counts; *peer_seg_count = 0; *peer_seg_offset = 0; switch (p->type) { case KN_PATTERN_REDUCE_SCATTERX: step_radix = ucc_kn_compute_step_radix(p); seg_index = ucc_kn_compute_seg_index(peer, p->radix_pow, p); *peer_seg_offset = ucc_buffer_block_offset(p->block_size_counts, step_radix, seg_index); *peer_seg_count = ucc_buffer_block_count(p->block_size_counts, step_radix, seg_index); return; case KN_PATTERN_REDUCE_SCATTER: ucc_kn_seg_desc_compute(p, &s, peer); block_offset_inv = ucc_knomial_pattern_loop_rank_inv(p, p->block_offset); peer_seg_offset_base = ucc_buffer_block_offset(p->count, p->size, s.seg_start); *peer_seg_count = ucc_buffer_block_offset(p->count, p->size, s.seg_end) - peer_seg_offset_base; block_offset_counts = ucc_buffer_block_offset(p->count, p->size, block_offset_inv); *peer_seg_offset = peer_seg_offset_base - block_offset_counts; return; case KN_PATTERN_REDUCE_SCATTERV: /* not implemented */ ucc_assert(0); default: ucc_assert(0); } } static inline void ucc_kn_rs_pattern_next_iter(ucc_knomial_pattern_t *p) { size_t bs; ptrdiff_t offset; ucc_kn_seg_desc_t s; ucc_kn_rs_pattern_peer_seg(p->rank, p, &bs, &offset); p->block_size_counts = bs; switch (p->type) { case KN_PATTERN_REDUCE_SCATTERX: p->block_offset += offset; ucc_knomial_pattern_next_iteration(p); return; case KN_PATTERN_REDUCE_SCATTER: ucc_kn_seg_desc_compute(p, &s, p->rank); p->block_size = s.seg_size; p->block_offset += s.seg_offset; ucc_knomial_pattern_next_iteration_backward(p); return; case KN_PATTERN_REDUCE_SCATTERV: /* not implemented */ ucc_assert(0); default: ucc_assert(0); } } static inline void ucc_kn_rs_pattern_extra_seg(ucc_knomial_pattern_t *p, size_t *seg_count, ptrdiff_t *seg_offset) { switch (p->type) { case KN_PATTERN_REDUCE_SCATTER: *seg_offset = ucc_buffer_block_count(p->count, p->size, p->rank); *seg_count = ucc_buffer_block_count( p->count, p->size, ucc_knomial_pattern_get_extra(p, p->rank)); return; default: ucc_assert(0); } } #endif ucc-1.8.0/src/coll_patterns/double_binary_tree.h0000664000175000017500000001550715211535620022134 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef DOUBLE_BINARY_TREE_H_ #define DOUBLE_BINARY_TREE_H_ enum { LEFT_CHILD, RIGHT_CHILD }; typedef struct ucc_dbt_single_tree { ucc_rank_t rank; ucc_rank_t size; ucc_rank_t root; ucc_rank_t parent; ucc_rank_t children[2]; int n_children; int height; int recv; } ucc_dbt_single_tree_t; static inline ucc_rank_t get_root(ucc_rank_t size) { ucc_rank_t r = 1; while (r <= size) { r *= 2; } return r/2 - 1; } static inline int get_height(ucc_rank_t rank) { int h = 1; if (rank % 2 == 0) { return 0; } rank++; while ((rank & (1 << h)) == 0) { h++; } return h; } static inline ucc_rank_t get_left_child(ucc_rank_t rank, int height) { ucc_rank_t sub_height; if (height == 0) { return UCC_RANK_INVALID; } sub_height = 1 << (height - 1); return rank - sub_height; } static inline ucc_rank_t get_right_child(ucc_rank_t size, ucc_rank_t rank, int height, ucc_rank_t root) { ucc_rank_t sub_right_root, sub_height; if (rank == size - 1 || height == 0) { return UCC_RANK_INVALID; } sub_right_root = get_root(size - rank - 1) + 1; sub_height = 1 << (height - 1); if (rank == root) { return rank + sub_right_root; } return (rank + sub_height < size) ? rank + sub_height : rank + sub_right_root; } static inline void get_children(ucc_rank_t size, ucc_rank_t rank, int height, ucc_rank_t root, ucc_rank_t *l_c, ucc_rank_t *r_c) { *l_c = get_left_child(rank, height); *r_c = get_right_child(size, rank, height, root); } static inline int get_n_children(ucc_rank_t l_c, ucc_rank_t r_c) { int n_children = 0; if (l_c != UCC_RANK_INVALID) { n_children++; } if (r_c != UCC_RANK_INVALID) { n_children++; } return n_children; } static inline ucc_rank_t get_parent(int vsize, int vrank, int height, int troot) { if (vrank == troot) { return UCC_RANK_INVALID; } else if (height == 0) { return ((((vrank/2) % 2 == 0) && (vrank + 1 != vsize))) ? vrank + 1 : vrank - 1; } else { vrank++; if ((((1<<(height+1)) & vrank) > 0) || (vrank + (1< vsize) { return vrank - (1<children[LEFT_CHILD], &t1->children[RIGHT_CHILD]); t1->n_children = get_n_children(t1->children[LEFT_CHILD], t1->children[RIGHT_CHILD]); t1->height = height; t1->parent = parent; t1->size = size; t1->rank = rank; t1->root = root; t1->recv = 0; } static inline ucc_rank_t ucc_dbt_convert_rank_for_shift(ucc_rank_t rank, ucc_rank_t size) { ucc_rank_t i; for (i = 0; i < size; i++) { if (rank == (i + 1) % size) { break; } } return i; } static inline ucc_rank_t ucc_dbt_convert_rank_for_mirror(ucc_rank_t rank, ucc_rank_t size) { ucc_rank_t i; for (i = 0; i < size; i++) { if (rank == size - 1 - i) { break; } } return i; } static inline void ucc_dbt_build_t2(ucc_rank_t rank, ucc_rank_t size, ucc_dbt_single_tree_t *t2) { ucc_rank_t temp_rank = (size % 2) ? ucc_dbt_convert_rank_for_shift(rank, size) : ucc_dbt_convert_rank_for_mirror(rank, size); ucc_dbt_single_tree_t t1_temp; ucc_dbt_build_t1(temp_rank, size, &t1_temp); if (size % 2) { ucc_dbt_build_t2_shift(t1_temp, t2); } else { ucc_dbt_build_t2_mirror(t1_temp, t2); } } static inline void ucc_dbt_build_trees(ucc_rank_t rank, ucc_rank_t size, ucc_dbt_single_tree_t *t1, ucc_dbt_single_tree_t *t2) { ucc_dbt_build_t1(rank, size, t1); ucc_dbt_build_t2(rank, size, t2); } #endif ucc-1.8.0/src/coll_patterns/bruck_alltoall.h0000664000175000017500000000235315211535620021264 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef BRUCK_ALLTOALL_H_ #define BRUCK_ALLTOALL_H_ #include "utils/ucc_math.h" #define GET_NEXT_BRUCK_NUM(_num, _radix, _pow) \ ((((_num) + 1) % (_pow))?((_num) + 1):(((_num) + 1) + (_pow) * ((_radix) - 1))) #define GET_PREV_BRUCK_NUM(_num, _radix, _pow) \ (((_num) % (_pow))?((_num) - 1):(((_num) - 1) - (_pow) * ((_radix) - 1))) static inline ucc_rank_t get_bruck_step_start(uint32_t pow, uint32_t d) { return pow * d; } static inline ucc_rank_t get_bruck_step_finish(ucc_rank_t n, uint32_t radix, uint32_t d, uint32_t pow) { ucc_assume(pow > 0 && radix > 0); return ucc_min(n + pow - 1 - (n - d * pow) % (pow * radix), n); } static inline ucc_rank_t get_bruck_recv_peer(ucc_rank_t trank, ucc_rank_t tsize, ucc_rank_t step, uint32_t digit) { return (trank - step * digit + tsize * digit) % tsize; } static inline ucc_rank_t get_bruck_send_peer(ucc_rank_t trank, ucc_rank_t tsize, ucc_rank_t step, uint32_t digit) { return (trank + step * digit) % tsize; } #endif ucc-1.8.0/src/coll_patterns/ring.c0000664000175000017500000002233215211535620017223 0ustar alastairalastair/** * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ring.h" #include "components/topo/ucc_topo.h" #include "ucc/api/ucc.h" static ucc_status_t ucc_ring_pattern_init_topo_host( ucc_topo_t *topo, ucc_ring_pattern_t *p) { ucc_rank_t size = ucc_subset_size(&topo->set); ucc_sbgp_t *sbgp; ucc_ep_map_t *maps; ucc_rank_t *rings_buf; ucc_rank_t i; sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_FULL_HOST_ORDERED); if (sbgp->status != UCC_SBGP_ENABLED) { return UCC_ERR_NOT_FOUND; } maps = ucc_malloc( sizeof(*maps) + size * sizeof(ucc_rank_t), "nvlink_ring_maps"); if (!maps) { return UCC_ERR_NO_MEMORY; } rings_buf = (ucc_rank_t *)(maps + 1); maps[0].type = UCC_EP_MAP_ARRAY; maps[0].ep_num = size; maps[0].array.map = rings_buf; maps[0].array.elem_size = sizeof(ucc_rank_t); /* Use ucc_ep_map_eval to get ranks from sbgp->map since rank_map may be NULL after ucc_ep_map_from_array took ownership */ for (i = 0; i < size; i++) { rings_buf[i] = ucc_ep_map_eval(sbgp->map, i); } ucc_ring_pattern_init_map(maps, 1, p); return UCC_OK; } static ucc_status_t ucc_topo_build_sbgp_ring( ucc_topo_t *topo, const ucc_sbgp_t *sbgp, ucc_rank_t **ring_buf, unsigned *num_rings) { ucc_rank_t gsize = sbgp->group_size; ucc_status_t status = UCC_OK; ucc_device_id_t *sbgp_nics; ucc_device_id_t *sbgp_devs; ucc_rank_t *sbgp_ranks; ucc_rank_t i, j, k, r, path_len, rank; unsigned num_sbgp_nics; int dist, best_dist; ucc_gpu_info_t *gpu_info; ucc_nic_info_t *nic_info; ucc_host_info_t *host_info; ucc_device_id_t best_dev; if (sbgp->status != UCC_SBGP_ENABLED) { return UCC_ERR_INVALID_PARAM; } if (sbgp->type != UCC_SBGP_NODE) { return UCC_ERR_NOT_SUPPORTED; } if (!ucc_topo_is_nvlink_fully_connected(topo, sbgp)) { return UCC_ERR_NOT_SUPPORTED; } sbgp_nics = ucc_malloc( sbgp->group_size * sizeof(*sbgp_nics), "nvlink_sbgp_nics"); if (!sbgp_nics) { status = UCC_ERR_NO_MEMORY; goto exit; } sbgp_devs = ucc_malloc( sbgp->group_size * sizeof(*sbgp_devs), "nvlink_sbgp_devs"); if (!sbgp_devs) { status = UCC_ERR_NO_MEMORY; goto free_sbgp_nics; } sbgp_ranks = ucc_malloc( sbgp->group_size * sizeof(*sbgp_ranks), "nvlink_sbgp_ranks"); if (!sbgp_ranks) { status = UCC_ERR_NO_MEMORY; goto free_sbgp_devs; } num_sbgp_nics = 0; for (j = 0; j < gsize; j++) { rank = ucc_ep_map_eval(sbgp->map, j); host_info = &topo->topo->hosts[rank]; gpu_info = &host_info->gpus[ucc_ilog2(host_info->visible_gpus)]; best_dist = INT_MAX; best_dev = UCC_DEVICE_ID_INVALID; ucc_for_each_bit (i, host_info->visible_nics) { nic_info = &host_info->nics[i]; dist = ucc_pci_distance(&gpu_info->pci, &nic_info->pci); if (dist < best_dist) { /* check if the nic is already in the sbgp_nics */ for (k = 0; k < j; k++) { if (sbgp_nics[k] == UCC_DEVICE_ID_INVALID) { continue; } if (ucc_compare_pci_info( &topo->topo->hosts[sbgp_ranks[k]] .nics[sbgp_nics[k]] .pci, &nic_info->pci)) { break; } } if (k == j) { /* no duplicate nic found, update the best dist and dev */ best_dist = dist; best_dev = i; } } } if (best_dev != UCC_DEVICE_ID_INVALID) { num_sbgp_nics++; } r = j; best_dist = INT_MAX; for (i = 0; i < j; i++) { dist = ucc_pci_distance( &gpu_info->pci, &topo->topo->hosts[sbgp_ranks[i]].gpus[sbgp_devs[i]].pci); if (dist < best_dist) { best_dist = dist; r = i + 1; } } for (i = j; i > r; i--) { sbgp_devs[i] = sbgp_devs[i - 1]; sbgp_nics[i] = sbgp_nics[i - 1]; sbgp_ranks[i] = sbgp_ranks[i - 1]; } sbgp_devs[r] = ucc_ilog2(host_info->visible_gpus); sbgp_nics[r] = best_dev; sbgp_ranks[r] = rank; } ucc_debug("found %d nvlink rings for sbgp %p", num_sbgp_nics, sbgp); *num_rings = num_sbgp_nics; if (*num_rings == 0) { status = UCC_ERR_NOT_FOUND; goto free_sbgp_ranks; } *ring_buf = ucc_malloc( *num_rings * gsize * sizeof(ucc_rank_t), "nvlink_sbgp_ring"); if (!(*ring_buf)) { status = UCC_ERR_NO_MEMORY; goto free_sbgp_ranks; } for (j = 0, r = 0; j < gsize; j++) { if (sbgp_nics[j] == UCC_DEVICE_ID_INVALID) { continue; } /* find next valid NIC in ring order after j */ k = (j + 1) % gsize; while (k != j && sbgp_nics[k] == UCC_DEVICE_ID_INVALID) { k = (k + 1) % gsize; } path_len = 1; (*ring_buf)[r*gsize] = sbgp_ranks[j]; i = (j - 1 + gsize) % gsize; while (i != j) { if (i != k) { (*ring_buf)[r*gsize + path_len++] = sbgp_ranks[i]; } i = (i - 1 + gsize) % gsize; } (*ring_buf)[r*gsize + path_len++] = sbgp_ranks[k]; r++; } free_sbgp_ranks: ucc_free(sbgp_ranks); free_sbgp_devs: ucc_free(sbgp_devs); free_sbgp_nics: ucc_free(sbgp_nics); exit: return status; } static ucc_status_t ucc_ring_pattern_init_topo_cuda( ucc_topo_t *topo, unsigned n_rings, ucc_ring_pattern_t *p) { ucc_rank_t size = ucc_subset_size(&topo->set); ucc_sbgp_t *sbgps = NULL; int n_sbgps = 0; ucc_rank_t i; ucc_ep_map_t *maps; ucc_rank_t *rings_buf; unsigned rings_avail; ucc_rank_t **sbgp_rings; ucc_status_t status; ucc_rank_t *ring; ucc_rank_t sbgp_id, sbgp_size; if (!ucc_topo_has_device_info(topo)) { goto fallback_host_ring; } status = ucc_topo_get_all_nodes(topo, &sbgps, &n_sbgps); if (status != UCC_OK) { if (status == UCC_ERR_INVALID_PARAM || status == UCC_ERR_NOT_FOUND) { goto fallback_host_ring; } return status; } sbgp_rings = ucc_calloc( n_sbgps, sizeof(*sbgp_rings), "nvlink_sbgp_rings_buf"); if (!sbgp_rings) { return UCC_ERR_NO_MEMORY; } ucc_debug("building nvlink rings for %d sbgps", n_sbgps); for (i = 0; i < n_sbgps; i++) { status = ucc_topo_build_sbgp_ring( topo, &sbgps[i], &sbgp_rings[i], &rings_avail); if (status != UCC_OK) { ucc_debug( "failed to build sbgp ring for sbgp %d (status=%d)", i, status); goto free_node_ring_info; } if (rings_avail == 0) { ucc_debug("no nvlink rings available for sbgp %d", i); goto free_node_ring_info; } n_rings = ucc_min(n_rings, rings_avail); } maps = ucc_malloc( n_rings * sizeof(*maps) + n_rings * size * sizeof(ucc_rank_t), "nvlink_ring_maps"); if (!maps) { goto free_node_ring_info; } rings_buf = (ucc_rank_t *)(maps + n_rings); for (i = 0; i < n_rings; i++) { maps[i].type = UCC_EP_MAP_ARRAY; maps[i].ep_num = size; maps[i].array.map = rings_buf + (size_t)i * size; maps[i].array.elem_size = sizeof(ucc_rank_t); } for (i = 0; i < n_rings; i++) { ring = (ucc_rank_t *)maps[i].array.map; for (sbgp_id = 0; sbgp_id < n_sbgps; sbgp_id++) { sbgp_size = sbgps[sbgp_id].group_size; memcpy( ring, PTR_OFFSET( sbgp_rings[sbgp_id], i * sbgp_size * sizeof(ucc_rank_t)), sbgp_size * sizeof(ucc_rank_t)); ring += sbgp_size; } } for (sbgp_id = 0; sbgp_id < n_sbgps; sbgp_id++) { ucc_free(sbgp_rings[sbgp_id]); } ucc_free(sbgp_rings); ucc_ring_pattern_init_map(maps, n_rings, p); return UCC_OK; free_node_ring_info: for (sbgp_id = 0; sbgp_id < n_sbgps; sbgp_id++) { ucc_free(sbgp_rings[sbgp_id]); } ucc_free(sbgp_rings); fallback_host_ring: ucc_debug("fallback to host ring for cuda ring"); return ucc_ring_pattern_init_topo_host(topo, p); } ucc_status_t ucc_ring_pattern_init_topo( ucc_topo_t *topo, ucc_memory_type_t mt, unsigned num_rings, ucc_ring_pattern_t *p) { if (!topo || !topo->topo || num_rings == 0) { return UCC_ERR_INVALID_PARAM; } switch (mt) { case UCC_MEMORY_TYPE_HOST: return ucc_ring_pattern_init_topo_host(topo, p); case UCC_MEMORY_TYPE_CUDA: return ucc_ring_pattern_init_topo_cuda(topo, num_rings, p); default: return UCC_ERR_NOT_SUPPORTED; } } ucc-1.8.0/src/Makefile.am0000664000175000017500000001164015211535620015303 0ustar alastairalastair# # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # See file LICENSE for terms. cl_dirs = components/cl/basic \ components/cl/hier mc_dirs = components/mc/cpu ec_dirs = components/ec/cpu topo_dirs = if HAVE_CUDA mc_dirs += components/mc/cuda ec_dirs += components/ec/cuda topo_dirs += components/topo/cuda endif if HAVE_DOCA_UROM cl_dirs += components/cl/doca_urom endif if HAVE_ROCM mc_dirs += components/mc/rocm ec_dirs += components/ec/rocm endif if HAVE_IBVERBS topo_dirs += components/topo/ib endif SUBDIRS = . $(cl_dirs) $(mc_dirs) $(ec_dirs) $(topo_dirs) include components/tl/makefile.am lib_LTLIBRARIES = libucc.la noinst_LIBRARIES = libucc_la_CPPFLAGS = $(AM_CPPFLAGS) $(BASE_CPPFLAGS) libucc_la_CFLAGS = -c $(BASE_CFLAGS) libucc_la_LDFLAGS = -version-info $(SOVERSION) --as-needed -pthread nobase_dist_libucc_la_HEADERS = \ ucc/api/ucc.h \ ucc/api/ucc_def.h \ ucc/api/ucc_version.h \ ucc/api/ucc_status.h noinst_HEADERS = \ core/ucc_global_opts.h \ core/ucc_lib.h \ core/ucc_context.h \ core/ucc_team.h \ core/ucc_ee.h \ core/ucc_progress_queue.h \ core/ucc_service_coll.h \ core/ucc_dt.h \ schedule/ucc_schedule.h \ schedule/ucc_schedule_pipelined.h \ coll_score/ucc_coll_score.h \ utils/arch/aarch64/cpu.h \ utils/arch/ppc64/cpu.h \ utils/arch/riscv64/cpu.h \ utils/arch/x86_64/cpu.h \ utils/arch/cpu.h \ utils/arch/cuda_def.h \ utils/ucc_compiler_def.h \ utils/ucc_log.h \ utils/ucc_parser.h \ utils/ucc_component.h \ utils/ucc_datastruct.h \ utils/ucc_math.h \ utils/ucc_coll_utils.h \ utils/ucc_list.h \ utils/ucc_string.h \ utils/ucc_queue.h \ utils/ucc_proc_info.h \ utils/khash.h \ utils/ini.h \ utils/ucc_spinlock.h \ utils/ucc_mpool.h \ utils/ucc_rcache.h \ utils/profile/ucc_profile.h \ utils/profile/ucc_profile_on.h \ utils/profile/ucc_profile_off.h \ utils/ucc_time.h \ utils/ucc_sys.h \ utils/ucc_assert.h \ utils/debug/log.h \ components/base/ucc_base_iface.h \ components/cl/ucc_cl.h \ components/cl/ucc_cl_log.h \ components/cl/ucc_cl_type.h \ components/tl/ucc_tl.h \ components/tl/ucc_tl_log.h \ components/mc/ucc_mc.h \ components/mc/base/ucc_mc_base.h \ components/mc/ucc_mc_log.h \ components/ec/ucc_ec.h \ components/ec/base/ucc_ec_base.h \ components/ec/ucc_ec_log.h \ coll_patterns/recursive_knomial.h \ coll_patterns/sra_knomial.h \ coll_patterns/bruck_alltoall.h \ coll_patterns/double_binary_tree.h \ components/topo/ucc_topo.h \ components/topo/ucc_sbgp.h \ components/topo/ucc_sysinfo.h \ components/topo/base/ucc_sysinfo_base.h libucc_la_SOURCES = \ core/ucc_lib.c \ core/ucc_constructor.c \ core/ucc_global_opts.c \ core/ucc_version.c \ core/ucc_context.c \ core/ucc_team.c \ core/ucc_ee.c \ core/ucc_coll.c \ core/ucc_progress_queue.c \ core/ucc_progress_queue_st.c \ core/ucc_progress_queue_mt.c \ core/ucc_service_coll.c \ core/ucc_dt.c \ coll_patterns/ring.c \ schedule/ucc_schedule.c \ schedule/ucc_schedule_pipelined.c \ coll_score/ucc_coll_score.c \ coll_score/ucc_coll_score_map.c \ utils/ini.c \ utils/ucc_component.c \ utils/ucc_datastruct.c \ utils/ucc_status.c \ utils/ucc_mpool.c \ utils/ucc_math.c \ utils/ucc_proc_info.c \ utils/ucc_string.c \ utils/ucc_coll_utils.c \ utils/ucc_parser.c \ utils/profile/ucc_profile.c \ utils/ucc_sys.c \ utils/arch/x86_64/cpu.c \ utils/arch/aarch64/cpu.c \ utils/ucc_assert.c \ utils/debug/log.c \ components/base/ucc_base_iface.c \ components/cl/ucc_cl.c \ components/tl/ucc_tl.c \ components/mc/ucc_mc.c \ components/mc/base/ucc_mc_base.c \ components/ec/ucc_ec.c \ components/ec/base/ucc_ec_base.c \ components/topo/ucc_topo.c \ components/topo/ucc_sysinfo.c \ components/topo/ucc_sbgp.c libucc_ladir = $(includedir) ucc-1.8.0/src/coll_score/0000775000175000017500000000000015211535620015371 5ustar alastairalastairucc-1.8.0/src/coll_score/ucc_coll_score.c0000664000175000017500000011315715211535620020523 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_coll_score.h" #include "utils/ucc_string.h" #include "utils/ucc_log.h" #include "utils/ucc_coll_utils.h" #include "utils/ucc_mem_type.h" char *ucc_score_to_str(ucc_score_t score, char *buf, size_t max) { if (score == UCC_SCORE_MAX) { ucc_strncpy_safe(buf, "inf", max); } else { ucc_snprintf_safe(buf, max, "%d", score); } return buf; } ucc_status_t ucc_coll_score_alloc(ucc_coll_score_t **score) { ucc_coll_score_t *s = ucc_malloc(sizeof(*s), "ucc_coll_score"); int i, j; if (!s) { ucc_error("failed to allocate %zd bytes for ucc_coll_score", sizeof(*s)); *score = NULL; return UCC_ERR_NO_MEMORY; } for (i = 0; i < UCC_COLL_TYPE_NUM; i++) { for (j = 0; j < UCC_MEMORY_TYPE_LAST; j++) { ucc_list_head_init(&s->scores[i][j]); } } *score = s; return UCC_OK; } static inline void ucc_msg_range_free(ucc_msg_range_t *r) { ucc_list_destruct(&r->fallback, ucc_coll_entry_t, ucc_free, list_elem); ucc_free(r); } static inline ucc_status_t coll_score_add_range(ucc_coll_score_t *score, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, size_t start, size_t end, ucc_score_t msg_score, ucc_base_coll_init_fn_t init, ucc_base_team_t *team) { ucc_msg_range_t *r; ucc_msg_range_t *range; ucc_list_link_t *list, *insert_pos, *next; if (start >= end) { return UCC_ERR_INVALID_PARAM; } r = ucc_malloc(sizeof(*r), "ucc_msg_range"); if (!r) { ucc_error("failed to allocate %zd bytes for ucc_msg_range", sizeof(*r)); return UCC_ERR_NO_MEMORY; } ucc_list_head_init(&r->fallback); r->start = start; r->end = end; r->super.score = msg_score; r->super.init = init; r->super.team = team; list = &score->scores[ucc_ilog2(coll_type)][mem_type]; insert_pos = list; ucc_list_for_each(range, list, super.list_elem) { if (start >= range->end) { insert_pos = &range->super.list_elem; } else { break; } } ucc_list_insert_after(insert_pos, &r->super.list_elem); /*sanity check: ranges shuold not overlap */ next = r->super.list_elem.next; if ((next != list) && (ucc_container_of(next, ucc_msg_range_t, super.list_elem)->start < r->end)) { ucc_error("attempt to add overlaping range"); ucc_list_del(&r->super.list_elem); ucc_msg_range_free(r); return UCC_ERR_INVALID_PARAM; } return UCC_OK; } ucc_status_t ucc_coll_score_add_range(ucc_coll_score_t *score, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, size_t start, size_t end, ucc_score_t msg_score, ucc_base_coll_init_fn_t init, ucc_base_team_t * team) { if (msg_score == 0) { /* score 0 means range is disabled, just skip */ return UCC_OK; } return coll_score_add_range(score, coll_type, mem_type, start, end, msg_score, init, team); } void ucc_coll_score_free(ucc_coll_score_t *score) { int i, j; if (!score) { return; } for (i = 0; i < UCC_COLL_TYPE_NUM; i++) { for (j = 0; j < UCC_MEMORY_TYPE_LAST; j++) { ucc_list_destruct(&score->scores[i][j], ucc_msg_range_t, ucc_msg_range_free, super.list_elem); } } ucc_free(score); } static ucc_status_t ucc_fallback_alloc(ucc_score_t score, ucc_base_coll_init_fn_t init, ucc_base_team_t *team, ucc_coll_entry_t **_fb) { ucc_coll_entry_t *fb; fb = ucc_malloc(sizeof(*fb), "fallback"); if (ucc_unlikely(!fb)) { ucc_error("failed to allocate %zd bytes for fallback", sizeof(ucc_coll_entry_t)); *_fb = NULL; return UCC_ERR_NO_MEMORY; } fb->score = score; fb->init = init; fb->team = team; *_fb = fb; return UCC_OK; } static inline void ucc_fallback_insert(ucc_list_link_t *list, ucc_coll_entry_t *fb) { ucc_list_link_t *insert_pos; ucc_coll_entry_t *f; insert_pos = list; ucc_list_for_each(f, list, list_elem) { if (fb->score == f->score && fb->init == f->init && fb->team == f->team) { ucc_free(fb); /* same fallback: skip */ return; } if (fb->score < f->score) { insert_pos = &f->list_elem; } else { break; } } ucc_list_insert_after(insert_pos, &fb->list_elem); } #define FB_ALLOC_INSERT(_fb_in, _fb_out, _dest, _status, _label) do { \ _status = \ ucc_fallback_alloc((_fb_in)->score, (_fb_in)->init, \ (_fb_in)->team, &(_fb_out)); \ if (ucc_unlikely(UCC_OK != _status)) { \ goto _label; \ } \ ucc_fallback_insert(&(_dest)->fallback, _fb_out); \ } while (0) static ucc_status_t ucc_fallback_copy(const ucc_msg_range_t *in, ucc_msg_range_t *out) { ucc_status_t status = UCC_OK; ucc_coll_entry_t *fb_in, *fb; ucc_list_for_each(fb_in, &in->fallback, list_elem) { FB_ALLOC_INSERT(fb_in, fb, out, status, out); } out: return status; } static ucc_status_t ucc_msg_range_dup(const ucc_msg_range_t *in, ucc_msg_range_t **out) { ucc_msg_range_t *r; ucc_status_t status; r = ucc_malloc(sizeof(*r), "msg_range"); if (ucc_unlikely(!r)) { *out = NULL; ucc_error("failed to allocate %zd bytes for msgrange", sizeof(*r)); return UCC_ERR_NO_MEMORY; } memcpy(r, in, sizeof(*r)); ucc_list_head_init(&r->fallback); status = ucc_fallback_copy(in, r); if (status != UCC_OK) { ucc_msg_range_free(r); r = NULL; } *out = r; return status; } #define MSG_RANGE_DUP(_r) \ ({ \ ucc_msg_range_t *_dup; \ status = ucc_msg_range_dup(_r, &_dup); \ if (UCC_OK != status) { \ goto out; \ } \ _dup; \ }) static ucc_status_t ucc_msg_range_add_fallback(const ucc_msg_range_t *in, ucc_msg_range_t *out) { ucc_coll_entry_t *fb; ucc_status_t status; if (in->super.init == out->super.init && in->super.team == out->super.team) { return UCC_OK; } status = ucc_fallback_alloc(in->super.score, in->super.init, in->super.team, &fb); if (ucc_unlikely(UCC_OK != status)) { return status; } status = ucc_fallback_copy(in, out); ucc_fallback_insert(&out->fallback, fb); return status; } #define ADD_FALLBACK(_in, _out) \ do { \ status = ucc_msg_range_add_fallback(_in, _out); \ if (UCC_OK != status) { \ goto out; \ } \ } while (0) static ucc_status_t ucc_score_list_dup(const ucc_list_link_t *src, ucc_list_link_t *dst) { ucc_msg_range_t *range, *r; ucc_status_t status; ucc_list_head_init(dst); ucc_list_for_each(range, src, super.list_elem) { r = MSG_RANGE_DUP(range); ucc_list_add_tail(dst, &r->super.list_elem); } return UCC_OK; out: ucc_list_for_each_safe(range, r, dst, super.list_elem) { ucc_list_del(&range->super.list_elem); ucc_msg_range_free(range); } ucc_assert(ucc_list_is_empty(dst)); return UCC_ERR_NO_MEMORY; } static inline int ucc_msg_range_fb_compare(ucc_msg_range_t *r1, ucc_msg_range_t *r2) { ucc_list_link_t *l1, *l2; ucc_coll_entry_t *fb1, *fb2; l1 = &r1->fallback; l2 = &r2->fallback; if (ucc_list_length(l1) != ucc_list_length(l2)) { return 0; } fb2 = ucc_list_head(l2, ucc_coll_entry_t, list_elem); ucc_list_for_each(fb1, l1, list_elem) { if (fb1->score != fb2->score || fb1->init != fb2->init || fb1->team != fb2->team) { return 0; } fb2 = ucc_list_next(&fb2->list_elem, ucc_coll_entry_t, list_elem); } return 1; } static ucc_status_t ucc_coll_score_merge_one(ucc_list_link_t *list1, ucc_list_link_t *list2, ucc_list_link_t *out) { ucc_list_link_t lst1, lst2; ucc_msg_range_t *r1, *r2, *left, *right, *best, *new; ucc_msg_range_t *range, *temp, *next; ucc_status_t status; if (ucc_list_is_empty(list1) && ucc_list_is_empty(list2)) { return UCC_OK; } else if (ucc_list_is_empty(list1)) { return ucc_score_list_dup(list2, out); } else if (ucc_list_is_empty(list2)) { return ucc_score_list_dup(list1, out); } /* list1 and list2 both non-empty: need to intersect ranges */ status = ucc_score_list_dup(list1, &lst1); if (UCC_OK != status) { return status; } status = ucc_score_list_dup(list2, &lst2); if (UCC_OK != status) { goto out; } while (!(ucc_list_is_empty(&lst1) && ucc_list_is_empty(&lst2))) { if (ucc_list_is_empty(&lst1)) { ucc_list_add_tail(out, &(ucc_list_extract_head(&lst2, ucc_coll_entry_t, list_elem)->list_elem)); continue; } if (ucc_list_is_empty(&lst2)) { ucc_list_add_tail(out, &(ucc_list_extract_head(&lst1, ucc_coll_entry_t, list_elem)->list_elem)); continue; } r1 = ucc_list_head(&lst1, ucc_msg_range_t, super.list_elem); r2 = ucc_list_head(&lst2, ucc_msg_range_t, super.list_elem); left = (r1->start < r2->start) ? r1 : r2; //NOLINT if (r1->start == r2->start) { if (r1->end == r2->end) { best = (r1->super.score > r2->super.score) ? r1 : r2; left = (best == r1) ? r2 : r1; ucc_list_del(&r1->super.list_elem); ucc_list_del(&r2->super.list_elem); ucc_list_add_tail(out, &best->super.list_elem); ADD_FALLBACK(left, best); ucc_msg_range_free(left); continue; } left = (r1->end < r2->end) ? r1 : r2; right = (left == r1) ? r2 : r1; new = MSG_RANGE_DUP(right); right->start = new->end = left->end; ucc_list_del(&left->super.list_elem); if (left->super.score < new->super.score) { SWAP(left, new, void *); } ADD_FALLBACK(new, left); ucc_msg_range_free(new); ucc_list_add_tail(out, &left->super.list_elem); continue; } right = (left == r1) ? r2 : r1; if (left->end <= right->start) { /* ranges don't overlap - copy over */ ucc_list_del(&left->super.list_elem); ucc_list_add_tail(out, &left->super.list_elem); } else { new = MSG_RANGE_DUP(left); new->end = right->start; ucc_list_add_tail(out, &new->super.list_elem); left->start = right->start; } } /* Merge consequtive ranges with the same score, same init fn, same team and same fallback sequence if any have been produced by the algorithm above */ ucc_list_for_each_safe(range, temp, out, super.list_elem) { if (range->super.list_elem.next != out) { next = ucc_container_of(range->super.list_elem.next, ucc_msg_range_t, super.list_elem); if (range->super.score == next->super.score && range->end == next->start && range->super.init == next->super.init && range->super.team == next->super.team && 1 == ucc_msg_range_fb_compare(range, next)) { next->start = range->start; ucc_list_del(&range->super.list_elem); ucc_msg_range_free(range); } } } return UCC_OK; out: ucc_list_destruct(&lst2, ucc_msg_range_t, ucc_msg_range_free, super.list_elem); ucc_list_destruct(&lst1, ucc_msg_range_t, ucc_msg_range_free, super.list_elem); ucc_list_destruct(out, ucc_msg_range_t, ucc_msg_range_free, super.list_elem); return status; } ucc_status_t ucc_coll_score_merge(ucc_coll_score_t * score1, ucc_coll_score_t * score2, ucc_coll_score_t **rst, int free_inputs) { ucc_coll_score_t *out; ucc_status_t status; int i, j; status = ucc_coll_score_alloc(&out); if (UCC_OK != status) { goto out; } for (i = 0; i < UCC_COLL_TYPE_NUM; i++) { for (j = 0; j < UCC_MEMORY_TYPE_LAST; j++) { status = ucc_coll_score_merge_one(&score1->scores[i][j], &score2->scores[i][j], &out->scores[i][j]); if (UCC_OK != status) { ucc_coll_score_free(out); goto out; } } } *rst = out; out: if (free_inputs) { ucc_coll_score_free(score1); ucc_coll_score_free(score2); } return status; } ucc_status_t ucc_coll_score_merge_in(ucc_coll_score_t **dst, ucc_coll_score_t *src) { ucc_coll_score_t *tmp = NULL; ucc_status_t status; status = ucc_coll_score_merge(src, *dst, &tmp, 1); *dst = tmp; return status; } static ucc_status_t str_to_coll_type(const char *str, unsigned *ct_n, ucc_coll_type_t **ct) { ucc_status_t status = UCC_OK; char **tokens; unsigned i, n_tokens; ucc_coll_type_t t; tokens = ucc_str_split(str, ","); if (!tokens) { status = UCC_ERR_INVALID_PARAM; goto out; } n_tokens = ucc_str_split_count(tokens); *ct = ucc_malloc(n_tokens * sizeof(ucc_coll_type_t), "ucc_coll_types"); if (!(*ct)) { ucc_error("failed to allocate %zd bytes for ucc_coll_types", sizeof(ucc_coll_type_t) * n_tokens); status = UCC_ERR_NO_MEMORY; goto out; } *ct_n = 0; for (i = 0; i < n_tokens; i++) { t = ucc_coll_type_from_str(tokens[i]); if (t == UCC_COLL_TYPE_LAST) { /* entry does not match any coll type name */ ucc_free(*ct); *ct = NULL; status = UCC_ERR_NOT_FOUND; goto out; } (*ct)[*ct_n] = t; (*ct_n)++; } out: ucc_str_split_free(tokens); return status; } static ucc_status_t str_to_score(const char *str, ucc_score_t *score) { if (0 == strcasecmp("inf", str)) { *score = UCC_SCORE_MAX; } else if (UCC_OK != ucc_str_is_number(str)) { return UCC_ERR_NOT_FOUND; } else { *score = (ucc_score_t)atoi(str); } return UCC_OK; } static ucc_status_t str_to_alg_id(const char *str, const char **alg_id) { if ('@' != str[0]) { return UCC_ERR_NOT_FOUND; } *alg_id = str + 1; return UCC_OK; } static ucc_status_t str_to_msgranges(const char *str, size_t **ranges, unsigned *n_ranges) { ucc_status_t status = UCC_OK; char **tokens; char **tokens2; unsigned i, n_tokens, n_tokens2; size_t m1, m2; tokens = ucc_str_split(str, ","); if (!tokens) { status = UCC_ERR_INVALID_PARAM; goto out; } n_tokens = ucc_str_split_count(tokens); *ranges = ucc_malloc(2 * n_tokens * sizeof(size_t), "ucc_msgsize_ranges"); if (!(*ranges)) { ucc_error("failed to allocate %zd bytes for ucc_msgsize_ranges", sizeof(size_t) * 2 * n_tokens); status = UCC_ERR_NO_MEMORY; goto out; } for (i = 0; i < n_tokens; i++) { tokens2 = ucc_str_split(tokens[i], "-"); if (!tokens2) { goto err; } n_tokens2 = ucc_str_split_count(tokens2); if (n_tokens2 != 2) { goto err; } if (UCC_OK != ucc_str_to_memunits(tokens2[0], &m1) || UCC_OK != ucc_str_to_memunits(tokens2[1], &m2)) { goto err; } ucc_str_split_free(tokens2); (*ranges)[2 * i] = m1; (*ranges)[2 * i + 1] = m2; } *n_ranges = i; out: ucc_str_split_free(tokens); return status; err: ucc_str_split_free(tokens2); ucc_free(*ranges); *ranges = NULL; status = UCC_ERR_NOT_FOUND; goto out; } static ucc_status_t str_to_tsizes(const char *str, ucc_rank_t **tsizes, unsigned *n_tsizes) { ucc_status_t status = UCC_OK; char ** tokens; char ** tokens2; unsigned i, n_tokens, n_tokens2; /* team_size qualifer should be enclosed in "[]". It it a coma-separated list of ranges start-end or single values (exact team size) */ if ('[' != str[0] || ']' != str[strlen(str) - 1]) { return UCC_ERR_NOT_FOUND; } tokens = ucc_str_split(str + 1, ","); if (!tokens) { status = UCC_ERR_INVALID_PARAM; goto out; } n_tokens = ucc_str_split_count(tokens); *tsizes = ucc_malloc(2 * n_tokens * sizeof(ucc_rank_t), "ucc_tsize_ranges"); if (!(*tsizes)) { ucc_error("failed to allocate %zd bytes for ucc_tsize_ranges", sizeof(ucc_rank_t) * 2 * n_tokens); status = UCC_ERR_NO_MEMORY; goto out; } ucc_assert(']' == tokens[n_tokens - 1][strlen(tokens[n_tokens - 1]) - 1]); /* remove last "]" from the parsed string, we have already checked it was present */ tokens[n_tokens - 1][strlen(tokens[n_tokens - 1]) - 1] = '\0'; for (i = 0; i < n_tokens; i++) { tokens2 = ucc_str_split(tokens[i], "-"); if (!tokens2) { status = UCC_ERR_INVALID_PARAM; goto err; } n_tokens2 = ucc_str_split_count(tokens2); if (n_tokens2 == 1) { /* exact team size - single value */ if (UCC_OK != ucc_str_is_number(tokens2[0])) { status = UCC_ERR_INVALID_PARAM; goto err; } (*tsizes)[2 * i] = (ucc_rank_t)atoi(tokens2[0]); (*tsizes)[2 * i + 1] = (ucc_rank_t)atoi(tokens2[0]); } else { if (n_tokens2 != 2) { status = UCC_ERR_INVALID_PARAM; goto err; } if (UCC_OK == ucc_str_is_number(tokens2[0])) { (*tsizes)[2 * i] = (ucc_rank_t)atoi(tokens2[0]); } else { status = UCC_ERR_INVALID_PARAM; goto err; } if (0 == strcasecmp("inf", tokens2[1])) { (*tsizes)[2 * i + 1] = UCC_RANK_MAX; } else if (UCC_OK == ucc_str_is_number(tokens2[1])) { (*tsizes)[2 * i + 1] = (ucc_rank_t)atoi(tokens2[1]); } else { status = UCC_ERR_INVALID_PARAM; goto err; } if ((*tsizes)[2 * i + 1] < (*tsizes)[2 * i]) { status = UCC_ERR_INVALID_PARAM; goto err; } } ucc_str_split_free(tokens2); } *n_tsizes = i; out: ucc_str_split_free(tokens); return status; err: ucc_str_split_free(tokens2); ucc_free(*tsizes); *tsizes = NULL; goto out; } static ucc_status_t ucc_coll_score_parse_str(const char *str, ucc_coll_score_t *score, ucc_rank_t team_size, ucc_base_coll_init_fn_t init, ucc_base_team_t *team, ucc_alg_id_to_init_fn_t alg_fn) { ucc_status_t status = UCC_OK; ucc_coll_type_t *ct = NULL; size_t *msg = NULL; ucc_rank_t *tsizes = NULL; ucc_base_coll_init_fn_t alg_init = NULL; const char* alg_id = NULL; ucc_score_t score_v = UCC_SCORE_INVALID; int ts_skip = 0; uint32_t mtypes = 0; char **tokens; unsigned i, n_tokens, ct_n, c, m, n_ranges, r, n_tsizes; ct_n = n_ranges = n_tsizes = 0; tokens = ucc_str_split(str, ":"); if (!tokens) { status = UCC_ERR_INVALID_PARAM; goto out; } n_tokens = ucc_str_split_count(tokens); for (i = 0; i < n_tokens; i++) { if (!ct && UCC_OK == str_to_coll_type(tokens[i], &ct_n, &ct)) { continue; } if (!mtypes && UCC_OK == ucc_str_to_mtype_map(tokens[i], ",", &mtypes)) { continue; } if ((UCC_SCORE_INVALID == score_v) && UCC_OK == str_to_score(tokens[i], &score_v)) { continue; } if (!msg && UCC_OK == str_to_msgranges(tokens[i], &msg, &n_ranges)) { continue; } if (!tsizes && UCC_OK == str_to_tsizes(tokens[i], &tsizes, &n_tsizes)) { continue; } if (!alg_id && UCC_OK == str_to_alg_id(tokens[i], &alg_id)) { continue; } /* if we get there then we could not match token to any field */ status = UCC_ERR_INVALID_PARAM; ucc_error("failed to parse token \'%s\' in \'%s\'", tokens[i], str); //TODO add parsing of msg ranges and team size ranges goto out; } if (tsizes) { /* Team size qualifier was provided: check if we should apply this str setting to the current team */ ts_skip = 1; for (i = 0; i < n_tsizes; i++) { if (team_size >= tsizes[2 * i] && team_size <= tsizes[2 * i + 1]) { ts_skip = 0; break; } } } if (!ts_skip && (UCC_SCORE_INVALID != score_v || NULL != alg_id)) { /* Score provided but not coll_types/mem_types. This means: apply score to ALL coll_types/mem_types */ if (!ct) { ct_n = UCC_COLL_TYPE_NUM; } if (!mtypes) { mtypes = UCC_MEM_TYPE_MASK_FULL; } if (!msg) { n_ranges = 1; } for (c = 0; c < ct_n; c++) { for (m = 0; m < UCC_MEMORY_TYPE_LAST; m++) { if (!(UCC_BIT(m) & mtypes)) { continue; } ucc_coll_type_t coll_type = ct ? ct[c] : (ucc_coll_type_t)UCC_BIT(c); ucc_memory_type_t mem_type = (ucc_memory_type_t)m; if (alg_id) { if (!alg_fn) { status = UCC_ERR_NOT_SUPPORTED; ucc_error("modifying algorithm id is not supported by " "component %s", team->context->lib->log_component.name); goto out; } ucc_assert(NULL != team); const char *alg_id_str = NULL; int alg_id_n = 0; if (UCC_OK == ucc_str_is_number(alg_id)) { alg_id_n = atoi(alg_id); } else { alg_id_str = alg_id; } status = alg_fn(alg_id_n, alg_id_str, coll_type, mem_type, &alg_init); if (UCC_ERR_INVALID_PARAM == status) { ucc_error("incorrect algorithm id provided: %s, %s, " "component %s", alg_id, str, team->context->lib->log_component.name); goto out; } else if (UCC_ERR_NOT_SUPPORTED == status) { ucc_error("modifying algorithm id is not supported for " "%s, alg %s, component %s", ucc_coll_type_str(coll_type), alg_id, team->context->lib->log_component.name); goto out; } else if (status < 0) { ucc_error("failed to map alg id to init: %s, %s, " "status %s, component %s", alg_id, str, ucc_status_string(status), team->context->lib->log_component.name); goto out; } } for (r = 0; r < n_ranges; r++) { size_t m_start = 0; size_t m_end = UCC_MSG_MAX; if (msg) { m_start = msg[r * 2]; m_end = msg[r * 2 + 1]; } status = coll_score_add_range( score, coll_type, mem_type, m_start, m_end, score_v, alg_init ? alg_init : init, team); } } } } out: ucc_free(ct); ucc_free(msg); ucc_free(tsizes); ucc_str_split_free(tokens); return status; } ucc_status_t ucc_coll_score_alloc_from_str(const char * str, ucc_coll_score_t ** score_p, ucc_rank_t team_size, ucc_base_coll_init_fn_t init, ucc_base_team_t * team, ucc_alg_id_to_init_fn_t alg_fn) { ucc_coll_score_t *score; ucc_status_t status; char **tokens; unsigned n_tokens, i; status = ucc_coll_score_alloc(&score); if (UCC_OK != status) { return status; } tokens = ucc_str_split(str, "#"); if (!tokens) { status = UCC_ERR_INVALID_PARAM; goto error; } n_tokens = ucc_str_split_count(tokens); for (i = 0; i < n_tokens; i++) { status = ucc_coll_score_parse_str(tokens[i], score, team_size, init, team, alg_fn); if (UCC_OK != status) { goto error_msg; } } ucc_str_split_free(tokens); *score_p = score; return UCC_OK; error_msg: ucc_error("failed to parse UCC_*_TUNE parameter: %s", tokens[i]); error: *score_p = NULL; ucc_coll_score_free(score); ucc_str_split_free(tokens); return status; } static ucc_status_t ucc_coll_score_update_one(ucc_list_link_t *dest, ucc_list_link_t *src, ucc_score_t default_score) { ucc_list_link_t *s = src->next; ucc_list_link_t *d = dest->next; ucc_msg_range_t *range, *tmp, *next, *rs, *rd, *new; ucc_coll_entry_t *fb; ucc_status_t status; if (ucc_list_is_empty(src) && ucc_list_is_empty(dest)) { return UCC_OK; } while (s != src && d != dest) { rs = ucc_container_of(s, ucc_msg_range_t, super.list_elem); rd = ucc_container_of(d, ucc_msg_range_t, super.list_elem); ucc_assert((NULL == rs->super.init) || (NULL != rs->super.team)); if (rd->start >= rs->end) { /* skip src range - no overlap */ s = s->next; if (rs->super.init) { new = MSG_RANGE_DUP(rs); if (new->super.score == UCC_SCORE_INVALID) { new->super.score = default_score; } ucc_list_insert_before(d, &new->super.list_elem); } } else if (rd->end <= rs->start) { /* no overlap - inverse case: skip dst range */ d = d->next; } else if (rd->start < rs->start) { new = MSG_RANGE_DUP(rd); new->end = rs->start; rd->start = rs->start; ucc_list_insert_before(d, &new->super.list_elem); } else if (rd->start > rs->start) { if (rs->super.init) { new = MSG_RANGE_DUP(rs); if (new->super.score == UCC_SCORE_INVALID) { new->super.score = default_score; } new->end = rd->start; ucc_list_insert_before(d, &new->super.list_elem); } rs->start = rd->start; } else { /* same start */ if (rs->end > rd->end) { if (UCC_SCORE_INVALID != rs->super.score) { rd->super.score = rs->super.score; } if (rs->super.init) { if (rs->super.init != rd->super.init) { /* User setting overrides existing init fn. Save it as a fallback */ FB_ALLOC_INSERT(&rd->super, fb, rd, status, out); } rd->super.init = rs->super.init; rd->super.team = rs->super.team; } rs->start = rd->end; d = d->next; } else if (rs->end < rd->end) { new = MSG_RANGE_DUP(rd); new->end = rs->end; if (UCC_SCORE_INVALID != rs->super.score) { new->super.score = rs->super.score; } if (rs->super.init) { if (rs->super.init != rd->super.init) { /* User setting overrides existing init fn. Save it as a fallback */ FB_ALLOC_INSERT(&rd->super, fb, new, status, out); } new->super.init = rs->super.init; new->super.team = rs->super.team; } ucc_list_insert_before(d, &new->super.list_elem); rd->start = rs->end; s = s->next; } else { if (UCC_SCORE_INVALID != rs->super.score) { rd->super.score = rs->super.score; } if (rs->super.init) { if (rs->super.init != rd->super.init) { /* User setting overrides existing init fn. Save it as a fallback */ FB_ALLOC_INSERT(&rd->super, fb, rd, status, out); } rd->super.init = rs->super.init; rd->super.team = rs->super.team; } s = s->next; d = d->next; } } } while (s != src) { rs = ucc_container_of(s, ucc_msg_range_t, super.list_elem); if (rs->super.init) { new = MSG_RANGE_DUP(rs); if (new->super.score == UCC_SCORE_INVALID) { new->super.score = default_score; } ucc_list_add_tail(dest, &new->super.list_elem); } s = s->next; } /* remove potentially disabled ranges */ ucc_list_for_each_safe(range, tmp, dest, super.list_elem) { if (0 == range->super.score) { ucc_list_del(&range->super.list_elem); ucc_msg_range_free(range); } } /* Merge consequtive ranges with the same score, same init fn, same team and same fallback sequence if any have been produced by the algorithm above */ ucc_list_for_each_safe(range, tmp, dest, super.list_elem) { //NOLINT if (range->super.list_elem.next != dest) { next = ucc_container_of(range->super.list_elem.next, ucc_msg_range_t, super.list_elem); //NOLINTNEXTLINE if (range->super.score == next->super.score && range->end == next->start && range->super.init == next->super.init && range->super.team == next->super.team && 1 == ucc_msg_range_fb_compare(range, next)) { next->start = range->start; ucc_list_del(&range->super.list_elem); ucc_msg_range_free(range); } } } return UCC_OK; out: return status; } ucc_status_t ucc_coll_score_update(ucc_coll_score_t *score, ucc_coll_score_t *update, ucc_score_t default_score, ucc_memory_type_t *mtypes, int mt_n, uint64_t colls) { ucc_status_t status; int i, j; ucc_memory_type_t mt; if (mt_n == 0) { mt_n = UCC_MEMORY_TYPE_LAST; } for (i = 0; i < UCC_COLL_TYPE_NUM; i++) { if (!(colls & UCS_BIT(i))) { continue; } for (j = 0; j < mt_n; j++) { mt = (mtypes == NULL) ? (ucc_memory_type_t)j : mtypes[j]; status = ucc_coll_score_update_one( &score->scores[i][mt], &update->scores[i][mt], default_score); if (UCC_OK != status) { return status; } } } return UCC_OK; } ucc_status_t ucc_coll_score_update_from_str(const char *str, const ucc_coll_score_team_info_t *info, ucc_base_team_t *team, ucc_coll_score_t *score) { ucc_status_t status; ucc_coll_score_t *score_str; status = ucc_coll_score_alloc_from_str(str, &score_str, info->size, info->init, team, info->alg_fn); if (UCC_OK != status) { return status; } status = ucc_coll_score_update(score, score_str, info->default_score, info->supported_mem_types, info->num_mem_types, info->supported_colls); ucc_coll_score_free(score_str); return status; } ucc_status_t ucc_coll_score_build_default(ucc_base_team_t *team, ucc_score_t default_score, ucc_base_coll_init_fn_t default_init, uint64_t coll_types, ucc_memory_type_t *mem_types, int mt_n, ucc_coll_score_t **score_p) { ucc_coll_score_t *score; ucc_status_t status; ucc_memory_type_t m; uint64_t c; ucc_coll_type_t ct; status = ucc_coll_score_alloc(&score); if (UCC_OK != status) { return status; } if (!mem_types) { mt_n = UCC_MEMORY_TYPE_LAST; } ucc_for_each_bit(c, coll_types) { for (m = UCC_MEMORY_TYPE_HOST; m < mt_n; m++) { ct = (ucc_coll_type_t)UCC_BIT(c); status = ucc_coll_score_add_range( score, ct, mem_types ? mem_types[m] : m, 0, UCC_MSG_MAX, default_score, default_init, team); if (UCC_OK != status) { ucc_coll_score_free(score); return status; } } } *score_p = score; return UCC_OK; } ucc_status_t ucc_coll_score_dup(const ucc_coll_score_t *in, ucc_coll_score_t ** out) { ucc_coll_score_t *score; ucc_status_t status; int i, j; status = ucc_coll_score_alloc(&score); if (UCC_OK != status) { return status; } for (i = 0; i < UCC_COLL_TYPE_NUM; i++) { for (j = 0; j < UCC_MEMORY_TYPE_LAST; j++) { status = ucc_score_list_dup(&in->scores[i][j], &score->scores[i][j]); if (UCC_OK != status) { return status; } } } *out = score; return status; } void ucc_coll_score_set(ucc_coll_score_t *score, ucc_score_t value) { int i, j; ucc_msg_range_t *range; for (i = 0; i < UCC_COLL_TYPE_NUM; i++) { for (j = 0; j < UCC_MEMORY_TYPE_LAST; j++) { ucc_list_for_each(range, &score->scores[i][j], super.list_elem) { range->super.score = value; } } } } ucc-1.8.0/src/coll_score/ucc_coll_score_map.c0000664000175000017500000002052015211535620021347 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_coll_score.h" #include "utils/ucc_coll_utils.h" #include "utils/ucc_string.h" #include "schedule/ucc_schedule.h" #include typedef struct ucc_score_map { ucc_coll_score_t *score; /* Size, rank of the process in the base_team associated with that score_map. It can be CL or TL team, which can be a subset of a core UCC team */ ucc_rank_t team_size; ucc_rank_t team_rank; } ucc_score_map_t; ucc_status_t ucc_coll_score_build_map(ucc_coll_score_t *score, ucc_score_map_t **map_p) { ucc_score_map_t *map; ucc_msg_range_t *range, *temp, *next; ucc_list_link_t *lst; int i, j; map = ucc_calloc(1, sizeof(*map), "ucc_score_map"); if (!map) { ucc_error("failed to allocate %zd bytes for score map", sizeof(*map)); return UCC_ERR_NO_MEMORY; } /* Resolve boundary between neighbour ranges: if ranges share a msg size as boundary leave that msgsize to the range with higher score. That way components that report higher scores do not get overwritten at range boundary */ for (i = 0; i < UCC_COLL_TYPE_NUM; i++) { for (j = 0; j < UCC_MEMORY_TYPE_LAST; j++) { lst = &score->scores[i][j]; if (!ucc_list_is_empty(lst) && map->team_size == 0) { /* For a given score_map all the entries refer to the base_teams (CL/TL) of the same size/rank. So we can take the first one. */ range = ucc_list_head(lst, ucc_msg_range_t, super.list_elem); map->team_size = range->super.team->params.size; map->team_rank = range->super.team->params.rank; } ucc_list_for_each_safe(range, temp, lst, super.list_elem) { if (range->super.list_elem.next != lst) { next = ucc_container_of(range->super.list_elem.next, ucc_msg_range_t, super.list_elem); if (range->end == next->start) { if (range->super.score > next->super.score) { next->start++; } else { range->end--; } } } } } } map->score = score; *map_p = map; return UCC_OK; } void ucc_coll_score_free_map(ucc_score_map_t *map) { ucc_coll_score_free(map->score); ucc_free(map); } static ucc_status_t ucc_coll_score_map_lookup(ucc_score_map_t *map, ucc_base_coll_args_t *bargs, ucc_msg_range_t **range) { ucc_memory_type_t mt = ucc_coll_args_mem_type(&bargs->args, map->team_rank); unsigned ct = ucc_ilog2(bargs->args.coll_type); size_t msgsize = ucc_coll_args_msgsize(&bargs->args, map->team_rank, map->team_size); ucc_list_link_t *list; ucc_msg_range_t *r; if (mt == UCC_MEMORY_TYPE_NOT_APPLY) { /* Temporary solution: for Barrier, Fanin, Fanout - use "host" range list */ mt = UCC_MEMORY_TYPE_HOST; } ucc_assert(ucc_coll_args_is_mem_symmetric(&bargs->args, map->team_rank)); if (msgsize == UCC_MSG_SIZE_INVALID || msgsize == UCC_MSG_SIZE_ASYMMETRIC) { /* These algorithms require global communication to get the same msgsize estimation. Can't use msg ranges. Use msize 0 (assuming the range list should only contain 1 range [0:inf]) */ msgsize = 0; } list = &map->score->scores[ct][mt]; ucc_list_for_each(r, list, super.list_elem) { if (msgsize >= r->start && msgsize <= r->end) { *range = r; return UCC_OK; } } return UCC_ERR_NOT_SUPPORTED; } ucc_status_t ucc_coll_init(ucc_score_map_t *map, ucc_base_coll_args_t *bargs, ucc_coll_task_t **task) { ucc_msg_range_t *r; ucc_coll_entry_t *fb; ucc_base_team_t *team; ucc_status_t status; status = ucc_coll_score_map_lookup(map, bargs, &r); if (ucc_unlikely(UCC_OK != status)) { ucc_debug("coll_score_map lookup failed %d (%s)", status, ucc_status_string(status)); return status; } team = r->super.team; status = r->super.init(bargs, team, task); if (UCC_OK == status) { return UCC_OK; } fb = ucc_list_head(&r->fallback, ucc_coll_entry_t, list_elem); while (&fb->list_elem != &r->fallback && (status == UCC_ERR_NOT_SUPPORTED || status == UCC_ERR_NOT_IMPLEMENTED)) { ucc_debug("coll %s is not supported for %s, fallback %s", ucc_coll_type_str(bargs->args.coll_type), team->context->lib->log_component.name, fb->team->context->lib->log_component.name); team = fb->team; status = fb->init(bargs, team, task); fb = ucc_list_next(&fb->list_elem, ucc_coll_entry_t, list_elem); } return status; } #define STR_APPEND(_str, _left, _tmp_size, _format, ...) { \ char _tmp[_tmp_size]; \ ucc_snprintf_safe(_tmp, _tmp_size, _format, ## __VA_ARGS__ ); \ strncat(_str, _tmp, _left - 1); \ _left = sizeof(_str) - strlen(_str); \ if (!_left) { \ return; \ } \ } static const char *get_fn_name(ucc_base_coll_init_fn_t init_fn) { int status; Dl_info info; const char *fn_ptr_str = "?"; status = dladdr(init_fn, &info); if (status && info.dli_sname != NULL) { fn_ptr_str = info.dli_sname; } return fn_ptr_str; } void ucc_coll_score_map_print_info(const ucc_score_map_t *map, int verbosity) { size_t left; ucc_msg_range_t *range; int i, j, all_empty; char score_str[32]; char range_str[128]; char coll_str[1024]; for (i = 0; i < UCC_COLL_TYPE_NUM; i++) { all_empty = 1; for (j = 0; j < UCC_MEMORY_TYPE_LAST; j++) { if (!ucc_list_is_empty(&map->score->scores[i][j])) { all_empty = 0; break; } } if (all_empty) { continue; } coll_str[0] = '\0'; left = sizeof(coll_str); STR_APPEND(coll_str, left, 32, "%s:\n", ucc_coll_type_str((ucc_coll_type_t)UCC_BIT(i))); for (j = 0; j < UCC_MEMORY_TYPE_LAST; j++) { if (ucc_list_is_empty(&map->score->scores[i][j])) { continue; } STR_APPEND(coll_str, left, 32, "\t%s: ", ucc_mem_type_str((ucc_memory_type_t)j)); ucc_list_for_each(range, &map->score->scores[i][j], super.list_elem) { ucc_memunits_range_str(range->start, range->end, range_str, sizeof(range_str)); ucc_score_to_str(range->super.score, score_str, sizeof(score_str)); if (verbosity >= UCC_LOG_LEVEL_DEBUG) { // Get the name of the init function through dladdr STR_APPEND(coll_str, left, 256, "{%s}:%s:%s=%s ", range_str, range->super.team->context->lib->log_component.name, score_str, get_fn_name(range->super.init)); } else { STR_APPEND(coll_str, left, 256, "{%s}:%s:%s ", range_str, range->super.team->context->lib->log_component.name, score_str); } } STR_APPEND(coll_str, left, 4, "\n"); } ucc_info("%s", coll_str); } } ucc-1.8.0/src/coll_score/ucc_coll_score.h0000664000175000017500000001724415211535620020530 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_COLL_SCORE_H_ #define UCC_COLL_SCORE_H_ #include "config.h" #include "utils/ucc_list.h" #include "components/base/ucc_base_iface.h" #include "utils/ucc_coll_utils.h" #include "utils/ucc_compiler_def.h" #include #define UCC_SCORE_MAX INT_MAX #define UCC_SCORE_MIN 0 #define UCC_SCORE_INVALID -1 #define UCC_MSG_MAX UINT64_MAX /* Callback that maps alg_id (int or str) to the "init" function. This callback is provided by the component (CL/TL) that uses ucc_coll_score_alloc_from_str. Return values: UCC_OK - input alg_id can be correctly mapped to the "init" fn UCC_ERR_NOT_SUPPORTED - CL/TL doesn't allow changing algorithms ids for the given coll_type, mem_type UCC_ERR_INVALID_PARAM - incorrect value of alg_id is provided */ typedef ucc_status_t (*ucc_alg_id_to_init_fn_t)(int alg_id, const char *alg_id_str, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, ucc_base_coll_init_fn_t *init); typedef struct ucc_coll_score_team_info { ucc_score_t default_score; ucc_rank_t size; uint64_t supported_colls; ucc_memory_type_t *supported_mem_types; int num_mem_types; ucc_base_coll_init_fn_t init; ucc_alg_id_to_init_fn_t alg_fn; } ucc_coll_score_team_info_t; typedef struct ucc_coll_entry { ucc_list_link_t list_elem; ucc_score_t score; ucc_base_coll_init_fn_t init; ucc_base_team_t *team; } ucc_coll_entry_t; typedef struct ucc_msg_range { ucc_coll_entry_t super; ucc_list_link_t fallback; size_t start; size_t end; } ucc_msg_range_t; typedef struct ucc_coll_score { ucc_list_link_t scores[UCC_COLL_TYPE_NUM][UCC_MEMORY_TYPE_LAST]; } ucc_coll_score_t; typedef struct ucc_score_map ucc_score_map_t; char *ucc_score_to_str(ucc_score_t score, char *buf, size_t max); /* Allocates empty score data structure */ ucc_status_t ucc_coll_score_alloc(ucc_coll_score_t **score); /* Adds a single score range to the storage. "init" must be either proper base_coll_init_fn or NULL. */ ucc_status_t ucc_coll_score_add_range(ucc_coll_score_t *score, ucc_coll_type_t coll_type, ucc_memory_type_t mem_type, size_t start, size_t end, ucc_score_t msg_score, ucc_base_coll_init_fn_t init, ucc_base_team_t *team); /* Releases the score data structure and all the score ranges stored there */ void ucc_coll_score_free(ucc_coll_score_t *score); /* Merges 2 scores score1 and score2 into the new score "rst" selecting larger score. Ie.: rst will contain a range from score1 if either score of that range in score1 is larger than that of score2 or that range does not overlap with score2. This fn is used by CL to merge scores from multiple TLs and produce a score map. As a result the produced score map will select TL with higher score.*/ ucc_status_t ucc_coll_score_merge(ucc_coll_score_t * score1, ucc_coll_score_t * score2, ucc_coll_score_t **rst, int free_inputs); /* Parses SCORE string (see ucc_base_iface.c for pattern description) and initializes score data structure. team_size is used to filter score ranges provided by user. "init" - default init function to be used if alg_id is not explicitly present in SCORE str. */ ucc_status_t ucc_coll_score_alloc_from_str(const char * str, ucc_coll_score_t ** score, ucc_rank_t team_size, ucc_base_coll_init_fn_t init, ucc_base_team_t * team, ucc_alg_id_to_init_fn_t alg_fn); /* Update existing score datastructure with the custom input specified in "str". Update applies the modifications specified in "str" to the existing "score": if some range in "str" overlaps with a range in "score" then the latter is modified according to "str" (in contrast to ucc_coll_score_merge where MAX score rule is used). If the new range is provided in "str" and it does not have "score" qualifier then def_score is used for it. If the new range is provided in "str" and it does not have "alg_id" qualifier than "init" fn is used otherwise "init" is taken from alg_fn mapper callback. "mtypes" parameter determines which memory types will be udpated. This function has 2 usages (see tl_ucp_team.c: ucc_tl_ucp_team_get_scores function): 1. Construct custom score table for component based on the built-in selection rules represented by string. In this case "init"" can be set to some generic init function (ucc_tl_ucp_coll_init). 2. Update existing score datastruct with user input: in this case "init" is set to NULL. User provided ranges without alg_id will not modify any existing "init" functions in that case and only change the score of existing ranges*/ ucc_status_t ucc_coll_score_update_from_str(const char *str, const ucc_coll_score_team_info_t *info, ucc_base_team_t *team, ucc_coll_score_t *score); ucc_status_t ucc_coll_score_merge_in(ucc_coll_score_t **dst, ucc_coll_score_t *src); /* Initializes the default score datastruct with a set of coll_types specified as a bitmap, mem_types passed as array, default score value and default init fn. The collective will have msg range 0-inf. */ ucc_status_t ucc_coll_score_build_default(ucc_base_team_t *team, ucc_score_t default_score, ucc_base_coll_init_fn_t default_init, uint64_t coll_types, ucc_memory_type_t *mem_types, int mt_n, ucc_coll_score_t **score_p); /* Builds optimized representation of a score for the faster lookup */ ucc_status_t ucc_coll_score_build_map(ucc_coll_score_t *score, ucc_score_map_t **map); void ucc_coll_score_free_map(ucc_score_map_t *map); /* Initializes task based on args selection and score map. Checks fallbacks if necessary. */ ucc_status_t ucc_coll_init(ucc_score_map_t *map, ucc_base_coll_args_t *bargs, ucc_coll_task_t **task); ucc_status_t ucc_coll_score_dup(const ucc_coll_score_t *in, ucc_coll_score_t **out); void ucc_coll_score_set(ucc_coll_score_t *score, ucc_score_t value); void ucc_coll_score_map_print_info(const ucc_score_map_t *score, int verbosity); ucc_status_t ucc_coll_score_update(ucc_coll_score_t *score, ucc_coll_score_t *update, ucc_score_t default_score, ucc_memory_type_t *mtypes, int mt_n, uint64_t colls); #endif ucc-1.8.0/src/ucc/0000775000175000017500000000000015211535620014017 5ustar alastairalastairucc-1.8.0/src/ucc/api/0000775000175000017500000000000015211535620014570 5ustar alastairalastairucc-1.8.0/src/ucc/api/ucc_status.h0000664000175000017500000000267515211535620017130 0ustar alastairalastair/** * @file ucc_status.h * @date 2020 * @copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_STATUS_H_ #define UCC_STATUS_H_ #ifdef __cplusplus # define BEGIN_C_DECLS extern "C" { # define END_C_DECLS } #else # define BEGIN_C_DECLS # define END_C_DECLS #endif BEGIN_C_DECLS /** * @ingroup UCC_UTILS * @brief Status codes for the UCC operations */ typedef enum { /* Operation completed successfully */ UCC_OK = 0, UCC_INPROGRESS = 1, /*!< Operation is posted and is in progress */ UCC_OPERATION_INITIALIZED = 2, /*!< Operation initialized but not posted */ /* Error status codes */ UCC_ERR_NOT_SUPPORTED = -1, UCC_ERR_NOT_IMPLEMENTED = -2, UCC_ERR_INVALID_PARAM = -3, UCC_ERR_NO_MEMORY = -4, UCC_ERR_NO_RESOURCE = -5, UCC_ERR_NO_MESSAGE = -6, /*!< General purpose return code without specific error */ UCC_ERR_NOT_FOUND = -7, UCC_ERR_TIMED_OUT = -8, UCC_ERR_IO_ERROR = -9, UCC_ERR_LAST = -100, } ucc_status_t; /** * @ingroup UCC_UTILS * @brief Routine to convert status code to string */ const char *ucc_status_string(ucc_status_t status); END_C_DECLS #endif ucc-1.8.0/src/ucc/api/ucc.h0000664000175000017500000024266215211535620015527 0ustar alastairalastair/** * @file ucc.h * @date 2020 * @copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * @copyright Copyright (C) Huawei Technologies Co., Ltd. 2020. ALL RIGHTS RESERVED. * @copyright Copyright (C) UChicago Argonne, LLC. 2022. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_H_ #define UCC_H_ #include #include #include #include BEGIN_C_DECLS /** Unified Collective Communications (UCC) Library Specification * * UCC is a collective communication operations API and library that is * flexible, complete, and feature-rich for current and emerging programming * models and runtimes. * * */ /** * @defgroup UCC_LIB_INIT_DT Library initialization data-structures * @{ * Library initialization parameters and data-structures * @} * */ /** * @defgroup UCC_DATATYPE Datatypes data-structures and functions * @{ * Datatypes data-structures and functions * @} * */ /** * @defgroup UCC_LIB Library initialization and finalization routines * @{ * Library initialization and finalization routines * @} */ /** * @defgroup UCC_LIB_INTERNAL Internal library routines * @{ * Internal library routines * @} */ /** * @defgroup UCC_CONTEXT_DT Context abstraction data-structures * @{ * Data-structures associated with context creation and management routines * @} */ /** * @defgroup UCC_CONTEXT Context abstraction routines * @{ * Context create and management routines * @} */ /** * @defgroup UCC_TEAM_DT Team abstraction data-structures * @{ * Data-structures associated with team create and management routines * @} */ /** * @defgroup UCC_TEAM Team abstraction routines * @{ * Team create and management routines * @} */ /** * @defgroup UCC_COLLECTIVES_DT Collective operations data-structures * @{ * Data-structures associated with collective operation creation, progress, and * finalize. * @} */ /** * @defgroup UCC_COLLECTIVES Collective Operations * @{ * Collective operations invocation and progress * @} */ /** * @defgroup UCC_EVENT_DT Events and Triggered operations' data-structures * @{ * Data-structures associated with event-driven collective execution * @} */ /** * @defgroup UCC_EVENT Events and Triggered Operations * @{ * Event-driven Collective Execution * @} */ /** * @defgroup UCC_UTILS Utility Operations * @{ * Helper functions to be used across the library * @} */ /** * ************************************************************* * Library initialization and finalize * ************************************************************* */ /** * * @ingroup UCC_LIB_INIT_DT * * @brief Enumeration representing the collective operations * * @parblock * * Description * * @ref ucc_coll_type_t represents the collective operations supported by the * UCC library. The exact set of supported collective operations depends on * UCC build flags, runtime configuration and available communication transports. * * @endparblock * */ typedef enum { UCC_COLL_TYPE_ALLGATHER = UCC_BIT(0), UCC_COLL_TYPE_ALLGATHERV = UCC_BIT(1), UCC_COLL_TYPE_ALLREDUCE = UCC_BIT(2), UCC_COLL_TYPE_ALLTOALL = UCC_BIT(3), UCC_COLL_TYPE_ALLTOALLV = UCC_BIT(4), UCC_COLL_TYPE_BARRIER = UCC_BIT(5), UCC_COLL_TYPE_BCAST = UCC_BIT(6), UCC_COLL_TYPE_FANIN = UCC_BIT(7), UCC_COLL_TYPE_FANOUT = UCC_BIT(8), UCC_COLL_TYPE_GATHER = UCC_BIT(9), UCC_COLL_TYPE_GATHERV = UCC_BIT(10), UCC_COLL_TYPE_REDUCE = UCC_BIT(11), UCC_COLL_TYPE_REDUCE_SCATTER = UCC_BIT(12), UCC_COLL_TYPE_REDUCE_SCATTERV = UCC_BIT(13), UCC_COLL_TYPE_SCATTER = UCC_BIT(14), UCC_COLL_TYPE_SCATTERV = UCC_BIT(15), UCC_COLL_TYPE_LAST } ucc_coll_type_t; /** * @ingroup UCC_COLLECTIVES_DT */ typedef enum ucc_memory_type { UCC_MEMORY_TYPE_HOST, /*!< Default system memory */ UCC_MEMORY_TYPE_CUDA, /*!< NVIDIA CUDA memory */ UCC_MEMORY_TYPE_CUDA_MANAGED, /*!< NVIDIA CUDA managed memory */ UCC_MEMORY_TYPE_ROCM, /*!< AMD ROCM memory */ UCC_MEMORY_TYPE_ROCM_MANAGED, /*!< AMD ROCM managed system memory */ UCC_MEMORY_TYPE_LAST, UCC_MEMORY_TYPE_UNKNOWN = UCC_MEMORY_TYPE_LAST } ucc_memory_type_t; /** * * @ingroup UCC_DATATYPE * * @brief Enumeration representing the UCC library's datatype * * @parblock * * Description * * @ref ucc_datatype_t represents the datatypes supported by the UCC library’s * collective and reduction operations. The predefined operations * are signed and unsigned integers of various sizes, float 16, 32, and 64, and * user-defined datatypes. User-defined datatypes are created using * @ref ucc_dt_create_generic interface and can support user-defined reduction * operations. Predefined reduction operations can be used only with * predefined datatypes. * * @endparblock * */ typedef uint64_t ucc_datatype_t; #define UCC_DT_INT8 UCC_PREDEFINED_DT(0) #define UCC_DT_INT16 UCC_PREDEFINED_DT(1) #define UCC_DT_INT32 UCC_PREDEFINED_DT(2) #define UCC_DT_INT64 UCC_PREDEFINED_DT(3) #define UCC_DT_INT128 UCC_PREDEFINED_DT(4) #define UCC_DT_UINT8 UCC_PREDEFINED_DT(5) #define UCC_DT_UINT16 UCC_PREDEFINED_DT(6) #define UCC_DT_UINT32 UCC_PREDEFINED_DT(7) #define UCC_DT_UINT64 UCC_PREDEFINED_DT(8) #define UCC_DT_UINT128 UCC_PREDEFINED_DT(9) #define UCC_DT_FLOAT16 UCC_PREDEFINED_DT(10) #define UCC_DT_FLOAT32 UCC_PREDEFINED_DT(11) #define UCC_DT_FLOAT64 UCC_PREDEFINED_DT(12) #define UCC_DT_BFLOAT16 UCC_PREDEFINED_DT(13) #define UCC_DT_FLOAT128 UCC_PREDEFINED_DT(14) #define UCC_DT_FLOAT32_COMPLEX UCC_PREDEFINED_DT(15) #define UCC_DT_FLOAT64_COMPLEX UCC_PREDEFINED_DT(16) #define UCC_DT_FLOAT128_COMPLEX UCC_PREDEFINED_DT(17) #define UCC_DT_PREDEFINED_LAST 18 /** * @ingroup UCC_DATATYPE */ enum ucc_generic_dt_ops_field { UCC_GENERIC_DT_OPS_FIELD_FLAGS = UCC_BIT(0), }; /** * @ingroup UCC_DATATYPE * @brief Flags that can be specified for generic datatype * */ typedef enum { UCC_GENERIC_DT_OPS_FLAG_CONTIG = UCC_BIT(0), /*!< If set, the created datatype represents a contiguous memory region with the size specified in @ref ucc_generic_dt_ops.contig_size field of @ref ucc_generic_dt_ops */ UCC_GENERIC_DT_OPS_FLAG_REDUCE = UCC_BIT(1), /*!< If set, the created datatype has user-defined reduction operation associated with it. reduce.cb and reduce.ctx fields of @ref ucc_generic_dt_ops must be initialized. Collective operations that involve reduction (allreduce, reduce, reduce_scatter/v) can use user-defined data-types only when this flag is set. */ } ucc_generic_dt_ops_flags_t; /** * @ingroup UCC_DATATYPE * @brief Descriptor of user-defined reduction callback * * This structure is the argument to the reduce.cb callback. It must implement * the reduction of n_vectors + 1 data vectors each containing "count" elements. * First vector is "src1", other n_vectors have start address * v_j = src2 + count * dt_extent * stride * j. * The result is stored in dst, so that * dst[i] = src1[i] + v0[i] + v1[i] + ... +v_nvectors[i], * for i in [0:count), where "+" represents user-defined reduction of 2 elements */ typedef struct ucc_reduce_cb_params { uint64_t mask; /*< for backward compatibility. currently ignored. */ void *src1; /*< input buffer */ void *src2; /*< input buffer: represents n_vectors buffers with offset "stride" between them */ void *dst; /*< destination buffer */ size_t n_vectors; /*< number of vectors from src2 to reduce */ size_t count; /*< number of elements in one vector */ size_t stride; /*< stride in bytes between the vectors in src2 */ ucc_dt_generic_t *dt; /*< pointer to user-defined datatype used for reduction */ void *cb_ctx; /*< user-defined context as defined by @ref ucc_generic_dt_ops::reduce.cb_ctx */ } ucc_reduce_cb_params_t; /** * @ingroup UCC_DATATYPE * @brief UCC generic data type descriptor * * This structure provides a generic datatype descriptor that is used to create * user-defined datatypes. */ typedef struct ucc_generic_dt_ops { uint64_t mask; uint64_t flags; size_t contig_size; /*!< size of the datatype if @ref UCC_GENERIC_DT_OPS_FLAG_CONTIG is set */ /** * @ingroup UCC_DATATYPE * @brief Start a packing request. * * The pointer refers to application defined start-to-pack routine. * * @param [in] context User-defined context. * @param [in] buffer Buffer to pack. * @param [in] count Number of elements to pack into the buffer. * * @return A custom state that is passed to the subsequent * @ref ucc_generic_dt_ops::pack "pack()" routine. */ void* (*start_pack)(void *context, const void *buffer, size_t count); /** * @ingroup UCC_DATATYPE * @brief Start an unpacking request. * * The pointer refers to application defined start-to-unpack routine. * * @param [in] context User-defined context. * @param [in] buffer Buffer to unpack to. * @param [in] count Number of elements to unpack in the buffer. * * @return A custom state that is passed later to the subsequent * @ref ucc_generic_dt_ops::unpack "unpack()" routine. */ void* (*start_unpack)(void *context, void *buffer, size_t count); /** * @ingroup UCC_DATATYPE * @brief Get the total size of packed data. * * The pointer refers to user defined routine that returns the size of data * in a packed format. * * @param [in] state State as returned by * @ref ucc_generic_dt_ops::start_pack * "start_pack()" routine. * * @return The size of the data in a packed form. */ size_t (*packed_size)(void *state); /** * @ingroup UCC_DATATYPE * @brief Pack data. * * The pointer refers to application defined pack routine. * * @param [in] state State as returned by * @ref ucc_generic_dt_ops::start_pack * "start_pack()" routine. * @param [in] offset Virtual offset in the output stream. * @param [in] dest Destination buffer to pack the data. * @param [in] max_length Maximum length to pack. * * @return The size of the data that was written to the destination buffer. * Must be less than or equal to @e max_length. */ size_t (*pack) (void *state, size_t offset, void *dest, size_t max_length); /** * @ingroup UCC_DATATYPE * @brief Unpack data. * * The pointer refers to application defined unpack routine. * * @param [in] state State as returned by * @ref ucc_generic_dt_ops::start_unpack * "start_unpack()" routine. * @param [in] offset Virtual offset in the input stream. * @param [in] src Source to unpack the data from. * @param [in] length Length to unpack. * * @return UCC_OK or an error if unpacking failed. */ ucc_status_t (*unpack)(void *state, size_t offset, const void *src, size_t length); /** * @ingroup UCC_DATATYPE * @brief Finish packing/unpacking. * * The pointer refers to application defined finish routine. * * @param [in] state State as returned by * @ref ucc_generic_dt_ops::start_pack * "start_pack()" * and * @ref ucc_generic_dt_ops::start_unpack * "start_unpack()" * routines. */ void (*finish)(void *state); /** * @ingroup UCC_DATATYPE * @brief User-defined reduction callback * * The pointer refers to user-defined reduction routine. * * @param [in] params reduction descriptor */ struct { ucc_status_t (*cb)(const ucc_reduce_cb_params_t *params); void *cb_ctx; } reduce; } ucc_generic_dt_ops_t; /** * @ingroup UCC_DATATYPE * @brief Create a generic datatype. * * This routine creates a generic datatype object. The generic datatype is * described by the @a ops @ref ucc_generic_dt_ops_t "object" which provides * a table of routines defining the operations for generic datatype manipulation. * Typically, generic datatypes are used for integration with datatype engines * provided with MPI implementations (MPICH, Open MPI, etc). The application * is responsible for releasing the @a datatype_p object using * @ref ucc_dt_destroy "ucc_dt_destroy()" routine. * * @param [in] ops Generic datatype function table as defined by * @ref ucc_generic_dt_ops_t . * @param [in] context Application defined context passed to this * routine. The context is passed as a parameter * to the routines in the @a ops table. * @param [out] datatype_p A pointer to datatype object. * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_dt_create_generic(const ucc_generic_dt_ops_t *ops, void *context, ucc_datatype_t *datatype_p); /** * @ingroup UCC_DATATYPE * @brief Destroy generic datatype */ void ucc_dt_destroy(ucc_datatype_t datatype); /** * * @ingroup UCC_LIB_INIT_DT * * @brief Enumeration representing the UCC reduction operations * * @parblock * * * Description * * @ref ucc_reduction_op_t represents the UCC reduction operations. It is used by the * library initialization routine @ref ucc_init to request the operations expected by the user. * It is used by the @ref ucc_lib_attr_t to communicate the operations supported by * the library. * * @endparblock * */ typedef enum { UCC_OP_SUM, UCC_OP_PROD, UCC_OP_MAX, UCC_OP_MIN, UCC_OP_LAND, UCC_OP_LOR, UCC_OP_LXOR, UCC_OP_BAND, UCC_OP_BOR, UCC_OP_BXOR, UCC_OP_MAXLOC, UCC_OP_MINLOC, UCC_OP_AVG, UCC_OP_LAST } ucc_reduction_op_t; /** * * @ingroup UCC_LIB_INIT_DT * * @brief Enumeration representing the UCC library's thread model * * @parblock * * Description * * @ref ucc_thread_mode_t is used to initialize the UCC library’s thread mode. * The UCC library can be configured in three thread modes UCC_THREAD_SINGLE, * UCC_THREAD_FUNNELED, and UCC_THREAD_MULTIPLE. In the UCC_THREAD_SINGLE * mode, the user program must not be multithreaded. In the UCC_THREAD_FUNNELED * mode, the user program may be multithreaded. However, all UCC interfaces * should be invoked from the same thread. In the UCC_THREAD_MULTIPLE mode, the * user program can be multithreaded and any thread may invoke the UCC * operations. * * @endparblock * */ typedef enum { UCC_THREAD_SINGLE = 0, /*!< Single-threaded library model */ UCC_THREAD_FUNNELED = 1, /*!< Funnel thread model */ UCC_THREAD_MULTIPLE = 2 /*!< Multithread library model */ } ucc_thread_mode_t; /** * * @ingroup UCC_LIB_INIT_DT * * @brief Enumeration representing the collective synchronization model * * @parblock * * Description * * @ref ucc_coll_sync_type_t represents the collective synchronization models. * Currently, it supports two synchronization models synchronous and * non-synchronous collective models. In the synchronous collective model, the * collective communication is not started until participants have not entered * the collective operation, and it is not completed until all participants have not * completed the collective. In the non-synchronous collective model, collective * communication can be started as soon as the participant enters the collective * operation and is completed as soon as it completes locally. * * @endparblock * */ typedef enum { UCC_NO_SYNC_COLLECTIVES = 0, /*!< Non-synchronous collectives */ UCC_SYNC_COLLECTIVES = 1 /*!< Synchronous collectives */ } ucc_coll_sync_type_t; /** * @brief UCC library initialization parameters */ /** * * @ingroup UCC_LIB_INIT_DT */ enum ucc_lib_params_field{ UCC_LIB_PARAM_FIELD_THREAD_MODE = UCC_BIT(0), UCC_LIB_PARAM_FIELD_COLL_TYPES = UCC_BIT(1), UCC_LIB_PARAM_FIELD_REDUCTION_TYPES = UCC_BIT(2), UCC_LIB_PARAM_FIELD_SYNC_TYPE = UCC_BIT(3) }; /** * * @ingroup UCC_LIB_INIT_DT */ enum ucc_lib_attr_field{ UCC_LIB_ATTR_FIELD_THREAD_MODE = UCC_BIT(0), UCC_LIB_ATTR_FIELD_COLL_TYPES = UCC_BIT(1), UCC_LIB_ATTR_FIELD_REDUCTION_TYPES = UCC_BIT(2), UCC_LIB_ATTR_FIELD_SYNC_TYPE = UCC_BIT(3) }; /** * * @ingroup UCC_LIB_INIT_DT * * @brief Structure representing the parameters to customize the library * * @parblock * * Description * * @ref ucc_lib_params_t defines the parameters that can be used to customize * the library. The bits in "mask" bit array is defined by @ref * ucc_lib_params_field, which correspond to fields in structure @ref * ucc_lib_params_t. The valid fields of the structure is specified by the * setting the bit to "1" in the bit-array "mask". When bits corresponding to * the fields is not set, the fields are not defined. * * @endparblock * */ typedef struct ucc_lib_params { uint64_t mask; ucc_thread_mode_t thread_mode; uint64_t coll_types; uint64_t reduction_types; ucc_coll_sync_type_t sync_type; } ucc_lib_params_t; /** * * @ingroup UCC_LIB_INIT_DT * * @brief Structure representing the attributes of the library * * @parblock * * Description * * @ref ucc_lib_attr_t defines the attributes of the library. The bits in * "mask" bit array is defined by @ref ucc_lib_attr_field, which correspond to * fields in structure @ref ucc_lib_attr_t. The valid fields of the structure * is specified by the setting the bit to "1" in the bit-array "mask". When * bits corresponding to the fields is not set, the fields are not defined. * * @endparblock * */ typedef struct ucc_lib_attr { uint64_t mask; ucc_thread_mode_t thread_mode; uint64_t coll_types; uint64_t reduction_types; ucc_coll_sync_type_t sync_type; } ucc_lib_attr_t; /** * @ingroup UCC_LIB * * @brief The @ref ucc_lib_config_read routine provides a method to read library * configuration from the environment and create configuration descriptor. * * @param [out] env_prefix If not NULL, the routine searches for the * environment variables with the prefix UCC_. Otherwise, the * routines search for the environment variables that start with the prefix @ * UCC_. * @param [in] filename If not NULL, read configuration values from the * file defined by @e filename. If the file does not exist, it will be ignored * and no error will be reported to the user. * @param [out] config Pointer to configuration descriptor as defined by * ucc_lib_config_h. * * @parblock * * @b Description * * @ref ucc_lib_config_read allocates the @ref ucc_lib_config_h handle and * fetches the configuration values from the run-time environment. The run-time * environment supported are environment variables or a configuration file. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_lib_config_read(const char *env_prefix, const char *filename, ucc_lib_config_h *config); /** * @ingroup UCC_LIB * * @brief The @ref ucc_lib_config_release routine releases the configuration descriptor * * @param [in] config Pointer to the configuration descriptor to be released. * Configuration descriptor as defined by @ref * ucc_lib_config_h. * * @parblock * * @b Description * * The routine releases the configuration descriptor that was allocated through * * @endparblock * * @ref ucc_lib_config_read "ucc_lib_config_read()" routine. * */ void ucc_lib_config_release(ucc_lib_config_h config); /** * @ingroup UCC_LIB * * @brief The @ref ucc_lib_config_print routine prints the configuration information * * @param [in] config ucc_lib_config_h "Configuration descriptor" * to print. * @param [in] stream Output stream to print the configuration to. * @param [in] title Configuration title to print. * @param [in] print_flags Flags that control various printing options. * * @parblock * * @b Description * * The routine prints the configuration information that is stored in * ucc_lib_config_h "configuration" descriptor. * * @endparblock * */ void ucc_lib_config_print(const ucc_lib_config_h config, FILE *stream, const char *title, ucc_config_print_flags_t print_flags); /** * @ingroup UCC_LIB * * @brief The @ref ucc_lib_config_modify routine modifies the runtime configuration as described by the * descriptor. * * @param [in] config Pointer to the configuration descriptor to be modified * @param [in] name Configuration variable to be modified * @param [in] value Configuration value to set * * @parblock * * @b Description * * The @ref ucc_lib_config_modify routine sets the value of identifier "name" * to "value". * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_lib_config_modify(ucc_lib_config_h config, const char *name, const char *value); /** * @ingroup UCC_LIB * @brief Get UCC library version. * * This routine returns the UCC library version. * * @param [out] major_version Filled with library major version. * @param [out] minor_version Filled with library minor version. * @param [out] release_number Filled with library release number. */ void ucc_get_version(unsigned *major_version, unsigned *minor_version, unsigned *release_number); /** * @ingroup UCC_LIB * @brief Get UCC library version as a string. * * This routine returns the UCC library version as a string which consists of: * "major.minor.release". */ const char *ucc_get_version_string(void); /** * @ingroup UCC_LIB_INTERNAL * * @brief The @ref ucc_init_version is an internal routine that checks * compatibility with a particular UCC API version. * @ref ucc_init should be used to create the UCC library handle. */ ucc_status_t ucc_init_version(unsigned api_major_version, unsigned api_minor_version, const ucc_lib_params_t *params, const ucc_lib_config_h config, ucc_lib_h *lib_p); /** * @ingroup UCC_LIB * * @brief The @ref ucc_init initializes the UCC library. * * @param [in] params User provided parameters to customize the library functionality * @param [in] config UCC configuration descriptor allocated through * @ref ucc_lib_config_read "ucc_config_read()" routine. * @param [out] lib_p UCC library handle * * @parblock * * @b Description * * A local operation to initialize and allocate the resources for the UCC * operations. The parameters passed using the ucc_lib_params_t and * @ref ucc_lib_config_h structures will customize and select the functionality of the * UCC library. The library can be customized for its interaction with the user * threads, types of collective operations, and reductions supported. * On success, the library object will be created and ucc_status_t will return * UCC_OK. On error, the library object will not be created and corresponding * error code as defined by @ref ucc_status_t is returned. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ static inline ucc_status_t ucc_init(const ucc_lib_params_t *params, const ucc_lib_config_h config, ucc_lib_h *lib_p) { return ucc_init_version(UCC_API_MAJOR, UCC_API_MINOR, params, config, lib_p); } /** * @ingroup UCC_LIB * * @brief The @ref ucc_finalize routine finalizes the UCC library. * * @param [in] lib_p Handle to ucc_lib_h * "UCC library". * * @parblock * * @b Description * * A local operation to release the resources and * cleanup. All participants that invoked @ref ucc_init should call this * routine. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_finalize(ucc_lib_h lib_p); /** * @ingroup UCC_LIB * * @brief The @ref ucc_lib_get_attr routine queries the library attributes. * * @param [out] lib_attr Library attributes * @param [in] lib_p Input library object * * @parblock * * @b Description * * A query operation to get the attributes of the library object. The * attributes are library configured values and reflect the choices made by the * library implementation. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_lib_get_attr(ucc_lib_h lib_p, ucc_lib_attr_t *lib_attr); /* * ************************************************************* * Context Section * ************************************************************* */ /** * * @ingroup UCC_CONTEXT_DT */ typedef enum { UCC_CONTEXT_EXCLUSIVE = 0 , UCC_CONTEXT_SHARED } ucc_context_type_t; /** * * @ingroup UCC_CONTEXT_DT */ enum ucc_context_params_field { UCC_CONTEXT_PARAM_FIELD_TYPE = UCC_BIT(0), UCC_CONTEXT_PARAM_FIELD_SYNC_TYPE = UCC_BIT(1), UCC_CONTEXT_PARAM_FIELD_OOB = UCC_BIT(2), UCC_CONTEXT_PARAM_FIELD_ID = UCC_BIT(3), UCC_CONTEXT_PARAM_FIELD_MEM_PARAMS = UCC_BIT(4) }; /** * * @ingroup UCC_CONTEXT_DT */ enum ucc_context_attr_field { UCC_CONTEXT_ATTR_FIELD_TYPE = UCC_BIT(0), UCC_CONTEXT_ATTR_FIELD_SYNC_TYPE = UCC_BIT(1), UCC_CONTEXT_ATTR_FIELD_CTX_ADDR = UCC_BIT(2), UCC_CONTEXT_ATTR_FIELD_CTX_ADDR_LEN = UCC_BIT(3), UCC_CONTEXT_ATTR_FIELD_WORK_BUFFER_SIZE = UCC_BIT(4) }; /** * @ingroup UCC_CONTEXT_DT * * @brief OOB collective operation for creating the context */ typedef struct ucc_oob_coll { ucc_status_t (*allgather)(void *src_buf, void *recv_buf, size_t size, void *allgather_info, void **request); ucc_status_t (*req_test)(void *request); ucc_status_t (*req_free)(void *request); void *coll_info; uint32_t n_oob_eps; /*!< Number of endpoints participating in the oob operation (e.g., number of processes representing a ucc team) */ uint32_t oob_ep; /*!< Integer value that represents the position of the calling processes in the given oob op: the data specified by "src_buf" will be placed at the offset "oob_ep*size" in the "recv_buf". oob_ep must be uniq at every calling process and should be in the range [0:n_oob_eps). */ } ucc_oob_coll_t; typedef ucc_oob_coll_t ucc_context_oob_coll_t; typedef ucc_oob_coll_t ucc_team_oob_coll_t; /** * * @ingroup UCC_CONTEXT_DT */ typedef struct ucc_mem_map { void * address; /*!< the address of a buffer to be attached to a UCC context */ size_t len; /*!< the length of the buffer */ } ucc_mem_map_t; /** * * @ingroup UCC_CONTEXT_DT */ typedef struct ucc_mem_map_params { ucc_mem_map_t *segments; /*!< array of ucc_mem_map elements */ uint64_t n_segments; /*!< the number of ucc_mem_map elements */ } ucc_mem_map_params_t; /** * * @ingroup UCC_CONTEXT_DT * * @brief Structure representing the parameters to customize the context * * @parblock * * Description * * @ref ucc_context_params_t defines the parameters that can be used to * customize the context. The "mask" bit array fields are defined by @ref * ucc_context_params_field. The bits in "mask" bit array is defined by @ref * ucc_context_params_field, which correspond to fields in structure @ref * ucc_context_params_t. The valid fields of the structure is specified by the * setting the bit to "1" in the bit-array "mask". When bits corresponding to * the fields is not set, the fields are not defined. * * * @endparblock * */ typedef struct ucc_context_params { uint64_t mask; ucc_context_type_t type; ucc_coll_sync_type_t sync_type; ucc_context_oob_coll_t oob; uint64_t ctx_id; ucc_mem_map_params_t mem_params; } ucc_context_params_t; /** * * @ingroup UCC_CONTEXT_DT * * @brief Structure representing context attributes * * @parblock * * Description * * @ref ucc_context_attr_t defines the attributes of the context. The bits in * "mask" bit array is defined by @ref ucc_context_attr_field, which correspond to * fields in structure @ref ucc_context_attr_t. The valid fields of the structure * is specified by the setting the bit to "1" in the bit-array "mask". When * bits corresponding to the fields is not set, the fields are not defined. * * @endparblock * */ typedef struct ucc_context_attr { uint64_t mask; ucc_context_type_t type; ucc_coll_sync_type_t sync_type; ucc_context_addr_h ctx_addr; ucc_context_addr_len_t ctx_addr_len; uint64_t global_work_buffer_size; } ucc_context_attr_t; /** * @ingroup UCC_CONTEXT * * @brief Routine reads the configuration information for contexts from the * runtime enviornment and creates the configuration descriptor. * * @param [in] lib_handle Library handle * @param [in] filename If not NULL, read configuration values from the * file defined by @e filename. If the file does not exist, * it will be ignored and no error will be reported to the user. * @param [out] config Pointer to configuration descriptor as defined by * @ref ucc_context_config_h. * * @parblock * * @b Description * * @ref ucc_context_config_read allocates the @ref ucc_lib_config_h handle and * fetches the configuration values from the run-time environment. The run-time * environment supported are environment variables or a configuration file. It uses * the env_prefix from @ref ucc_lib_config_read. If env_prefix is not NULL, the routine * searches for the environment variables with the prefix UCC_. Otherwise, the * routines search for the environment variables that start with the prefix @ UCC_. * * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_context_config_read(ucc_lib_h lib_handle, const char *filename, ucc_context_config_h *config); /** * @ingroup UCC_CONTEXT * * @brief The @ref ucc_context_config_release routine releases the configuration descriptor. * * @param [in] config Pointer to the configuration descriptor to be released. * Configuration descriptor as defined by @ref ucc_context_config_h * * @parblock * * @b Description * * The routine releases the configuration descriptor that was allocated through * @ref ucc_context_config_read "ucc_context_config_read()" routine. * * @endparblock * */ void ucc_context_config_release(ucc_context_config_h config); /** * @ingroup UCC_CONTEXT * * @brief The @ref ucc_context_config_print routine prints the configuration information * * @param [in] config ucc_context_config_h "Configuration descriptor" * to print. * @param [in] stream Output stream to print the configuration to. * @param [in] title Configuration title to print. * @param [in] print_flags Flags that control various printing options. * * @parblock * * @b Description * * The routine prints the configuration information that is stored in * ucc_context_config_h "configuration" descriptor. * * @endparblock * */ void ucc_context_config_print(const ucc_context_config_h config, FILE *stream, const char *title, ucc_config_print_flags_t print_flags); /** * @ingroup UCC_CONTEXT * * @brief The @ref ucc_context_config_modify routine modifies the runtime configuration * of UCC context (optionally for a given CLS) * * @param [in] config Pointer to the configuration descriptor to be modified * @param [in] component CL/TL component (e.g. "tl/ucp" or "cl/basic") or NULL. If NULL then core context config is modified. * @param [in] name Configuration variable to be modified * @param [in] value Configuration value to set * * @parblock * * @b Description * * The @ref ucc_context_config_modify routine sets the value of identifier "name" * to "value" for a specified CL. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_context_config_modify(ucc_context_config_h config, const char *component, const char *name, const char *value); /** * @ingroup UCC_CONTEXT * * @brief The @ref ucc_context_create routine creates the context handle. * * @param [in] lib_handle Library handle * @param [in] params Customizations for the communication context * @param [in] config Configuration for the communication context to read * from environment * @param [out] context Pointer to the newly created communication context * * @parblock * * @b Description * * The @ref ucc_context_create creates the context and @ref ucc_context_destroy * releases the resources and destroys the context state. The creation of * context does not necessarily indicate its readiness to be used for * collective or other group operations. On success, the context handle will be * created and ucc_status_t will return UCC_OK. On error, the context object * will not be created and corresponding error code as defined by * @ref ucc_status_t is returned. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_context_create(ucc_lib_h lib_handle, const ucc_context_params_t *params, const ucc_context_config_h config, ucc_context_h *context); /** * @ingroup UCC_CONTEXT * * @brief The @ref ucc_context_progress routine progresses the operations * on the context handle. * * @param [in] context Communication context handle to be progressed * * @parblock * * @b Description * * The @ref ucc_context_progress routine progresses the operations on the * content handle. It does not block for lack of resources or communication. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_context_progress(ucc_context_h context); /** * @ingroup UCC_CONTEXT * * @brief The @ref ucc_context_destroy routine frees the context handle. * * @param [in] context Communication context handle to be released * * @parblock * * @b Description * * @ref ucc_context_destroy routine releases the resources associated * with the handle @e context. All teams associated with the context should be * released before this. It is invalid to associate any team with this handle * after the routine is called. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_context_destroy(ucc_context_h context); /** * @ingroup UCC_CONTEXT * * @brief The routine queries the attributes of the context handle. * * @param [in] context Communication context * @param [out] context_attr Attributes of the communication context * * @parblock * * @b Description * * @brief @ref ucc_context_get_attr routine queries the context handle * attributes described by @ref ucc_context_attr. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_context_get_attr(ucc_context_h context, ucc_context_attr_t *context_attr); /* * ************************************************************* * Teams Section * ************************************************************* */ /** * * @ingroup UCC_TEAM_DT */ enum ucc_team_params_field { UCC_TEAM_PARAM_FIELD_ORDERING = UCC_BIT(0), UCC_TEAM_PARAM_FIELD_OUTSTANDING_COLLS = UCC_BIT(1), UCC_TEAM_PARAM_FIELD_EP = UCC_BIT(2), UCC_TEAM_PARAM_FIELD_EP_LIST = UCC_BIT(3), UCC_TEAM_PARAM_FIELD_EP_RANGE = UCC_BIT(4), UCC_TEAM_PARAM_FIELD_TEAM_SIZE = UCC_BIT(5), UCC_TEAM_PARAM_FIELD_SYNC_TYPE = UCC_BIT(6), UCC_TEAM_PARAM_FIELD_OOB = UCC_BIT(7), UCC_TEAM_PARAM_FIELD_P2P_CONN = UCC_BIT(8), UCC_TEAM_PARAM_FIELD_MEM_PARAMS = UCC_BIT(9), UCC_TEAM_PARAM_FIELD_EP_MAP = UCC_BIT(10), UCC_TEAM_PARAM_FIELD_ID = UCC_BIT(11), UCC_TEAM_PARAM_FIELD_FLAGS = UCC_BIT(12) }; /** * * @ingroup UCC_TEAM_DT */ enum ucc_team_attr_field { UCC_TEAM_ATTR_FIELD_POST_ORDERING = UCC_BIT(0), UCC_TEAM_ATTR_FIELD_OUTSTANDING_CALLS = UCC_BIT(1), UCC_TEAM_ATTR_FIELD_EP = UCC_BIT(2), UCC_TEAM_ATTR_FIELD_EP_RANGE = UCC_BIT(3), UCC_TEAM_ATTR_FIELD_SYNC_TYPE = UCC_BIT(4), UCC_TEAM_ATTR_FIELD_MEM_PARAMS = UCC_BIT(5), UCC_TEAM_ATTR_FIELD_SIZE = UCC_BIT(6), UCC_TEAM_ATTR_FIELD_EPS = UCC_BIT(7) }; /** * * @ingroup UCC_TEAM_DT */ enum ucc_team_flags { UCC_TEAM_FLAG_COLL_WORK_BUFFER = UCC_BIT(0) /*< If set, this indicates the user will provide a scratchpad buffer for use in one-sided collectives. Otherwise, an internal buffer will used. */ }; /** * * @ingroup UCC_TEAM_DT */ typedef struct ucc_team_p2p_conn { int (*conn_info_lookup)(void *conn_ctx, uint64_t ep, ucc_p2p_conn_t **conn_info, void *request); int (*conn_info_release)(ucc_p2p_conn_t *conn_info); void *conn_ctx; ucc_status_t (*req_test)(void *request); ucc_status_t (*req_free)(void *request); } ucc_team_p2p_conn_t; /** * * @ingroup UCC_TEAM_DT */ typedef enum { /** * When set to this value, the collective participants shall post the operation * in the same order. */ UCC_COLLECTIVE_POST_ORDERED = 0, /** * When set to this value, the collective participants shall post the operation * in any order. */ UCC_COLLECTIVE_POST_UNORDERED = 1, /** * When set to this value, the collective participants shall initialize the operation * in the same order. */ UCC_COLLECTIVE_INIT_ORDERED = 2, /** * When set to this value, the collective participants shall initialize the operation * in any order. */ UCC_COLLECTIVE_INIT_UNORDERED = 3, /** * When set to this value, the collective participants shall initialize and * post the operation in the same order. */ UCC_COLLECTIVE_INIT_AND_POST_ORDERED = 4, /** * When set to this value, the collective participants shall initialize and * post the operation in any order. */ UCC_COLLECTIVE_INIT_AND_POST_UNORDERED = 5 } ucc_post_ordering_t; /** * * @ingroup UCC_TEAM_DT */ typedef enum { UCC_COLLECTIVE_EP_RANGE_CONTIG = 0, UCC_COLLECTIVE_EP_RANGE_NONCONTIG = 1 } ucc_ep_range_type_t; /** * * @ingroup UCC_TEAM_DT */ struct ucc_ep_map_strided { uint64_t start; int64_t stride; }; /** * * @ingroup UCC_TEAM_DT */ struct ucc_ep_map_array { void *map; size_t elem_size; /*!< 4 if array is int, 8 if e.g. uint64_t */ }; /** * * @ingroup UCC_TEAM_DT */ struct ucc_ep_map_cb { uint64_t (*cb)(uint64_t ep, void *cb_ctx); void *cb_ctx; }; /** * * @ingroup UCC_TEAM_DT */ typedef enum { UCC_EP_MAP_FULL = 1, /*!< The ep range of the team spans all eps from a context. */ UCC_EP_MAP_STRIDED = 2, /*!< The ep range of the team can be described by the 2 values: start, stride.*/ UCC_EP_MAP_ARRAY = 3, /*!< The ep range is given as an array of intergers that map the ep in the team to the team_context rank. */ UCC_EP_MAP_CB = 4, /*!< The ep range mapping is defined as callback provided by the UCC user. */ } ucc_ep_map_type_t; /** * * @ingroup UCC_TEAM_DT */ typedef struct ucc_ep_map_t { ucc_ep_map_type_t type; uint64_t ep_num; /*!< number of eps mapped to ctx */ union { struct ucc_ep_map_strided strided; struct ucc_ep_map_array array; struct ucc_ep_map_cb cb; }; } ucc_ep_map_t; /** * * @ingroup UCC_TEAM_DT * * @brief Structure representing the parameters to customize the team * * @parblock * * Description * * @ref ucc_team_params_t defines the parameters that can be used to customize * the team. The "mask" bit array fields are defined by @ref * ucc_team_params_field. The bits in "mask" bit array is defined by @ref * ucc_team_params_field, which correspond to fields in structure @ref * ucc_team_params_t. The valid fields of the structure is specified by the * setting the bit to "1" in the bit-array "mask". When bits corresponding to * the fields is not set, the fields are not defined. * * * * @endparblock * */ typedef struct ucc_team_params { uint64_t mask; uint64_t flags; /** @ref ucc_team_params.ordering is set to one the values defined by @ref * ucc_post_ordering_t */ ucc_post_ordering_t ordering; /** @ref ucc_team_params.outstanding_colls represents the number of outstanding non-blocking * calls the user expects to post to the team. If the user posts more non-blocking * calls than set, the behavior is undefined. If not set, there is no limit on * the number of outstanding calls to be posted. */ uint64_t outstanding_colls; /** @ref ucc_team_params.ep The endpoint is a non-negative unique integer identifying the * participant in the collective. If ep is not set, and @ref ucc_team_params.oob is not set, the * library generates the ep. The generated ep can be queried using the @ref * ucc_team_get_attr interface. */ uint64_t ep; /** @ref ucc_team_params.ep_list The endpoint list provides the list of eps participating to * create the team. */ uint64_t *ep_list; /** @ref ucc_team_params.ep_range can be either contiguous or not * contiguous. It is a hint to the library. */ ucc_ep_range_type_t ep_range; /** @ref ucc_team_params.team_size The team size is the number of participants in the team. If * @ref ucc_team_params.oob is provided, the team size and @ref * ucc_oob_coll.n_oob_eps should be the same. */ uint64_t team_size; /** * @ref ucc_team_params.sync_type The options for sync_type are provided by @ref * ucc_coll_sync_type_t */ ucc_coll_sync_type_t sync_type; /** @ref ucc_team_params.oob The signature of the function is defined by @ref * ucc_oob_coll_t * . The oob is used for exchanging information between the team * participants during team creation. The user is * responsible for implementing the oob operation. The relation between @ref * ucc_team_params.ep and @ref ucc_oob_coll.oob_ep is defined as below: * * - When both are not provided. The library is responsible for generating the ep, * which can be then queried via the @ref ucc_team_get_attr interface. This * requires, however, ucc_params_t ep_map to be set and context created by * @ref ucc_oob_coll. The behavior is undefined, when neither @ref * ucc_team_params.ep or @ref ucc_team_params.ep_map, or @ref * ucc_team_params.oob is not set. * * - When @ref ucc_team_params.ep is provided and @ref ucc_team_params.oob is * not provided. The “ep” is the unique integer for the participant. * * - When @ref ucc_oob_coll.oob_ep is provided and @ref ucc_team_params.ep * is not provided. The “ep” will be equivalent to @ref ucc_oob_coll.oob_ep. * * - When both are provided, the @ref ucc_oob_coll.oob_ep and @ref * ucc_team_params_t.ep should be same. Otherwise, it * is undefined. */ ucc_team_oob_coll_t oob; /** @ref ucc_team_params.p2p_conn is a callback function for the gathering * the point-to-point communication information. */ ucc_team_p2p_conn_t p2p_conn; /** @ref ucc_team_params.mem_params provides an ability to attach a buffer * to the team. This can be used as input/output or control buffer for the * team. Typically, it can be useful for one-sided collective * implementation. */ ucc_mem_map_params_t mem_params; /** @ref ucc_team_params.ep_map provides a mapping between @ref * ucc_oob_coll.oob_ep used by * the team and @ref ucc_oob_coll.oob_ep * used by the context. The mapping options are defined by @ref * ucc_ep_map_t. The definition is valid only when context is created with * an @ref ucc_oob_coll. */ ucc_ep_map_t ep_map; /** @ref ucc_team_params.id * The team id is a unique integer identifying the team that is active. The * integer is unique within the process and not the job .i.e., any two active * non-overlapping teams can have the same id. This semantic helps to avoid a * global information exchange .i.e, the processes or threads not * participating in the particular, need not participate in the team * creation. If not provided, the team id is created internally. For the MPI * programming model, this can be inherited from the MPI communicator id. */ uint64_t id; } ucc_team_params_t; /** * * @ingroup UCC_TEAM_DT * * @brief Structure representing the team attributes * * @parblock * * Description * * @ref ucc_team_attr_t defines the attributes of the team. The bits in * "mask" bit array is defined by @ref ucc_team_attr_field, which correspond to * fields in structure @ref ucc_team_attr_t. The valid fields of the structure * is specified by the setting the bit to "1" in the bit-array "mask". When * bits corresponding to the fields is not set, the fields are not defined. * * @endparblock * */ typedef struct ucc_team_attr { uint64_t mask; ucc_post_ordering_t ordering; uint64_t outstanding_colls; uint64_t ep; ucc_ep_range_type_t ep_range; ucc_coll_sync_type_t sync_type; ucc_mem_map_params_t mem_params; uint32_t size; uint64_t *eps; } ucc_team_attr_t; /** * @ingroup UCC_TEAM * * @brief The routine is a method to create the team. * * @param [in] contexts Communication contexts abstracting the resources * @param [in] num_contexts Number of contexts passed for the create operation * @param [in] team_params User defined configurations for the team * @param [out] new_team Team handle * * @parblock * * @b Description * * @ref ucc_team_create_post is a nonblocking collective operation to create * the team handle. Overlapping of multiple ucc\_team\_create\_post operations * are invalid. The post takes in parameters ucc_context_h and ucc_team_params_t. * The ucc_team_params_t provides user configuration to customize the team and, * ucc_context_h provides the resources for the team and collectives. * The routine returns immediately after posting the operation with the * new team handle. However, the team handle is not ready for posting * the collective operation. ucc_team_create_test operation is used to learn * the status of the new team handle. On error, the team handle will not * be created and corresponding error code as defined by @ref ucc_status_t is * returned. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_team_create_post(ucc_context_h *contexts, uint32_t num_contexts, const ucc_team_params_t *team_params, ucc_team_h *new_team); /** * @ingroup UCC_TEAM * * @brief The routine queries the status of the team creation operation. * * @param [in] team Team handle to test * * @parblock * * @b Description * * @ref ucc_team_create_test routines tests the status of team handle. * If required it can progress the communication but cannot block on the * communications. On error, the team handle becomes invalid, user is responsible * to call ucc_team_destroy to destroy team and free allocated resources. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_team_create_test(ucc_team_h team); /** * @ingroup UCC_TEAM * * @brief The team frees the team handle. * * @param [in] team Destroy previously created team and release all resources * associated with it. * * @parblock * * @b Description * * @ref ucc_team_destroy is a nonblocking collective operation to release all * resources associated with the team handle, and destroy the team handle. It is * invalid to post a collective operation after the ucc_team_destroy operation. * It is invalid to call @ref ucc_team_destroy operation while @ref * ucc_team_create_post is in progress. It is the user's responsibility to ensure * there is one outstanding @ref ucc_team_create_post or @ref ucc_team_destroy * operation is in progress. * * * @endparblock * * @return Error code as defined by @ref ucc_status_t * */ ucc_status_t ucc_team_destroy(ucc_team_h team); /** * @ingroup UCC_TEAM * * @brief The routine returns the attributes of the team. * * @param [in] team Team handle * @param [out] team_attr Attributes of the team * * @parblock * * @b Description * * @brief @ref ucc_team_get_attr routine queries the team handle * attributes. The attributes of the team handle are described by the team * attributes @ref ucc_team_attr_t * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_team_get_attr(ucc_team_h team, ucc_team_attr_t *team_attr); /** * @ingroup UCC_TEAM * * @brief The routine creates a new team from the parent team. * * @param [in] my_ep Endpoint of the process/thread calling the split operation * @param [in] parent_team Parent team handle from which a new team handle is created * @param [in] included Variable indicating whether a * process/thread participates in the newly created team; * value 1 indicates the participation and value 0 indicates * otherwise * @param [out] new_team Pointer to the new team handle * * @parblock * * @b Description * * @brief ucc_team_create_from_parent is a nonblocking collective operation, * which creates a new team from the parent team. If a participant intends to * participate in the new team, it passes a TRUE value for the "included" * parameter. Otherwise, it passes FALSE. The routine returns immediately after * the post-operation. To learn the completion of the team create operation, the * ucc_team_create_test operation is used. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_team_create_from_parent(uint64_t my_ep, uint32_t included, ucc_team_h parent_team, ucc_team_h *new_team); /* * ************************************************************* * Collectives Section * ************************************************************* */ /** * @ingroup UCC_COLLECTIVES_DT */ typedef enum { UCC_COLL_ARGS_FLAG_IN_PLACE = UCC_BIT(0), /*!< If set, the output buffer is identical to the input buffer.*/ UCC_COLL_ARGS_FLAG_PERSISTENT = UCC_BIT(1), /*!< If set, the collective is considered persistent. Only, the persistent collective can be called multiple times with the same request. */ UCC_COLL_ARGS_FLAG_COUNT_64BIT = UCC_BIT(2), /*!< If set, the count is 64bit, otherwise, it is 32 bit. */ UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT = UCC_BIT(3), /*!< If set, the displacement is 64bit, otherwise, it is 32 bit. */ UCC_COLL_ARGS_FLAG_CONTIG_SRC_BUFFER = UCC_BIT(4), /*!< If set, the src buffer is considered contiguous. Particularly, useful for alltoallv operation.*/ UCC_COLL_ARGS_FLAG_CONTIG_DST_BUFFER = UCC_BIT(5), /*!status; } /** * @ingroup UCC_COLLECTIVES * * @brief The routine to release the collective operation associated with the request object. * * @param [in] request - Request handle * * @parblock * * @b Description * * @ref ucc_collective_finalize operation releases all resources * associated with the collective operation represented by the request handle. * In UCC_THREAD_MULTIPLE mode, the user is responsible for ensuring that * @ref ucc_collective_finalize is called after the status is UCC_OK and after * completing the execution of any callback registered with @ref ucc_coll_args_t. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_collective_finalize(ucc_coll_req_h request); /** * @ingroup UCC_EVENT_DT * */ typedef enum ucc_event_type { UCC_EVENT_COLLECTIVE_POST = UCC_BIT(0), UCC_EVENT_COLLECTIVE_COMPLETE = UCC_BIT(1), UCC_EVENT_COMPUTE_COMPLETE = UCC_BIT(2), UCC_EVENT_OVERFLOW = UCC_BIT(3) } ucc_event_type_t; /** * @ingroup UCC_EVENT_DT * */ typedef enum ucc_ee_type { UCC_EE_FIRST = 0, UCC_EE_CUDA_STREAM = UCC_EE_FIRST, UCC_EE_CPU_THREAD, UCC_EE_ROCM_STREAM, UCC_EE_LAST, UCC_EE_UNKNOWN = UCC_EE_LAST } ucc_ee_type_t; /** * @ingroup UCC_EVENT_DT * */ typedef struct ucc_event { ucc_event_type_t ev_type; void * ev_context; size_t ev_context_size; ucc_coll_req_h req; } ucc_ev_t; /** * @ingroup UCC_EVENT_DT * */ typedef struct ucc_ee_params { ucc_ee_type_t ee_type; void * ee_context; size_t ee_context_size; } ucc_ee_params_t; /** * @ingroup UCC_EVENT * * @brief The routine creates the execution context for collective operations. * * @param [in] team Team handle * @param [in] params User provided params to customize the execution engine * @param [out] ee Execution engine handle * * @parblock * * @b Description * * @ref ucc_ee_create creates the execution engine. It enables event-driven * collective execution. @ref ucc_ee_params_t allows the execution engine to be * configured to abstract either GPU and CPU threads. The execution engine is * created and coupled with the team. There can be many execution engines * coupled to the team. However, attaching the same execution engine to multiple * teams is not allowed. The execution engine is created after the team is * created and destroyed before the team is destroyed. It is the user's * responsibility to destroy the execution engines before the team. If the team * is destroyed before the execution engine is destroyed, the result is * undefined. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_ee_create(ucc_team_h team, const ucc_ee_params_t *params, ucc_ee_h *ee); /** * @ingroup UCC_EVENT * * @brief The routine destroys the execution context created for collective operations. * * @param [in] ee Execution engine handle * * @parblock * * @b Description * * @ref ucc_ee_destroy releases the resources attached with the * execution engine and destroys the execution engine. All events and triggered * operations related to this ee are invalid after the destroy operation. To * avoid race between the creation and destroying the execution engine, for a * given ee, the @ref ucc_ee_create and @ref ucc_ee_destroy must be invoked from * the same thread. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_ee_destroy(ucc_ee_h ee); /** * @ingroup UCC_EVENT * * @brief The routine gets the event from the event queue. * * @param [in] ee Execution engine handle * @param [out] ev Event structure fetched from the event queue * * @parblock * * @b Description * * @ref ucc_ee_get_event fetches the events from the execution engine. If there * are no events posted on the ee, it returns immediately without waiting for * events. All events must be acknowledged using the @ref ucc_ee_ack_event * interface. The event acknowledged is destroyed by the library. An event * fetched with @ref ucc_ee_get_event but not acknowledged might consume * resources in the library. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_ee_get_event(ucc_ee_h ee, ucc_ev_t **ev); /** * @ingroup UCC_EVENT * * @brief The routine acks the events from the event queue. * * @param [in] ee Execution engine handle * @param [in] ev Event to be acked * * @parblock * * @b Description * * An event acknowledged by the user using @ref ucc_ee_ack_event is destroyed by * the library. Any triggered operations on the event should be completed before * calling this interface. The behavior is undefined if the user acknowledges * the event while waiting on the event or triggering operations on the event. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_ee_ack_event(ucc_ee_h ee, ucc_ev_t *ev); /** * @ingroup UCC_EVENT * * @brief The routine to set the event to the tail of the queue. * * @param [in] ee Execution engine handle * @param [in] ev Event structure fetched from the event queue * * @parblock * * @b Description * * @ref ucc_ee_set_event sets the event on the execution engine. If the * operations are waiting on the event when the user sets the event, the * operations are launched. The events created by the user need to be destroyed * by the user. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_ee_set_event(ucc_ee_h ee, ucc_ev_t *ev); /** * @ingroup UCC_EVENT * * @brief The routine blocks the calling thread until there is an event on the queue. * * @param [in] ee Execution engine handle * @param [out] ev Event structure fetched from the event queue * * @parblock * * @b Description * * The user thread invoking the @ref ucc_ee_wait interface is blocked until an * event is posted to the execution engine. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_ee_wait(ucc_ee_h ee, ucc_ev_t *ev); /** * @ingroup UCC_EVENT * * @brief The routine posts the collective operation on the execution engine, which is * launched on the event. * * @param [in] ee Execution engine handle * @param [in] ee_event Event triggering the post operation * * @parblock * * @b Description * * @ref ucc_collective_triggered_post allow the users to schedule a collective * operation that executes in the future when an event occurs on the execution * engine. On error, request handle associated with event becomes invalid, * user is responsible to call ucc_collective_finalize to free allocated resources. * * @endparblock * * @return Error code as defined by @ref ucc_status_t */ ucc_status_t ucc_collective_triggered_post(ucc_ee_h ee, ucc_ev_t *ee_event); /** * @ingroup UCC_DATATYPE */ typedef enum { UCC_MEM_MAP_MODE_EXPORT = 0, /*!< Indicate ucc_mem_map() should export memory handles from TLs used by context */ UCC_MEM_MAP_MODE_IMPORT = 1, /*!< Indicate ucc_mem_map() should import memory handles from user memory handle */ UCC_MEM_MAP_MODE_EXPORT_OFFLOAD = 2, /*!< Indicate ucc_mem_map() should export memory handles from TLs used by a context on an offloaded device */ UCC_MEM_MAP_MODE_IMPORT_OFFLOAD = 3, /*!< Indicate ucc_mem_map() should import memory handles from a host's memory handle to an offloaded device */ UCC_MEM_MAP_MODE_LAST = 4 } ucc_mem_map_mode_t; /** * @ingroup UCC_CONTEXT * @brief Routine registers or maps memory for use in future collectives. * * This local routine maps a user-specified memory segment with a * ucc_context_h. The segment is considered "mapped" with the context until * the user calls ucc_mem_unmap. It is the user's responsibility to unmap all * mapped segments prior to calling ucc_context_destroy(). A handle to the * mapped memory is provided in memh. If the mode UCC_MEM_MAP_MODE_EXPORT is used, * the memory will be mapped and memory handles from TLs will be generated and * stored in the memh. If the mode UCC_MEM_MAP_MODE_IMPORT is used, the user must * provide a valid memh, otherwise behavior is undefined. * * @param [in] context Context mapped memory is associated with * @param [in] mode mode dictating the behavior of the routine * @param [in] params parameters indicating the address and length of * memory to map * @param [out] memh_size Size of memory handle when exported * @param [inout] *memh Handle for the registered memory * * @return Error code as defined by ucc_status_t. */ ucc_status_t ucc_mem_map(ucc_context_h context, ucc_mem_map_mode_t mode, const ucc_mem_map_params_t *params, size_t *memh_size, ucc_mem_map_mem_h *memh); /** * @ingroup UCC_CONTEXT * @brief Routine unmaps memory from a context * * This is a local routine that unmaps memory and all resources * associated with the memory from a context. The memh object is freed and * cannot be reused. * * @param [in] *memh Handle of the registered memory * * @return Error code as defined by ucc_status_t. */ ucc_status_t ucc_mem_unmap(ucc_mem_map_mem_h *memh); END_C_DECLS #endif ucc-1.8.0/src/ucc/api/ucc_version.h.in0000664000175000017500000000112715211535620017666 0ustar alastairalastair/** * @file ucc_version.h * @date 2020 * @copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #define UCC_VERSION(_major, _minor) \ (((_major) << UCC_VERSION_MAJOR_SHIFT) | \ ((_minor) << UCC_VERSION_MINOR_SHIFT)) #define UCC_VERSION_MAJOR_SHIFT 24 #define UCC_VERSION_MINOR_SHIFT 16 #define UCC_API_MAJOR @MAJOR_VERSION@ #define UCC_API_MINOR @MINOR_VERSION@ #define UCC_API_VERSION UCC_VERSION(@MAJOR_VERSION@, @MINOR_VERSION@) #define UCC_VERSION_STRING "@VERSION@" #define UCC_GIT_REVISION "@UCC_GIT_SHA@" ucc-1.8.0/src/ucc/api/ucc_def.h0000664000175000017500000001130015211535620016324 0ustar alastairalastair/** * @file ucc_def.h * @date 2020 * @copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * @copyright Copyright (C) Huawei Technologies Co., Ltd. 2020. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ #ifndef UCC_DEF_H_ #define UCC_DEF_H_ #include #include #include /** * @ingroup UCC_LIB_INIT_DT * @brief UCC library handle * * The ucc library handle is an opaque handle created by the library. It * abstracts the collective library. It holds the global information and * resources associated with the library. The library handle cannot be passed * from one library instance to another. */ typedef struct ucc_lib_info* ucc_lib_h; /** * @ingroup UCC_CONTEXT_DT * @brief UCC context * * The UCC context is an opaque handle to abstract the network resources for * collective operations. The network resources could be either software or * hardware. Based on the type of the context, the resources can be shared or * either be exclusively used. The UCC context is required but not sufficient to * execute a collective operation. */ typedef struct ucc_context* ucc_context_h; /** * @ingroup UCC_TEAM_DT * @brief UCC team handle * * The UCC team handle is an opaque handle created by the library. It abstracts * the group resources required for the collective operations and participants * of the collective operation. The participants of the collective operation can * be an OS process or thread. */ typedef struct ucc_team* ucc_team_h; /** * @ingroup UCC_COLLECTIVES_DT * @brief UCC collective request handle * * The UCC request handle is an opaque handle created by the library during the * invocation of the collective operation. The request may be used to learn the * status of the collective operation, progress, or complete the collective * operation. */ typedef struct ucc_coll_req* ucc_coll_req_h; typedef struct ucc_coll_req { ucc_status_t status; } ucc_coll_req_t; /** * @ingroup UCC_COLLECTIVES_DT * @brief UCC collective completion callback * * The callback is invoked whenever the collective operation is completed. * It is not allowed to call UCC APIs from the completion callback except * for @ref ucc_collective_finalize. */ typedef struct ucc_coll_callback { void (*cb)(void *data, ucc_status_t status); void *data; } ucc_coll_callback_t; /** * @ingroup UCC_COLLECTIVES * @brief UCC memory handle * * The UCC memory handle is an opaque handle created by the library representing * the buffer and address. */ typedef struct ucc_mem_handle* ucc_mem_h; /** * @ingroup UCC_LIB_INIT_DT * * @brief UCC library configuration handle */ typedef struct ucc_lib_config* ucc_lib_config_h; /** * @ingroup UCC_CONTEXT_DT * * @brief UCC context configuration handle */ typedef struct ucc_context_config* ucc_context_config_h; /** * @ingroup UCC_COLLECTIVES_DT * @brief Count datatype to support both small (32 bit) and large counts (64 bit) */ typedef uint64_t ucc_count_t; /** * @ingroup UCC_COLLECTIVES_DT * @brief Datatype to support both small (32 bit) and large address offsets (64 bit) */ typedef uint64_t ucc_aint_t; /* Reflects the definition in UCS - The i-th bit */ #define UCC_BIT(i) (1ul << (i)) #define UCC_MASK(i) (UCC_BIT(i) - 1) /* Reflects the definition in UCS */ /** * @ingroup UCC_UTILS * @brief Print configurations */ typedef enum { UCC_CONFIG_PRINT_CONFIG = UCC_BIT(0), UCC_CONFIG_PRINT_HEADER = UCC_BIT(1), UCC_CONFIG_PRINT_DOC = UCC_BIT(2), UCC_CONFIG_PRINT_HIDDEN = UCC_BIT(3) } ucc_config_print_flags_t; /** * @ingroup UCC_COLLECTIVES_DT * @brief Datatype for collective tags */ typedef uint16_t ucc_coll_id_t ; /** * @ingroup UCC_TEAM_DT */ typedef void *ucc_p2p_conn_t; /** * @ingroup UCC_TEAM_DT */ typedef void* ucc_context_addr_h; /** * @ingroup UCC_TEAM_DT */ typedef size_t ucc_context_addr_len_t; /** * @ingroup UCC_EVENT_DT * @brief UCC execution engine handle * * The UCC execution engine handle is an opaque handle created by the library representing * the execution context and related queues. */ typedef struct ucc_ee* ucc_ee_h; typedef struct ucc_dt_generic ucc_dt_generic_t; /** * @ingroup UCC_DATATYPE * @brief Helper enum for generic/predefined datatype representation * */ typedef enum { UCC_DATATYPE_PREDEFINED = 0, UCC_DATATYPE_GENERIC = UCC_BIT(0), UCC_DATATYPE_SHIFT = 3, UCC_DATATYPE_CLASS_MASK = UCC_MASK(UCC_DATATYPE_SHIFT) } ucc_dt_type_t; #define UCC_PREDEFINED_DT(_id) \ (ucc_datatype_t)((((uint64_t)(_id)) << UCC_DATATYPE_SHIFT) | \ (UCC_DATATYPE_PREDEFINED)) #endif ucc-1.8.0/src/core/0000775000175000017500000000000015211535620014175 5ustar alastairalastairucc-1.8.0/src/core/ucc_dt.c0000664000175000017500000000402215211535620015600 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "ucc_dt.h" #include "utils/ucc_malloc.h" #include "utils/ucc_math.h" size_t ucc_dt_predefined_sizes[UCC_DT_PREDEFINED_LAST] = { [UCC_DT_PREDEFINED_ID(UCC_DT_INT8)] = 1, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT8)] = 1, [UCC_DT_PREDEFINED_ID(UCC_DT_INT16)] = 2, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT16)] = 2, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT16)] = 2, [UCC_DT_PREDEFINED_ID(UCC_DT_BFLOAT16)] = 2, [UCC_DT_PREDEFINED_ID(UCC_DT_INT32)] = 4, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT32)] = 4, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT32)] = 4, [UCC_DT_PREDEFINED_ID(UCC_DT_INT64)] = 8, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT64)] = 8, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT64)] = 8, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT128)] = 16, [UCC_DT_PREDEFINED_ID(UCC_DT_INT128)] = 16, [UCC_DT_PREDEFINED_ID(UCC_DT_UINT128)] = 16, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT32_COMPLEX)] = 8, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT64_COMPLEX)] = 16, [UCC_DT_PREDEFINED_ID(UCC_DT_FLOAT128_COMPLEX)] = 32}; ucc_status_t ucc_dt_create_generic(const ucc_generic_dt_ops_t *ops, void *context, ucc_datatype_t *datatype_p) { ucc_dt_generic_t *dt_gen; int ret; ret = ucc_posix_memalign((void **)&dt_gen, ucc_max(sizeof(void *), UCC_BIT(UCC_DATATYPE_SHIFT)), sizeof(*dt_gen), "generic_dt"); if (ret != 0) { return UCC_ERR_NO_MEMORY; } dt_gen->ops = *ops; dt_gen->context = context; *datatype_p = ucc_dt_from_generic(dt_gen); return UCC_OK; } void ucc_dt_destroy(ucc_datatype_t datatype) { ucc_dt_generic_t *dt_gen; if (UCC_DT_IS_GENERIC(datatype)) { dt_gen = ucc_dt_to_generic(datatype); ucc_free(dt_gen); } } ucc-1.8.0/src/core/ucc_service_coll.c0000664000175000017500000005502515211535620017653 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_service_coll.h" #include "ucc_team.h" #include "ucc_global_opts.h" #include "schedule/ucc_schedule.h" #include "utils/ucc_malloc.h" #include "utils/ucc_coll_utils.h" uint64_t ucc_service_coll_map_cb(uint64_t ep, void *cb_ctx) { ucc_service_coll_req_t *req = cb_ctx; ucc_team_t *team = req->team; ucc_rank_t team_rank; team_rank = ucc_ep_map_eval(req->subset.map, (ucc_rank_t)ep); return ucc_ep_map_eval(team->ctx_map, team_rank); } static inline ucc_status_t ucc_service_coll_req_init(ucc_team_t *team, ucc_subset_t *subset, ucc_tl_team_t **service_team, ucc_service_coll_req_t **_req) { ucc_context_t *ctx = team->contexts[0]; ucc_service_coll_req_t *req; *service_team = NULL; req = ucc_malloc(sizeof(*req), "service_req"); if (!req) { ucc_error("failed to allocate %zd bytes for service coll req", sizeof(*req)); return UCC_ERR_NO_MEMORY; } req->team = team; req->subset = *subset; if (ctx->service_team) { *service_team = ctx->service_team; subset->map.type = UCC_EP_MAP_CB; subset->map.cb.cb = ucc_service_coll_map_cb; subset->map.cb.cb_ctx = req; } else { ucc_assert(team->service_team != NULL); *service_team = team->service_team; } *_req = req; return UCC_OK; } ucc_status_t ucc_service_allreduce(ucc_team_t *team, void *sbuf, void *rbuf, ucc_datatype_t dt, size_t count, ucc_reduction_op_t op, ucc_subset_t subset, ucc_service_coll_req_t **req) { ucc_tl_team_t *steam; ucc_tl_iface_t *tl_iface; ucc_status_t status; status = ucc_service_coll_req_init(team, &subset, &steam, req); if (UCC_OK != status) { return status; } tl_iface = UCC_TL_TEAM_IFACE(steam); status = tl_iface->scoll.allreduce(&steam->super, sbuf, rbuf, dt, count, op, subset, &(*req)->task); if (status < 0) { ucc_free(*req); ucc_error("failed to start service allreduce for team %p: %s", team, ucc_status_string(status)); return status; } return UCC_OK; } ucc_status_t ucc_service_allgather(ucc_team_t *team, void *sbuf, void *rbuf, size_t msgsize, ucc_subset_t subset, ucc_service_coll_req_t **req) { ucc_tl_team_t *steam; ucc_tl_iface_t *tl_iface; ucc_status_t status; status = ucc_service_coll_req_init(team, &subset, &steam, req); if (UCC_OK != status) { return status; } tl_iface = UCC_TL_TEAM_IFACE(steam); status = tl_iface->scoll.allgather(&steam->super, sbuf, rbuf, msgsize, subset, &(*req)->task); if (status < 0) { ucc_free(*req); ucc_error("failed to start service allreduce for team %p: %s", team, ucc_status_string(status)); return status; } return UCC_OK; } ucc_status_t ucc_service_bcast(ucc_team_t *team, void *buf, size_t msgsize, ucc_rank_t root, ucc_subset_t subset, ucc_service_coll_req_t **req) { ucc_tl_team_t *steam; ucc_tl_iface_t *tl_iface; ucc_status_t status; status = ucc_service_coll_req_init(team, &subset, &steam, req); if (UCC_OK != status) { return status; } tl_iface = UCC_TL_TEAM_IFACE(steam); status = tl_iface->scoll.bcast(&steam->super, buf, msgsize, root, subset, &(*req)->task); if (status < 0) { ucc_free(*req); ucc_error("failed to start service bcast for team %p: %s", team, ucc_status_string(status)); return status; } return UCC_OK; } ucc_status_t ucc_service_coll_test(ucc_service_coll_req_t *req) { ucc_status_t status; status = ucc_collective_test(&req->task->super); if (UCC_INPROGRESS == status) { ucc_context_progress(req->team->contexts[0]); } return status; } ucc_status_t ucc_service_coll_finalize(ucc_service_coll_req_t *req) { ucc_status_t status; status = ucc_collective_finalize_internal(req->task); ucc_free(req); return status; } typedef struct ucc_internal_oob_coll_info { ucc_team_t *team; ucc_subset_t subset; } ucc_internal_oob_coll_info_t; static ucc_status_t ucc_internal_oob_allgather(void *sbuf, void *rbuf, size_t size, void *coll_info, void **request) { ucc_internal_oob_coll_info_t *ci = coll_info; ucc_service_coll_req_t *req = NULL; ucc_status_t status; status = ucc_service_allgather(ci->team, sbuf, rbuf, size, ci->subset, &req); *request = (void *)req; return status; } static ucc_status_t ucc_internal_oob_test(void *request) { ucc_service_coll_req_t *req = request; return ucc_service_coll_test(req); } static ucc_status_t ucc_internal_oob_free(void *request) { ucc_service_coll_req_t *req = request; return ucc_service_coll_finalize(req); } ucc_status_t ucc_internal_oob_init(ucc_team_t *team, ucc_subset_t subset, ucc_team_oob_coll_t *oob) { ucc_internal_oob_coll_info_t *ci; ci = ucc_malloc(sizeof(*ci), "internal_coll_info"); if (!ci) { ucc_error("failed to allocate %zd bytes for internal_coll_info", sizeof(*ci)); return UCC_ERR_NO_MEMORY; } ci->team = team; ci->subset = subset; oob->coll_info = ci; oob->allgather = ucc_internal_oob_allgather; oob->req_test = ucc_internal_oob_test; oob->req_free = ucc_internal_oob_free; oob->n_oob_eps = (uint32_t)subset.map.ep_num; oob->oob_ep = (uint32_t)subset.myrank; return UCC_OK; } void ucc_internal_oob_finalize(ucc_team_oob_coll_t *oob) { ucc_free(oob->coll_info); } /* Helper macro to get dt_check from schedule */ #define UCC_DT_CHECK_SCHEDULE(_task) \ ucc_derived_of((_task)->schedule, ucc_dt_check_schedule_t) #define UCC_DT_CHECK_FROM_TASK(_task) \ (&UCC_DT_CHECK_SCHEDULE(_task)->dt_check) /** * Validate allreduced datatype values using min/max trick * * After MIN allreduce on [dt, -dt, mem, -mem]: * - values[0] contains min(dt) across all ranks * - values[1] contains min(-dt) = -max(dt) across all ranks * - If values[0] == -values[1], all ranks have identical dt * - Same logic applies to memory type with values[2] and values[3] */ static ucc_status_t ucc_dt_validate_results(ucc_dt_check_state_t *dt_check) { int16_t *values; /* Safety checks */ if (!dt_check) { return UCC_ERR_INVALID_PARAM; } values = dt_check->values; /* Check if any rank has non-contiguous datatype */ if (values[0] == (int16_t) UCC_ERR_NOT_SUPPORTED) { return UCC_ERR_NOT_SUPPORTED; } /* Check if all ranks have the same datatype using min/max trick */ if (values[0] != -values[1]) { return UCC_ERR_INVALID_PARAM; } /* Check if all ranks have the same memory type */ if (values[2] != -values[3]) { return UCC_ERR_INVALID_PARAM; } return UCC_OK; } /** * Post function for allreduce wrapper task using service allreduce * * Starts the service allreduce (MIN) to detect datatype mismatches across all ranks. */ static ucc_status_t ucc_dt_check_allreduce_post(ucc_coll_task_t *allreduce_wrapper) { ucc_dt_check_state_t *dt_check = UCC_DT_CHECK_FROM_TASK(allreduce_wrapper); ucc_team_t *team = allreduce_wrapper->bargs.team; ucc_status_t status; /* Safety check */ if (!dt_check) { allreduce_wrapper->status = UCC_ERR_INVALID_PARAM; return UCC_ERR_INVALID_PARAM; } /* Start in-place service allreduce with MIN operation on 4 int16_t values */ status = ucc_service_allreduce(team, dt_check->values, dt_check->values, UCC_DT_INT16, 4, UCC_OP_MIN, dt_check->subset, &dt_check->check_req); if (status != UCC_OK) { allreduce_wrapper->status = status; return status; } allreduce_wrapper->status = UCC_INPROGRESS; /* Enqueue wrapper task for progress */ return ucc_progress_queue_enqueue(team->contexts[0]->pq, allreduce_wrapper); } /** * Progress function for allreduce wrapper task using service allreduce * * Progresses service allreduce, and when complete, validates the reduced datatypes. */ static void ucc_dt_check_allreduce_progress(ucc_coll_task_t *allreduce_wrapper) { ucc_dt_check_state_t *dt_check = UCC_DT_CHECK_FROM_TASK(allreduce_wrapper); ucc_coll_task_t *ar_task; ucc_status_t status; /* Safety check */ if (!dt_check || !dt_check->check_req) { allreduce_wrapper->status = UCC_ERR_INVALID_PARAM; return; } /* Check status of the service allreduce */ ar_task = dt_check->check_req->task; status = ar_task->super.status; if (status == UCC_INPROGRESS) { allreduce_wrapper->status = UCC_INPROGRESS; return; } /* Service allreduce completed (or failed) - finalize it */ ucc_service_coll_finalize(dt_check->check_req); dt_check->check_req = NULL; /* If service allreduce failed, mark validation as failed */ if (status != UCC_OK) { dt_check->validated = 0; allreduce_wrapper->status = UCC_OK; return; } /* Service allreduce succeeded - validate using min/max check */ status = ucc_dt_validate_results(dt_check); dt_check->validated = (status == UCC_OK); /* Completes with UCC_OK so schedule continues to actual wrapper */ allreduce_wrapper->status = UCC_OK; } /** * Finalize function for allreduce wrapper task */ static ucc_status_t ucc_dt_check_allreduce_finalize(ucc_coll_task_t *allreduce_wrapper) { ucc_dt_check_state_t *dt_check = UCC_DT_CHECK_FROM_TASK(allreduce_wrapper); /* Clean up check_req if it wasn't finalized in progress */ if (dt_check && dt_check->check_req) { ucc_service_coll_finalize(dt_check->check_req); dt_check->check_req = NULL; } /* dt_check is embedded in schedule, no need to free */ /* Return wrapper task to memory pool */ ucc_mpool_put(allreduce_wrapper); return UCC_OK; } /** * Post function for actual task wrapper * * Checks validation result and only posts actual task if validation succeeded. */ static ucc_status_t ucc_dt_check_actual_wrapper_post(ucc_coll_task_t *wrapper) { ucc_dt_check_state_t *dt_check = UCC_DT_CHECK_FROM_TASK(wrapper); ucc_coll_task_t *actual_task = dt_check->actual_task; ucc_schedule_t *schedule = wrapper->schedule; ucc_status_t status; /* Check if validation succeeded */ if (!dt_check->validated) { /* Validation failed - propagate error to schedule and complete wrapper */ wrapper->status = UCC_ERR_NOT_SUPPORTED; if (schedule) { schedule->super.status = UCC_ERR_NOT_SUPPORTED; schedule->super.super.status = UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } /* Validation succeeded - post the actual task */ status = actual_task->post(actual_task); if (status < 0) { wrapper->status = status; if (schedule) { schedule->super.status = status; schedule->super.super.status = status; } return status; } wrapper->status = UCC_INPROGRESS; return ucc_progress_queue_enqueue(wrapper->bargs.team->contexts[0]->pq, wrapper); } /** * Progress function for actual task wrapper * * Checks the actual task status. The actual task progresses itself via its own * progress queue enqueued by its post() function. */ static void ucc_dt_check_actual_wrapper_progress(ucc_coll_task_t *wrapper) { ucc_dt_check_state_t *dt_check = UCC_DT_CHECK_FROM_TASK(wrapper); ucc_coll_task_t *actual_task = dt_check->actual_task; /* Just copy status from actual task to wrapper * The actual task progresses itself since it was enqueued by its post() */ wrapper->status = actual_task->status; } /** * Finalize function for actual task wrapper */ static ucc_status_t ucc_dt_check_actual_wrapper_finalize(ucc_coll_task_t *wrapper) { ucc_dt_check_state_t *dt_check = UCC_DT_CHECK_FROM_TASK(wrapper); ucc_coll_task_t *actual_task = dt_check->actual_task; ucc_status_t status = UCC_OK; /* Finalize the actual task */ if (actual_task && actual_task->finalize) { status = actual_task->finalize(actual_task); } /* dt_check is embedded in schedule, will be freed with schedule */ /* Wrapper is from memory pool */ ucc_mpool_put(wrapper); return status; } /** * Finalize function for dt_check schedule * * Finalizes all tasks in the schedule and frees the schedule itself. */ static ucc_status_t ucc_dt_check_schedule_finalize(ucc_coll_task_t *task) { ucc_dt_check_schedule_t *dt_schedule = ucc_derived_of(task, ucc_dt_check_schedule_t); ucc_status_t status; /* Finalize all tasks in the schedule */ status = ucc_schedule_finalize(task); /* Destruct and free the schedule itself (including embedded dt_check) */ ucc_coll_task_destruct(&dt_schedule->super.super); ucc_free(dt_schedule); return status; } ucc_coll_task_t* ucc_service_dt_check(ucc_team_t *team, ucc_coll_task_t *task, ucc_status_t *status_out) { ucc_dt_check_schedule_t *dt_schedule; ucc_dt_check_state_t *dt_check; ucc_coll_task_t *allreduce_wrapper; ucc_coll_task_t *actual_wrapper; ucc_base_coll_args_t empty_bargs; const ucc_coll_args_t *args = &task->bargs.args; ucc_rank_t rank = team->rank; ucc_rank_t root = args->root; ucc_coll_type_t coll_type = args->coll_type; ucc_datatype_t local_dt = UCC_DT_INT8; ucc_memory_type_t local_mem_type = UCC_MEMORY_TYPE_UNKNOWN; ucc_status_t status; /* If check is disabled, return original task */ if (!ucc_global_config.check_asymmetric_dt) { return task; } /* Determine which datatype and memory type to check based on operation and rank */ if (rank == root) { switch (coll_type) { case UCC_COLL_TYPE_GATHER: if (UCC_IS_INPLACE(*args)) { local_dt = args->dst.info.datatype; local_mem_type = args->dst.info.mem_type; } else { local_dt = args->src.info.datatype; local_mem_type = args->src.info.mem_type; } break; case UCC_COLL_TYPE_GATHERV: if (UCC_IS_INPLACE(*args)) { local_dt = args->dst.info_v.datatype; local_mem_type = args->dst.info_v.mem_type; } else { local_dt = args->src.info.datatype; local_mem_type = args->src.info.mem_type; } break; case UCC_COLL_TYPE_SCATTER: if (UCC_IS_INPLACE(*args)) { local_dt = args->src.info.datatype; local_mem_type = args->src.info.mem_type; } else { local_dt = args->dst.info.datatype; local_mem_type = args->dst.info.mem_type; } break; case UCC_COLL_TYPE_SCATTERV: if (UCC_IS_INPLACE(*args)) { local_dt = args->src.info_v.datatype; local_mem_type = args->src.info_v.mem_type; } else { local_dt = args->dst.info.datatype; local_mem_type = args->dst.info.mem_type; } break; default: /* Not a rooted collective, no validation needed */ return task; } } else { switch (coll_type) { case UCC_COLL_TYPE_GATHER: case UCC_COLL_TYPE_GATHERV: local_dt = args->src.info.datatype; local_mem_type = args->src.info.mem_type; break; case UCC_COLL_TYPE_SCATTER: case UCC_COLL_TYPE_SCATTERV: local_dt = args->dst.info.datatype; local_mem_type = args->dst.info.mem_type; break; default: /* Not a rooted collective, no validation needed */ return task; } } /* Allocate schedule with embedded dt_check */ dt_schedule = (ucc_dt_check_schedule_t *)ucc_malloc(sizeof(*dt_schedule), "dt_check_schedule"); if (!dt_schedule) { ucc_error("failed to allocate dt_check_schedule"); if (status_out) { *status_out = UCC_ERR_NO_MEMORY; } return NULL; } memset(dt_schedule, 0, sizeof(*dt_schedule)); /* Initialize schedule with task's bargs and TL/CL team */ ucc_coll_task_construct(&dt_schedule->super.super); status = ucc_schedule_init(&dt_schedule->super, &task->bargs, task->team); if (status != UCC_OK) { ucc_error("failed to initialize dt_check schedule: %s", ucc_status_string(status)); ucc_free(dt_schedule); if (status_out) { *status_out = status; } return NULL; } /* Setup embedded dt_check state */ dt_check = &dt_schedule->dt_check; /* Setup values for min/max trick: [dt, -dt, mem, -mem] */ if (!UCC_DT_IS_PREDEFINED(local_dt)) { /* Generic or invalid datatype - reject to prevent int16 overflow */ dt_check->values[0] = (int16_t) UCC_ERR_NOT_SUPPORTED; dt_check->values[1] = -(int16_t) UCC_ERR_NOT_SUPPORTED; } else { /* Predefined datatypes are always contiguous - safe to cast to int16 */ dt_check->values[0] = (int16_t) local_dt; dt_check->values[1] = -(int16_t) local_dt; } dt_check->values[2] = (int16_t) local_mem_type; dt_check->values[3] = -(int16_t) local_mem_type; /* Setup subset for full team */ dt_check->subset.myrank = team->rank; dt_check->subset.map.type = UCC_EP_MAP_FULL; dt_check->subset.map.ep_num = team->size; /* Initialize check_req to NULL */ dt_check->check_req = NULL; dt_check->validated = 0; dt_check->actual_task = task; /* Create allreduce wrapper task from memory pool */ allreduce_wrapper = ucc_mpool_get(&task->bargs.team->contexts[0]->lib->stub_tasks_mp); if (!allreduce_wrapper) { ucc_error("failed to allocate allreduce wrapper task from mpool"); status = UCC_ERR_NO_MEMORY; goto error_schedule; } /* Initialize allreduce wrapper task with minimal bargs (only needs team) */ memset(&empty_bargs, 0, sizeof(empty_bargs)); empty_bargs.team = task->bargs.team; status = ucc_coll_task_init(allreduce_wrapper, &empty_bargs, task->team); if (status != UCC_OK) { ucc_error("failed to init allreduce wrapper task: %s", ucc_status_string(status)); ucc_mpool_put(allreduce_wrapper); goto error_schedule; } /* Set allreduce wrapper functions */ allreduce_wrapper->post = ucc_dt_check_allreduce_post; allreduce_wrapper->progress = ucc_dt_check_allreduce_progress; allreduce_wrapper->finalize = ucc_dt_check_allreduce_finalize; allreduce_wrapper->status = UCC_OPERATION_INITIALIZED; /* Add allreduce wrapper to schedule - starts when schedule starts */ status = ucc_task_subscribe_dep(&dt_schedule->super.super, allreduce_wrapper, UCC_EVENT_SCHEDULE_STARTED); if (status != UCC_OK) { ucc_error("failed to subscribe allreduce wrapper: %s", ucc_status_string(status)); goto error_allreduce_wrapper; } status = ucc_schedule_add_task(&dt_schedule->super, allreduce_wrapper); if (status != UCC_OK) { ucc_error("failed to add allreduce wrapper to schedule: %s", ucc_status_string(status)); goto error_allreduce_wrapper; } /* Create actual task wrapper that conditionally posts the actual task */ actual_wrapper = ucc_mpool_get(&task->bargs.team->contexts[0]->lib->stub_tasks_mp); if (!actual_wrapper) { ucc_error("failed to allocate actual wrapper task from mpool"); status = UCC_ERR_NO_MEMORY; goto error_allreduce_wrapper; } status = ucc_coll_task_init(actual_wrapper, &empty_bargs, task->team); if (status != UCC_OK) { ucc_error("failed to init actual wrapper task: %s", ucc_status_string(status)); ucc_mpool_put(actual_wrapper); goto error_allreduce_wrapper; } /* Set actual wrapper functions */ actual_wrapper->post = ucc_dt_check_actual_wrapper_post; actual_wrapper->progress = ucc_dt_check_actual_wrapper_progress; actual_wrapper->finalize = ucc_dt_check_actual_wrapper_finalize; actual_wrapper->status = UCC_OPERATION_INITIALIZED; /* Add actual wrapper to schedule - depends on allreduce completing */ status = ucc_task_subscribe_dep(allreduce_wrapper, actual_wrapper, UCC_EVENT_COMPLETED); if (status != UCC_OK) { ucc_error("failed to subscribe actual wrapper dependency: %s", ucc_status_string(status)); ucc_mpool_put(actual_wrapper); goto error_allreduce_wrapper; } status = ucc_schedule_add_task(&dt_schedule->super, actual_wrapper); if (status != UCC_OK) { ucc_error("failed to add actual wrapper to schedule: %s", ucc_status_string(status)); ucc_mpool_put(actual_wrapper); goto error_allreduce_wrapper; } /* Propagate executor requirement from actual_task to schedule */ if (task->flags & UCC_COLL_TASK_FLAG_EXECUTOR) { dt_schedule->super.super.flags |= UCC_COLL_TASK_FLAG_EXECUTOR; } /* Set schedule functions */ dt_schedule->super.super.post = ucc_schedule_start; dt_schedule->super.super.progress = NULL; /* Sub-tasks have their own progress functions */ dt_schedule->super.super.finalize = ucc_dt_check_schedule_finalize; return &dt_schedule->super.super; error_allreduce_wrapper: ucc_coll_task_destruct(allreduce_wrapper); ucc_mpool_put(allreduce_wrapper); error_schedule: ucc_coll_task_destruct(&dt_schedule->super.super); ucc_free(dt_schedule); if (status_out) { *status_out = status; } return NULL; } ucc-1.8.0/src/core/ucc_lib.c0000664000175000017500000004404415211535620015747 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "ucc_global_opts.h" #include "ucc_lib.h" #include "components/topo/ucc_sysinfo.h" #include "utils/ucc_malloc.h" #include "utils/ucc_parser.h" #include "components/cl/ucc_cl.h" #include "components/tl/ucc_tl.h" #include "components/mc/ucc_mc.h" #include "components/ec/ucc_ec.h" UCS_CONFIG_DEFINE_ARRAY(cl_types, sizeof(ucc_cl_type_t), UCS_CONFIG_TYPE_ENUM(ucc_cl_names)); static ucc_config_field_t ucc_lib_config_table[] = { {"CLS", "basic", "Comma separated list of CL components to be used", ucc_offsetof(ucc_lib_config_t, cls), UCC_CONFIG_TYPE_ARRAY(cl_types)}, {NULL} }; UCC_CONFIG_REGISTER_TABLE(ucc_lib_config_table, "UCC", NULL, ucc_lib_config_t, &ucc_config_global_list) static inline int ucc_cl_requested(const ucc_lib_config_t *cfg, int cl_type) { int i; for (i = 0; i < cfg->cls.count; i++) { if (cfg->cls.types[i] == cl_type) { return 1; } } return 0; } static inline void ucc_copy_lib_params(ucc_lib_params_t *dst, const ucc_lib_params_t *src) { dst->mask = src->mask; UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_LIB_PARAM_FIELD_THREAD_MODE, thread_mode); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_LIB_PARAM_FIELD_COLL_TYPES, coll_types); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_LIB_PARAM_FIELD_REDUCTION_TYPES, reduction_types); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_LIB_PARAM_FIELD_SYNC_TYPE, sync_type); } /* Core logic for the selection of CL components: 1. If user does not provide a set of required CLs then we try to use only those components that support the required input params. This means, if some component does not pass params check it is skipped. 2. In contrast, if user explicitly requests a list of CLs to use, then we try to load ALL of them and report the supported attributes based on that selection. */ static ucc_status_t ucc_cl_lib_init(const ucc_lib_params_t *user_params, const ucc_lib_config_t *config, ucc_lib_info_t *lib) { int n_cls = ucc_global_config.cl_framework.n_components; uint64_t supported_coll_types = 0; ucc_thread_mode_t highest_tm = UCC_THREAD_SINGLE; ucc_thread_mode_t lowest_tm = UCC_THREAD_MULTIPLE; ucc_lib_params_t params = *user_params; ucc_cl_lib_config_t *cl_config = NULL; ucc_cl_iface_t *cl_iface; ucc_cl_lib_attr_t *attrs; ucc_base_lib_t * b_lib; ucc_base_lib_params_t b_params; ucc_cl_lib_t *cl_lib; ucc_status_t status; int i; lib->cl_libs = (ucc_cl_lib_t **)ucc_malloc(sizeof(ucc_cl_lib_t *) * n_cls, "cl_libs"); if (!lib->cl_libs) { ucc_error("failed to allocate %zd bytes for cl_libs", sizeof(ucc_cl_lib_t *) * n_cls); status = UCC_ERR_NO_MEMORY; goto error; } lib->cl_attrs = (ucc_cl_lib_attr_t *) ucc_calloc(n_cls, sizeof(ucc_cl_lib_attr_t), "cl_attrs"); if (!lib->cl_attrs) { ucc_error("failed to allocate %zd bytes for cl_attrs", sizeof(ucc_cl_lib_attr_t) * n_cls); status = UCC_ERR_NO_MEMORY; goto error; } attrs = lib->cl_attrs; if (!(params.mask & UCC_LIB_PARAM_FIELD_THREAD_MODE)) { params.mask |= UCC_LIB_PARAM_FIELD_THREAD_MODE; params.thread_mode = UCC_THREAD_SINGLE; } memset(&b_params, 0, sizeof(ucc_base_lib_params_t)); ucc_copy_lib_params(&b_params.params, ¶ms); ucc_assert(config->cls.count >= 1); lib->specific_cls_requested = (0 == ucc_cl_requested(config, UCC_CL_ALL)); lib->n_cl_libs_opened = 0; for (i = 0; i < n_cls; i++) { cl_iface = ucc_derived_of(ucc_global_config.cl_framework.components[i], ucc_cl_iface_t); /* User requested specific list of CLs and current cl_iface is not part of the list: skip it. */ if (lib->specific_cls_requested && (0 == ucc_cl_requested(config, cl_iface->type))) { continue; } status = ucc_cl_lib_config_read(cl_iface, lib->full_prefix, &cl_config); if (UCC_OK != status) { ucc_error("failed to read CL \"%s\" lib configuration", cl_iface->super.name); goto error_cfg_read; } // coverity[overrun-buffer-val:FALSE] status = cl_iface->lib.init(&b_params, &cl_config->super.super, &b_lib); if (UCC_OK != status) { if (lib->specific_cls_requested) { ucc_error("lib_init failed for component: %s", cl_iface->super.name); goto error_cl_init; } else { ucc_debug("lib_init failed for component: %s, skipping", cl_iface->super.name); ucc_base_config_release(&cl_config->super.super); continue; } } ucc_base_config_release(&cl_config->super.super); cl_lib = ucc_derived_of(b_lib, ucc_cl_lib_t); lib->cl_libs[lib->n_cl_libs_opened] = cl_lib; status = cl_iface->lib.get_attr(&cl_lib->super, &attrs[lib->n_cl_libs_opened].super); if (UCC_OK != status) { ucc_error("failed to query cl lib %s attr", cl_lib->iface->super.name); return status; } ucc_debug("lib_prefix \"%s\": initialized component \"%s\" score %d", config->full_prefix, cl_iface->super.name, cl_iface->super.score); if (attrs[lib->n_cl_libs_opened].super.attr.thread_mode > highest_tm) { highest_tm = attrs[lib->n_cl_libs_opened].super.attr.thread_mode; } if (attrs[i].super.attr.thread_mode < lowest_tm) { lowest_tm = attrs[lib->n_cl_libs_opened].super.attr.thread_mode; } lib->n_cl_libs_opened++; } if (lib->n_cl_libs_opened == 0) { ucc_error("lib_init failed: no CL libs were opened"); status = UCC_ERR_NO_MESSAGE; goto error; } if (highest_tm < params.thread_mode) { /* No CL can provide the thread_mode that user required. Leave all the selected components and set library thread_mode to lowest_tm */ ucc_info("selected set of CLs does not provide the requested " "thread_mode"); lib->attr.thread_mode = lowest_tm; } else if (lowest_tm < highest_tm) { /* Some CLs can support the required thread_mode but some can't. Lets try to satisfy user request and remove all the CLs with thread_mode < required */ int n_cl_libs_filtered = 0; lib->attr.thread_mode = params.thread_mode; for (i = 0; i < lib->n_cl_libs_opened; i++) { if (attrs[i].super.attr.thread_mode >= params.thread_mode) { lib->cl_libs[n_cl_libs_filtered] = lib->cl_libs[i]; attrs[n_cl_libs_filtered] = attrs[i]; n_cl_libs_filtered++; } } lib->n_cl_libs_opened = n_cl_libs_filtered; ucc_assert(n_cl_libs_filtered > 0); } else { /* All opened CLs can support required thread mode: leave them all*/ lib->attr.thread_mode = params.thread_mode; } for (i = 0; i < lib->n_cl_libs_opened; i++) { supported_coll_types |= attrs[i].super.attr.coll_types; } /* Check if the combination of the selected CLs provides all the requested coll_types: not an error, just print a message if not all the colls are supported */ if (params.mask & UCC_LIB_PARAM_FIELD_COLL_TYPES && ((params.coll_types & supported_coll_types) != params.coll_types)) { ucc_debug("selected set of CLs does not provide all the requested " "coll_types"); } lib->attr.coll_types = supported_coll_types; return UCC_OK; error_cl_init: ucc_base_config_release(&cl_config->super.super); error_cfg_read: for (i = 0; i < lib->n_cl_libs_opened; i++) { lib->cl_libs[i]->iface->lib.finalize(&lib->cl_libs[i]->super); } error: ucc_free(lib->cl_attrs); ucc_free(lib->cl_libs); return status; } int ucc_tl_is_required(ucc_lib_info_t *lib, ucc_tl_iface_t *tl_iface, int forced) { int i; for (i = 0; i < lib->n_cl_libs_opened; i++) { if (ucc_config_names_search(forced ? lib->cl_attrs[i].tls_forced : lib->cl_attrs[i].tls, tl_iface->super.name) >= 0) { return 1; } } return 0; } static ucc_status_t ucc_tl_lib_init(const ucc_lib_params_t *user_params, ucc_lib_info_t *lib) { ucc_status_t status; int i, n_tls; ucc_tl_lib_t *tl_lib; ucc_tl_lib_config_t *tl_config; ucc_base_lib_t * b_lib; ucc_base_lib_params_t b_params; ucc_tl_iface_t *tl_iface; ucc_copy_lib_params(&b_params.params, user_params); b_params.full_prefix = lib->full_prefix; n_tls = ucc_global_config.tl_framework.n_components; lib->tl_libs = (ucc_tl_lib_t **)ucc_malloc(sizeof(ucc_tl_lib_t *) * n_tls, "tl_libs"); lib->n_tl_libs_opened = 0; if (!lib->tl_libs) { ucc_error("failed to allocate %zd bytes for tl_libs", sizeof(ucc_tl_lib_t *) * n_tls); return UCC_ERR_NO_MEMORY; } for (i=0; ifull_prefix, &tl_config); if (UCC_OK != status) { ucc_warn("failed to read TL \"%s\" lib configuration", tl_iface->super.name); continue; } // coverity[overrun-buffer-val:FALSE] status = tl_iface->lib.init(&b_params, &tl_config->super.super, &b_lib); ucc_base_config_release(&tl_config->super.super); if (UCC_OK != status) { ucc_debug("lib_init failed for component: %s, skipping", tl_iface->super.name); continue; } tl_lib = ucc_derived_of(b_lib, ucc_tl_lib_t); lib->tl_libs[lib->n_tl_libs_opened++] = tl_lib; } } return UCC_OK; } ucc_status_t ucc_init_version(unsigned api_major_version, unsigned api_minor_version, const ucc_lib_params_t *params, const ucc_lib_config_h config, ucc_lib_h *lib_p) { unsigned major_version, minor_version, release_number; ucc_status_t status; ucc_lib_info_t *lib; ucc_mc_params_t mc_params = { .thread_mode = params->thread_mode, }; ucc_ec_params_t ec_params = { .thread_mode = params->thread_mode, }; *lib_p = NULL; if (UCC_OK != (status = ucc_constructor())) { return status; } if (UCC_OK != (status = ucc_mc_init(&mc_params))) { return status; } if (UCC_OK != (status = ucc_ec_init(&ec_params))) { return status; } if (UCC_OK != (status = ucc_sysinfo_init())) { return status; } if (UCC_OK != (status = ucc_sysinfo_get_host_info(&ucc_local_host))) { return status; } ucc_get_version(&major_version, &minor_version, &release_number); if ((api_major_version != major_version) || ((api_major_version == major_version) && (api_minor_version > minor_version))) { ucc_warn( "UCC version is incompatible, required: %d.%d, actual: %d.%d.%d", api_major_version, api_minor_version, major_version, minor_version, release_number); } lib = ucc_calloc(1, sizeof(ucc_lib_info_t), "lib_info"); if (!lib) { ucc_error("failed to allocate %zd bytes for lib_info", sizeof(ucc_lib_info_t)); return UCC_ERR_NO_MEMORY; } lib->full_prefix = strdup(config->full_prefix); if (!lib->full_prefix) { ucc_error("failed strdup for full_prefix"); status = UCC_ERR_NO_MEMORY; goto error; } /* Initialize ucc lib handle using requirements from the user provided via params/config and available CLs in the CL component framework. The lib_p object will contain the array of ucc_cl_lib_t objects that are allocated using CL init/finalize interface. */ status = ucc_cl_lib_init(params, config, lib); if (UCC_OK != status) { goto error; } status = ucc_tl_lib_init(params, lib); if (UCC_OK != status) { goto error; } status = ucc_mpool_init(&lib->stub_tasks_mp, 0, sizeof(ucc_coll_task_t), 0, UCC_CACHE_LINE_SIZE, 8, UINT_MAX, &ucc_coll_task_mpool_ops, UCC_THREAD_MULTIPLE, "stub_tasks"); if (status != UCC_OK) { goto error; } *lib_p = lib; return UCC_OK; error: ucc_free(lib); return status; } ucc_status_t ucc_lib_config_read(const char *env_prefix, const char *filename, ucc_lib_config_t **config_p) { ucc_lib_config_t *config; ucc_status_t status; size_t full_prefix_len; const char *base_prefix = "UCC_"; if (UCC_OK != (status = ucc_constructor())) { return status; } if (filename != NULL) { ucc_error("read from file is not implemented"); status = UCC_ERR_NOT_IMPLEMENTED; goto err; } config = ucc_malloc(sizeof(*config), "lib_config"); if (config == NULL) { ucc_error("failed to allocate %zd bytes for lib_config", sizeof(*config)); status = UCC_ERR_NO_MEMORY; goto err; } /* full_prefix for a UCC lib config is either just (i) "UCC_" - if no "env_prefix" is provided, or (ii) "AAA_UCC" - where AAA is the value of "env_prefix". Allocate space to build prefix str and two characters ("_" and "\0") */ full_prefix_len = strlen(base_prefix) + (env_prefix ? strlen(env_prefix) : 0) + 2; config->full_prefix = ucc_malloc(full_prefix_len, "full_prefix"); if (!config->full_prefix) { ucc_error("failed to allocate %zd bytes for full_prefix", full_prefix_len); status = UCC_ERR_NO_MEMORY; goto err_free_config; } if (env_prefix) { ucc_snprintf_safe(config->full_prefix, full_prefix_len, "%s_%s", env_prefix, base_prefix); } else { ucc_strncpy_safe(config->full_prefix, base_prefix, strlen(base_prefix) + 1); } status = ucc_config_parser_fill_opts(config, UCC_CONFIG_GET_TABLE(ucc_lib_config_table), config->full_prefix, 0); if (status != UCC_OK) { ucc_error("failed to read UCC lib config"); goto err_free_prefix; } *config_p = config; return UCC_OK; err_free_prefix: ucc_free(config->full_prefix); err_free_config: ucc_free(config); err: return status; } void ucc_lib_config_release(ucc_lib_config_t *config) { ucc_config_parser_release_opts(config, ucc_lib_config_table); ucc_free(config->full_prefix); ucc_free(config); } void ucc_lib_config_print(const ucc_lib_config_h config, FILE *stream, const char *title, ucc_config_print_flags_t print_flags) { ucc_config_parser_print_opts(stream, title, config, ucc_lib_config_table, NULL, config->full_prefix, print_flags); } ucc_status_t ucc_lib_config_modify(ucc_lib_config_h config, const char *name, const char *value) { return ucc_config_parser_set_value(config, ucc_lib_config_table, name, value); } ucc_status_t ucc_lib_get_attr(ucc_lib_h lib_p, ucc_lib_attr_t *lib_attr) { ucc_lib_info_t *lib = (ucc_lib_info_t *)lib_p; if (lib_attr->mask & UCC_LIB_ATTR_FIELD_THREAD_MODE) { lib_attr->thread_mode = lib->attr.thread_mode; } if (lib_attr->mask & UCC_LIB_ATTR_FIELD_COLL_TYPES) { lib_attr->coll_types = lib->attr.coll_types; } if ((lib_attr->mask & UCC_LIB_ATTR_FIELD_REDUCTION_TYPES) || (lib_attr->mask & UCC_LIB_ATTR_FIELD_SYNC_TYPE)) { return UCC_ERR_NOT_SUPPORTED; } return UCC_OK; } ucc_status_t ucc_finalize(ucc_lib_info_t *lib) { int i; ucc_status_t status, gl_status; gl_status = UCC_OK; ucc_assert(lib->n_cl_libs_opened > 0); ucc_assert(lib->cl_libs != NULL); ucc_mpool_cleanup(&lib->stub_tasks_mp, 1); for (i = 0; i < lib->n_tl_libs_opened; i++) { lib->tl_libs[i]->iface->lib.finalize(&lib->tl_libs[i]->super); } for (i = 0; i < lib->n_cl_libs_opened; i++) { lib->cl_libs[i]->iface->lib.finalize(&lib->cl_libs[i]->super); } status = ucc_mc_finalize(); if (status != UCC_OK) { gl_status = status; } status = ucc_ec_finalize(); if (status != UCC_OK) { gl_status = status; } ucc_free(lib->tl_libs); ucc_free(lib->cl_libs); ucc_free(lib->full_prefix); ucc_free(lib->cl_attrs); ucc_free(lib); return gl_status; } ucc-1.8.0/src/core/ucc_dt.h0000664000175000017500000000433215211535620015611 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_DT_H_ #define UCC_DT_H_ #include "config.h" #include "ucc/api/ucc.h" #include "utils/ucc_assert.h" typedef struct ucc_dt_generic { void *context; ucc_generic_dt_ops_t ops; } ucc_dt_generic_t; #define UCC_DT_PREDEFINED_ID(_dt) ((_dt) >> UCC_DATATYPE_SHIFT) #define UCC_DT_IS_GENERIC(_dt) \ (((_dt) & UCC_DATATYPE_CLASS_MASK) == UCC_DATATYPE_GENERIC) #define UCC_DT_IS_PREDEFINED(_dt) \ (((_dt) & UCC_DATATYPE_CLASS_MASK) == UCC_DATATYPE_PREDEFINED) #define UCC_DT_GENERIC_IS_CONTIG(_dt) (((_dt)->ops.mask & UCC_GENERIC_DT_OPS_FIELD_FLAGS) && \ ((_dt)->ops.flags & UCC_GENERIC_DT_OPS_FLAG_CONTIG)) #define UCC_DT_GENERIC_HAS_REDUCE(_dt) (((_dt)->ops.mask & UCC_GENERIC_DT_OPS_FIELD_FLAGS) && \ ((_dt)->ops.flags & UCC_GENERIC_DT_OPS_FLAG_REDUCE)) #define UCC_DT_IS_CONTIG(_dt) (UCC_DT_IS_GENERIC(_dt) && \ UCC_DT_GENERIC_IS_CONTIG(ucc_dt_to_generic(_dt))) #define UCC_DT_HAS_REDUCE(_dt) (UCC_DT_IS_GENERIC(_dt) && \ UCC_DT_GENERIC_HAS_REDUCE(ucc_dt_to_generic(_dt))) static inline ucc_dt_generic_t* ucc_dt_to_generic(ucc_datatype_t datatype) { return (ucc_dt_generic_t*)(void*)(datatype & ~UCC_DATATYPE_CLASS_MASK); } static inline ucc_datatype_t ucc_dt_from_generic(ucc_dt_generic_t* dt_gen) { return ((uintptr_t)dt_gen) | UCC_DATATYPE_GENERIC; } static inline size_t ucc_contig_dt_size(ucc_datatype_t datatype) { return ucc_dt_to_generic(datatype)->ops.contig_size; } extern size_t ucc_dt_predefined_sizes[UCC_DT_PREDEFINED_LAST]; static inline size_t ucc_dt_size(ucc_datatype_t dt) { if (UCC_DT_IS_PREDEFINED(dt)) { return ucc_dt_predefined_sizes[UCC_DT_PREDEFINED_ID(dt)]; } else if (UCC_DT_IS_CONTIG(dt)) { return ucc_contig_dt_size(dt); } /* GENERIC callback pack/unpack TODO remove ucc_likely once custom datatype is implemented does not matter what to return - we should not get here */ ucc_assert(0); return SIZE_MAX; } #endif ucc-1.8.0/src/core/ucc_progress_queue.h0000664000175000017500000000326715211535620020260 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_PROGRESS_QUEUE_H_ #define UCC_PROGRESS_QUEUE_H_ #include "ucc/api/ucc.h" #include "schedule/ucc_schedule.h" typedef struct ucc_progress_queue ucc_progress_queue_t; struct ucc_progress_queue { void (*enqueue)(ucc_progress_queue_t *pq, ucc_coll_task_t *task); void (*dequeue)(ucc_progress_queue_t *pq, ucc_coll_task_t **task); int (*progress)(ucc_progress_queue_t *pq); int (*is_empty)(ucc_progress_queue_t *pq); void (*finalize)(ucc_progress_queue_t *pq); }; ucc_status_t ucc_progress_queue_init(ucc_progress_queue_t **pq, ucc_thread_mode_t tm, uint32_t lock_free_progress_q); static inline void ucc_progress_enqueue(ucc_progress_queue_t *pq, ucc_coll_task_t *task) { pq->enqueue(pq, task); } static inline ucc_status_t ucc_progress_queue_enqueue(ucc_progress_queue_t *pq, ucc_coll_task_t *task) { task->progress(task); if (task->status != UCC_INPROGRESS) { /* task completed immediately, don't add it to the progress queue */ return ucc_task_complete(task); } /* set user visible status */ task->super.status = UCC_INPROGRESS; pq->enqueue(pq, task); return UCC_OK; } static inline int ucc_progress_queue(ucc_progress_queue_t *pq) { return pq->progress(pq); } static inline int ucc_progress_queue_is_empty(ucc_progress_queue_t *pq) { return pq->is_empty(pq); } void ucc_progress_queue_finalize(ucc_progress_queue_t *pq); #endif ucc-1.8.0/src/core/ucc_team.h0000664000175000017500000001163215211535620016131 0ustar alastairalastair/** * Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TEAM_H_ #define UCC_TEAM_H_ #include "ucc/api/ucc.h" #include "utils/ucc_datastruct.h" #include "utils/ucc_coll_utils.h" #include "ucc_context.h" #include "utils/ucc_math.h" #include "components/base/ucc_base_iface.h" #include "components/cl/ucc_cl.h" #include "components/tl/ucc_tl.h" #include "coll_score/ucc_coll_score.h" typedef struct ucc_service_coll_req ucc_service_coll_req_t; typedef enum { UCC_TEAM_ADDR_EXCHANGE, UCC_TEAM_SERVICE_TEAM, UCC_TEAM_ALLOC_ID, UCC_TEAM_CL_CREATE, UCC_TEAM_ACTIVE, } ucc_team_state_t; typedef struct ucc_team { ucc_team_state_t state; ucc_context_t ** contexts; uint32_t num_contexts; ucc_base_team_params_t bp; ucc_team_oob_coll_t runtime_oob; ucc_cl_team_t ** cl_teams; int n_cl_teams; int last_team_create_posted; uint16_t id; /*< context-uniq team identifier */ ucc_rank_t rank; ucc_rank_t size; ucc_tl_team_t * service_team; ucc_service_coll_req_t *sreq; ucc_addr_storage_t addr_storage; /*< addresses of team endpoints */ ucc_rank_t * ctx_ranks; void * oob_req; ucc_ep_map_t ctx_map; /*< map to the ctx ranks, defined if CTX type is global (oob provided) */ ucc_topo_t *topo; ucc_score_map_t *score_map; /*< score map of CLs */ uint32_t seq_num; } ucc_team_t; /* If the bit is set then team_id is provided by the user */ #define UCC_TEAM_ID_EXTERNAL_BIT ((uint16_t)UCC_BIT(15)) #define UCC_TEAM_ID_IS_EXTERNAL(_team) (team->id & UCC_TEAM_ID_EXTERNAL_BIT) #define UCC_TEAM_ID_MAX ((uint16_t)UCC_BIT(15) - 1) void ucc_copy_team_params(ucc_team_params_t *dst, const ucc_team_params_t *src); /* Returns addressing information for "rank" in a team. If ucc context was created with OOB then addr storage is located on context. In that case we need to map rank to ctx_rank first. Otherwise, addr storage is per-team: just use rank then. The returned value is "header": it stores proc_info, host info, ctx_id and addresses of TL/CL components.*/ static inline ucc_context_addr_header_t * ucc_get_team_ep_header(ucc_context_t *context, ucc_team_t *team, ucc_rank_t rank) { ucc_addr_storage_t *storage = context->addr_storage.storage ? &context->addr_storage : &team->addr_storage; ucc_rank_t storage_rank = context->addr_storage.storage ? (team ? ucc_ep_map_eval(team->ctx_map, rank) : rank) : rank; return UCC_ADDR_STORAGE_RANK_HEADER(storage, storage_rank); } /* Gets the component specific address of rank in a team. First we get the header, and then find the component address by offset */ static inline void *ucc_get_team_ep_addr(ucc_context_t *context, ucc_team_t *team, ucc_rank_t rank, unsigned long component_id) { ucc_context_addr_header_t *h = ucc_get_team_ep_header(context, team, rank); void *addr = NULL; int i; for (i = 0; i < h->n_components; i++) { if (h->components[i].id == component_id) { addr = PTR_OFFSET(h, h->components[i].offset); break; } } ucc_assert(NULL != addr); return addr; } static inline ucc_rank_t ucc_get_ctx_rank(ucc_team_t *team, ucc_rank_t team_rank) { return ucc_ep_map_eval(team->ctx_map, team_rank); } static inline ucc_host_id_t ucc_team_rank_host_id(ucc_rank_t rank, ucc_team_t *team) { return team->topo->topo->procs[ucc_get_ctx_rank(team, rank)].host_id; } static inline int ucc_team_ranks_on_same_node(ucc_rank_t rank1, ucc_rank_t rank2, ucc_team_t *team) { ucc_context_addr_header_t *h1 = ucc_get_team_ep_header(team->contexts[0], team, rank1); ucc_context_addr_header_t *h2 = ucc_get_team_ep_header(team->contexts[0], team, rank2); return h1->ctx_id.pi.host_hash == h2->ctx_id.pi.host_hash; } static inline int ucc_team_map_is_single_node(ucc_team_t *team, ucc_ep_map_t map) { uint64_t i; ucc_rank_t r, r0; r0 = ucc_ep_map_eval(map, 0); for (i = 1; i < map.ep_num; i++) { r = ucc_ep_map_eval(map, i); if (!ucc_team_ranks_on_same_node(r0, r, team)) { return 0; } } return 1; } #endif ucc-1.8.0/src/core/ucc_global_opts.h0000664000175000017500000000421215211535620017504 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_GLOBAL_OPTS_H_ #define UCC_GLOBAL_OPTS_H_ #include "config.h" #include "utils/ucc_component.h" #include "utils/ucc_parser.h" #include "utils/debug/types.h" typedef struct ucc_global_config { /* Log level above which log messages will be printed*/ ucc_log_component_config_t log_component; /* Print collective info for each initialized collective */ ucc_log_component_config_t coll_trace; ucc_component_framework_t cl_framework; ucc_component_framework_t tl_framework; ucc_component_framework_t mc_framework; ucc_component_framework_t ec_framework; ucc_component_framework_t sysinfo_framework; /* Coll component libraries path */ char *component_path; char *install_path; int initialized; /* Profiling mode */ uint64_t profile_mode; /* Profiling output file name */ char *profile_file; /* Limit for profiling log size */ size_t profile_log_size; char *cfg_filename; ucc_file_config_t *file_cfg; /* Log file */ char *log_file; /* Maximal log file size */ size_t log_file_size; /* Maximal backup log files count that could be created by log infrastructure */ unsigned log_file_rotate; /* Size of log buffer for one message */ size_t log_buffer_size; /* Maximal amount of packet data to print per packet */ size_t log_data_size; /* Enable ucc_print() output */ int log_print_enable; /* Log level to trigger error handling */ ucc_log_level_t log_level_trigger; /* Check for asymmetric datatypes in rooted collectives */ int check_asymmetric_dt; } ucc_global_config_t; extern ucc_global_config_t ucc_global_config; extern ucc_config_field_t ucc_global_config_table[]; ucc_status_t ucc_constructor(void); extern ucs_list_link_t ucc_config_global_list; #endif ucc-1.8.0/src/core/ucc_lib.h0000664000175000017500000000222715211535620015751 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_LIB_H_ #define UCC_LIB_H_ #include "config.h" #include "ucc/api/ucc.h" #include "components/cl/ucc_cl_type.h" #include "utils/ucc_parser.h" #include "utils/ucc_mpool.h" typedef struct ucc_cl_lib ucc_cl_lib_t; typedef struct ucc_tl_lib ucc_tl_lib_t; typedef struct ucc_cl_lib_attr ucc_cl_lib_attr_t; typedef struct ucc_tl_iface ucc_tl_iface_t; typedef struct ucc_lib_config { char *full_prefix; struct { ucc_cl_type_t *types; unsigned count; } cls; } ucc_lib_config_t; typedef struct ucc_lib_info { char *full_prefix; int n_cl_libs_opened; int n_tl_libs_opened; ucc_cl_lib_t **cl_libs; ucc_tl_lib_t **tl_libs; ucc_lib_attr_t attr; int specific_cls_requested; ucc_cl_lib_attr_t *cl_attrs; ucc_mpool_t stub_tasks_mp; } ucc_lib_info_t; int ucc_tl_is_required(ucc_lib_info_t *lib, ucc_tl_iface_t *tl_iface, int forced); #endif ucc-1.8.0/src/core/ucc_context.h0000664000175000017500000001617715211535620016700 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_CONTEXT_H_ #define UCC_CONTEXT_H_ #include "ucc/api/ucc.h" #include "ucc_progress_queue.h" #include "utils/ucc_list.h" #include "utils/ucc_proc_info.h" #include "components/topo/ucc_topo.h" #define UCC_MEM_MAP_TL_NAME_LEN 8 typedef struct ucc_lib_info ucc_lib_info_t; typedef struct ucc_cl_context ucc_cl_context_t; typedef struct ucc_tl_context ucc_tl_context_t; typedef struct ucc_cl_context_config ucc_cl_context_config_t; typedef struct ucc_tl_context_config ucc_tl_context_config_t; typedef struct ucc_tl_team ucc_tl_team_t; typedef unsigned (*ucc_context_progress_fn_t)(void *progress_arg); typedef struct ucc_team_id_pool { uint64_t *pool; uint32_t pool_size; } ucc_team_id_pool_t; typedef struct ucc_context_id { ucc_proc_info_t pi; uint32_t seq_num; } ucc_context_id_t; #define UCC_CTX_ID_EQUAL(_id1, _id2) (UCC_PROC_INFO_EQUAL((_id1).pi, (_id2).pi) \ && (_id1).seq_num == (_id2).seq_num) enum { /* all ranks have identical set of TLs*/ UCC_ADDR_STORAGE_FLAG_TLS_SYMMETRIC = UCC_BIT(0), }; typedef struct ucc_addr_storage { void *storage; void *oob_req; size_t addr_len; ucc_rank_t size; ucc_rank_t rank; uint64_t flags; } ucc_addr_storage_t; typedef struct ucc_context { ucc_lib_info_t *lib; ucc_context_params_t params; ucc_context_attr_t attr; ucc_thread_mode_t thread_mode; ucc_cl_context_t **cl_ctx; ucc_tl_context_t **tl_ctx; ucc_tl_context_t *service_ctx; unsigned n_cl_ctx; unsigned n_tl_ctx; /** * number of TL/CL components whose addresses are packed into * ucc_context->attr.addr */ int n_addr_packed; ucc_config_names_array_t all_tls; ucc_config_names_array_t net_devices; ucc_list_link_t progress_list; ucc_progress_queue_t *pq; ucc_team_id_pool_t ids; ucc_context_id_t id; ucc_addr_storage_t addr_storage; /** * rank of a process in the "global" (with OOB) context */ ucc_rank_t rank; ucc_context_topo_t *topo; uint64_t cl_flags; ucc_tl_team_t *service_team; int32_t throttle_progress; } ucc_context_t; typedef struct ucc_context_config { ucc_lib_info_t *lib; ucc_cl_context_config_t **cl_cfgs; ucc_tl_context_config_t **tl_cfgs; int n_cl_cfg; int n_tl_cfg; uint32_t team_ids_pool_size; uint32_t estimated_num_eps; uint32_t estimated_num_ppn; uint32_t lock_free_progress_q; uint32_t internal_oob; uint32_t throttle_progress; ucs_config_names_array_t net_devices; unsigned long node_local_id; } ucc_context_config_t; typedef struct ucc_mem_map_tl_t { size_t packed_size; char tl_name[UCC_MEM_MAP_TL_NAME_LEN]; void *tl_data; /* tl specific data */ } ucc_mem_map_tl_t; typedef struct ucc_mem_map_memh_t { ucc_mem_map_mode_t mode; ucc_context_h context; void *address; size_t len; ucc_mem_map_tl_t *tl_h; int num_tls; char pack_buffer[0]; } ucc_mem_map_memh_t; /* Internal function for context creation that takes explicit pointer for proc_info */ ucc_status_t ucc_context_create_proc_info(ucc_lib_h lib, const ucc_context_params_t *params, const ucc_context_config_h config, ucc_context_h *context, ucc_proc_info_t *proc_info); /* Any internal UCC component (TL, CL, etc) may register its own progress callback fn (and argument for the callback) into core ucc context. Those callbacks will be triggered as part of ucc_context_progress. Any progress callback fn inserted is required to be thread safe. If not, we need to add to this engine a thread safe mechanism. */ ucc_status_t ucc_context_progress_register(ucc_context_t *ctx, ucc_context_progress_fn_t fn, void *progress_arg); ucc_status_t ucc_context_progress_deregister(ucc_context_t *ctx, ucc_context_progress_fn_t fn, void *progress_arg); /* Performs address exchange between the processes group defined by OOB. This function can be used either at context creation time (if ctx is global) or at team creation time. The function is non-blocking and can return UCC_INPROGRESS. If caller needs a blocking behavior then the function must be called until UCC_OK is returned. The addresses are stored in the addr_storage data structure. The addressing data of rank "i" (according to OOB) can be accessed with UCC_ADDR_STORAGE_RANK_HEADER macro defined below. */ ucc_status_t ucc_core_addr_exchange(ucc_context_t *context, ucc_oob_coll_t *oob, ucc_addr_storage_t *addr_storage); /* UCC context packed address layout: -------------------------------------------------------------------------- |ctx_id|host_info|n_components|id0|offset0|id1|offset1|..| |idN|offsetN|data0|data1|..|dataN| -------------------------------------------------------------------------- each component can extract its own addressing using offset into data. Offset is found by id. */ typedef struct ucc_context_addr_header { ucc_context_id_t ctx_id; ucc_host_info_t host_info; int n_components; // Number of CL/TL components whose address is packed struct { unsigned long id; // id of component computed during framework load ptrdiff_t offset; // offset of the address of the component in the // packed data array. Component from the start of the header } components[1]; } ucc_context_addr_header_t; #define UCC_CONTEXT_ADDR_HEADER_SIZE(_n_components) \ ({ \ ucc_context_addr_header_t _h; \ size_t _size; \ _size = sizeof(_h) + sizeof(_h.components[0]) * (_n_components - 1); \ _size; \ }) #define UCC_CONTEXT_ADDR_DATA(_header) \ PTR_OFFSET(_header, UCC_CONTEXT_ADDR_HEADER_SIZE(_header->n_components)) #define UCC_ADDR_STORAGE_RANK_HEADER(_storage, _rank) \ (ucc_context_addr_header_t *)PTR_OFFSET((_storage)->storage, \ (_storage)->addr_len *(_rank)) #endif ucc-1.8.0/src/core/ucc_progress_queue_st.c0000664000175000017500000000545215211535620020757 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "ucc_progress_queue.h" #include "utils/ucc_malloc.h" #include "utils/ucc_log.h" #include "utils/ucc_time.h" #include "utils/ucc_coll_utils.h" typedef struct ucc_pq_st { ucc_progress_queue_t super; ucc_list_link_t list; } ucc_pq_st_t; static int ucc_pq_st_progress(ucc_progress_queue_t *pq) { ucc_pq_st_t *pq_st = ucc_derived_of(pq, ucc_pq_st_t); int n_progressed = 0; double timestamp = -1; ucc_coll_task_t *task, *tmp; ucc_status_t status; ucc_list_for_each_safe(task, tmp, &pq_st->list, list_elem) { ucc_assert((task->status != UCC_OPERATION_INITIALIZED) && (task->super.status != UCC_OPERATION_INITIALIZED)); if (task->progress) { ucc_assert(task->status != UCC_OK); task->progress(task); } if (UCC_INPROGRESS == task->status) { if (UCC_COLL_TIMEOUT_REQUIRED(task)) { if (timestamp < 0) { timestamp = ucc_get_time(); } if (ucc_unlikely(timestamp - task->start_time > task->bargs.args.timeout)) { task->status = UCC_ERR_TIMED_OUT; ucc_list_del(&task->list_elem); ucc_task_complete(task); return UCC_ERR_TIMED_OUT; } } continue; } ucc_list_del(&task->list_elem); n_progressed++; if (0 > (status = ucc_task_complete(task))) { return status; } } return n_progressed; } static void ucc_pq_st_enqueue(ucc_progress_queue_t *pq, ucc_coll_task_t *task) { ucc_pq_st_t *pq_st = ucc_derived_of(pq, ucc_pq_st_t); ucc_list_add_tail(&pq_st->list, &task->list_elem); } static void ucc_pq_st_finalize(ucc_progress_queue_t *pq) { ucc_pq_st_t *pq_st = ucc_derived_of(pq, ucc_pq_st_t); ucc_free(pq_st); } static int ucc_pq_st_is_empty(ucc_progress_queue_t *pq) { ucc_pq_st_t *pq_st = ucc_derived_of(pq, ucc_pq_st_t); return ucc_list_is_empty(&pq_st->list); } ucc_status_t ucc_pq_st_init(ucc_progress_queue_t **pq) { ucc_pq_st_t *pq_st = ucc_malloc(sizeof(*pq_st), "pq_st"); if (!pq_st) { ucc_error("failed to allocate %zd bytes for pq_st", sizeof(*pq_st)); return UCC_ERR_NO_MEMORY; } ucc_list_head_init(&pq_st->list); pq_st->super.enqueue = ucc_pq_st_enqueue; pq_st->super.dequeue = NULL; pq_st->super.progress = ucc_pq_st_progress; pq_st->super.finalize = ucc_pq_st_finalize; pq_st->super.is_empty = ucc_pq_st_is_empty; *pq = &pq_st->super; return UCC_OK; } ucc-1.8.0/src/core/ucc_progress_queue_mt.c0000664000175000017500000001233015211535620020742 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "ucc_progress_queue.h" #include "utils/ucc_malloc.h" #include "utils/ucc_log.h" #include "utils/ucc_time.h" #include "utils/ucc_spinlock.h" #include "utils/ucc_list.h" #include "utils/ucc_lock_free_queue.h" #include "utils/ucc_coll_utils.h" typedef struct ucc_pq_mt { ucc_progress_queue_t super; ucc_lf_queue_t lf_queue; } ucc_pq_mt_t; typedef struct ucc_pq_mt_locked { ucc_progress_queue_t super; ucc_spinlock_t queue_lock; ucc_list_link_t queue; } ucc_pq_mt_locked_t; static void ucc_pq_locked_mt_enqueue(ucc_progress_queue_t *pq, ucc_coll_task_t *task) { ucc_pq_mt_locked_t *pq_mt = ucc_derived_of(pq, ucc_pq_mt_locked_t); ucc_spin_lock(&pq_mt->queue_lock); ucc_list_add_tail(&pq_mt->queue, &task->list_elem); ucc_spin_unlock(&pq_mt->queue_lock); } static void ucc_pq_mt_enqueue(ucc_progress_queue_t *pq, ucc_coll_task_t *task) { ucc_pq_mt_t *pq_mt = ucc_derived_of(pq, ucc_pq_mt_t); ucc_lf_queue_enqueue(&pq_mt->lf_queue, &task->lf_elem); } static void ucc_pq_locked_mt_dequeue(ucc_progress_queue_t *pq, ucc_coll_task_t **popped_task) { ucc_pq_mt_locked_t *pq_mt = ucc_derived_of(pq, ucc_pq_mt_locked_t); *popped_task = NULL; ucc_spin_lock(&pq_mt->queue_lock); if (!ucc_list_is_empty(&pq_mt->queue)) { *popped_task = ucc_list_extract_head(&pq_mt->queue, ucc_coll_task_t, list_elem); } ucc_spin_unlock(&pq_mt->queue_lock); } static void ucc_pq_mt_dequeue(ucc_progress_queue_t *pq, ucc_coll_task_t **popped_task) { ucc_pq_mt_t *pq_mt = ucc_derived_of(pq, ucc_pq_mt_t); ucc_lf_queue_elem_t *elem = ucc_lf_queue_dequeue(&pq_mt->lf_queue, 1); *popped_task = elem ? ucc_container_of(elem, ucc_coll_task_t, lf_elem) : NULL; } static int ucc_pq_mt_progress(ucc_progress_queue_t *pq) { int n_progressed = 0; double timestamp = -1; ucc_coll_task_t *task; ucc_status_t status; pq->dequeue(pq, &task); if (task) { if (task->progress) { task->progress(task); } if (UCC_INPROGRESS == task->status) { if (UCC_COLL_TIMEOUT_REQUIRED(task)) { if (timestamp < 0) { timestamp = ucc_get_time(); } if (ucc_unlikely(timestamp - task->start_time > task->bargs.args.timeout)) { task->status = UCC_ERR_TIMED_OUT; ucc_task_complete(task); return UCC_ERR_TIMED_OUT; } } pq->enqueue(pq, task); return n_progressed; } n_progressed++; if (ucc_unlikely(0 > (status = ucc_task_complete(task)))) { return status; } } return n_progressed; } static int ucc_pq_locked_mt_is_empty(ucc_progress_queue_t *pq) { ucc_pq_mt_locked_t *pq_mt = ucc_derived_of(pq, ucc_pq_mt_locked_t); /* this function should not be very accurate for the purpose of progress throttling */ return ucc_list_is_empty(&pq_mt->queue); } static int ucc_pq_mt_is_empty(ucc_progress_queue_t *pq) //NOLINT: pq is unused { /* lock free progress queue never use throttling */ return 0; } static void ucc_pq_locked_mt_finalize(ucc_progress_queue_t *pq) { ucc_pq_mt_locked_t *pq_mt = ucc_derived_of(pq, ucc_pq_mt_locked_t); ucc_spinlock_destroy(&pq_mt->queue_lock); ucc_free(pq_mt); } static void ucc_pq_mt_finalize(ucc_progress_queue_t *pq) { ucc_pq_mt_t *pq_mt = ucc_derived_of(pq, ucc_pq_mt_t); ucc_lf_queue_destroy(&pq_mt->lf_queue); ucc_free(pq_mt); } ucc_status_t ucc_pq_mt_init(ucc_progress_queue_t **pq, uint32_t lock_free_progress_q) { if (lock_free_progress_q) { ucc_pq_mt_t *pq_mt = ucc_malloc(sizeof(*pq_mt), "pq_mt"); if (!pq_mt) { ucc_error("failed to allocate %zd bytes for pq_mt", sizeof(*pq_mt)); return UCC_ERR_NO_MEMORY; } ucc_lf_queue_init(&pq_mt->lf_queue); pq_mt->super.enqueue = ucc_pq_mt_enqueue; pq_mt->super.dequeue = ucc_pq_mt_dequeue; pq_mt->super.progress = ucc_pq_mt_progress; pq_mt->super.finalize = ucc_pq_mt_finalize; pq_mt->super.is_empty = ucc_pq_mt_is_empty; *pq = &pq_mt->super; } else { ucc_pq_mt_locked_t *pq_mt = ucc_malloc(sizeof(*pq_mt), "pq_mt"); if (!pq_mt) { ucc_error("failed to allocate %zd bytes for pq_mt", sizeof(*pq_mt)); return UCC_ERR_NO_MEMORY; } ucc_spinlock_init(&pq_mt->queue_lock, 0); ucc_list_head_init(&pq_mt->queue); pq_mt->super.enqueue = ucc_pq_locked_mt_enqueue; pq_mt->super.dequeue = ucc_pq_locked_mt_dequeue; pq_mt->super.progress = ucc_pq_mt_progress; pq_mt->super.finalize = ucc_pq_locked_mt_finalize; pq_mt->super.is_empty = ucc_pq_locked_mt_is_empty; *pq = &pq_mt->super; } return UCC_OK; } ucc-1.8.0/src/core/ucc_progress_queue.c0000664000175000017500000000142715211535620020247 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "config.h" #include "ucc_progress_queue.h" ucc_status_t ucc_pq_st_init(ucc_progress_queue_t **pq); ucc_status_t ucc_pq_mt_init(ucc_progress_queue_t **pq, uint32_t lock_free_progress_q); ucc_status_t ucc_progress_queue_init(ucc_progress_queue_t **pq, ucc_thread_mode_t tm, uint32_t lock_free_progress_q) { if (tm == UCC_THREAD_SINGLE) { return ucc_pq_st_init(pq); } else { // TODO also for UCC_THREAD_FUNNELED? return ucc_pq_mt_init(pq, lock_free_progress_q); } } void ucc_progress_queue_finalize(ucc_progress_queue_t *pq) { return pq->finalize(pq); } ucc-1.8.0/src/core/ucc_version.c.in0000664000175000017500000000070615211535620017270 0ustar alastairalastair/* * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ void ucc_get_version(unsigned *major_version, unsigned *minor_version, unsigned *release_number) { *major_version = @MAJOR_VERSION@; *minor_version = @MINOR_VERSION@; *release_number = @PATCH_VERSION@; } const char *ucc_get_version_string() { return "@MAJOR_VERSION@.@MINOR_VERSION@.@PATCH_VERSION@"; } ucc-1.8.0/src/core/ucc_coll.c0000664000175000017500000006121015211535620016124 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "ucc_team.h" #include "ucc_context.h" #include "components/mc/ucc_mc.h" #include "components/ec/ucc_ec.h" #include "components/cl/ucc_cl.h" #include "utils/ucc_malloc.h" #include "utils/ucc_log.h" #include "utils/ucc_coll_utils.h" #include "utils/ucc_time.h" #include "utils/profile/ucc_profile_core.h" #include "schedule/ucc_schedule.h" #include "coll_score/ucc_coll_score.h" #include "ucc_ee.h" #include "ucc_global_opts.h" #include "ucc_service_coll.h" #define UCC_BUFFER_INFO_CHECK_MEM_TYPE(_info) do { \ if ((_info).mem_type == UCC_MEMORY_TYPE_UNKNOWN) { \ ucc_mem_attr_t mem_attr; \ ucc_status_t st; \ mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; \ st = ucc_mc_get_mem_attr((_info).buffer, &mem_attr); \ if (ucc_unlikely(st != UCC_OK)) { \ return st; \ } \ (_info).mem_type = mem_attr.mem_type; \ } \ } while(0) #define UCC_BUFFER_INFO_CHECK_DATATYPE(_info1, _info2) do { \ if ((_info1).datatype != (_info2).datatype) { \ ucc_error("datatype missmatch"); \ return UCC_ERR_INVALID_PARAM; \ } \ } while(0) #if ENABLE_DEBUG == 1 static ucc_status_t ucc_check_coll_args(const ucc_coll_args_t *coll_args, ucc_rank_t rank) { switch (coll_args->coll_type) { case UCC_COLL_TYPE_ALLREDUCE: case UCC_COLL_TYPE_REDUCE_SCATTER: if (!UCC_IS_INPLACE(*coll_args)) { UCC_BUFFER_INFO_CHECK_DATATYPE(coll_args->src.info, coll_args->dst.info); } break; case UCC_COLL_TYPE_REDUCE: if (!UCC_IS_INPLACE(*coll_args) && UCC_IS_ROOT(*coll_args, rank)) { UCC_BUFFER_INFO_CHECK_DATATYPE(coll_args->src.info, coll_args->dst.info); } break; case UCC_COLL_TYPE_BCAST: case UCC_COLL_TYPE_BARRIER: case UCC_COLL_TYPE_FANIN: case UCC_COLL_TYPE_FANOUT: if (UCC_IS_INPLACE(*coll_args)) { ucc_warn("Inplace flag for %s is not defined by UCC API", ucc_coll_type_str(coll_args->coll_type)); } break; default: return UCC_OK; } return UCC_OK; } #else #define ucc_check_coll_args(...) UCC_OK #endif static ucc_status_t ucc_coll_args_check_mem_type(ucc_coll_args_t *coll_args, ucc_rank_t rank) { switch (coll_args->coll_type) { case UCC_COLL_TYPE_BARRIER: case UCC_COLL_TYPE_FANIN: case UCC_COLL_TYPE_FANOUT: return UCC_OK; case UCC_COLL_TYPE_BCAST: UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->src.info); return UCC_OK; case UCC_COLL_TYPE_ALLREDUCE: UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->dst.info); if (!UCC_IS_INPLACE(*coll_args)) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->src.info); } else { coll_args->src.info.mem_type = coll_args->dst.info.mem_type; } return UCC_OK; case UCC_COLL_TYPE_ALLGATHER: case UCC_COLL_TYPE_ALLTOALL: case UCC_COLL_TYPE_REDUCE_SCATTER: UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->dst.info); if (!UCC_IS_INPLACE(*coll_args)) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->src.info); } return UCC_OK; case UCC_COLL_TYPE_ALLGATHERV: case UCC_COLL_TYPE_REDUCE_SCATTERV: UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->dst.info_v); if (!UCC_IS_INPLACE(*coll_args)) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->src.info); } return UCC_OK; case UCC_COLL_TYPE_ALLTOALLV: UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->dst.info_v); if (!UCC_IS_INPLACE(*coll_args)) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->src.info_v); } return UCC_OK; case UCC_COLL_TYPE_GATHER: // TODO: Check logic for Gather once implemented case UCC_COLL_TYPE_REDUCE: if (!UCC_IS_ROOT(*coll_args, rank)) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->src.info); } else { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->dst.info); if (!UCC_IS_INPLACE(*coll_args)) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->src.info); } } return UCC_OK; case UCC_COLL_TYPE_GATHERV: if (UCC_IS_ROOT(*coll_args, rank)) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->dst.info_v); } if (!(UCC_IS_INPLACE(*coll_args) && UCC_IS_ROOT(*coll_args, rank))) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->src.info); } return UCC_OK; case UCC_COLL_TYPE_SCATTER: if (UCC_IS_ROOT(*coll_args, rank)) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->src.info); } if (!(UCC_IS_INPLACE(*coll_args) && UCC_IS_ROOT(*coll_args, rank))) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->dst.info); } return UCC_OK; case UCC_COLL_TYPE_SCATTERV: if (UCC_IS_ROOT(*coll_args, rank)) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->src.info_v); } if (!(UCC_IS_INPLACE(*coll_args) && UCC_IS_ROOT(*coll_args, rank))) { UCC_BUFFER_INFO_CHECK_MEM_TYPE(coll_args->dst.info); } return UCC_OK; default: ucc_error("unknown collective type"); return UCC_ERR_INVALID_PARAM; }; } #define UCC_COLL_TYPE_SKIP_ZERO_SIZE \ (UCC_COLL_TYPE_ALLREDUCE | \ UCC_COLL_TYPE_ALLGATHER | \ UCC_COLL_TYPE_ALLTOALL | \ UCC_COLL_TYPE_BCAST | \ UCC_COLL_TYPE_GATHER | \ UCC_COLL_TYPE_REDUCE | \ UCC_COLL_TYPE_SCATTER) UCC_CORE_PROFILE_FUNC(ucc_status_t, ucc_collective_init, (coll_args, request, team), ucc_coll_args_t *coll_args, ucc_coll_req_h *request, ucc_team_h team) { ucc_base_coll_args_t op_args = {0}; ucc_coll_task_t *task; ucc_status_t status; ucc_ee_executor_params_t params; ucc_memory_type_t coll_mem_type; ucc_ee_type_t coll_ee_type; size_t coll_size; if (ucc_unlikely(team->state != UCC_TEAM_ACTIVE)) { ucc_error("team %p is used before team create is completed", team); return UCC_ERR_INVALID_PARAM; } /* Global check to reduce the amount of checks throughout all TLs */ if (UCC_COLL_TYPE_SKIP_ZERO_SIZE & coll_args->coll_type) { coll_size = ucc_coll_args_msgsize(coll_args, team->rank, team->size); if (coll_size == 0) { task = ucc_mpool_get(&team->contexts[0]->lib->stub_tasks_mp); if (ucc_unlikely(!task)) { ucc_error("failed to allocate dummy task"); return UCC_ERR_NO_MEMORY; } op_args.mask = 0; memcpy(&op_args.args, coll_args, sizeof(ucc_coll_args_t)); op_args.team = team; op_args.args.flags = 0; UCC_COPY_PARAM_BY_FIELD(&op_args.args, coll_args, UCC_COLL_ARGS_FIELD_FLAGS, flags); ucc_coll_task_init(task, &op_args, NULL); goto print_trace; } } if (UCC_COLL_ARGS_ACTIVE_SET(coll_args) && (UCC_COLL_TYPE_BCAST != coll_args->coll_type)) { ucc_warn("Active Sets are only supported for bcast"); return UCC_ERR_NOT_SUPPORTED; } status = ucc_coll_args_check_mem_type(coll_args, team->rank); if (ucc_unlikely(status != UCC_OK)) { ucc_error("memory type detection failed"); return status; } status = ucc_check_coll_args(coll_args, team->rank); if (ucc_unlikely(status != UCC_OK)) { ucc_error("collective arguments check failed"); return status; } /* TO discuss: maybe we want to pass around user pointer ? */ memcpy(&op_args.args, coll_args, sizeof(ucc_coll_args_t)); op_args.team = team; op_args.args.flags = 0; UCC_COPY_PARAM_BY_FIELD(&op_args.args, coll_args, UCC_COLL_ARGS_FIELD_FLAGS, flags); if (!ucc_coll_args_is_mem_symmetric(&op_args.args, team->rank) && ucc_coll_args_is_rooted(op_args.args.coll_type)) { status = ucc_coll_args_init_asymmetric_buffer(&op_args.args, team, &op_args.asymmetric_save_info); if (ucc_unlikely(status != UCC_OK)) { ucc_error("handling asymmetric memory failed"); return status; } } else { op_args.asymmetric_save_info.scratch = NULL; } status = ucc_coll_init(team->score_map, &op_args, &task); if (UCC_ERR_NOT_SUPPORTED == status) { ucc_debug("failed to init collective: not supported"); goto free_scratch; } else if (ucc_unlikely(status < 0)) { char coll_args_str[256] = {0}; ucc_coll_args_str(&op_args.args, team->rank, team->size, coll_args_str, sizeof(coll_args_str)); ucc_error("failed to init collective: %s, err: (%d) %s", coll_args_str, status, ucc_status_string(status)); goto free_scratch; } /* Setup non-blocking datatype check for rooted collectives * * This implements transparent validation using a schedule with two tasks: * 1. Allreduce validation task: uses MIN reduction with min/max trick to detect mismatches * 2. Actual collective task: the real gather/scatter operation * * Validation uses allreduce (MIN) on [dt, -dt, mem, -mem]: * - Message size: 8 bytes (4 × int16_t, doesn't scale with number of ranks) * - After reduction: min(dt) == -min(-dt) means all ranks have same datatype * * Dependencies: allreduce validation → actual task * If validation fails, the dependency mechanism prevents the actual task from posting. */ if (coll_args->coll_type == UCC_COLL_TYPE_GATHER || coll_args->coll_type == UCC_COLL_TYPE_GATHERV || coll_args->coll_type == UCC_COLL_TYPE_SCATTER || coll_args->coll_type == UCC_COLL_TYPE_SCATTERV) { /* Check if datatype validation is needed and create schedule if so */ ucc_coll_task_t *validated_task; validated_task = ucc_service_dt_check(team, task, &status); if (!validated_task) { ucc_error("failed to create dt_check schedule: %s", ucc_status_string(status)); goto coll_finalize; } /* Return schedule if validation was needed, or original task if not */ task = validated_task; } /* Setup top-level task (actual task or schedule) */ task->flags |= UCC_COLL_TASK_FLAG_TOP_LEVEL; if (task->flags & UCC_COLL_TASK_FLAG_EXECUTOR) { task->flags |= UCC_COLL_TASK_FLAG_EXECUTOR_STOP; coll_mem_type = ucc_coll_args_mem_type(&op_args.args, team->rank); switch(coll_mem_type) { case UCC_MEMORY_TYPE_CUDA: case UCC_MEMORY_TYPE_CUDA_MANAGED: coll_ee_type = UCC_EE_CUDA_STREAM; break; case UCC_MEMORY_TYPE_ROCM: coll_ee_type = UCC_EE_ROCM_STREAM; break; case UCC_MEMORY_TYPE_HOST: coll_ee_type = UCC_EE_CPU_THREAD; break; default: ucc_error("no suitable executor available for memory type %s", ucc_memory_type_names[coll_mem_type]); status = UCC_ERR_INVALID_PARAM; goto coll_finalize; } params.mask = UCC_EE_EXECUTOR_PARAM_FIELD_TYPE; params.ee_type = coll_ee_type; status = ucc_ee_executor_init(¶ms, &task->executor); if (UCC_OK != status) { ucc_error("failed to init executor: %s", ucc_status_string(status)); goto coll_finalize; } } if (coll_args->mask & UCC_COLL_ARGS_FIELD_CB) { task->cb = coll_args->cb; task->flags |= UCC_COLL_TASK_FLAG_CB; } task->seq_num = team->seq_num++; ucc_assert(task->super.status == UCC_OPERATION_INITIALIZED); print_trace: *request = &task->super; if (ucc_unlikely(ucc_global_config.coll_trace.log_level >= UCC_LOG_LEVEL_DIAG)) { char coll_str[256]; ucc_coll_str(task, coll_str, sizeof(coll_str), ucc_global_config.coll_trace.log_level); if (ucc_global_config.coll_trace.log_level <= UCC_LOG_LEVEL_INFO) { if (team->rank == 0) { ucc_log_component_collective_trace( ucc_global_config.coll_trace.log_level, "coll_init: %s", coll_str); } } else { ucc_coll_trace_debug("coll_init: %s", coll_str); } } return UCC_OK; coll_finalize: task->finalize(task); free_scratch: if (op_args.asymmetric_save_info.scratch != NULL) { ucc_mc_free(op_args.asymmetric_save_info.scratch); } return status; } /* Check if user is trying to post the request which is either in completed, inprogress or error state. The only allowed case is: request is completed and has a persistent flag. Otherwise: bad usage. */ #define COLL_POST_STATUS_CHECK(_task) do { \ if ((_task)->super.status != UCC_OPERATION_INITIALIZED) { \ if (!(UCC_OK == (_task)->super.status && \ UCC_IS_PERSISTENT((_task)->bargs.args))) { \ ucc_error("%s request with invalid status %s is being posted", \ UCC_IS_PERSISTENT((_task)->bargs.args) \ ? "persistent" : "non-persistent", \ ucc_status_string((_task)->super.status)); \ return UCC_ERR_INVALID_PARAM; \ } \ } \ } while(0) UCC_CORE_PROFILE_FUNC(ucc_status_t, ucc_collective_post, (request), ucc_coll_req_h request) { ucc_coll_task_t *task = ucc_derived_of(request, ucc_coll_task_t); ucc_status_t status; if (ucc_global_config.coll_trace.log_level >= UCC_LOG_LEVEL_DEBUG) { ucc_rank_t rank = task->bargs.team->rank; if (ucc_global_config.coll_trace.log_level == UCC_LOG_LEVEL_DEBUG) { if (rank == 0) { ucc_log_component_collective_trace( ucc_global_config.coll_trace.log_level, "coll post: req %p, seq_num %u", task, task->seq_num); } } else { ucc_log_component_collective_trace( ucc_global_config.coll_trace.log_level, "coll post: rank %d req %p, seq_num %u", rank, task, task->seq_num); } } if (task->bargs.asymmetric_save_info.scratch != NULL && (task->bargs.args.coll_type == UCC_COLL_TYPE_SCATTER || task->bargs.args.coll_type == UCC_COLL_TYPE_SCATTERV)) { status = ucc_copy_asymmetric_buffer(task); if (status != UCC_OK) { ucc_error("failure copying in asymmetric buffer: %s", ucc_status_string(status)); return status; } } COLL_POST_STATUS_CHECK(task); if (UCC_COLL_TIMEOUT_REQUIRED(task)) { task->start_time = ucc_get_time(); } if (task->flags & UCC_COLL_TASK_FLAG_EXECUTOR) { status = ucc_ee_executor_start(task->executor, NULL); if (ucc_unlikely(status != UCC_OK)) { ucc_error("failed to start executor: %s", ucc_status_string(status)); } } return task->post(task); } ucc_status_t ucc_collective_triggered_post(ucc_ee_h ee, ucc_ev_t *ev) { ucc_coll_task_t *task = ucc_derived_of(ev->req, ucc_coll_task_t); if (ucc_global_config.coll_trace.log_level >= UCC_LOG_LEVEL_DEBUG) { ucc_rank_t rank = task->bargs.team->rank; if (ucc_global_config.coll_trace.log_level == UCC_LOG_LEVEL_DEBUG) { if (rank == 0) { ucc_log_component_collective_trace( ucc_global_config.coll_trace.log_level, "coll triggered_post: req %p, seq_num %u", task, task->seq_num); } } else { ucc_log_component_collective_trace( ucc_global_config.coll_trace.log_level, "coll triggered_post: rank %d req %p, seq_num %u", rank, task, task->seq_num); } } COLL_POST_STATUS_CHECK(task); if (UCC_COLL_TIMEOUT_REQUIRED(task)) { task->start_time = ucc_get_time(); } return task->triggered_post(ee, ev, task); } UCC_CORE_PROFILE_FUNC(ucc_status_t, ucc_collective_init_and_post, (coll_args, request, team), ucc_coll_args_t *coll_args, //NOLINT ucc_coll_req_h *request, ucc_team_h team) //NOLINT { ucc_error("ucc_collective_init_and_post() is not implemented"); return UCC_ERR_NOT_IMPLEMENTED; } ucc_status_t ucc_collective_finalize_internal(ucc_coll_task_t *task) { ucc_status_t st; if (ucc_unlikely(task->super.status == UCC_INPROGRESS)) { ucc_error("attempt to finalize task with status UCC_INPROGRESS"); return UCC_ERR_INVALID_PARAM; } if (task->bargs.asymmetric_save_info.scratch) { st = ucc_coll_args_free_asymmetric_buffer(task); if (ucc_unlikely(st != UCC_OK)) { ucc_error("error freeing asymmetric buf: %s", ucc_status_string(st)); } } if (task->executor) { st = ucc_ee_executor_finalize(task->executor); if (ucc_unlikely(st != UCC_OK)) { ucc_error("executor finalize error: %s", ucc_status_string(st)); } } return task->finalize(task); } UCC_CORE_PROFILE_FUNC(ucc_status_t, ucc_collective_finalize, (request), ucc_coll_req_h request) { ucc_coll_task_t *task = ucc_derived_of(request, ucc_coll_task_t); if (ucc_global_config.coll_trace.log_level >= UCC_LOG_LEVEL_DEBUG) { if (task->team) { ucc_rank_t rank = task->team->params.team->rank; if (ucc_global_config.coll_trace.log_level == UCC_LOG_LEVEL_DEBUG) { if (rank == 0) { ucc_log_component_collective_trace( ucc_global_config.coll_trace.log_level, "coll finalize: req %p, seq_num %u", task, task->seq_num); } } else { ucc_log_component_collective_trace( ucc_global_config.coll_trace.log_level, "coll finalize: rank %d req %p, seq_num %u", rank, task, task->seq_num); } } } return ucc_collective_finalize_internal(task); } static ucc_status_t ucc_triggered_task_finalize(ucc_coll_task_t *task) { ucc_trace("finalizing triggered ev task %p", task); ucc_free(task); return UCC_OK; } //NOLINTNEXTLINE static void ucc_triggered_task_cb(void *task, ucc_status_t st) { ucc_triggered_task_finalize((ucc_coll_task_t*)task); } static ucc_status_t ucc_trigger_complete(ucc_coll_task_t *parent_task, ucc_coll_task_t *task) { ucc_status_t status; ucc_trace("event triggered, ev_task %p, coll_task %p, seq_num %u", parent_task, task, task->seq_num); if (!(task->flags & UCC_COLL_TASK_FLAG_EXECUTOR)) { task->executor = parent_task->executor; task->flags |= (UCC_COLL_TASK_FLAG_EXECUTOR_STOP | UCC_COLL_TASK_FLAG_EXECUTOR_DESTROY); } status = task->post(task); if (ucc_unlikely(status != UCC_OK)) { ucc_error("failed to post triggered coll, task %p, seq_num %u, %s", task, task->seq_num, ucc_status_string(status)); } return status; } static void ucc_trigger_test(ucc_coll_task_t *task) { ucc_status_t status; ucc_ev_t post_event; ucc_ev_t *ev; ucc_ee_executor_params_t params; if (task->ev == NULL) { if ((task->ee->ee_type == UCC_EE_CUDA_STREAM) || (task->ee->ee_type == UCC_EE_ROCM_STREAM)) { /* implicit event triggered */ task->ev = (ucc_ev_t *) 0xFFFF; /* dummy event */ task->executor = NULL; } else if (UCC_OK == ucc_ee_get_event_internal(task->ee, &ev, &task->ee->event_in_queue)) { ucc_trace("triggered event arrived, ev_task %p", task); task->ev = ev; task->executor = NULL; } else { task->status = UCC_OK; return; } } if (task->executor == NULL) { if (task->triggered_task->triggered_post_setup) { status = task->triggered_task->triggered_post_setup(task->triggered_task); if (ucc_unlikely(status != UCC_OK)) { ucc_error("error in triggered post setup, %s", ucc_status_string(status)); task->status = status; return; } } if (task->triggered_task->executor) { task->executor = task->triggered_task->executor; } else { /* triggered task doesn't need executor, init and start executor on * trigger task */ params.mask = UCC_EE_EXECUTOR_PARAM_FIELD_TYPE | UCC_EE_EXECUTOR_PARAM_FIELD_TASK_TYPES; params.ee_type = task->ee->ee_type; params.task_types = 0; status = ucc_ee_executor_init(¶ms, &task->executor); if (ucc_unlikely(status != UCC_OK)) { ucc_error("error in ee executor init, %s", ucc_status_string(status)); task->status = status; return; } } status = ucc_ee_executor_start(task->executor, task->ee->ee_context); if (ucc_unlikely(status != UCC_OK)) { ucc_error("error in ee executor start, %s", ucc_status_string(status)); task->status = status; return; } post_event.ev_type = UCC_EVENT_COLLECTIVE_POST; post_event.ev_context_size = 0; post_event.ev_context = NULL; post_event.req = &task->triggered_task->super; ucc_ee_set_event_internal(task->ee, &post_event, &task->ee->event_out_queue); } if (ucc_ee_executor_status(task->executor) == UCC_OK) { task->status = UCC_OK; } } ucc_status_t ucc_triggered_post(ucc_ee_h ee, ucc_ev_t *ev, ucc_coll_task_t *task) { ucc_coll_task_t *ev_task; ucc_status_t status; if (ev->ev_type != UCC_EVENT_COMPUTE_COMPLETE) { ucc_error("event type %d is not supported", ev->ev_type); return UCC_ERR_NOT_IMPLEMENTED; } task->ee = ee; task->super.status = UCC_OPERATION_INITIALIZED; ev_task = ucc_malloc(sizeof(*ev_task), "ev_task"); if (!ev_task) { ucc_error("failed to allocate %zd bytes for ev_task", sizeof(*ev_task)); return UCC_ERR_NO_MEMORY; } ucc_coll_task_construct(ev_task); ucc_coll_task_init(ev_task, NULL, task->team); ev_task->ee = ee; ev_task->ev = NULL; ev_task->triggered_task = task; ev_task->flags = UCC_COLL_TASK_FLAG_CB; ev_task->cb.cb = ucc_triggered_task_cb; ev_task->cb.data = ev_task; ev_task->finalize = ucc_triggered_task_finalize; ev_task->progress = ucc_trigger_test; ev_task->status = UCC_INPROGRESS; if (UCC_COLL_TIMEOUT_REQUIRED(task)) { UCC_COLL_SET_TIMEOUT(ev_task, task->bargs.args.timeout); } status = ucc_event_manager_subscribe(ev_task, UCC_EVENT_COMPLETED, task, ucc_trigger_complete); if (ucc_unlikely(UCC_OK != status)) { return status; } return ucc_progress_queue_enqueue(task->bargs.team->contexts[0]->pq, ev_task); } ucc-1.8.0/src/core/ucc_service_coll.h0000664000175000017500000000425015211535620017652 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_SERVICE_COLL_H_ #define UCC_SERVICE_COLL_H_ #include "ucc/api/ucc.h" #include "components/tl/ucc_tl.h" typedef struct ucc_service_coll_req { ucc_coll_task_t *task; ucc_team_t *team; void * data; ucc_subset_t subset; } ucc_service_coll_req_t; ucc_status_t ucc_service_allreduce(ucc_team_t *team, void *sbuf, void *rbuf, ucc_datatype_t dt, size_t count, ucc_reduction_op_t op, ucc_subset_t subset, ucc_service_coll_req_t **req); ucc_status_t ucc_service_allgather(ucc_team_t *team, void *sbuf, void *rbuf, size_t msgsize, ucc_subset_t subset, ucc_service_coll_req_t **req); ucc_status_t ucc_service_bcast(ucc_team_t *team, void *buf, size_t msgsize, ucc_rank_t root, ucc_subset_t subset, ucc_service_coll_req_t **req); ucc_status_t ucc_service_coll_test(ucc_service_coll_req_t *req); ucc_status_t ucc_service_coll_finalize(ucc_service_coll_req_t *req); ucc_status_t ucc_internal_oob_init(ucc_team_t *team, ucc_subset_t subset, ucc_team_oob_coll_t *oob); void ucc_internal_oob_finalize(ucc_team_oob_coll_t *oob); ucc_status_t ucc_collective_finalize_internal(ucc_coll_task_t *task); /** * Create datatype validation schedule for rooted collectives * * This function checks if datatype validation is needed and creates a schedule * with validation logic if required. If validation is not needed, returns the * original task unchanged. * * @param team The UCC team * @param task The actual collective task (already created by TL/CL) * @param status_out If non-NULL, set to the error status when returning NULL * @return Schedule with validation (as ucc_coll_task_t*), or original task, or NULL on error */ ucc_coll_task_t* ucc_service_dt_check(ucc_team_t *team, ucc_coll_task_t *task, ucc_status_t *status_out); #endif ucc-1.8.0/src/core/ucc_context.c0000664000175000017500000016553715211535620016700 0ustar alastairalastair/** * Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "ucc_context.h" #include "utils/ucc_proc_info.h" #include "components/cl/ucc_cl.h" #include "components/tl/ucc_tl.h" #include "components/topo/ucc_sysinfo.h" #include "utils/ucc_malloc.h" #include "utils/ucc_log.h" #include "utils/ucc_list.h" #include "utils/ucc_string.h" #include "utils/ucc_debug.h" #include "ucc_progress_queue.h" static uint32_t ucc_context_seq_num = 0; static ucc_config_field_t ucc_context_config_table[] = { {"ESTIMATED_NUM_EPS", "0", "An optimization hint of how many endpoints will be created on this " "context", ucc_offsetof(ucc_context_config_t, estimated_num_eps), UCC_CONFIG_TYPE_UINT}, {"LOCK_FREE_PROGRESS_Q", "0", "Enable lock free progress queue optimization", ucc_offsetof(ucc_context_config_t, lock_free_progress_q), UCC_CONFIG_TYPE_UINT}, {"ESTIMATED_NUM_PPN", "0", "An optimization hint of how many endpoints created on this context reside" " on the same node", ucc_offsetof(ucc_context_config_t, estimated_num_ppn), UCC_CONFIG_TYPE_UINT}, {"TEAM_IDS_POOL_SIZE", "32", "Defines the size of the team_id_pool. The number of coexisting unique " "team ids for a single process is team_ids_pool_size*64. This parameter " "is relevant when internal team id allocation takes place.", ucc_offsetof(ucc_context_config_t, team_ids_pool_size), UCC_CONFIG_TYPE_UINT}, {"INTERNAL_OOB", "1", "Use internal OOB transport for team creation. Available for ucc_context " "is configured with OOB (global mode). 0 - disable, 1 - try, 2 - force.", ucc_offsetof(ucc_context_config_t, internal_oob), UCC_CONFIG_TYPE_UINT}, {"THROTTLE_PROGRESS", "1000", "Throttle UCC progress to every th invocation", ucc_offsetof(ucc_context_config_t, throttle_progress), UCC_CONFIG_TYPE_UINT}, {"NET_DEVICES", "all", "Specifies which network device(s) to use. The order is not meaningful.\n" "\"all\" would use all available devices. Only TLs that support this " "parameter will be affected. The parameter is only supported by UCX TL.", ucc_offsetof(ucc_context_config_t, net_devices), UCC_CONFIG_TYPE_STRING_ARRAY}, {"NODE_LOCAL_ID", "auto", "An optimization hint for the local identificator on a single node.", ucc_offsetof(ucc_context_config_t, node_local_id), UCC_CONFIG_TYPE_ULUNITS}, {NULL}}; UCC_CONFIG_REGISTER_TABLE(ucc_context_config_table, "UCC context", NULL, ucc_context_config_t, &ucc_config_global_list); ucc_status_t ucc_context_config_read(ucc_lib_info_t *lib, const char *filename, ucc_context_config_t **config_p) { ucc_cl_context_config_t *cl_config = NULL; ucc_tl_context_config_t *tl_config = NULL; int i; ucc_status_t status; ucc_context_config_t *config; if (filename != NULL) { ucc_error("read from file is not implemented"); return UCC_ERR_NOT_IMPLEMENTED; } config = (ucc_context_config_t *)ucc_malloc(sizeof(ucc_context_config_t), "ctx_config"); if (config == NULL) { ucc_error("failed to allocate %zd bytes for context config", sizeof(ucc_context_config_t)); return UCC_ERR_NO_MEMORY; } status = ucc_config_parser_fill_opts(config, UCC_CONFIG_GET_TABLE(ucc_context_config_table), lib->full_prefix, 0); if (status != UCC_OK) { ucc_error("failed to read UCC core context config"); goto err_config; } config->lib = lib; config->cl_cfgs = (ucc_cl_context_config_t **)ucc_calloc( lib->n_cl_libs_opened, sizeof(ucc_cl_context_config_t *), "cl_configs_array"); if (config->cl_cfgs == NULL) { ucc_error("failed to allocate %zd bytes for cl configs array", sizeof(ucc_cl_context_config_t *)); status = UCC_ERR_NO_MEMORY; goto err_config; } config->tl_cfgs = (ucc_tl_context_config_t **)ucc_calloc( lib->n_tl_libs_opened, sizeof(ucc_tl_context_config_t *), "tl_configs_array"); if (config->tl_cfgs == NULL) { ucc_error("failed to allocate %zd bytes for tl configs array", sizeof(ucc_tl_context_config_t *)); status = UCC_ERR_NO_MEMORY; goto err_configs; } config->n_cl_cfg = 0; for (i = 0; i < lib->n_cl_libs_opened; i++) { ucc_assert(NULL != lib->cl_libs[i]->iface->cl_context_config.table); status = ucc_cl_context_config_read(lib->cl_libs[i], config, &cl_config); if (UCC_OK != status) { ucc_error("failed to read CL \"%s\" context configuration", lib->cl_libs[i]->iface->super.name); goto err_cl_config; } config->cl_cfgs[config->n_cl_cfg] = cl_config; config->n_cl_cfg++; } config->n_tl_cfg = 0; for (i = 0; i < lib->n_tl_libs_opened; i++) { ucc_assert(NULL != lib->tl_libs[i]->iface->tl_context_config.table); status = ucc_tl_context_config_read(lib->tl_libs[i], config, &tl_config); if (UCC_OK != status) { ucc_error("failed to read TL \"%s\" context configuration", lib->tl_libs[i]->iface->super.name); goto err_tl_config; } config->tl_cfgs[config->n_tl_cfg] = tl_config; config->n_tl_cfg++; } *config_p = config; return UCC_OK; err_tl_config: for (i = 0; i < config->n_tl_cfg; i++) { ucc_base_config_release(&config->tl_cfgs[i]->super.super); } err_cl_config: for (i = 0; i < config->n_cl_cfg; i++) { ucc_base_config_release(&config->cl_cfgs[i]->super.super); } ucc_free(config->tl_cfgs); err_configs: ucc_free(config->cl_cfgs); err_config: ucc_free(config); return status; } static inline ucc_cl_context_config_t * find_cl_context_config(ucc_context_config_t *cfg, const char *name) { int i; for (i = 0; i < cfg->n_cl_cfg; i++) { if (cfg->cl_cfgs[i] && (0 == strcmp(cfg->cl_cfgs[i]->cl_lib->iface->super.name, name))) { return cfg->cl_cfgs[i]; } } return NULL; } static inline ucc_tl_context_config_t * find_tl_context_config(ucc_context_config_t *cfg, const char *name) { int i; for (i = 0; i < cfg->n_tl_cfg; i++) { if (cfg->tl_cfgs[i] && (0 == strcmp(cfg->tl_cfgs[i]->tl_lib->iface->super.name, name))) { return cfg->tl_cfgs[i]; } } return NULL; } /* Modifies the ucc_context configuration. If user sets cls="all" then this means that the parameter "name" should be modified in ALL available CLS. In this case we loop over all of them, and if error is reported by any CL we bail and report error to the user. If user passes a comma separated list of CLs, then we go over the list and apply modifications to the specified CLs only. */ ucc_status_t ucc_context_config_modify(ucc_context_config_t *config, const char *components, const char *name, const char *value) { int i; ucc_status_t status; ucc_cl_context_config_t *cl_cfg; ucc_tl_context_config_t *tl_cfg; char **tokens; const char *component; unsigned n_tokens; if (NULL == components) { /* cls is NULL means modify core ucc context config */ status = ucc_config_parser_set_value(config, ucc_context_config_table, name, value); if (UCC_OK != status) { ucc_error("failed to modify CORE configuration, name %s, value %s", name, value);; return status; } } else if (0 != strcmp(components, "all")) { tokens = ucc_str_split(components, ","); if (!tokens) { return UCC_ERR_INVALID_PARAM; } n_tokens = ucc_str_split_count(tokens); for (i = 0; i < n_tokens; i++) { if (strlen(tokens[i]) < 4) { status = UCC_ERR_INVALID_PARAM; goto err_input; } component = tokens[i] + 3; if (0 == strncmp(tokens[i], "cl/", 3)) { cl_cfg = find_cl_context_config(config, component); if (!cl_cfg) { ucc_info("required CL %s is not part of the context", component); status = UCC_ERR_NOT_FOUND; goto err; } status = ucc_config_parser_set_value( cl_cfg, cl_cfg->cl_lib->iface->cl_context_config.table, name, value); if (UCC_OK != status) { ucc_error("failed to modify CL \"%s\" configuration, name %s, " "value %s", component, name, value); goto err; } } else if (0 == strncmp(tokens[i], "tl/", 3)) { tl_cfg = find_tl_context_config(config, component); if (!tl_cfg) { ucc_info("required TL %s is not part of the context", component); status = UCC_ERR_NOT_FOUND; goto err; } status = ucc_config_parser_set_value( tl_cfg, tl_cfg->tl_lib->iface->tl_context_config.table, name, value); if (UCC_OK != status) { ucc_error("failed to modify TL \"%s\" configuration, name %s, " "value %s", component, name, value); goto err; } } else { status = UCC_ERR_INVALID_PARAM; ucc_error("invalid component name %s", tokens[i]); goto err_input; } } ucc_str_split_free(tokens); } else { for (i = 0; i < config->n_cl_cfg; i++) { if (config->cl_cfgs[i]) { status = ucc_config_parser_set_value( config->cl_cfgs[i], config->cl_cfgs[i]->cl_lib->iface->cl_context_config.table, name, value); if (UCC_OK != status) { ucc_error("failed to modify CL \"%s\" configuration, name " "%s, value %s", config->cl_cfgs[i]->cl_lib->iface->super.name, name, value); return status; } } } } return UCC_OK; err_input: ucc_error("incorrect component name %s is provided. " "expected cl/ or tl/.", tokens[i]); err: ucc_str_split_free(tokens); return status; } void ucc_context_config_release(ucc_context_config_t *config) { int i; for (i = 0; i < config->n_tl_cfg; i++) { ucc_base_config_release(&config->tl_cfgs[i]->super.super); } for (i = 0; i < config->n_cl_cfg; i++) { ucc_base_config_release(&config->cl_cfgs[i]->super.super); } ucc_config_parser_release_opts(config, ucc_context_config_table); ucc_free(config->tl_cfgs); ucc_free(config->cl_cfgs); ucc_free(config); } /* The function prints the configuration of UCC context. The ucc_context is a combination of contexts of different (potentially multiple) CLs. If HEADER flag is required - print it once passing user "title" variable. For other cl_contexts use CL name as title so that the printed output was clear for a user */ void ucc_context_config_print(const ucc_context_config_h config, FILE *stream, const char *title, ucc_config_print_flags_t print_flags) { int i; int print_header = print_flags & UCC_CONFIG_PRINT_HEADER; int flags = print_flags; /* cl_context_configs will always be printed with HEADER using CL name as title */ flags |= UCC_CONFIG_PRINT_HEADER; if (print_header) { ucc_config_parser_print_opts( stream, title, config, ucc_context_config_table, "", config->lib->full_prefix, UCC_CONFIG_PRINT_HEADER); } ucc_config_parser_print_opts( stream, "CORE", config, ucc_context_config_table, "", config->lib->full_prefix, (ucc_config_print_flags_t)flags); for (i = 0; i < config->n_cl_cfg; i++) { ucc_config_parser_print_opts( stream, config->lib->cl_libs[i]->iface->cl_context_config.name, config->cl_cfgs[i], config->lib->cl_libs[i]->iface->cl_context_config.table, config->lib->cl_libs[i]->iface->cl_context_config.prefix, config->lib->full_prefix, (ucc_config_print_flags_t)flags); } } static inline void ucc_copy_context_params(ucc_context_params_t *dst, const ucc_context_params_t *src) { dst->mask = src->mask; UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_CONTEXT_PARAM_FIELD_TYPE, type); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_CONTEXT_PARAM_FIELD_OOB, oob); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_CONTEXT_PARAM_FIELD_ID, ctx_id); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_CONTEXT_PARAM_FIELD_SYNC_TYPE, sync_type); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_CONTEXT_PARAM_FIELD_MEM_PARAMS, mem_params); } static ucc_status_t ucc_create_tl_contexts(ucc_context_t *ctx, ucc_context_config_t *ctx_config, ucc_base_context_params_t b_params) { ucc_lib_info_t *lib = ctx->lib; ucc_tl_lib_t *tl_lib; ucc_base_context_t *b_ctx; int i, num_tls; ucc_status_t status; ucc_base_lib_attr_t attr; int ctx_service_team; ctx_service_team = ctx_config->internal_oob && b_params.params.mask & UCC_CONTEXT_PARAM_FIELD_OOB; num_tls = ctx_config->n_tl_cfg; ctx->tl_ctx = (ucc_tl_context_t **)ucc_malloc( sizeof(ucc_tl_context_t *) * num_tls, "tl_ctx_array"); if (!ctx->tl_ctx) { ucc_error("failed to allocate %zd bytes for tl_ctx array", sizeof(ucc_tl_context_t *) * num_tls); return UCC_ERR_NO_MEMORY; } ctx->n_tl_ctx = 0; for (i = 0; i < num_tls; i++) { tl_lib = ctx_config->tl_cfgs[i]->tl_lib; status = tl_lib->iface->lib.get_attr(&tl_lib->super, &attr); if (UCC_OK != status) { ucc_error("failed to query tl lib %s attr", tl_lib->iface->super.name); return status; } if ((attr.flags & UCC_BASE_LIB_FLAG_CTX_SERVICE_TEAM_REQUIRED) && (!ctx_service_team)) { ucc_debug("can not create tl/%s context because context service " "team is not available for it", tl_lib->iface->super.name); continue; } // coverity[overrun-buffer-val:FALSE] status = tl_lib->iface->context.create( &b_params, &ctx_config->tl_cfgs[i]->super.super, &b_ctx); if (UCC_OK != status) { /* UCC_ERR_LAST means component was disabled via TUNE param: don't print warning. */ if (UCC_ERR_LAST != status) { if (ucc_tl_is_required(lib, tl_lib->iface, 1)) { ucc_error("failed to create tl context for %s", tl_lib->iface->super.name); } else { ucc_debug("failed to create tl context for %s", tl_lib->iface->super.name); } } continue; } ctx->tl_ctx[ctx->n_tl_ctx] = ucc_derived_of(b_ctx, ucc_tl_context_t); ctx->n_tl_ctx++; } if (ctx->n_tl_ctx == 0) { ucc_error("no tl contexts were created"); status = UCC_ERR_NOT_FOUND; goto err; } /* build the list of names of all available tl contexts. This is a convenience data struct for CLs */ ctx->all_tls.count = ctx->n_tl_ctx; ctx->all_tls.names = ucc_malloc(sizeof(char*) * ctx->n_tl_ctx, "all_tls"); if (!ctx->all_tls.names) { ucc_error("failed to allocate %zd bytes for all_tls names", sizeof(char*) * ctx->n_tl_ctx); status = UCC_ERR_NO_MEMORY; goto err; } for (i = 0; i < ctx->n_tl_ctx; i++) { ctx->all_tls.names[i] = (char*)ucc_derived_of(ctx->tl_ctx[i]->super.lib, ucc_tl_lib_t)->iface->super.name; } return UCC_OK; err: for (i = 0; i < ctx->n_tl_ctx; i++) { tl_lib = ucc_derived_of(ctx->tl_ctx[i]->super.lib, ucc_tl_lib_t); tl_lib->iface->context.destroy(&ctx->tl_ctx[i]->super); } ucc_free(ctx->tl_ctx); return status; } ucc_status_t ucc_core_addr_exchange( ucc_context_t *context, ucc_oob_coll_t *oob, ucc_addr_storage_t *addr_storage) { size_t *addr_lens; ucc_context_attr_t attr; ucc_status_t status; ucc_rank_t i; size_t max_addrlen; poll: if (addr_storage->oob_req) { status = oob->req_test(addr_storage->oob_req); if (status < 0) { oob->req_free(addr_storage->oob_req); ucc_error("oob req test failed during team addr exchange"); return status; } else if (UCC_INPROGRESS == status) { return status; } oob->req_free(addr_storage->oob_req); addr_storage->oob_req = NULL; } if (0 == addr_storage->addr_len) { if (NULL == addr_storage->storage) { addr_storage->size = oob->n_oob_eps; attr.mask = UCC_CONTEXT_ATTR_FIELD_CTX_ADDR_LEN | UCC_CONTEXT_ATTR_FIELD_CTX_ADDR; status = ucc_context_get_attr(context, &attr); if (UCC_OK != status) { ucc_error("failed to query ctx address"); return status; } addr_storage->storage = ucc_malloc( addr_storage->size * sizeof(size_t), "max_addrlen_tmp"); if (!addr_storage->storage) { ucc_error( "failed to allocate %zd bytes for max_addrlen tmp storage", addr_storage->size * sizeof(size_t)); return UCC_ERR_NO_MEMORY; } status = oob->allgather( &context->attr.ctx_addr_len, addr_storage->storage, sizeof(size_t), oob->coll_info, &addr_storage->oob_req); if (UCC_OK != status) { ucc_error("failed to start oob allgather"); return status; } goto poll; } addr_lens = (size_t *)addr_storage->storage; ucc_assert(addr_storage->storage != NULL); for (i = 0; i < addr_storage->size; i++) { if (addr_lens[i] > addr_storage->addr_len) { addr_storage->addr_len = addr_lens[i]; } } if (addr_storage->addr_len == 0) { ucc_free(addr_storage->storage); addr_storage->storage = NULL; return UCC_OK; } max_addrlen = addr_storage->addr_len; addr_storage->storage = ucc_realloc( addr_storage->storage, (addr_storage->size + 1) * max_addrlen, "addr_storage"); if (!addr_storage->storage) { ucc_error( "failed to allocate %zd bytes for addr storage", addr_storage->size * max_addrlen); return UCC_ERR_NO_MEMORY; } memcpy( PTR_OFFSET(addr_storage->storage, max_addrlen * addr_storage->size), context->attr.ctx_addr, context->attr.ctx_addr_len); status = oob->allgather( PTR_OFFSET(addr_storage->storage, max_addrlen * addr_storage->size), addr_storage->storage, max_addrlen, oob->coll_info, &addr_storage->oob_req); if (UCC_OK != status) { ucc_error("failed to start oob allgather"); return status; } goto poll; } ucc_assert(addr_storage->addr_len); { /* Compute storage rank and check proc info uniqeness */ ucc_rank_t r = UCC_RANK_MAX; int j; ucc_context_addr_header_t *h, *h0; addr_storage->flags = UCC_ADDR_STORAGE_FLAG_TLS_SYMMETRIC; h0 = UCC_ADDR_STORAGE_RANK_HEADER(addr_storage, 0); for (i = 0; i < addr_storage->size; i++) { h = UCC_ADDR_STORAGE_RANK_HEADER(addr_storage, i); if (UCC_CTX_ID_EQUAL(context->id, h->ctx_id)) { /* We should find local id only once. However, due to node hashing there is tiny chance of collision: check it */ if (r != UCC_RANK_MAX) { ucc_error("proc info collision: %d %d", r, i); return UCC_ERR_NO_MESSAGE; } r = i; } if (h->n_components == h0->n_components) { /*check if TLs array is the same*/ for (j = 0; j < h->n_components; j++) { if (h->components[j].id != h0->components[j].id) { addr_storage->flags = 0; break; } } } else { addr_storage->flags = 0; } } addr_storage->rank = r; } return UCC_OK; } static void remove_tl_ctx_from_array(ucc_tl_context_t **array, unsigned *size, ucc_tl_context_t *tl_ctx) { int i; for (i = 0; i < (*size); i++) { if (array[i] == tl_ctx) { break; } } if (i == (*size)) { /* given tl_ctx is not part of array */ return; } /* decrement array size and do cyclic shift */ (*size)--; for (; i < (*size); i++) { array[i] = array[i + 1]; } } ucc_status_t ucc_context_create_proc_info( ucc_lib_h lib, const ucc_context_params_t *params, const ucc_context_config_h config, ucc_context_h *context, ucc_proc_info_t *proc_info) { uint32_t topo_required = 0; uint64_t created_ctx_counter = 0; ucc_base_context_params_t b_params; ucc_base_context_t *b_ctx; ucc_base_ctx_attr_t c_attr; ucc_cl_lib_attr_t l_attr; ucc_cl_lib_t *cl_lib; ucc_tl_context_t *tl_ctx; ucc_tl_lib_t *tl_lib; ucc_context_t *ctx; ucc_status_t status; uint64_t i, j, n_tl_ctx; int num_cls; num_cls = config->n_cl_cfg; ctx = ucc_calloc(1, sizeof(ucc_context_t), "ucc_context"); if (!ctx) { ucc_error("failed to allocate %zd bytes for ucc_context", sizeof(ucc_context_t)); status = UCC_ERR_NO_MEMORY; goto error; } ctx->net_devices.count = 0; ucc_config_names_array_dup(&ctx->net_devices, &config->net_devices); ctx->throttle_progress = config->throttle_progress; ctx->rank = UCC_RANK_MAX; ctx->lib = lib; ctx->ids.pool_size = config->team_ids_pool_size; ucc_list_head_init(&ctx->progress_list); ucc_copy_context_params(&ctx->params, params); ucc_copy_context_params(&b_params.params, params); b_params.context = ctx; b_params.estimated_num_eps = config->estimated_num_eps; b_params.estimated_num_ppn = config->estimated_num_ppn; b_params.node_local_id = config->node_local_id; b_params.prefix = lib->full_prefix; b_params.thread_mode = lib->attr.thread_mode; if (params->mask & UCC_CONTEXT_PARAM_FIELD_OOB) { ctx->rank = params->oob.oob_ep; #ifdef ENABLE_DEBUG ucc_check_wait_for_debugger(ctx->rank); #endif } status = ucc_create_tl_contexts(ctx, config, b_params); if (UCC_OK != status) { /* only critical error could have happened - bail */ ucc_error("failed to create tl contexts"); goto error_ctx; } ctx->cl_ctx = (ucc_cl_context_t **)ucc_malloc( sizeof(ucc_cl_context_t *) * num_cls, "cl_ctx_array"); if (!ctx->cl_ctx) { ucc_error("failed to allocate %zd bytes for cl_ctx array", sizeof(ucc_cl_context_t *) * num_cls); status = UCC_ERR_NO_MEMORY; goto error_ctx; } ctx->n_cl_ctx = 0; ctx->cl_flags = 0; for (i = 0; i < num_cls; i++) { cl_lib = config->cl_cfgs[i]->cl_lib; // coverity[overrun-buffer-val:FALSE] status = cl_lib->iface->context.create( &b_params, &config->cl_cfgs[i]->super.super, &b_ctx); if (UCC_OK != status) { if (lib->specific_cls_requested) { ucc_error("failed to create cl context for %s", cl_lib->iface->super.name); goto error_ctx_create; } else { /* UCC_ERR_LAST means component was disabled via TUNE param: don't print warning. */ if (UCC_ERR_LAST != status) { ucc_warn("failed to create cl context for %s, skipping", cl_lib->iface->super.name); } continue; } } ctx->cl_ctx[ctx->n_cl_ctx] = ucc_derived_of(b_ctx, ucc_cl_context_t); ctx->n_cl_ctx++; memset(&c_attr, 0, sizeof(c_attr)); status = cl_lib->iface->context.get_attr(b_ctx, &c_attr); if (status != UCC_OK) { ucc_error("failed to query context attributes for %s", cl_lib->iface->super.name); goto error_ctx_create; } if (c_attr.topo_required) { topo_required = 1; } memset(&l_attr, 0, sizeof(l_attr)); status = cl_lib->iface->lib.get_attr(&cl_lib->super, &l_attr.super); if (UCC_OK != status) { ucc_error("failed to query lib %s attr", cl_lib->iface->super.name); goto error_ctx_create; } ctx->cl_flags |= l_attr.super.flags; } if (0 == ctx->n_cl_ctx) { ucc_error("no CL context created in ucc_context_create"); status = UCC_ERR_NO_MESSAGE; goto error_ctx; } /* Initialize ctx thread mode: if context is EXCLUSIVE then thread_mode is always SINGLE, otherwise it is inherited from lib */ ctx->thread_mode = ((params->type == UCC_CONTEXT_EXCLUSIVE) && (params->mask & UCC_CONTEXT_PARAM_FIELD_TYPE)) ? UCC_THREAD_SINGLE : lib->attr.thread_mode; status = ucc_progress_queue_init(&ctx->pq, ctx->thread_mode, config->lock_free_progress_q); if (UCC_OK != status) { ucc_error("failed to init progress queue for context %p", ctx); goto error_ctx_create; } ctx->id.pi = *proc_info; ctx->id.seq_num = ucc_atomic_fadd32(&ucc_context_seq_num, 1); if (params->mask & UCC_CONTEXT_PARAM_FIELD_OOB && params->oob.n_oob_eps > 1) { do { /* UCC context create is blocking fn, so we can wait here for the completion of addr exchange */ status = ucc_core_addr_exchange(ctx, &ctx->params.oob, &ctx->addr_storage); if (status < 0) { ucc_error("failed to exchange addresses during context " "creation with status: %s", ucc_status_string(status)); goto error_ctx_create; } } while (status == UCC_INPROGRESS); if (topo_required) { /* At least one available CL context reported it needs topo info */ status = ucc_context_topo_init(&ctx->addr_storage, &ctx->topo); if (UCC_OK != status) { ucc_error("failed to init ctx topo"); ucc_free(ctx->addr_storage.storage); goto error_ctx_create; } } ucc_assert(ctx->addr_storage.rank == params->oob.oob_ep); } if (config->internal_oob) { if (params->mask & UCC_CONTEXT_PARAM_FIELD_OOB && params->oob.n_oob_eps > 1) { ucc_base_team_params_t t_params; ucc_base_team_t * b_team; status = ucc_tl_context_get(ctx, "ucp", &ctx->service_ctx); if (UCC_OK != status) { if (config->internal_oob == 2) { ucc_error( "TL UCP context is not available, service team can " "not be created but was force requested"); goto error_ctx_create; } ucc_debug("TL UCP context is not available, " "service team can not be created"); } else { memset(&t_params.map, 0, sizeof(ucc_ep_map_t)); memset(&t_params.params, 0, sizeof(ucc_team_params_t)); t_params.params.mask = UCC_TEAM_PARAM_FIELD_EP | UCC_TEAM_PARAM_FIELD_EP_RANGE | UCC_TEAM_PARAM_FIELD_OOB; t_params.params.oob.allgather = ctx->params.oob.allgather; t_params.params.oob.req_test = ctx->params.oob.req_test; t_params.params.oob.req_free = ctx->params.oob.req_free; t_params.params.oob.coll_info = ctx->params.oob.coll_info; t_params.params.oob.n_oob_eps = ctx->params.oob.n_oob_eps; t_params.params.ep_range = UCC_COLLECTIVE_EP_RANGE_CONTIG; t_params.params.ep = ctx->rank; t_params.rank = ctx->rank; t_params.size = ctx->params.oob.n_oob_eps; /* CORE scope id - never overlaps with CL type */ t_params.scope = UCC_CL_LAST + 1; t_params.scope_id = 0; t_params.id = 0; t_params.team = NULL; t_params.map.type = UCC_EP_MAP_FULL; t_params.map.ep_num = t_params.size; status = UCC_TL_CTX_IFACE(ctx->service_ctx) ->team.create_post(&ctx->service_ctx->super, &t_params, &b_team); if (UCC_OK != status) { ucc_error("ctx service team create post failed with status: %s", ucc_status_string(status)); goto error_ctx_create; } do { status = UCC_TL_CTX_IFACE(ctx->service_ctx) ->team.create_test(b_team); } while (UCC_INPROGRESS == status); if (status < 0) { ucc_error("failed to create ctx service team with status: %s", ucc_status_string(status)); goto error_ctx_create; } ctx->service_team = ucc_derived_of(b_team, ucc_tl_team_t); } } else if (config->internal_oob == 2) { ucc_error("UCC_INTERNAL_OOB was force requested for context " "without OOB"); goto error_ctx_create; } } n_tl_ctx = ctx->n_tl_ctx; for (i = 0; i < n_tl_ctx; i++) { tl_ctx = ctx->tl_ctx[i]; tl_lib = ucc_derived_of(tl_ctx->super.lib, ucc_tl_lib_t); if (tl_lib->iface->context.create_epilog) { status = tl_lib->iface->context.create_epilog(&tl_ctx->super); if (UCC_OK == status) { created_ctx_counter++; } else { if (ucc_tl_is_required(lib, tl_lib->iface, 1)) { ucc_error("ctx create epilog for %s failed with status: %s", tl_lib->iface->super.name, ucc_status_string(status)); goto error_ctx_create_epilog; } else { ucc_debug("ctx create epilog for %s failed with status: %s", tl_lib->iface->super.name, ucc_status_string(status)); tl_lib->iface->context.destroy(&tl_ctx->super); for (j = 0; j < ctx->n_cl_ctx; j++) { remove_tl_ctx_from_array(ctx->cl_ctx[j]->tl_ctxs, &ctx->cl_ctx[j]->n_tl_ctxs, tl_ctx); } remove_tl_ctx_from_array(ctx->tl_ctx, &ctx->n_tl_ctx, tl_ctx); } } } else { created_ctx_counter++; } } if (0 == created_ctx_counter) { ucc_error("no TL context created"); status = UCC_ERR_NO_RESOURCE; goto error_ctx_create_epilog; } ucc_debug("created ucc context %p for lib %s: type %s, thread mode %s, oob %s, num eps %d, num ppn %d", ctx, lib->full_prefix, params->mask & UCC_CONTEXT_PARAM_FIELD_TYPE ? ucc_context_type_str(params->type) : "n/a", ucc_thread_mode_str(ctx->thread_mode), params->mask & UCC_CONTEXT_PARAM_FIELD_OOB ? "true" : "false", config->estimated_num_eps, config->estimated_num_ppn); *context = ctx; return UCC_OK; error_ctx_create_epilog: for (j = 0; j < created_ctx_counter; j++) { tl_ctx = ctx->tl_ctx[j]; tl_lib = ucc_derived_of(tl_ctx->super.lib, ucc_tl_lib_t); tl_lib->iface->context.destroy(&tl_ctx->super); } error_ctx_create: for (i = 0; i < ctx->n_cl_ctx; i++) { config->cl_cfgs[i]->cl_lib->iface->context.destroy( &ctx->cl_ctx[i]->super); } ucc_free(ctx->cl_ctx); error_ctx: ucc_free(ctx); error: return status; } ucc_status_t ucc_context_create(ucc_lib_h lib, const ucc_context_params_t *params, const ucc_context_config_h config, ucc_context_h *context) { return ucc_context_create_proc_info(lib, params, config, context, &ucc_local_proc); } static ucc_status_t ucc_context_free_attr(ucc_context_attr_t *context_attr) { if (context_attr->mask & UCC_CONTEXT_ATTR_FIELD_CTX_ADDR) { ucc_free(context_attr->ctx_addr); } return UCC_OK; } ucc_status_t ucc_context_destroy(ucc_context_t *context) { ucc_cl_context_t *cl_ctx; ucc_cl_lib_t *cl_lib; ucc_tl_context_t *tl_ctx; ucc_tl_lib_t *tl_lib; int i; ucc_status_t status; if (UCC_OK != ucc_context_free_attr(&context->attr)) { ucc_error("failed to free context attributes"); } for (i = 0; i < context->n_cl_ctx; i++) { cl_ctx = context->cl_ctx[i]; cl_lib = ucc_derived_of(cl_ctx->super.lib, ucc_cl_lib_t); cl_lib->iface->context.destroy(&cl_ctx->super); } ucc_free(context->cl_ctx); for (i = 0; i < context->n_tl_ctx; i++) { tl_ctx = context->tl_ctx[i]; if ((context->service_team) && (tl_ctx == context->service_ctx)) { /* skip service context cause service team might be in use by other contexts */ continue; } tl_lib = ucc_derived_of(tl_ctx->super.lib, ucc_tl_lib_t); if (tl_ctx->ref_count != 0 ) { ucc_info("tl ctx %s is still in use", tl_lib->iface->super.name); } tl_lib->iface->context.destroy(&tl_ctx->super); } if (context->service_team) { /* service team can now be safely destroyed since all other contexts have been cleaned up and can no longer use the service team */ while (UCC_INPROGRESS == (status = UCC_TL_CTX_IFACE(context->service_ctx) ->team.destroy(&context->service_team->super))) { ucc_context_progress(context); } if (status < 0) { ucc_error("failed to destroy ctx service team"); } ucc_tl_context_put(context->service_ctx); tl_lib = ucc_derived_of(context->service_ctx->super.lib, ucc_tl_lib_t); if (context->service_ctx->ref_count != 0 ) { ucc_info("tl ctx %s is still in use", tl_lib->iface->super.name); } tl_lib->iface->context.destroy(&context->service_ctx->super); } ucc_config_names_array_free(&context->net_devices); ucc_context_topo_cleanup(context->topo); ucc_progress_queue_finalize(context->pq); ucc_free(context->addr_storage.storage); ucc_free(context->all_tls.names); ucc_free(context->tl_ctx); ucc_free(context->ids.pool); ucc_free(context); return UCC_OK; } typedef struct ucc_context_progress_entry { ucc_list_link_t list_elem; ucc_context_progress_fn_t fn; void *arg; } ucc_context_progress_entry_t; ucc_status_t ucc_context_progress_register(ucc_context_t *ctx, ucc_context_progress_fn_t fn, void *progress_arg) { ucc_context_progress_entry_t *entry = ucc_malloc(sizeof(*entry), "progress_entry"); if (!entry) { ucc_error("failed to allocate %zd bytes for progress entry", sizeof(*entry)); return UCC_ERR_NO_MEMORY; } entry->fn = fn; entry->arg = progress_arg; ucc_list_add_tail(&ctx->progress_list, &entry->list_elem); return UCC_OK; } ucc_status_t ucc_context_progress_deregister(ucc_context_t *ctx, ucc_context_progress_fn_t fn, void *progress_arg) { ucc_context_progress_entry_t *entry, *tmp; ucc_list_for_each_safe(entry, tmp, &ctx->progress_list, list_elem) { if (entry->fn == fn && entry->arg == progress_arg) { ucc_list_del(&entry->list_elem); ucc_free(entry); return UCC_OK; } } return UCC_ERR_NOT_FOUND; } ucc_status_t ucc_context_progress(ucc_context_h context) { static int call_num = 0; ucc_status_t status; ucc_context_progress_entry_t *entry; int is_empty; is_empty = ucc_progress_queue_is_empty(context->pq); if (ucc_likely(is_empty)) { call_num--; if (ucc_likely(call_num >= 0)) { return UCC_OK; } /* progress registered progress fns */ ucc_list_for_each(entry, &context->progress_list, list_elem) { entry->fn(entry->arg); } call_num = context->throttle_progress; return UCC_OK; } /* the fn below returns int - number of completed tasks. TODO : do we need to handle it ? Maybe return to user as int as well? */ status = (ucc_status_t)ucc_progress_queue(context->pq); return (status >= 0 ? UCC_OK : status); } static ucc_status_t ucc_context_pack_addr(ucc_context_t *context, ucc_context_addr_len_t *addr_len, int *n_packed, ucc_context_addr_header_t *h) { ucc_context_addr_len_t total_len = 0; ptrdiff_t offset = 0; int i, packed, last_packed; ucc_base_ctx_attr_t attr; ucc_cl_lib_t *cl_lib; ucc_tl_lib_t *tl_lib; ucc_status_t status; attr.attr.mask = UCC_CONTEXT_ATTR_FIELD_CTX_ADDR_LEN; if (h) { attr.attr.mask |= UCC_CONTEXT_ATTR_FIELD_CTX_ADDR; offset = (ptrdiff_t)UCC_CONTEXT_ADDR_DATA(h) - (ptrdiff_t)h; last_packed = 0; } packed = 0; for (i = 0; i < context->n_cl_ctx; i++) { cl_lib = ucc_derived_of(context->cl_ctx[i]->super.lib, ucc_cl_lib_t); attr.attr.ctx_addr = PTR_OFFSET(h, offset); status = cl_lib->iface->context.get_attr(&context->cl_ctx[i]->super, &attr); if (UCC_OK != status) { ucc_error("failed to query addr len from %s", cl_lib->super.log_component.name); return status; } total_len += attr.attr.ctx_addr_len; packed++; if (h) { h->components[last_packed].id = cl_lib->iface->super.id; h->components[last_packed].offset = offset; last_packed++; offset += attr.attr.ctx_addr_len; } } for (i = 0; i < context->n_tl_ctx; i++) { tl_lib = ucc_derived_of(context->tl_ctx[i]->super.lib, ucc_tl_lib_t); attr.attr.ctx_addr = PTR_OFFSET(h, offset); status = tl_lib->iface->context.get_attr(&context->tl_ctx[i]->super, &attr); if (UCC_OK != status) { ucc_error("failed to query addr len from %s", tl_lib->super.log_component.name); return status; } total_len += attr.attr.ctx_addr_len; packed++; if (h) { h->components[last_packed].id = tl_lib->iface->super.id; h->components[last_packed].offset = offset; last_packed++; offset += attr.attr.ctx_addr_len; } } if (addr_len) { *addr_len = total_len + UCC_CONTEXT_ADDR_HEADER_SIZE(packed); } if (n_packed) { *n_packed = packed; } return UCC_OK; } ucc_status_t ucc_context_get_attr( ucc_context_t *context, ucc_context_attr_t *context_attr) { ucc_status_t status = UCC_OK; ucc_status_t vis_status = UCC_OK; ucc_context_addr_header_t *h; if (context_attr->mask & (UCC_CONTEXT_ATTR_FIELD_CTX_ADDR_LEN | UCC_CONTEXT_ATTR_FIELD_CTX_ADDR)) { if (!(context->attr.mask & UCC_CONTEXT_ATTR_FIELD_CTX_ADDR_LEN)) { /* addrlen is not computed yet - do it once and cache on context->attr */ status = ucc_context_pack_addr(context, &context->attr.ctx_addr_len, &context->n_addr_packed, NULL); if (UCC_OK != status) { ucc_error("failed to calc ucc context address length"); return status; } context->attr.mask |= UCC_CONTEXT_ATTR_FIELD_CTX_ADDR_LEN; } context_attr->ctx_addr_len = context->attr.ctx_addr_len; } if (context_attr->mask & UCC_CONTEXT_ATTR_FIELD_CTX_ADDR) { if (!(context->attr.mask & UCC_CONTEXT_ATTR_FIELD_CTX_ADDR)) { /* addr_len and n_packed are computed already */ h = ucc_malloc(context->attr.ctx_addr_len, "ucc_context_address"); if (!h) { ucc_error("failed to allocate %zd bytes for ucc_context_address", context->attr.ctx_addr_len); return UCC_ERR_NO_MEMORY; } h->ctx_id = context->id; h->host_info = ucc_local_host; vis_status = ucc_sysinfo_set_visible_devices( &h->host_info, UCC_SYSINFO_TYPE_NVLINK, NULL); if (vis_status != UCC_OK) { ucc_debug("failed to set visible GPU devices: %d", vis_status); } vis_status = ucc_sysinfo_set_visible_devices( &h->host_info, UCC_SYSINFO_TYPE_IB, &context->net_devices); if (vis_status != UCC_OK) { ucc_debug("failed to set visible IB devices: %d", vis_status); } h->n_components = context->n_addr_packed; status = ucc_context_pack_addr(context, NULL, NULL, h); if (UCC_OK != status) { ucc_error("failed to calc ucc context address length"); return status; } context->attr.mask |= UCC_CONTEXT_ATTR_FIELD_CTX_ADDR; context->attr.ctx_addr = (ucc_context_addr_h)h; } context_attr->ctx_addr = context->attr.ctx_addr; } if (context_attr->mask & UCC_CONTEXT_ATTR_FIELD_WORK_BUFFER_SIZE) { uint64_t max_buffer_size = 0; int i; ucc_base_ctx_attr_t attr; ucc_tl_lib_t * tl_lib; memset(&attr.attr, 0, sizeof(ucc_context_attr_t)); attr.attr.mask = UCC_CONTEXT_ATTR_FIELD_WORK_BUFFER_SIZE; attr.attr.global_work_buffer_size = 0; for (i = 0; i < context->n_tl_ctx; i++) { tl_lib = ucc_derived_of(context->tl_ctx[i]->super.lib, ucc_tl_lib_t); status = tl_lib->iface->context.get_attr(&context->tl_ctx[i]->super, &attr); if (UCC_OK != status) { ucc_error("failed to obtain global work buffer size"); return status; } if (attr.attr.global_work_buffer_size > max_buffer_size) { max_buffer_size = attr.attr.global_work_buffer_size; } } context_attr->global_work_buffer_size = max_buffer_size; } return status; } ucc_status_t ucc_mem_map_import(ucc_context_h context, ucc_mem_map_mode_t mode, const ucc_mem_map_params_t *params, size_t *memh_size, ucc_mem_map_mem_h *memh) { ucc_context_t *ctx = (ucc_context_t *)context; ucc_status_t status = UCC_OK; ucc_config_names_array_t *tls = &ctx->all_tls; size_t offset = 0; int i; ucc_mem_map_memh_t *local_memh; ucc_tl_lib_t *tl_lib; if (!memh) { ucc_error("cannot import NULL memory handle"); return UCC_ERR_INVALID_PARAM; } if (!params) { ucc_error("params cannot be NULL"); return UCC_ERR_INVALID_PARAM; } local_memh = *memh; if (ctx->n_tl_ctx == 0) { ucc_debug("No TL contexts available for import"); local_memh->mode = mode; local_memh->context = ctx; *memh_size = 0; return UCC_OK; } /* memh should have been used in exchanges or from a remote process, addresses, etc. likely garbage. fix it */ local_memh->tl_h = (ucc_mem_map_tl_t *)ucc_calloc( ctx->n_tl_ctx, sizeof(ucc_mem_map_tl_t), "tl memh"); if (!local_memh->tl_h) { ucc_error("failed to allocate tl memh for import"); return UCC_ERR_NO_MEMORY; } for (i = 0; i < ctx->n_tl_ctx; i++) { offset = 0; for (int j = 0; j < local_memh->num_tls; j++) { char *name = PTR_OFFSET(local_memh->pack_buffer, offset); size_t *packed_size = PTR_OFFSET(local_memh->pack_buffer, offset + UCC_MEM_MAP_TL_NAME_LEN); if (strcmp(name, tls->names[i]) == 0) { tl_lib = ucc_derived_of(ctx->tl_ctx[i]->super.lib, ucc_tl_lib_t); strncpy(local_memh->tl_h[j].tl_name, tls->names[i], UCC_MEM_MAP_TL_NAME_LEN - 1); status = tl_lib->iface->context.mem_map( (const ucc_base_context_t *)ctx->tl_ctx[i], mode, local_memh, &local_memh->tl_h[j]); if (status < UCC_ERR_NOT_IMPLEMENTED) { ucc_error("failed to import mem map memh %d", status); ucc_free(local_memh->tl_h); return status; } } offset += UCC_MEM_MAP_TL_NAME_LEN + sizeof(size_t) + *packed_size; } } local_memh->mode = mode; /* fix context as it will be incorrect on a different system */ local_memh->context = ctx; *memh_size = 0; return status; } ucc_status_t ucc_mem_map_export(ucc_context_h context, ucc_mem_map_mode_t mode, const ucc_mem_map_params_t *params, size_t *memh_size, ucc_mem_map_mem_h *memh) { ucc_context_t *ctx = (ucc_context_t *)context; size_t total_pack_size = 0; int packed_tls = 0; size_t offset = 0; ucc_config_names_array_t *tls_names = &ctx->all_tls; ucc_mem_map_memh_t *local_memh; ucc_mem_map_memh_t *exported_memh; void **packed_buffers; ucc_status_t status; ucc_tl_lib_t *tl_lib; int i; int tls; int tlh_index; if (!params) { ucc_error("params cannot be NULL"); return UCC_ERR_INVALID_PARAM; } if (ctx->n_tl_ctx == 0) { ucc_debug("No TL contexts available for export"); return UCC_OK; } if (mode == UCC_MEM_MAP_MODE_EXPORT) { local_memh = (ucc_mem_map_memh_t *)ucc_calloc(1, sizeof(ucc_mem_map_memh_t), "local memh"); if (!local_memh) { ucc_error("failed to allocate a local memory handle"); return UCC_ERR_NO_MEMORY; } local_memh->tl_h = (ucc_mem_map_tl_t *)ucc_calloc( ctx->n_tl_ctx, sizeof(ucc_mem_map_tl_t), "tl memh"); if (!local_memh->tl_h) { ucc_error("failed to allocate a local memory handle"); ucc_free(local_memh); return UCC_ERR_NO_MEMORY; } local_memh->address = params->segments[0].address; local_memh->len = params->segments[0].len; } else { if (!memh) { ucc_error("unable to export map NULL memory handle"); return UCC_ERR_INVALID_PARAM; } local_memh = *memh; } if (mode != UCC_MEM_MAP_MODE_EXPORT_OFFLOAD) { /* map all the memory */ for (i = 0; i < ctx->n_tl_ctx; i++) { tl_lib = ucc_derived_of(ctx->tl_ctx[i]->super.lib, ucc_tl_lib_t); /* always treat as a local mem handle */ status = tl_lib->iface->context.mem_map( (const ucc_base_context_t *)ctx->tl_ctx[i], mode, local_memh, &local_memh->tl_h[i]); if (status != UCC_OK) { if (status < UCC_ERR_NOT_IMPLEMENTED) { ucc_error("failed to map memory"); goto failed_mem_map; } if (status == UCC_ERR_NOT_IMPLEMENTED || status == UCC_ERR_NOT_SUPPORTED) { /* either not implemented or not supported, set memh to null */ local_memh->tl_h[i].packed_size = 0; } } } } packed_buffers = (void **)ucc_calloc(ctx->n_tl_ctx, sizeof(void *), "packed buffers"); if (!packed_buffers) { if (mode == UCC_MEM_MAP_MODE_EXPORT) { ucc_free(local_memh->tl_h); ucc_free(local_memh); } ucc_error("failed to allocate space for packed buffers"); return UCC_ERR_NO_MEMORY; } /* now pack all the memories */ for (i = 0; i < ctx->n_tl_ctx; i++) { tl_lib = ucc_derived_of(ctx->tl_ctx[i]->super.lib, ucc_tl_lib_t); if (mode != UCC_MEM_MAP_MODE_EXPORT_OFFLOAD) { tlh_index = i; } else { tlh_index = -1; for (int j = 0; j < local_memh->num_tls; j++) { if (strcmp(local_memh->tl_h[j].tl_name, tls_names->names[i]) == 0) { tlh_index = j; break; } } if (tlh_index == -1) { /* this TL is not in the memory handle, goto next */ continue; } } /* tl should set packed_size, allocate buffer, pack memh */ status = tl_lib->iface->context.memh_pack( (const ucc_base_context_t *)ctx->tl_ctx[i], mode, &local_memh->tl_h[tlh_index], &packed_buffers[tlh_index]); if (status != UCC_OK) { if (status < UCC_ERR_NOT_IMPLEMENTED) { ucc_error("failed to pack memory handles"); goto failed_pack; } } if (local_memh->tl_h[tlh_index].packed_size) { ++packed_tls; } total_pack_size += local_memh->tl_h[tlh_index].packed_size; } if (packed_tls == 0) { ucc_debug("No TLs available for packing"); ucc_free(packed_buffers); if (mode == UCC_MEM_MAP_MODE_EXPORT) { ucc_free(local_memh->tl_h); ucc_free(local_memh); } return UCC_OK; } /* allocate exported memh, copy items over */ exported_memh = (ucc_mem_map_memh_t *)ucc_calloc( 1, sizeof(ucc_mem_map_memh_t) + total_pack_size + 2 * sizeof(size_t) * packed_tls, "exported memh"); if (!exported_memh) { ucc_error("failed to allocate handle for exported buffers"); status = UCC_ERR_NO_MEMORY; goto failed_pack; } /* copy or allocate depending on the mode */ if (mode == UCC_MEM_MAP_MODE_EXPORT) { exported_memh->tl_h = (ucc_mem_map_tl_t *)ucc_calloc( packed_tls, sizeof(ucc_mem_map_tl_t), "packed tl memh"); if (!exported_memh->tl_h) { ucc_error("failed to allocate handle for exported buffers' tl handles"); status = UCC_ERR_NO_MEMORY; ucc_free(exported_memh); goto failed_pack; } for (i = 0, offset = 0, tls = 0; i < ctx->n_tl_ctx; i++) { if (local_memh->tl_h[i].packed_size) { strncpy(PTR_OFFSET(exported_memh->pack_buffer, offset), local_memh->tl_h[i].tl_name, UCC_MEM_MAP_TL_NAME_LEN); offset += UCC_MEM_MAP_TL_NAME_LEN; memcpy(PTR_OFFSET(exported_memh->pack_buffer, offset), &local_memh->tl_h[i].packed_size, sizeof(size_t)); offset += sizeof(size_t); memcpy(PTR_OFFSET(exported_memh->pack_buffer, offset), packed_buffers[i], local_memh->tl_h[i].packed_size); ucc_free(packed_buffers[i]); offset += local_memh->tl_h[i].packed_size; memcpy(&exported_memh->tl_h[tls++], &local_memh->tl_h[i], sizeof(ucc_mem_map_tl_t)); } } } else { exported_memh->tl_h = local_memh->tl_h; for (i = 0; i < local_memh->num_tls; i++) { /* these are already packed in order */ strncpy(PTR_OFFSET(exported_memh->pack_buffer, offset), local_memh->tl_h[i].tl_name, UCC_MEM_MAP_TL_NAME_LEN); offset += UCC_MEM_MAP_TL_NAME_LEN; memcpy(PTR_OFFSET(exported_memh->pack_buffer, offset), &local_memh->tl_h[i].packed_size, sizeof(size_t)); offset += sizeof(size_t); memcpy(PTR_OFFSET(exported_memh->pack_buffer, offset), packed_buffers[i], local_memh->tl_h[i].packed_size); ucc_free(packed_buffers[i]); offset += local_memh->tl_h[i].packed_size; } } exported_memh->mode = mode; exported_memh->context = ctx; exported_memh->address = local_memh->address; exported_memh->len = local_memh->len; exported_memh->num_tls = packed_tls; *memh = exported_memh; *memh_size = sizeof(ucc_mem_map_memh_t) + offset; if (mode == UCC_MEM_MAP_MODE_EXPORT) { ucc_free(local_memh->tl_h); } ucc_free(local_memh); ucc_free(packed_buffers); return UCC_OK; failed_pack: for (int j = 0; j < i; j++) { ucc_free(packed_buffers[j]); } i = ctx->n_tl_ctx; failed_mem_map: for (int j = 0; j < i; j++) { tl_lib = ucc_derived_of(ctx->tl_ctx[i]->super.lib, ucc_tl_lib_t); tl_lib->iface->context.mem_unmap((const ucc_base_context_t *)ctx, UCC_MEM_MAP_MODE_EXPORT, &local_memh->tl_h[j]); } if (mode == UCC_MEM_MAP_MODE_EXPORT) { ucc_free(local_memh->tl_h); } ucc_free(local_memh); *memh = NULL; *memh_size = 0; return status; } ucc_status_t ucc_mem_map(ucc_context_h context, ucc_mem_map_mode_t mode, const ucc_mem_map_params_t *params, size_t *memh_size, ucc_mem_map_mem_h *memh) { if (mode >= UCC_MEM_MAP_MODE_LAST) { ucc_error("Invalid memory map mode: %d", mode); return UCC_ERR_INVALID_PARAM; } if (mode == UCC_MEM_MAP_MODE_IMPORT || mode == UCC_MEM_MAP_MODE_IMPORT_OFFLOAD) { return ucc_mem_map_import(context, mode, params, memh_size, memh); } if (!params) { ucc_error("params cannot be NULL"); return UCC_ERR_INVALID_PARAM; } if (params->n_segments > 1) { ucc_error("UCC only supports one mapping per call"); return UCC_ERR_INVALID_PARAM; } return ucc_mem_map_export(context, mode, params, memh_size, memh); } ucc_status_t ucc_mem_unmap(ucc_mem_map_mem_h *memh) { ucc_context_t *ctx; ucc_status_t status; ucc_mem_map_memh_t *lmemh; ucc_tl_lib_t *tl_lib; int i; int j; ucc_config_names_array_t *tls; if (!memh) { ucc_warn("unable to free NULL memh"); return UCC_ERR_INVALID_PARAM; } if (!*memh) { ucc_warn("unable to free NULL memory handle"); return UCC_ERR_INVALID_PARAM; } lmemh = *memh; ctx = (ucc_context_t *)lmemh->context; tls = &ctx->all_tls; for (i = 0; i < ctx->n_tl_ctx; i++) { for (j = 0; j < lmemh->num_tls; j++) { if (strcmp(lmemh->tl_h[j].tl_name, tls->names[i]) == 0) { tl_lib = ucc_derived_of(ctx->tl_ctx[i]->super.lib, ucc_tl_lib_t); status = tl_lib->iface->context.mem_unmap( (const ucc_base_context_t *)ctx->tl_ctx[i], lmemh->mode, &lmemh->tl_h[j]); if (status < UCC_ERR_NOT_IMPLEMENTED) { ucc_error("error during unmap operation on TL %s", lmemh->tl_h[j].tl_name); return status; } } } } /* Free the TL handles array if it was allocated separately */ if (lmemh->tl_h && (lmemh->mode == UCC_MEM_MAP_MODE_EXPORT || lmemh->mode == UCC_MEM_MAP_MODE_IMPORT)) { ucc_free(lmemh->tl_h); } /* Free the memory handle structure */ ucc_free(lmemh); *memh = NULL; return UCC_OK; } ucc-1.8.0/src/core/ucc_ee.c0000664000175000017500000000655715211535620015601 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "ucc_team.h" #include "ucc_ee.h" #include "ucc_lib.h" #include "components/cl/ucc_cl.h" #include "components/tl/ucc_tl.h" const char *ucc_ee_ev_names[] = { [UCC_EVENT_COLLECTIVE_POST] = "COLL_POST", [UCC_EVENT_COLLECTIVE_COMPLETE] = "COLL_COMPLETE", [UCC_EVENT_COMPUTE_COMPLETE] = "COMPUTE_COMPLETE", [UCC_EVENT_OVERFLOW] = "OVERFLOW", }; ucc_status_t ucc_ee_create(ucc_team_h team, const ucc_ee_params_t *params, ucc_ee_h *ee_p) { ucc_ee_t *ee; ee = ucc_malloc(sizeof(ucc_ee_t), "ucc execution engine"); if (!ee) { ucc_error("failed to allocate %zd bytes for ucc execution engine", sizeof(ucc_ee_t)); return UCC_ERR_NO_MEMORY; } ee->team = team; ee->ee_type = params->ee_type; ee->ee_context_size = params->ee_context_size; ee->ee_context = params->ee_context; ucc_spinlock_init(&ee->lock, 0); ucc_queue_head_init(&ee->event_in_queue); ucc_queue_head_init(&ee->event_out_queue); *ee_p = ee; ucc_debug("ee is created: %p ee_context: %p", ee, params->ee_context); return UCC_OK; } ucc_status_t ucc_ee_destroy(ucc_ee_h ee) { ucc_debug("ee is destroyed: %p", ee); ucc_spinlock_destroy(&ee->lock); ucc_free(ee); return UCC_OK; } ucc_status_t ucc_ee_get_event_internal(ucc_ee_h ee, ucc_ev_t **ev, ucc_queue_head_t *queue) { ucc_event_desc_t *event_desc; ucc_queue_elem_t *elem; ucc_spin_lock(&ee->lock); if (ucc_queue_is_empty(queue)) { ucc_spin_unlock(&ee->lock); return UCC_ERR_NOT_FOUND; } elem = ucc_queue_pull(queue); ucc_spin_unlock(&ee->lock); ucc_assert(elem != NULL); event_desc = ucc_container_of(elem, ucc_event_desc_t, queue); *ev = &event_desc->ev; ucc_debug("EE Event Get. ee:%p, queue:%p ev_type:%s ", ee, queue, ucc_ee_ev_names[event_desc->ev.ev_type]); return UCC_OK; } ucc_status_t ucc_ee_get_event(ucc_ee_h ee, ucc_ev_t **ev) { return ucc_ee_get_event_internal(ee, ev, &ee->event_out_queue); } ucc_status_t ucc_ee_ack_event(ucc_ee_h ee, //NOLINT ucc_ev_t *ev) { ucc_event_desc_t *event_desc; event_desc = ucc_container_of(ev, ucc_event_desc_t, ev); /* TODO destroy event context */ ucc_free(event_desc); return UCC_OK; } ucc_status_t ucc_ee_set_event_internal(ucc_ee_h ee, ucc_ev_t *ev, ucc_queue_head_t *queue) { ucc_event_desc_t *event_desc; event_desc = ucc_malloc(sizeof(ucc_event_desc_t), "event descriptor"); if (ucc_unlikely(!event_desc)) { ucc_error("failed to allocate ucc event descriptor"); return UCC_ERR_NO_MEMORY; } event_desc->ev = *ev; ucc_spin_lock(&ee->lock); ucc_queue_push(queue, &event_desc->queue); ucc_spin_unlock(&ee->lock); ucc_debug("EE Event Set. ee:%p, queue:%p ev_type:%s ", ee, queue, ucc_ee_ev_names[ev->ev_type]); return UCC_OK; } ucc_status_t ucc_ee_set_event(ucc_ee_h ee, ucc_ev_t *ev) { return ucc_ee_set_event_internal(ee, ev, &ee->event_in_queue); } ucc_status_t ucc_ee_wait(ucc_ee_h ee, ucc_ev_t *ev) { while(UCC_OK != ucc_ee_get_event(ee, &ev)) { ucc_progress_queue(ee->team->contexts[0]->pq); } return UCC_OK; } ucc-1.8.0/src/core/ucc_global_opts.c0000664000175000017500000001130615211535620017501 0ustar alastairalastair/** * Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "ucc_global_opts.h" #include "utils/debug/types.h" #include "utils/ucc_compiler_def.h" #include "utils/ucc_datastruct.h" #include "utils/debug/log.h" #include "utils/profile/ucc_profile.h" UCC_LIST_HEAD(ucc_config_global_list); ucc_global_config_t ucc_global_config = { .log_component = {UCC_LOG_LEVEL_WARN, "UCC", "*"}, .coll_trace = {UCC_LOG_LEVEL_WARN, "UCC_COLL", "*"}, .component_path = NULL, .install_path = NULL, .initialized = 0, .profile_mode = 0, .profile_file = "", .profile_log_size = 0, .file_cfg = 0, .log_file = "", .log_file_size = SIZE_MAX, .log_file_rotate = 0, .log_buffer_size = 1024, .log_data_size = 0, .log_print_enable = 0, .log_level_trigger = UCC_LOG_LEVEL_FATAL}; ucc_config_field_t ucc_global_config_table[] = { {"LOG_LEVEL", "warn", "UCC logging level. Messages with a level higher or equal to the selected " "will be printed.\n" "Possible values are: fatal, error, warn, info, debug, trace, data, func, " "poll.", ucc_offsetof(ucc_global_config_t, log_component), UCC_CONFIG_TYPE_LOG_COMP}, {"COLL_TRACE", "warn", "UCC collective logging level. Higher level will result in more verbose " "collective info. \n " "Possible values are: fatal, error, warn, info, debug, trace, data, func, " "poll.", ucc_offsetof(ucc_global_config_t, coll_trace), UCC_CONFIG_TYPE_LOG_COMP }, {"PROFILE_MODE", "", "Profile collection modes. If none is specified, profiling is disabled.\n" " - log - Record all timestamps.\n" " - accum - Accumulate measurements per location.\n", ucc_offsetof(ucc_global_config_t, profile_mode), UCC_CONFIG_TYPE_BITMAP(ucs_profile_mode_names)}, {"PROFILE_FILE", "ucc_%h_%p.prof", "File name to dump profiling data to.\n" "Substitutions: %h: host, %p: pid, %c: cpu, %t: time, %u: user, %e: " "exe.\n", ucc_offsetof(ucc_global_config_t, profile_file), UCC_CONFIG_TYPE_STRING}, {"PROFILE_LOG_SIZE", "4m", "Maximal size of profiling log. New records will replace old records.", ucc_offsetof(ucc_global_config_t, profile_log_size), UCC_CONFIG_TYPE_MEMUNITS}, {"CONFIG_FILE", "auto", "Location of configuration file.\n" "auto - config file is searched in $HOME/ucc.conf first, then, if not " "found, in /share/ucc.conf.\n" "empty string \"\" - disable use of config file", ucc_offsetof(ucc_global_config_t, cfg_filename), UCC_CONFIG_TYPE_STRING}, {"LOG_FILE", "", "If not empty, UCC will print log messages to the specified file instead of stdout.\n" "The following substitutions are performed on this string:\n" " %p - Replaced with process ID\n" " %h - Replaced with host name", ucc_offsetof(ucc_global_config_t, log_file), UCC_CONFIG_TYPE_STRING}, {"LOG_FILE_SIZE", "inf", "The maximal size of log file. The maximal log file size has to be >= LOG_BUFFER.", ucc_offsetof(ucc_global_config_t, log_file_size), UCC_CONFIG_TYPE_MEMUNITS}, {"LOG_FILE_ROTATE", "0", "The maximal number of backup log files that could be created to save logs\n" "after the previous ones (if any) are completely filled. The value has to be\n" "less than the maximal signed integer value.", ucc_offsetof(ucc_global_config_t, log_file_rotate), UCC_CONFIG_TYPE_UINT}, {"LOG_BUFFER", "1024", "Buffer size for a single log message.", ucc_offsetof(ucc_global_config_t, log_buffer_size), UCC_CONFIG_TYPE_MEMUNITS}, {"LOG_DATA_SIZE", "0", "How much packet payload to print, at most, in data mode.", ucc_offsetof(ucc_global_config_t, log_data_size), UCC_CONFIG_TYPE_ULONG}, {"LOG_PRINT_ENABLE", "n", "Enable output of ucc_print(). This option is intended for use by the library developers.", ucc_offsetof(ucc_global_config_t, log_print_enable), UCC_CONFIG_TYPE_BOOL}, {"LOG_LEVEL_TRIGGER", "fatal", "Log level to trigger error handling.", ucc_offsetof(ucc_global_config_t, log_level_trigger), UCC_CONFIG_TYPE_ENUM(ucc_log_level_names)}, {"CHECK_ASYMMETRIC_DT", "n", "Enable asymmetric datatype checking for rooted collectives\n" "(gather, gatherv, scatter, scatterv). Uses allreduce to verify\n" "all ranks use the same datatype. Disabled by default for performance.\n" "Enable for debugging or when OMPI needs UCC to detect asymmetric\n" "datatypes for proper fallback behavior.", ucc_offsetof(ucc_global_config_t, check_asymmetric_dt), UCC_CONFIG_TYPE_BOOL}, {NULL} }; ucc-1.8.0/src/core/ucc_constructor.c0000664000175000017500000001742015211535620017564 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "config.h" #include "ucc_global_opts.h" #include "utils/debug/log_def.h" #include "utils/ucc_malloc.h" #include "utils/ucc_component.h" #include "utils/ucc_log.h" #include "utils/ucc_sys.h" #include "utils/ucc_string.h" #include "utils/ucc_proc_info.h" #include "utils/profile/ucc_profile.h" #include "ucc/api/ucc_version.h" #include #include static ucc_status_t ucc_check_config_file(void) { ucc_global_config_t *cfg = &ucc_global_config; ucc_status_t status = UCC_OK; const char * default_share_name = "/share/ucc.conf"; const char * default_home_name = "/ucc.conf"; const char * home; char * filename; /* First check the UCC_CONFIG_FILE - most precedence */ if (strlen(cfg->cfg_filename) == 0) { /* file configuration disabled */ return UCC_OK; } if (0 != strcasecmp(cfg->cfg_filename, "auto")) { /* Actual file path is provided */ status = ucc_parse_file_config(cfg->cfg_filename, &ucc_global_config.file_cfg); if (UCC_ERR_NOT_FOUND == status) { /* File ENV value provided but not available */ ucc_warn("failed to open config file: %s", cfg->cfg_filename); } return status; } /* CONFIG_FILE is set to AUTO. Search HOME and install/share */ if (NULL != (home = getenv("HOME"))) { if (UCC_OK != (status = ucc_str_concat(home ,default_home_name, &filename))) { return status; } status = ucc_parse_file_config(filename, &ucc_global_config.file_cfg); ucc_free(filename); if (UCC_ERR_NOT_FOUND != status) { /* either OK or fatal error, NOT_FOUND means no file in HOME - just continue */ return status; } } /* Finally, try to find config file in the library install/share */ if (UCC_OK != (status = ucc_str_concat(cfg->install_path, default_share_name, &filename))) { return status; } status = ucc_parse_file_config(filename, &ucc_global_config.file_cfg); ucc_free(filename); return status; } static ucc_status_t init_lib_paths(void) { const char *so_path = ucc_sys_get_lib_path(); ucc_status_t status; char *lib_path; if (!so_path) { return UCC_ERR_NOT_FOUND; } status = ucc_sys_dirname(so_path, &lib_path); if (UCC_OK != status) { return status; } status = ucc_sys_dirname(lib_path, &ucc_global_config.install_path); if (UCC_OK != status) { goto out; } status = ucc_sys_path_join(lib_path, UCC_MODULE_SUBDIR, &ucc_global_config.component_path); out: free(lib_path); return status; } UCC_CONFIG_REGISTER_TABLE(ucc_global_config_table, "UCC global", NULL, ucc_global_config, &ucc_config_global_list) static pthread_mutex_t ucc_constructor_mutex = PTHREAD_MUTEX_INITIALIZER; ucc_status_t ucc_constructor(void) { ucc_global_config_t *cfg = &ucc_global_config; ucc_status_t status = UCC_OK; Dl_info dl_info; int ret; pthread_mutex_lock(&ucc_constructor_mutex); if (cfg->initialized) { goto exit_unlock_mutex; } ucc_log_early_init(); status = ucc_config_parser_fill_opts(&ucc_global_config, UCC_CONFIG_GET_TABLE(ucc_global_config_table), "UCC_", 1); if (UCC_OK != status) { ucc_error("failed to parse global options"); goto exit_unlock_mutex; } ucc_log_init(); cfg->initialized = 1; if (UCC_OK != (status = init_lib_paths())) { ucc_error("failed to init ucc components path"); goto exit_unlock_mutex; } status = ucc_check_config_file(); if (UCC_OK != status && UCC_ERR_NOT_FOUND != status) { /* bail only in case of real error */ goto exit_unlock_mutex; } status = ucc_components_load("cl", &cfg->cl_framework); if (UCC_OK != status) { ucc_error("no CL components were found in the " "ucc modules dir: %s", cfg->component_path); goto exit_unlock_mutex; } status = ucc_component_check_scores_uniq(&cfg->cl_framework); if (UCC_OK != status) { ucc_error("CLs must have distinct uniq default scores"); goto exit_unlock_mutex; } status = ucc_components_load("tl", &cfg->tl_framework); if (UCC_OK != status) { /* not critical - some CLs may operate w/o use of TL */ ucc_debug("no TL components were found in the " "ucc modules dir: %s", cfg->component_path); } status = ucc_component_check_scores_uniq(&cfg->tl_framework); if (UCC_OK != status) { ucc_error("TLs must have distinct uniq default scores"); goto exit_unlock_mutex; } status = ucc_components_load("mc", &cfg->mc_framework); if (UCC_OK != status) { ucc_error("no memory components were found in the " "ucc modules dir: %s", cfg->component_path); goto exit_unlock_mutex; } status = ucc_components_load("ec", &cfg->ec_framework); if (status != UCC_OK) { if (status == UCC_ERR_NOT_FOUND) { ucc_info("no execution components were found in the " "ucc modules dir: %s. " "Triggered operations might not work", cfg->component_path); } else { ucc_error("failed to load execution components %d (%s)", status, ucc_status_string(status)); goto exit_unlock_mutex; } } status = ucc_components_load("sysinfo", &cfg->sysinfo_framework); if (status != UCC_OK) { if (status == UCC_ERR_NOT_FOUND) { ucc_info("no sysinfo components were found in the " "ucc modules dir: %s. " "some sysinfo features will be disabled", cfg->component_path); status = UCC_OK; } else { ucc_error("failed to load sysinfo components %d (%s)", status, ucc_status_string(status)); goto exit_unlock_mutex; } } if (UCC_OK != ucc_local_proc_info_init()) { ucc_error("failed to initialize local proc info"); goto exit_unlock_mutex; } #ifdef HAVE_PROFILING ucc_profile_init(cfg->profile_mode, cfg->profile_file, cfg->profile_log_size); #endif if (ucc_global_config.log_component.log_level >= UCC_LOG_LEVEL_INFO) { ret = dladdr(ucc_init_version, &dl_info); if (ret == 0) { ucc_error("failed to get ucc_init_version handler"); status = UCC_ERR_NO_RESOURCE; goto exit_unlock_mutex; } ucc_info("version: %s, loaded from: %s, cfg file: %s", ucc_get_version_string(), dl_info.dli_fname, ucc_global_config.file_cfg ? ucc_global_config.file_cfg->filename: "n/a"); } exit_unlock_mutex: pthread_mutex_unlock(&ucc_constructor_mutex); return status; } __attribute__((destructor)) static void ucc_destructor(void) { if (ucc_global_config.initialized) { ucc_log_cleanup(); #ifdef HAVE_PROFILING ucc_profile_cleanup(); #endif ucc_config_parser_release_opts(&ucc_global_config, ucc_global_config_table); if (ucc_global_config.file_cfg) { ucc_release_file_config(ucc_global_config.file_cfg); } free(ucc_global_config.component_path); free(ucc_global_config.install_path); } } ucc-1.8.0/src/core/ucc_team.c0000664000175000017500000005622615211535620016134 0ustar alastairalastair/** * Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (c) Meta Platforms, Inc. and affiliates. 2022. * * See file LICENSE for terms. */ #include "config.h" #include "ucc_team.h" #include "ucc_lib.h" #include "components/cl/ucc_cl.h" #include "components/tl/ucc_tl.h" #include "ucc_service_coll.h" static ucc_status_t ucc_team_alloc_id(ucc_team_t *team); static void ucc_team_release_id(ucc_team_t *team); void ucc_copy_team_params(ucc_team_params_t *dst, const ucc_team_params_t *src) { dst->mask = src->mask; UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_TEAM_PARAM_FIELD_ORDERING, ordering); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_TEAM_PARAM_FIELD_OUTSTANDING_COLLS, outstanding_colls); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_TEAM_PARAM_FIELD_EP, ep); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_TEAM_PARAM_FIELD_EP_RANGE, ep_range); //TODO do we need to copy ep_list ? UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_TEAM_PARAM_FIELD_TEAM_SIZE, team_size); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_TEAM_PARAM_FIELD_SYNC_TYPE, sync_type); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_TEAM_PARAM_FIELD_OOB, oob); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_TEAM_PARAM_FIELD_P2P_CONN, p2p_conn); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_TEAM_PARAM_FIELD_MEM_PARAMS, mem_params); UCC_COPY_PARAM_BY_FIELD(dst, src, UCC_TEAM_PARAM_FIELD_EP_MAP, ep_map); } ucc_status_t ucc_team_get_attr(ucc_team_h team, ucc_team_attr_t *team_attr) { uint64_t supported_fields = UCC_TEAM_ATTR_FIELD_SIZE | UCC_TEAM_ATTR_FIELD_EP; if (team_attr->mask & ~supported_fields) { ucc_error("ucc_team_get_attr() is not implemented for specified field"); return UCC_ERR_NOT_IMPLEMENTED; } if (team_attr->mask & UCC_TEAM_ATTR_FIELD_SIZE) { team_attr->size = team->size; } if (team_attr->mask & UCC_TEAM_ATTR_FIELD_EP) { team_attr->ep = team->rank; } return UCC_OK; } static ucc_status_t ucc_team_create_post_single(ucc_context_t *context, ucc_team_t *team) { ucc_status_t status; if (context->service_team && team->size > 1) { /* Use internal service team for OOB, skip OOB if team size is 1 */ ucc_subset_t subset = {.myrank = team->rank, .map.ep_num = team->size, .map.type = UCC_EP_MAP_FULL}; status = ucc_internal_oob_init(team, subset, &team->bp.params.oob); if (UCC_OK != status) { return status; } team->bp.params.mask |= UCC_TEAM_PARAM_FIELD_OOB; } team->cl_teams = ucc_malloc(sizeof(ucc_cl_team_t *) * context->n_cl_ctx); if (!team->cl_teams) { ucc_error("failed to allocate %zd bytes for cl teams array", sizeof(ucc_cl_team_t *) * context->n_cl_ctx); return UCC_ERR_NO_MEMORY; } team->bp.rank = team->rank; team->bp.size = team->size; team->bp.team = team; team->bp.map.type = UCC_EP_MAP_FULL; team->bp.map.ep_num = team->size; team->state = (team->size > 1) ? UCC_TEAM_ADDR_EXCHANGE : UCC_TEAM_CL_CREATE; team->last_team_create_posted = -1; return UCC_OK; } ucc_status_t ucc_team_create_post(ucc_context_h *contexts, uint32_t num_contexts, const ucc_team_params_t *params, ucc_team_h *new_team) { uint64_t team_size = 0; uint64_t team_rank = UINT64_MAX; ucc_team_t *team; ucc_status_t status; if (num_contexts < 1) { return UCC_ERR_INVALID_PARAM; } else if (num_contexts > 1) { ucc_error("team creation from multiple contexts is not supported yet"); return UCC_ERR_NOT_SUPPORTED; } if (params->mask & UCC_TEAM_PARAM_FIELD_TEAM_SIZE) { team_size = params->team_size; } if (params->mask & UCC_TEAM_PARAM_FIELD_OOB) { if (team_size > 0 && params->oob.n_oob_eps != team_size) { ucc_error( "inconsistent team_sizes provided as params.team_size %llu " "and params.oob.n_oob_eps %llu", (unsigned long long)params->team_size, (unsigned long long)params->oob.n_oob_eps); return UCC_ERR_INVALID_PARAM; } team_size = params->oob.n_oob_eps; } if (params->mask & UCC_TEAM_PARAM_FIELD_EP_MAP) { if (team_size > 0 && params->ep_map.ep_num != team_size) { ucc_error( "inconsistent team_sizes provided as params.team_size %llu " "and/or params.oob.n_oob_eps %llu and/or ep_map.ep_num %llu", (unsigned long long)params->team_size, (unsigned long long)params->oob.n_oob_eps, (unsigned long long)params->ep_map.ep_num); return UCC_ERR_INVALID_PARAM; } team_size = params->ep_map.ep_num; } if (team_size < 1) { ucc_warn("minimal size of UCC team is 1, provided %llu", (unsigned long long)team_size); return UCC_ERR_INVALID_PARAM; } if ((params->mask & UCC_TEAM_PARAM_FIELD_EP) && (params->mask & UCC_TEAM_PARAM_FIELD_EP_RANGE) && (params->ep_range == UCC_COLLECTIVE_EP_RANGE_CONTIG)) { if ((params->mask & UCC_TEAM_PARAM_FIELD_OOB) && (params->oob.oob_ep != params->ep)) { ucc_error( "inconsistent EP value is provided as params.ep %llu " "and params.oob.oob_ep %llu", (unsigned long long)params->ep, (unsigned long long)params->oob.oob_ep); return UCC_ERR_INVALID_PARAM; } team_rank = params->ep; } else if (params->mask & UCC_TEAM_PARAM_FIELD_OOB) { team_rank = params->oob.oob_ep; } if (team_rank == UINT64_MAX) { /* Neither EP nor OOB_EP is provided, can't assign the rank */ ucc_error("either UCC_TEAM_PARAM_FIELD_EP(RANGE) " "or UCC_TEAM_PARAM_FIELD_OOB must be provided"); return UCC_ERR_INVALID_PARAM; } if (team_size > (uint64_t)UCC_RANK_MAX) { ucc_error("team size is too large: %llu, max supported %u", (unsigned long long)team_size, UCC_RANK_MAX); return UCC_ERR_INVALID_PARAM; } if (team_rank > (uint64_t)UCC_RANK_MAX) { ucc_error("team rank is too large: %llu, max supported %u", (unsigned long long)team_rank, UCC_RANK_MAX); return UCC_ERR_INVALID_PARAM; } team = ucc_calloc(1, sizeof(ucc_team_t), "ucc_team"); if (!team) { ucc_error("failed to allocate %zd bytes for ucc team", sizeof(ucc_team_t)); return UCC_ERR_NO_MEMORY; } team->runtime_oob = params->oob; team->num_contexts = num_contexts; team->size = (ucc_rank_t)team_size; team->rank = (ucc_rank_t)team_rank; team->seq_num = 0; team->contexts = ucc_malloc(sizeof(ucc_context_t *) * num_contexts, "ucc_team_ctx"); if (!team->contexts) { ucc_error("failed to allocate %zd bytes for ucc team contexts array", sizeof(ucc_context_t) * num_contexts); status = UCC_ERR_NO_MEMORY; goto err_ctx_alloc; } memcpy(team->contexts, contexts, sizeof(ucc_context_t *) * num_contexts); ucc_copy_team_params(&team->bp.params, params); /* check if user provides team id and if it is not too large */ if ((params->mask & UCC_TEAM_PARAM_FIELD_ID) && (params->id <= UCC_TEAM_ID_MAX)) { team->id = ((uint16_t)params->id) | UCC_TEAM_ID_EXTERNAL_BIT; } status = ucc_team_create_post_single(contexts[0], team); *new_team = team; return status; err_ctx_alloc: *new_team = NULL; ucc_free(team); return status; } static ucc_status_t ucc_team_create_service_team(ucc_context_t *context, ucc_team_t *team) { ucc_status_t status; if (context->service_team) { /* Global single service team is allocated on ucc_context. UCC_INTERNAL_OOB is enabled. Don't need another service team */ return UCC_OK; } if (!team->service_team) { ucc_base_team_params_t b_params; ucc_base_team_t * b_team; status = ucc_tl_context_get(context, "ucp", &context->service_ctx); if (UCC_OK != status) { ucc_warn("TL UCP context is not available, " "service team can not be created"); return status; } memcpy(&b_params, &team->bp, sizeof(ucc_base_team_params_t)); b_params.scope = UCC_CL_LAST + 1; // CORE scope id - never overlaps with CL type b_params.scope_id = 0; b_params.id = 0; b_params.team = team; b_params.map.type = UCC_EP_MAP_FULL; status = UCC_TL_CTX_IFACE(context->service_ctx) ->team.create_post(&context->service_ctx->super, &b_params, &b_team); if (UCC_OK != status) { ucc_error("tl ucp service team create post failed"); return status; } team->service_team = ucc_derived_of(b_team, ucc_tl_team_t); } status = UCC_TL_CTX_IFACE(context->service_ctx) ->team.create_test(&team->service_team->super); if (status < 0) { team->service_team = NULL; ucc_error("failed to create service tl ucp team"); } return status; } static ucc_status_t ucc_team_create_cls(ucc_context_t *context, ucc_team_t *team) { ucc_cl_iface_t *cl_iface; ucc_base_team_t *b_team; ucc_status_t status; ucc_subset_t subset; int i; if (context->topo && !team->topo && team->size > 1) { /* Context->topo is not NULL if any of the enabled CLs reported topo_required through the lib_attr */ subset.map = team->ctx_map; subset.myrank = team->rank; status = ucc_topo_init(subset, context->topo, &team->topo); if (UCC_OK != status) { ucc_warn("failed to init team topo"); } } if (team->last_team_create_posted >= 0) { cl_iface = UCC_CL_CTX_IFACE(context->cl_ctx[team->last_team_create_posted]); b_team = &team->cl_teams[team->last_team_create_posted]->super; status = cl_iface->team.create_test(b_team); if (status < 0) { team->n_cl_teams--; ucc_debug("failed to create CL %s team", cl_iface->super.name); cl_iface->team.destroy(b_team); } else if (status == UCC_INPROGRESS) { return status; } } for (i = team->last_team_create_posted + 1; i < context->n_cl_ctx; i++) { cl_iface = UCC_CL_CTX_IFACE(context->cl_ctx[i]); status = cl_iface->team.create_post(&context->cl_ctx[i]->super, &team->bp, &b_team); if (status != UCC_OK) { ucc_debug("failed to create CL %s team", cl_iface->super.name); continue; } status = cl_iface->team.create_test(b_team); if (status < 0) { ucc_debug("failed to create CL %s team", cl_iface->super.name); cl_iface->team.destroy(b_team); continue; } team->cl_teams[team->n_cl_teams++] = ucc_derived_of(b_team, ucc_cl_team_t); if (status == UCC_INPROGRESS) { team->last_team_create_posted = i; /* workaround to fix oob allgather issue if multiple teams use it simultaneously*/ return UCC_INPROGRESS; } } if (0 == team->n_cl_teams) { ucc_error("no CL teams were created"); return UCC_ERR_NO_MESSAGE; } return UCC_OK; } static inline ucc_status_t ucc_team_exchange(ucc_context_t *context, ucc_team_t * team) { ucc_team_oob_coll_t oob = team->runtime_oob; ucc_status_t status; if (!context->addr_storage.storage) { /* There is no addresses collected on the context (can be, e.g., if user did not pass OOB for ctx creation). Need to exchange addresses here */ return ucc_core_addr_exchange(context, &oob, &team->addr_storage); } /* We only need to exchange ctx_ranks and build map to ctx array */ ucc_assert(context->addr_storage.storage != NULL); if (team->bp.params.mask & UCC_TEAM_PARAM_FIELD_EP_MAP) { team->ctx_map = team->bp.params.ep_map; } else { if (!team->ctx_ranks) { team->ctx_ranks = ucc_malloc(team->size * sizeof(ucc_rank_t), "ctx_ranks"); if (!team->ctx_ranks) { ucc_error("failed to allocate %zd bytes for ctx ranks array", team->size * sizeof(ucc_rank_t)); return UCC_ERR_NO_MEMORY; } status = oob.allgather(&context->rank, team->ctx_ranks, sizeof(ucc_rank_t), oob.coll_info, &team->oob_req); if (UCC_OK != status) { ucc_error("failed to start oob allgather for proc info exchange"); ucc_free(team->ctx_ranks); return status; } } status = oob.req_test(team->oob_req); if (status < 0) { oob.req_free(team->oob_req); ucc_error("oob req test failed during team proc info exchange"); return status; } else if (UCC_INPROGRESS == status) { return status; } oob.req_free(team->oob_req); ucc_assert(team->size >= 2); team->ctx_map = ucc_ep_map_from_array(&team->ctx_ranks, team->size, context->addr_storage.size, 1); } ucc_debug("team %p rank %d, ctx_rank %d, map_type %d", team, team->rank, context->rank, team->ctx_map.type); return UCC_OK; } static ucc_status_t ucc_team_build_score_map(ucc_team_t *team) { ucc_coll_score_t *score, *score_merge, *score_next; ucc_status_t status; int i; ucc_assert(team->n_cl_teams > 0); status = UCC_CL_TEAM_IFACE(team->cl_teams[0]) ->team.get_scores(&team->cl_teams[0]->super, &score); if (UCC_OK != status) { ucc_error("failed to get cl %s scores", UCC_CL_TEAM_IFACE(team->cl_teams[0])->super.name); return status; } for (i = 1; i < team->n_cl_teams; i++) { status = UCC_CL_TEAM_IFACE(team->cl_teams[i]) ->team.get_scores(&team->cl_teams[i]->super, &score_next); if (UCC_OK != status) { ucc_error("failed to get cl %s scores", UCC_CL_TEAM_IFACE(team->cl_teams[i])->super.name); ucc_coll_score_free(score); return status; } status = ucc_coll_score_merge(score, score_next, &score_merge, 1); if (UCC_OK != status) { ucc_error("failed to merge scores"); ucc_coll_score_free(score); ucc_coll_score_free(score_next); return status; } score = score_merge; } status = ucc_coll_score_build_map(score, &team->score_map); if (UCC_OK != status) { ucc_error("failed to build score map"); } return status; } ucc_status_t ucc_team_create_test_single(ucc_context_t *context, ucc_team_t *team) { ucc_status_t status = UCC_OK; switch (team->state) { case UCC_TEAM_ADDR_EXCHANGE: status = ucc_team_exchange(context, team); if (UCC_OK != status) { goto out; } team->state = UCC_TEAM_SERVICE_TEAM; /* fall through */ case UCC_TEAM_SERVICE_TEAM: if ((context->cl_flags & UCC_BASE_LIB_FLAG_SERVICE_TEAM_REQUIRED) || ((context->cl_flags & UCC_BASE_LIB_FLAG_TEAM_ID_REQUIRED) && (team->id == 0))) { /* We need service team either when it is explicitly required * by any CL/TL (e.g. CL/HIER) or if TEAM_ID is required but * not provided by the user */ status = ucc_team_create_service_team(context, team); if (UCC_OK != status) { goto out; } } team->state = UCC_TEAM_ALLOC_ID; /* fall through */ case UCC_TEAM_ALLOC_ID: if (context->cl_flags & UCC_BASE_LIB_FLAG_TEAM_ID_REQUIRED) { status = ucc_team_alloc_id(team); if (UCC_OK != status) { goto out; } } team->bp.id = team->id; team->state = UCC_TEAM_CL_CREATE; if (team->service_team) { /* update service team id */ UCC_TL_TEAM_IFACE(team->service_team)->scoll.update_id (&team->service_team->super, team->id); } /* fall through */ case UCC_TEAM_CL_CREATE: status = ucc_team_create_cls(context, team); break; case UCC_TEAM_ACTIVE: return UCC_OK; } out: if (UCC_OK == status) { team->state = UCC_TEAM_ACTIVE; status = ucc_team_build_score_map(team); } if (UCC_OK == status && ucc_global_config.log_component.log_level >= UCC_LOG_LEVEL_INFO && team->rank == 0) { ucc_info("===== COLL_SCORE_MAP (team_id %d, size %u) =====", team->id, team->size); ucc_coll_score_map_print_info(team->score_map, ucc_global_config.log_component.log_level); ucc_info("================================================"); } /* TODO: add team/coll selection and check if some teams are never used after selection and clean them up */ return status; } ucc_status_t ucc_team_create_test(ucc_team_h team) { if (NULL == team) { ucc_error("ucc_team_create_test: invalid team handle: NULL"); return UCC_ERR_INVALID_PARAM; } /* we don't support multiple contexts per team yet */ ucc_assert(team->num_contexts == 1); if (team->state == UCC_TEAM_ACTIVE) { return UCC_OK; } return ucc_team_create_test_single(team->contexts[0], team); } static ucc_status_t ucc_team_destroy_single(ucc_team_h team) { ucc_cl_iface_t *cl_iface; int i; ucc_status_t status; if (team->service_team) { if (UCC_OK != (status = UCC_TL_CTX_IFACE(team->contexts[0]->service_ctx) ->team.destroy(&team->service_team->super))) { return status; } team->service_team = NULL; ucc_tl_context_put(team->contexts[0]->service_ctx); } for (i = 0; i < team->n_cl_teams; i++) { if (!team->cl_teams[i]) continue; cl_iface = UCC_CL_TEAM_IFACE(team->cl_teams[i]); if (UCC_OK != (status = cl_iface->team.destroy(&team->cl_teams[i]->super))) { return status; } team->cl_teams[i] = NULL; } ucc_topo_cleanup(team->topo); if (team->contexts[0]->service_team && team->size > 1) { ucc_internal_oob_finalize(&team->bp.params.oob); } if ((ucc_global_config.log_component.log_level >= UCC_LOG_LEVEL_INFO) && (team->rank == 0)) { ucc_info("team destroyed, team_id %d", team->id); } ucc_coll_score_free_map(team->score_map); ucc_free(team->addr_storage.storage); ucc_free(team->ctx_ranks); ucc_team_release_id(team); ucc_free(team->cl_teams); ucc_free(team->contexts); ucc_free(team); return UCC_OK; } ucc_status_t ucc_team_destroy(ucc_team_h team) { if (NULL == team) { ucc_error("ucc_team_destroy: invalid team handle: NULL"); return UCC_ERR_INVALID_PARAM; } if (team->state != UCC_TEAM_ACTIVE) { ucc_error("team %p is used before team_create is completed", team); return UCC_ERR_INVALID_PARAM; } /* we don't support multiple contexts per team yet */ ucc_assert(team->num_contexts == 1); return ucc_team_destroy_single(team); } static inline int find_first_set_and_zero(uint64_t *value) { int i; for (i=0; i<64; i++) { if (*value & ((uint64_t)1 << i)) { *value &= ~((uint64_t)1 << i); return i+1; } } return 0; } static inline void set_id_bit(uint64_t *local, int id) { int map_pos = id / 64; int pos = (id-1) % 64; ucc_assert(id >= 1); local[map_pos] |= ((uint64_t)1 << pos); } static ucc_status_t ucc_team_alloc_id(ucc_team_t *team) { /* at least 1 ctx is always available */ ucc_context_t *ctx = team->contexts[0]; uint64_t *local, *global; ucc_status_t status; int pos, i; if (team->id > 0) { ucc_assert(UCC_TEAM_ID_IS_EXTERNAL(team)); return UCC_OK; } if (!ctx->ids.pool) { ctx->ids.pool = ucc_malloc(ctx->ids.pool_size*2*sizeof(uint64_t), "ids_pool"); if (!ctx->ids.pool) { ucc_error("failed to allocate %zd bytes for team_ids_pool", ctx->ids.pool_size*2*sizeof(uint64_t)); return UCC_ERR_NO_MEMORY; } /* init all bits to 1 - all available */ memset(ctx->ids.pool, 255, ctx->ids.pool_size*2*sizeof(uint64_t)); } local = ctx->ids.pool; global = ctx->ids.pool + ctx->ids.pool_size; if (!team->sreq) { ucc_subset_t subset = {.map.type = UCC_EP_MAP_FULL, .map.ep_num = team->size, .myrank = team->rank}; status = ucc_service_allreduce(team, local, global, UCC_DT_UINT64, ctx->ids.pool_size, UCC_OP_BAND, subset, &team->sreq); if (status < 0) { return status; } } ucc_context_progress(ctx); status = ucc_service_coll_test(team->sreq); if (status < 0) { ucc_error("service allreduce test failure: %s", ucc_status_string(status)); return status; } else if (status != UCC_OK) { return status; } ucc_service_coll_finalize(team->sreq); team->sreq = NULL; memcpy(local, global, ctx->ids.pool_size*sizeof(uint64_t)); pos = 0; for (i=0; iids.pool_size; i++) { if ((pos = find_first_set_and_zero(&local[i])) > 0) { break; } } if (pos > 0) { ucc_assert(pos <= 64); team->id = (uint16_t)(i*64+pos); ucc_debug("allocated ID %d for team %p", team->id, team); } else { ucc_warn("could not allocate team id, whole id space is occupied, " "try increasing UCC_TEAM_IDS_POOL_SIZE"); return UCC_ERR_NO_RESOURCE; } ucc_assert(team->id > 0); return UCC_OK; } static void ucc_team_release_id(ucc_team_t *team) { ucc_context_t *ctx = team->contexts[0]; /* release the id pool bit if it was not provided by user */ if (0 != team->id && !UCC_TEAM_ID_IS_EXTERNAL(team)) { set_id_bit(ctx->ids.pool, team->id); } } ucc-1.8.0/src/core/ucc_ee.h0000664000175000017500000000155615211535620015600 0ustar alastairalastair/** * Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_EE_H_ #define UCC_EE_H_ #include "ucc/api/ucc.h" #include "utils/ucc_datastruct.h" #include "utils/ucc_queue.h" #include "utils/ucc_spinlock.h" extern const char *ucc_ee_ev_names[]; typedef struct ucc_ee { ucc_team_h team; ucc_ee_type_t ee_type; ucc_spinlock_t lock; ucc_queue_head_t event_in_queue; ucc_queue_head_t event_out_queue; size_t ee_context_size; char *ee_context; } ucc_ee_t; typedef struct ucc_event_desc { ucc_queue_elem_t queue; ucc_ev_t ev; } ucc_event_desc_t; ucc_status_t ucc_ee_get_event_internal(ucc_ee_h ee, ucc_ev_t **ev, ucc_queue_head_t *queue); ucc_status_t ucc_ee_set_event_internal(ucc_ee_h ee, ucc_ev_t *ev, ucc_queue_head_t *queue); #endif ucc-1.8.0/.azure/0000775000175000017500000000000015211535620013662 5ustar alastairalastairucc-1.8.0/.azure/azure-pipelines-pr.yml0000664000175000017500000000605115211535620020142 0ustar alastairalastair# See https://aka.ms/yaml # This pipeline to be run on PRs trigger: none pr: branches: include: - master - v*.*.x paths: exclude: - .gitignore - .readthedocs.yaml - contrib/pr_merge_check.py - docs/source - docs/CodeStyle.md - docs/LoggingStyle.md - docs/OptimizationStyle.md - README.md - NEWS - AUTHORS resources: containers: - container: fedora image: rdmz-harbor.rdmz.labs.mlnx/ucx/fedora33:1 stages: - stage: Codestyle jobs: # Check that the code is formatted according to the code style guidelines - job: format displayName: format code pool: name: MLNX demands: - ucx_docker -equals yes container: fedora steps: - checkout: self clean: true fetchDepth: 100 - bash: | set -x git log -1 HEAD git log -1 HEAD^ BASE_SOURCEVERSION=$(git rev-parse HEAD^) echo "Checking code format on diff ${BASE_SOURCEVERSION}..${BUILD_SOURCEVERSION}" git-clang-format --diff ${BASE_SOURCEVERSION} ${BUILD_SOURCEVERSION} > format.patch echo "Generated patch file:" cat format.patch if [ "`cat format.patch`" = "no modified files to format" ]; then exit fi git apply format.patch if ! git diff --quiet --exit-code then url="https://github.com/openucx/ucx/wiki/Code-style-checking" echo "##vso[task.complete result=SucceededWithIssues;]DONE'Code is not formatted according to the code style, see $url for more info.'" echo "##vso[task.logissue type=warning]'Code is not formatted'" fi - stage: Test jobs: - job: Gtest timeoutInMinutes: 120 workspace: clean: all pool: name: MLNX demands: - ucx_bf -equals yes steps: - checkout: self - bash: | set -eE git clone --depth 1 -b master https://github.com/openucx/ucx.git ucx cd ucx ./autogen.sh mkdir -p ucx_build_dir cd ucx_build_dir ../configure --without-java --prefix=$(Build.Repository.LocalPath)/install_ucx gcc -v make -s -j `nproc` make install displayName: Build ucx artifact timeoutInMinutes: 40 - bash: | set -xEe ./autogen.sh mkdir -p build cd build ../configure --with-ucx=$(Build.Repository.LocalPath)/install_ucx \ --prefix=$(Build.Repository.LocalPath)/install --enable-gtest make -j install displayName: Build ucc artifact timeoutInMinutes: 60 - bash: | cd build make gtest displayName: Launch Gtest timeoutInMinutes: 120 ucc-1.8.0/configure.ac0000664000175000017500000002370115211535620014747 0ustar alastairalastair# # Copyright (c) 2001-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # This software product is a proprietary product of Mellanox Technologies Ltd. # (the "Company") and all right, title, and interest and to the software product, # including all associated intellectual property rights, are and shall # remain exclusively with the Company. # # This software product is governed by the End User License Agreement # provided with the software product. # $COPYRIGHT$ # $HEADER$ AC_PREREQ([2.63]) define([ucc_ver_major], 1) define([ucc_ver_minor], 8) define([ucc_ver_patch], 0) define([ucc_ver_micro], esyscmd([sh -c "git rev-list HEAD | wc -l | sed -e 's/ *//g' | xargs -n1 printf"])) define([ucc_git_sha], esyscmd([sh -c "git rev-parse --short HEAD"])) # This is the API version (see libtool library versioning) # http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html # current:rev:age define([libucc_so_version], 1:0:0) AC_INIT([ucc], [ucc_ver_major.ucc_ver_minor]) : ${CPPFLAGS=""} : ${CFLAGS=""} : ${CXXFLAGS=""} config_flags="$*" AC_USE_SYSTEM_EXTENSIONS AC_CONFIG_HEADERS([config.h]) # # Save config flags for version dump tool # AC_DEFINE_UNQUOTED([UCC_CONFIGURE_FLAGS], ["$config_flags"], [UCC configure flags]) UCC_TOP_BUILDDIR="`pwd`" AC_SUBST(UCC_TOP_BUILDDIR) cd "$srcdir" UCC_TOP_SRCDIR="`pwd`" AC_SUBST(UCC_TOP_SRCDIR) cd "$UCC_TOP_BUILDDIR" AC_MSG_NOTICE([builddir: $UCC_TOP_BUILDDIR]) AC_MSG_NOTICE([srcdir: $UCC_TOP_SRCDIR]) AM_INIT_AUTOMAKE([1.10 foreign tar-ustar silent-rules subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AM_MAINTAINER_MODE AC_CONFIG_MACRO_DIR([m4]) define([ucc_prefix], [/opt/ucf/ucc]) AC_PREFIX_DEFAULT([ucc_prefix]) RPM_RELEASE=1 MAJOR_VERSION=ucc_ver_major MINOR_VERSION=ucc_ver_minor MICRO_VERSION=ucc_ver_micro PATCH_VERSION=ucc_ver_patch UCC_GIT_SHA=ucc_git_sha VERSION=$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION SOVERSION=libucc_so_version AC_SUBST(MAJOR_VERSION) AC_SUBST(MINOR_VERSION) AC_SUBST(MICRO_VERSION) AC_SUBST(PATCH_VERSION) AC_SUBST(RPM_RELEASE) AC_SUBST(SOVERSION) AC_SUBST(UCC_GIT_SHA) AC_MSG_NOTICE([git sha: $UCC_GIT_SHA]) CFLAGS_save="$CFLAGS" AC_PROG_CC AC_PROG_CXX AM_PROG_AS AC_PROG_LN_S AC_PROG_MKDIR_P AC_PROG_INSTALL LT_INIT CFLAGS="$CFLAGS_save" AC_CHECK_SIZEOF(float) AC_CHECK_SIZEOF(double) AC_CHECK_SIZEOF(long double) AC_CHECK_SIZEOF(float _Complex) AC_CHECK_SIZEOF(double _Complex) AC_CHECK_SIZEOF(long double _Complex) # # Check if 'ln' supports creating relative links # AC_MSG_CHECKING([if ${LN_S} supports --relative]) AS_IF([${LN_S} --relative symlinktest 2>/dev/null], [AC_MSG_RESULT([yes]) AC_SUBST([LN_RS], ["${LN_S} --relative"]) rm symlinktest], [AC_MSG_RESULT([no]) AC_SUBST([LN_RS], [${LN_S}])]) AC_SUBST([modulesubdir], [${PACKAGE_NAME}]) # module directory names AC_SUBST([moduledir], [${libdir}/${modulesubdir}]) # module installation directory AC_SUBST([localmoduledir], ['$(abs_top_builddir)/modules']) # local directory for module symlinks AC_SUBST([objdir], [${objdir}]) # libtool objects dir, usually .libs AC_SUBST([shrext], [${shrext_cmds}]) # libtool shared library extension AC_DEFINE_UNQUOTED([UCC_MODULE_SUBDIR], ["${modulesubdir}"], [UCC module sub-directory]) # # Additional m4 files # m4_include([config/m4/ax_prog_doxygen.m4]) AC_ARG_WITH([docs_only], AS_HELP_STRING([--with-docs-only], [Compile only documentation for UCC. [default=NO]]), ,[:],[with_docs_only=no]) AC_DEFUN([UCC_DX_ENABLE_CHECK], [AS_IF([DX_TEST_FEATURE($1)], [], [AS_IF([test "x$enable_doxygen_$1" == xyes], [AC_MSG_ERROR([--enable-doxygen-$1 was specified, but $1 doxygen was not found])], [])])]) # # Doxygen options # DX_PS_FEATURE(OFF) DX_HTML_FEATURE(ON) DX_MAN_FEATURE(ON) DX_PDF_FEATURE(ON) DX_INIT_DOXYGEN([UCC],[docs/doxygen/ucc_doxygen.conf],[docs/doxygen-doc]) AS_IF([test "x$with_docs_only" = xyes], [AS_MESSAGE([Documents only requested]) AS_IF([DX_TEST_FEATURE(doc)], [], [AC_MSG_ERROR([--with-only-docs was requested, but doxygen was not found])]) UCC_DX_ENABLE_CHECK([html]) UCC_DX_ENABLE_CHECK([man]) UCC_DX_ENABLE_CHECK([pdf]) AM_CONDITIONAL([DOCS_ONLY], [true]) AM_CONDITIONAL([HAVE_GTEST], [false]) AM_CONDITIONAL([HAVE_CXX11], [false]) AM_CONDITIONAL([HAVE_CXX17], [false]) AM_CONDITIONAL([HAVE_GNUXX11], [false]) AM_CONDITIONAL([HAVE_GLIBCXX_NOTHROW], [false]) AM_CONDITIONAL([HAVE_AARCH64_THUNDERX2], [false]) AM_CONDITIONAL([HAVE_AARCH64_THUNDERX1], [false]) AM_CONDITIONAL([HAVE_AARCH64_HI1620], [false]) AM_CONDITIONAL([HAVE_UCX], [false]) AM_CONDITIONAL([HAVE_CUDA], [false]) AM_CONDITIONAL([HAVE_ROCM], [false]) AM_CONDITIONAL([HAVE_HIP], [false]) AM_CONDITIONAL([HAVE_NVML], [false]) AM_CONDITIONAL([HAVE_MPI], [false]) AM_CONDITIONAL([HAVE_MPIRUN], [false]) AM_CONDITIONAL([HAVE_MPICC], [false]) AM_CONDITIONAL([HAVE_MPICXX], [false]) AM_CONDITIONAL([HAVE_PROFILING],[false]) AM_CONDITIONAL([HAVE_IBVERBS],[false]) AM_CONDITIONAL([HAVE_RDMACM],[false]) AM_CONDITIONAL([HAVE_MLX5DV],[false]) AM_CONDITIONAL([HAVE_DOCA_UROM], [false]) ], [ AM_CONDITIONAL([DOCS_ONLY], [false]) m4_include([config/m4/compiler.m4]) m4_include([config/m4/gtest.m4]) m4_include([config/m4/sysdep.m4]) m4_include([config/m4/ucx.m4]) m4_include([config/m4/cuda.m4]) m4_include([config/m4/nvls.m4]) m4_include([config/m4/nccl.m4]) m4_include([config/m4/rocm.m4]) m4_include([config/m4/doca_urom.m4]) m4_include([config/m4/rccl.m4]) m4_include([config/m4/sharp.m4]) m4_include([config/m4/mpi.m4]) m4_include([config/m4/ibverbs.m4]) m4_include([config/m4/rdmacm.m4]) m4_include([config/m4/configure.m4]) m4_include([config/m4/tl_coll_plugins.m4]) m4_include([config/m4/check_tls.m4]) mc_modules=":cpu" tl_modules="" tlcp_modules="" AC_MSG_RESULT([MPI perftest: ${mpi_enable}]) CHECK_UCX AC_MSG_RESULT([UCX support: $ucx_happy]) if test $ucx_happy != "yes"; then AC_MSG_ERROR([UCX is not available]) fi CHECK_CUDA AC_MSG_RESULT([CUDA support: $cuda_happy; $CUDA_CPPFLAGS $CUDA_LDFLAGS]) if test $cuda_happy = "yes"; then mc_modules="${mc_modules}:cuda" fi CHECK_NVLS AC_MSG_RESULT([NVLS support: $nvls_happy]) CHECK_ROCM AC_MSG_RESULT([ROCM support: $rocm_happy; $ROCM_CPPFLAGS $ROCM_LDFLAGS]) AC_MSG_RESULT([HIP support: $hip_happy; $HIP_CPPFLAGS $HIP_LDFLAGS]) if test $rocm_happy = "yes"; then mc_modules="${mc_modules}:rocm" fi CHECK_DOCA_UROM AC_MSG_RESULT([DOCA_UROM support: $doca_urom_happy]) CHECK_GTEST AC_MSG_RESULT([GTEST support: $gtest_happy]) CHECK_IBVERBS AC_MSG_RESULT([IBVERBS support: $ibverbs_happy, MLX5DV support: $mlx5dv_happy]) CHECK_RDMACM AC_MSG_RESULT([RDMACM support: $rdmacm_happy]) ]) # Docs only CHECK_TLS CFLAGS="$CFLAGS -std=gnu11" CPPFLAGS="$CPPFLAGS $UCS_CPPFLAGS $includes" LDFLAGS="$LDFLAGS $UCS_LDFLAGS $UCS_LIBADD" CHECK_TL_COLL_PLUGINS AC_CONFIG_FILES([ Makefile contrib/Makefile contrib/doca_urom_ucc_plugin/Makefile src/Makefile src/ucc/api/ucc_version.h src/core/ucc_version.c src/components/cl/basic/Makefile src/components/cl/hier/Makefile src/components/cl/doca_urom/Makefile src/components/mc/cpu/Makefile src/components/mc/cuda/Makefile src/components/topo/cuda/Makefile src/components/topo/ib/Makefile src/components/ec/cpu/Makefile src/components/ec/cuda/Makefile src/components/ec/cuda/kernel/Makefile src/components/mc/rocm/Makefile src/components/ec/rocm/Makefile src/components/ec/rocm/kernel/Makefile src/components/tl/cuda/kernels/Makefile test/mpi/Makefile test/gtest/Makefile tools/info/Makefile tools/perf/Makefile cmake/Makefile cmake/ucc-config-version.cmake cmake/ucc-config.cmake cmake/ucc-targets.cmake ucc.pc ]) AC_OUTPUT # # Print build condiguration # AC_MSG_NOTICE([=========================================================]) AS_IF([test "x$with_docs_only" = xyes], [ AC_MSG_NOTICE([Building documents only]) ], [ AC_MSG_NOTICE([UCC build configuration:]) AC_MSG_NOTICE([ Build prefix: ${prefix}]) AC_MSG_NOTICE([Preprocessor flags: ${CPPFLAGS} ${BASE_CPPFLAGS}]) AC_MSG_NOTICE([ C compiler: ${CC} ${CFLAGS} ${BASE_CFLAGS}]) AC_MSG_NOTICE([ C++ compiler: ${CXX} ${CXXFLAGS} ${BASE_CXXFLAGS}]) AS_IF([test "x$cuda_happy" = "xyes"],[ AC_MSG_NOTICE([ NVCC gencodes: ${NVCC_ARCH}]) ]) AC_MSG_NOTICE([ DOCA UROM enabled: ${doca_urom_happy}]) AS_IF([test "x$nvls_happy" = "xyes"],[ AC_MSG_NOTICE([NVLS (NVLINK SHARP): enabled]) AC_MSG_NOTICE([NVCC NVLS gencodes: ${NVCC_ARCH_NVLS}]) ]) AS_IF([test "x$rocm_happy" = xyes],[ AC_MSG_NOTICE([ROCM architectures: ${ROCM_ARCH}]) ]) AC_MSG_NOTICE([ Perftest: ${mpi_enable}]) AC_MSG_NOTICE([ Gtest: ${gtest_enable}]) AC_MSG_NOTICE([ MC modules: <$(echo ${mc_modules}|tr ':' ' ') >]) AC_MSG_NOTICE([ TL modules: <$(echo ${tl_modules}|tr ':' ' ') >]) AC_MSG_NOTICE([ TLCP modules: <$(echo ${tlcp_modules}|tr ':' ' ') >]) AS_IF([test "x$enable_profiling" = xyes],[ AC_MSG_NOTICE([ Profiling modules: <$(echo ${prof_modules}|tr ':' ' ') >]) ]) ]) AC_MSG_NOTICE([=========================================================]) ucc-1.8.0/test/0000775000175000017500000000000015211535620013435 5ustar alastairalastairucc-1.8.0/test/gtest/0000775000175000017500000000000015211535620014563 5ustar alastairalastairucc-1.8.0/test/gtest/common/0000775000175000017500000000000015211535620016053 5ustar alastairalastairucc-1.8.0/test/gtest/common/test_ucc.h0000664000175000017500000002173315211535620020043 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef TEST_UCC_H #define TEST_UCC_H #include "test.h" extern "C" { #include "components/mc/ucc_mc.h" #include "utils/ucc_malloc.h" #include #include } #include #include #include #include #include #include #include typedef struct { ucc_mc_buffer_header_t *dst_mc_header; ucc_mc_buffer_header_t *src_mc_header; void *init_buf; size_t rbuf_size; ucc_coll_args_t *args; } gtest_ucc_coll_ctx_t; typedef std::vector UccCollCtxVec; typedef enum { TEST_NO_INPLACE, TEST_INPLACE } gtest_ucc_inplace_t; class UccCollArgs { protected: ucc_memory_type_t mem_type; gtest_ucc_inplace_t inplace; bool is_contig; void alltoallx_init_buf(int src_rank, int dst_rank, uint8_t *buf, size_t len) { for (int i = 0; i < len; i++) { buf[i] = (uint8_t)(((src_rank + len - i) * (dst_rank + 1)) % UINT8_MAX); } } int alltoallx_validate_buf(int src_rank, int dst_rank, uint8_t *buf, size_t len) { int err = 0; for (int i = 0; i < len; i ++) { uint8_t expected = (uint8_t) (((dst_rank + len - i) * (src_rank + 1)) % UINT8_MAX); if (buf[i] != expected) { err++; } } return err; } public: UccCollArgs() { // defaults mem_type = UCC_MEMORY_TYPE_HOST; inplace = TEST_NO_INPLACE; } virtual ~UccCollArgs() {} virtual void data_init(int nprocs, ucc_datatype_t dtype, size_t count, UccCollCtxVec &args, bool persistent = false) = 0; virtual void data_fini(UccCollCtxVec args) = 0; virtual bool data_validate(UccCollCtxVec args) = 0; void set_mem_type(ucc_memory_type_t _mt); void set_inplace(gtest_ucc_inplace_t _inplace); void set_contig(bool _contig); }; #define SET_MEM_TYPE(_mt) do { \ if (UCC_OK != ucc_mc_available(_mt)) { \ GTEST_SKIP(); \ } \ this->mem_type = _mt; \ } while (0) class ThreadAllgather; class ThreadAllgatherReq { public: ThreadAllgather *ta; int rank; ucc_status_t status; std::thread t; ThreadAllgatherReq(ThreadAllgather *_ta, int _rank) : ta(_ta), rank(_rank) { status = UCC_OPERATION_INITIALIZED; }; }; class ThreadAllgather { public: int n_procs; std::atomic ready_count; void *buffer; std::mutex lock; std::vector reqs; ThreadAllgather(int _n_procs) : n_procs(_n_procs), ready_count(0), buffer(NULL) { for (auto i = 0; i < _n_procs; i++) { reqs.push_back(ThreadAllgatherReq(this, i)); } }; ~ThreadAllgather() { buffer = NULL; ready_count = 0; } }; /* A single processes in a Job that runs UCC. It has context and lib object */ class UccProcess { public: ucc_context_params_t ctx_params; static constexpr ucc_lib_params_t default_lib_params = { .mask = UCC_LIB_PARAM_FIELD_THREAD_MODE | UCC_LIB_PARAM_FIELD_COLL_TYPES, .thread_mode = UCC_THREAD_SINGLE, .coll_types = UCC_COLL_TYPE_BARRIER | UCC_COLL_TYPE_ALLTOALL | UCC_COLL_TYPE_ALLTOALLV | UCC_COLL_TYPE_ALLREDUCE | UCC_COLL_TYPE_ALLGATHER | UCC_COLL_TYPE_ALLGATHERV | UCC_COLL_TYPE_REDUCE | UCC_COLL_TYPE_BCAST | UCC_COLL_TYPE_GATHER | UCC_COLL_TYPE_SCATTER}; static constexpr ucc_context_params_t default_ctx_params = { .mask = UCC_CONTEXT_PARAM_FIELD_TYPE, .type = UCC_CONTEXT_EXCLUSIVE }; ucc_lib_h lib_h; ucc_context_h ctx_h; void * onesided_buf[3]; int job_rank; UccProcess(int _job_rank, const ucc_lib_params_t &lp = default_lib_params, const ucc_context_params_t &cp = default_ctx_params); ~UccProcess(); }; typedef std::shared_ptr UccProcess_h; /* Ucc team that consists of several processes. The team is created from UccJob environment */ class UccTeam { struct proc { UccProcess_h p; ucc_team_h team; proc(){}; proc(UccProcess_h _p) : p(_p) {}; }; typedef enum { AG_INIT, AG_READY, AG_COPY_DONE, AG_COMPLETE } allgather_phase_t; struct allgather_data { void *sbuf; void *rbuf; size_t len; allgather_phase_t phase; }; typedef struct allgather_coll_info { int my_rank; UccTeam *self; } allgather_coll_info_t; std::vector ag; void init_team(bool use_team_ep_map, bool use_ep_range, bool is_onesided); void destroy_team(); void test_allgather(size_t msglen); static ucc_status_t allgather(void *src_buf, void *recv_buf, size_t size, void *coll_info, void **request); static ucc_status_t req_test(void *request); static ucc_status_t req_free(void *request); int copy_complete_count; public: int n_procs; void progress(); std::vector procs; UccTeam(std::vector &_procs, bool use_team_ep_map = false, bool use_ep_range = true, bool is_onesided = false); ~UccTeam(); }; typedef std::shared_ptr UccTeam_h; typedef std::pair ucc_env_var_t; typedef std::vector ucc_job_env_t; /* UccJob - environent that has n_procs processes. Multiple UccTeams can be created from UccJob */ class UccJob { static UccJob* staticUccJob; static std::vector staticTeams; ThreadAllgather ta; public: typedef enum { UCC_JOB_CTX_LOCAL, UCC_JOB_CTX_GLOBAL, /*< ucc ctx create with OOB */ UCC_JOB_CTX_GLOBAL_ONESIDED } ucc_job_ctx_mode_t; static const int nStaticTeams = 5; static const int staticUccJobSize = 16; static constexpr int staticTeamSizes[nStaticTeams] = {1, 2, 8, 11, staticUccJobSize}; static void cleanup(); static UccJob* getStaticJob(); static const std::vector &getStaticTeams(); int n_procs; UccJob(int _n_procs = 2, ucc_job_ctx_mode_t _ctx_mode = UCC_JOB_CTX_GLOBAL, ucc_job_env_t vars = ucc_job_env_t()); ~UccJob(); std::vector procs; UccTeam_h create_team(int n_procs, bool use_team_ep_map = false, bool use_ep_range = true, bool is_onesided = false); UccTeam_h create_team(std::vector &ranks, bool use_team_ep_map = false, bool use_ep_range = true, bool is_onesided = false); void create_context(); ucc_job_ctx_mode_t ctx_mode; }; class UccReq { UccTeam_h team; /* Make copy constructor and = private, to avoid req leak */ public: ucc_status_t status; UccReq(const UccReq&) = delete; UccReq& operator=(const UccReq&) = delete; UccReq(UccReq&& source) : team(source.team), status(source.status) { reqs.swap(source.reqs); }; std::vector reqs; UccReq(UccTeam_h _team, ucc_coll_args_t *args); UccReq(UccTeam_h _team, UccCollCtxVec args); ~UccReq(); void start(void); ucc_status_t wait(); ucc_status_t test(void); static void waitall(std::vector &reqs); static void startall(std::vector &reqs); }; #define DATA_FINI_ALL(_test, _ctx) for (auto &c : ctxs) { _test->data_fini(c); } #define CHECK_REQ_NOT_SUPPORTED_SKIP(_UccReq, _action) do{ \ if ((_UccReq).status == UCC_ERR_NOT_SUPPORTED) { \ _action; \ GTEST_SKIP(); \ } \ ASSERT_EQ(UCC_OK, (_UccReq).status); \ } while(0) void clear_buffer(void *_buf, size_t size, ucc_memory_type_t mt, uint8_t value); #define PREDEFINED_DTYPES \ ::testing::Values( \ UCC_DT_INT8, UCC_DT_INT16, UCC_DT_INT32, UCC_DT_INT64, UCC_DT_INT128, \ UCC_DT_UINT8, UCC_DT_UINT16, UCC_DT_UINT32, UCC_DT_UINT64, \ UCC_DT_UINT128, UCC_DT_FLOAT16, UCC_DT_FLOAT32, UCC_DT_FLOAT64, \ UCC_DT_BFLOAT16, UCC_DT_FLOAT128, UCC_DT_FLOAT32_COMPLEX, \ UCC_DT_FLOAT64_COMPLEX, UCC_DT_FLOAT128_COMPLEX) #define UCC_TEST_N_MEM_SEGMENTS 3 #define UCC_TEST_MEM_SEGMENT_SIZE (1 << 20) bool tl_self_available(); #endif ucc-1.8.0/test/gtest/common/gtest.h0000664000175000017500000231324115211535620017360 0ustar alastairalastair// Copyright 2005, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // The Google C++ Testing and Mocking Framework (Google Test) // // This header file defines the public API for Google Test. It should be // included by any test program that uses Google Test. // // IMPORTANT NOTE: Due to limitation of the C++ language, we have to // leave some internal implementation details in this header file. // They are clearly marked by comments like this: // // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. // // Such code is NOT meant to be used by a user directly, and is subject // to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user // program! // // Acknowledgment: Google Test borrowed the idea of automatic test // registration from Barthelemy Dagenais' (barthelemy@prologique.com) // easyUnit framework. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_GTEST_H_ #define GTEST_INCLUDE_GTEST_GTEST_H_ #include #include #include #include #include #include // Copyright 2005, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // The Google C++ Testing and Mocking Framework (Google Test) // // This header file declares functions and macros used internally by // Google Test. They are subject to change without notice. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ // Copyright 2005, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Low-level types and utilities for porting Google Test to various // platforms. All macros ending with _ and symbols defined in an // internal namespace are subject to change without notice. Code // outside Google Test MUST NOT USE THEM DIRECTLY. Macros that don't // end with _ are part of Google Test's public API and can be used by // code outside Google Test. // // This file is fundamental to Google Test. All other Google Test source // files are expected to #include this. Therefore, it cannot #include // any other Google Test header. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ // Environment-describing macros // ----------------------------- // // Google Test can be used in many different environments. Macros in // this section tell Google Test what kind of environment it is being // used in, such that Google Test can provide environment-specific // features and implementations. // // Google Test tries to automatically detect the properties of its // environment, so users usually don't need to worry about these // macros. However, the automatic detection is not perfect. // Sometimes it's necessary for a user to define some of the following // macros in the build script to override Google Test's decisions. // // If the user doesn't define a macro in the list, Google Test will // provide a default definition. After this header is #included, all // macros in this list will be defined to either 1 or 0. // // Notes to maintainers: // - Each macro here is a user-tweakable knob; do not grow the list // lightly. // - Use #if to key off these macros. Don't use #ifdef or "#if // defined(...)", which will not work as these macros are ALWAYS // defined. // // GTEST_HAS_CLONE - Define it to 1/0 to indicate that clone(2) // is/isn't available. // GTEST_HAS_EXCEPTIONS - Define it to 1/0 to indicate that exceptions // are enabled. // GTEST_HAS_POSIX_RE - Define it to 1/0 to indicate that POSIX regular // expressions are/aren't available. // GTEST_HAS_PTHREAD - Define it to 1/0 to indicate that // is/isn't available. // GTEST_HAS_RTTI - Define it to 1/0 to indicate that RTTI is/isn't // enabled. // GTEST_HAS_STD_WSTRING - Define it to 1/0 to indicate that // std::wstring does/doesn't work (Google Test can // be used where std::wstring is unavailable). // GTEST_HAS_SEH - Define it to 1/0 to indicate whether the // compiler supports Microsoft's "Structured // Exception Handling". // GTEST_HAS_STREAM_REDIRECTION // - Define it to 1/0 to indicate whether the // platform supports I/O stream redirection using // dup() and dup2(). // GTEST_LINKED_AS_SHARED_LIBRARY // - Define to 1 when compiling tests that use // Google Test as a shared library (known as // DLL on Windows). // GTEST_CREATE_SHARED_LIBRARY // - Define to 1 when compiling Google Test itself // as a shared library. // GTEST_DEFAULT_DEATH_TEST_STYLE // - The default value of --gtest_death_test_style. // The legacy default has been "fast" in the open // source version since 2008. The recommended value // is "threadsafe", and can be set in // custom/gtest-port.h. // Platform-indicating macros // -------------------------- // // Macros indicating the platform on which Google Test is being used // (a macro is defined to 1 if compiled on the given platform; // otherwise UNDEFINED -- it's never defined to 0.). Google Test // defines these macros automatically. Code outside Google Test MUST // NOT define them. // // GTEST_OS_AIX - IBM AIX // GTEST_OS_CYGWIN - Cygwin // GTEST_OS_DRAGONFLY - DragonFlyBSD // GTEST_OS_FREEBSD - FreeBSD // GTEST_OS_FUCHSIA - Fuchsia // GTEST_OS_GNU_KFREEBSD - GNU/kFreeBSD // GTEST_OS_HAIKU - Haiku // GTEST_OS_HPUX - HP-UX // GTEST_OS_LINUX - Linux // GTEST_OS_LINUX_ANDROID - Google Android // GTEST_OS_MAC - Mac OS X // GTEST_OS_IOS - iOS // GTEST_OS_NACL - Google Native Client (NaCl) // GTEST_OS_NETBSD - NetBSD // GTEST_OS_OPENBSD - OpenBSD // GTEST_OS_OS2 - OS/2 // GTEST_OS_QNX - QNX // GTEST_OS_SOLARIS - Sun Solaris // GTEST_OS_WINDOWS - Windows (Desktop, MinGW, or Mobile) // GTEST_OS_WINDOWS_DESKTOP - Windows Desktop // GTEST_OS_WINDOWS_MINGW - MinGW // GTEST_OS_WINDOWS_MOBILE - Windows Mobile // GTEST_OS_WINDOWS_PHONE - Windows Phone // GTEST_OS_WINDOWS_RT - Windows Store App/WinRT // GTEST_OS_ZOS - z/OS // // Among the platforms, Cygwin, Linux, Mac OS X, and Windows have the // most stable support. Since core members of the Google Test project // don't have access to other platforms, support for them may be less // stable. If you notice any problems on your platform, please notify // googletestframework@googlegroups.com (patches for fixing them are // even more welcome!). // // It is possible that none of the GTEST_OS_* macros are defined. // Feature-indicating macros // ------------------------- // // Macros indicating which Google Test features are available (a macro // is defined to 1 if the corresponding feature is supported; // otherwise UNDEFINED -- it's never defined to 0.). Google Test // defines these macros automatically. Code outside Google Test MUST // NOT define them. // // These macros are public so that portable tests can be written. // Such tests typically surround code using a feature with an #if // which controls that code. For example: // // #if GTEST_HAS_DEATH_TEST // EXPECT_DEATH(DoSomethingDeadly()); // #endif // // GTEST_HAS_DEATH_TEST - death tests // GTEST_HAS_TYPED_TEST - typed tests // GTEST_HAS_TYPED_TEST_P - type-parameterized tests // GTEST_IS_THREADSAFE - Google Test is thread-safe. // GOOGLETEST_CM0007 DO NOT DELETE // GTEST_USES_POSIX_RE - enhanced POSIX regex is used. Do not confuse with // GTEST_HAS_POSIX_RE (see above) which users can // define themselves. // GTEST_USES_SIMPLE_RE - our own simple regex is used; // the above RE\b(s) are mutually exclusive. // Misc public macros // ------------------ // // GTEST_FLAG(flag_name) - references the variable corresponding to // the given Google Test flag. // Internal utilities // ------------------ // // The following macros and utilities are for Google Test's INTERNAL // use only. Code outside Google Test MUST NOT USE THEM DIRECTLY. // // Macros for basic C++ coding: // GTEST_AMBIGUOUS_ELSE_BLOCKER_ - for disabling a gcc warning. // GTEST_ATTRIBUTE_UNUSED_ - declares that a class' instances or a // variable don't have to be used. // GTEST_DISALLOW_ASSIGN_ - disables operator=. // GTEST_DISALLOW_COPY_AND_ASSIGN_ - disables copy ctor and operator=. // GTEST_MUST_USE_RESULT_ - declares that a function's result must be used. // GTEST_INTENTIONAL_CONST_COND_PUSH_ - start code section where MSVC C4127 is // suppressed (constant conditional). // GTEST_INTENTIONAL_CONST_COND_POP_ - finish code section where MSVC C4127 // is suppressed. // // Synchronization: // Mutex, MutexLock, ThreadLocal, GetThreadCount() // - synchronization primitives. // // Regular expressions: // RE - a simple regular expression class using the POSIX // Extended Regular Expression syntax on UNIX-like platforms // GOOGLETEST_CM0008 DO NOT DELETE // or a reduced regular exception syntax on other // platforms, including Windows. // Logging: // GTEST_LOG_() - logs messages at the specified severity level. // LogToStderr() - directs all log messages to stderr. // FlushInfoLog() - flushes informational log messages. // // Stdout and stderr capturing: // CaptureStdout() - starts capturing stdout. // GetCapturedStdout() - stops capturing stdout and returns the captured // string. // CaptureStderr() - starts capturing stderr. // GetCapturedStderr() - stops capturing stderr and returns the captured // string. // // Integer types: // TypeWithSize - maps an integer to a int type. // Int32, UInt32, Int64, UInt64, TimeInMillis // - integers of known sizes. // BiggestInt - the biggest signed integer type. // // Command-line utilities: // GTEST_DECLARE_*() - declares a flag. // GTEST_DEFINE_*() - defines a flag. // GetInjectableArgvs() - returns the command line as a vector of strings. // // Environment variable utilities: // GetEnv() - gets the value of an environment variable. // BoolFromGTestEnv() - parses a bool environment variable. // Int32FromGTestEnv() - parses an Int32 environment variable. // StringFromGTestEnv() - parses a string environment variable. // // Deprecation warnings: // GTEST_INTERNAL_DEPRECATED(message) - attribute marking a function as // deprecated; calling a marked function // should generate a compiler warning #include // for isspace, etc #include // for ptrdiff_t #include #include #include #include #include #ifndef _WIN32_WCE # include # include #endif // !_WIN32_WCE #if defined __APPLE__ # include # include #endif #include // NOLINT #include // NOLINT #include // NOLINT #include // NOLINT #include #include #include // NOLINT // Copyright 2015, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // The Google C++ Testing and Mocking Framework (Google Test) // // This header file defines the GTEST_OS_* macro. // It is separate from gtest-port.h so that custom/gtest-port.h can include it. #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ // Determines the platform on which Google Test is compiled. #ifdef __CYGWIN__ # define GTEST_OS_CYGWIN 1 # elif defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__) # define GTEST_OS_WINDOWS_MINGW 1 # define GTEST_OS_WINDOWS 1 #elif defined _WIN32 # define GTEST_OS_WINDOWS 1 # ifdef _WIN32_WCE # define GTEST_OS_WINDOWS_MOBILE 1 # elif defined(WINAPI_FAMILY) # include # if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) # define GTEST_OS_WINDOWS_DESKTOP 1 # elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PHONE_APP) # define GTEST_OS_WINDOWS_PHONE 1 # elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) # define GTEST_OS_WINDOWS_RT 1 # elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_TV_TITLE) # define GTEST_OS_WINDOWS_PHONE 1 # define GTEST_OS_WINDOWS_TV_TITLE 1 # else // WINAPI_FAMILY defined but no known partition matched. // Default to desktop. # define GTEST_OS_WINDOWS_DESKTOP 1 # endif # else # define GTEST_OS_WINDOWS_DESKTOP 1 # endif // _WIN32_WCE #elif defined __OS2__ # define GTEST_OS_OS2 1 #elif defined __APPLE__ # define GTEST_OS_MAC 1 # if TARGET_OS_IPHONE # define GTEST_OS_IOS 1 # endif #elif defined __DragonFly__ # define GTEST_OS_DRAGONFLY 1 #elif defined __FreeBSD__ # define GTEST_OS_FREEBSD 1 #elif defined __Fuchsia__ # define GTEST_OS_FUCHSIA 1 #elif defined(__GLIBC__) && defined(__FreeBSD_kernel__) # define GTEST_OS_GNU_KFREEBSD 1 #elif defined __linux__ # define GTEST_OS_LINUX 1 # if defined __ANDROID__ # define GTEST_OS_LINUX_ANDROID 1 # endif #elif defined __MVS__ # define GTEST_OS_ZOS 1 #elif defined(__sun) && defined(__SVR4) # define GTEST_OS_SOLARIS 1 #elif defined(_AIX) # define GTEST_OS_AIX 1 #elif defined(__hpux) # define GTEST_OS_HPUX 1 #elif defined __native_client__ # define GTEST_OS_NACL 1 #elif defined __NetBSD__ # define GTEST_OS_NETBSD 1 #elif defined __OpenBSD__ # define GTEST_OS_OPENBSD 1 #elif defined __QNX__ # define GTEST_OS_QNX 1 #elif defined(__HAIKU__) #define GTEST_OS_HAIKU 1 #endif // __CYGWIN__ #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ // Copyright 2015, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Injection point for custom user configurations. See README for details // // ** Custom implementation starts here ** #ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ #endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ #if !defined(GTEST_DEV_EMAIL_) # define GTEST_DEV_EMAIL_ "googletestframework@@googlegroups.com" # define GTEST_FLAG_PREFIX_ "gtest_" # define GTEST_FLAG_PREFIX_DASH_ "gtest-" # define GTEST_FLAG_PREFIX_UPPER_ "GTEST_" # define GTEST_NAME_ "Google Test" # define GTEST_PROJECT_URL_ "https://github.com/google/googletest/" #endif // !defined(GTEST_DEV_EMAIL_) #if !defined(GTEST_INIT_GOOGLE_TEST_NAME_) # define GTEST_INIT_GOOGLE_TEST_NAME_ "testing::InitGoogleTest" #endif // !defined(GTEST_INIT_GOOGLE_TEST_NAME_) // Determines the version of gcc that is used to compile this. #ifdef __GNUC__ // 40302 means version 4.3.2. # define GTEST_GCC_VER_ \ (__GNUC__*10000 + __GNUC_MINOR__*100 + __GNUC_PATCHLEVEL__) #endif // __GNUC__ // Macros for disabling Microsoft Visual C++ warnings. // // GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 4385) // /* code that triggers warnings C4800 and C4385 */ // GTEST_DISABLE_MSC_WARNINGS_POP_() #if defined(_MSC_VER) # define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) \ __pragma(warning(push)) \ __pragma(warning(disable: warnings)) # define GTEST_DISABLE_MSC_WARNINGS_POP_() \ __pragma(warning(pop)) #else // Not all compilers are MSVC # define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) # define GTEST_DISABLE_MSC_WARNINGS_POP_() #endif // Clang on Windows does not understand MSVC's pragma warning. // We need clang-specific way to disable function deprecation warning. #ifdef __clang__ # define GTEST_DISABLE_MSC_DEPRECATED_PUSH_() \ _Pragma("clang diagnostic push") \ _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") \ _Pragma("clang diagnostic ignored \"-Wdeprecated-implementations\"") #define GTEST_DISABLE_MSC_DEPRECATED_POP_() \ _Pragma("clang diagnostic pop") #else # define GTEST_DISABLE_MSC_DEPRECATED_PUSH_() \ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996) # define GTEST_DISABLE_MSC_DEPRECATED_POP_() \ GTEST_DISABLE_MSC_WARNINGS_POP_() #endif // Brings in definitions for functions used in the testing::internal::posix // namespace (read, write, close, chdir, isatty, stat). We do not currently // use them on Windows Mobile. #if GTEST_OS_WINDOWS # if !GTEST_OS_WINDOWS_MOBILE # include # include # endif // In order to avoid having to include , use forward declaration #if GTEST_OS_WINDOWS_MINGW && !defined(__MINGW64_VERSION_MAJOR) // MinGW defined _CRITICAL_SECTION and _RTL_CRITICAL_SECTION as two // separate (equivalent) structs, instead of using typedef typedef struct _CRITICAL_SECTION GTEST_CRITICAL_SECTION; #else // Assume CRITICAL_SECTION is a typedef of _RTL_CRITICAL_SECTION. // This assumption is verified by // WindowsTypesTest.CRITICAL_SECTIONIs_RTL_CRITICAL_SECTION. typedef struct _RTL_CRITICAL_SECTION GTEST_CRITICAL_SECTION; #endif #else // This assumes that non-Windows OSes provide unistd.h. For OSes where this // is not the case, we need to include headers that provide the functions // mentioned above. # include # include #endif // GTEST_OS_WINDOWS #if GTEST_OS_LINUX_ANDROID // Used to define __ANDROID_API__ matching the target NDK API level. # include // NOLINT #endif // Defines this to true if and only if Google Test can use POSIX regular // expressions. #ifndef GTEST_HAS_POSIX_RE # if GTEST_OS_LINUX_ANDROID // On Android, is only available starting with Gingerbread. # define GTEST_HAS_POSIX_RE (__ANDROID_API__ >= 9) # else # define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS) # endif #endif #if GTEST_USES_PCRE // The appropriate headers have already been included. #elif GTEST_HAS_POSIX_RE // On some platforms, needs someone to define size_t, and // won't compile otherwise. We can #include it here as we already // included , which is guaranteed to define size_t through // . # include // NOLINT # define GTEST_USES_POSIX_RE 1 #elif GTEST_OS_WINDOWS // is not available on Windows. Use our own simple regex // implementation instead. # define GTEST_USES_SIMPLE_RE 1 #else // may not be available on this platform. Use our own // simple regex implementation instead. # define GTEST_USES_SIMPLE_RE 1 #endif // GTEST_USES_PCRE #ifndef GTEST_HAS_EXCEPTIONS // The user didn't tell us whether exceptions are enabled, so we need // to figure it out. # if defined(_MSC_VER) && defined(_CPPUNWIND) // MSVC defines _CPPUNWIND to 1 if and only if exceptions are enabled. # define GTEST_HAS_EXCEPTIONS 1 # elif defined(__BORLANDC__) // C++Builder's implementation of the STL uses the _HAS_EXCEPTIONS // macro to enable exceptions, so we'll do the same. // Assumes that exceptions are enabled by default. # ifndef _HAS_EXCEPTIONS # define _HAS_EXCEPTIONS 1 # endif // _HAS_EXCEPTIONS # define GTEST_HAS_EXCEPTIONS _HAS_EXCEPTIONS # elif defined(__clang__) // clang defines __EXCEPTIONS if and only if exceptions are enabled before clang // 220714, but if and only if cleanups are enabled after that. In Obj-C++ files, // there can be cleanups for ObjC exceptions which also need cleanups, even if // C++ exceptions are disabled. clang has __has_feature(cxx_exceptions) which // checks for C++ exceptions starting at clang r206352, but which checked for // cleanups prior to that. To reliably check for C++ exception availability with // clang, check for // __EXCEPTIONS && __has_feature(cxx_exceptions). # define GTEST_HAS_EXCEPTIONS (__EXCEPTIONS && __has_feature(cxx_exceptions)) # elif defined(__GNUC__) && __EXCEPTIONS // gcc defines __EXCEPTIONS to 1 if and only if exceptions are enabled. # define GTEST_HAS_EXCEPTIONS 1 # elif defined(__SUNPRO_CC) // Sun Pro CC supports exceptions. However, there is no compile-time way of // detecting whether they are enabled or not. Therefore, we assume that // they are enabled unless the user tells us otherwise. # define GTEST_HAS_EXCEPTIONS 1 # elif defined(__IBMCPP__) && __EXCEPTIONS // xlC defines __EXCEPTIONS to 1 if and only if exceptions are enabled. # define GTEST_HAS_EXCEPTIONS 1 # elif defined(__HP_aCC) // Exception handling is in effect by default in HP aCC compiler. It has to // be turned of by +noeh compiler option if desired. # define GTEST_HAS_EXCEPTIONS 1 # else // For other compilers, we assume exceptions are disabled to be // conservative. # define GTEST_HAS_EXCEPTIONS 0 # endif // defined(_MSC_VER) || defined(__BORLANDC__) #endif // GTEST_HAS_EXCEPTIONS #if !defined(GTEST_HAS_STD_STRING) // Even though we don't use this macro any longer, we keep it in case // some clients still depend on it. # define GTEST_HAS_STD_STRING 1 #elif !GTEST_HAS_STD_STRING // The user told us that ::std::string isn't available. # error "::std::string isn't available." #endif // !defined(GTEST_HAS_STD_STRING) #ifndef GTEST_HAS_STD_WSTRING // The user didn't tell us whether ::std::wstring is available, so we need // to figure it out. // Cygwin 1.7 and below doesn't support ::std::wstring. // Solaris' libc++ doesn't support it either. Android has // no support for it at least as recent as Froyo (2.2). #define GTEST_HAS_STD_WSTRING \ (!(GTEST_OS_LINUX_ANDROID || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \ GTEST_OS_HAIKU)) #endif // GTEST_HAS_STD_WSTRING // Determines whether RTTI is available. #ifndef GTEST_HAS_RTTI // The user didn't tell us whether RTTI is enabled, so we need to // figure it out. # ifdef _MSC_VER #ifdef _CPPRTTI // MSVC defines this macro if and only if RTTI is enabled. # define GTEST_HAS_RTTI 1 # else # define GTEST_HAS_RTTI 0 # endif // Starting with version 4.3.2, gcc defines __GXX_RTTI if and only if RTTI is // enabled. # elif defined(__GNUC__) # ifdef __GXX_RTTI // When building against STLport with the Android NDK and with // -frtti -fno-exceptions, the build fails at link time with undefined // references to __cxa_bad_typeid. Note sure if STL or toolchain bug, // so disable RTTI when detected. # if GTEST_OS_LINUX_ANDROID && defined(_STLPORT_MAJOR) && \ !defined(__EXCEPTIONS) # define GTEST_HAS_RTTI 0 # else # define GTEST_HAS_RTTI 1 # endif // GTEST_OS_LINUX_ANDROID && __STLPORT_MAJOR && !__EXCEPTIONS # else # define GTEST_HAS_RTTI 0 # endif // __GXX_RTTI // Clang defines __GXX_RTTI starting with version 3.0, but its manual recommends // using has_feature instead. has_feature(cxx_rtti) is supported since 2.7, the // first version with C++ support. # elif defined(__clang__) # define GTEST_HAS_RTTI __has_feature(cxx_rtti) // Starting with version 9.0 IBM Visual Age defines __RTTI_ALL__ to 1 if // both the typeid and dynamic_cast features are present. # elif defined(__IBMCPP__) && (__IBMCPP__ >= 900) # ifdef __RTTI_ALL__ # define GTEST_HAS_RTTI 1 # else # define GTEST_HAS_RTTI 0 # endif # else // For all other compilers, we assume RTTI is enabled. # define GTEST_HAS_RTTI 1 # endif // _MSC_VER #endif // GTEST_HAS_RTTI // It's this header's responsibility to #include when RTTI // is enabled. #if GTEST_HAS_RTTI # include #endif // Determines whether Google Test can use the pthreads library. #ifndef GTEST_HAS_PTHREAD // The user didn't tell us explicitly, so we make reasonable assumptions about // which platforms have pthreads support. // // To disable threading support in Google Test, add -DGTEST_HAS_PTHREAD=0 // to your compiler flags. #define GTEST_HAS_PTHREAD \ (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_HPUX || GTEST_OS_QNX || \ GTEST_OS_FREEBSD || GTEST_OS_NACL || GTEST_OS_NETBSD || GTEST_OS_FUCHSIA || \ GTEST_OS_DRAGONFLY || GTEST_OS_GNU_KFREEBSD || GTEST_OS_OPENBSD || \ GTEST_OS_HAIKU) #endif // GTEST_HAS_PTHREAD #if GTEST_HAS_PTHREAD // gtest-port.h guarantees to #include when GTEST_HAS_PTHREAD is // true. # include // NOLINT // For timespec and nanosleep, used below. # include // NOLINT #endif // Determines whether clone(2) is supported. // Usually it will only be available on Linux, excluding // Linux on the Itanium architecture. // Also see http://linux.die.net/man/2/clone. #ifndef GTEST_HAS_CLONE // The user didn't tell us, so we need to figure it out. # if GTEST_OS_LINUX && !defined(__ia64__) # if GTEST_OS_LINUX_ANDROID // On Android, clone() became available at different API levels for each 32-bit // architecture. # if defined(__LP64__) || \ (defined(__arm__) && __ANDROID_API__ >= 9) || \ (defined(__mips__) && __ANDROID_API__ >= 12) || \ (defined(__i386__) && __ANDROID_API__ >= 17) # define GTEST_HAS_CLONE 1 # else # define GTEST_HAS_CLONE 0 # endif # else # define GTEST_HAS_CLONE 1 # endif # else # define GTEST_HAS_CLONE 0 # endif // GTEST_OS_LINUX && !defined(__ia64__) #endif // GTEST_HAS_CLONE // Determines whether to support stream redirection. This is used to test // output correctness and to implement death tests. #ifndef GTEST_HAS_STREAM_REDIRECTION // By default, we assume that stream redirection is supported on all // platforms except known mobile ones. # if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT # define GTEST_HAS_STREAM_REDIRECTION 0 # else # define GTEST_HAS_STREAM_REDIRECTION 1 # endif // !GTEST_OS_WINDOWS_MOBILE #endif // GTEST_HAS_STREAM_REDIRECTION // Determines whether to support death tests. // pops up a dialog window that cannot be suppressed programmatically. #if (GTEST_OS_LINUX || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \ (GTEST_OS_MAC && !GTEST_OS_IOS) || \ (GTEST_OS_WINDOWS_DESKTOP && _MSC_VER) || GTEST_OS_WINDOWS_MINGW || \ GTEST_OS_AIX || GTEST_OS_HPUX || GTEST_OS_OPENBSD || GTEST_OS_QNX || \ GTEST_OS_FREEBSD || GTEST_OS_NETBSD || GTEST_OS_FUCHSIA || \ GTEST_OS_DRAGONFLY || GTEST_OS_GNU_KFREEBSD || GTEST_OS_HAIKU) # define GTEST_HAS_DEATH_TEST 1 #endif // Determines whether to support type-driven tests. // Typed tests need and variadic macros, which GCC, VC++ 8.0, // Sun Pro CC, IBM Visual Age, and HP aCC support. #if defined(__GNUC__) || defined(_MSC_VER) || defined(__SUNPRO_CC) || \ defined(__IBMCPP__) || defined(__HP_aCC) # define GTEST_HAS_TYPED_TEST 1 # define GTEST_HAS_TYPED_TEST_P 1 #endif // Determines whether the system compiler uses UTF-16 for encoding wide strings. #define GTEST_WIDE_STRING_USES_UTF16_ \ (GTEST_OS_WINDOWS || GTEST_OS_CYGWIN || GTEST_OS_AIX || GTEST_OS_OS2) // Determines whether test results can be streamed to a socket. #if GTEST_OS_LINUX || GTEST_OS_GNU_KFREEBSD || GTEST_OS_DRAGONFLY || \ GTEST_OS_FREEBSD || GTEST_OS_NETBSD || GTEST_OS_OPENBSD # define GTEST_CAN_STREAM_RESULTS_ 1 #endif // Defines some utility macros. // The GNU compiler emits a warning if nested "if" statements are followed by // an "else" statement and braces are not used to explicitly disambiguate the // "else" binding. This leads to problems with code like: // // if (gate) // ASSERT_*(condition) << "Some message"; // // The "switch (0) case 0:" idiom is used to suppress this. #ifdef __INTEL_COMPILER # define GTEST_AMBIGUOUS_ELSE_BLOCKER_ #else # define GTEST_AMBIGUOUS_ELSE_BLOCKER_ switch (0) case 0: default: // NOLINT #endif // Use this annotation at the end of a struct/class definition to // prevent the compiler from optimizing away instances that are never // used. This is useful when all interesting logic happens inside the // c'tor and / or d'tor. Example: // // struct Foo { // Foo() { ... } // } GTEST_ATTRIBUTE_UNUSED_; // // Also use it after a variable or parameter declaration to tell the // compiler the variable/parameter does not have to be used. #if defined(__GNUC__) && !defined(COMPILER_ICC) # define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) #elif defined(__clang__) # if __has_attribute(unused) # define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) # endif #endif #ifndef GTEST_ATTRIBUTE_UNUSED_ # define GTEST_ATTRIBUTE_UNUSED_ #endif // Use this annotation before a function that takes a printf format string. #if (defined(__GNUC__) || defined(__clang__)) && !defined(COMPILER_ICC) # if defined(__MINGW_PRINTF_FORMAT) // MinGW has two different printf implementations. Ensure the format macro // matches the selected implementation. See // https://sourceforge.net/p/mingw-w64/wiki2/gnu%20printf/. # define GTEST_ATTRIBUTE_PRINTF_(string_index, first_to_check) \ __attribute__((__format__(__MINGW_PRINTF_FORMAT, string_index, \ first_to_check))) # else # define GTEST_ATTRIBUTE_PRINTF_(string_index, first_to_check) \ __attribute__((__format__(__printf__, string_index, first_to_check))) # endif #else # define GTEST_ATTRIBUTE_PRINTF_(string_index, first_to_check) #endif // A macro to disallow operator= // This should be used in the private: declarations for a class. #define GTEST_DISALLOW_ASSIGN_(type) \ void operator=(type const &) = delete // A macro to disallow copy constructor and operator= // This should be used in the private: declarations for a class. #define GTEST_DISALLOW_COPY_AND_ASSIGN_(type) \ type(type const &) = delete; \ GTEST_DISALLOW_ASSIGN_(type) // Tell the compiler to warn about unused return values for functions declared // with this macro. The macro should be used on function declarations // following the argument list: // // Sprocket* AllocateSprocket() GTEST_MUST_USE_RESULT_; #if defined(__GNUC__) && !defined(COMPILER_ICC) # define GTEST_MUST_USE_RESULT_ __attribute__ ((warn_unused_result)) #else # define GTEST_MUST_USE_RESULT_ #endif // __GNUC__ && !COMPILER_ICC // MS C++ compiler emits warning when a conditional expression is compile time // constant. In some contexts this warning is false positive and needs to be // suppressed. Use the following two macros in such cases: // // GTEST_INTENTIONAL_CONST_COND_PUSH_() // while (true) { // GTEST_INTENTIONAL_CONST_COND_POP_() // } # define GTEST_INTENTIONAL_CONST_COND_PUSH_() \ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4127) # define GTEST_INTENTIONAL_CONST_COND_POP_() \ GTEST_DISABLE_MSC_WARNINGS_POP_() // Determine whether the compiler supports Microsoft's Structured Exception // Handling. This is supported by several Windows compilers but generally // does not exist on any other system. #ifndef GTEST_HAS_SEH // The user didn't tell us, so we need to figure it out. # if defined(_MSC_VER) || defined(__BORLANDC__) // These two compilers are known to support SEH. # define GTEST_HAS_SEH 1 # else // Assume no SEH. # define GTEST_HAS_SEH 0 # endif #endif // GTEST_HAS_SEH #ifndef GTEST_IS_THREADSAFE #define GTEST_IS_THREADSAFE \ (GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ || \ (GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT) || \ GTEST_HAS_PTHREAD) #endif // GTEST_IS_THREADSAFE // GTEST_API_ qualifies all symbols that must be exported. The definitions below // are guarded by #ifndef to give embedders a chance to define GTEST_API_ in // gtest/internal/custom/gtest-port.h #ifndef GTEST_API_ #ifdef _MSC_VER # if GTEST_LINKED_AS_SHARED_LIBRARY # define GTEST_API_ __declspec(dllimport) # elif GTEST_CREATE_SHARED_LIBRARY # define GTEST_API_ __declspec(dllexport) # endif #elif __GNUC__ >= 4 || defined(__clang__) # define GTEST_API_ __attribute__((visibility ("default"))) #endif // _MSC_VER #endif // GTEST_API_ #ifndef GTEST_API_ # define GTEST_API_ #endif // GTEST_API_ #ifndef GTEST_DEFAULT_DEATH_TEST_STYLE # define GTEST_DEFAULT_DEATH_TEST_STYLE "fast" #endif // GTEST_DEFAULT_DEATH_TEST_STYLE #ifdef __GNUC__ // Ask the compiler to never inline a given function. # define GTEST_NO_INLINE_ __attribute__((noinline)) #else # define GTEST_NO_INLINE_ #endif // _LIBCPP_VERSION is defined by the libc++ library from the LLVM project. #if !defined(GTEST_HAS_CXXABI_H_) # if defined(__GLIBCXX__) || (defined(_LIBCPP_VERSION) && !defined(_MSC_VER)) # define GTEST_HAS_CXXABI_H_ 1 # else # define GTEST_HAS_CXXABI_H_ 0 # endif #endif // A function level attribute to disable checking for use of uninitialized // memory when built with MemorySanitizer. #if defined(__clang__) # if __has_feature(memory_sanitizer) # define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ \ __attribute__((no_sanitize_memory)) # else # define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ # endif // __has_feature(memory_sanitizer) #else # define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ #endif // __clang__ // A function level attribute to disable AddressSanitizer instrumentation. #if defined(__clang__) # if __has_feature(address_sanitizer) # define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ \ __attribute__((no_sanitize_address)) # else # define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ # endif // __has_feature(address_sanitizer) #else # define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ #endif // __clang__ // A function level attribute to disable HWAddressSanitizer instrumentation. #if defined(__clang__) # if __has_feature(hwaddress_sanitizer) # define GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ \ __attribute__((no_sanitize("hwaddress"))) # else # define GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ # endif // __has_feature(hwaddress_sanitizer) #else # define GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ #endif // __clang__ // A function level attribute to disable ThreadSanitizer instrumentation. #if defined(__clang__) # if __has_feature(thread_sanitizer) # define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ \ __attribute__((no_sanitize_thread)) # else # define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ # endif // __has_feature(thread_sanitizer) #else # define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ #endif // __clang__ namespace testing { class Message; // Legacy imports for backwards compatibility. // New code should use std:: names directly. using std::get; using std::make_tuple; using std::tuple; using std::tuple_element; using std::tuple_size; namespace internal { // A secret type that Google Test users don't know about. It has no // definition on purpose. Therefore it's impossible to create a // Secret object, which is what we want. class Secret; // The GTEST_COMPILE_ASSERT_ is a legacy macro used to verify that a compile // time expression is true (in new code, use static_assert instead). For // example, you could use it to verify the size of a static array: // // GTEST_COMPILE_ASSERT_(GTEST_ARRAY_SIZE_(names) == NUM_NAMES, // names_incorrect_size); // // The second argument to the macro must be a valid C++ identifier. If the // expression is false, compiler will issue an error containing this identifier. #define GTEST_COMPILE_ASSERT_(expr, msg) static_assert(expr, #msg) // Evaluates to the number of elements in 'array'. #define GTEST_ARRAY_SIZE_(array) (sizeof(array) / sizeof(array[0])) // A helper for suppressing warnings on constant condition. It just // returns 'condition'. GTEST_API_ bool IsTrue(bool condition); // Defines RE. #if GTEST_USES_PCRE // if used, PCRE is injected by custom/gtest-port.h #elif GTEST_USES_POSIX_RE || GTEST_USES_SIMPLE_RE // A simple C++ wrapper for . It uses the POSIX Extended // Regular Expression syntax. class GTEST_API_ RE { public: // A copy constructor is required by the Standard to initialize object // references from r-values. RE(const RE& other) { Init(other.pattern()); } // Constructs an RE from a string. RE(const ::std::string& regex) { Init(regex.c_str()); } // NOLINT RE(const char* regex) { Init(regex); } // NOLINT ~RE(); // Returns the string representation of the regex. const char* pattern() const { return pattern_; } // FullMatch(str, re) returns true if and only if regular expression re // matches the entire str. // PartialMatch(str, re) returns true if and only if regular expression re // matches a substring of str (including str itself). static bool FullMatch(const ::std::string& str, const RE& re) { return FullMatch(str.c_str(), re); } static bool PartialMatch(const ::std::string& str, const RE& re) { return PartialMatch(str.c_str(), re); } static bool FullMatch(const char* str, const RE& re); static bool PartialMatch(const char* str, const RE& re); private: void Init(const char* regex); const char* pattern_; bool is_valid_; # if GTEST_USES_POSIX_RE regex_t full_regex_; // For FullMatch(). regex_t partial_regex_; // For PartialMatch(). # else // GTEST_USES_SIMPLE_RE const char* full_pattern_; // For FullMatch(); # endif GTEST_DISALLOW_ASSIGN_(RE); }; #endif // GTEST_USES_PCRE // Formats a source file path and a line number as they would appear // in an error message from the compiler used to compile this code. GTEST_API_ ::std::string FormatFileLocation(const char* file, int line); // Formats a file location for compiler-independent XML output. // Although this function is not platform dependent, we put it next to // FormatFileLocation in order to contrast the two functions. GTEST_API_ ::std::string FormatCompilerIndependentFileLocation(const char* file, int line); // Defines logging utilities: // GTEST_LOG_(severity) - logs messages at the specified severity level. The // message itself is streamed into the macro. // LogToStderr() - directs all log messages to stderr. // FlushInfoLog() - flushes informational log messages. enum GTestLogSeverity { GTEST_INFO, GTEST_WARNING, GTEST_ERROR, GTEST_FATAL }; // Formats log entry severity, provides a stream object for streaming the // log message, and terminates the message with a newline when going out of // scope. class GTEST_API_ GTestLog { public: GTestLog(GTestLogSeverity severity, const char* file, int line); // Flushes the buffers and, if severity is GTEST_FATAL, aborts the program. ~GTestLog(); ::std::ostream& GetStream() { return ::std::cerr; } private: const GTestLogSeverity severity_; GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestLog); }; #if !defined(GTEST_LOG_) # define GTEST_LOG_(severity) \ ::testing::internal::GTestLog(::testing::internal::GTEST_##severity, \ __FILE__, __LINE__).GetStream() inline void LogToStderr() {} inline void FlushInfoLog() { fflush(nullptr); } #endif // !defined(GTEST_LOG_) #if !defined(GTEST_CHECK_) // INTERNAL IMPLEMENTATION - DO NOT USE. // // GTEST_CHECK_ is an all-mode assert. It aborts the program if the condition // is not satisfied. // Synopsys: // GTEST_CHECK_(boolean_condition); // or // GTEST_CHECK_(boolean_condition) << "Additional message"; // // This checks the condition and if the condition is not satisfied // it prints message about the condition violation, including the // condition itself, plus additional message streamed into it, if any, // and then it aborts the program. It aborts the program irrespective of // whether it is built in the debug mode or not. # define GTEST_CHECK_(condition) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (::testing::internal::IsTrue(condition)) \ ; \ else \ GTEST_LOG_(FATAL) << "Condition " #condition " failed. " #endif // !defined(GTEST_CHECK_) // An all-mode assert to verify that the given POSIX-style function // call returns 0 (indicating success). Known limitation: this // doesn't expand to a balanced 'if' statement, so enclose the macro // in {} if you need to use it as the only statement in an 'if' // branch. #define GTEST_CHECK_POSIX_SUCCESS_(posix_call) \ if (const int gtest_error = (posix_call)) \ GTEST_LOG_(FATAL) << #posix_call << "failed with error " \ << gtest_error // Transforms "T" into "const T&" according to standard reference collapsing // rules (this is only needed as a backport for C++98 compilers that do not // support reference collapsing). Specifically, it transforms: // // char ==> const char& // const char ==> const char& // char& ==> char& // const char& ==> const char& // // Note that the non-const reference will not have "const" added. This is // standard, and necessary so that "T" can always bind to "const T&". template struct ConstRef { typedef const T& type; }; template struct ConstRef { typedef T& type; }; // The argument T must depend on some template parameters. #define GTEST_REFERENCE_TO_CONST_(T) \ typename ::testing::internal::ConstRef::type // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Use ImplicitCast_ as a safe version of static_cast for upcasting in // the type hierarchy (e.g. casting a Foo* to a SuperclassOfFoo* or a // const Foo*). When you use ImplicitCast_, the compiler checks that // the cast is safe. Such explicit ImplicitCast_s are necessary in // surprisingly many situations where C++ demands an exact type match // instead of an argument type convertable to a target type. // // The syntax for using ImplicitCast_ is the same as for static_cast: // // ImplicitCast_(expr) // // ImplicitCast_ would have been part of the C++ standard library, // but the proposal was submitted too late. It will probably make // its way into the language in the future. // // This relatively ugly name is intentional. It prevents clashes with // similar functions users may have (e.g., implicit_cast). The internal // namespace alone is not enough because the function can be found by ADL. template inline To ImplicitCast_(To x) { return x; } // When you upcast (that is, cast a pointer from type Foo to type // SuperclassOfFoo), it's fine to use ImplicitCast_<>, since upcasts // always succeed. When you downcast (that is, cast a pointer from // type Foo to type SubclassOfFoo), static_cast<> isn't safe, because // how do you know the pointer is really of type SubclassOfFoo? It // could be a bare Foo, or of type DifferentSubclassOfFoo. Thus, // when you downcast, you should use this macro. In debug mode, we // use dynamic_cast<> to double-check the downcast is legal (we die // if it's not). In normal mode, we do the efficient static_cast<> // instead. Thus, it's important to test in debug mode to make sure // the cast is legal! // This is the only place in the code we should use dynamic_cast<>. // In particular, you SHOULDN'T be using dynamic_cast<> in order to // do RTTI (eg code like this: // if (dynamic_cast(foo)) HandleASubclass1Object(foo); // if (dynamic_cast(foo)) HandleASubclass2Object(foo); // You should design the code some other way not to need this. // // This relatively ugly name is intentional. It prevents clashes with // similar functions users may have (e.g., down_cast). The internal // namespace alone is not enough because the function can be found by ADL. template // use like this: DownCast_(foo); inline To DownCast_(From* f) { // so we only accept pointers // Ensures that To is a sub-type of From *. This test is here only // for compile-time type checking, and has no overhead in an // optimized build at run-time, as it will be optimized away // completely. GTEST_INTENTIONAL_CONST_COND_PUSH_() if (false) { GTEST_INTENTIONAL_CONST_COND_POP_() const To to = nullptr; ::testing::internal::ImplicitCast_(to); } #if GTEST_HAS_RTTI // RTTI: debug mode only! GTEST_CHECK_(f == nullptr || dynamic_cast(f) != nullptr); #endif return static_cast(f); } // Downcasts the pointer of type Base to Derived. // Derived must be a subclass of Base. The parameter MUST // point to a class of type Derived, not any subclass of it. // When RTTI is available, the function performs a runtime // check to enforce this. template Derived* CheckedDowncastToActualType(Base* base) { #if GTEST_HAS_RTTI GTEST_CHECK_(typeid(*base) == typeid(Derived)); #endif #if GTEST_HAS_DOWNCAST_ return ::down_cast(base); #elif GTEST_HAS_RTTI return dynamic_cast(base); // NOLINT #else return static_cast(base); // Poor man's downcast. #endif } #if GTEST_HAS_STREAM_REDIRECTION // Defines the stderr capturer: // CaptureStdout - starts capturing stdout. // GetCapturedStdout - stops capturing stdout and returns the captured string. // CaptureStderr - starts capturing stderr. // GetCapturedStderr - stops capturing stderr and returns the captured string. // GTEST_API_ void CaptureStdout(); GTEST_API_ std::string GetCapturedStdout(); GTEST_API_ void CaptureStderr(); GTEST_API_ std::string GetCapturedStderr(); #endif // GTEST_HAS_STREAM_REDIRECTION // Returns the size (in bytes) of a file. GTEST_API_ size_t GetFileSize(FILE* file); // Reads the entire content of a file as a string. GTEST_API_ std::string ReadEntireFile(FILE* file); // All command line arguments. GTEST_API_ std::vector GetArgvs(); #if GTEST_HAS_DEATH_TEST std::vector GetInjectableArgvs(); // Deprecated: pass the args vector by value instead. void SetInjectableArgvs(const std::vector* new_argvs); void SetInjectableArgvs(const std::vector& new_argvs); void ClearInjectableArgvs(); #endif // GTEST_HAS_DEATH_TEST // Defines synchronization primitives. #if GTEST_IS_THREADSAFE # if GTEST_HAS_PTHREAD // Sleeps for (roughly) n milliseconds. This function is only for testing // Google Test's own constructs. Don't use it in user tests, either // directly or indirectly. inline void SleepMilliseconds(int n) { const timespec time = { 0, // 0 seconds. n * 1000L * 1000L, // And n ms. }; nanosleep(&time, nullptr); } # endif // GTEST_HAS_PTHREAD # if GTEST_HAS_NOTIFICATION_ // Notification has already been imported into the namespace. // Nothing to do here. # elif GTEST_HAS_PTHREAD // Allows a controller thread to pause execution of newly created // threads until notified. Instances of this class must be created // and destroyed in the controller thread. // // This class is only for testing Google Test's own constructs. Do not // use it in user tests, either directly or indirectly. class Notification { public: Notification() : notified_(false) { GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, nullptr)); } ~Notification() { pthread_mutex_destroy(&mutex_); } // Notifies all threads created with this notification to start. Must // be called from the controller thread. void Notify() { pthread_mutex_lock(&mutex_); notified_ = true; pthread_mutex_unlock(&mutex_); } // Blocks until the controller thread notifies. Must be called from a test // thread. void WaitForNotification() { for (;;) { pthread_mutex_lock(&mutex_); const bool notified = notified_; pthread_mutex_unlock(&mutex_); if (notified) break; SleepMilliseconds(10); } } private: pthread_mutex_t mutex_; bool notified_; GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); }; # elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT GTEST_API_ void SleepMilliseconds(int n); // Provides leak-safe Windows kernel handle ownership. // Used in death tests and in threading support. class GTEST_API_ AutoHandle { public: // Assume that Win32 HANDLE type is equivalent to void*. Doing so allows us to // avoid including in this header file. Including is // undesirable because it defines a lot of symbols and macros that tend to // conflict with client code. This assumption is verified by // WindowsTypesTest.HANDLEIsVoidStar. typedef void* Handle; AutoHandle(); explicit AutoHandle(Handle handle); ~AutoHandle(); Handle Get() const; void Reset(); void Reset(Handle handle); private: // Returns true if and only if the handle is a valid handle object that can be // closed. bool IsCloseable() const; Handle handle_; GTEST_DISALLOW_COPY_AND_ASSIGN_(AutoHandle); }; // Allows a controller thread to pause execution of newly created // threads until notified. Instances of this class must be created // and destroyed in the controller thread. // // This class is only for testing Google Test's own constructs. Do not // use it in user tests, either directly or indirectly. class GTEST_API_ Notification { public: Notification(); void Notify(); void WaitForNotification(); private: AutoHandle event_; GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); }; # endif // GTEST_HAS_NOTIFICATION_ // On MinGW, we can have both GTEST_OS_WINDOWS and GTEST_HAS_PTHREAD // defined, but we don't want to use MinGW's pthreads implementation, which // has conformance problems with some versions of the POSIX standard. # if GTEST_HAS_PTHREAD && !GTEST_OS_WINDOWS_MINGW // As a C-function, ThreadFuncWithCLinkage cannot be templated itself. // Consequently, it cannot select a correct instantiation of ThreadWithParam // in order to call its Run(). Introducing ThreadWithParamBase as a // non-templated base class for ThreadWithParam allows us to bypass this // problem. class ThreadWithParamBase { public: virtual ~ThreadWithParamBase() {} virtual void Run() = 0; }; // pthread_create() accepts a pointer to a function type with the C linkage. // According to the Standard (7.5/1), function types with different linkages // are different even if they are otherwise identical. Some compilers (for // example, SunStudio) treat them as different types. Since class methods // cannot be defined with C-linkage we need to define a free C-function to // pass into pthread_create(). extern "C" inline void* ThreadFuncWithCLinkage(void* thread) { static_cast(thread)->Run(); return nullptr; } // Helper class for testing Google Test's multi-threading constructs. // To use it, write: // // void ThreadFunc(int param) { /* Do things with param */ } // Notification thread_can_start; // ... // // The thread_can_start parameter is optional; you can supply NULL. // ThreadWithParam thread(&ThreadFunc, 5, &thread_can_start); // thread_can_start.Notify(); // // These classes are only for testing Google Test's own constructs. Do // not use them in user tests, either directly or indirectly. template class ThreadWithParam : public ThreadWithParamBase { public: typedef void UserThreadFunc(T); ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start) : func_(func), param_(param), thread_can_start_(thread_can_start), finished_(false) { ThreadWithParamBase* const base = this; // The thread can be created only after all fields except thread_ // have been initialized. GTEST_CHECK_POSIX_SUCCESS_( pthread_create(&thread_, nullptr, &ThreadFuncWithCLinkage, base)); } ~ThreadWithParam() override { Join(); } void Join() { if (!finished_) { GTEST_CHECK_POSIX_SUCCESS_(pthread_join(thread_, nullptr)); finished_ = true; } } void Run() override { if (thread_can_start_ != nullptr) thread_can_start_->WaitForNotification(); func_(param_); } private: UserThreadFunc* const func_; // User-supplied thread function. const T param_; // User-supplied parameter to the thread function. // When non-NULL, used to block execution until the controller thread // notifies. Notification* const thread_can_start_; bool finished_; // true if and only if we know that the thread function has // finished. pthread_t thread_; // The native thread object. GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); }; # endif // !GTEST_OS_WINDOWS && GTEST_HAS_PTHREAD || // GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ # if GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ // Mutex and ThreadLocal have already been imported into the namespace. // Nothing to do here. # elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT // Mutex implements mutex on Windows platforms. It is used in conjunction // with class MutexLock: // // Mutex mutex; // ... // MutexLock lock(&mutex); // Acquires the mutex and releases it at the // // end of the current scope. // // A static Mutex *must* be defined or declared using one of the following // macros: // GTEST_DEFINE_STATIC_MUTEX_(g_some_mutex); // GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex); // // (A non-static Mutex is defined/declared in the usual way). class GTEST_API_ Mutex { public: enum MutexType { kStatic = 0, kDynamic = 1 }; // We rely on kStaticMutex being 0 as it is to what the linker initializes // type_ in static mutexes. critical_section_ will be initialized lazily // in ThreadSafeLazyInit(). enum StaticConstructorSelector { kStaticMutex = 0 }; // This constructor intentionally does nothing. It relies on type_ being // statically initialized to 0 (effectively setting it to kStatic) and on // ThreadSafeLazyInit() to lazily initialize the rest of the members. explicit Mutex(StaticConstructorSelector /*dummy*/) {} Mutex(); ~Mutex(); void Lock(); void Unlock(); // Does nothing if the current thread holds the mutex. Otherwise, crashes // with high probability. void AssertHeld(); private: // Initializes owner_thread_id_ and critical_section_ in static mutexes. void ThreadSafeLazyInit(); // Per https://blogs.msdn.microsoft.com/oldnewthing/20040223-00/?p=40503, // we assume that 0 is an invalid value for thread IDs. unsigned int owner_thread_id_; // For static mutexes, we rely on these members being initialized to zeros // by the linker. MutexType type_; long critical_section_init_phase_; // NOLINT GTEST_CRITICAL_SECTION* critical_section_; GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); }; # define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ extern ::testing::internal::Mutex mutex # define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ ::testing::internal::Mutex mutex(::testing::internal::Mutex::kStaticMutex) // We cannot name this class MutexLock because the ctor declaration would // conflict with a macro named MutexLock, which is defined on some // platforms. That macro is used as a defensive measure to prevent against // inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than // "MutexLock l(&mu)". Hence the typedef trick below. class GTestMutexLock { public: explicit GTestMutexLock(Mutex* mutex) : mutex_(mutex) { mutex_->Lock(); } ~GTestMutexLock() { mutex_->Unlock(); } private: Mutex* const mutex_; GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock); }; typedef GTestMutexLock MutexLock; // Base class for ValueHolder. Allows a caller to hold and delete a value // without knowing its type. class ThreadLocalValueHolderBase { public: virtual ~ThreadLocalValueHolderBase() {} }; // Provides a way for a thread to send notifications to a ThreadLocal // regardless of its parameter type. class ThreadLocalBase { public: // Creates a new ValueHolder object holding a default value passed to // this ThreadLocal's constructor and returns it. It is the caller's // responsibility not to call this when the ThreadLocal instance already // has a value on the current thread. virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const = 0; protected: ThreadLocalBase() {} virtual ~ThreadLocalBase() {} private: GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocalBase); }; // Maps a thread to a set of ThreadLocals that have values instantiated on that // thread and notifies them when the thread exits. A ThreadLocal instance is // expected to persist until all threads it has values on have terminated. class GTEST_API_ ThreadLocalRegistry { public: // Registers thread_local_instance as having value on the current thread. // Returns a value that can be used to identify the thread from other threads. static ThreadLocalValueHolderBase* GetValueOnCurrentThread( const ThreadLocalBase* thread_local_instance); // Invoked when a ThreadLocal instance is destroyed. static void OnThreadLocalDestroyed( const ThreadLocalBase* thread_local_instance); }; class GTEST_API_ ThreadWithParamBase { public: void Join(); protected: class Runnable { public: virtual ~Runnable() {} virtual void Run() = 0; }; ThreadWithParamBase(Runnable *runnable, Notification* thread_can_start); virtual ~ThreadWithParamBase(); private: AutoHandle thread_; }; // Helper class for testing Google Test's multi-threading constructs. template class ThreadWithParam : public ThreadWithParamBase { public: typedef void UserThreadFunc(T); ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start) : ThreadWithParamBase(new RunnableImpl(func, param), thread_can_start) { } virtual ~ThreadWithParam() {} private: class RunnableImpl : public Runnable { public: RunnableImpl(UserThreadFunc* func, T param) : func_(func), param_(param) { } virtual ~RunnableImpl() {} virtual void Run() { func_(param_); } private: UserThreadFunc* const func_; const T param_; GTEST_DISALLOW_COPY_AND_ASSIGN_(RunnableImpl); }; GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); }; // Implements thread-local storage on Windows systems. // // // Thread 1 // ThreadLocal tl(100); // 100 is the default value for each thread. // // // Thread 2 // tl.set(150); // Changes the value for thread 2 only. // EXPECT_EQ(150, tl.get()); // // // Thread 1 // EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value. // tl.set(200); // EXPECT_EQ(200, tl.get()); // // The template type argument T must have a public copy constructor. // In addition, the default ThreadLocal constructor requires T to have // a public default constructor. // // The users of a TheadLocal instance have to make sure that all but one // threads (including the main one) using that instance have exited before // destroying it. Otherwise, the per-thread objects managed for them by the // ThreadLocal instance are not guaranteed to be destroyed on all platforms. // // Google Test only uses global ThreadLocal objects. That means they // will die after main() has returned. Therefore, no per-thread // object managed by Google Test will be leaked as long as all threads // using Google Test have exited when main() returns. template class ThreadLocal : public ThreadLocalBase { public: ThreadLocal() : default_factory_(new DefaultValueHolderFactory()) {} explicit ThreadLocal(const T& value) : default_factory_(new InstanceValueHolderFactory(value)) {} ~ThreadLocal() { ThreadLocalRegistry::OnThreadLocalDestroyed(this); } T* pointer() { return GetOrCreateValue(); } const T* pointer() const { return GetOrCreateValue(); } const T& get() const { return *pointer(); } void set(const T& value) { *pointer() = value; } private: // Holds a value of T. Can be deleted via its base class without the caller // knowing the type of T. class ValueHolder : public ThreadLocalValueHolderBase { public: ValueHolder() : value_() {} explicit ValueHolder(const T& value) : value_(value) {} T* pointer() { return &value_; } private: T value_; GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); }; T* GetOrCreateValue() const { return static_cast( ThreadLocalRegistry::GetValueOnCurrentThread(this))->pointer(); } virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const { return default_factory_->MakeNewHolder(); } class ValueHolderFactory { public: ValueHolderFactory() {} virtual ~ValueHolderFactory() {} virtual ValueHolder* MakeNewHolder() const = 0; private: GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolderFactory); }; class DefaultValueHolderFactory : public ValueHolderFactory { public: DefaultValueHolderFactory() {} virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(); } private: GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultValueHolderFactory); }; class InstanceValueHolderFactory : public ValueHolderFactory { public: explicit InstanceValueHolderFactory(const T& value) : value_(value) {} virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(value_); } private: const T value_; // The value for each thread. GTEST_DISALLOW_COPY_AND_ASSIGN_(InstanceValueHolderFactory); }; std::unique_ptr default_factory_; GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); }; # elif GTEST_HAS_PTHREAD // MutexBase and Mutex implement mutex on pthreads-based platforms. class MutexBase { public: // Acquires this mutex. void Lock() { GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_lock(&mutex_)); owner_ = pthread_self(); has_owner_ = true; } // Releases this mutex. void Unlock() { // Since the lock is being released the owner_ field should no longer be // considered valid. We don't protect writing to has_owner_ here, as it's // the caller's responsibility to ensure that the current thread holds the // mutex when this is called. has_owner_ = false; GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_unlock(&mutex_)); } // Does nothing if the current thread holds the mutex. Otherwise, crashes // with high probability. void AssertHeld() const { GTEST_CHECK_(has_owner_ && pthread_equal(owner_, pthread_self())) << "The current thread is not holding the mutex @" << this; } // A static mutex may be used before main() is entered. It may even // be used before the dynamic initialization stage. Therefore we // must be able to initialize a static mutex object at link time. // This means MutexBase has to be a POD and its member variables // have to be public. public: pthread_mutex_t mutex_; // The underlying pthread mutex. // has_owner_ indicates whether the owner_ field below contains a valid thread // ID and is therefore safe to inspect (e.g., to use in pthread_equal()). All // accesses to the owner_ field should be protected by a check of this field. // An alternative might be to memset() owner_ to all zeros, but there's no // guarantee that a zero'd pthread_t is necessarily invalid or even different // from pthread_self(). bool has_owner_; pthread_t owner_; // The thread holding the mutex. }; // Forward-declares a static mutex. # define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ extern ::testing::internal::MutexBase mutex // Defines and statically (i.e. at link time) initializes a static mutex. // The initialization list here does not explicitly initialize each field, // instead relying on default initialization for the unspecified fields. In // particular, the owner_ field (a pthread_t) is not explicitly initialized. // This allows initialization to work whether pthread_t is a scalar or struct. // The flag -Wmissing-field-initializers must not be specified for this to work. #define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ ::testing::internal::MutexBase mutex = {PTHREAD_MUTEX_INITIALIZER, false, 0} // The Mutex class can only be used for mutexes created at runtime. It // shares its API with MutexBase otherwise. class Mutex : public MutexBase { public: Mutex() { GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, nullptr)); has_owner_ = false; } ~Mutex() { GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_destroy(&mutex_)); } private: GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); }; // We cannot name this class MutexLock because the ctor declaration would // conflict with a macro named MutexLock, which is defined on some // platforms. That macro is used as a defensive measure to prevent against // inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than // "MutexLock l(&mu)". Hence the typedef trick below. class GTestMutexLock { public: explicit GTestMutexLock(MutexBase* mutex) : mutex_(mutex) { mutex_->Lock(); } ~GTestMutexLock() { mutex_->Unlock(); } private: MutexBase* const mutex_; GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock); }; typedef GTestMutexLock MutexLock; // Helpers for ThreadLocal. // pthread_key_create() requires DeleteThreadLocalValue() to have // C-linkage. Therefore it cannot be templatized to access // ThreadLocal. Hence the need for class // ThreadLocalValueHolderBase. class ThreadLocalValueHolderBase { public: virtual ~ThreadLocalValueHolderBase() {} }; // Called by pthread to delete thread-local data stored by // pthread_setspecific(). extern "C" inline void DeleteThreadLocalValue(void* value_holder) { delete static_cast(value_holder); } // Implements thread-local storage on pthreads-based systems. template class GTEST_API_ ThreadLocal { public: ThreadLocal() : key_(CreateKey()), default_factory_(new DefaultValueHolderFactory()) {} explicit ThreadLocal(const T& value) : key_(CreateKey()), default_factory_(new InstanceValueHolderFactory(value)) {} ~ThreadLocal() { // Destroys the managed object for the current thread, if any. DeleteThreadLocalValue(pthread_getspecific(key_)); // Releases resources associated with the key. This will *not* // delete managed objects for other threads. GTEST_CHECK_POSIX_SUCCESS_(pthread_key_delete(key_)); } T* pointer() { return GetOrCreateValue(); } const T* pointer() const { return GetOrCreateValue(); } const T& get() const { return *pointer(); } void set(const T& value) { *pointer() = value; } private: // Holds a value of type T. class ValueHolder : public ThreadLocalValueHolderBase { public: ValueHolder() : value_() {} explicit ValueHolder(const T& value) : value_(value) {} T* pointer() { return &value_; } private: T value_; GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); }; static pthread_key_t CreateKey() { pthread_key_t key; // When a thread exits, DeleteThreadLocalValue() will be called on // the object managed for that thread. GTEST_CHECK_POSIX_SUCCESS_( pthread_key_create(&key, &DeleteThreadLocalValue)); return key; } T* GetOrCreateValue() const { ThreadLocalValueHolderBase* const holder = static_cast(pthread_getspecific(key_)); if (holder != nullptr) { return CheckedDowncastToActualType(holder)->pointer(); } ValueHolder* const new_holder = default_factory_->MakeNewHolder(); ThreadLocalValueHolderBase* const holder_base = new_holder; GTEST_CHECK_POSIX_SUCCESS_(pthread_setspecific(key_, holder_base)); return new_holder->pointer(); } class ValueHolderFactory { public: ValueHolderFactory() {} virtual ~ValueHolderFactory() {} virtual ValueHolder* MakeNewHolder() const = 0; private: GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolderFactory); }; class DefaultValueHolderFactory : public ValueHolderFactory { public: DefaultValueHolderFactory() {} virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(); } private: GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultValueHolderFactory); }; class InstanceValueHolderFactory : public ValueHolderFactory { public: explicit InstanceValueHolderFactory(const T& value) : value_(value) {} virtual ValueHolder* MakeNewHolder() const { return new ValueHolder(value_); } private: const T value_; // The value for each thread. GTEST_DISALLOW_COPY_AND_ASSIGN_(InstanceValueHolderFactory); }; // A key pthreads uses for looking up per-thread values. const pthread_key_t key_; std::unique_ptr default_factory_; GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); }; # endif // GTEST_HAS_MUTEX_AND_THREAD_LOCAL_ #else // GTEST_IS_THREADSAFE // A dummy implementation of synchronization primitives (mutex, lock, // and thread-local variable). Necessary for compiling Google Test where // mutex is not supported - using Google Test in multiple threads is not // supported on such platforms. class Mutex { public: Mutex() {} void Lock() {} void Unlock() {} void AssertHeld() const {} }; # define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ extern ::testing::internal::Mutex mutex # define GTEST_DEFINE_STATIC_MUTEX_(mutex) ::testing::internal::Mutex mutex // We cannot name this class MutexLock because the ctor declaration would // conflict with a macro named MutexLock, which is defined on some // platforms. That macro is used as a defensive measure to prevent against // inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than // "MutexLock l(&mu)". Hence the typedef trick below. class GTestMutexLock { public: explicit GTestMutexLock(Mutex*) {} // NOLINT }; typedef GTestMutexLock MutexLock; template class GTEST_API_ ThreadLocal { public: ThreadLocal() : value_() {} explicit ThreadLocal(const T& value) : value_(value) {} T* pointer() { return &value_; } const T* pointer() const { return &value_; } const T& get() const { return value_; } void set(const T& value) { value_ = value; } private: T value_; }; #endif // GTEST_IS_THREADSAFE // Returns the number of threads running in the process, or 0 to indicate that // we cannot detect it. GTEST_API_ size_t GetThreadCount(); template using bool_constant = std::integral_constant; #if GTEST_OS_WINDOWS # define GTEST_PATH_SEP_ "\\" # define GTEST_HAS_ALT_PATH_SEP_ 1 // The biggest signed integer type the compiler supports. typedef __int64 BiggestInt; #else # define GTEST_PATH_SEP_ "/" # define GTEST_HAS_ALT_PATH_SEP_ 0 typedef long long BiggestInt; // NOLINT #endif // GTEST_OS_WINDOWS // Utilities for char. // isspace(int ch) and friends accept an unsigned char or EOF. char // may be signed, depending on the compiler (or compiler flags). // Therefore we need to cast a char to unsigned char before calling // isspace(), etc. inline bool IsAlpha(char ch) { return isalpha(static_cast(ch)) != 0; } inline bool IsAlNum(char ch) { return isalnum(static_cast(ch)) != 0; } inline bool IsDigit(char ch) { return isdigit(static_cast(ch)) != 0; } inline bool IsLower(char ch) { return islower(static_cast(ch)) != 0; } inline bool IsSpace(char ch) { return isspace(static_cast(ch)) != 0; } inline bool IsUpper(char ch) { return isupper(static_cast(ch)) != 0; } inline bool IsXDigit(char ch) { return isxdigit(static_cast(ch)) != 0; } inline bool IsXDigit(wchar_t ch) { const unsigned char low_byte = static_cast(ch); return ch == low_byte && isxdigit(low_byte) != 0; } inline char ToLower(char ch) { return static_cast(tolower(static_cast(ch))); } inline char ToUpper(char ch) { return static_cast(toupper(static_cast(ch))); } inline std::string StripTrailingSpaces(std::string str) { std::string::iterator it = str.end(); while (it != str.begin() && IsSpace(*--it)) it = str.erase(it); return str; } // The testing::internal::posix namespace holds wrappers for common // POSIX functions. These wrappers hide the differences between // Windows/MSVC and POSIX systems. Since some compilers define these // standard functions as macros, the wrapper cannot have the same name // as the wrapped function. namespace posix { // Functions with a different name on Windows. #if GTEST_OS_WINDOWS typedef struct _stat StatStruct; # ifdef __BORLANDC__ inline int IsATTY(int fd) { return isatty(fd); } inline int StrCaseCmp(const char* s1, const char* s2) { return stricmp(s1, s2); } inline char* StrDup(const char* src) { return strdup(src); } # else // !__BORLANDC__ # if GTEST_OS_WINDOWS_MOBILE inline int IsATTY(int /* fd */) { return 0; } # else inline int IsATTY(int fd) { return _isatty(fd); } # endif // GTEST_OS_WINDOWS_MOBILE inline int StrCaseCmp(const char* s1, const char* s2) { return _stricmp(s1, s2); } inline char* StrDup(const char* src) { return _strdup(src); } # endif // __BORLANDC__ # if GTEST_OS_WINDOWS_MOBILE inline int FileNo(FILE* file) { return reinterpret_cast(_fileno(file)); } // Stat(), RmDir(), and IsDir() are not needed on Windows CE at this // time and thus not defined there. # else inline int FileNo(FILE* file) { return _fileno(file); } inline int Stat(const char* path, StatStruct* buf) { return _stat(path, buf); } inline int RmDir(const char* dir) { return _rmdir(dir); } inline bool IsDir(const StatStruct& st) { return (_S_IFDIR & st.st_mode) != 0; } # endif // GTEST_OS_WINDOWS_MOBILE #else typedef struct stat StatStruct; inline int FileNo(FILE* file) { return fileno(file); } inline int IsATTY(int fd) { return isatty(fd); } inline int Stat(const char* path, StatStruct* buf) { return stat(path, buf); } inline int StrCaseCmp(const char* s1, const char* s2) { return strcasecmp(s1, s2); } inline char* StrDup(const char* src) { return strdup(src); } inline int RmDir(const char* dir) { return rmdir(dir); } inline bool IsDir(const StatStruct& st) { return S_ISDIR(st.st_mode); } #endif // GTEST_OS_WINDOWS // Functions deprecated by MSVC 8.0. GTEST_DISABLE_MSC_DEPRECATED_PUSH_() inline const char* StrNCpy(char* dest, const char* src, size_t n) { return strncpy(dest, src, n); } // ChDir(), FReopen(), FDOpen(), Read(), Write(), Close(), and // StrError() aren't needed on Windows CE at this time and thus not // defined there. #if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT inline int ChDir(const char* dir) { return chdir(dir); } #endif inline FILE* FOpen(const char* path, const char* mode) { return fopen(path, mode); } #if !GTEST_OS_WINDOWS_MOBILE inline FILE *FReopen(const char* path, const char* mode, FILE* stream) { return freopen(path, mode, stream); } inline FILE* FDOpen(int fd, const char* mode) { return fdopen(fd, mode); } #endif inline int FClose(FILE* fp) { return fclose(fp); } #if !GTEST_OS_WINDOWS_MOBILE inline int Read(int fd, void* buf, unsigned int count) { return static_cast(read(fd, buf, count)); } inline int Write(int fd, const void* buf, unsigned int count) { return static_cast(write(fd, buf, count)); } inline int Close(int fd) { return close(fd); } inline const char* StrError(int errnum) { return strerror(errnum); } #endif inline const char* GetEnv(const char* name) { #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT // We are on Windows CE, which has no environment variables. static_cast(name); // To prevent 'unused argument' warning. return nullptr; #elif defined(__BORLANDC__) || defined(__SunOS_5_8) || defined(__SunOS_5_9) // Environment variables which we programmatically clear will be set to the // empty string rather than unset (NULL). Handle that case. const char* const env = getenv(name); return (env != nullptr && env[0] != '\0') ? env : nullptr; #else return getenv(name); #endif } GTEST_DISABLE_MSC_DEPRECATED_POP_() #if GTEST_OS_WINDOWS_MOBILE // Windows CE has no C library. The abort() function is used in // several places in Google Test. This implementation provides a reasonable // imitation of standard behaviour. [[noreturn]] void Abort(); #else [[noreturn]] inline void Abort() { abort(); } #endif // GTEST_OS_WINDOWS_MOBILE } // namespace posix // MSVC "deprecates" snprintf and issues warnings wherever it is used. In // order to avoid these warnings, we need to use _snprintf or _snprintf_s on // MSVC-based platforms. We map the GTEST_SNPRINTF_ macro to the appropriate // function in order to achieve that. We use macro definition here because // snprintf is a variadic function. #if _MSC_VER && !GTEST_OS_WINDOWS_MOBILE // MSVC 2005 and above support variadic macros. # define GTEST_SNPRINTF_(buffer, size, format, ...) \ _snprintf_s(buffer, size, size, format, __VA_ARGS__) #elif defined(_MSC_VER) // Windows CE does not define _snprintf_s # define GTEST_SNPRINTF_ _snprintf #else # define GTEST_SNPRINTF_ snprintf #endif // The maximum number a BiggestInt can represent. This definition // works no matter BiggestInt is represented in one's complement or // two's complement. // // We cannot rely on numeric_limits in STL, as __int64 and long long // are not part of standard C++ and numeric_limits doesn't need to be // defined for them. const BiggestInt kMaxBiggestInt = ~(static_cast(1) << (8*sizeof(BiggestInt) - 1)); // This template class serves as a compile-time function from size to // type. It maps a size in bytes to a primitive type with that // size. e.g. // // TypeWithSize<4>::UInt // // is typedef-ed to be unsigned int (unsigned integer made up of 4 // bytes). // // Such functionality should belong to STL, but I cannot find it // there. // // Google Test uses this class in the implementation of floating-point // comparison. // // For now it only handles UInt (unsigned int) as that's all Google Test // needs. Other types can be easily added in the future if need // arises. template class TypeWithSize { public: // This prevents the user from using TypeWithSize with incorrect // values of N. typedef void UInt; }; // The specialization for size 4. template <> class TypeWithSize<4> { public: // unsigned int has size 4 in both gcc and MSVC. // // As base/basictypes.h doesn't compile on Windows, we cannot use // uint32, uint64, and etc here. typedef int Int; typedef unsigned int UInt; }; // The specialization for size 8. template <> class TypeWithSize<8> { public: #if GTEST_OS_WINDOWS typedef __int64 Int; typedef unsigned __int64 UInt; #else typedef long long Int; // NOLINT typedef unsigned long long UInt; // NOLINT #endif // GTEST_OS_WINDOWS }; // Integer types of known sizes. typedef TypeWithSize<4>::Int Int32; typedef TypeWithSize<4>::UInt UInt32; typedef TypeWithSize<8>::Int Int64; typedef TypeWithSize<8>::UInt UInt64; typedef TypeWithSize<8>::Int TimeInMillis; // Represents time in milliseconds. // Utilities for command line flags and environment variables. // Macro for referencing flags. #if !defined(GTEST_FLAG) # define GTEST_FLAG(name) FLAGS_gtest_##name #endif // !defined(GTEST_FLAG) #if !defined(GTEST_USE_OWN_FLAGFILE_FLAG_) # define GTEST_USE_OWN_FLAGFILE_FLAG_ 1 #endif // !defined(GTEST_USE_OWN_FLAGFILE_FLAG_) #if !defined(GTEST_DECLARE_bool_) # define GTEST_FLAG_SAVER_ ::testing::internal::GTestFlagSaver // Macros for declaring flags. # define GTEST_DECLARE_bool_(name) GTEST_API_ extern bool GTEST_FLAG(name) # define GTEST_DECLARE_int32_(name) \ GTEST_API_ extern ::testing::internal::Int32 GTEST_FLAG(name) # define GTEST_DECLARE_string_(name) \ GTEST_API_ extern ::std::string GTEST_FLAG(name) // Macros for defining flags. # define GTEST_DEFINE_bool_(name, default_val, doc) \ GTEST_API_ bool GTEST_FLAG(name) = (default_val) # define GTEST_DEFINE_int32_(name, default_val, doc) \ GTEST_API_ ::testing::internal::Int32 GTEST_FLAG(name) = (default_val) # define GTEST_DEFINE_string_(name, default_val, doc) \ GTEST_API_ ::std::string GTEST_FLAG(name) = (default_val) #endif // !defined(GTEST_DECLARE_bool_) // Thread annotations #if !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_) # define GTEST_EXCLUSIVE_LOCK_REQUIRED_(locks) # define GTEST_LOCK_EXCLUDED_(locks) #endif // !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_) // Parses 'str' for a 32-bit signed integer. If successful, writes the result // to *value and returns true; otherwise leaves *value unchanged and returns // false. bool ParseInt32(const Message& src_text, const char* str, Int32* value); // Parses a bool/Int32/string from the environment variable // corresponding to the given Google Test flag. bool BoolFromGTestEnv(const char* flag, bool default_val); GTEST_API_ Int32 Int32FromGTestEnv(const char* flag, Int32 default_val); std::string OutputFlagAlsoCheckEnvVar(); const char* StringFromGTestEnv(const char* flag, const char* default_val); } // namespace internal } // namespace testing #if !defined(GTEST_INTERNAL_DEPRECATED) // Internal Macro to mark an API deprecated, for googletest usage only // Usage: class GTEST_INTERNAL_DEPRECATED(message) MyClass or // GTEST_INTERNAL_DEPRECATED(message) myFunction(); Every usage of // a deprecated entity will trigger a warning when compiled with // `-Wdeprecated-declarations` option (clang, gcc, any __GNUC__ compiler). // For msvc /W3 option will need to be used // Note that for 'other' compilers this macro evaluates to nothing to prevent // compilations errors. #if defined(_MSC_VER) #define GTEST_INTERNAL_DEPRECATED(message) __declspec(deprecated(message)) #elif defined(__GNUC__) #define GTEST_INTERNAL_DEPRECATED(message) __attribute__((deprecated(message))) #else #define GTEST_INTERNAL_DEPRECATED(message) #endif #endif // !defined(GTEST_INTERNAL_DEPRECATED) #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ #if GTEST_OS_LINUX # include # include # include # include #endif // GTEST_OS_LINUX #if GTEST_HAS_EXCEPTIONS # include #endif #include #include #include #include #include #include #include #include #include #include // Copyright 2005, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // The Google C++ Testing and Mocking Framework (Google Test) // // This header file defines the Message class. // // IMPORTANT NOTE: Due to limitation of the C++ language, we have to // leave some internal implementation details in this header file. // They are clearly marked by comments like this: // // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. // // Such code is NOT meant to be used by a user directly, and is subject // to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user // program! // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ #define GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ #include #include GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) // Ensures that there is at least one operator<< in the global namespace. // See Message& operator<<(...) below for why. void operator<<(const testing::internal::Secret&, int); namespace testing { // The Message class works like an ostream repeater. // // Typical usage: // // 1. You stream a bunch of values to a Message object. // It will remember the text in a stringstream. // 2. Then you stream the Message object to an ostream. // This causes the text in the Message to be streamed // to the ostream. // // For example; // // testing::Message foo; // foo << 1 << " != " << 2; // std::cout << foo; // // will print "1 != 2". // // Message is not intended to be inherited from. In particular, its // destructor is not virtual. // // Note that stringstream behaves differently in gcc and in MSVC. You // can stream a NULL char pointer to it in the former, but not in the // latter (it causes an access violation if you do). The Message // class hides this difference by treating a NULL char pointer as // "(null)". class GTEST_API_ Message { private: // The type of basic IO manipulators (endl, ends, and flush) for // narrow streams. typedef std::ostream& (*BasicNarrowIoManip)(std::ostream&); public: // Constructs an empty Message. Message(); // Copy constructor. Message(const Message& msg) : ss_(new ::std::stringstream) { // NOLINT *ss_ << msg.GetString(); } // Constructs a Message from a C-string. explicit Message(const char* str) : ss_(new ::std::stringstream) { *ss_ << str; } // Streams a non-pointer value to this object. template inline Message& operator <<(const T& val) { // Some libraries overload << for STL containers. These // overloads are defined in the global namespace instead of ::std. // // C++'s symbol lookup rule (i.e. Koenig lookup) says that these // overloads are visible in either the std namespace or the global // namespace, but not other namespaces, including the testing // namespace which Google Test's Message class is in. // // To allow STL containers (and other types that has a << operator // defined in the global namespace) to be used in Google Test // assertions, testing::Message must access the custom << operator // from the global namespace. With this using declaration, // overloads of << defined in the global namespace and those // visible via Koenig lookup are both exposed in this function. using ::operator <<; *ss_ << val; return *this; } // Streams a pointer value to this object. // // This function is an overload of the previous one. When you // stream a pointer to a Message, this definition will be used as it // is more specialized. (The C++ Standard, section // [temp.func.order].) If you stream a non-pointer, then the // previous definition will be used. // // The reason for this overload is that streaming a NULL pointer to // ostream is undefined behavior. Depending on the compiler, you // may get "0", "(nil)", "(null)", or an access violation. To // ensure consistent result across compilers, we always treat NULL // as "(null)". template inline Message& operator <<(T* const& pointer) { // NOLINT if (pointer == nullptr) { *ss_ << "(null)"; } else { *ss_ << pointer; } return *this; } // Since the basic IO manipulators are overloaded for both narrow // and wide streams, we have to provide this specialized definition // of operator <<, even though its body is the same as the // templatized version above. Without this definition, streaming // endl or other basic IO manipulators to Message will confuse the // compiler. Message& operator <<(BasicNarrowIoManip val) { *ss_ << val; return *this; } // Instead of 1/0, we want to see true/false for bool values. Message& operator <<(bool b) { return *this << (b ? "true" : "false"); } // These two overloads allow streaming a wide C string to a Message // using the UTF-8 encoding. Message& operator <<(const wchar_t* wide_c_str); Message& operator <<(wchar_t* wide_c_str); #if GTEST_HAS_STD_WSTRING // Converts the given wide string to a narrow string using the UTF-8 // encoding, and streams the result to this Message object. Message& operator <<(const ::std::wstring& wstr); #endif // GTEST_HAS_STD_WSTRING // Gets the text streamed to this object so far as an std::string. // Each '\0' character in the buffer is replaced with "\\0". // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. std::string GetString() const; private: // We'll hold the text streamed to this object here. const std::unique_ptr< ::std::stringstream> ss_; // We declare (but don't implement) this to prevent the compiler // from implementing the assignment operator. void operator=(const Message&); }; // Streams a Message to an ostream. inline std::ostream& operator <<(std::ostream& os, const Message& sb) { return os << sb.GetString(); } namespace internal { // Converts a streamable value to an std::string. A NULL pointer is // converted to "(null)". When the input value is a ::string, // ::std::string, ::wstring, or ::std::wstring object, each NUL // character in it is replaced with "\\0". template std::string StreamableToString(const T& streamable) { return (Message() << streamable).GetString(); } } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 #endif // GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ // Copyright 2008, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Google Test filepath utilities // // This header file declares classes and functions used internally by // Google Test. They are subject to change without notice. // // This file is #included in gtest/internal/gtest-internal.h. // Do not include this header file separately! // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ // Copyright 2005, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // The Google C++ Testing and Mocking Framework (Google Test) // // This header file declares the String class and functions used internally by // Google Test. They are subject to change without notice. They should not used // by code external to Google Test. // // This header file is #included by gtest-internal.h. // It should not be #included by other files. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ #ifdef __BORLANDC__ // string.h is not guaranteed to provide strcpy on C++ Builder. # include #endif #include #include namespace testing { namespace internal { // String - an abstract class holding static string utilities. class GTEST_API_ String { public: // Static utility methods // Clones a 0-terminated C string, allocating memory using new. The // caller is responsible for deleting the return value using // delete[]. Returns the cloned string, or NULL if the input is // NULL. // // This is different from strdup() in string.h, which allocates // memory using malloc(). static const char* CloneCString(const char* c_str); #if GTEST_OS_WINDOWS_MOBILE // Windows CE does not have the 'ANSI' versions of Win32 APIs. To be // able to pass strings to Win32 APIs on CE we need to convert them // to 'Unicode', UTF-16. // Creates a UTF-16 wide string from the given ANSI string, allocating // memory using new. The caller is responsible for deleting the return // value using delete[]. Returns the wide string, or NULL if the // input is NULL. // // The wide string is created using the ANSI codepage (CP_ACP) to // match the behaviour of the ANSI versions of Win32 calls and the // C runtime. static LPCWSTR AnsiToUtf16(const char* c_str); // Creates an ANSI string from the given wide string, allocating // memory using new. The caller is responsible for deleting the return // value using delete[]. Returns the ANSI string, or NULL if the // input is NULL. // // The returned string is created using the ANSI codepage (CP_ACP) to // match the behaviour of the ANSI versions of Win32 calls and the // C runtime. static const char* Utf16ToAnsi(LPCWSTR utf16_str); #endif // Compares two C strings. Returns true if and only if they have the same // content. // // Unlike strcmp(), this function can handle NULL argument(s). A // NULL C string is considered different to any non-NULL C string, // including the empty string. static bool CStringEquals(const char* lhs, const char* rhs); // Converts a wide C string to a String using the UTF-8 encoding. // NULL will be converted to "(null)". If an error occurred during // the conversion, "(failed to convert from wide string)" is // returned. static std::string ShowWideCString(const wchar_t* wide_c_str); // Compares two wide C strings. Returns true if and only if they have the // same content. // // Unlike wcscmp(), this function can handle NULL argument(s). A // NULL C string is considered different to any non-NULL C string, // including the empty string. static bool WideCStringEquals(const wchar_t* lhs, const wchar_t* rhs); // Compares two C strings, ignoring case. Returns true if and only if // they have the same content. // // Unlike strcasecmp(), this function can handle NULL argument(s). // A NULL C string is considered different to any non-NULL C string, // including the empty string. static bool CaseInsensitiveCStringEquals(const char* lhs, const char* rhs); // Compares two wide C strings, ignoring case. Returns true if and only if // they have the same content. // // Unlike wcscasecmp(), this function can handle NULL argument(s). // A NULL C string is considered different to any non-NULL wide C string, // including the empty string. // NB: The implementations on different platforms slightly differ. // On windows, this method uses _wcsicmp which compares according to LC_CTYPE // environment variable. On GNU platform this method uses wcscasecmp // which compares according to LC_CTYPE category of the current locale. // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the // current locale. static bool CaseInsensitiveWideCStringEquals(const wchar_t* lhs, const wchar_t* rhs); // Returns true if and only if the given string ends with the given suffix, // ignoring case. Any string is considered to end with an empty suffix. static bool EndsWithCaseInsensitive( const std::string& str, const std::string& suffix); // Formats an int value as "%02d". static std::string FormatIntWidth2(int value); // "%02d" for width == 2 // Formats an int value as "%X". static std::string FormatHexInt(int value); // Formats an int value as "%X". static std::string FormatHexUInt32(UInt32 value); // Formats a byte as "%02X". static std::string FormatByte(unsigned char value); private: String(); // Not meant to be instantiated. }; // class String // Gets the content of the stringstream's buffer as an std::string. Each '\0' // character in the buffer is replaced with "\\0". GTEST_API_ std::string StringStreamToString(::std::stringstream* stream); } // namespace internal } // namespace testing #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) namespace testing { namespace internal { // FilePath - a class for file and directory pathname manipulation which // handles platform-specific conventions (like the pathname separator). // Used for helper functions for naming files in a directory for xml output. // Except for Set methods, all methods are const or static, which provides an // "immutable value object" -- useful for peace of mind. // A FilePath with a value ending in a path separator ("like/this/") represents // a directory, otherwise it is assumed to represent a file. In either case, // it may or may not represent an actual file or directory in the file system. // Names are NOT checked for syntax correctness -- no checking for illegal // characters, malformed paths, etc. class GTEST_API_ FilePath { public: FilePath() : pathname_("") { } FilePath(const FilePath& rhs) : pathname_(rhs.pathname_) { } explicit FilePath(const std::string& pathname) : pathname_(pathname) { Normalize(); } FilePath& operator=(const FilePath& rhs) { Set(rhs); return *this; } void Set(const FilePath& rhs) { pathname_ = rhs.pathname_; } const std::string& string() const { return pathname_; } const char* c_str() const { return pathname_.c_str(); } // Returns the current working directory, or "" if unsuccessful. static FilePath GetCurrentDir(); // Given directory = "dir", base_name = "test", number = 0, // extension = "xml", returns "dir/test.xml". If number is greater // than zero (e.g., 12), returns "dir/test_12.xml". // On Windows platform, uses \ as the separator rather than /. static FilePath MakeFileName(const FilePath& directory, const FilePath& base_name, int number, const char* extension); // Given directory = "dir", relative_path = "test.xml", // returns "dir/test.xml". // On Windows, uses \ as the separator rather than /. static FilePath ConcatPaths(const FilePath& directory, const FilePath& relative_path); // Returns a pathname for a file that does not currently exist. The pathname // will be directory/base_name.extension or // directory/base_name_.extension if directory/base_name.extension // already exists. The number will be incremented until a pathname is found // that does not already exist. // Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'. // There could be a race condition if two or more processes are calling this // function at the same time -- they could both pick the same filename. static FilePath GenerateUniqueFileName(const FilePath& directory, const FilePath& base_name, const char* extension); // Returns true if and only if the path is "". bool IsEmpty() const { return pathname_.empty(); } // If input name has a trailing separator character, removes it and returns // the name, otherwise return the name string unmodified. // On Windows platform, uses \ as the separator, other platforms use /. FilePath RemoveTrailingPathSeparator() const; // Returns a copy of the FilePath with the directory part removed. // Example: FilePath("path/to/file").RemoveDirectoryName() returns // FilePath("file"). If there is no directory part ("just_a_file"), it returns // the FilePath unmodified. If there is no file part ("just_a_dir/") it // returns an empty FilePath (""). // On Windows platform, '\' is the path separator, otherwise it is '/'. FilePath RemoveDirectoryName() const; // RemoveFileName returns the directory path with the filename removed. // Example: FilePath("path/to/file").RemoveFileName() returns "path/to/". // If the FilePath is "a_file" or "/a_file", RemoveFileName returns // FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does // not have a file, like "just/a/dir/", it returns the FilePath unmodified. // On Windows platform, '\' is the path separator, otherwise it is '/'. FilePath RemoveFileName() const; // Returns a copy of the FilePath with the case-insensitive extension removed. // Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns // FilePath("dir/file"). If a case-insensitive extension is not // found, returns a copy of the original FilePath. FilePath RemoveExtension(const char* extension) const; // Creates directories so that path exists. Returns true if successful or if // the directories already exist; returns false if unable to create // directories for any reason. Will also return false if the FilePath does // not represent a directory (that is, it doesn't end with a path separator). bool CreateDirectoriesRecursively() const; // Create the directory so that path exists. Returns true if successful or // if the directory already exists; returns false if unable to create the // directory for any reason, including if the parent directory does not // exist. Not named "CreateDirectory" because that's a macro on Windows. bool CreateFolder() const; // Returns true if FilePath describes something in the file-system, // either a file, directory, or whatever, and that something exists. bool FileOrDirectoryExists() const; // Returns true if pathname describes a directory in the file-system // that exists. bool DirectoryExists() const; // Returns true if FilePath ends with a path separator, which indicates that // it is intended to represent a directory. Returns false otherwise. // This does NOT check that a directory (or file) actually exists. bool IsDirectory() const; // Returns true if pathname describes a root directory. (Windows has one // root directory per disk drive.) bool IsRootDirectory() const; // Returns true if pathname describes an absolute path. bool IsAbsolutePath() const; private: // Replaces multiple consecutive separators with a single separator. // For example, "bar///foo" becomes "bar/foo". Does not eliminate other // redundancies that might be in a pathname involving "." or "..". // // A pathname with multiple consecutive separators may occur either through // user error or as a result of some scripts or APIs that generate a pathname // with a trailing separator. On other platforms the same API or script // may NOT generate a pathname with a trailing "/". Then elsewhere that // pathname may have another "/" and pathname components added to it, // without checking for the separator already being there. // The script language and operating system may allow paths like "foo//bar" // but some of the functions in FilePath will not handle that correctly. In // particular, RemoveTrailingPathSeparator() only removes one separator, and // it is called in CreateDirectoriesRecursively() assuming that it will change // a pathname from directory syntax (trailing separator) to filename syntax. // // On Windows this method also replaces the alternate path separator '/' with // the primary path separator '\\', so that for example "bar\\/\\foo" becomes // "bar\\foo". void Normalize(); // Returns a pointer to the last occurence of a valid path separator in // the FilePath. On Windows, for example, both '/' and '\' are valid path // separators. Returns NULL if no path separator was found. const char* FindLastPathSeparator() const; std::string pathname_; }; // class FilePath } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ // This file was GENERATED by command: // pump.py gtest-type-util.h.pump // DO NOT EDIT BY HAND!!! // Copyright 2008 Google Inc. // All Rights Reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Type utilities needed for implementing typed and type-parameterized // tests. This file is generated by a SCRIPT. DO NOT EDIT BY HAND! // // Currently we support at most 50 types in a list, and at most 50 // type-parameterized tests in one type-parameterized test suite. // Please contact googletestframework@googlegroups.com if you need // more. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ // #ifdef __GNUC__ is too general here. It is possible to use gcc without using // libstdc++ (which is where cxxabi.h comes from). # if GTEST_HAS_CXXABI_H_ # include # elif defined(__HP_aCC) # include # endif // GTEST_HASH_CXXABI_H_ namespace testing { namespace internal { // Canonicalizes a given name with respect to the Standard C++ Library. // This handles removing the inline namespace within `std` that is // used by various standard libraries (e.g., `std::__1`). Names outside // of namespace std are returned unmodified. inline std::string CanonicalizeForStdLibVersioning(std::string s) { static const char prefix[] = "std::__"; if (s.compare(0, strlen(prefix), prefix) == 0) { std::string::size_type end = s.find("::", strlen(prefix)); if (end != s.npos) { // Erase everything between the initial `std` and the second `::`. s.erase(strlen("std"), end - strlen("std")); } } return s; } // GetTypeName() returns a human-readable name of type T. // NB: This function is also used in Google Mock, so don't move it inside of // the typed-test-only section below. template std::string GetTypeName() { # if GTEST_HAS_RTTI const char* const name = typeid(T).name(); # if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) int status = 0; // gcc's implementation of typeid(T).name() mangles the type name, // so we have to demangle it. # if GTEST_HAS_CXXABI_H_ using abi::__cxa_demangle; # endif // GTEST_HAS_CXXABI_H_ char* const readable_name = __cxa_demangle(name, nullptr, nullptr, &status); const std::string name_str(status == 0 ? readable_name : name); free(readable_name); return CanonicalizeForStdLibVersioning(name_str); # else return name; # endif // GTEST_HAS_CXXABI_H_ || __HP_aCC # else return ""; # endif // GTEST_HAS_RTTI } #if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P // A unique type used as the default value for the arguments of class // template Types. This allows us to simulate variadic templates // (e.g. Types, Type, and etc), which C++ doesn't // support directly. struct None {}; // The following family of struct and struct templates are used to // represent type lists. In particular, TypesN // represents a type list with N types (T1, T2, ..., and TN) in it. // Except for Types0, every struct in the family has two member types: // Head for the first type in the list, and Tail for the rest of the // list. // The empty type list. struct Types0 {}; // Type lists of length 1, 2, 3, and so on. template struct Types1 { typedef T1 Head; typedef Types0 Tail; }; template struct Types2 { typedef T1 Head; typedef Types1 Tail; }; template struct Types3 { typedef T1 Head; typedef Types2 Tail; }; template struct Types4 { typedef T1 Head; typedef Types3 Tail; }; template struct Types5 { typedef T1 Head; typedef Types4 Tail; }; template struct Types6 { typedef T1 Head; typedef Types5 Tail; }; template struct Types7 { typedef T1 Head; typedef Types6 Tail; }; template struct Types8 { typedef T1 Head; typedef Types7 Tail; }; template struct Types9 { typedef T1 Head; typedef Types8 Tail; }; template struct Types10 { typedef T1 Head; typedef Types9 Tail; }; template struct Types11 { typedef T1 Head; typedef Types10 Tail; }; template struct Types12 { typedef T1 Head; typedef Types11 Tail; }; template struct Types13 { typedef T1 Head; typedef Types12 Tail; }; template struct Types14 { typedef T1 Head; typedef Types13 Tail; }; template struct Types15 { typedef T1 Head; typedef Types14 Tail; }; template struct Types16 { typedef T1 Head; typedef Types15 Tail; }; template struct Types17 { typedef T1 Head; typedef Types16 Tail; }; template struct Types18 { typedef T1 Head; typedef Types17 Tail; }; template struct Types19 { typedef T1 Head; typedef Types18 Tail; }; template struct Types20 { typedef T1 Head; typedef Types19 Tail; }; template struct Types21 { typedef T1 Head; typedef Types20 Tail; }; template struct Types22 { typedef T1 Head; typedef Types21 Tail; }; template struct Types23 { typedef T1 Head; typedef Types22 Tail; }; template struct Types24 { typedef T1 Head; typedef Types23 Tail; }; template struct Types25 { typedef T1 Head; typedef Types24 Tail; }; template struct Types26 { typedef T1 Head; typedef Types25 Tail; }; template struct Types27 { typedef T1 Head; typedef Types26 Tail; }; template struct Types28 { typedef T1 Head; typedef Types27 Tail; }; template struct Types29 { typedef T1 Head; typedef Types28 Tail; }; template struct Types30 { typedef T1 Head; typedef Types29 Tail; }; template struct Types31 { typedef T1 Head; typedef Types30 Tail; }; template struct Types32 { typedef T1 Head; typedef Types31 Tail; }; template struct Types33 { typedef T1 Head; typedef Types32 Tail; }; template struct Types34 { typedef T1 Head; typedef Types33 Tail; }; template struct Types35 { typedef T1 Head; typedef Types34 Tail; }; template struct Types36 { typedef T1 Head; typedef Types35 Tail; }; template struct Types37 { typedef T1 Head; typedef Types36 Tail; }; template struct Types38 { typedef T1 Head; typedef Types37 Tail; }; template struct Types39 { typedef T1 Head; typedef Types38 Tail; }; template struct Types40 { typedef T1 Head; typedef Types39 Tail; }; template struct Types41 { typedef T1 Head; typedef Types40 Tail; }; template struct Types42 { typedef T1 Head; typedef Types41 Tail; }; template struct Types43 { typedef T1 Head; typedef Types42 Tail; }; template struct Types44 { typedef T1 Head; typedef Types43 Tail; }; template struct Types45 { typedef T1 Head; typedef Types44 Tail; }; template struct Types46 { typedef T1 Head; typedef Types45 Tail; }; template struct Types47 { typedef T1 Head; typedef Types46 Tail; }; template struct Types48 { typedef T1 Head; typedef Types47 Tail; }; template struct Types49 { typedef T1 Head; typedef Types48 Tail; }; template struct Types50 { typedef T1 Head; typedef Types49 Tail; }; } // namespace internal // We don't want to require the users to write TypesN<...> directly, // as that would require them to count the length. Types<...> is much // easier to write, but generates horrible messages when there is a // compiler error, as gcc insists on printing out each template // argument, even if it has the default value (this means Types // will appear as Types in the compiler // errors). // // Our solution is to combine the best part of the two approaches: a // user would write Types, and Google Test will translate // that to TypesN internally to make error messages // readable. The translation is done by the 'type' member of the // Types template. template struct Types { typedef internal::Types50 type; }; template <> struct Types { typedef internal::Types0 type; }; template struct Types { typedef internal::Types1 type; }; template struct Types { typedef internal::Types2 type; }; template struct Types { typedef internal::Types3 type; }; template struct Types { typedef internal::Types4 type; }; template struct Types { typedef internal::Types5 type; }; template struct Types { typedef internal::Types6 type; }; template struct Types { typedef internal::Types7 type; }; template struct Types { typedef internal::Types8 type; }; template struct Types { typedef internal::Types9 type; }; template struct Types { typedef internal::Types10 type; }; template struct Types { typedef internal::Types11 type; }; template struct Types { typedef internal::Types12 type; }; template struct Types { typedef internal::Types13 type; }; template struct Types { typedef internal::Types14 type; }; template struct Types { typedef internal::Types15 type; }; template struct Types { typedef internal::Types16 type; }; template struct Types { typedef internal::Types17 type; }; template struct Types { typedef internal::Types18 type; }; template struct Types { typedef internal::Types19 type; }; template struct Types { typedef internal::Types20 type; }; template struct Types { typedef internal::Types21 type; }; template struct Types { typedef internal::Types22 type; }; template struct Types { typedef internal::Types23 type; }; template struct Types { typedef internal::Types24 type; }; template struct Types { typedef internal::Types25 type; }; template struct Types { typedef internal::Types26 type; }; template struct Types { typedef internal::Types27 type; }; template struct Types { typedef internal::Types28 type; }; template struct Types { typedef internal::Types29 type; }; template struct Types { typedef internal::Types30 type; }; template struct Types { typedef internal::Types31 type; }; template struct Types { typedef internal::Types32 type; }; template struct Types { typedef internal::Types33 type; }; template struct Types { typedef internal::Types34 type; }; template struct Types { typedef internal::Types35 type; }; template struct Types { typedef internal::Types36 type; }; template struct Types { typedef internal::Types37 type; }; template struct Types { typedef internal::Types38 type; }; template struct Types { typedef internal::Types39 type; }; template struct Types { typedef internal::Types40 type; }; template struct Types { typedef internal::Types41 type; }; template struct Types { typedef internal::Types42 type; }; template struct Types { typedef internal::Types43 type; }; template struct Types { typedef internal::Types44 type; }; template struct Types { typedef internal::Types45 type; }; template struct Types { typedef internal::Types46 type; }; template struct Types { typedef internal::Types47 type; }; template struct Types { typedef internal::Types48 type; }; template struct Types { typedef internal::Types49 type; }; namespace internal { # define GTEST_TEMPLATE_ template class // The template "selector" struct TemplateSel is used to // represent Tmpl, which must be a class template with one type // parameter, as a type. TemplateSel::Bind::type is defined // as the type Tmpl. This allows us to actually instantiate the // template "selected" by TemplateSel. // // This trick is necessary for simulating typedef for class templates, // which C++ doesn't support directly. template struct TemplateSel { template struct Bind { typedef Tmpl type; }; }; # define GTEST_BIND_(TmplSel, T) \ TmplSel::template Bind::type // A unique struct template used as the default value for the // arguments of class template Templates. This allows us to simulate // variadic templates (e.g. Templates, Templates, // and etc), which C++ doesn't support directly. template struct NoneT {}; // The following family of struct and struct templates are used to // represent template lists. In particular, TemplatesN represents a list of N templates (T1, T2, ..., and TN). Except // for Templates0, every struct in the family has two member types: // Head for the selector of the first template in the list, and Tail // for the rest of the list. // The empty template list. struct Templates0 {}; // Template lists of length 1, 2, 3, and so on. template struct Templates1 { typedef TemplateSel Head; typedef Templates0 Tail; }; template struct Templates2 { typedef TemplateSel Head; typedef Templates1 Tail; }; template struct Templates3 { typedef TemplateSel Head; typedef Templates2 Tail; }; template struct Templates4 { typedef TemplateSel Head; typedef Templates3 Tail; }; template struct Templates5 { typedef TemplateSel Head; typedef Templates4 Tail; }; template struct Templates6 { typedef TemplateSel Head; typedef Templates5 Tail; }; template struct Templates7 { typedef TemplateSel Head; typedef Templates6 Tail; }; template struct Templates8 { typedef TemplateSel Head; typedef Templates7 Tail; }; template struct Templates9 { typedef TemplateSel Head; typedef Templates8 Tail; }; template struct Templates10 { typedef TemplateSel Head; typedef Templates9 Tail; }; template struct Templates11 { typedef TemplateSel Head; typedef Templates10 Tail; }; template struct Templates12 { typedef TemplateSel Head; typedef Templates11 Tail; }; template struct Templates13 { typedef TemplateSel Head; typedef Templates12 Tail; }; template struct Templates14 { typedef TemplateSel Head; typedef Templates13 Tail; }; template struct Templates15 { typedef TemplateSel Head; typedef Templates14 Tail; }; template struct Templates16 { typedef TemplateSel Head; typedef Templates15 Tail; }; template struct Templates17 { typedef TemplateSel Head; typedef Templates16 Tail; }; template struct Templates18 { typedef TemplateSel Head; typedef Templates17 Tail; }; template struct Templates19 { typedef TemplateSel Head; typedef Templates18 Tail; }; template struct Templates20 { typedef TemplateSel Head; typedef Templates19 Tail; }; template struct Templates21 { typedef TemplateSel Head; typedef Templates20 Tail; }; template struct Templates22 { typedef TemplateSel Head; typedef Templates21 Tail; }; template struct Templates23 { typedef TemplateSel Head; typedef Templates22 Tail; }; template struct Templates24 { typedef TemplateSel Head; typedef Templates23 Tail; }; template struct Templates25 { typedef TemplateSel Head; typedef Templates24 Tail; }; template struct Templates26 { typedef TemplateSel Head; typedef Templates25 Tail; }; template struct Templates27 { typedef TemplateSel Head; typedef Templates26 Tail; }; template struct Templates28 { typedef TemplateSel Head; typedef Templates27 Tail; }; template struct Templates29 { typedef TemplateSel Head; typedef Templates28 Tail; }; template struct Templates30 { typedef TemplateSel Head; typedef Templates29 Tail; }; template struct Templates31 { typedef TemplateSel Head; typedef Templates30 Tail; }; template struct Templates32 { typedef TemplateSel Head; typedef Templates31 Tail; }; template struct Templates33 { typedef TemplateSel Head; typedef Templates32 Tail; }; template struct Templates34 { typedef TemplateSel Head; typedef Templates33 Tail; }; template struct Templates35 { typedef TemplateSel Head; typedef Templates34 Tail; }; template struct Templates36 { typedef TemplateSel Head; typedef Templates35 Tail; }; template struct Templates37 { typedef TemplateSel Head; typedef Templates36 Tail; }; template struct Templates38 { typedef TemplateSel Head; typedef Templates37 Tail; }; template struct Templates39 { typedef TemplateSel Head; typedef Templates38 Tail; }; template struct Templates40 { typedef TemplateSel Head; typedef Templates39 Tail; }; template struct Templates41 { typedef TemplateSel Head; typedef Templates40 Tail; }; template struct Templates42 { typedef TemplateSel Head; typedef Templates41 Tail; }; template struct Templates43 { typedef TemplateSel Head; typedef Templates42 Tail; }; template struct Templates44 { typedef TemplateSel Head; typedef Templates43 Tail; }; template struct Templates45 { typedef TemplateSel Head; typedef Templates44 Tail; }; template struct Templates46 { typedef TemplateSel Head; typedef Templates45 Tail; }; template struct Templates47 { typedef TemplateSel Head; typedef Templates46 Tail; }; template struct Templates48 { typedef TemplateSel Head; typedef Templates47 Tail; }; template struct Templates49 { typedef TemplateSel Head; typedef Templates48 Tail; }; template struct Templates50 { typedef TemplateSel Head; typedef Templates49 Tail; }; // We don't want to require the users to write TemplatesN<...> directly, // as that would require them to count the length. Templates<...> is much // easier to write, but generates horrible messages when there is a // compiler error, as gcc insists on printing out each template // argument, even if it has the default value (this means Templates // will appear as Templates in the compiler // errors). // // Our solution is to combine the best part of the two approaches: a // user would write Templates, and Google Test will translate // that to TemplatesN internally to make error messages // readable. The translation is done by the 'type' member of the // Templates template. template struct Templates { typedef Templates50 type; }; template <> struct Templates { typedef Templates0 type; }; template struct Templates { typedef Templates1 type; }; template struct Templates { typedef Templates2 type; }; template struct Templates { typedef Templates3 type; }; template struct Templates { typedef Templates4 type; }; template struct Templates { typedef Templates5 type; }; template struct Templates { typedef Templates6 type; }; template struct Templates { typedef Templates7 type; }; template struct Templates { typedef Templates8 type; }; template struct Templates { typedef Templates9 type; }; template struct Templates { typedef Templates10 type; }; template struct Templates { typedef Templates11 type; }; template struct Templates { typedef Templates12 type; }; template struct Templates { typedef Templates13 type; }; template struct Templates { typedef Templates14 type; }; template struct Templates { typedef Templates15 type; }; template struct Templates { typedef Templates16 type; }; template struct Templates { typedef Templates17 type; }; template struct Templates { typedef Templates18 type; }; template struct Templates { typedef Templates19 type; }; template struct Templates { typedef Templates20 type; }; template struct Templates { typedef Templates21 type; }; template struct Templates { typedef Templates22 type; }; template struct Templates { typedef Templates23 type; }; template struct Templates { typedef Templates24 type; }; template struct Templates { typedef Templates25 type; }; template struct Templates { typedef Templates26 type; }; template struct Templates { typedef Templates27 type; }; template struct Templates { typedef Templates28 type; }; template struct Templates { typedef Templates29 type; }; template struct Templates { typedef Templates30 type; }; template struct Templates { typedef Templates31 type; }; template struct Templates { typedef Templates32 type; }; template struct Templates { typedef Templates33 type; }; template struct Templates { typedef Templates34 type; }; template struct Templates { typedef Templates35 type; }; template struct Templates { typedef Templates36 type; }; template struct Templates { typedef Templates37 type; }; template struct Templates { typedef Templates38 type; }; template struct Templates { typedef Templates39 type; }; template struct Templates { typedef Templates40 type; }; template struct Templates { typedef Templates41 type; }; template struct Templates { typedef Templates42 type; }; template struct Templates { typedef Templates43 type; }; template struct Templates { typedef Templates44 type; }; template struct Templates { typedef Templates45 type; }; template struct Templates { typedef Templates46 type; }; template struct Templates { typedef Templates47 type; }; template struct Templates { typedef Templates48 type; }; template struct Templates { typedef Templates49 type; }; // The TypeList template makes it possible to use either a single type // or a Types<...> list in TYPED_TEST_SUITE() and // INSTANTIATE_TYPED_TEST_SUITE_P(). template struct TypeList { typedef Types1 type; }; template struct TypeList > { typedef typename Types::type type; }; #endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P } // namespace internal } // namespace testing #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ // Due to C++ preprocessor weirdness, we need double indirection to // concatenate two tokens when one of them is __LINE__. Writing // // foo ## __LINE__ // // will result in the token foo__LINE__, instead of foo followed by // the current line number. For more details, see // http://www.parashift.com/c++-faq-lite/misc-technical-issues.html#faq-39.6 #define GTEST_CONCAT_TOKEN_(foo, bar) GTEST_CONCAT_TOKEN_IMPL_(foo, bar) #define GTEST_CONCAT_TOKEN_IMPL_(foo, bar) foo ## bar // Stringifies its argument. #define GTEST_STRINGIFY_(name) #name namespace proto2 { class Message; } namespace testing { // Forward declarations. class AssertionResult; // Result of an assertion. class Message; // Represents a failure message. class Test; // Represents a test. class TestInfo; // Information about a test. class TestPartResult; // Result of a test part. class UnitTest; // A collection of test suites. template ::std::string PrintToString(const T& value); namespace internal { struct TraceInfo; // Information about a trace point. class TestInfoImpl; // Opaque implementation of TestInfo class UnitTestImpl; // Opaque implementation of UnitTest // The text used in failure messages to indicate the start of the // stack trace. GTEST_API_ extern const char kStackTraceMarker[]; // An IgnoredValue object can be implicitly constructed from ANY value. class IgnoredValue { struct Sink {}; public: // This constructor template allows any value to be implicitly // converted to IgnoredValue. The object has no data member and // doesn't try to remember anything about the argument. We // deliberately omit the 'explicit' keyword in order to allow the // conversion to be implicit. // Disable the conversion if T already has a magical conversion operator. // Otherwise we get ambiguity. template ::value, int>::type = 0> IgnoredValue(const T& /* ignored */) {} // NOLINT(runtime/explicit) }; // Appends the user-supplied message to the Google-Test-generated message. GTEST_API_ std::string AppendUserMessage( const std::string& gtest_msg, const Message& user_msg); #if GTEST_HAS_EXCEPTIONS GTEST_DISABLE_MSC_WARNINGS_PUSH_(4275 \ /* an exported class was derived from a class that was not exported */) // This exception is thrown by (and only by) a failed Google Test // assertion when GTEST_FLAG(throw_on_failure) is true (if exceptions // are enabled). We derive it from std::runtime_error, which is for // errors presumably detectable only at run time. Since // std::runtime_error inherits from std::exception, many testing // frameworks know how to extract and print the message inside it. class GTEST_API_ GoogleTestFailureException : public ::std::runtime_error { public: explicit GoogleTestFailureException(const TestPartResult& failure); }; GTEST_DISABLE_MSC_WARNINGS_POP_() // 4275 #endif // GTEST_HAS_EXCEPTIONS namespace edit_distance { // Returns the optimal edits to go from 'left' to 'right'. // All edits cost the same, with replace having lower priority than // add/remove. // Simple implementation of the Wagner-Fischer algorithm. // See http://en.wikipedia.org/wiki/Wagner-Fischer_algorithm enum EditType { kMatch, kAdd, kRemove, kReplace }; GTEST_API_ std::vector CalculateOptimalEdits( const std::vector& left, const std::vector& right); // Same as above, but the input is represented as strings. GTEST_API_ std::vector CalculateOptimalEdits( const std::vector& left, const std::vector& right); // Create a diff of the input strings in Unified diff format. GTEST_API_ std::string CreateUnifiedDiff(const std::vector& left, const std::vector& right, size_t context = 2); } // namespace edit_distance // Calculate the diff between 'left' and 'right' and return it in unified diff // format. // If not null, stores in 'total_line_count' the total number of lines found // in left + right. GTEST_API_ std::string DiffStrings(const std::string& left, const std::string& right, size_t* total_line_count); // Constructs and returns the message for an equality assertion // (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. // // The first four parameters are the expressions used in the assertion // and their values, as strings. For example, for ASSERT_EQ(foo, bar) // where foo is 5 and bar is 6, we have: // // expected_expression: "foo" // actual_expression: "bar" // expected_value: "5" // actual_value: "6" // // The ignoring_case parameter is true if and only if the assertion is a // *_STRCASEEQ*. When it's true, the string " (ignoring case)" will // be inserted into the message. GTEST_API_ AssertionResult EqFailure(const char* expected_expression, const char* actual_expression, const std::string& expected_value, const std::string& actual_value, bool ignoring_case); // Constructs a failure message for Boolean assertions such as EXPECT_TRUE. GTEST_API_ std::string GetBoolAssertionFailureMessage( const AssertionResult& assertion_result, const char* expression_text, const char* actual_predicate_value, const char* expected_predicate_value); // This template class represents an IEEE floating-point number // (either single-precision or double-precision, depending on the // template parameters). // // The purpose of this class is to do more sophisticated number // comparison. (Due to round-off error, etc, it's very unlikely that // two floating-points will be equal exactly. Hence a naive // comparison by the == operation often doesn't work.) // // Format of IEEE floating-point: // // The most-significant bit being the leftmost, an IEEE // floating-point looks like // // sign_bit exponent_bits fraction_bits // // Here, sign_bit is a single bit that designates the sign of the // number. // // For float, there are 8 exponent bits and 23 fraction bits. // // For double, there are 11 exponent bits and 52 fraction bits. // // More details can be found at // http://en.wikipedia.org/wiki/IEEE_floating-point_standard. // // Template parameter: // // RawType: the raw floating-point type (either float or double) template class FloatingPoint { public: // Defines the unsigned integer type that has the same size as the // floating point number. typedef typename TypeWithSize::UInt Bits; // Constants. // # of bits in a number. static const size_t kBitCount = 8*sizeof(RawType); // # of fraction bits in a number. static const size_t kFractionBitCount = std::numeric_limits::digits - 1; // # of exponent bits in a number. static const size_t kExponentBitCount = kBitCount - 1 - kFractionBitCount; // The mask for the sign bit. static const Bits kSignBitMask = static_cast(1) << (kBitCount - 1); // The mask for the fraction bits. static const Bits kFractionBitMask = ~static_cast(0) >> (kExponentBitCount + 1); // The mask for the exponent bits. static const Bits kExponentBitMask = ~(kSignBitMask | kFractionBitMask); // How many ULP's (Units in the Last Place) we want to tolerate when // comparing two numbers. The larger the value, the more error we // allow. A 0 value means that two numbers must be exactly the same // to be considered equal. // // The maximum error of a single floating-point operation is 0.5 // units in the last place. On Intel CPU's, all floating-point // calculations are done with 80-bit precision, while double has 64 // bits. Therefore, 4 should be enough for ordinary use. // // See the following article for more details on ULP: // http://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/ static const size_t kMaxUlps = 4; // Constructs a FloatingPoint from a raw floating-point number. // // On an Intel CPU, passing a non-normalized NAN (Not a Number) // around may change its bits, although the new value is guaranteed // to be also a NAN. Therefore, don't expect this constructor to // preserve the bits in x when x is a NAN. explicit FloatingPoint(const RawType& x) { u_.value_ = x; } // Static methods // Reinterprets a bit pattern as a floating-point number. // // This function is needed to test the AlmostEquals() method. static RawType ReinterpretBits(const Bits bits) { FloatingPoint fp(0); fp.u_.bits_ = bits; return fp.u_.value_; } // Returns the floating-point number that represent positive infinity. static RawType Infinity() { return ReinterpretBits(kExponentBitMask); } // Returns the maximum representable finite floating-point number. static RawType Max(); // Non-static methods // Returns the bits that represents this number. const Bits &bits() const { return u_.bits_; } // Returns the exponent bits of this number. Bits exponent_bits() const { return kExponentBitMask & u_.bits_; } // Returns the fraction bits of this number. Bits fraction_bits() const { return kFractionBitMask & u_.bits_; } // Returns the sign bit of this number. Bits sign_bit() const { return kSignBitMask & u_.bits_; } // Returns true if and only if this is NAN (not a number). bool is_nan() const { // It's a NAN if the exponent bits are all ones and the fraction // bits are not entirely zeros. return (exponent_bits() == kExponentBitMask) && (fraction_bits() != 0); } // Returns true if and only if this number is at most kMaxUlps ULP's away // from rhs. In particular, this function: // // - returns false if either number is (or both are) NAN. // - treats really large numbers as almost equal to infinity. // - thinks +0.0 and -0.0 are 0 DLP's apart. bool AlmostEquals(const FloatingPoint& rhs) const { // The IEEE standard says that any comparison operation involving // a NAN must return false. if (is_nan() || rhs.is_nan()) return false; return DistanceBetweenSignAndMagnitudeNumbers(u_.bits_, rhs.u_.bits_) <= kMaxUlps; } private: // The data type used to store the actual floating-point number. union FloatingPointUnion { RawType value_; // The raw floating-point number. Bits bits_; // The bits that represent the number. }; // Converts an integer from the sign-and-magnitude representation to // the biased representation. More precisely, let N be 2 to the // power of (kBitCount - 1), an integer x is represented by the // unsigned number x + N. // // For instance, // // -N + 1 (the most negative number representable using // sign-and-magnitude) is represented by 1; // 0 is represented by N; and // N - 1 (the biggest number representable using // sign-and-magnitude) is represented by 2N - 1. // // Read http://en.wikipedia.org/wiki/Signed_number_representations // for more details on signed number representations. static Bits SignAndMagnitudeToBiased(const Bits &sam) { if (kSignBitMask & sam) { // sam represents a negative number. return ~sam + 1; } else { // sam represents a positive number. return kSignBitMask | sam; } } // Given two numbers in the sign-and-magnitude representation, // returns the distance between them as an unsigned number. static Bits DistanceBetweenSignAndMagnitudeNumbers(const Bits &sam1, const Bits &sam2) { const Bits biased1 = SignAndMagnitudeToBiased(sam1); const Bits biased2 = SignAndMagnitudeToBiased(sam2); return (biased1 >= biased2) ? (biased1 - biased2) : (biased2 - biased1); } FloatingPointUnion u_; }; // We cannot use std::numeric_limits::max() as it clashes with the max() // macro defined by . template <> inline float FloatingPoint::Max() { return FLT_MAX; } template <> inline double FloatingPoint::Max() { return DBL_MAX; } // Typedefs the instances of the FloatingPoint template class that we // care to use. typedef FloatingPoint Float; typedef FloatingPoint Double; // In order to catch the mistake of putting tests that use different // test fixture classes in the same test suite, we need to assign // unique IDs to fixture classes and compare them. The TypeId type is // used to hold such IDs. The user should treat TypeId as an opaque // type: the only operation allowed on TypeId values is to compare // them for equality using the == operator. typedef const void* TypeId; template class TypeIdHelper { public: // dummy_ must not have a const type. Otherwise an overly eager // compiler (e.g. MSVC 7.1 & 8.0) may try to merge // TypeIdHelper::dummy_ for different Ts as an "optimization". static bool dummy_; }; template bool TypeIdHelper::dummy_ = false; // GetTypeId() returns the ID of type T. Different values will be // returned for different types. Calling the function twice with the // same type argument is guaranteed to return the same ID. template TypeId GetTypeId() { // The compiler is required to allocate a different // TypeIdHelper::dummy_ variable for each T used to instantiate // the template. Therefore, the address of dummy_ is guaranteed to // be unique. return &(TypeIdHelper::dummy_); } // Returns the type ID of ::testing::Test. Always call this instead // of GetTypeId< ::testing::Test>() to get the type ID of // ::testing::Test, as the latter may give the wrong result due to a // suspected linker bug when compiling Google Test as a Mac OS X // framework. GTEST_API_ TypeId GetTestTypeId(); // Defines the abstract factory interface that creates instances // of a Test object. class TestFactoryBase { public: virtual ~TestFactoryBase() {} // Creates a test instance to run. The instance is both created and destroyed // within TestInfoImpl::Run() virtual Test* CreateTest() = 0; protected: TestFactoryBase() {} private: GTEST_DISALLOW_COPY_AND_ASSIGN_(TestFactoryBase); }; // This class provides implementation of TeastFactoryBase interface. // It is used in TEST and TEST_F macros. template class TestFactoryImpl : public TestFactoryBase { public: Test* CreateTest() override { return new TestClass; } }; #if GTEST_OS_WINDOWS // Predicate-formatters for implementing the HRESULT checking macros // {ASSERT|EXPECT}_HRESULT_{SUCCEEDED|FAILED} // We pass a long instead of HRESULT to avoid causing an // include dependency for the HRESULT type. GTEST_API_ AssertionResult IsHRESULTSuccess(const char* expr, long hr); // NOLINT GTEST_API_ AssertionResult IsHRESULTFailure(const char* expr, long hr); // NOLINT #endif // GTEST_OS_WINDOWS // Types of SetUpTestSuite() and TearDownTestSuite() functions. using SetUpTestSuiteFunc = void (*)(); using TearDownTestSuiteFunc = void (*)(); struct CodeLocation { CodeLocation(const std::string& a_file, int a_line) : file(a_file), line(a_line) {} std::string file; int line; }; // Helper to identify which setup function for TestCase / TestSuite to call. // Only one function is allowed, either TestCase or TestSute but not both. // Utility functions to help SuiteApiResolver using SetUpTearDownSuiteFuncType = void (*)(); inline SetUpTearDownSuiteFuncType GetNotDefaultOrNull( SetUpTearDownSuiteFuncType a, SetUpTearDownSuiteFuncType def) { return a == def ? nullptr : a; } template // Note that SuiteApiResolver inherits from T because // SetUpTestSuite()/TearDownTestSuite() could be protected. Ths way // SuiteApiResolver can access them. struct SuiteApiResolver : T { // testing::Test is only forward declared at this point. So we make it a // dependend class for the compiler to be OK with it. using Test = typename std::conditional::type; static SetUpTearDownSuiteFuncType GetSetUpCaseOrSuite(const char* filename, int line_num) { SetUpTearDownSuiteFuncType test_case_fp = GetNotDefaultOrNull(&T::SetUpTestCase, &Test::SetUpTestCase); SetUpTearDownSuiteFuncType test_suite_fp = GetNotDefaultOrNull(&T::SetUpTestSuite, &Test::SetUpTestSuite); GTEST_CHECK_(!test_case_fp || !test_suite_fp) << "Test can not provide both SetUpTestSuite and SetUpTestCase, please " "make sure there is only one present at " << filename << ":" << line_num; return test_case_fp != nullptr ? test_case_fp : test_suite_fp; } static SetUpTearDownSuiteFuncType GetTearDownCaseOrSuite(const char* filename, int line_num) { SetUpTearDownSuiteFuncType test_case_fp = GetNotDefaultOrNull(&T::TearDownTestCase, &Test::TearDownTestCase); SetUpTearDownSuiteFuncType test_suite_fp = GetNotDefaultOrNull(&T::TearDownTestSuite, &Test::TearDownTestSuite); GTEST_CHECK_(!test_case_fp || !test_suite_fp) << "Test can not provide both TearDownTestSuite and TearDownTestCase," " please make sure there is only one present at" << filename << ":" << line_num; return test_case_fp != nullptr ? test_case_fp : test_suite_fp; } }; // Creates a new TestInfo object and registers it with Google Test; // returns the created object. // // Arguments: // // test_suite_name: name of the test suite // name: name of the test // type_param the name of the test's type parameter, or NULL if // this is not a typed or a type-parameterized test. // value_param text representation of the test's value parameter, // or NULL if this is not a type-parameterized test. // code_location: code location where the test is defined // fixture_class_id: ID of the test fixture class // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite // factory: pointer to the factory that creates a test object. // The newly created TestInfo instance will assume // ownership of the factory object. GTEST_API_ TestInfo* MakeAndRegisterTestInfo( const char* test_suite_name, const char* name, const char* type_param, const char* value_param, CodeLocation code_location, TypeId fixture_class_id, SetUpTestSuiteFunc set_up_tc, TearDownTestSuiteFunc tear_down_tc, TestFactoryBase* factory); // If *pstr starts with the given prefix, modifies *pstr to be right // past the prefix and returns true; otherwise leaves *pstr unchanged // and returns false. None of pstr, *pstr, and prefix can be NULL. GTEST_API_ bool SkipPrefix(const char* prefix, const char** pstr); #if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) // State of the definition of a type-parameterized test suite. class GTEST_API_ TypedTestSuitePState { public: TypedTestSuitePState() : registered_(false) {} // Adds the given test name to defined_test_names_ and return true // if the test suite hasn't been registered; otherwise aborts the // program. bool AddTestName(const char* file, int line, const char* case_name, const char* test_name) { if (registered_) { fprintf(stderr, "%s Test %s must be defined before " "REGISTER_TYPED_TEST_SUITE_P(%s, ...).\n", FormatFileLocation(file, line).c_str(), test_name, case_name); fflush(stderr); posix::Abort(); } registered_tests_.insert( ::std::make_pair(test_name, CodeLocation(file, line))); return true; } bool TestExists(const std::string& test_name) const { return registered_tests_.count(test_name) > 0; } const CodeLocation& GetCodeLocation(const std::string& test_name) const { RegisteredTestsMap::const_iterator it = registered_tests_.find(test_name); GTEST_CHECK_(it != registered_tests_.end()); return it->second; } // Verifies that registered_tests match the test names in // defined_test_names_; returns registered_tests if successful, or // aborts the program otherwise. const char* VerifyRegisteredTestNames( const char* file, int line, const char* registered_tests); private: typedef ::std::map RegisteredTestsMap; bool registered_; RegisteredTestsMap registered_tests_; }; // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ using TypedTestCasePState = TypedTestSuitePState; #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 // Skips to the first non-space char after the first comma in 'str'; // returns NULL if no comma is found in 'str'. inline const char* SkipComma(const char* str) { const char* comma = strchr(str, ','); if (comma == nullptr) { return nullptr; } while (IsSpace(*(++comma))) {} return comma; } // Returns the prefix of 'str' before the first comma in it; returns // the entire string if it contains no comma. inline std::string GetPrefixUntilComma(const char* str) { const char* comma = strchr(str, ','); return comma == nullptr ? str : std::string(str, comma); } // Splits a given string on a given delimiter, populating a given // vector with the fields. void SplitString(const ::std::string& str, char delimiter, ::std::vector< ::std::string>* dest); // The default argument to the template below for the case when the user does // not provide a name generator. struct DefaultNameGenerator { template static std::string GetName(int i) { return StreamableToString(i); } }; template struct NameGeneratorSelector { typedef Provided type; }; template void GenerateNamesRecursively(Types0, std::vector*, int) {} template void GenerateNamesRecursively(Types, std::vector* result, int i) { result->push_back(NameGenerator::template GetName(i)); GenerateNamesRecursively(typename Types::Tail(), result, i + 1); } template std::vector GenerateNames() { std::vector result; GenerateNamesRecursively(Types(), &result, 0); return result; } // TypeParameterizedTest::Register() // registers a list of type-parameterized tests with Google Test. The // return value is insignificant - we just need to return something // such that we can call this function in a namespace scope. // // Implementation note: The GTEST_TEMPLATE_ macro declares a template // template parameter. It's defined in gtest-type-util.h. template class TypeParameterizedTest { public: // 'index' is the index of the test in the type list 'Types' // specified in INSTANTIATE_TYPED_TEST_SUITE_P(Prefix, TestSuite, // Types). Valid values for 'index' are [0, N - 1] where N is the // length of Types. static bool Register(const char* prefix, const CodeLocation& code_location, const char* case_name, const char* test_names, int index, const std::vector& type_names = GenerateNames()) { typedef typename Types::Head Type; typedef Fixture FixtureClass; typedef typename GTEST_BIND_(TestSel, Type) TestClass; // First, registers the first type-parameterized test in the type // list. MakeAndRegisterTestInfo( (std::string(prefix) + (prefix[0] == '\0' ? "" : "/") + case_name + "/" + type_names[static_cast(index)]) .c_str(), StripTrailingSpaces(GetPrefixUntilComma(test_names)).c_str(), GetTypeName().c_str(), nullptr, // No value parameter. code_location, GetTypeId(), SuiteApiResolver::GetSetUpCaseOrSuite( code_location.file.c_str(), code_location.line), SuiteApiResolver::GetTearDownCaseOrSuite( code_location.file.c_str(), code_location.line), new TestFactoryImpl); // Next, recurses (at compile time) with the tail of the type list. return TypeParameterizedTest::Register(prefix, code_location, case_name, test_names, index + 1, type_names); } }; // The base case for the compile time recursion. template class TypeParameterizedTest { public: static bool Register(const char* /*prefix*/, const CodeLocation&, const char* /*case_name*/, const char* /*test_names*/, int /*index*/, const std::vector& = std::vector() /*type_names*/) { return true; } }; // TypeParameterizedTestSuite::Register() // registers *all combinations* of 'Tests' and 'Types' with Google // Test. The return value is insignificant - we just need to return // something such that we can call this function in a namespace scope. template class TypeParameterizedTestSuite { public: static bool Register(const char* prefix, CodeLocation code_location, const TypedTestSuitePState* state, const char* case_name, const char* test_names, const std::vector& type_names = GenerateNames()) { std::string test_name = StripTrailingSpaces( GetPrefixUntilComma(test_names)); if (!state->TestExists(test_name)) { fprintf(stderr, "Failed to get code location for test %s.%s at %s.", case_name, test_name.c_str(), FormatFileLocation(code_location.file.c_str(), code_location.line).c_str()); fflush(stderr); posix::Abort(); } const CodeLocation& test_location = state->GetCodeLocation(test_name); typedef typename Tests::Head Head; // First, register the first test in 'Test' for each type in 'Types'. TypeParameterizedTest::Register( prefix, test_location, case_name, test_names, 0, type_names); // Next, recurses (at compile time) with the tail of the test list. return TypeParameterizedTestSuite::Register(prefix, code_location, state, case_name, SkipComma(test_names), type_names); } }; // The base case for the compile time recursion. template class TypeParameterizedTestSuite { public: static bool Register(const char* /*prefix*/, const CodeLocation&, const TypedTestSuitePState* /*state*/, const char* /*case_name*/, const char* /*test_names*/, const std::vector& = std::vector() /*type_names*/) { return true; } }; #endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P // Returns the current OS stack trace as an std::string. // // The maximum number of stack frames to be included is specified by // the gtest_stack_trace_depth flag. The skip_count parameter // specifies the number of top frames to be skipped, which doesn't // count against the number of frames to be included. // // For example, if Foo() calls Bar(), which in turn calls // GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in // the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. GTEST_API_ std::string GetCurrentOsStackTraceExceptTop( UnitTest* unit_test, int skip_count); // Helpers for suppressing warnings on unreachable code or constant // condition. // Always returns true. GTEST_API_ bool AlwaysTrue(); // Always returns false. inline bool AlwaysFalse() { return !AlwaysTrue(); } // Helper for suppressing false warning from Clang on a const char* // variable declared in a conditional expression always being NULL in // the else branch. struct GTEST_API_ ConstCharPtr { ConstCharPtr(const char* str) : value(str) {} operator bool() const { return true; } const char* value; }; // A simple Linear Congruential Generator for generating random // numbers with a uniform distribution. Unlike rand() and srand(), it // doesn't use global state (and therefore can't interfere with user // code). Unlike rand_r(), it's portable. An LCG isn't very random, // but it's good enough for our purposes. class GTEST_API_ Random { public: static const UInt32 kMaxRange = 1u << 31; explicit Random(UInt32 seed) : state_(seed) {} void Reseed(UInt32 seed) { state_ = seed; } // Generates a random number from [0, range). Crashes if 'range' is // 0 or greater than kMaxRange. UInt32 Generate(UInt32 range); private: UInt32 state_; GTEST_DISALLOW_COPY_AND_ASSIGN_(Random); }; // Turns const U&, U&, const U, and U all into U. #define GTEST_REMOVE_REFERENCE_AND_CONST_(T) \ typename std::remove_const::type>::type // IsAProtocolMessage::value is a compile-time bool constant that's // true if and only if T is type proto2::Message or a subclass of it. template struct IsAProtocolMessage : public bool_constant< std::is_convertible::value> {}; // When the compiler sees expression IsContainerTest(0), if C is an // STL-style container class, the first overload of IsContainerTest // will be viable (since both C::iterator* and C::const_iterator* are // valid types and NULL can be implicitly converted to them). It will // be picked over the second overload as 'int' is a perfect match for // the type of argument 0. If C::iterator or C::const_iterator is not // a valid type, the first overload is not viable, and the second // overload will be picked. Therefore, we can determine whether C is // a container class by checking the type of IsContainerTest(0). // The value of the expression is insignificant. // // In C++11 mode we check the existence of a const_iterator and that an // iterator is properly implemented for the container. // // For pre-C++11 that we look for both C::iterator and C::const_iterator. // The reason is that C++ injects the name of a class as a member of the // class itself (e.g. you can refer to class iterator as either // 'iterator' or 'iterator::iterator'). If we look for C::iterator // only, for example, we would mistakenly think that a class named // iterator is an STL container. // // Also note that the simpler approach of overloading // IsContainerTest(typename C::const_iterator*) and // IsContainerTest(...) doesn't work with Visual Age C++ and Sun C++. typedef int IsContainer; template ().begin()), class = decltype(::std::declval().end()), class = decltype(++::std::declval()), class = decltype(*::std::declval()), class = typename C::const_iterator> IsContainer IsContainerTest(int /* dummy */) { return 0; } typedef char IsNotContainer; template IsNotContainer IsContainerTest(long /* dummy */) { return '\0'; } // Trait to detect whether a type T is a hash table. // The heuristic used is that the type contains an inner type `hasher` and does // not contain an inner type `reverse_iterator`. // If the container is iterable in reverse, then order might actually matter. template struct IsHashTable { private: template static char test(typename U::hasher*, typename U::reverse_iterator*); template static int test(typename U::hasher*, ...); template static char test(...); public: static const bool value = sizeof(test(nullptr, nullptr)) == sizeof(int); }; template const bool IsHashTable::value; template (0)) == sizeof(IsContainer)> struct IsRecursiveContainerImpl; template struct IsRecursiveContainerImpl : public std::false_type {}; // Since the IsRecursiveContainerImpl depends on the IsContainerTest we need to // obey the same inconsistencies as the IsContainerTest, namely check if // something is a container is relying on only const_iterator in C++11 and // is relying on both const_iterator and iterator otherwise template struct IsRecursiveContainerImpl { using value_type = decltype(*std::declval()); using type = std::is_same::type>::type, C>; }; // IsRecursiveContainer is a unary compile-time predicate that // evaluates whether C is a recursive container type. A recursive container // type is a container type whose value_type is equal to the container type // itself. An example for a recursive container type is // boost::filesystem::path, whose iterator has a value_type that is equal to // boost::filesystem::path. template struct IsRecursiveContainer : public IsRecursiveContainerImpl::type {}; // Utilities for native arrays. // ArrayEq() compares two k-dimensional native arrays using the // elements' operator==, where k can be any integer >= 0. When k is // 0, ArrayEq() degenerates into comparing a single pair of values. template bool ArrayEq(const T* lhs, size_t size, const U* rhs); // This generic version is used when k is 0. template inline bool ArrayEq(const T& lhs, const U& rhs) { return lhs == rhs; } // This overload is used when k >= 1. template inline bool ArrayEq(const T(&lhs)[N], const U(&rhs)[N]) { return internal::ArrayEq(lhs, N, rhs); } // This helper reduces code bloat. If we instead put its logic inside // the previous ArrayEq() function, arrays with different sizes would // lead to different copies of the template code. template bool ArrayEq(const T* lhs, size_t size, const U* rhs) { for (size_t i = 0; i != size; i++) { if (!internal::ArrayEq(lhs[i], rhs[i])) return false; } return true; } // Finds the first element in the iterator range [begin, end) that // equals elem. Element may be a native array type itself. template Iter ArrayAwareFind(Iter begin, Iter end, const Element& elem) { for (Iter it = begin; it != end; ++it) { if (internal::ArrayEq(*it, elem)) return it; } return end; } // CopyArray() copies a k-dimensional native array using the elements' // operator=, where k can be any integer >= 0. When k is 0, // CopyArray() degenerates into copying a single value. template void CopyArray(const T* from, size_t size, U* to); // This generic version is used when k is 0. template inline void CopyArray(const T& from, U* to) { *to = from; } // This overload is used when k >= 1. template inline void CopyArray(const T(&from)[N], U(*to)[N]) { internal::CopyArray(from, N, *to); } // This helper reduces code bloat. If we instead put its logic inside // the previous CopyArray() function, arrays with different sizes // would lead to different copies of the template code. template void CopyArray(const T* from, size_t size, U* to) { for (size_t i = 0; i != size; i++) { internal::CopyArray(from[i], to + i); } } // The relation between an NativeArray object (see below) and the // native array it represents. // We use 2 different structs to allow non-copyable types to be used, as long // as RelationToSourceReference() is passed. struct RelationToSourceReference {}; struct RelationToSourceCopy {}; // Adapts a native array to a read-only STL-style container. Instead // of the complete STL container concept, this adaptor only implements // members useful for Google Mock's container matchers. New members // should be added as needed. To simplify the implementation, we only // support Element being a raw type (i.e. having no top-level const or // reference modifier). It's the client's responsibility to satisfy // this requirement. Element can be an array type itself (hence // multi-dimensional arrays are supported). template class NativeArray { public: // STL-style container typedefs. typedef Element value_type; typedef Element* iterator; typedef const Element* const_iterator; // Constructs from a native array. References the source. NativeArray(const Element* array, size_t count, RelationToSourceReference) { InitRef(array, count); } // Constructs from a native array. Copies the source. NativeArray(const Element* array, size_t count, RelationToSourceCopy) { InitCopy(array, count); } // Copy constructor. NativeArray(const NativeArray& rhs) { (this->*rhs.clone_)(rhs.array_, rhs.size_); } ~NativeArray() { if (clone_ != &NativeArray::InitRef) delete[] array_; } // STL-style container methods. size_t size() const { return size_; } const_iterator begin() const { return array_; } const_iterator end() const { return array_ + size_; } bool operator==(const NativeArray& rhs) const { return size() == rhs.size() && ArrayEq(begin(), size(), rhs.begin()); } private: static_assert(!std::is_const::value, "Type must not be const"); static_assert(!std::is_reference::value, "Type must not be a reference"); // Initializes this object with a copy of the input. void InitCopy(const Element* array, size_t a_size) { Element* const copy = new Element[a_size]; CopyArray(array, a_size, copy); array_ = copy; size_ = a_size; clone_ = &NativeArray::InitCopy; } // Initializes this object with a reference of the input. void InitRef(const Element* array, size_t a_size) { array_ = array; size_ = a_size; clone_ = &NativeArray::InitRef; } const Element* array_; size_t size_; void (NativeArray::*clone_)(const Element*, size_t); GTEST_DISALLOW_ASSIGN_(NativeArray); }; // Backport of std::index_sequence. template struct IndexSequence { using type = IndexSequence; }; // Double the IndexSequence, and one if plus_one is true. template struct DoubleSequence; template struct DoubleSequence, sizeofT> { using type = IndexSequence; }; template struct DoubleSequence, sizeofT> { using type = IndexSequence; }; // Backport of std::make_index_sequence. // It uses O(ln(N)) instantiation depth. template struct MakeIndexSequence : DoubleSequence::type, N / 2>::type {}; template <> struct MakeIndexSequence<0> : IndexSequence<> {}; // FIXME: This implementation of ElemFromList is O(1) in instantiation depth, // but it is O(N^2) in total instantiations. Not sure if this is the best // tradeoff, as it will make it somewhat slow to compile. template struct ElemFromListImpl {}; template struct ElemFromListImpl { using type = T; }; // Get the Nth element from T... // It uses O(1) instantiation depth. template struct ElemFromList; template struct ElemFromList, T...> : ElemFromListImpl... {}; template class FlatTuple; template struct FlatTupleElemBase; template struct FlatTupleElemBase, I> { using value_type = typename ElemFromList::type, T...>::type; FlatTupleElemBase() = default; explicit FlatTupleElemBase(value_type t) : value(std::move(t)) {} value_type value; }; template struct FlatTupleBase; template struct FlatTupleBase, IndexSequence> : FlatTupleElemBase, Idx>... { using Indices = IndexSequence; FlatTupleBase() = default; explicit FlatTupleBase(T... t) : FlatTupleElemBase, Idx>(std::move(t))... {} }; // Analog to std::tuple but with different tradeoffs. // This class minimizes the template instantiation depth, thus allowing more // elements that std::tuple would. std::tuple has been seen to require an // instantiation depth of more than 10x the number of elements in some // implementations. // FlatTuple and ElemFromList are not recursive and have a fixed depth // regardless of T... // MakeIndexSequence, on the other hand, it is recursive but with an // instantiation depth of O(ln(N)). template class FlatTuple : private FlatTupleBase, typename MakeIndexSequence::type> { using Indices = typename FlatTuple::FlatTupleBase::Indices; public: FlatTuple() = default; explicit FlatTuple(T... t) : FlatTuple::FlatTupleBase(std::move(t)...) {} template const typename ElemFromList::type& Get() const { return static_cast*>(this)->value; } template typename ElemFromList::type& Get() { return static_cast*>(this)->value; } }; // Utility functions to be called with static_assert to induce deprecation // warnings. GTEST_INTERNAL_DEPRECATED( "INSTANTIATE_TEST_CASE_P is deprecated, please use " "INSTANTIATE_TEST_SUITE_P") constexpr bool InstantiateTestCase_P_IsDeprecated() { return true; } GTEST_INTERNAL_DEPRECATED( "TYPED_TEST_CASE_P is deprecated, please use " "TYPED_TEST_SUITE_P") constexpr bool TypedTestCase_P_IsDeprecated() { return true; } GTEST_INTERNAL_DEPRECATED( "TYPED_TEST_CASE is deprecated, please use " "TYPED_TEST_SUITE") constexpr bool TypedTestCaseIsDeprecated() { return true; } GTEST_INTERNAL_DEPRECATED( "REGISTER_TYPED_TEST_CASE_P is deprecated, please use " "REGISTER_TYPED_TEST_SUITE_P") constexpr bool RegisterTypedTestCase_P_IsDeprecated() { return true; } GTEST_INTERNAL_DEPRECATED( "INSTANTIATE_TYPED_TEST_CASE_P is deprecated, please use " "INSTANTIATE_TYPED_TEST_SUITE_P") constexpr bool InstantiateTypedTestCase_P_IsDeprecated() { return true; } } // namespace internal } // namespace testing #define GTEST_MESSAGE_AT_(file, line, message, result_type) \ ::testing::internal::AssertHelper(result_type, file, line, message) \ = ::testing::Message() #define GTEST_MESSAGE_(message, result_type) \ GTEST_MESSAGE_AT_(__FILE__, __LINE__, message, result_type) #define GTEST_FATAL_FAILURE_(message) \ return GTEST_MESSAGE_(message, ::testing::TestPartResult::kFatalFailure) #define GTEST_NONFATAL_FAILURE_(message) \ GTEST_MESSAGE_(message, ::testing::TestPartResult::kNonFatalFailure) #define GTEST_SUCCESS_(message) \ GTEST_MESSAGE_(message, ::testing::TestPartResult::kSuccess) #define GTEST_SKIP_(message) \ return GTEST_MESSAGE_(message, ::testing::TestPartResult::kSkip) // Suppress MSVC warning 4072 (unreachable code) for the code following // statement if it returns or throws (or doesn't return or throw in some // situations). #define GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) \ if (::testing::internal::AlwaysTrue()) { statement; } #define GTEST_TEST_THROW_(statement, expected_exception, fail) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (::testing::internal::ConstCharPtr gtest_msg = "") { \ bool gtest_caught_expected = false; \ try { \ GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ } \ catch (expected_exception const&) { \ gtest_caught_expected = true; \ } \ catch (...) { \ gtest_msg.value = \ "Expected: " #statement " throws an exception of type " \ #expected_exception ".\n Actual: it throws a different type."; \ goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ } \ if (!gtest_caught_expected) { \ gtest_msg.value = \ "Expected: " #statement " throws an exception of type " \ #expected_exception ".\n Actual: it throws nothing."; \ goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ } \ } else \ GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__): \ fail(gtest_msg.value) #define GTEST_TEST_NO_THROW_(statement, fail) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (::testing::internal::AlwaysTrue()) { \ try { \ GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ } \ catch (...) { \ goto GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__); \ } \ } else \ GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__): \ fail("Expected: " #statement " doesn't throw an exception.\n" \ " Actual: it throws.") #define GTEST_TEST_ANY_THROW_(statement, fail) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (::testing::internal::AlwaysTrue()) { \ bool gtest_caught_any = false; \ try { \ GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ } \ catch (...) { \ gtest_caught_any = true; \ } \ if (!gtest_caught_any) { \ goto GTEST_CONCAT_TOKEN_(gtest_label_testanythrow_, __LINE__); \ } \ } else \ GTEST_CONCAT_TOKEN_(gtest_label_testanythrow_, __LINE__): \ fail("Expected: " #statement " throws an exception.\n" \ " Actual: it doesn't.") // Implements Boolean test assertions such as EXPECT_TRUE. expression can be // either a boolean expression or an AssertionResult. text is a textual // represenation of expression as it was passed into the EXPECT_TRUE. #define GTEST_TEST_BOOLEAN_(expression, text, actual, expected, fail) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (const ::testing::AssertionResult gtest_ar_ = \ ::testing::AssertionResult(expression)) \ ; \ else \ fail(::testing::internal::GetBoolAssertionFailureMessage(\ gtest_ar_, text, #actual, #expected).c_str()) #define GTEST_TEST_NO_FATAL_FAILURE_(statement, fail) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (::testing::internal::AlwaysTrue()) { \ ::testing::internal::HasNewFatalFailureHelper gtest_fatal_failure_checker; \ GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ if (gtest_fatal_failure_checker.has_new_fatal_failure()) { \ goto GTEST_CONCAT_TOKEN_(gtest_label_testnofatal_, __LINE__); \ } \ } else \ GTEST_CONCAT_TOKEN_(gtest_label_testnofatal_, __LINE__): \ fail("Expected: " #statement " doesn't generate new fatal " \ "failures in the current thread.\n" \ " Actual: it does.") // Expands to the name of the class that implements the given test. #define GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) \ test_suite_name##_##test_name##_Test // Helper macro for defining tests. #define GTEST_TEST_(test_suite_name, test_name, parent_class, parent_id) \ static_assert(sizeof(GTEST_STRINGIFY_(test_suite_name)) > 1, \ "test_suite_name must not be empty"); \ static_assert(sizeof(GTEST_STRINGIFY_(test_name)) > 1, \ "test_name must not be empty"); \ class GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) \ : public parent_class { \ public: \ GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() {} \ \ private: \ virtual void TestBody(); \ static ::testing::TestInfo* const test_info_ GTEST_ATTRIBUTE_UNUSED_; \ GTEST_DISALLOW_COPY_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name, \ test_name)); \ }; \ \ ::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(test_suite_name, \ test_name)::test_info_ = \ ::testing::internal::MakeAndRegisterTestInfo( \ #test_suite_name, #test_name, nullptr, nullptr, \ ::testing::internal::CodeLocation(__FILE__, __LINE__), (parent_id), \ ::testing::internal::SuiteApiResolver< \ parent_class>::GetSetUpCaseOrSuite(__FILE__, __LINE__), \ ::testing::internal::SuiteApiResolver< \ parent_class>::GetTearDownCaseOrSuite(__FILE__, __LINE__), \ new ::testing::internal::TestFactoryImpl); \ void GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)::TestBody() #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ // Copyright 2005, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // The Google C++ Testing and Mocking Framework (Google Test) // // This header file defines the public API for death tests. It is // #included by gtest.h so a user doesn't need to include this // directly. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ #define GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ // Copyright 2005, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // The Google C++ Testing and Mocking Framework (Google Test) // // This header file defines internal utilities needed for implementing // death tests. They are subject to change without notice. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ // Copyright 2007, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // The Google C++ Testing and Mocking Framework (Google Test) // // This file implements just enough of the matcher interface to allow // EXPECT_DEATH and friends to accept a matcher argument. // IWYU pragma: private, include "testing/base/public/gunit.h" // IWYU pragma: friend third_party/googletest/googlemock/.* // IWYU pragma: friend third_party/googletest/googletest/.* #ifndef GTEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ #define GTEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ #include #include #include #include // Copyright 2007, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Google Test - The Google C++ Testing and Mocking Framework // // This file implements a universal value printer that can print a // value of any type T: // // void ::testing::internal::UniversalPrinter::Print(value, ostream_ptr); // // A user can teach this function how to print a class type T by // defining either operator<<() or PrintTo() in the namespace that // defines T. More specifically, the FIRST defined function in the // following list will be used (assuming T is defined in namespace // foo): // // 1. foo::PrintTo(const T&, ostream*) // 2. operator<<(ostream&, const T&) defined in either foo or the // global namespace. // // However if T is an STL-style container then it is printed element-wise // unless foo::PrintTo(const T&, ostream*) is defined. Note that // operator<<() is ignored for container types. // // If none of the above is defined, it will print the debug string of // the value if it is a protocol buffer, or print the raw bytes in the // value otherwise. // // To aid debugging: when T is a reference type, the address of the // value is also printed; when T is a (const) char pointer, both the // pointer value and the NUL-terminated string it points to are // printed. // // We also provide some convenient wrappers: // // // Prints a value to a string. For a (const or not) char // // pointer, the NUL-terminated string (but not the pointer) is // // printed. // std::string ::testing::PrintToString(const T& value); // // // Prints a value tersely: for a reference type, the referenced // // value (but not the address) is printed; for a (const or not) char // // pointer, the NUL-terminated string (but not the pointer) is // // printed. // void ::testing::internal::UniversalTersePrint(const T& value, ostream*); // // // Prints value using the type inferred by the compiler. The difference // // from UniversalTersePrint() is that this function prints both the // // pointer and the NUL-terminated string for a (const or not) char pointer. // void ::testing::internal::UniversalPrint(const T& value, ostream*); // // // Prints the fields of a tuple tersely to a string vector, one // // element for each field. Tuple support must be enabled in // // gtest-port.h. // std::vector UniversalTersePrintTupleFieldsToStrings( // const Tuple& value); // // Known limitation: // // The print primitives print the elements of an STL-style container // using the compiler-inferred type of *iter where iter is a // const_iterator of the container. When const_iterator is an input // iterator but not a forward iterator, this inferred type may not // match value_type, and the print output may be incorrect. In // practice, this is rarely a problem as for most containers // const_iterator is a forward iterator. We'll fix this if there's an // actual need for it. Note that this fix cannot rely on value_type // being defined as many user-defined container types don't have // value_type. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ #define GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ #include #include // NOLINT #include #include #include #include #include #include #if GTEST_HAS_ABSL #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/variant.h" #endif // GTEST_HAS_ABSL namespace testing { // Definitions in the 'internal' and 'internal2' name spaces are // subject to change without notice. DO NOT USE THEM IN USER CODE! namespace internal2 { // Prints the given number of bytes in the given object to the given // ostream. GTEST_API_ void PrintBytesInObjectTo(const unsigned char* obj_bytes, size_t count, ::std::ostream* os); // For selecting which printer to use when a given type has neither << // nor PrintTo(). enum TypeKind { kProtobuf, // a protobuf type kConvertibleToInteger, // a type implicitly convertible to BiggestInt // (e.g. a named or unnamed enum type) #if GTEST_HAS_ABSL kConvertibleToStringView, // a type implicitly convertible to // absl::string_view #endif kOtherType // anything else }; // TypeWithoutFormatter::PrintValue(value, os) is called // by the universal printer to print a value of type T when neither // operator<< nor PrintTo() is defined for T, where kTypeKind is the // "kind" of T as defined by enum TypeKind. template class TypeWithoutFormatter { public: // This default version is called when kTypeKind is kOtherType. static void PrintValue(const T& value, ::std::ostream* os) { PrintBytesInObjectTo( static_cast( reinterpret_cast(std::addressof(value))), sizeof(value), os); } }; // We print a protobuf using its ShortDebugString() when the string // doesn't exceed this many characters; otherwise we print it using // DebugString() for better readability. const size_t kProtobufOneLinerMaxLength = 50; template class TypeWithoutFormatter { public: static void PrintValue(const T& value, ::std::ostream* os) { std::string pretty_str = value.ShortDebugString(); if (pretty_str.length() > kProtobufOneLinerMaxLength) { pretty_str = "\n" + value.DebugString(); } *os << ("<" + pretty_str + ">"); } }; template class TypeWithoutFormatter { public: // Since T has no << operator or PrintTo() but can be implicitly // converted to BiggestInt, we print it as a BiggestInt. // // Most likely T is an enum type (either named or unnamed), in which // case printing it as an integer is the desired behavior. In case // T is not an enum, printing it as an integer is the best we can do // given that it has no user-defined printer. static void PrintValue(const T& value, ::std::ostream* os) { const internal::BiggestInt kBigInt = value; *os << kBigInt; } }; #if GTEST_HAS_ABSL template class TypeWithoutFormatter { public: // Since T has neither operator<< nor PrintTo() but can be implicitly // converted to absl::string_view, we print it as a absl::string_view. // // Note: the implementation is further below, as it depends on // internal::PrintTo symbol which is defined later in the file. static void PrintValue(const T& value, ::std::ostream* os); }; #endif // Prints the given value to the given ostream. If the value is a // protocol message, its debug string is printed; if it's an enum or // of a type implicitly convertible to BiggestInt, it's printed as an // integer; otherwise the bytes in the value are printed. This is // what UniversalPrinter::Print() does when it knows nothing about // type T and T has neither << operator nor PrintTo(). // // A user can override this behavior for a class type Foo by defining // a << operator in the namespace where Foo is defined. // // We put this operator in namespace 'internal2' instead of 'internal' // to simplify the implementation, as much code in 'internal' needs to // use << in STL, which would conflict with our own << were it defined // in 'internal'. // // Note that this operator<< takes a generic std::basic_ostream type instead of the more restricted std::ostream. If // we define it to take an std::ostream instead, we'll get an // "ambiguous overloads" compiler error when trying to print a type // Foo that supports streaming to std::basic_ostream, as the compiler cannot tell whether // operator<<(std::ostream&, const T&) or // operator<<(std::basic_stream, const Foo&) is more // specific. template ::std::basic_ostream& operator<<( ::std::basic_ostream& os, const T& x) { TypeWithoutFormatter::value ? kProtobuf : std::is_convertible< const T&, internal::BiggestInt>::value ? kConvertibleToInteger : #if GTEST_HAS_ABSL std::is_convertible< const T&, absl::string_view>::value ? kConvertibleToStringView : #endif kOtherType)>::PrintValue(x, &os); return os; } } // namespace internal2 } // namespace testing // This namespace MUST NOT BE NESTED IN ::testing, or the name look-up // magic needed for implementing UniversalPrinter won't work. namespace testing_internal { // Used to print a value that is not an STL-style container when the // user doesn't define PrintTo() for it. template void DefaultPrintNonContainerTo(const T& value, ::std::ostream* os) { // With the following statement, during unqualified name lookup, // testing::internal2::operator<< appears as if it was declared in // the nearest enclosing namespace that contains both // ::testing_internal and ::testing::internal2, i.e. the global // namespace. For more details, refer to the C++ Standard section // 7.3.4-1 [namespace.udir]. This allows us to fall back onto // testing::internal2::operator<< in case T doesn't come with a << // operator. // // We cannot write 'using ::testing::internal2::operator<<;', which // gcc 3.3 fails to compile due to a compiler bug. using namespace ::testing::internal2; // NOLINT // Assuming T is defined in namespace foo, in the next statement, // the compiler will consider all of: // // 1. foo::operator<< (thanks to Koenig look-up), // 2. ::operator<< (as the current namespace is enclosed in ::), // 3. testing::internal2::operator<< (thanks to the using statement above). // // The operator<< whose type matches T best will be picked. // // We deliberately allow #2 to be a candidate, as sometimes it's // impossible to define #1 (e.g. when foo is ::std, defining // anything in it is undefined behavior unless you are a compiler // vendor.). *os << value; } } // namespace testing_internal namespace testing { namespace internal { // FormatForComparison::Format(value) formats a // value of type ToPrint that is an operand of a comparison assertion // (e.g. ASSERT_EQ). OtherOperand is the type of the other operand in // the comparison, and is used to help determine the best way to // format the value. In particular, when the value is a C string // (char pointer) and the other operand is an STL string object, we // want to format the C string as a string, since we know it is // compared by value with the string object. If the value is a char // pointer but the other operand is not an STL string object, we don't // know whether the pointer is supposed to point to a NUL-terminated // string, and thus want to print it as a pointer to be safe. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. // The default case. template class FormatForComparison { public: static ::std::string Format(const ToPrint& value) { return ::testing::PrintToString(value); } }; // Array. template class FormatForComparison { public: static ::std::string Format(const ToPrint* value) { return FormatForComparison::Format(value); } }; // By default, print C string as pointers to be safe, as we don't know // whether they actually point to a NUL-terminated string. #define GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(CharType) \ template \ class FormatForComparison { \ public: \ static ::std::string Format(CharType* value) { \ return ::testing::PrintToString(static_cast(value)); \ } \ } GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char); GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char); GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(wchar_t); GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const wchar_t); #undef GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_ // If a C string is compared with an STL string object, we know it's meant // to point to a NUL-terminated string, and thus can print it as a string. #define GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(CharType, OtherStringType) \ template <> \ class FormatForComparison { \ public: \ static ::std::string Format(CharType* value) { \ return ::testing::PrintToString(value); \ } \ } GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::std::string); GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::std::string); #if GTEST_HAS_STD_WSTRING GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::std::wstring); GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::std::wstring); #endif #undef GTEST_IMPL_FORMAT_C_STRING_AS_STRING_ // Formats a comparison assertion (e.g. ASSERT_EQ, EXPECT_LT, and etc) // operand to be used in a failure message. The type (but not value) // of the other operand may affect the format. This allows us to // print a char* as a raw pointer when it is compared against another // char* or void*, and print it as a C string when it is compared // against an std::string object, for example. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. template std::string FormatForComparisonFailureMessage( const T1& value, const T2& /* other_operand */) { return FormatForComparison::Format(value); } // UniversalPrinter::Print(value, ostream_ptr) prints the given // value to the given ostream. The caller must ensure that // 'ostream_ptr' is not NULL, or the behavior is undefined. // // We define UniversalPrinter as a class template (as opposed to a // function template), as we need to partially specialize it for // reference types, which cannot be done with function templates. template class UniversalPrinter; template void UniversalPrint(const T& value, ::std::ostream* os); enum DefaultPrinterType { kPrintContainer, kPrintPointer, kPrintFunctionPointer, kPrintOther, }; template struct WrapPrinterType {}; // Used to print an STL-style container when the user doesn't define // a PrintTo() for it. template void DefaultPrintTo(WrapPrinterType /* dummy */, const C& container, ::std::ostream* os) { const size_t kMaxCount = 32; // The maximum number of elements to print. *os << '{'; size_t count = 0; for (typename C::const_iterator it = container.begin(); it != container.end(); ++it, ++count) { if (count > 0) { *os << ','; if (count == kMaxCount) { // Enough has been printed. *os << " ..."; break; } } *os << ' '; // We cannot call PrintTo(*it, os) here as PrintTo() doesn't // handle *it being a native array. internal::UniversalPrint(*it, os); } if (count > 0) { *os << ' '; } *os << '}'; } // Used to print a pointer that is neither a char pointer nor a member // pointer, when the user doesn't define PrintTo() for it. (A member // variable pointer or member function pointer doesn't really point to // a location in the address space. Their representation is // implementation-defined. Therefore they will be printed as raw // bytes.) template void DefaultPrintTo(WrapPrinterType /* dummy */, T* p, ::std::ostream* os) { if (p == nullptr) { *os << "NULL"; } else { // T is not a function type. We just call << to print p, // relying on ADL to pick up user-defined << for their pointer // types, if any. *os << p; } } template void DefaultPrintTo(WrapPrinterType /* dummy */, T* p, ::std::ostream* os) { if (p == nullptr) { *os << "NULL"; } else { // T is a function type, so '*os << p' doesn't do what we want // (it just prints p as bool). We want to print p as a const // void*. *os << reinterpret_cast(p); } } // Used to print a non-container, non-pointer value when the user // doesn't define PrintTo() for it. template void DefaultPrintTo(WrapPrinterType /* dummy */, const T& value, ::std::ostream* os) { ::testing_internal::DefaultPrintNonContainerTo(value, os); } // Prints the given value using the << operator if it has one; // otherwise prints the bytes in it. This is what // UniversalPrinter::Print() does when PrintTo() is not specialized // or overloaded for type T. // // A user can override this behavior for a class type Foo by defining // an overload of PrintTo() in the namespace where Foo is defined. We // give the user this option as sometimes defining a << operator for // Foo is not desirable (e.g. the coding style may prevent doing it, // or there is already a << operator but it doesn't do what the user // wants). template void PrintTo(const T& value, ::std::ostream* os) { // DefaultPrintTo() is overloaded. The type of its first argument // determines which version will be picked. // // Note that we check for container types here, prior to we check // for protocol message types in our operator<<. The rationale is: // // For protocol messages, we want to give people a chance to // override Google Mock's format by defining a PrintTo() or // operator<<. For STL containers, other formats can be // incompatible with Google Mock's format for the container // elements; therefore we check for container types here to ensure // that our format is used. // // Note that MSVC and clang-cl do allow an implicit conversion from // pointer-to-function to pointer-to-object, but clang-cl warns on it. // So don't use ImplicitlyConvertible if it can be helped since it will // cause this warning, and use a separate overload of DefaultPrintTo for // function pointers so that the `*os << p` in the object pointer overload // doesn't cause that warning either. DefaultPrintTo( WrapPrinterType < (sizeof(IsContainerTest(0)) == sizeof(IsContainer)) && !IsRecursiveContainer::value ? kPrintContainer : !std::is_pointer::value ? kPrintOther : std::is_function::type>::value ? kPrintFunctionPointer : kPrintPointer > (), value, os); } // The following list of PrintTo() overloads tells // UniversalPrinter::Print() how to print standard types (built-in // types, strings, plain arrays, and pointers). // Overloads for various char types. GTEST_API_ void PrintTo(unsigned char c, ::std::ostream* os); GTEST_API_ void PrintTo(signed char c, ::std::ostream* os); inline void PrintTo(char c, ::std::ostream* os) { // When printing a plain char, we always treat it as unsigned. This // way, the output won't be affected by whether the compiler thinks // char is signed or not. PrintTo(static_cast(c), os); } // Overloads for other simple built-in types. inline void PrintTo(bool x, ::std::ostream* os) { *os << (x ? "true" : "false"); } // Overload for wchar_t type. // Prints a wchar_t as a symbol if it is printable or as its internal // code otherwise and also as its decimal code (except for L'\0'). // The L'\0' char is printed as "L'\\0'". The decimal code is printed // as signed integer when wchar_t is implemented by the compiler // as a signed type and is printed as an unsigned integer when wchar_t // is implemented as an unsigned type. GTEST_API_ void PrintTo(wchar_t wc, ::std::ostream* os); // Overloads for C strings. GTEST_API_ void PrintTo(const char* s, ::std::ostream* os); inline void PrintTo(char* s, ::std::ostream* os) { PrintTo(ImplicitCast_(s), os); } // signed/unsigned char is often used for representing binary data, so // we print pointers to it as void* to be safe. inline void PrintTo(const signed char* s, ::std::ostream* os) { PrintTo(ImplicitCast_(s), os); } inline void PrintTo(signed char* s, ::std::ostream* os) { PrintTo(ImplicitCast_(s), os); } inline void PrintTo(const unsigned char* s, ::std::ostream* os) { PrintTo(ImplicitCast_(s), os); } inline void PrintTo(unsigned char* s, ::std::ostream* os) { PrintTo(ImplicitCast_(s), os); } // MSVC can be configured to define wchar_t as a typedef of unsigned // short. It defines _NATIVE_WCHAR_T_DEFINED when wchar_t is a native // type. When wchar_t is a typedef, defining an overload for const // wchar_t* would cause unsigned short* be printed as a wide string, // possibly causing invalid memory accesses. #if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) // Overloads for wide C strings GTEST_API_ void PrintTo(const wchar_t* s, ::std::ostream* os); inline void PrintTo(wchar_t* s, ::std::ostream* os) { PrintTo(ImplicitCast_(s), os); } #endif // Overload for C arrays. Multi-dimensional arrays are printed // properly. // Prints the given number of elements in an array, without printing // the curly braces. template void PrintRawArrayTo(const T a[], size_t count, ::std::ostream* os) { UniversalPrint(a[0], os); for (size_t i = 1; i != count; i++) { *os << ", "; UniversalPrint(a[i], os); } } // Overloads for ::std::string. GTEST_API_ void PrintStringTo(const ::std::string&s, ::std::ostream* os); inline void PrintTo(const ::std::string& s, ::std::ostream* os) { PrintStringTo(s, os); } // Overloads for ::std::wstring. #if GTEST_HAS_STD_WSTRING GTEST_API_ void PrintWideStringTo(const ::std::wstring&s, ::std::ostream* os); inline void PrintTo(const ::std::wstring& s, ::std::ostream* os) { PrintWideStringTo(s, os); } #endif // GTEST_HAS_STD_WSTRING #if GTEST_HAS_ABSL // Overload for absl::string_view. inline void PrintTo(absl::string_view sp, ::std::ostream* os) { PrintTo(::std::string(sp), os); } #endif // GTEST_HAS_ABSL inline void PrintTo(std::nullptr_t, ::std::ostream* os) { *os << "(nullptr)"; } template void PrintTo(std::reference_wrapper ref, ::std::ostream* os) { UniversalPrinter::Print(ref.get(), os); } // Helper function for printing a tuple. T must be instantiated with // a tuple type. template void PrintTupleTo(const T&, std::integral_constant, ::std::ostream*) {} template void PrintTupleTo(const T& t, std::integral_constant, ::std::ostream* os) { PrintTupleTo(t, std::integral_constant(), os); GTEST_INTENTIONAL_CONST_COND_PUSH_() if (I > 1) { GTEST_INTENTIONAL_CONST_COND_POP_() *os << ", "; } UniversalPrinter::type>::Print( std::get(t), os); } template void PrintTo(const ::std::tuple& t, ::std::ostream* os) { *os << "("; PrintTupleTo(t, std::integral_constant(), os); *os << ")"; } // Overload for std::pair. template void PrintTo(const ::std::pair& value, ::std::ostream* os) { *os << '('; // We cannot use UniversalPrint(value.first, os) here, as T1 may be // a reference type. The same for printing value.second. UniversalPrinter::Print(value.first, os); *os << ", "; UniversalPrinter::Print(value.second, os); *os << ')'; } // Implements printing a non-reference type T by letting the compiler // pick the right overload of PrintTo() for T. template class UniversalPrinter { public: // MSVC warns about adding const to a function type, so we want to // disable the warning. GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180) // Note: we deliberately don't call this PrintTo(), as that name // conflicts with ::testing::internal::PrintTo in the body of the // function. static void Print(const T& value, ::std::ostream* os) { // By default, ::testing::internal::PrintTo() is used for printing // the value. // // Thanks to Koenig look-up, if T is a class and has its own // PrintTo() function defined in its namespace, that function will // be visible here. Since it is more specific than the generic ones // in ::testing::internal, it will be picked by the compiler in the // following statement - exactly what we want. PrintTo(value, os); } GTEST_DISABLE_MSC_WARNINGS_POP_() }; #if GTEST_HAS_ABSL // Printer for absl::optional template class UniversalPrinter<::absl::optional> { public: static void Print(const ::absl::optional& value, ::std::ostream* os) { *os << '('; if (!value) { *os << "nullopt"; } else { UniversalPrint(*value, os); } *os << ')'; } }; // Printer for absl::variant template class UniversalPrinter<::absl::variant> { public: static void Print(const ::absl::variant& value, ::std::ostream* os) { *os << '('; absl::visit(Visitor{os}, value); *os << ')'; } private: struct Visitor { template void operator()(const U& u) const { *os << "'" << GetTypeName() << "' with value "; UniversalPrint(u, os); } ::std::ostream* os; }; }; #endif // GTEST_HAS_ABSL // UniversalPrintArray(begin, len, os) prints an array of 'len' // elements, starting at address 'begin'. template void UniversalPrintArray(const T* begin, size_t len, ::std::ostream* os) { if (len == 0) { *os << "{}"; } else { *os << "{ "; const size_t kThreshold = 18; const size_t kChunkSize = 8; // If the array has more than kThreshold elements, we'll have to // omit some details by printing only the first and the last // kChunkSize elements. if (len <= kThreshold) { PrintRawArrayTo(begin, len, os); } else { PrintRawArrayTo(begin, kChunkSize, os); *os << ", ..., "; PrintRawArrayTo(begin + len - kChunkSize, kChunkSize, os); } *os << " }"; } } // This overload prints a (const) char array compactly. GTEST_API_ void UniversalPrintArray( const char* begin, size_t len, ::std::ostream* os); // This overload prints a (const) wchar_t array compactly. GTEST_API_ void UniversalPrintArray( const wchar_t* begin, size_t len, ::std::ostream* os); // Implements printing an array type T[N]. template class UniversalPrinter { public: // Prints the given array, omitting some elements when there are too // many. static void Print(const T (&a)[N], ::std::ostream* os) { UniversalPrintArray(a, N, os); } }; // Implements printing a reference type T&. template class UniversalPrinter { public: // MSVC warns about adding const to a function type, so we want to // disable the warning. GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180) static void Print(const T& value, ::std::ostream* os) { // Prints the address of the value. We use reinterpret_cast here // as static_cast doesn't compile when T is a function type. *os << "@" << reinterpret_cast(&value) << " "; // Then prints the value itself. UniversalPrint(value, os); } GTEST_DISABLE_MSC_WARNINGS_POP_() }; // Prints a value tersely: for a reference type, the referenced value // (but not the address) is printed; for a (const) char pointer, the // NUL-terminated string (but not the pointer) is printed. template class UniversalTersePrinter { public: static void Print(const T& value, ::std::ostream* os) { UniversalPrint(value, os); } }; template class UniversalTersePrinter { public: static void Print(const T& value, ::std::ostream* os) { UniversalPrint(value, os); } }; template class UniversalTersePrinter { public: static void Print(const T (&value)[N], ::std::ostream* os) { UniversalPrinter::Print(value, os); } }; template <> class UniversalTersePrinter { public: static void Print(const char* str, ::std::ostream* os) { if (str == nullptr) { *os << "NULL"; } else { UniversalPrint(std::string(str), os); } } }; template <> class UniversalTersePrinter { public: static void Print(char* str, ::std::ostream* os) { UniversalTersePrinter::Print(str, os); } }; #if GTEST_HAS_STD_WSTRING template <> class UniversalTersePrinter { public: static void Print(const wchar_t* str, ::std::ostream* os) { if (str == nullptr) { *os << "NULL"; } else { UniversalPrint(::std::wstring(str), os); } } }; #endif template <> class UniversalTersePrinter { public: static void Print(wchar_t* str, ::std::ostream* os) { UniversalTersePrinter::Print(str, os); } }; template void UniversalTersePrint(const T& value, ::std::ostream* os) { UniversalTersePrinter::Print(value, os); } // Prints a value using the type inferred by the compiler. The // difference between this and UniversalTersePrint() is that for a // (const) char pointer, this prints both the pointer and the // NUL-terminated string. template void UniversalPrint(const T& value, ::std::ostream* os) { // A workarond for the bug in VC++ 7.1 that prevents us from instantiating // UniversalPrinter with T directly. typedef T T1; UniversalPrinter::Print(value, os); } typedef ::std::vector< ::std::string> Strings; // Tersely prints the first N fields of a tuple to a string vector, // one element for each field. template void TersePrintPrefixToStrings(const Tuple&, std::integral_constant, Strings*) {} template void TersePrintPrefixToStrings(const Tuple& t, std::integral_constant, Strings* strings) { TersePrintPrefixToStrings(t, std::integral_constant(), strings); ::std::stringstream ss; UniversalTersePrint(std::get(t), &ss); strings->push_back(ss.str()); } // Prints the fields of a tuple tersely to a string vector, one // element for each field. See the comment before // UniversalTersePrint() for how we define "tersely". template Strings UniversalTersePrintTupleFieldsToStrings(const Tuple& value) { Strings result; TersePrintPrefixToStrings( value, std::integral_constant::value>(), &result); return result; } } // namespace internal #if GTEST_HAS_ABSL namespace internal2 { template void TypeWithoutFormatter::PrintValue( const T& value, ::std::ostream* os) { internal::PrintTo(absl::string_view(value), os); } } // namespace internal2 #endif template ::std::string PrintToString(const T& value) { ::std::stringstream ss; internal::UniversalTersePrinter::Print(value, &ss); return ss.str(); } } // namespace testing // Include any custom printer added by the local installation. // We must include this header at the end to make sure it can use the // declarations from this file. // Copyright 2015, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // This file provides an injection point for custom printers in a local // installation of gTest. // It will be included from gtest-printers.h and the overrides in this file // will be visible to everyone. // // Injection point for custom user configurations. See README for details // // ** Custom implementation starts here ** #ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ #endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ #endif // GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ // MSVC warning C5046 is new as of VS2017 version 15.8. #if defined(_MSC_VER) && _MSC_VER >= 1915 #define GTEST_MAYBE_5046_ 5046 #else #define GTEST_MAYBE_5046_ #endif GTEST_DISABLE_MSC_WARNINGS_PUSH_( 4251 GTEST_MAYBE_5046_ /* class A needs to have dll-interface to be used by clients of class B */ /* Symbol involving type with internal linkage not defined */) namespace testing { // To implement a matcher Foo for type T, define: // 1. a class FooMatcherImpl that implements the // MatcherInterface interface, and // 2. a factory function that creates a Matcher object from a // FooMatcherImpl*. // // The two-level delegation design makes it possible to allow a user // to write "v" instead of "Eq(v)" where a Matcher is expected, which // is impossible if we pass matchers by pointers. It also eases // ownership management as Matcher objects can now be copied like // plain values. // MatchResultListener is an abstract class. Its << operator can be // used by a matcher to explain why a value matches or doesn't match. // class MatchResultListener { public: // Creates a listener object with the given underlying ostream. The // listener does not own the ostream, and does not dereference it // in the constructor or destructor. explicit MatchResultListener(::std::ostream* os) : stream_(os) {} virtual ~MatchResultListener() = 0; // Makes this class abstract. // Streams x to the underlying ostream; does nothing if the ostream // is NULL. template MatchResultListener& operator<<(const T& x) { if (stream_ != nullptr) *stream_ << x; return *this; } // Returns the underlying ostream. ::std::ostream* stream() { return stream_; } // Returns true if and only if the listener is interested in an explanation // of the match result. A matcher's MatchAndExplain() method can use // this information to avoid generating the explanation when no one // intends to hear it. bool IsInterested() const { return stream_ != nullptr; } private: ::std::ostream* const stream_; GTEST_DISALLOW_COPY_AND_ASSIGN_(MatchResultListener); }; inline MatchResultListener::~MatchResultListener() { } // An instance of a subclass of this knows how to describe itself as a // matcher. class MatcherDescriberInterface { public: virtual ~MatcherDescriberInterface() {} // Describes this matcher to an ostream. The function should print // a verb phrase that describes the property a value matching this // matcher should have. The subject of the verb phrase is the value // being matched. For example, the DescribeTo() method of the Gt(7) // matcher prints "is greater than 7". virtual void DescribeTo(::std::ostream* os) const = 0; // Describes the negation of this matcher to an ostream. For // example, if the description of this matcher is "is greater than // 7", the negated description could be "is not greater than 7". // You are not required to override this when implementing // MatcherInterface, but it is highly advised so that your matcher // can produce good error messages. virtual void DescribeNegationTo(::std::ostream* os) const { *os << "not ("; DescribeTo(os); *os << ")"; } }; // The implementation of a matcher. template class MatcherInterface : public MatcherDescriberInterface { public: // Returns true if and only if the matcher matches x; also explains the // match result to 'listener' if necessary (see the next paragraph), in // the form of a non-restrictive relative clause ("which ...", // "whose ...", etc) that describes x. For example, the // MatchAndExplain() method of the Pointee(...) matcher should // generate an explanation like "which points to ...". // // Implementations of MatchAndExplain() should add an explanation of // the match result *if and only if* they can provide additional // information that's not already present (or not obvious) in the // print-out of x and the matcher's description. Whether the match // succeeds is not a factor in deciding whether an explanation is // needed, as sometimes the caller needs to print a failure message // when the match succeeds (e.g. when the matcher is used inside // Not()). // // For example, a "has at least 10 elements" matcher should explain // what the actual element count is, regardless of the match result, // as it is useful information to the reader; on the other hand, an // "is empty" matcher probably only needs to explain what the actual // size is when the match fails, as it's redundant to say that the // size is 0 when the value is already known to be empty. // // You should override this method when defining a new matcher. // // It's the responsibility of the caller (Google Test) to guarantee // that 'listener' is not NULL. This helps to simplify a matcher's // implementation when it doesn't care about the performance, as it // can talk to 'listener' without checking its validity first. // However, in order to implement dummy listeners efficiently, // listener->stream() may be NULL. virtual bool MatchAndExplain(T x, MatchResultListener* listener) const = 0; // Inherits these methods from MatcherDescriberInterface: // virtual void DescribeTo(::std::ostream* os) const = 0; // virtual void DescribeNegationTo(::std::ostream* os) const; }; namespace internal { // Converts a MatcherInterface to a MatcherInterface. template class MatcherInterfaceAdapter : public MatcherInterface { public: explicit MatcherInterfaceAdapter(const MatcherInterface* impl) : impl_(impl) {} ~MatcherInterfaceAdapter() override { delete impl_; } void DescribeTo(::std::ostream* os) const override { impl_->DescribeTo(os); } void DescribeNegationTo(::std::ostream* os) const override { impl_->DescribeNegationTo(os); } bool MatchAndExplain(const T& x, MatchResultListener* listener) const override { return impl_->MatchAndExplain(x, listener); } private: const MatcherInterface* const impl_; GTEST_DISALLOW_COPY_AND_ASSIGN_(MatcherInterfaceAdapter); }; struct AnyEq { template bool operator()(const A& a, const B& b) const { return a == b; } }; struct AnyNe { template bool operator()(const A& a, const B& b) const { return a != b; } }; struct AnyLt { template bool operator()(const A& a, const B& b) const { return a < b; } }; struct AnyGt { template bool operator()(const A& a, const B& b) const { return a > b; } }; struct AnyLe { template bool operator()(const A& a, const B& b) const { return a <= b; } }; struct AnyGe { template bool operator()(const A& a, const B& b) const { return a >= b; } }; // A match result listener that ignores the explanation. class DummyMatchResultListener : public MatchResultListener { public: DummyMatchResultListener() : MatchResultListener(nullptr) {} private: GTEST_DISALLOW_COPY_AND_ASSIGN_(DummyMatchResultListener); }; // A match result listener that forwards the explanation to a given // ostream. The difference between this and MatchResultListener is // that the former is concrete. class StreamMatchResultListener : public MatchResultListener { public: explicit StreamMatchResultListener(::std::ostream* os) : MatchResultListener(os) {} private: GTEST_DISALLOW_COPY_AND_ASSIGN_(StreamMatchResultListener); }; // An internal class for implementing Matcher, which will derive // from it. We put functionalities common to all Matcher // specializations here to avoid code duplication. template class MatcherBase { public: // Returns true if and only if the matcher matches x; also explains the // match result to 'listener'. bool MatchAndExplain(const T& x, MatchResultListener* listener) const { return impl_->MatchAndExplain(x, listener); } // Returns true if and only if this matcher matches x. bool Matches(const T& x) const { DummyMatchResultListener dummy; return MatchAndExplain(x, &dummy); } // Describes this matcher to an ostream. void DescribeTo(::std::ostream* os) const { impl_->DescribeTo(os); } // Describes the negation of this matcher to an ostream. void DescribeNegationTo(::std::ostream* os) const { impl_->DescribeNegationTo(os); } // Explains why x matches, or doesn't match, the matcher. void ExplainMatchResultTo(const T& x, ::std::ostream* os) const { StreamMatchResultListener listener(os); MatchAndExplain(x, &listener); } // Returns the describer for this matcher object; retains ownership // of the describer, which is only guaranteed to be alive when // this matcher object is alive. const MatcherDescriberInterface* GetDescriber() const { return impl_.get(); } protected: MatcherBase() {} // Constructs a matcher from its implementation. explicit MatcherBase(const MatcherInterface* impl) : impl_(impl) {} template explicit MatcherBase( const MatcherInterface* impl, typename std::enable_if::value>::type* = nullptr) : impl_(new internal::MatcherInterfaceAdapter(impl)) {} MatcherBase(const MatcherBase&) = default; MatcherBase& operator=(const MatcherBase&) = default; MatcherBase(MatcherBase&&) = default; MatcherBase& operator=(MatcherBase&&) = default; virtual ~MatcherBase() {} private: std::shared_ptr> impl_; }; } // namespace internal // A Matcher is a copyable and IMMUTABLE (except by assignment) // object that can check whether a value of type T matches. The // implementation of Matcher is just a std::shared_ptr to const // MatcherInterface. Don't inherit from Matcher! template class Matcher : public internal::MatcherBase { public: // Constructs a null matcher. Needed for storing Matcher objects in STL // containers. A default-constructed matcher is not yet initialized. You // cannot use it until a valid value has been assigned to it. explicit Matcher() {} // NOLINT // Constructs a matcher from its implementation. explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} template explicit Matcher( const MatcherInterface* impl, typename std::enable_if::value>::type* = nullptr) : internal::MatcherBase(impl) {} // Implicit constructor here allows people to write // EXPECT_CALL(foo, Bar(5)) instead of EXPECT_CALL(foo, Bar(Eq(5))) sometimes Matcher(T value); // NOLINT }; // The following two specializations allow the user to write str // instead of Eq(str) and "foo" instead of Eq("foo") when a std::string // matcher is expected. template <> class GTEST_API_ Matcher : public internal::MatcherBase { public: Matcher() {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} // Allows the user to write str instead of Eq(str) sometimes, where // str is a std::string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. Matcher(const char* s); // NOLINT }; template <> class GTEST_API_ Matcher : public internal::MatcherBase { public: Matcher() {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} // Allows the user to write str instead of Eq(str) sometimes, where // str is a string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. Matcher(const char* s); // NOLINT }; #if GTEST_HAS_ABSL // The following two specializations allow the user to write str // instead of Eq(str) and "foo" instead of Eq("foo") when a absl::string_view // matcher is expected. template <> class GTEST_API_ Matcher : public internal::MatcherBase { public: Matcher() {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} // Allows the user to write str instead of Eq(str) sometimes, where // str is a std::string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. Matcher(const char* s); // NOLINT // Allows the user to pass absl::string_views directly. Matcher(absl::string_view s); // NOLINT }; template <> class GTEST_API_ Matcher : public internal::MatcherBase { public: Matcher() {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} // Allows the user to write str instead of Eq(str) sometimes, where // str is a std::string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. Matcher(const char* s); // NOLINT // Allows the user to pass absl::string_views directly. Matcher(absl::string_view s); // NOLINT }; #endif // GTEST_HAS_ABSL // Prints a matcher in a human-readable format. template std::ostream& operator<<(std::ostream& os, const Matcher& matcher) { matcher.DescribeTo(&os); return os; } // The PolymorphicMatcher class template makes it easy to implement a // polymorphic matcher (i.e. a matcher that can match values of more // than one type, e.g. Eq(n) and NotNull()). // // To define a polymorphic matcher, a user should provide an Impl // class that has a DescribeTo() method and a DescribeNegationTo() // method, and define a member function (or member function template) // // bool MatchAndExplain(const Value& value, // MatchResultListener* listener) const; // // See the definition of NotNull() for a complete example. template class PolymorphicMatcher { public: explicit PolymorphicMatcher(const Impl& an_impl) : impl_(an_impl) {} // Returns a mutable reference to the underlying matcher // implementation object. Impl& mutable_impl() { return impl_; } // Returns an immutable reference to the underlying matcher // implementation object. const Impl& impl() const { return impl_; } template operator Matcher() const { return Matcher(new MonomorphicImpl(impl_)); } private: template class MonomorphicImpl : public MatcherInterface { public: explicit MonomorphicImpl(const Impl& impl) : impl_(impl) {} virtual void DescribeTo(::std::ostream* os) const { impl_.DescribeTo(os); } virtual void DescribeNegationTo(::std::ostream* os) const { impl_.DescribeNegationTo(os); } virtual bool MatchAndExplain(T x, MatchResultListener* listener) const { return impl_.MatchAndExplain(x, listener); } private: const Impl impl_; }; Impl impl_; }; // Creates a matcher from its implementation. // DEPRECATED: Especially in the generic code, prefer: // Matcher(new MyMatcherImpl(...)); // // MakeMatcher may create a Matcher that accepts its argument by value, which // leads to unnecessary copies & lack of support for non-copyable types. template inline Matcher MakeMatcher(const MatcherInterface* impl) { return Matcher(impl); } // Creates a polymorphic matcher from its implementation. This is // easier to use than the PolymorphicMatcher constructor as it // doesn't require you to explicitly write the template argument, e.g. // // MakePolymorphicMatcher(foo); // vs // PolymorphicMatcher(foo); template inline PolymorphicMatcher MakePolymorphicMatcher(const Impl& impl) { return PolymorphicMatcher(impl); } namespace internal { // Implements a matcher that compares a given value with a // pre-supplied value using one of the ==, <=, <, etc, operators. The // two values being compared don't have to have the same type. // // The matcher defined here is polymorphic (for example, Eq(5) can be // used to match an int, a short, a double, etc). Therefore we use // a template type conversion operator in the implementation. // // The following template definition assumes that the Rhs parameter is // a "bare" type (i.e. neither 'const T' nor 'T&'). template class ComparisonBase { public: explicit ComparisonBase(const Rhs& rhs) : rhs_(rhs) {} template operator Matcher() const { return Matcher(new Impl(rhs_)); } private: template static const T& Unwrap(const T& v) { return v; } template static const T& Unwrap(std::reference_wrapper v) { return v; } template class Impl : public MatcherInterface { public: explicit Impl(const Rhs& rhs) : rhs_(rhs) {} bool MatchAndExplain(Lhs lhs, MatchResultListener* /* listener */) const override { return Op()(lhs, Unwrap(rhs_)); } void DescribeTo(::std::ostream* os) const override { *os << D::Desc() << " "; UniversalPrint(Unwrap(rhs_), os); } void DescribeNegationTo(::std::ostream* os) const override { *os << D::NegatedDesc() << " "; UniversalPrint(Unwrap(rhs_), os); } private: Rhs rhs_; }; Rhs rhs_; }; template class EqMatcher : public ComparisonBase, Rhs, AnyEq> { public: explicit EqMatcher(const Rhs& rhs) : ComparisonBase, Rhs, AnyEq>(rhs) { } static const char* Desc() { return "is equal to"; } static const char* NegatedDesc() { return "isn't equal to"; } }; template class NeMatcher : public ComparisonBase, Rhs, AnyNe> { public: explicit NeMatcher(const Rhs& rhs) : ComparisonBase, Rhs, AnyNe>(rhs) { } static const char* Desc() { return "isn't equal to"; } static const char* NegatedDesc() { return "is equal to"; } }; template class LtMatcher : public ComparisonBase, Rhs, AnyLt> { public: explicit LtMatcher(const Rhs& rhs) : ComparisonBase, Rhs, AnyLt>(rhs) { } static const char* Desc() { return "is <"; } static const char* NegatedDesc() { return "isn't <"; } }; template class GtMatcher : public ComparisonBase, Rhs, AnyGt> { public: explicit GtMatcher(const Rhs& rhs) : ComparisonBase, Rhs, AnyGt>(rhs) { } static const char* Desc() { return "is >"; } static const char* NegatedDesc() { return "isn't >"; } }; template class LeMatcher : public ComparisonBase, Rhs, AnyLe> { public: explicit LeMatcher(const Rhs& rhs) : ComparisonBase, Rhs, AnyLe>(rhs) { } static const char* Desc() { return "is <="; } static const char* NegatedDesc() { return "isn't <="; } }; template class GeMatcher : public ComparisonBase, Rhs, AnyGe> { public: explicit GeMatcher(const Rhs& rhs) : ComparisonBase, Rhs, AnyGe>(rhs) { } static const char* Desc() { return "is >="; } static const char* NegatedDesc() { return "isn't >="; } }; // Implements polymorphic matchers MatchesRegex(regex) and // ContainsRegex(regex), which can be used as a Matcher as long as // T can be converted to a string. class MatchesRegexMatcher { public: MatchesRegexMatcher(const RE* regex, bool full_match) : regex_(regex), full_match_(full_match) {} #if GTEST_HAS_ABSL bool MatchAndExplain(const absl::string_view& s, MatchResultListener* listener) const { return MatchAndExplain(std::string(s), listener); } #endif // GTEST_HAS_ABSL // Accepts pointer types, particularly: // const char* // char* // const wchar_t* // wchar_t* template bool MatchAndExplain(CharType* s, MatchResultListener* listener) const { return s != nullptr && MatchAndExplain(std::string(s), listener); } // Matches anything that can convert to std::string. // // This is a template, not just a plain function with const std::string&, // because absl::string_view has some interfering non-explicit constructors. template bool MatchAndExplain(const MatcheeStringType& s, MatchResultListener* /* listener */) const { const std::string& s2(s); return full_match_ ? RE::FullMatch(s2, *regex_) : RE::PartialMatch(s2, *regex_); } void DescribeTo(::std::ostream* os) const { *os << (full_match_ ? "matches" : "contains") << " regular expression "; UniversalPrinter::Print(regex_->pattern(), os); } void DescribeNegationTo(::std::ostream* os) const { *os << "doesn't " << (full_match_ ? "match" : "contain") << " regular expression "; UniversalPrinter::Print(regex_->pattern(), os); } private: const std::shared_ptr regex_; const bool full_match_; }; } // namespace internal // Matches a string that fully matches regular expression 'regex'. // The matcher takes ownership of 'regex'. inline PolymorphicMatcher MatchesRegex( const internal::RE* regex) { return MakePolymorphicMatcher(internal::MatchesRegexMatcher(regex, true)); } inline PolymorphicMatcher MatchesRegex( const std::string& regex) { return MatchesRegex(new internal::RE(regex)); } // Matches a string that contains regular expression 'regex'. // The matcher takes ownership of 'regex'. inline PolymorphicMatcher ContainsRegex( const internal::RE* regex) { return MakePolymorphicMatcher(internal::MatchesRegexMatcher(regex, false)); } inline PolymorphicMatcher ContainsRegex( const std::string& regex) { return ContainsRegex(new internal::RE(regex)); } // Creates a polymorphic matcher that matches anything equal to x. // Note: if the parameter of Eq() were declared as const T&, Eq("foo") // wouldn't compile. template inline internal::EqMatcher Eq(T x) { return internal::EqMatcher(x); } // Constructs a Matcher from a 'value' of type T. The constructed // matcher matches any value that's equal to 'value'. template Matcher::Matcher(T value) { *this = Eq(value); } // Creates a monomorphic matcher that matches anything with type Lhs // and equal to rhs. A user may need to use this instead of Eq(...) // in order to resolve an overloading ambiguity. // // TypedEq(x) is just a convenient short-hand for Matcher(Eq(x)) // or Matcher(x), but more readable than the latter. // // We could define similar monomorphic matchers for other comparison // operations (e.g. TypedLt, TypedGe, and etc), but decided not to do // it yet as those are used much less than Eq() in practice. A user // can always write Matcher(Lt(5)) to be explicit about the type, // for example. template inline Matcher TypedEq(const Rhs& rhs) { return Eq(rhs); } // Creates a polymorphic matcher that matches anything >= x. template inline internal::GeMatcher Ge(Rhs x) { return internal::GeMatcher(x); } // Creates a polymorphic matcher that matches anything > x. template inline internal::GtMatcher Gt(Rhs x) { return internal::GtMatcher(x); } // Creates a polymorphic matcher that matches anything <= x. template inline internal::LeMatcher Le(Rhs x) { return internal::LeMatcher(x); } // Creates a polymorphic matcher that matches anything < x. template inline internal::LtMatcher Lt(Rhs x) { return internal::LtMatcher(x); } // Creates a polymorphic matcher that matches anything != x. template inline internal::NeMatcher Ne(Rhs x) { return internal::NeMatcher(x); } } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 5046 #endif // GTEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ #include #include namespace testing { namespace internal { GTEST_DECLARE_string_(internal_run_death_test); // Names of the flags (needed for parsing Google Test flags). const char kDeathTestStyleFlag[] = "death_test_style"; const char kDeathTestUseFork[] = "death_test_use_fork"; const char kInternalRunDeathTestFlag[] = "internal_run_death_test"; #if GTEST_HAS_DEATH_TEST GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) // DeathTest is a class that hides much of the complexity of the // GTEST_DEATH_TEST_ macro. It is abstract; its static Create method // returns a concrete class that depends on the prevailing death test // style, as defined by the --gtest_death_test_style and/or // --gtest_internal_run_death_test flags. // In describing the results of death tests, these terms are used with // the corresponding definitions: // // exit status: The integer exit information in the format specified // by wait(2) // exit code: The integer code passed to exit(3), _exit(2), or // returned from main() class GTEST_API_ DeathTest { public: // Create returns false if there was an error determining the // appropriate action to take for the current death test; for example, // if the gtest_death_test_style flag is set to an invalid value. // The LastMessage method will return a more detailed message in that // case. Otherwise, the DeathTest pointer pointed to by the "test" // argument is set. If the death test should be skipped, the pointer // is set to NULL; otherwise, it is set to the address of a new concrete // DeathTest object that controls the execution of the current test. static bool Create(const char* statement, Matcher matcher, const char* file, int line, DeathTest** test); DeathTest(); virtual ~DeathTest() { } // A helper class that aborts a death test when it's deleted. class ReturnSentinel { public: explicit ReturnSentinel(DeathTest* test) : test_(test) { } ~ReturnSentinel() { test_->Abort(TEST_ENCOUNTERED_RETURN_STATEMENT); } private: DeathTest* const test_; GTEST_DISALLOW_COPY_AND_ASSIGN_(ReturnSentinel); } GTEST_ATTRIBUTE_UNUSED_; // An enumeration of possible roles that may be taken when a death // test is encountered. EXECUTE means that the death test logic should // be executed immediately. OVERSEE means that the program should prepare // the appropriate environment for a child process to execute the death // test, then wait for it to complete. enum TestRole { OVERSEE_TEST, EXECUTE_TEST }; // An enumeration of the three reasons that a test might be aborted. enum AbortReason { TEST_ENCOUNTERED_RETURN_STATEMENT, TEST_THREW_EXCEPTION, TEST_DID_NOT_DIE }; // Assumes one of the above roles. virtual TestRole AssumeRole() = 0; // Waits for the death test to finish and returns its status. virtual int Wait() = 0; // Returns true if the death test passed; that is, the test process // exited during the test, its exit status matches a user-supplied // predicate, and its stderr output matches a user-supplied regular // expression. // The user-supplied predicate may be a macro expression rather // than a function pointer or functor, or else Wait and Passed could // be combined. virtual bool Passed(bool exit_status_ok) = 0; // Signals that the death test did not die as expected. virtual void Abort(AbortReason reason) = 0; // Returns a human-readable outcome message regarding the outcome of // the last death test. static const char* LastMessage(); static void set_last_death_test_message(const std::string& message); private: // A string containing a description of the outcome of the last death test. static std::string last_death_test_message_; GTEST_DISALLOW_COPY_AND_ASSIGN_(DeathTest); }; GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 // Factory interface for death tests. May be mocked out for testing. class DeathTestFactory { public: virtual ~DeathTestFactory() { } virtual bool Create(const char* statement, Matcher matcher, const char* file, int line, DeathTest** test) = 0; }; // A concrete DeathTestFactory implementation for normal use. class DefaultDeathTestFactory : public DeathTestFactory { public: bool Create(const char* statement, Matcher matcher, const char* file, int line, DeathTest** test) override; }; // Returns true if exit_status describes a process that was terminated // by a signal, or exited normally with a nonzero exit code. GTEST_API_ bool ExitedUnsuccessfully(int exit_status); // A string passed to EXPECT_DEATH (etc.) is caught by one of these overloads // and interpreted as a regex (rather than an Eq matcher) for legacy // compatibility. inline Matcher MakeDeathTestMatcher( ::testing::internal::RE regex) { return ContainsRegex(regex.pattern()); } inline Matcher MakeDeathTestMatcher(const char* regex) { return ContainsRegex(regex); } inline Matcher MakeDeathTestMatcher( const ::std::string& regex) { return ContainsRegex(regex); } // If a Matcher is passed to EXPECT_DEATH (etc.), it's // used directly. inline Matcher MakeDeathTestMatcher( Matcher matcher) { return matcher; } // Traps C++ exceptions escaping statement and reports them as test // failures. Note that trapping SEH exceptions is not implemented here. # if GTEST_HAS_EXCEPTIONS # define GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, death_test) \ try { \ GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ } catch (const ::std::exception& gtest_exception) { \ fprintf(\ stderr, \ "\n%s: Caught std::exception-derived exception escaping the " \ "death test statement. Exception message: %s\n", \ ::testing::internal::FormatFileLocation(__FILE__, __LINE__).c_str(), \ gtest_exception.what()); \ fflush(stderr); \ death_test->Abort(::testing::internal::DeathTest::TEST_THREW_EXCEPTION); \ } catch (...) { \ death_test->Abort(::testing::internal::DeathTest::TEST_THREW_EXCEPTION); \ } # else # define GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, death_test) \ GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) # endif // This macro is for implementing ASSERT_DEATH*, EXPECT_DEATH*, // ASSERT_EXIT*, and EXPECT_EXIT*. #define GTEST_DEATH_TEST_(statement, predicate, regex_or_matcher, fail) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (::testing::internal::AlwaysTrue()) { \ ::testing::internal::DeathTest* gtest_dt; \ if (!::testing::internal::DeathTest::Create( \ #statement, \ ::testing::internal::MakeDeathTestMatcher(regex_or_matcher), \ __FILE__, __LINE__, >est_dt)) { \ goto GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__); \ } \ if (gtest_dt != nullptr) { \ std::unique_ptr< ::testing::internal::DeathTest> gtest_dt_ptr(gtest_dt); \ switch (gtest_dt->AssumeRole()) { \ case ::testing::internal::DeathTest::OVERSEE_TEST: \ if (!gtest_dt->Passed(predicate(gtest_dt->Wait()))) { \ goto GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__); \ } \ break; \ case ::testing::internal::DeathTest::EXECUTE_TEST: { \ ::testing::internal::DeathTest::ReturnSentinel gtest_sentinel( \ gtest_dt); \ GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, gtest_dt); \ gtest_dt->Abort(::testing::internal::DeathTest::TEST_DID_NOT_DIE); \ break; \ } \ default: \ break; \ } \ } \ } else \ GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__) \ : fail(::testing::internal::DeathTest::LastMessage()) // The symbol "fail" here expands to something into which a message // can be streamed. // This macro is for implementing ASSERT/EXPECT_DEBUG_DEATH when compiled in // NDEBUG mode. In this case we need the statements to be executed and the macro // must accept a streamed message even though the message is never printed. // The regex object is not evaluated, but it is used to prevent "unused" // warnings and to avoid an expression that doesn't compile in debug mode. #define GTEST_EXECUTE_STATEMENT_(statement, regex_or_matcher) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (::testing::internal::AlwaysTrue()) { \ GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ } else if (!::testing::internal::AlwaysTrue()) { \ ::testing::internal::MakeDeathTestMatcher(regex_or_matcher); \ } else \ ::testing::Message() // A class representing the parsed contents of the // --gtest_internal_run_death_test flag, as it existed when // RUN_ALL_TESTS was called. class InternalRunDeathTestFlag { public: InternalRunDeathTestFlag(const std::string& a_file, int a_line, int an_index, int a_write_fd) : file_(a_file), line_(a_line), index_(an_index), write_fd_(a_write_fd) {} ~InternalRunDeathTestFlag() { if (write_fd_ >= 0) posix::Close(write_fd_); } const std::string& file() const { return file_; } int line() const { return line_; } int index() const { return index_; } int write_fd() const { return write_fd_; } private: std::string file_; int line_; int index_; int write_fd_; GTEST_DISALLOW_COPY_AND_ASSIGN_(InternalRunDeathTestFlag); }; // Returns a newly created InternalRunDeathTestFlag object with fields // initialized from the GTEST_FLAG(internal_run_death_test) flag if // the flag is specified; otherwise returns NULL. InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag(); #endif // GTEST_HAS_DEATH_TEST } // namespace internal } // namespace testing #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ namespace testing { // This flag controls the style of death tests. Valid values are "threadsafe", // meaning that the death test child process will re-execute the test binary // from the start, running only a single death test, or "fast", // meaning that the child process will execute the test logic immediately // after forking. GTEST_DECLARE_string_(death_test_style); #if GTEST_HAS_DEATH_TEST namespace internal { // Returns a Boolean value indicating whether the caller is currently // executing in the context of the death test child process. Tools such as // Valgrind heap checkers may need this to modify their behavior in death // tests. IMPORTANT: This is an internal utility. Using it may break the // implementation of death tests. User code MUST NOT use it. GTEST_API_ bool InDeathTestChild(); } // namespace internal // The following macros are useful for writing death tests. // Here's what happens when an ASSERT_DEATH* or EXPECT_DEATH* is // executed: // // 1. It generates a warning if there is more than one active // thread. This is because it's safe to fork() or clone() only // when there is a single thread. // // 2. The parent process clone()s a sub-process and runs the death // test in it; the sub-process exits with code 0 at the end of the // death test, if it hasn't exited already. // // 3. The parent process waits for the sub-process to terminate. // // 4. The parent process checks the exit code and error message of // the sub-process. // // Examples: // // ASSERT_DEATH(server.SendMessage(56, "Hello"), "Invalid port number"); // for (int i = 0; i < 5; i++) { // EXPECT_DEATH(server.ProcessRequest(i), // "Invalid request .* in ProcessRequest()") // << "Failed to die on request " << i; // } // // ASSERT_EXIT(server.ExitNow(), ::testing::ExitedWithCode(0), "Exiting"); // // bool KilledBySIGHUP(int exit_code) { // return WIFSIGNALED(exit_code) && WTERMSIG(exit_code) == SIGHUP; // } // // ASSERT_EXIT(client.HangUpServer(), KilledBySIGHUP, "Hanging up!"); // // On the regular expressions used in death tests: // // GOOGLETEST_CM0005 DO NOT DELETE // On POSIX-compliant systems (*nix), we use the library, // which uses the POSIX extended regex syntax. // // On other platforms (e.g. Windows or Mac), we only support a simple regex // syntax implemented as part of Google Test. This limited // implementation should be enough most of the time when writing // death tests; though it lacks many features you can find in PCRE // or POSIX extended regex syntax. For example, we don't support // union ("x|y"), grouping ("(xy)"), brackets ("[xy]"), and // repetition count ("x{5,7}"), among others. // // Below is the syntax that we do support. We chose it to be a // subset of both PCRE and POSIX extended regex, so it's easy to // learn wherever you come from. In the following: 'A' denotes a // literal character, period (.), or a single \\ escape sequence; // 'x' and 'y' denote regular expressions; 'm' and 'n' are for // natural numbers. // // c matches any literal character c // \\d matches any decimal digit // \\D matches any character that's not a decimal digit // \\f matches \f // \\n matches \n // \\r matches \r // \\s matches any ASCII whitespace, including \n // \\S matches any character that's not a whitespace // \\t matches \t // \\v matches \v // \\w matches any letter, _, or decimal digit // \\W matches any character that \\w doesn't match // \\c matches any literal character c, which must be a punctuation // . matches any single character except \n // A? matches 0 or 1 occurrences of A // A* matches 0 or many occurrences of A // A+ matches 1 or many occurrences of A // ^ matches the beginning of a string (not that of each line) // $ matches the end of a string (not that of each line) // xy matches x followed by y // // If you accidentally use PCRE or POSIX extended regex features // not implemented by us, you will get a run-time failure. In that // case, please try to rewrite your regular expression within the // above syntax. // // This implementation is *not* meant to be as highly tuned or robust // as a compiled regex library, but should perform well enough for a // death test, which already incurs significant overhead by launching // a child process. // // Known caveats: // // A "threadsafe" style death test obtains the path to the test // program from argv[0] and re-executes it in the sub-process. For // simplicity, the current implementation doesn't search the PATH // when launching the sub-process. This means that the user must // invoke the test program via a path that contains at least one // path separator (e.g. path/to/foo_test and // /absolute/path/to/bar_test are fine, but foo_test is not). This // is rarely a problem as people usually don't put the test binary // directory in PATH. // // Asserts that a given statement causes the program to exit, with an // integer exit status that satisfies predicate, and emitting error output // that matches regex. # define ASSERT_EXIT(statement, predicate, regex) \ GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_FATAL_FAILURE_) // Like ASSERT_EXIT, but continues on to successive tests in the // test suite, if any: # define EXPECT_EXIT(statement, predicate, regex) \ GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_NONFATAL_FAILURE_) // Asserts that a given statement causes the program to exit, either by // explicitly exiting with a nonzero exit code or being killed by a // signal, and emitting error output that matches regex. # define ASSERT_DEATH(statement, regex) \ ASSERT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) // Like ASSERT_DEATH, but continues on to successive tests in the // test suite, if any: # define EXPECT_DEATH(statement, regex) \ EXPECT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) // Two predicate classes that can be used in {ASSERT,EXPECT}_EXIT*: // Tests that an exit code describes a normal exit with a given exit code. class GTEST_API_ ExitedWithCode { public: explicit ExitedWithCode(int exit_code); bool operator()(int exit_status) const; private: // No implementation - assignment is unsupported. void operator=(const ExitedWithCode& other); const int exit_code_; }; # if !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA // Tests that an exit code describes an exit due to termination by a // given signal. // GOOGLETEST_CM0006 DO NOT DELETE class GTEST_API_ KilledBySignal { public: explicit KilledBySignal(int signum); bool operator()(int exit_status) const; private: const int signum_; }; # endif // !GTEST_OS_WINDOWS // EXPECT_DEBUG_DEATH asserts that the given statements die in debug mode. // The death testing framework causes this to have interesting semantics, // since the sideeffects of the call are only visible in opt mode, and not // in debug mode. // // In practice, this can be used to test functions that utilize the // LOG(DFATAL) macro using the following style: // // int DieInDebugOr12(int* sideeffect) { // if (sideeffect) { // *sideeffect = 12; // } // LOG(DFATAL) << "death"; // return 12; // } // // TEST(TestSuite, TestDieOr12WorksInDgbAndOpt) { // int sideeffect = 0; // // Only asserts in dbg. // EXPECT_DEBUG_DEATH(DieInDebugOr12(&sideeffect), "death"); // // #ifdef NDEBUG // // opt-mode has sideeffect visible. // EXPECT_EQ(12, sideeffect); // #else // // dbg-mode no visible sideeffect. // EXPECT_EQ(0, sideeffect); // #endif // } // // This will assert that DieInDebugReturn12InOpt() crashes in debug // mode, usually due to a DCHECK or LOG(DFATAL), but returns the // appropriate fallback value (12 in this case) in opt mode. If you // need to test that a function has appropriate side-effects in opt // mode, include assertions against the side-effects. A general // pattern for this is: // // EXPECT_DEBUG_DEATH({ // // Side-effects here will have an effect after this statement in // // opt mode, but none in debug mode. // EXPECT_EQ(12, DieInDebugOr12(&sideeffect)); // }, "death"); // # ifdef NDEBUG # define EXPECT_DEBUG_DEATH(statement, regex) \ GTEST_EXECUTE_STATEMENT_(statement, regex) # define ASSERT_DEBUG_DEATH(statement, regex) \ GTEST_EXECUTE_STATEMENT_(statement, regex) # else # define EXPECT_DEBUG_DEATH(statement, regex) \ EXPECT_DEATH(statement, regex) # define ASSERT_DEBUG_DEATH(statement, regex) \ ASSERT_DEATH(statement, regex) # endif // NDEBUG for EXPECT_DEBUG_DEATH #endif // GTEST_HAS_DEATH_TEST // This macro is used for implementing macros such as // EXPECT_DEATH_IF_SUPPORTED and ASSERT_DEATH_IF_SUPPORTED on systems where // death tests are not supported. Those macros must compile on such systems // if and only if EXPECT_DEATH and ASSERT_DEATH compile with the same parameters // on systems that support death tests. This allows one to write such a macro on // a system that does not support death tests and be sure that it will compile // on a death-test supporting system. It is exposed publicly so that systems // that have death-tests with stricter requirements than GTEST_HAS_DEATH_TEST // can write their own equivalent of EXPECT_DEATH_IF_SUPPORTED and // ASSERT_DEATH_IF_SUPPORTED. // // Parameters: // statement - A statement that a macro such as EXPECT_DEATH would test // for program termination. This macro has to make sure this // statement is compiled but not executed, to ensure that // EXPECT_DEATH_IF_SUPPORTED compiles with a certain // parameter if and only if EXPECT_DEATH compiles with it. // regex - A regex that a macro such as EXPECT_DEATH would use to test // the output of statement. This parameter has to be // compiled but not evaluated by this macro, to ensure that // this macro only accepts expressions that a macro such as // EXPECT_DEATH would accept. // terminator - Must be an empty statement for EXPECT_DEATH_IF_SUPPORTED // and a return statement for ASSERT_DEATH_IF_SUPPORTED. // This ensures that ASSERT_DEATH_IF_SUPPORTED will not // compile inside functions where ASSERT_DEATH doesn't // compile. // // The branch that has an always false condition is used to ensure that // statement and regex are compiled (and thus syntactically correct) but // never executed. The unreachable code macro protects the terminator // statement from generating an 'unreachable code' warning in case // statement unconditionally returns or throws. The Message constructor at // the end allows the syntax of streaming additional messages into the // macro, for compilational compatibility with EXPECT_DEATH/ASSERT_DEATH. # define GTEST_UNSUPPORTED_DEATH_TEST(statement, regex, terminator) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (::testing::internal::AlwaysTrue()) { \ GTEST_LOG_(WARNING) \ << "Death tests are not supported on this platform.\n" \ << "Statement '" #statement "' cannot be verified."; \ } else if (::testing::internal::AlwaysFalse()) { \ ::testing::internal::RE::PartialMatch(".*", (regex)); \ GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ terminator; \ } else \ ::testing::Message() // EXPECT_DEATH_IF_SUPPORTED(statement, regex) and // ASSERT_DEATH_IF_SUPPORTED(statement, regex) expand to real death tests if // death tests are supported; otherwise they just issue a warning. This is // useful when you are combining death test assertions with normal test // assertions in one test. #if GTEST_HAS_DEATH_TEST # define EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ EXPECT_DEATH(statement, regex) # define ASSERT_DEATH_IF_SUPPORTED(statement, regex) \ ASSERT_DEATH(statement, regex) #else # define EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ GTEST_UNSUPPORTED_DEATH_TEST(statement, regex, ) # define ASSERT_DEATH_IF_SUPPORTED(statement, regex) \ GTEST_UNSUPPORTED_DEATH_TEST(statement, regex, return) #endif } // namespace testing #endif // GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ // Copyright 2008, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Macros and functions for implementing parameterized tests // in Google C++ Testing and Mocking Framework (Google Test) // // This file is generated by a SCRIPT. DO NOT EDIT BY HAND! // // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ #define GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ // Value-parameterized tests allow you to test your code with different // parameters without writing multiple copies of the same test. // // Here is how you use value-parameterized tests: #if 0 // To write value-parameterized tests, first you should define a fixture // class. It is usually derived from testing::TestWithParam (see below for // another inheritance scheme that's sometimes useful in more complicated // class hierarchies), where the type of your parameter values. // TestWithParam is itself derived from testing::Test. T can be any // copyable type. If it's a raw pointer, you are responsible for managing the // lifespan of the pointed values. class FooTest : public ::testing::TestWithParam { // You can implement all the usual class fixture members here. }; // Then, use the TEST_P macro to define as many parameterized tests // for this fixture as you want. The _P suffix is for "parameterized" // or "pattern", whichever you prefer to think. TEST_P(FooTest, DoesBlah) { // Inside a test, access the test parameter with the GetParam() method // of the TestWithParam class: EXPECT_TRUE(foo.Blah(GetParam())); ... } TEST_P(FooTest, HasBlahBlah) { ... } // Finally, you can use INSTANTIATE_TEST_SUITE_P to instantiate the test // case with any set of parameters you want. Google Test defines a number // of functions for generating test parameters. They return what we call // (surprise!) parameter generators. Here is a summary of them, which // are all in the testing namespace: // // // Range(begin, end [, step]) - Yields values {begin, begin+step, // begin+step+step, ...}. The values do not // include end. step defaults to 1. // Values(v1, v2, ..., vN) - Yields values {v1, v2, ..., vN}. // ValuesIn(container) - Yields values from a C-style array, an STL // ValuesIn(begin,end) container, or an iterator range [begin, end). // Bool() - Yields sequence {false, true}. // Combine(g1, g2, ..., gN) - Yields all combinations (the Cartesian product // for the math savvy) of the values generated // by the N generators. // // For more details, see comments at the definitions of these functions below // in this file. // // The following statement will instantiate tests from the FooTest test suite // each with parameter values "meeny", "miny", and "moe". INSTANTIATE_TEST_SUITE_P(InstantiationName, FooTest, Values("meeny", "miny", "moe")); // To distinguish different instances of the pattern, (yes, you // can instantiate it more than once) the first argument to the // INSTANTIATE_TEST_SUITE_P macro is a prefix that will be added to the // actual test suite name. Remember to pick unique prefixes for different // instantiations. The tests from the instantiation above will have // these names: // // * InstantiationName/FooTest.DoesBlah/0 for "meeny" // * InstantiationName/FooTest.DoesBlah/1 for "miny" // * InstantiationName/FooTest.DoesBlah/2 for "moe" // * InstantiationName/FooTest.HasBlahBlah/0 for "meeny" // * InstantiationName/FooTest.HasBlahBlah/1 for "miny" // * InstantiationName/FooTest.HasBlahBlah/2 for "moe" // // You can use these names in --gtest_filter. // // This statement will instantiate all tests from FooTest again, each // with parameter values "cat" and "dog": const char* pets[] = {"cat", "dog"}; INSTANTIATE_TEST_SUITE_P(AnotherInstantiationName, FooTest, ValuesIn(pets)); // The tests from the instantiation above will have these names: // // * AnotherInstantiationName/FooTest.DoesBlah/0 for "cat" // * AnotherInstantiationName/FooTest.DoesBlah/1 for "dog" // * AnotherInstantiationName/FooTest.HasBlahBlah/0 for "cat" // * AnotherInstantiationName/FooTest.HasBlahBlah/1 for "dog" // // Please note that INSTANTIATE_TEST_SUITE_P will instantiate all tests // in the given test suite, whether their definitions come before or // AFTER the INSTANTIATE_TEST_SUITE_P statement. // // Please also note that generator expressions (including parameters to the // generators) are evaluated in InitGoogleTest(), after main() has started. // This allows the user on one hand, to adjust generator parameters in order // to dynamically determine a set of tests to run and on the other hand, // give the user a chance to inspect the generated tests with Google Test // reflection API before RUN_ALL_TESTS() is executed. // // You can see samples/sample7_unittest.cc and samples/sample8_unittest.cc // for more examples. // // In the future, we plan to publish the API for defining new parameter // generators. But for now this interface remains part of the internal // implementation and is subject to change. // // // A parameterized test fixture must be derived from testing::Test and from // testing::WithParamInterface, where T is the type of the parameter // values. Inheriting from TestWithParam satisfies that requirement because // TestWithParam inherits from both Test and WithParamInterface. In more // complicated hierarchies, however, it is occasionally useful to inherit // separately from Test and WithParamInterface. For example: class BaseTest : public ::testing::Test { // You can inherit all the usual members for a non-parameterized test // fixture here. }; class DerivedTest : public BaseTest, public ::testing::WithParamInterface { // The usual test fixture members go here too. }; TEST_F(BaseTest, HasFoo) { // This is an ordinary non-parameterized test. } TEST_P(DerivedTest, DoesBlah) { // GetParam works just the same here as if you inherit from TestWithParam. EXPECT_TRUE(foo.Blah(GetParam())); } #endif // 0 #include #include // Copyright 2008 Google Inc. // All Rights Reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Type and function utilities for implementing parameterized tests. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ #include #include #include #include #include #include #include #include namespace testing { // Input to a parameterized test name generator, describing a test parameter. // Consists of the parameter value and the integer parameter index. template struct TestParamInfo { TestParamInfo(const ParamType& a_param, size_t an_index) : param(a_param), index(an_index) {} ParamType param; size_t index; }; // A builtin parameterized test name generator which returns the result of // testing::PrintToString. struct PrintToStringParamName { template std::string operator()(const TestParamInfo& info) const { return PrintToString(info.param); } }; namespace internal { // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // Utility Functions // Outputs a message explaining invalid registration of different // fixture class for the same test suite. This may happen when // TEST_P macro is used to define two tests with the same name // but in different namespaces. GTEST_API_ void ReportInvalidTestSuiteType(const char* test_suite_name, CodeLocation code_location); template class ParamGeneratorInterface; template class ParamGenerator; // Interface for iterating over elements provided by an implementation // of ParamGeneratorInterface. template class ParamIteratorInterface { public: virtual ~ParamIteratorInterface() {} // A pointer to the base generator instance. // Used only for the purposes of iterator comparison // to make sure that two iterators belong to the same generator. virtual const ParamGeneratorInterface* BaseGenerator() const = 0; // Advances iterator to point to the next element // provided by the generator. The caller is responsible // for not calling Advance() on an iterator equal to // BaseGenerator()->End(). virtual void Advance() = 0; // Clones the iterator object. Used for implementing copy semantics // of ParamIterator. virtual ParamIteratorInterface* Clone() const = 0; // Dereferences the current iterator and provides (read-only) access // to the pointed value. It is the caller's responsibility not to call // Current() on an iterator equal to BaseGenerator()->End(). // Used for implementing ParamGenerator::operator*(). virtual const T* Current() const = 0; // Determines whether the given iterator and other point to the same // element in the sequence generated by the generator. // Used for implementing ParamGenerator::operator==(). virtual bool Equals(const ParamIteratorInterface& other) const = 0; }; // Class iterating over elements provided by an implementation of // ParamGeneratorInterface. It wraps ParamIteratorInterface // and implements the const forward iterator concept. template class ParamIterator { public: typedef T value_type; typedef const T& reference; typedef ptrdiff_t difference_type; // ParamIterator assumes ownership of the impl_ pointer. ParamIterator(const ParamIterator& other) : impl_(other.impl_->Clone()) {} ParamIterator& operator=(const ParamIterator& other) { if (this != &other) impl_.reset(other.impl_->Clone()); return *this; } const T& operator*() const { return *impl_->Current(); } const T* operator->() const { return impl_->Current(); } // Prefix version of operator++. ParamIterator& operator++() { impl_->Advance(); return *this; } // Postfix version of operator++. ParamIterator operator++(int /*unused*/) { ParamIteratorInterface* clone = impl_->Clone(); impl_->Advance(); return ParamIterator(clone); } bool operator==(const ParamIterator& other) const { return impl_.get() == other.impl_.get() || impl_->Equals(*other.impl_); } bool operator!=(const ParamIterator& other) const { return !(*this == other); } private: friend class ParamGenerator; explicit ParamIterator(ParamIteratorInterface* impl) : impl_(impl) {} std::unique_ptr > impl_; }; // ParamGeneratorInterface is the binary interface to access generators // defined in other translation units. template class ParamGeneratorInterface { public: typedef T ParamType; virtual ~ParamGeneratorInterface() {} // Generator interface definition virtual ParamIteratorInterface* Begin() const = 0; virtual ParamIteratorInterface* End() const = 0; }; // Wraps ParamGeneratorInterface and provides general generator syntax // compatible with the STL Container concept. // This class implements copy initialization semantics and the contained // ParamGeneratorInterface instance is shared among all copies // of the original object. This is possible because that instance is immutable. template class ParamGenerator { public: typedef ParamIterator iterator; explicit ParamGenerator(ParamGeneratorInterface* impl) : impl_(impl) {} ParamGenerator(const ParamGenerator& other) : impl_(other.impl_) {} ParamGenerator& operator=(const ParamGenerator& other) { impl_ = other.impl_; return *this; } iterator begin() const { return iterator(impl_->Begin()); } iterator end() const { return iterator(impl_->End()); } private: std::shared_ptr > impl_; }; // Generates values from a range of two comparable values. Can be used to // generate sequences of user-defined types that implement operator+() and // operator<(). // This class is used in the Range() function. template class RangeGenerator : public ParamGeneratorInterface { public: RangeGenerator(T begin, T end, IncrementT step) : begin_(begin), end_(end), step_(step), end_index_(CalculateEndIndex(begin, end, step)) {} ~RangeGenerator() override {} ParamIteratorInterface* Begin() const override { return new Iterator(this, begin_, 0, step_); } ParamIteratorInterface* End() const override { return new Iterator(this, end_, end_index_, step_); } private: class Iterator : public ParamIteratorInterface { public: Iterator(const ParamGeneratorInterface* base, T value, int index, IncrementT step) : base_(base), value_(value), index_(index), step_(step) {} ~Iterator() override {} const ParamGeneratorInterface* BaseGenerator() const override { return base_; } void Advance() override { value_ = static_cast(value_ + step_); index_++; } ParamIteratorInterface* Clone() const override { return new Iterator(*this); } const T* Current() const override { return &value_; } bool Equals(const ParamIteratorInterface& other) const override { // Having the same base generator guarantees that the other // iterator is of the same type and we can downcast. GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) << "The program attempted to compare iterators " << "from different generators." << std::endl; const int other_index = CheckedDowncastToActualType(&other)->index_; return index_ == other_index; } private: Iterator(const Iterator& other) : ParamIteratorInterface(), base_(other.base_), value_(other.value_), index_(other.index_), step_(other.step_) {} // No implementation - assignment is unsupported. void operator=(const Iterator& other); const ParamGeneratorInterface* const base_; T value_; int index_; const IncrementT step_; }; // class RangeGenerator::Iterator static int CalculateEndIndex(const T& begin, const T& end, const IncrementT& step) { int end_index = 0; for (T i = begin; i < end; i = static_cast(i + step)) end_index++; return end_index; } // No implementation - assignment is unsupported. void operator=(const RangeGenerator& other); const T begin_; const T end_; const IncrementT step_; // The index for the end() iterator. All the elements in the generated // sequence are indexed (0-based) to aid iterator comparison. const int end_index_; }; // class RangeGenerator // Generates values from a pair of STL-style iterators. Used in the // ValuesIn() function. The elements are copied from the source range // since the source can be located on the stack, and the generator // is likely to persist beyond that stack frame. template class ValuesInIteratorRangeGenerator : public ParamGeneratorInterface { public: template ValuesInIteratorRangeGenerator(ForwardIterator begin, ForwardIterator end) : container_(begin, end) {} ~ValuesInIteratorRangeGenerator() override {} ParamIteratorInterface* Begin() const override { return new Iterator(this, container_.begin()); } ParamIteratorInterface* End() const override { return new Iterator(this, container_.end()); } private: typedef typename ::std::vector ContainerType; class Iterator : public ParamIteratorInterface { public: Iterator(const ParamGeneratorInterface* base, typename ContainerType::const_iterator iterator) : base_(base), iterator_(iterator) {} ~Iterator() override {} const ParamGeneratorInterface* BaseGenerator() const override { return base_; } void Advance() override { ++iterator_; value_.reset(); } ParamIteratorInterface* Clone() const override { return new Iterator(*this); } // We need to use cached value referenced by iterator_ because *iterator_ // can return a temporary object (and of type other then T), so just // having "return &*iterator_;" doesn't work. // value_ is updated here and not in Advance() because Advance() // can advance iterator_ beyond the end of the range, and we cannot // detect that fact. The client code, on the other hand, is // responsible for not calling Current() on an out-of-range iterator. const T* Current() const override { if (value_.get() == nullptr) value_.reset(new T(*iterator_)); return value_.get(); } bool Equals(const ParamIteratorInterface& other) const override { // Having the same base generator guarantees that the other // iterator is of the same type and we can downcast. GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) << "The program attempted to compare iterators " << "from different generators." << std::endl; return iterator_ == CheckedDowncastToActualType(&other)->iterator_; } private: Iterator(const Iterator& other) // The explicit constructor call suppresses a false warning // emitted by gcc when supplied with the -Wextra option. : ParamIteratorInterface(), base_(other.base_), iterator_(other.iterator_) {} const ParamGeneratorInterface* const base_; typename ContainerType::const_iterator iterator_; // A cached value of *iterator_. We keep it here to allow access by // pointer in the wrapping iterator's operator->(). // value_ needs to be mutable to be accessed in Current(). // Use of std::unique_ptr helps manage cached value's lifetime, // which is bound by the lifespan of the iterator itself. mutable std::unique_ptr value_; }; // class ValuesInIteratorRangeGenerator::Iterator // No implementation - assignment is unsupported. void operator=(const ValuesInIteratorRangeGenerator& other); const ContainerType container_; }; // class ValuesInIteratorRangeGenerator // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Default parameterized test name generator, returns a string containing the // integer test parameter index. template std::string DefaultParamName(const TestParamInfo& info) { Message name_stream; name_stream << info.index; return name_stream.GetString(); } template void TestNotEmpty() { static_assert(sizeof(T) == 0, "Empty arguments are not allowed."); } template void TestNotEmpty(const T&) {} // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Stores a parameter value and later creates tests parameterized with that // value. template class ParameterizedTestFactory : public TestFactoryBase { public: typedef typename TestClass::ParamType ParamType; explicit ParameterizedTestFactory(ParamType parameter) : parameter_(parameter) {} Test* CreateTest() override { TestClass::SetParam(¶meter_); return new TestClass(); } private: const ParamType parameter_; GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestFactory); }; // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // TestMetaFactoryBase is a base class for meta-factories that create // test factories for passing into MakeAndRegisterTestInfo function. template class TestMetaFactoryBase { public: virtual ~TestMetaFactoryBase() {} virtual TestFactoryBase* CreateTestFactory(ParamType parameter) = 0; }; // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // TestMetaFactory creates test factories for passing into // MakeAndRegisterTestInfo function. Since MakeAndRegisterTestInfo receives // ownership of test factory pointer, same factory object cannot be passed // into that method twice. But ParameterizedTestSuiteInfo is going to call // it for each Test/Parameter value combination. Thus it needs meta factory // creator class. template class TestMetaFactory : public TestMetaFactoryBase { public: using ParamType = typename TestSuite::ParamType; TestMetaFactory() {} TestFactoryBase* CreateTestFactory(ParamType parameter) override { return new ParameterizedTestFactory(parameter); } private: GTEST_DISALLOW_COPY_AND_ASSIGN_(TestMetaFactory); }; // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // ParameterizedTestSuiteInfoBase is a generic interface // to ParameterizedTestSuiteInfo classes. ParameterizedTestSuiteInfoBase // accumulates test information provided by TEST_P macro invocations // and generators provided by INSTANTIATE_TEST_SUITE_P macro invocations // and uses that information to register all resulting test instances // in RegisterTests method. The ParameterizeTestSuiteRegistry class holds // a collection of pointers to the ParameterizedTestSuiteInfo objects // and calls RegisterTests() on each of them when asked. class ParameterizedTestSuiteInfoBase { public: virtual ~ParameterizedTestSuiteInfoBase() {} // Base part of test suite name for display purposes. virtual const std::string& GetTestSuiteName() const = 0; // Test case id to verify identity. virtual TypeId GetTestSuiteTypeId() const = 0; // UnitTest class invokes this method to register tests in this // test suite right before running them in RUN_ALL_TESTS macro. // This method should not be called more than once on any single // instance of a ParameterizedTestSuiteInfoBase derived class. virtual void RegisterTests() = 0; protected: ParameterizedTestSuiteInfoBase() {} private: GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestSuiteInfoBase); }; // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // ParameterizedTestSuiteInfo accumulates tests obtained from TEST_P // macro invocations for a particular test suite and generators // obtained from INSTANTIATE_TEST_SUITE_P macro invocations for that // test suite. It registers tests with all values generated by all // generators when asked. template class ParameterizedTestSuiteInfo : public ParameterizedTestSuiteInfoBase { public: // ParamType and GeneratorCreationFunc are private types but are required // for declarations of public methods AddTestPattern() and // AddTestSuiteInstantiation(). using ParamType = typename TestSuite::ParamType; // A function that returns an instance of appropriate generator type. typedef ParamGenerator(GeneratorCreationFunc)(); using ParamNameGeneratorFunc = std::string(const TestParamInfo&); explicit ParameterizedTestSuiteInfo(const char* name, CodeLocation code_location) : test_suite_name_(name), code_location_(code_location) {} // Test case base name for display purposes. const std::string& GetTestSuiteName() const override { return test_suite_name_; } // Test case id to verify identity. TypeId GetTestSuiteTypeId() const override { return GetTypeId(); } // TEST_P macro uses AddTestPattern() to record information // about a single test in a LocalTestInfo structure. // test_suite_name is the base name of the test suite (without invocation // prefix). test_base_name is the name of an individual test without // parameter index. For the test SequenceA/FooTest.DoBar/1 FooTest is // test suite base name and DoBar is test base name. void AddTestPattern(const char* test_suite_name, const char* test_base_name, TestMetaFactoryBase* meta_factory) { tests_.push_back(std::shared_ptr( new TestInfo(test_suite_name, test_base_name, meta_factory))); } // INSTANTIATE_TEST_SUITE_P macro uses AddGenerator() to record information // about a generator. int AddTestSuiteInstantiation(const std::string& instantiation_name, GeneratorCreationFunc* func, ParamNameGeneratorFunc* name_func, const char* file, int line) { instantiations_.push_back( InstantiationInfo(instantiation_name, func, name_func, file, line)); return 0; // Return value used only to run this method in namespace scope. } // UnitTest class invokes this method to register tests in this test suite // test suites right before running tests in RUN_ALL_TESTS macro. // This method should not be called more than once on any single // instance of a ParameterizedTestSuiteInfoBase derived class. // UnitTest has a guard to prevent from calling this method more than once. void RegisterTests() override { for (typename TestInfoContainer::iterator test_it = tests_.begin(); test_it != tests_.end(); ++test_it) { std::shared_ptr test_info = *test_it; for (typename InstantiationContainer::iterator gen_it = instantiations_.begin(); gen_it != instantiations_.end(); ++gen_it) { const std::string& instantiation_name = gen_it->name; ParamGenerator generator((*gen_it->generator)()); ParamNameGeneratorFunc* name_func = gen_it->name_func; const char* file = gen_it->file; int line = gen_it->line; std::string test_suite_name; if ( !instantiation_name.empty() ) test_suite_name = instantiation_name + "/"; test_suite_name += test_info->test_suite_base_name; size_t i = 0; std::set test_param_names; for (typename ParamGenerator::iterator param_it = generator.begin(); param_it != generator.end(); ++param_it, ++i) { Message test_name_stream; std::string param_name = name_func( TestParamInfo(*param_it, i)); GTEST_CHECK_(IsValidParamName(param_name)) << "Parameterized test name '" << param_name << "' is invalid, in " << file << " line " << line << std::endl; GTEST_CHECK_(test_param_names.count(param_name) == 0) << "Duplicate parameterized test name '" << param_name << "', in " << file << " line " << line << std::endl; test_param_names.insert(param_name); if (!test_info->test_base_name.empty()) { test_name_stream << test_info->test_base_name << "/"; } test_name_stream << param_name; MakeAndRegisterTestInfo( test_suite_name.c_str(), test_name_stream.GetString().c_str(), nullptr, // No type parameter. PrintToString(*param_it).c_str(), code_location_, GetTestSuiteTypeId(), SuiteApiResolver::GetSetUpCaseOrSuite(file, line), SuiteApiResolver::GetTearDownCaseOrSuite(file, line), test_info->test_meta_factory->CreateTestFactory(*param_it)); } // for param_it } // for gen_it } // for test_it } // RegisterTests private: // LocalTestInfo structure keeps information about a single test registered // with TEST_P macro. struct TestInfo { TestInfo(const char* a_test_suite_base_name, const char* a_test_base_name, TestMetaFactoryBase* a_test_meta_factory) : test_suite_base_name(a_test_suite_base_name), test_base_name(a_test_base_name), test_meta_factory(a_test_meta_factory) {} const std::string test_suite_base_name; const std::string test_base_name; const std::unique_ptr > test_meta_factory; }; using TestInfoContainer = ::std::vector >; // Records data received from INSTANTIATE_TEST_SUITE_P macros: // struct InstantiationInfo { InstantiationInfo(const std::string &name_in, GeneratorCreationFunc* generator_in, ParamNameGeneratorFunc* name_func_in, const char* file_in, int line_in) : name(name_in), generator(generator_in), name_func(name_func_in), file(file_in), line(line_in) {} std::string name; GeneratorCreationFunc* generator; ParamNameGeneratorFunc* name_func; const char* file; int line; }; typedef ::std::vector InstantiationContainer; static bool IsValidParamName(const std::string& name) { // Check for empty string if (name.empty()) return false; // Check for invalid characters for (std::string::size_type index = 0; index < name.size(); ++index) { if (!isalnum(name[index]) && name[index] != '_') return false; } return true; } const std::string test_suite_name_; CodeLocation code_location_; TestInfoContainer tests_; InstantiationContainer instantiations_; GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestSuiteInfo); }; // class ParameterizedTestSuiteInfo // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ template using ParameterizedTestCaseInfo = ParameterizedTestSuiteInfo; #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // ParameterizedTestSuiteRegistry contains a map of // ParameterizedTestSuiteInfoBase classes accessed by test suite names. TEST_P // and INSTANTIATE_TEST_SUITE_P macros use it to locate their corresponding // ParameterizedTestSuiteInfo descriptors. class ParameterizedTestSuiteRegistry { public: ParameterizedTestSuiteRegistry() {} ~ParameterizedTestSuiteRegistry() { for (auto& test_suite_info : test_suite_infos_) { delete test_suite_info; } } // Looks up or creates and returns a structure containing information about // tests and instantiations of a particular test suite. template ParameterizedTestSuiteInfo* GetTestSuitePatternHolder( const char* test_suite_name, CodeLocation code_location) { ParameterizedTestSuiteInfo* typed_test_info = nullptr; for (auto& test_suite_info : test_suite_infos_) { if (test_suite_info->GetTestSuiteName() == test_suite_name) { if (test_suite_info->GetTestSuiteTypeId() != GetTypeId()) { // Complain about incorrect usage of Google Test facilities // and terminate the program since we cannot guaranty correct // test suite setup and tear-down in this case. ReportInvalidTestSuiteType(test_suite_name, code_location); posix::Abort(); } else { // At this point we are sure that the object we found is of the same // type we are looking for, so we downcast it to that type // without further checks. typed_test_info = CheckedDowncastToActualType< ParameterizedTestSuiteInfo >(test_suite_info); } break; } } if (typed_test_info == nullptr) { typed_test_info = new ParameterizedTestSuiteInfo( test_suite_name, code_location); test_suite_infos_.push_back(typed_test_info); } return typed_test_info; } void RegisterTests() { for (auto& test_suite_info : test_suite_infos_) { test_suite_info->RegisterTests(); } } // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ template ParameterizedTestCaseInfo* GetTestCasePatternHolder( const char* test_case_name, CodeLocation code_location) { return GetTestSuitePatternHolder(test_case_name, code_location); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ private: using TestSuiteInfoContainer = ::std::vector; TestSuiteInfoContainer test_suite_infos_; GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestSuiteRegistry); }; } // namespace internal // Forward declarations of ValuesIn(), which is implemented in // include/gtest/gtest-param-test.h. template internal::ParamGenerator ValuesIn( const Container& container); namespace internal { // Used in the Values() function to provide polymorphic capabilities. template class ValueArray { public: ValueArray(Ts... v) : v_{std::move(v)...} {} template operator ParamGenerator() const { // NOLINT return ValuesIn(MakeVector(MakeIndexSequence())); } private: template std::vector MakeVector(IndexSequence) const { return std::vector{static_cast(v_.template Get())...}; } FlatTuple v_; }; template class CartesianProductGenerator : public ParamGeneratorInterface<::std::tuple> { public: typedef ::std::tuple ParamType; CartesianProductGenerator(const std::tuple...>& g) : generators_(g) {} ~CartesianProductGenerator() override {} ParamIteratorInterface* Begin() const override { return new Iterator(this, generators_, false); } ParamIteratorInterface* End() const override { return new Iterator(this, generators_, true); } private: template class IteratorImpl; template class IteratorImpl> : public ParamIteratorInterface { public: IteratorImpl(const ParamGeneratorInterface* base, const std::tuple...>& generators, bool is_end) : base_(base), begin_(std::get(generators).begin()...), end_(std::get(generators).end()...), current_(is_end ? end_ : begin_) { ComputeCurrentValue(); } ~IteratorImpl() override {} const ParamGeneratorInterface* BaseGenerator() const override { return base_; } // Advance should not be called on beyond-of-range iterators // so no component iterators must be beyond end of range, either. void Advance() override { assert(!AtEnd()); // Advance the last iterator. ++std::get(current_); // if that reaches end, propagate that up. AdvanceIfEnd(); ComputeCurrentValue(); } ParamIteratorInterface* Clone() const override { return new IteratorImpl(*this); } const ParamType* Current() const override { return current_value_.get(); } bool Equals(const ParamIteratorInterface& other) const override { // Having the same base generator guarantees that the other // iterator is of the same type and we can downcast. GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) << "The program attempted to compare iterators " << "from different generators." << std::endl; const IteratorImpl* typed_other = CheckedDowncastToActualType(&other); // We must report iterators equal if they both point beyond their // respective ranges. That can happen in a variety of fashions, // so we have to consult AtEnd(). if (AtEnd() && typed_other->AtEnd()) return true; bool same = true; bool dummy[] = { (same = same && std::get(current_) == std::get(typed_other->current_))...}; (void)dummy; return same; } private: template void AdvanceIfEnd() { if (std::get(current_) != std::get(end_)) return; bool last = ThisI == 0; if (last) { // We are done. Nothing else to propagate. return; } constexpr size_t NextI = ThisI - (ThisI != 0); std::get(current_) = std::get(begin_); ++std::get(current_); AdvanceIfEnd(); } void ComputeCurrentValue() { if (!AtEnd()) current_value_ = std::make_shared(*std::get(current_)...); } bool AtEnd() const { bool at_end = false; bool dummy[] = { (at_end = at_end || std::get(current_) == std::get(end_))...}; (void)dummy; return at_end; } const ParamGeneratorInterface* const base_; std::tuple::iterator...> begin_; std::tuple::iterator...> end_; std::tuple::iterator...> current_; std::shared_ptr current_value_; }; using Iterator = IteratorImpl::type>; std::tuple...> generators_; }; template class CartesianProductHolder { public: CartesianProductHolder(const Gen&... g) : generators_(g...) {} template operator ParamGenerator<::std::tuple>() const { return ParamGenerator<::std::tuple>( new CartesianProductGenerator(generators_)); } private: std::tuple generators_; }; } // namespace internal } // namespace testing #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ namespace testing { // Functions producing parameter generators. // // Google Test uses these generators to produce parameters for value- // parameterized tests. When a parameterized test suite is instantiated // with a particular generator, Google Test creates and runs tests // for each element in the sequence produced by the generator. // // In the following sample, tests from test suite FooTest are instantiated // each three times with parameter values 3, 5, and 8: // // class FooTest : public TestWithParam { ... }; // // TEST_P(FooTest, TestThis) { // } // TEST_P(FooTest, TestThat) { // } // INSTANTIATE_TEST_SUITE_P(TestSequence, FooTest, Values(3, 5, 8)); // // Range() returns generators providing sequences of values in a range. // // Synopsis: // Range(start, end) // - returns a generator producing a sequence of values {start, start+1, // start+2, ..., }. // Range(start, end, step) // - returns a generator producing a sequence of values {start, start+step, // start+step+step, ..., }. // Notes: // * The generated sequences never include end. For example, Range(1, 5) // returns a generator producing a sequence {1, 2, 3, 4}. Range(1, 9, 2) // returns a generator producing {1, 3, 5, 7}. // * start and end must have the same type. That type may be any integral or // floating-point type or a user defined type satisfying these conditions: // * It must be assignable (have operator=() defined). // * It must have operator+() (operator+(int-compatible type) for // two-operand version). // * It must have operator<() defined. // Elements in the resulting sequences will also have that type. // * Condition start < end must be satisfied in order for resulting sequences // to contain any elements. // template internal::ParamGenerator Range(T start, T end, IncrementT step) { return internal::ParamGenerator( new internal::RangeGenerator(start, end, step)); } template internal::ParamGenerator Range(T start, T end) { return Range(start, end, 1); } // ValuesIn() function allows generation of tests with parameters coming from // a container. // // Synopsis: // ValuesIn(const T (&array)[N]) // - returns a generator producing sequences with elements from // a C-style array. // ValuesIn(const Container& container) // - returns a generator producing sequences with elements from // an STL-style container. // ValuesIn(Iterator begin, Iterator end) // - returns a generator producing sequences with elements from // a range [begin, end) defined by a pair of STL-style iterators. These // iterators can also be plain C pointers. // // Please note that ValuesIn copies the values from the containers // passed in and keeps them to generate tests in RUN_ALL_TESTS(). // // Examples: // // This instantiates tests from test suite StringTest // each with C-string values of "foo", "bar", and "baz": // // const char* strings[] = {"foo", "bar", "baz"}; // INSTANTIATE_TEST_SUITE_P(StringSequence, StringTest, ValuesIn(strings)); // // This instantiates tests from test suite StlStringTest // each with STL strings with values "a" and "b": // // ::std::vector< ::std::string> GetParameterStrings() { // ::std::vector< ::std::string> v; // v.push_back("a"); // v.push_back("b"); // return v; // } // // INSTANTIATE_TEST_SUITE_P(CharSequence, // StlStringTest, // ValuesIn(GetParameterStrings())); // // // This will also instantiate tests from CharTest // each with parameter values 'a' and 'b': // // ::std::list GetParameterChars() { // ::std::list list; // list.push_back('a'); // list.push_back('b'); // return list; // } // ::std::list l = GetParameterChars(); // INSTANTIATE_TEST_SUITE_P(CharSequence2, // CharTest, // ValuesIn(l.begin(), l.end())); // template internal::ParamGenerator< typename std::iterator_traits::value_type> ValuesIn(ForwardIterator begin, ForwardIterator end) { typedef typename std::iterator_traits::value_type ParamType; return internal::ParamGenerator( new internal::ValuesInIteratorRangeGenerator(begin, end)); } template internal::ParamGenerator ValuesIn(const T (&array)[N]) { return ValuesIn(array, array + N); } template internal::ParamGenerator ValuesIn( const Container& container) { return ValuesIn(container.begin(), container.end()); } // Values() allows generating tests from explicitly specified list of // parameters. // // Synopsis: // Values(T v1, T v2, ..., T vN) // - returns a generator producing sequences with elements v1, v2, ..., vN. // // For example, this instantiates tests from test suite BarTest each // with values "one", "two", and "three": // // INSTANTIATE_TEST_SUITE_P(NumSequence, // BarTest, // Values("one", "two", "three")); // // This instantiates tests from test suite BazTest each with values 1, 2, 3.5. // The exact type of values will depend on the type of parameter in BazTest. // // INSTANTIATE_TEST_SUITE_P(FloatingNumbers, BazTest, Values(1, 2, 3.5)); // // template internal::ValueArray Values(T... v) { return internal::ValueArray(std::move(v)...); } // Bool() allows generating tests with parameters in a set of (false, true). // // Synopsis: // Bool() // - returns a generator producing sequences with elements {false, true}. // // It is useful when testing code that depends on Boolean flags. Combinations // of multiple flags can be tested when several Bool()'s are combined using // Combine() function. // // In the following example all tests in the test suite FlagDependentTest // will be instantiated twice with parameters false and true. // // class FlagDependentTest : public testing::TestWithParam { // virtual void SetUp() { // external_flag = GetParam(); // } // } // INSTANTIATE_TEST_SUITE_P(BoolSequence, FlagDependentTest, Bool()); // inline internal::ParamGenerator Bool() { return Values(false, true); } // Combine() allows the user to combine two or more sequences to produce // values of a Cartesian product of those sequences' elements. // // Synopsis: // Combine(gen1, gen2, ..., genN) // - returns a generator producing sequences with elements coming from // the Cartesian product of elements from the sequences generated by // gen1, gen2, ..., genN. The sequence elements will have a type of // std::tuple where T1, T2, ..., TN are the types // of elements from sequences produces by gen1, gen2, ..., genN. // // Combine can have up to 10 arguments. // // Example: // // This will instantiate tests in test suite AnimalTest each one with // the parameter values tuple("cat", BLACK), tuple("cat", WHITE), // tuple("dog", BLACK), and tuple("dog", WHITE): // // enum Color { BLACK, GRAY, WHITE }; // class AnimalTest // : public testing::TestWithParam > {...}; // // TEST_P(AnimalTest, AnimalLooksNice) {...} // // INSTANTIATE_TEST_SUITE_P(AnimalVariations, AnimalTest, // Combine(Values("cat", "dog"), // Values(BLACK, WHITE))); // // This will instantiate tests in FlagDependentTest with all variations of two // Boolean flags: // // class FlagDependentTest // : public testing::TestWithParam > { // virtual void SetUp() { // // Assigns external_flag_1 and external_flag_2 values from the tuple. // std::tie(external_flag_1, external_flag_2) = GetParam(); // } // }; // // TEST_P(FlagDependentTest, TestFeature1) { // // Test your code using external_flag_1 and external_flag_2 here. // } // INSTANTIATE_TEST_SUITE_P(TwoBoolSequence, FlagDependentTest, // Combine(Bool(), Bool())); // template internal::CartesianProductHolder Combine(const Generator&... g) { return internal::CartesianProductHolder(g...); } #define TEST_P(test_suite_name, test_name) \ class GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) \ : public test_suite_name { \ public: \ GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() {} \ virtual void TestBody(); \ \ private: \ static int AddToRegistry() { \ ::testing::UnitTest::GetInstance() \ ->parameterized_test_registry() \ .GetTestSuitePatternHolder( \ #test_suite_name, \ ::testing::internal::CodeLocation(__FILE__, __LINE__)) \ ->AddTestPattern( \ GTEST_STRINGIFY_(test_suite_name), GTEST_STRINGIFY_(test_name), \ new ::testing::internal::TestMetaFactory()); \ return 0; \ } \ static int gtest_registering_dummy_ GTEST_ATTRIBUTE_UNUSED_; \ GTEST_DISALLOW_COPY_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name, \ test_name)); \ }; \ int GTEST_TEST_CLASS_NAME_(test_suite_name, \ test_name)::gtest_registering_dummy_ = \ GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)::AddToRegistry(); \ void GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)::TestBody() // The last argument to INSTANTIATE_TEST_SUITE_P allows the user to specify // generator and an optional function or functor that generates custom test name // suffixes based on the test parameters. Such a function or functor should // accept one argument of type testing::TestParamInfo, and // return std::string. // // testing::PrintToStringParamName is a builtin test suffix generator that // returns the value of testing::PrintToString(GetParam()). // // Note: test names must be non-empty, unique, and may only contain ASCII // alphanumeric characters or underscore. Because PrintToString adds quotes // to std::string and C strings, it won't work for these types. #define GTEST_EXPAND_(arg) arg #define GTEST_GET_FIRST_(first, ...) first #define GTEST_GET_SECOND_(first, second, ...) second #define INSTANTIATE_TEST_SUITE_P(prefix, test_suite_name, ...) \ static ::testing::internal::ParamGenerator \ gtest_##prefix##test_suite_name##_EvalGenerator_() { \ return GTEST_EXPAND_(GTEST_GET_FIRST_(__VA_ARGS__, DUMMY_PARAM_)); \ } \ static ::std::string gtest_##prefix##test_suite_name##_EvalGenerateName_( \ const ::testing::TestParamInfo& info) { \ if (::testing::internal::AlwaysFalse()) { \ ::testing::internal::TestNotEmpty(GTEST_EXPAND_(GTEST_GET_SECOND_( \ __VA_ARGS__, \ ::testing::internal::DefaultParamName, \ DUMMY_PARAM_))); \ auto t = std::make_tuple(__VA_ARGS__); \ static_assert(std::tuple_size::value <= 2, \ "Too Many Args!"); \ } \ return ((GTEST_EXPAND_(GTEST_GET_SECOND_( \ __VA_ARGS__, \ ::testing::internal::DefaultParamName, \ DUMMY_PARAM_))))(info); \ } \ static int gtest_##prefix##test_suite_name##_dummy_ \ GTEST_ATTRIBUTE_UNUSED_ = \ ::testing::UnitTest::GetInstance() \ ->parameterized_test_registry() \ .GetTestSuitePatternHolder( \ #test_suite_name, \ ::testing::internal::CodeLocation(__FILE__, __LINE__)) \ ->AddTestSuiteInstantiation( \ #prefix, >est_##prefix##test_suite_name##_EvalGenerator_, \ >est_##prefix##test_suite_name##_EvalGenerateName_, \ __FILE__, __LINE__) // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ #define INSTANTIATE_TEST_CASE_P \ static_assert(::testing::internal::InstantiateTestCase_P_IsDeprecated(), \ ""); \ INSTANTIATE_TEST_SUITE_P #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ } // namespace testing #endif // GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ // Copyright 2006, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Google C++ Testing and Mocking Framework definitions useful in production code. // GOOGLETEST_CM0003 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_GTEST_PROD_H_ #define GTEST_INCLUDE_GTEST_GTEST_PROD_H_ // When you need to test the private or protected members of a class, // use the FRIEND_TEST macro to declare your tests as friends of the // class. For example: // // class MyClass { // private: // void PrivateMethod(); // FRIEND_TEST(MyClassTest, PrivateMethodWorks); // }; // // class MyClassTest : public testing::Test { // // ... // }; // // TEST_F(MyClassTest, PrivateMethodWorks) { // // Can call MyClass::PrivateMethod() here. // } // // Note: The test class must be in the same namespace as the class being tested. // For example, putting MyClassTest in an anonymous namespace will not work. #define FRIEND_TEST(test_case_name, test_name)\ friend class test_case_name##_##test_name##_Test #endif // GTEST_INCLUDE_GTEST_GTEST_PROD_H_ // Copyright 2008, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ #define GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ #include #include GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) namespace testing { // A copyable object representing the result of a test part (i.e. an // assertion or an explicit FAIL(), ADD_FAILURE(), or SUCCESS()). // // Don't inherit from TestPartResult as its destructor is not virtual. class GTEST_API_ TestPartResult { public: // The possible outcomes of a test part (i.e. an assertion or an // explicit SUCCEED(), FAIL(), or ADD_FAILURE()). enum Type { kSuccess, // Succeeded. kNonFatalFailure, // Failed but the test can continue. kFatalFailure, // Failed and the test should be terminated. kSkip // Skipped. }; // C'tor. TestPartResult does NOT have a default constructor. // Always use this constructor (with parameters) to create a // TestPartResult object. TestPartResult(Type a_type, const char* a_file_name, int a_line_number, const char* a_message) : type_(a_type), file_name_(a_file_name == nullptr ? "" : a_file_name), line_number_(a_line_number), summary_(ExtractSummary(a_message)), message_(a_message) {} // Gets the outcome of the test part. Type type() const { return type_; } // Gets the name of the source file where the test part took place, or // NULL if it's unknown. const char* file_name() const { return file_name_.empty() ? nullptr : file_name_.c_str(); } // Gets the line in the source file where the test part took place, // or -1 if it's unknown. int line_number() const { return line_number_; } // Gets the summary of the failure message. const char* summary() const { return summary_.c_str(); } // Gets the message associated with the test part. const char* message() const { return message_.c_str(); } // Returns true if and only if the test part was skipped. bool skipped() const { return type_ == kSkip; } // Returns true if and only if the test part passed. bool passed() const { return type_ == kSuccess; } // Returns true if and only if the test part non-fatally failed. bool nonfatally_failed() const { return type_ == kNonFatalFailure; } // Returns true if and only if the test part fatally failed. bool fatally_failed() const { return type_ == kFatalFailure; } // Returns true if and only if the test part failed. bool failed() const { return fatally_failed() || nonfatally_failed(); } private: Type type_; // Gets the summary of the failure message by omitting the stack // trace in it. static std::string ExtractSummary(const char* message); // The name of the source file where the test part took place, or // "" if the source file is unknown. std::string file_name_; // The line in the source file where the test part took place, or -1 // if the line number is unknown. int line_number_; std::string summary_; // The test failure summary. std::string message_; // The test failure message. }; // Prints a TestPartResult object. std::ostream& operator<<(std::ostream& os, const TestPartResult& result); // An array of TestPartResult objects. // // Don't inherit from TestPartResultArray as its destructor is not // virtual. class GTEST_API_ TestPartResultArray { public: TestPartResultArray() {} // Appends the given TestPartResult to the array. void Append(const TestPartResult& result); // Returns the TestPartResult at the given index (0-based). const TestPartResult& GetTestPartResult(int index) const; // Returns the number of TestPartResult objects in the array. int size() const; private: std::vector array_; GTEST_DISALLOW_COPY_AND_ASSIGN_(TestPartResultArray); }; // This interface knows how to report a test part result. class GTEST_API_ TestPartResultReporterInterface { public: virtual ~TestPartResultReporterInterface() {} virtual void ReportTestPartResult(const TestPartResult& result) = 0; }; namespace internal { // This helper class is used by {ASSERT|EXPECT}_NO_FATAL_FAILURE to check if a // statement generates new fatal failures. To do so it registers itself as the // current test part result reporter. Besides checking if fatal failures were // reported, it only delegates the reporting to the former result reporter. // The original result reporter is restored in the destructor. // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. class GTEST_API_ HasNewFatalFailureHelper : public TestPartResultReporterInterface { public: HasNewFatalFailureHelper(); ~HasNewFatalFailureHelper() override; void ReportTestPartResult(const TestPartResult& result) override; bool has_new_fatal_failure() const { return has_new_fatal_failure_; } private: bool has_new_fatal_failure_; TestPartResultReporterInterface* original_reporter_; GTEST_DISALLOW_COPY_AND_ASSIGN_(HasNewFatalFailureHelper); }; } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 #endif // GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ // Copyright 2008 Google Inc. // All Rights Reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ #define GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ // This header implements typed tests and type-parameterized tests. // Typed (aka type-driven) tests repeat the same test for types in a // list. You must know which types you want to test with when writing // typed tests. Here's how you do it: #if 0 // First, define a fixture class template. It should be parameterized // by a type. Remember to derive it from testing::Test. template class FooTest : public testing::Test { public: ... typedef std::list List; static T shared_; T value_; }; // Next, associate a list of types with the test suite, which will be // repeated for each type in the list. The typedef is necessary for // the macro to parse correctly. typedef testing::Types MyTypes; TYPED_TEST_SUITE(FooTest, MyTypes); // If the type list contains only one type, you can write that type // directly without Types<...>: // TYPED_TEST_SUITE(FooTest, int); // Then, use TYPED_TEST() instead of TEST_F() to define as many typed // tests for this test suite as you want. TYPED_TEST(FooTest, DoesBlah) { // Inside a test, refer to the special name TypeParam to get the type // parameter. Since we are inside a derived class template, C++ requires // us to visit the members of FooTest via 'this'. TypeParam n = this->value_; // To visit static members of the fixture, add the TestFixture:: // prefix. n += TestFixture::shared_; // To refer to typedefs in the fixture, add the "typename // TestFixture::" prefix. typename TestFixture::List values; values.push_back(n); ... } TYPED_TEST(FooTest, HasPropertyA) { ... } // TYPED_TEST_SUITE takes an optional third argument which allows to specify a // class that generates custom test name suffixes based on the type. This should // be a class which has a static template function GetName(int index) returning // a string for each type. The provided integer index equals the index of the // type in the provided type list. In many cases the index can be ignored. // // For example: // class MyTypeNames { // public: // template // static std::string GetName(int) { // if (std::is_same()) return "char"; // if (std::is_same()) return "int"; // if (std::is_same()) return "unsignedInt"; // } // }; // TYPED_TEST_SUITE(FooTest, MyTypes, MyTypeNames); #endif // 0 // Type-parameterized tests are abstract test patterns parameterized // by a type. Compared with typed tests, type-parameterized tests // allow you to define the test pattern without knowing what the type // parameters are. The defined pattern can be instantiated with // different types any number of times, in any number of translation // units. // // If you are designing an interface or concept, you can define a // suite of type-parameterized tests to verify properties that any // valid implementation of the interface/concept should have. Then, // each implementation can easily instantiate the test suite to verify // that it conforms to the requirements, without having to write // similar tests repeatedly. Here's an example: #if 0 // First, define a fixture class template. It should be parameterized // by a type. Remember to derive it from testing::Test. template class FooTest : public testing::Test { ... }; // Next, declare that you will define a type-parameterized test suite // (the _P suffix is for "parameterized" or "pattern", whichever you // prefer): TYPED_TEST_SUITE_P(FooTest); // Then, use TYPED_TEST_P() to define as many type-parameterized tests // for this type-parameterized test suite as you want. TYPED_TEST_P(FooTest, DoesBlah) { // Inside a test, refer to TypeParam to get the type parameter. TypeParam n = 0; ... } TYPED_TEST_P(FooTest, HasPropertyA) { ... } // Now the tricky part: you need to register all test patterns before // you can instantiate them. The first argument of the macro is the // test suite name; the rest are the names of the tests in this test // case. REGISTER_TYPED_TEST_SUITE_P(FooTest, DoesBlah, HasPropertyA); // Finally, you are free to instantiate the pattern with the types you // want. If you put the above code in a header file, you can #include // it in multiple C++ source files and instantiate it multiple times. // // To distinguish different instances of the pattern, the first // argument to the INSTANTIATE_* macro is a prefix that will be added // to the actual test suite name. Remember to pick unique prefixes for // different instances. typedef testing::Types MyTypes; INSTANTIATE_TYPED_TEST_SUITE_P(My, FooTest, MyTypes); // If the type list contains only one type, you can write that type // directly without Types<...>: // INSTANTIATE_TYPED_TEST_SUITE_P(My, FooTest, int); // // Similar to the optional argument of TYPED_TEST_SUITE above, // INSTANTIATE_TEST_SUITE_P takes an optional fourth argument which allows to // generate custom names. // INSTANTIATE_TYPED_TEST_SUITE_P(My, FooTest, MyTypes, MyTypeNames); #endif // 0 // Implements typed tests. #if GTEST_HAS_TYPED_TEST // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Expands to the name of the typedef for the type parameters of the // given test suite. #define GTEST_TYPE_PARAMS_(TestSuiteName) gtest_type_params_##TestSuiteName##_ // Expands to the name of the typedef for the NameGenerator, responsible for // creating the suffixes of the name. #define GTEST_NAME_GENERATOR_(TestSuiteName) \ gtest_type_params_##TestSuiteName##_NameGenerator #define TYPED_TEST_SUITE(CaseName, Types, ...) \ typedef ::testing::internal::TypeList::type GTEST_TYPE_PARAMS_( \ CaseName); \ typedef ::testing::internal::NameGeneratorSelector<__VA_ARGS__>::type \ GTEST_NAME_GENERATOR_(CaseName) # define TYPED_TEST(CaseName, TestName) \ template \ class GTEST_TEST_CLASS_NAME_(CaseName, TestName) \ : public CaseName { \ private: \ typedef CaseName TestFixture; \ typedef gtest_TypeParam_ TypeParam; \ virtual void TestBody(); \ }; \ static bool gtest_##CaseName##_##TestName##_registered_ \ GTEST_ATTRIBUTE_UNUSED_ = \ ::testing::internal::TypeParameterizedTest< \ CaseName, \ ::testing::internal::TemplateSel, \ GTEST_TYPE_PARAMS_( \ CaseName)>::Register("", \ ::testing::internal::CodeLocation( \ __FILE__, __LINE__), \ #CaseName, #TestName, 0, \ ::testing::internal::GenerateNames< \ GTEST_NAME_GENERATOR_(CaseName), \ GTEST_TYPE_PARAMS_(CaseName)>()); \ template \ void GTEST_TEST_CLASS_NAME_(CaseName, \ TestName)::TestBody() // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ #define TYPED_TEST_CASE \ static_assert(::testing::internal::TypedTestCaseIsDeprecated(), ""); \ TYPED_TEST_SUITE #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ #endif // GTEST_HAS_TYPED_TEST // Implements type-parameterized tests. #if GTEST_HAS_TYPED_TEST_P // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Expands to the namespace name that the type-parameterized tests for // the given type-parameterized test suite are defined in. The exact // name of the namespace is subject to change without notice. #define GTEST_SUITE_NAMESPACE_(TestSuiteName) gtest_suite_##TestSuiteName##_ // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Expands to the name of the variable used to remember the names of // the defined tests in the given test suite. #define GTEST_TYPED_TEST_SUITE_P_STATE_(TestSuiteName) \ gtest_typed_test_suite_p_state_##TestSuiteName##_ // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE DIRECTLY. // // Expands to the name of the variable used to remember the names of // the registered tests in the given test suite. #define GTEST_REGISTERED_TEST_NAMES_(TestSuiteName) \ gtest_registered_test_names_##TestSuiteName##_ // The variables defined in the type-parameterized test macros are // static as typically these macros are used in a .h file that can be // #included in multiple translation units linked together. #define TYPED_TEST_SUITE_P(SuiteName) \ static ::testing::internal::TypedTestSuitePState \ GTEST_TYPED_TEST_SUITE_P_STATE_(SuiteName) // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ #define TYPED_TEST_CASE_P \ static_assert(::testing::internal::TypedTestCase_P_IsDeprecated(), ""); \ TYPED_TEST_SUITE_P #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ #define TYPED_TEST_P(SuiteName, TestName) \ namespace GTEST_SUITE_NAMESPACE_(SuiteName) { \ template \ class TestName : public SuiteName { \ private: \ typedef SuiteName TestFixture; \ typedef gtest_TypeParam_ TypeParam; \ virtual void TestBody(); \ }; \ static bool gtest_##TestName##_defined_ GTEST_ATTRIBUTE_UNUSED_ = \ GTEST_TYPED_TEST_SUITE_P_STATE_(SuiteName).AddTestName( \ __FILE__, __LINE__, #SuiteName, #TestName); \ } \ template \ void GTEST_SUITE_NAMESPACE_( \ SuiteName)::TestName::TestBody() #define REGISTER_TYPED_TEST_SUITE_P(SuiteName, ...) \ namespace GTEST_SUITE_NAMESPACE_(SuiteName) { \ typedef ::testing::internal::Templates<__VA_ARGS__>::type gtest_AllTests_; \ } \ static const char* const GTEST_REGISTERED_TEST_NAMES_( \ SuiteName) GTEST_ATTRIBUTE_UNUSED_ = \ GTEST_TYPED_TEST_SUITE_P_STATE_(SuiteName).VerifyRegisteredTestNames( \ __FILE__, __LINE__, #__VA_ARGS__) // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ #define REGISTER_TYPED_TEST_CASE_P \ static_assert(::testing::internal::RegisterTypedTestCase_P_IsDeprecated(), \ ""); \ REGISTER_TYPED_TEST_SUITE_P #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ #define INSTANTIATE_TYPED_TEST_SUITE_P(Prefix, SuiteName, Types, ...) \ static bool gtest_##Prefix##_##SuiteName GTEST_ATTRIBUTE_UNUSED_ = \ ::testing::internal::TypeParameterizedTestSuite< \ SuiteName, GTEST_SUITE_NAMESPACE_(SuiteName)::gtest_AllTests_, \ ::testing::internal::TypeList::type>:: \ Register(#Prefix, \ ::testing::internal::CodeLocation(__FILE__, __LINE__), \ >EST_TYPED_TEST_SUITE_P_STATE_(SuiteName), #SuiteName, \ GTEST_REGISTERED_TEST_NAMES_(SuiteName), \ ::testing::internal::GenerateNames< \ ::testing::internal::NameGeneratorSelector< \ __VA_ARGS__>::type, \ ::testing::internal::TypeList::type>()) // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ #define INSTANTIATE_TYPED_TEST_CASE_P \ static_assert( \ ::testing::internal::InstantiateTypedTestCase_P_IsDeprecated(), ""); \ INSTANTIATE_TYPED_TEST_SUITE_P #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ #endif // GTEST_HAS_TYPED_TEST_P #endif // GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) namespace testing { // Silence C4100 (unreferenced formal parameter) and 4805 // unsafe mix of type 'const int' and type 'const bool' #ifdef _MSC_VER # pragma warning(push) # pragma warning(disable:4805) # pragma warning(disable:4100) #endif // Declares the flags. // This flag temporary enables the disabled tests. GTEST_DECLARE_bool_(also_run_disabled_tests); // This flag brings the debugger on an assertion failure. GTEST_DECLARE_bool_(break_on_failure); // This flag controls whether Google Test catches all test-thrown exceptions // and logs them as failures. GTEST_DECLARE_bool_(catch_exceptions); // This flag enables using colors in terminal output. Available values are // "yes" to enable colors, "no" (disable colors), or "auto" (the default) // to let Google Test decide. GTEST_DECLARE_string_(color); // This flag sets up the filter to select by name using a glob pattern // the tests to run. If the filter is not given all tests are executed. GTEST_DECLARE_string_(filter); // This flag controls whether Google Test installs a signal handler that dumps // debugging information when fatal signals are raised. GTEST_DECLARE_bool_(install_failure_signal_handler); // This flag causes the Google Test to list tests. None of the tests listed // are actually run if the flag is provided. GTEST_DECLARE_bool_(list_tests); // This flag controls whether Google Test emits a detailed XML report to a file // in addition to its normal textual output. GTEST_DECLARE_string_(output); // This flags control whether Google Test prints the elapsed time for each // test. GTEST_DECLARE_bool_(print_time); // This flags control whether Google Test prints UTF8 characters as text. GTEST_DECLARE_bool_(print_utf8); // This flag specifies the random number seed. GTEST_DECLARE_int32_(random_seed); // This flag sets how many times the tests are repeated. The default value // is 1. If the value is -1 the tests are repeating forever. GTEST_DECLARE_int32_(repeat); // This flag controls whether Google Test includes Google Test internal // stack frames in failure stack traces. GTEST_DECLARE_bool_(show_internal_stack_frames); // When this flag is specified, tests' order is randomized on every iteration. GTEST_DECLARE_bool_(shuffle); // This flag specifies the maximum number of stack frames to be // printed in a failure message. GTEST_DECLARE_int32_(stack_trace_depth); // When this flag is specified, a failed assertion will throw an // exception if exceptions are enabled, or exit the program with a // non-zero code otherwise. For use with an external test framework. GTEST_DECLARE_bool_(throw_on_failure); // When this flag is specified, list of skipped test names is printed in // summary GTEST_DECLARE_bool_(print_skipped); // When this flag is set with a "host:port" string, on supported // platforms test results are streamed to the specified port on // the specified host machine. GTEST_DECLARE_string_(stream_result_to); #if GTEST_USE_OWN_FLAGFILE_FLAG_ GTEST_DECLARE_string_(flagfile); #endif // GTEST_USE_OWN_FLAGFILE_FLAG_ // The upper limit for valid stack trace depths. const int kMaxStackTraceDepth = 100; namespace internal { class AssertHelper; class DefaultGlobalTestPartResultReporter; class ExecDeathTest; class NoExecDeathTest; class FinalSuccessChecker; class GTestFlagSaver; class StreamingListenerTest; class TestResultAccessor; class TestEventListenersAccessor; class TestEventRepeater; class UnitTestRecordPropertyTestHelper; class WindowsDeathTest; class FuchsiaDeathTest; class UnitTestImpl* GetUnitTestImpl(); void ReportFailureInUnknownLocation(TestPartResult::Type result_type, const std::string& message); } // namespace internal // The friend relationship of some of these classes is cyclic. // If we don't forward declare them the compiler might confuse the classes // in friendship clauses with same named classes on the scope. class Test; class TestSuite; // Old API is still available but deprecated #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ using TestCase = TestSuite; #endif class TestInfo; class UnitTest; // A class for indicating whether an assertion was successful. When // the assertion wasn't successful, the AssertionResult object // remembers a non-empty message that describes how it failed. // // To create an instance of this class, use one of the factory functions // (AssertionSuccess() and AssertionFailure()). // // This class is useful for two purposes: // 1. Defining predicate functions to be used with Boolean test assertions // EXPECT_TRUE/EXPECT_FALSE and their ASSERT_ counterparts // 2. Defining predicate-format functions to be // used with predicate assertions (ASSERT_PRED_FORMAT*, etc). // // For example, if you define IsEven predicate: // // testing::AssertionResult IsEven(int n) { // if ((n % 2) == 0) // return testing::AssertionSuccess(); // else // return testing::AssertionFailure() << n << " is odd"; // } // // Then the failed expectation EXPECT_TRUE(IsEven(Fib(5))) // will print the message // // Value of: IsEven(Fib(5)) // Actual: false (5 is odd) // Expected: true // // instead of a more opaque // // Value of: IsEven(Fib(5)) // Actual: false // Expected: true // // in case IsEven is a simple Boolean predicate. // // If you expect your predicate to be reused and want to support informative // messages in EXPECT_FALSE and ASSERT_FALSE (negative assertions show up // about half as often as positive ones in our tests), supply messages for // both success and failure cases: // // testing::AssertionResult IsEven(int n) { // if ((n % 2) == 0) // return testing::AssertionSuccess() << n << " is even"; // else // return testing::AssertionFailure() << n << " is odd"; // } // // Then a statement EXPECT_FALSE(IsEven(Fib(6))) will print // // Value of: IsEven(Fib(6)) // Actual: true (8 is even) // Expected: false // // NB: Predicates that support negative Boolean assertions have reduced // performance in positive ones so be careful not to use them in tests // that have lots (tens of thousands) of positive Boolean assertions. // // To use this class with EXPECT_PRED_FORMAT assertions such as: // // // Verifies that Foo() returns an even number. // EXPECT_PRED_FORMAT1(IsEven, Foo()); // // you need to define: // // testing::AssertionResult IsEven(const char* expr, int n) { // if ((n % 2) == 0) // return testing::AssertionSuccess(); // else // return testing::AssertionFailure() // << "Expected: " << expr << " is even\n Actual: it's " << n; // } // // If Foo() returns 5, you will see the following message: // // Expected: Foo() is even // Actual: it's 5 // class GTEST_API_ AssertionResult { public: // Copy constructor. // Used in EXPECT_TRUE/FALSE(assertion_result). AssertionResult(const AssertionResult& other); #if defined(_MSC_VER) && _MSC_VER < 1910 GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 /* forcing value to bool */) #endif // Used in the EXPECT_TRUE/FALSE(bool_expression). // // T must be contextually convertible to bool. // // The second parameter prevents this overload from being considered if // the argument is implicitly convertible to AssertionResult. In that case // we want AssertionResult's copy constructor to be used. template explicit AssertionResult( const T& success, typename std::enable_if< !std::is_convertible::value>::type* /*enabler*/ = nullptr) : success_(success) {} #if defined(_MSC_VER) && _MSC_VER < 1910 GTEST_DISABLE_MSC_WARNINGS_POP_() #endif // Assignment operator. AssertionResult& operator=(AssertionResult other) { swap(other); return *this; } // Returns true if and only if the assertion succeeded. operator bool() const { return success_; } // NOLINT // Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. AssertionResult operator!() const; // Returns the text streamed into this AssertionResult. Test assertions // use it when they fail (i.e., the predicate's outcome doesn't match the // assertion's expectation). When nothing has been streamed into the // object, returns an empty string. const char* message() const { return message_.get() != nullptr ? message_->c_str() : ""; } // Deprecated; please use message() instead. const char* failure_message() const { return message(); } // Streams a custom failure message into this object. template AssertionResult& operator<<(const T& value) { AppendMessage(Message() << value); return *this; } // Allows streaming basic output manipulators such as endl or flush into // this object. AssertionResult& operator<<( ::std::ostream& (*basic_manipulator)(::std::ostream& stream)) { AppendMessage(Message() << basic_manipulator); return *this; } private: // Appends the contents of message to message_. void AppendMessage(const Message& a_message) { if (message_.get() == nullptr) message_.reset(new ::std::string); message_->append(a_message.GetString().c_str()); } // Swap the contents of this AssertionResult with other. void swap(AssertionResult& other); // Stores result of the assertion predicate. bool success_; // Stores the message describing the condition in case the expectation // construct is not satisfied with the predicate's outcome. // Referenced via a pointer to avoid taking too much stack frame space // with test assertions. std::unique_ptr< ::std::string> message_; }; // Makes a successful assertion result. GTEST_API_ AssertionResult AssertionSuccess(); // Makes a failed assertion result. GTEST_API_ AssertionResult AssertionFailure(); // Makes a failed assertion result with the given failure message. // Deprecated; use AssertionFailure() << msg. GTEST_API_ AssertionResult AssertionFailure(const Message& msg); } // namespace testing // Includes the auto-generated header that implements a family of generic // predicate assertion macros. This include comes late because it relies on // APIs declared above. // Copyright 2006, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // This file is AUTOMATICALLY GENERATED on 01/02/2019 by command // 'gen_gtest_pred_impl.py 5'. DO NOT EDIT BY HAND! // // Implements a family of generic predicate assertion macros. // GOOGLETEST_CM0001 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ #define GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ namespace testing { // This header implements a family of generic predicate assertion // macros: // // ASSERT_PRED_FORMAT1(pred_format, v1) // ASSERT_PRED_FORMAT2(pred_format, v1, v2) // ... // // where pred_format is a function or functor that takes n (in the // case of ASSERT_PRED_FORMATn) values and their source expression // text, and returns a testing::AssertionResult. See the definition // of ASSERT_EQ in gtest.h for an example. // // If you don't care about formatting, you can use the more // restrictive version: // // ASSERT_PRED1(pred, v1) // ASSERT_PRED2(pred, v1, v2) // ... // // where pred is an n-ary function or functor that returns bool, // and the values v1, v2, ..., must support the << operator for // streaming to std::ostream. // // We also define the EXPECT_* variations. // // For now we only support predicates whose arity is at most 5. // Please email googletestframework@googlegroups.com if you need // support for higher arities. // GTEST_ASSERT_ is the basic statement to which all of the assertions // in this file reduce. Don't use this in your code. #define GTEST_ASSERT_(expression, on_failure) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (const ::testing::AssertionResult gtest_ar = (expression)) \ ; \ else \ on_failure(gtest_ar.failure_message()) // Helper function for implementing {EXPECT|ASSERT}_PRED1. Don't use // this in your code. template AssertionResult AssertPred1Helper(const char* pred_text, const char* e1, Pred pred, const T1& v1) { if (pred(v1)) return AssertionSuccess(); return AssertionFailure() << pred_text << "(" << e1 << ") evaluates to false, where" << "\n" << e1 << " evaluates to " << ::testing::PrintToString(v1); } // Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT1. // Don't use this in your code. #define GTEST_PRED_FORMAT1_(pred_format, v1, on_failure)\ GTEST_ASSERT_(pred_format(#v1, v1), \ on_failure) // Internal macro for implementing {EXPECT|ASSERT}_PRED1. Don't use // this in your code. #define GTEST_PRED1_(pred, v1, on_failure)\ GTEST_ASSERT_(::testing::AssertPred1Helper(#pred, \ #v1, \ pred, \ v1), on_failure) // Unary predicate assertion macros. #define EXPECT_PRED_FORMAT1(pred_format, v1) \ GTEST_PRED_FORMAT1_(pred_format, v1, GTEST_NONFATAL_FAILURE_) #define EXPECT_PRED1(pred, v1) \ GTEST_PRED1_(pred, v1, GTEST_NONFATAL_FAILURE_) #define ASSERT_PRED_FORMAT1(pred_format, v1) \ GTEST_PRED_FORMAT1_(pred_format, v1, GTEST_FATAL_FAILURE_) #define ASSERT_PRED1(pred, v1) \ GTEST_PRED1_(pred, v1, GTEST_FATAL_FAILURE_) // Helper function for implementing {EXPECT|ASSERT}_PRED2. Don't use // this in your code. template AssertionResult AssertPred2Helper(const char* pred_text, const char* e1, const char* e2, Pred pred, const T1& v1, const T2& v2) { if (pred(v1, v2)) return AssertionSuccess(); return AssertionFailure() << pred_text << "(" << e1 << ", " << e2 << ") evaluates to false, where" << "\n" << e1 << " evaluates to " << ::testing::PrintToString(v1) << "\n" << e2 << " evaluates to " << ::testing::PrintToString(v2); } // Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT2. // Don't use this in your code. #define GTEST_PRED_FORMAT2_(pred_format, v1, v2, on_failure)\ GTEST_ASSERT_(pred_format(#v1, #v2, v1, v2), \ on_failure) // Internal macro for implementing {EXPECT|ASSERT}_PRED2. Don't use // this in your code. #define GTEST_PRED2_(pred, v1, v2, on_failure)\ GTEST_ASSERT_(::testing::AssertPred2Helper(#pred, \ #v1, \ #v2, \ pred, \ v1, \ v2), on_failure) // Binary predicate assertion macros. #define EXPECT_PRED_FORMAT2(pred_format, v1, v2) \ GTEST_PRED_FORMAT2_(pred_format, v1, v2, GTEST_NONFATAL_FAILURE_) #define EXPECT_PRED2(pred, v1, v2) \ GTEST_PRED2_(pred, v1, v2, GTEST_NONFATAL_FAILURE_) #define ASSERT_PRED_FORMAT2(pred_format, v1, v2) \ GTEST_PRED_FORMAT2_(pred_format, v1, v2, GTEST_FATAL_FAILURE_) #define ASSERT_PRED2(pred, v1, v2) \ GTEST_PRED2_(pred, v1, v2, GTEST_FATAL_FAILURE_) // Helper function for implementing {EXPECT|ASSERT}_PRED3. Don't use // this in your code. template AssertionResult AssertPred3Helper(const char* pred_text, const char* e1, const char* e2, const char* e3, Pred pred, const T1& v1, const T2& v2, const T3& v3) { if (pred(v1, v2, v3)) return AssertionSuccess(); return AssertionFailure() << pred_text << "(" << e1 << ", " << e2 << ", " << e3 << ") evaluates to false, where" << "\n" << e1 << " evaluates to " << ::testing::PrintToString(v1) << "\n" << e2 << " evaluates to " << ::testing::PrintToString(v2) << "\n" << e3 << " evaluates to " << ::testing::PrintToString(v3); } // Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT3. // Don't use this in your code. #define GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, on_failure)\ GTEST_ASSERT_(pred_format(#v1, #v2, #v3, v1, v2, v3), \ on_failure) // Internal macro for implementing {EXPECT|ASSERT}_PRED3. Don't use // this in your code. #define GTEST_PRED3_(pred, v1, v2, v3, on_failure)\ GTEST_ASSERT_(::testing::AssertPred3Helper(#pred, \ #v1, \ #v2, \ #v3, \ pred, \ v1, \ v2, \ v3), on_failure) // Ternary predicate assertion macros. #define EXPECT_PRED_FORMAT3(pred_format, v1, v2, v3) \ GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, GTEST_NONFATAL_FAILURE_) #define EXPECT_PRED3(pred, v1, v2, v3) \ GTEST_PRED3_(pred, v1, v2, v3, GTEST_NONFATAL_FAILURE_) #define ASSERT_PRED_FORMAT3(pred_format, v1, v2, v3) \ GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, GTEST_FATAL_FAILURE_) #define ASSERT_PRED3(pred, v1, v2, v3) \ GTEST_PRED3_(pred, v1, v2, v3, GTEST_FATAL_FAILURE_) // Helper function for implementing {EXPECT|ASSERT}_PRED4. Don't use // this in your code. template AssertionResult AssertPred4Helper(const char* pred_text, const char* e1, const char* e2, const char* e3, const char* e4, Pred pred, const T1& v1, const T2& v2, const T3& v3, const T4& v4) { if (pred(v1, v2, v3, v4)) return AssertionSuccess(); return AssertionFailure() << pred_text << "(" << e1 << ", " << e2 << ", " << e3 << ", " << e4 << ") evaluates to false, where" << "\n" << e1 << " evaluates to " << ::testing::PrintToString(v1) << "\n" << e2 << " evaluates to " << ::testing::PrintToString(v2) << "\n" << e3 << " evaluates to " << ::testing::PrintToString(v3) << "\n" << e4 << " evaluates to " << ::testing::PrintToString(v4); } // Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT4. // Don't use this in your code. #define GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, on_failure)\ GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, v1, v2, v3, v4), \ on_failure) // Internal macro for implementing {EXPECT|ASSERT}_PRED4. Don't use // this in your code. #define GTEST_PRED4_(pred, v1, v2, v3, v4, on_failure)\ GTEST_ASSERT_(::testing::AssertPred4Helper(#pred, \ #v1, \ #v2, \ #v3, \ #v4, \ pred, \ v1, \ v2, \ v3, \ v4), on_failure) // 4-ary predicate assertion macros. #define EXPECT_PRED_FORMAT4(pred_format, v1, v2, v3, v4) \ GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, GTEST_NONFATAL_FAILURE_) #define EXPECT_PRED4(pred, v1, v2, v3, v4) \ GTEST_PRED4_(pred, v1, v2, v3, v4, GTEST_NONFATAL_FAILURE_) #define ASSERT_PRED_FORMAT4(pred_format, v1, v2, v3, v4) \ GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, GTEST_FATAL_FAILURE_) #define ASSERT_PRED4(pred, v1, v2, v3, v4) \ GTEST_PRED4_(pred, v1, v2, v3, v4, GTEST_FATAL_FAILURE_) // Helper function for implementing {EXPECT|ASSERT}_PRED5. Don't use // this in your code. template AssertionResult AssertPred5Helper(const char* pred_text, const char* e1, const char* e2, const char* e3, const char* e4, const char* e5, Pred pred, const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5) { if (pred(v1, v2, v3, v4, v5)) return AssertionSuccess(); return AssertionFailure() << pred_text << "(" << e1 << ", " << e2 << ", " << e3 << ", " << e4 << ", " << e5 << ") evaluates to false, where" << "\n" << e1 << " evaluates to " << ::testing::PrintToString(v1) << "\n" << e2 << " evaluates to " << ::testing::PrintToString(v2) << "\n" << e3 << " evaluates to " << ::testing::PrintToString(v3) << "\n" << e4 << " evaluates to " << ::testing::PrintToString(v4) << "\n" << e5 << " evaluates to " << ::testing::PrintToString(v5); } // Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT5. // Don't use this in your code. #define GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, on_failure)\ GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, #v5, v1, v2, v3, v4, v5), \ on_failure) // Internal macro for implementing {EXPECT|ASSERT}_PRED5. Don't use // this in your code. #define GTEST_PRED5_(pred, v1, v2, v3, v4, v5, on_failure)\ GTEST_ASSERT_(::testing::AssertPred5Helper(#pred, \ #v1, \ #v2, \ #v3, \ #v4, \ #v5, \ pred, \ v1, \ v2, \ v3, \ v4, \ v5), on_failure) // 5-ary predicate assertion macros. #define EXPECT_PRED_FORMAT5(pred_format, v1, v2, v3, v4, v5) \ GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, GTEST_NONFATAL_FAILURE_) #define EXPECT_PRED5(pred, v1, v2, v3, v4, v5) \ GTEST_PRED5_(pred, v1, v2, v3, v4, v5, GTEST_NONFATAL_FAILURE_) #define ASSERT_PRED_FORMAT5(pred_format, v1, v2, v3, v4, v5) \ GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, GTEST_FATAL_FAILURE_) #define ASSERT_PRED5(pred, v1, v2, v3, v4, v5) \ GTEST_PRED5_(pred, v1, v2, v3, v4, v5, GTEST_FATAL_FAILURE_) } // namespace testing #endif // GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ namespace testing { // The abstract class that all tests inherit from. // // In Google Test, a unit test program contains one or many TestSuites, and // each TestSuite contains one or many Tests. // // When you define a test using the TEST macro, you don't need to // explicitly derive from Test - the TEST macro automatically does // this for you. // // The only time you derive from Test is when defining a test fixture // to be used in a TEST_F. For example: // // class FooTest : public testing::Test { // protected: // void SetUp() override { ... } // void TearDown() override { ... } // ... // }; // // TEST_F(FooTest, Bar) { ... } // TEST_F(FooTest, Baz) { ... } // // Test is not copyable. class GTEST_API_ Test { public: friend class TestInfo; // The d'tor is virtual as we intend to inherit from Test. virtual ~Test(); // Sets up the stuff shared by all tests in this test case. // // Google Test will call Foo::SetUpTestSuite() before running the first // test in test case Foo. Hence a sub-class can define its own // SetUpTestSuite() method to shadow the one defined in the super // class. // Failures that happen during SetUpTestSuite are logged but otherwise // ignored. static void SetUpTestSuite() {} // Tears down the stuff shared by all tests in this test suite. // // Google Test will call Foo::TearDownTestSuite() after running the last // test in test case Foo. Hence a sub-class can define its own // TearDownTestSuite() method to shadow the one defined in the super // class. // Failures that happen during TearDownTestSuite are logged but otherwise // ignored. static void TearDownTestSuite() {} // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ static void TearDownTestCase() {} static void SetUpTestCase() {} #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // Returns true if and only if the current test has a fatal failure. static bool HasFatalFailure(); // Returns true if and only if the current test has a non-fatal failure. static bool HasNonfatalFailure(); // Returns true if and only if the current test was skipped. static bool IsSkipped(); // Returns true if and only if the current test has a (either fatal or // non-fatal) failure. static bool HasFailure() { return HasFatalFailure() || HasNonfatalFailure(); } // Logs a property for the current test, test suite, or for the entire // invocation of the test program when used outside of the context of a // test suite. Only the last value for a given key is remembered. These // are public static so they can be called from utility functions that are // not members of the test fixture. Calls to RecordProperty made during // lifespan of the test (from the moment its constructor starts to the // moment its destructor finishes) will be output in XML as attributes of // the element. Properties recorded from fixture's // SetUpTestSuite or TearDownTestSuite are logged as attributes of the // corresponding element. Calls to RecordProperty made in the // global context (before or after invocation of RUN_ALL_TESTS and from // SetUp/TearDown method of Environment objects registered with Google // Test) will be output as attributes of the element. static void RecordProperty(const std::string& key, const std::string& value); static void RecordProperty(const std::string& key, int value); protected: // Creates a Test object. Test(); // Sets up the test fixture. virtual void SetUp(); // Tears down the test fixture. virtual void TearDown(); private: // Returns true if and only if the current test has the same fixture class // as the first test in the current test suite. static bool HasSameFixtureClass(); // Runs the test after the test fixture has been set up. // // A sub-class must implement this to define the test logic. // // DO NOT OVERRIDE THIS FUNCTION DIRECTLY IN A USER PROGRAM. // Instead, use the TEST or TEST_F macro. virtual void TestBody() = 0; // Sets up, executes, and tears down the test. void Run(); // Deletes self. We deliberately pick an unusual name for this // internal method to avoid clashing with names used in user TESTs. void DeleteSelf_() { delete this; } const std::unique_ptr gtest_flag_saver_; // Often a user misspells SetUp() as Setup() and spends a long time // wondering why it is never called by Google Test. The declaration of // the following method is solely for catching such an error at // compile time: // // - The return type is deliberately chosen to be not void, so it // will be a conflict if void Setup() is declared in the user's // test fixture. // // - This method is private, so it will be another compiler error // if the method is called from the user's test fixture. // // DO NOT OVERRIDE THIS FUNCTION. // // If you see an error about overriding the following function or // about it being private, you have mis-spelled SetUp() as Setup(). struct Setup_should_be_spelled_SetUp {}; virtual Setup_should_be_spelled_SetUp* Setup() { return nullptr; } // We disallow copying Tests. GTEST_DISALLOW_COPY_AND_ASSIGN_(Test); }; typedef internal::TimeInMillis TimeInMillis; // A copyable object representing a user specified test property which can be // output as a key/value string pair. // // Don't inherit from TestProperty as its destructor is not virtual. class TestProperty { public: // C'tor. TestProperty does NOT have a default constructor. // Always use this constructor (with parameters) to create a // TestProperty object. TestProperty(const std::string& a_key, const std::string& a_value) : key_(a_key), value_(a_value) { } // Gets the user supplied key. const char* key() const { return key_.c_str(); } // Gets the user supplied value. const char* value() const { return value_.c_str(); } // Sets a new value, overriding the one supplied in the constructor. void SetValue(const std::string& new_value) { value_ = new_value; } private: // The key supplied by the user. std::string key_; // The value supplied by the user. std::string value_; }; // The result of a single Test. This includes a list of // TestPartResults, a list of TestProperties, a count of how many // death tests there are in the Test, and how much time it took to run // the Test. // // TestResult is not copyable. class GTEST_API_ TestResult { public: // Creates an empty TestResult. TestResult(); // D'tor. Do not inherit from TestResult. ~TestResult(); // Gets the number of all test parts. This is the sum of the number // of successful test parts and the number of failed test parts. int total_part_count() const; // Returns the number of the test properties. int test_property_count() const; // Returns true if and only if the test passed (i.e. no test part failed). bool Passed() const { return !Skipped() && !Failed(); } // Returns true if and only if the test was skipped. bool Skipped() const; // Returns true if and only if the test failed. bool Failed() const; // Returns true if and only if the test fatally failed. bool HasFatalFailure() const; // Returns true if and only if the test has a non-fatal failure. bool HasNonfatalFailure() const; // Returns the elapsed time, in milliseconds. TimeInMillis elapsed_time() const { return elapsed_time_; } // Gets the time of the test case start, in ms from the start of the // UNIX epoch. TimeInMillis start_timestamp() const { return start_timestamp_; } // Returns the i-th test part result among all the results. i can range from 0 // to total_part_count() - 1. If i is not in that range, aborts the program. const TestPartResult& GetTestPartResult(int i) const; // Returns the i-th test property. i can range from 0 to // test_property_count() - 1. If i is not in that range, aborts the // program. const TestProperty& GetTestProperty(int i) const; private: friend class TestInfo; friend class TestSuite; friend class UnitTest; friend class internal::DefaultGlobalTestPartResultReporter; friend class internal::ExecDeathTest; friend class internal::TestResultAccessor; friend class internal::UnitTestImpl; friend class internal::WindowsDeathTest; friend class internal::FuchsiaDeathTest; // Gets the vector of TestPartResults. const std::vector& test_part_results() const { return test_part_results_; } // Gets the vector of TestProperties. const std::vector& test_properties() const { return test_properties_; } // Sets the start time. void set_start_timestamp(TimeInMillis start) { start_timestamp_ = start; } // Sets the elapsed time. void set_elapsed_time(TimeInMillis elapsed) { elapsed_time_ = elapsed; } // Adds a test property to the list. The property is validated and may add // a non-fatal failure if invalid (e.g., if it conflicts with reserved // key names). If a property is already recorded for the same key, the // value will be updated, rather than storing multiple values for the same // key. xml_element specifies the element for which the property is being // recorded and is used for validation. void RecordProperty(const std::string& xml_element, const TestProperty& test_property); // Adds a failure if the key is a reserved attribute of Google Test // testsuite tags. Returns true if the property is valid. // FIXME: Validate attribute names are legal and human readable. static bool ValidateTestProperty(const std::string& xml_element, const TestProperty& test_property); // Adds a test part result to the list. void AddTestPartResult(const TestPartResult& test_part_result); // Returns the death test count. int death_test_count() const { return death_test_count_; } // Increments the death test count, returning the new count. int increment_death_test_count() { return ++death_test_count_; } // Clears the test part results. void ClearTestPartResults(); // Clears the object. void Clear(); // Protects mutable state of the property vector and of owned // properties, whose values may be updated. internal::Mutex test_properites_mutex_; // The vector of TestPartResults std::vector test_part_results_; // The vector of TestProperties std::vector test_properties_; // Running count of death tests. int death_test_count_; // The start time, in milliseconds since UNIX Epoch. TimeInMillis start_timestamp_; // The elapsed time, in milliseconds. TimeInMillis elapsed_time_; // We disallow copying TestResult. GTEST_DISALLOW_COPY_AND_ASSIGN_(TestResult); }; // class TestResult // A TestInfo object stores the following information about a test: // // Test suite name // Test name // Whether the test should be run // A function pointer that creates the test object when invoked // Test result // // The constructor of TestInfo registers itself with the UnitTest // singleton such that the RUN_ALL_TESTS() macro knows which tests to // run. class GTEST_API_ TestInfo { public: // Destructs a TestInfo object. This function is not virtual, so // don't inherit from TestInfo. ~TestInfo(); // Returns the test suite name. const char* test_suite_name() const { return test_suite_name_.c_str(); } // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ const char* test_case_name() const { return test_suite_name(); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // Returns the test name. const char* name() const { return name_.c_str(); } // Returns the name of the parameter type, or NULL if this is not a typed // or a type-parameterized test. const char* type_param() const { if (type_param_.get() != nullptr) return type_param_->c_str(); return nullptr; } // Returns the text representation of the value parameter, or NULL if this // is not a value-parameterized test. const char* value_param() const { if (value_param_.get() != nullptr) return value_param_->c_str(); return nullptr; } // Returns the file name where this test is defined. const char* file() const { return location_.file.c_str(); } // Returns the line where this test is defined. int line() const { return location_.line; } // Return true if this test should not be run because it's in another shard. bool is_in_another_shard() const { return is_in_another_shard_; } // Returns true if this test should run, that is if the test is not // disabled (or it is disabled but the also_run_disabled_tests flag has // been specified) and its full name matches the user-specified filter. // // Google Test allows the user to filter the tests by their full names. // The full name of a test Bar in test suite Foo is defined as // "Foo.Bar". Only the tests that match the filter will run. // // A filter is a colon-separated list of glob (not regex) patterns, // optionally followed by a '-' and a colon-separated list of // negative patterns (tests to exclude). A test is run if it // matches one of the positive patterns and does not match any of // the negative patterns. // // For example, *A*:Foo.* is a filter that matches any string that // contains the character 'A' or starts with "Foo.". bool should_run() const { return should_run_; } // Returns true if and only if this test will appear in the XML report. bool is_reportable() const { // The XML report includes tests matching the filter, excluding those // run in other shards. return matches_filter_ && !is_in_another_shard_; } // Returns the result of the test. const TestResult* result() const { return &result_; } private: #if GTEST_HAS_DEATH_TEST friend class internal::DefaultDeathTestFactory; #endif // GTEST_HAS_DEATH_TEST friend class Test; friend class TestSuite; friend class internal::UnitTestImpl; friend class internal::StreamingListenerTest; friend TestInfo* internal::MakeAndRegisterTestInfo( const char* test_suite_name, const char* name, const char* type_param, const char* value_param, internal::CodeLocation code_location, internal::TypeId fixture_class_id, internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc, internal::TestFactoryBase* factory); // Constructs a TestInfo object. The newly constructed instance assumes // ownership of the factory object. TestInfo(const std::string& test_suite_name, const std::string& name, const char* a_type_param, // NULL if not a type-parameterized test const char* a_value_param, // NULL if not a value-parameterized test internal::CodeLocation a_code_location, internal::TypeId fixture_class_id, internal::TestFactoryBase* factory); // Increments the number of death tests encountered in this test so // far. int increment_death_test_count() { return result_.increment_death_test_count(); } // Creates the test object, runs it, records its result, and then // deletes it. void Run(); static void ClearTestResult(TestInfo* test_info) { test_info->result_.Clear(); } // These fields are immutable properties of the test. const std::string test_suite_name_; // test suite name const std::string name_; // Test name // Name of the parameter type, or NULL if this is not a typed or a // type-parameterized test. const std::unique_ptr type_param_; // Text representation of the value parameter, or NULL if this is not a // value-parameterized test. const std::unique_ptr value_param_; internal::CodeLocation location_; const internal::TypeId fixture_class_id_; // ID of the test fixture class bool should_run_; // True if and only if this test should run bool is_disabled_; // True if and only if this test is disabled bool matches_filter_; // True if this test matches the // user-specified filter. bool is_in_another_shard_; // Will be run in another shard. internal::TestFactoryBase* const factory_; // The factory that creates // the test object // This field is mutable and needs to be reset before running the // test for the second time. TestResult result_; GTEST_DISALLOW_COPY_AND_ASSIGN_(TestInfo); }; // A test suite, which consists of a vector of TestInfos. // // TestSuite is not copyable. class GTEST_API_ TestSuite { public: // Creates a TestSuite with the given name. // // TestSuite does NOT have a default constructor. Always use this // constructor to create a TestSuite object. // // Arguments: // // name: name of the test suite // a_type_param: the name of the test's type parameter, or NULL if // this is not a type-parameterized test. // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite TestSuite(const char* name, const char* a_type_param, internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc); // Destructor of TestSuite. virtual ~TestSuite(); // Gets the name of the TestSuite. const char* name() const { return name_.c_str(); } // Returns the name of the parameter type, or NULL if this is not a // type-parameterized test suite. const char* type_param() const { if (type_param_.get() != nullptr) return type_param_->c_str(); return nullptr; } // Returns true if any test in this test suite should run. bool should_run() const { return should_run_; } // Gets the number of successful tests in this test suite. int successful_test_count() const; // Gets the number of skipped tests in this test suite. int skipped_test_count() const; // Gets the number of failed tests in this test suite. int failed_test_count() const; // Gets the number of disabled tests that will be reported in the XML report. int reportable_disabled_test_count() const; // Gets the number of disabled tests in this test suite. int disabled_test_count() const; // Gets the number of tests to be printed in the XML report. int reportable_test_count() const; // Get the number of tests in this test suite that should run. int test_to_run_count() const; // Gets the number of all tests in this test suite. int total_test_count() const; // Returns true if and only if the test suite passed. bool Passed() const { return !Failed(); } // Returns true if and only if the test suite failed. bool Failed() const { return failed_test_count() > 0; } // Returns the elapsed time, in milliseconds. TimeInMillis elapsed_time() const { return elapsed_time_; } // Gets the time of the test suite start, in ms from the start of the // UNIX epoch. TimeInMillis start_timestamp() const { return start_timestamp_; } // Returns the i-th test among all the tests. i can range from 0 to // total_test_count() - 1. If i is not in that range, returns NULL. const TestInfo* GetTestInfo(int i) const; // Returns the TestResult that holds test properties recorded during // execution of SetUpTestSuite and TearDownTestSuite. const TestResult& ad_hoc_test_result() const { return ad_hoc_test_result_; } private: friend class Test; friend class internal::UnitTestImpl; // Gets the (mutable) vector of TestInfos in this TestSuite. std::vector& test_info_list() { return test_info_list_; } // Gets the (immutable) vector of TestInfos in this TestSuite. const std::vector& test_info_list() const { return test_info_list_; } // Returns the i-th test among all the tests. i can range from 0 to // total_test_count() - 1. If i is not in that range, returns NULL. TestInfo* GetMutableTestInfo(int i); // Sets the should_run member. void set_should_run(bool should) { should_run_ = should; } // Adds a TestInfo to this test suite. Will delete the TestInfo upon // destruction of the TestSuite object. void AddTestInfo(TestInfo * test_info); // Clears the results of all tests in this test suite. void ClearResult(); // Clears the results of all tests in the given test suite. static void ClearTestSuiteResult(TestSuite* test_suite) { test_suite->ClearResult(); } // Runs every test in this TestSuite. void Run(); // Runs SetUpTestSuite() for this TestSuite. This wrapper is needed // for catching exceptions thrown from SetUpTestSuite(). void RunSetUpTestSuite() { if (set_up_tc_ != nullptr) { (*set_up_tc_)(); } } // Runs TearDownTestSuite() for this TestSuite. This wrapper is // needed for catching exceptions thrown from TearDownTestSuite(). void RunTearDownTestSuite() { if (tear_down_tc_ != nullptr) { (*tear_down_tc_)(); } } // Returns true if and only if test passed. static bool TestPassed(const TestInfo* test_info) { return test_info->should_run() && test_info->result()->Passed(); } // Returns true if and only if test skipped. static bool TestSkipped(const TestInfo* test_info) { return test_info->should_run() && test_info->result()->Skipped(); } // Returns true if and only if test failed. static bool TestFailed(const TestInfo* test_info) { return test_info->should_run() && test_info->result()->Failed(); } // Returns true if and only if the test is disabled and will be reported in // the XML report. static bool TestReportableDisabled(const TestInfo* test_info) { return test_info->is_reportable() && test_info->is_disabled_; } // Returns true if and only if test is disabled. static bool TestDisabled(const TestInfo* test_info) { return test_info->is_disabled_; } // Returns true if and only if this test will appear in the XML report. static bool TestReportable(const TestInfo* test_info) { return test_info->is_reportable(); } // Returns true if the given test should run. static bool ShouldRunTest(const TestInfo* test_info) { return test_info->should_run(); } // Shuffles the tests in this test suite. void ShuffleTests(internal::Random* random); // Restores the test order to before the first shuffle. void UnshuffleTests(); // Name of the test suite. std::string name_; // Name of the parameter type, or NULL if this is not a typed or a // type-parameterized test. const std::unique_ptr type_param_; // The vector of TestInfos in their original order. It owns the // elements in the vector. std::vector test_info_list_; // Provides a level of indirection for the test list to allow easy // shuffling and restoring the test order. The i-th element in this // vector is the index of the i-th test in the shuffled test list. std::vector test_indices_; // Pointer to the function that sets up the test suite. internal::SetUpTestSuiteFunc set_up_tc_; // Pointer to the function that tears down the test suite. internal::TearDownTestSuiteFunc tear_down_tc_; // True if and only if any test in this test suite should run. bool should_run_; // The start time, in milliseconds since UNIX Epoch. TimeInMillis start_timestamp_; // Elapsed time, in milliseconds. TimeInMillis elapsed_time_; // Holds test properties recorded during execution of SetUpTestSuite and // TearDownTestSuite. TestResult ad_hoc_test_result_; // We disallow copying TestSuites. GTEST_DISALLOW_COPY_AND_ASSIGN_(TestSuite); }; // An Environment object is capable of setting up and tearing down an // environment. You should subclass this to define your own // environment(s). // // An Environment object does the set-up and tear-down in virtual // methods SetUp() and TearDown() instead of the constructor and the // destructor, as: // // 1. You cannot safely throw from a destructor. This is a problem // as in some cases Google Test is used where exceptions are enabled, and // we may want to implement ASSERT_* using exceptions where they are // available. // 2. You cannot use ASSERT_* directly in a constructor or // destructor. class Environment { public: // The d'tor is virtual as we need to subclass Environment. virtual ~Environment() {} // Override this to define how to set up the environment. virtual void SetUp() {} // Override this to define how to tear down the environment. virtual void TearDown() {} private: // If you see an error about overriding the following function or // about it being private, you have mis-spelled SetUp() as Setup(). struct Setup_should_be_spelled_SetUp {}; virtual Setup_should_be_spelled_SetUp* Setup() { return nullptr; } }; #if GTEST_HAS_EXCEPTIONS // Exception which can be thrown from TestEventListener::OnTestPartResult. class GTEST_API_ AssertionException : public internal::GoogleTestFailureException { public: explicit AssertionException(const TestPartResult& result) : GoogleTestFailureException(result) {} }; #endif // GTEST_HAS_EXCEPTIONS // The interface for tracing execution of tests. The methods are organized in // the order the corresponding events are fired. class TestEventListener { public: virtual ~TestEventListener() {} // Fired before any test activity starts. virtual void OnTestProgramStart(const UnitTest& unit_test) = 0; // Fired before each iteration of tests starts. There may be more than // one iteration if GTEST_FLAG(repeat) is set. iteration is the iteration // index, starting from 0. virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration) = 0; // Fired before environment set-up for each iteration of tests starts. virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test) = 0; // Fired after environment set-up for each iteration of tests ends. virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test) = 0; // Fired before the test suite starts. virtual void OnTestSuiteStart(const TestSuite& /*test_suite*/) {} // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ virtual void OnTestCaseStart(const TestCase& /*test_case*/) {} #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // Fired before the test starts. virtual void OnTestStart(const TestInfo& test_info) = 0; // Fired after a failed assertion or a SUCCEED() invocation. // If you want to throw an exception from this function to skip to the next // TEST, it must be AssertionException defined above, or inherited from it. virtual void OnTestPartResult(const TestPartResult& test_part_result) = 0; // Fired after the test ends. virtual void OnTestEnd(const TestInfo& test_info) = 0; // Fired after the test suite ends. virtual void OnTestSuiteEnd(const TestSuite& /*test_suite*/) {} // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ virtual void OnTestCaseEnd(const TestCase& /*test_case*/) {} #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // Fired before environment tear-down for each iteration of tests starts. virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test) = 0; // Fired after environment tear-down for each iteration of tests ends. virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test) = 0; // Fired after each iteration of tests finishes. virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration) = 0; // Fired after all test activities have ended. virtual void OnTestProgramEnd(const UnitTest& unit_test) = 0; }; // The convenience class for users who need to override just one or two // methods and are not concerned that a possible change to a signature of // the methods they override will not be caught during the build. For // comments about each method please see the definition of TestEventListener // above. class EmptyTestEventListener : public TestEventListener { public: void OnTestProgramStart(const UnitTest& /*unit_test*/) override {} void OnTestIterationStart(const UnitTest& /*unit_test*/, int /*iteration*/) override {} void OnEnvironmentsSetUpStart(const UnitTest& /*unit_test*/) override {} void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) override {} void OnTestSuiteStart(const TestSuite& /*test_suite*/) override {} // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnTestCaseStart(const TestCase& /*test_case*/) override {} #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnTestStart(const TestInfo& /*test_info*/) override {} void OnTestPartResult(const TestPartResult& /*test_part_result*/) override {} void OnTestEnd(const TestInfo& /*test_info*/) override {} void OnTestSuiteEnd(const TestSuite& /*test_suite*/) override {} #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnTestCaseEnd(const TestCase& /*test_case*/) override {} #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnEnvironmentsTearDownStart(const UnitTest& /*unit_test*/) override {} void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) override {} void OnTestIterationEnd(const UnitTest& /*unit_test*/, int /*iteration*/) override {} void OnTestProgramEnd(const UnitTest& /*unit_test*/) override {} }; // TestEventListeners lets users add listeners to track events in Google Test. class GTEST_API_ TestEventListeners { public: TestEventListeners(); ~TestEventListeners(); // Appends an event listener to the end of the list. Google Test assumes // the ownership of the listener (i.e. it will delete the listener when // the test program finishes). void Append(TestEventListener* listener); // Removes the given event listener from the list and returns it. It then // becomes the caller's responsibility to delete the listener. Returns // NULL if the listener is not found in the list. TestEventListener* Release(TestEventListener* listener); // Returns the standard listener responsible for the default console // output. Can be removed from the listeners list to shut down default // console output. Note that removing this object from the listener list // with Release transfers its ownership to the caller and makes this // function return NULL the next time. TestEventListener* default_result_printer() const { return default_result_printer_; } // Returns the standard listener responsible for the default XML output // controlled by the --gtest_output=xml flag. Can be removed from the // listeners list by users who want to shut down the default XML output // controlled by this flag and substitute it with custom one. Note that // removing this object from the listener list with Release transfers its // ownership to the caller and makes this function return NULL the next // time. TestEventListener* default_xml_generator() const { return default_xml_generator_; } private: friend class TestSuite; friend class TestInfo; friend class internal::DefaultGlobalTestPartResultReporter; friend class internal::NoExecDeathTest; friend class internal::TestEventListenersAccessor; friend class internal::UnitTestImpl; // Returns repeater that broadcasts the TestEventListener events to all // subscribers. TestEventListener* repeater(); // Sets the default_result_printer attribute to the provided listener. // The listener is also added to the listener list and previous // default_result_printer is removed from it and deleted. The listener can // also be NULL in which case it will not be added to the list. Does // nothing if the previous and the current listener objects are the same. void SetDefaultResultPrinter(TestEventListener* listener); // Sets the default_xml_generator attribute to the provided listener. The // listener is also added to the listener list and previous // default_xml_generator is removed from it and deleted. The listener can // also be NULL in which case it will not be added to the list. Does // nothing if the previous and the current listener objects are the same. void SetDefaultXmlGenerator(TestEventListener* listener); // Controls whether events will be forwarded by the repeater to the // listeners in the list. bool EventForwardingEnabled() const; void SuppressEventForwarding(); // The actual list of listeners. internal::TestEventRepeater* repeater_; // Listener responsible for the standard result output. TestEventListener* default_result_printer_; // Listener responsible for the creation of the XML output file. TestEventListener* default_xml_generator_; // We disallow copying TestEventListeners. GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventListeners); }; // A UnitTest consists of a vector of TestSuites. // // This is a singleton class. The only instance of UnitTest is // created when UnitTest::GetInstance() is first called. This // instance is never deleted. // // UnitTest is not copyable. // // This class is thread-safe as long as the methods are called // according to their specification. class GTEST_API_ UnitTest { public: // Gets the singleton UnitTest object. The first time this method // is called, a UnitTest object is constructed and returned. // Consecutive calls will return the same object. static UnitTest* GetInstance(); // Runs all tests in this UnitTest object and prints the result. // Returns 0 if successful, or 1 otherwise. // // This method can only be called from the main thread. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. int Run() GTEST_MUST_USE_RESULT_; // Returns the working directory when the first TEST() or TEST_F() // was executed. The UnitTest object owns the string. const char* original_working_dir() const; // Returns the TestSuite object for the test that's currently running, // or NULL if no test is running. const TestSuite* current_test_suite() const GTEST_LOCK_EXCLUDED_(mutex_); // Legacy API is still available but deprecated #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ const TestCase* current_test_case() const GTEST_LOCK_EXCLUDED_(mutex_); #endif // Returns the TestInfo object for the test that's currently running, // or NULL if no test is running. const TestInfo* current_test_info() const GTEST_LOCK_EXCLUDED_(mutex_); // Returns the random seed used at the start of the current test run. int random_seed() const; // Returns the ParameterizedTestSuiteRegistry object used to keep track of // value-parameterized tests and instantiate and register them. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. internal::ParameterizedTestSuiteRegistry& parameterized_test_registry() GTEST_LOCK_EXCLUDED_(mutex_); // Gets the number of successful test suites. int successful_test_suite_count() const; // Gets the number of failed test suites. int failed_test_suite_count() const; // Gets the number of all test suites. int total_test_suite_count() const; // Gets the number of all test suites that contain at least one test // that should run. int test_suite_to_run_count() const; // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ int successful_test_case_count() const; int failed_test_case_count() const; int total_test_case_count() const; int test_case_to_run_count() const; #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // Gets the number of successful tests. int successful_test_count() const; // Gets the number of skipped tests. int skipped_test_count() const; // Gets the number of failed tests. int failed_test_count() const; // Gets the number of disabled tests that will be reported in the XML report. int reportable_disabled_test_count() const; // Gets the number of disabled tests. int disabled_test_count() const; // Gets the number of tests to be printed in the XML report. int reportable_test_count() const; // Gets the number of all tests. int total_test_count() const; // Gets the number of tests that should run. int test_to_run_count() const; // Gets the time of the test program start, in ms from the start of the // UNIX epoch. TimeInMillis start_timestamp() const; // Gets the elapsed time, in milliseconds. TimeInMillis elapsed_time() const; // Returns true if and only if the unit test passed (i.e. all test suites // passed). bool Passed() const; // Returns true if and only if the unit test failed (i.e. some test suite // failed or something outside of all tests failed). bool Failed() const; // Gets the i-th test suite among all the test suites. i can range from 0 to // total_test_suite_count() - 1. If i is not in that range, returns NULL. const TestSuite* GetTestSuite(int i) const; // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ const TestCase* GetTestCase(int i) const; #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // Returns the TestResult containing information on test failures and // properties logged outside of individual test suites. const TestResult& ad_hoc_test_result() const; // Returns the list of event listeners that can be used to track events // inside Google Test. TestEventListeners& listeners(); private: // Registers and returns a global test environment. When a test // program is run, all global test environments will be set-up in // the order they were registered. After all tests in the program // have finished, all global test environments will be torn-down in // the *reverse* order they were registered. // // The UnitTest object takes ownership of the given environment. // // This method can only be called from the main thread. Environment* AddEnvironment(Environment* env); // Adds a TestPartResult to the current TestResult object. All // Google Test assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) // eventually call this to report their results. The user code // should use the assertion macros instead of calling this directly. void AddTestPartResult(TestPartResult::Type result_type, const char* file_name, int line_number, const std::string& message, const std::string& os_stack_trace) GTEST_LOCK_EXCLUDED_(mutex_); // Adds a TestProperty to the current TestResult object when invoked from // inside a test, to current TestSuite's ad_hoc_test_result_ when invoked // from SetUpTestSuite or TearDownTestSuite, or to the global property set // when invoked elsewhere. If the result already contains a property with // the same key, the value will be updated. void RecordProperty(const std::string& key, const std::string& value); // Gets the i-th test suite among all the test suites. i can range from 0 to // total_test_suite_count() - 1. If i is not in that range, returns NULL. TestSuite* GetMutableTestSuite(int i); // Accessors for the implementation object. internal::UnitTestImpl* impl() { return impl_; } const internal::UnitTestImpl* impl() const { return impl_; } // These classes and functions are friends as they need to access private // members of UnitTest. friend class ScopedTrace; friend class Test; friend class internal::AssertHelper; friend class internal::StreamingListenerTest; friend class internal::UnitTestRecordPropertyTestHelper; friend Environment* AddGlobalTestEnvironment(Environment* env); friend internal::UnitTestImpl* internal::GetUnitTestImpl(); friend void internal::ReportFailureInUnknownLocation( TestPartResult::Type result_type, const std::string& message); // Creates an empty UnitTest. UnitTest(); // D'tor virtual ~UnitTest(); // Pushes a trace defined by SCOPED_TRACE() on to the per-thread // Google Test trace stack. void PushGTestTrace(const internal::TraceInfo& trace) GTEST_LOCK_EXCLUDED_(mutex_); // Pops a trace from the per-thread Google Test trace stack. void PopGTestTrace() GTEST_LOCK_EXCLUDED_(mutex_); // Protects mutable state in *impl_. This is mutable as some const // methods need to lock it too. mutable internal::Mutex mutex_; // Opaque implementation object. This field is never changed once // the object is constructed. We don't mark it as const here, as // doing so will cause a warning in the constructor of UnitTest. // Mutable state in *impl_ is protected by mutex_. internal::UnitTestImpl* impl_; // We disallow copying UnitTest. GTEST_DISALLOW_COPY_AND_ASSIGN_(UnitTest); }; // A convenient wrapper for adding an environment for the test // program. // // You should call this before RUN_ALL_TESTS() is called, probably in // main(). If you use gtest_main, you need to call this before main() // starts for it to take effect. For example, you can define a global // variable like this: // // testing::Environment* const foo_env = // testing::AddGlobalTestEnvironment(new FooEnvironment); // // However, we strongly recommend you to write your own main() and // call AddGlobalTestEnvironment() there, as relying on initialization // of global variables makes the code harder to read and may cause // problems when you register multiple environments from different // translation units and the environments have dependencies among them // (remember that the compiler doesn't guarantee the order in which // global variables from different translation units are initialized). inline Environment* AddGlobalTestEnvironment(Environment* env) { return UnitTest::GetInstance()->AddEnvironment(env); } // Initializes Google Test. This must be called before calling // RUN_ALL_TESTS(). In particular, it parses a command line for the // flags that Google Test recognizes. Whenever a Google Test flag is // seen, it is removed from argv, and *argc is decremented. // // No value is returned. Instead, the Google Test flag variables are // updated. // // Calling the function for the second time has no user-visible effect. GTEST_API_ void InitGoogleTest(int* argc, char** argv); // This overloaded version can be used in Windows programs compiled in // UNICODE mode. GTEST_API_ void InitGoogleTest(int* argc, wchar_t** argv); // This overloaded version can be used on Arduino/embedded platforms where // there is no argc/argv. GTEST_API_ void InitGoogleTest(); namespace internal { // Separate the error generating code from the code path to reduce the stack // frame size of CmpHelperEQ. This helps reduce the overhead of some sanitizers // when calling EXPECT_* in a tight loop. template AssertionResult CmpHelperEQFailure(const char* lhs_expression, const char* rhs_expression, const T1& lhs, const T2& rhs) { return EqFailure(lhs_expression, rhs_expression, FormatForComparisonFailureMessage(lhs, rhs), FormatForComparisonFailureMessage(rhs, lhs), false); } // This block of code defines operator==/!= // to block lexical scope lookup. // It prevents using invalid operator==/!= defined at namespace scope. struct faketype {}; inline bool operator==(faketype, faketype) { return true; } inline bool operator!=(faketype, faketype) { return false; } // The helper function for {ASSERT|EXPECT}_EQ. template AssertionResult CmpHelperEQ(const char* lhs_expression, const char* rhs_expression, const T1& lhs, const T2& rhs) { if (lhs == rhs) { return AssertionSuccess(); } return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs); } // With this overloaded version, we allow anonymous enums to be used // in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous enums // can be implicitly cast to BiggestInt. GTEST_API_ AssertionResult CmpHelperEQ(const char* lhs_expression, const char* rhs_expression, BiggestInt lhs, BiggestInt rhs); class EqHelper { public: // This templatized version is for the general case. template < typename T1, typename T2, // Disable this overload for cases where one argument is a pointer // and the other is the null pointer constant. typename std::enable_if::value || !std::is_pointer::value>::type* = nullptr> static AssertionResult Compare(const char* lhs_expression, const char* rhs_expression, const T1& lhs, const T2& rhs) { return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); } // With this overloaded version, we allow anonymous enums to be used // in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous // enums can be implicitly cast to BiggestInt. // // Even though its body looks the same as the above version, we // cannot merge the two, as it will make anonymous enums unhappy. static AssertionResult Compare(const char* lhs_expression, const char* rhs_expression, BiggestInt lhs, BiggestInt rhs) { return CmpHelperEQ(lhs_expression, rhs_expression, lhs, rhs); } template static AssertionResult Compare( const char* lhs_expression, const char* rhs_expression, // Handle cases where '0' is used as a null pointer literal. std::nullptr_t /* lhs */, T* rhs) { // We already know that 'lhs' is a null pointer. return CmpHelperEQ(lhs_expression, rhs_expression, static_cast(nullptr), rhs); } }; // Separate the error generating code from the code path to reduce the stack // frame size of CmpHelperOP. This helps reduce the overhead of some sanitizers // when calling EXPECT_OP in a tight loop. template AssertionResult CmpHelperOpFailure(const char* expr1, const char* expr2, const T1& val1, const T2& val2, const char* op) { return AssertionFailure() << "Expected: (" << expr1 << ") " << op << " (" << expr2 << "), actual: " << FormatForComparisonFailureMessage(val1, val2) << " vs " << FormatForComparisonFailureMessage(val2, val1); } // A macro for implementing the helper functions needed to implement // ASSERT_?? and EXPECT_??. It is here just to avoid copy-and-paste // of similar code. // // For each templatized helper function, we also define an overloaded // version for BiggestInt in order to reduce code bloat and allow // anonymous enums to be used with {ASSERT|EXPECT}_?? when compiled // with gcc 4. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. #define GTEST_IMPL_CMP_HELPER_(op_name, op)\ template \ AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ const T1& val1, const T2& val2) {\ if (val1 op val2) {\ return AssertionSuccess();\ } else {\ return CmpHelperOpFailure(expr1, expr2, val1, val2, #op);\ }\ }\ GTEST_API_ AssertionResult CmpHelper##op_name(\ const char* expr1, const char* expr2, BiggestInt val1, BiggestInt val2) // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. // Implements the helper function for {ASSERT|EXPECT}_NE GTEST_IMPL_CMP_HELPER_(NE, !=); // Implements the helper function for {ASSERT|EXPECT}_LE GTEST_IMPL_CMP_HELPER_(LE, <=); // Implements the helper function for {ASSERT|EXPECT}_LT GTEST_IMPL_CMP_HELPER_(LT, <); // Implements the helper function for {ASSERT|EXPECT}_GE GTEST_IMPL_CMP_HELPER_(GE, >=); // Implements the helper function for {ASSERT|EXPECT}_GT GTEST_IMPL_CMP_HELPER_(GT, >); #undef GTEST_IMPL_CMP_HELPER_ // The helper function for {ASSERT|EXPECT}_STREQ. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. GTEST_API_ AssertionResult CmpHelperSTREQ(const char* s1_expression, const char* s2_expression, const char* s1, const char* s2); // The helper function for {ASSERT|EXPECT}_STRCASEEQ. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. GTEST_API_ AssertionResult CmpHelperSTRCASEEQ(const char* s1_expression, const char* s2_expression, const char* s1, const char* s2); // The helper function for {ASSERT|EXPECT}_STRNE. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. GTEST_API_ AssertionResult CmpHelperSTRNE(const char* s1_expression, const char* s2_expression, const char* s1, const char* s2); // The helper function for {ASSERT|EXPECT}_STRCASENE. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. GTEST_API_ AssertionResult CmpHelperSTRCASENE(const char* s1_expression, const char* s2_expression, const char* s1, const char* s2); // Helper function for *_STREQ on wide strings. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. GTEST_API_ AssertionResult CmpHelperSTREQ(const char* s1_expression, const char* s2_expression, const wchar_t* s1, const wchar_t* s2); // Helper function for *_STRNE on wide strings. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. GTEST_API_ AssertionResult CmpHelperSTRNE(const char* s1_expression, const char* s2_expression, const wchar_t* s1, const wchar_t* s2); } // namespace internal // IsSubstring() and IsNotSubstring() are intended to be used as the // first argument to {EXPECT,ASSERT}_PRED_FORMAT2(), not by // themselves. They check whether needle is a substring of haystack // (NULL is considered a substring of itself only), and return an // appropriate error message when they fail. // // The {needle,haystack}_expr arguments are the stringified // expressions that generated the two real arguments. GTEST_API_ AssertionResult IsSubstring( const char* needle_expr, const char* haystack_expr, const char* needle, const char* haystack); GTEST_API_ AssertionResult IsSubstring( const char* needle_expr, const char* haystack_expr, const wchar_t* needle, const wchar_t* haystack); GTEST_API_ AssertionResult IsNotSubstring( const char* needle_expr, const char* haystack_expr, const char* needle, const char* haystack); GTEST_API_ AssertionResult IsNotSubstring( const char* needle_expr, const char* haystack_expr, const wchar_t* needle, const wchar_t* haystack); GTEST_API_ AssertionResult IsSubstring( const char* needle_expr, const char* haystack_expr, const ::std::string& needle, const ::std::string& haystack); GTEST_API_ AssertionResult IsNotSubstring( const char* needle_expr, const char* haystack_expr, const ::std::string& needle, const ::std::string& haystack); #if GTEST_HAS_STD_WSTRING GTEST_API_ AssertionResult IsSubstring( const char* needle_expr, const char* haystack_expr, const ::std::wstring& needle, const ::std::wstring& haystack); GTEST_API_ AssertionResult IsNotSubstring( const char* needle_expr, const char* haystack_expr, const ::std::wstring& needle, const ::std::wstring& haystack); #endif // GTEST_HAS_STD_WSTRING namespace internal { // Helper template function for comparing floating-points. // // Template parameter: // // RawType: the raw floating-point type (either float or double) // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. template AssertionResult CmpHelperFloatingPointEQ(const char* lhs_expression, const char* rhs_expression, RawType lhs_value, RawType rhs_value) { const FloatingPoint lhs(lhs_value), rhs(rhs_value); if (lhs.AlmostEquals(rhs)) { return AssertionSuccess(); } ::std::stringstream lhs_ss; lhs_ss << std::setprecision(std::numeric_limits::digits10 + 2) << lhs_value; ::std::stringstream rhs_ss; rhs_ss << std::setprecision(std::numeric_limits::digits10 + 2) << rhs_value; return EqFailure(lhs_expression, rhs_expression, StringStreamToString(&lhs_ss), StringStreamToString(&rhs_ss), false); } // Helper function for implementing ASSERT_NEAR. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. GTEST_API_ AssertionResult DoubleNearPredFormat(const char* expr1, const char* expr2, const char* abs_error_expr, double val1, double val2, double abs_error); // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // A class that enables one to stream messages to assertion macros class GTEST_API_ AssertHelper { public: // Constructor. AssertHelper(TestPartResult::Type type, const char* file, int line, const char* message); ~AssertHelper(); // Message assignment is a semantic trick to enable assertion // streaming; see the GTEST_MESSAGE_ macro below. void operator=(const Message& message) const; private: // We put our data in a struct so that the size of the AssertHelper class can // be as small as possible. This is important because gcc is incapable of // re-using stack space even for temporary variables, so every EXPECT_EQ // reserves stack space for another AssertHelper. struct AssertHelperData { AssertHelperData(TestPartResult::Type t, const char* srcfile, int line_num, const char* msg) : type(t), file(srcfile), line(line_num), message(msg) { } TestPartResult::Type const type; const char* const file; int const line; std::string const message; private: GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelperData); }; AssertHelperData* const data_; GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelper); }; enum GTestColor { COLOR_DEFAULT, COLOR_RED, COLOR_GREEN, COLOR_YELLOW }; GTEST_API_ GTEST_ATTRIBUTE_PRINTF_(2, 3) void ColoredPrintf(GTestColor color, const char* fmt, ...); } // namespace internal // The pure interface class that all value-parameterized tests inherit from. // A value-parameterized class must inherit from both ::testing::Test and // ::testing::WithParamInterface. In most cases that just means inheriting // from ::testing::TestWithParam, but more complicated test hierarchies // may need to inherit from Test and WithParamInterface at different levels. // // This interface has support for accessing the test parameter value via // the GetParam() method. // // Use it with one of the parameter generator defining functions, like Range(), // Values(), ValuesIn(), Bool(), and Combine(). // // class FooTest : public ::testing::TestWithParam { // protected: // FooTest() { // // Can use GetParam() here. // } // ~FooTest() override { // // Can use GetParam() here. // } // void SetUp() override { // // Can use GetParam() here. // } // void TearDown override { // // Can use GetParam() here. // } // }; // TEST_P(FooTest, DoesBar) { // // Can use GetParam() method here. // Foo foo; // ASSERT_TRUE(foo.DoesBar(GetParam())); // } // INSTANTIATE_TEST_SUITE_P(OneToTenRange, FooTest, ::testing::Range(1, 10)); template class WithParamInterface { public: typedef T ParamType; virtual ~WithParamInterface() {} // The current parameter value. Is also available in the test fixture's // constructor. static const ParamType& GetParam() { GTEST_CHECK_(parameter_ != nullptr) << "GetParam() can only be called inside a value-parameterized test " << "-- did you intend to write TEST_P instead of TEST_F?"; return *parameter_; } private: // Sets parameter value. The caller is responsible for making sure the value // remains alive and unchanged throughout the current test. static void SetParam(const ParamType* parameter) { parameter_ = parameter; } // Static value used for accessing parameter during a test lifetime. static const ParamType* parameter_; // TestClass must be a subclass of WithParamInterface and Test. template friend class internal::ParameterizedTestFactory; }; template const T* WithParamInterface::parameter_ = nullptr; // Most value-parameterized classes can ignore the existence of // WithParamInterface, and can just inherit from ::testing::TestWithParam. template class TestWithParam : public Test, public WithParamInterface { }; // Macros for indicating success/failure in test code. // Skips test in runtime. // Skipping test aborts current function. // Skipped tests are neither successful nor failed. #define GTEST_SKIP() GTEST_SKIP_("Skipped") // ADD_FAILURE unconditionally adds a failure to the current test. // SUCCEED generates a success - it doesn't automatically make the // current test successful, as a test is only successful when it has // no failure. // // EXPECT_* verifies that a certain condition is satisfied. If not, // it behaves like ADD_FAILURE. In particular: // // EXPECT_TRUE verifies that a Boolean condition is true. // EXPECT_FALSE verifies that a Boolean condition is false. // // FAIL and ASSERT_* are similar to ADD_FAILURE and EXPECT_*, except // that they will also abort the current function on failure. People // usually want the fail-fast behavior of FAIL and ASSERT_*, but those // writing data-driven tests often find themselves using ADD_FAILURE // and EXPECT_* more. // Generates a nonfatal failure with a generic message. #define ADD_FAILURE() GTEST_NONFATAL_FAILURE_("Failed") // Generates a nonfatal failure at the given source file location with // a generic message. #define ADD_FAILURE_AT(file, line) \ GTEST_MESSAGE_AT_(file, line, "Failed", \ ::testing::TestPartResult::kNonFatalFailure) // Generates a fatal failure with a generic message. #define GTEST_FAIL() GTEST_FATAL_FAILURE_("Failed") // Like GTEST_FAIL(), but at the given source file location. #define GTEST_FAIL_AT(file, line) \ GTEST_MESSAGE_AT_(file, line, "Failed", \ ::testing::TestPartResult::kFatalFailure) // Define this macro to 1 to omit the definition of FAIL(), which is a // generic name and clashes with some other libraries. #if !GTEST_DONT_DEFINE_FAIL # define FAIL() GTEST_FAIL() #endif // Generates a success with a generic message. #define GTEST_SUCCEED() GTEST_SUCCESS_("Succeeded") // Define this macro to 1 to omit the definition of SUCCEED(), which // is a generic name and clashes with some other libraries. #if !GTEST_DONT_DEFINE_SUCCEED # define SUCCEED() GTEST_SUCCEED() #endif // Macros for testing exceptions. // // * {ASSERT|EXPECT}_THROW(statement, expected_exception): // Tests that the statement throws the expected exception. // * {ASSERT|EXPECT}_NO_THROW(statement): // Tests that the statement doesn't throw any exception. // * {ASSERT|EXPECT}_ANY_THROW(statement): // Tests that the statement throws an exception. #define EXPECT_THROW(statement, expected_exception) \ GTEST_TEST_THROW_(statement, expected_exception, GTEST_NONFATAL_FAILURE_) #define EXPECT_NO_THROW(statement) \ GTEST_TEST_NO_THROW_(statement, GTEST_NONFATAL_FAILURE_) #define EXPECT_ANY_THROW(statement) \ GTEST_TEST_ANY_THROW_(statement, GTEST_NONFATAL_FAILURE_) #define ASSERT_THROW(statement, expected_exception) \ GTEST_TEST_THROW_(statement, expected_exception, GTEST_FATAL_FAILURE_) #define ASSERT_NO_THROW(statement) \ GTEST_TEST_NO_THROW_(statement, GTEST_FATAL_FAILURE_) #define ASSERT_ANY_THROW(statement) \ GTEST_TEST_ANY_THROW_(statement, GTEST_FATAL_FAILURE_) // Boolean assertions. Condition can be either a Boolean expression or an // AssertionResult. For more information on how to use AssertionResult with // these macros see comments on that class. #define EXPECT_TRUE(condition) \ GTEST_TEST_BOOLEAN_(condition, #condition, false, true, \ GTEST_NONFATAL_FAILURE_) #define EXPECT_FALSE(condition) \ GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ GTEST_NONFATAL_FAILURE_) #define ASSERT_TRUE(condition) \ GTEST_TEST_BOOLEAN_(condition, #condition, false, true, \ GTEST_FATAL_FAILURE_) #define ASSERT_FALSE(condition) \ GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ GTEST_FATAL_FAILURE_) // Macros for testing equalities and inequalities. // // * {ASSERT|EXPECT}_EQ(v1, v2): Tests that v1 == v2 // * {ASSERT|EXPECT}_NE(v1, v2): Tests that v1 != v2 // * {ASSERT|EXPECT}_LT(v1, v2): Tests that v1 < v2 // * {ASSERT|EXPECT}_LE(v1, v2): Tests that v1 <= v2 // * {ASSERT|EXPECT}_GT(v1, v2): Tests that v1 > v2 // * {ASSERT|EXPECT}_GE(v1, v2): Tests that v1 >= v2 // // When they are not, Google Test prints both the tested expressions and // their actual values. The values must be compatible built-in types, // or you will get a compiler error. By "compatible" we mean that the // values can be compared by the respective operator. // // Note: // // 1. It is possible to make a user-defined type work with // {ASSERT|EXPECT}_??(), but that requires overloading the // comparison operators and is thus discouraged by the Google C++ // Usage Guide. Therefore, you are advised to use the // {ASSERT|EXPECT}_TRUE() macro to assert that two objects are // equal. // // 2. The {ASSERT|EXPECT}_??() macros do pointer comparisons on // pointers (in particular, C strings). Therefore, if you use it // with two C strings, you are testing how their locations in memory // are related, not how their content is related. To compare two C // strings by content, use {ASSERT|EXPECT}_STR*(). // // 3. {ASSERT|EXPECT}_EQ(v1, v2) is preferred to // {ASSERT|EXPECT}_TRUE(v1 == v2), as the former tells you // what the actual value is when it fails, and similarly for the // other comparisons. // // 4. Do not depend on the order in which {ASSERT|EXPECT}_??() // evaluate their arguments, which is undefined. // // 5. These macros evaluate their arguments exactly once. // // Examples: // // EXPECT_NE(Foo(), 5); // EXPECT_EQ(a_pointer, NULL); // ASSERT_LT(i, array_size); // ASSERT_GT(records.size(), 0) << "There is no record left."; #define EXPECT_EQ(val1, val2) \ EXPECT_PRED_FORMAT2(::testing::internal::EqHelper::Compare, val1, val2) #define EXPECT_NE(val1, val2) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperNE, val1, val2) #define EXPECT_LE(val1, val2) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2) #define EXPECT_LT(val1, val2) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2) #define EXPECT_GE(val1, val2) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2) #define EXPECT_GT(val1, val2) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) #define GTEST_ASSERT_EQ(val1, val2) \ ASSERT_PRED_FORMAT2(::testing::internal::EqHelper::Compare, val1, val2) #define GTEST_ASSERT_NE(val1, val2) \ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperNE, val1, val2) #define GTEST_ASSERT_LE(val1, val2) \ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2) #define GTEST_ASSERT_LT(val1, val2) \ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2) #define GTEST_ASSERT_GE(val1, val2) \ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2) #define GTEST_ASSERT_GT(val1, val2) \ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) // Define macro GTEST_DONT_DEFINE_ASSERT_XY to 1 to omit the definition of // ASSERT_XY(), which clashes with some users' own code. #if !GTEST_DONT_DEFINE_ASSERT_EQ # define ASSERT_EQ(val1, val2) GTEST_ASSERT_EQ(val1, val2) #endif #if !GTEST_DONT_DEFINE_ASSERT_NE # define ASSERT_NE(val1, val2) GTEST_ASSERT_NE(val1, val2) #endif #if !GTEST_DONT_DEFINE_ASSERT_LE # define ASSERT_LE(val1, val2) GTEST_ASSERT_LE(val1, val2) #endif #if !GTEST_DONT_DEFINE_ASSERT_LT # define ASSERT_LT(val1, val2) GTEST_ASSERT_LT(val1, val2) #endif #if !GTEST_DONT_DEFINE_ASSERT_GE # define ASSERT_GE(val1, val2) GTEST_ASSERT_GE(val1, val2) #endif #if !GTEST_DONT_DEFINE_ASSERT_GT # define ASSERT_GT(val1, val2) GTEST_ASSERT_GT(val1, val2) #endif // C-string Comparisons. All tests treat NULL and any non-NULL string // as different. Two NULLs are equal. // // * {ASSERT|EXPECT}_STREQ(s1, s2): Tests that s1 == s2 // * {ASSERT|EXPECT}_STRNE(s1, s2): Tests that s1 != s2 // * {ASSERT|EXPECT}_STRCASEEQ(s1, s2): Tests that s1 == s2, ignoring case // * {ASSERT|EXPECT}_STRCASENE(s1, s2): Tests that s1 != s2, ignoring case // // For wide or narrow string objects, you can use the // {ASSERT|EXPECT}_??() macros. // // Don't depend on the order in which the arguments are evaluated, // which is undefined. // // These macros evaluate their arguments exactly once. #define EXPECT_STREQ(s1, s2) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTREQ, s1, s2) #define EXPECT_STRNE(s1, s2) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRNE, s1, s2) #define EXPECT_STRCASEEQ(s1, s2) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASEEQ, s1, s2) #define EXPECT_STRCASENE(s1, s2)\ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASENE, s1, s2) #define ASSERT_STREQ(s1, s2) \ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTREQ, s1, s2) #define ASSERT_STRNE(s1, s2) \ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRNE, s1, s2) #define ASSERT_STRCASEEQ(s1, s2) \ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASEEQ, s1, s2) #define ASSERT_STRCASENE(s1, s2)\ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASENE, s1, s2) // Macros for comparing floating-point numbers. // // * {ASSERT|EXPECT}_FLOAT_EQ(val1, val2): // Tests that two float values are almost equal. // * {ASSERT|EXPECT}_DOUBLE_EQ(val1, val2): // Tests that two double values are almost equal. // * {ASSERT|EXPECT}_NEAR(v1, v2, abs_error): // Tests that v1 and v2 are within the given distance to each other. // // Google Test uses ULP-based comparison to automatically pick a default // error bound that is appropriate for the operands. See the // FloatingPoint template class in gtest-internal.h if you are // interested in the implementation details. #define EXPECT_FLOAT_EQ(val1, val2)\ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ val1, val2) #define EXPECT_DOUBLE_EQ(val1, val2)\ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ val1, val2) #define ASSERT_FLOAT_EQ(val1, val2)\ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ val1, val2) #define ASSERT_DOUBLE_EQ(val1, val2)\ ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ val1, val2) #define EXPECT_NEAR(val1, val2, abs_error)\ EXPECT_PRED_FORMAT3(::testing::internal::DoubleNearPredFormat, \ val1, val2, abs_error) #define ASSERT_NEAR(val1, val2, abs_error)\ ASSERT_PRED_FORMAT3(::testing::internal::DoubleNearPredFormat, \ val1, val2, abs_error) // These predicate format functions work on floating-point values, and // can be used in {ASSERT|EXPECT}_PRED_FORMAT2*(), e.g. // // EXPECT_PRED_FORMAT2(testing::DoubleLE, Foo(), 5.0); // Asserts that val1 is less than, or almost equal to, val2. Fails // otherwise. In particular, it fails if either val1 or val2 is NaN. GTEST_API_ AssertionResult FloatLE(const char* expr1, const char* expr2, float val1, float val2); GTEST_API_ AssertionResult DoubleLE(const char* expr1, const char* expr2, double val1, double val2); #if GTEST_OS_WINDOWS // Macros that test for HRESULT failure and success, these are only useful // on Windows, and rely on Windows SDK macros and APIs to compile. // // * {ASSERT|EXPECT}_HRESULT_{SUCCEEDED|FAILED}(expr) // // When expr unexpectedly fails or succeeds, Google Test prints the // expected result and the actual result with both a human-readable // string representation of the error, if available, as well as the // hex result code. # define EXPECT_HRESULT_SUCCEEDED(expr) \ EXPECT_PRED_FORMAT1(::testing::internal::IsHRESULTSuccess, (expr)) # define ASSERT_HRESULT_SUCCEEDED(expr) \ ASSERT_PRED_FORMAT1(::testing::internal::IsHRESULTSuccess, (expr)) # define EXPECT_HRESULT_FAILED(expr) \ EXPECT_PRED_FORMAT1(::testing::internal::IsHRESULTFailure, (expr)) # define ASSERT_HRESULT_FAILED(expr) \ ASSERT_PRED_FORMAT1(::testing::internal::IsHRESULTFailure, (expr)) #endif // GTEST_OS_WINDOWS // Macros that execute statement and check that it doesn't generate new fatal // failures in the current thread. // // * {ASSERT|EXPECT}_NO_FATAL_FAILURE(statement); // // Examples: // // EXPECT_NO_FATAL_FAILURE(Process()); // ASSERT_NO_FATAL_FAILURE(Process()) << "Process() failed"; // #define ASSERT_NO_FATAL_FAILURE(statement) \ GTEST_TEST_NO_FATAL_FAILURE_(statement, GTEST_FATAL_FAILURE_) #define EXPECT_NO_FATAL_FAILURE(statement) \ GTEST_TEST_NO_FATAL_FAILURE_(statement, GTEST_NONFATAL_FAILURE_) // Causes a trace (including the given source file path and line number, // and the given message) to be included in every test failure message generated // by code in the scope of the lifetime of an instance of this class. The effect // is undone with the destruction of the instance. // // The message argument can be anything streamable to std::ostream. // // Example: // testing::ScopedTrace trace("file.cc", 123, "message"); // class GTEST_API_ ScopedTrace { public: // The c'tor pushes the given source file location and message onto // a trace stack maintained by Google Test. // Template version. Uses Message() to convert the values into strings. // Slow, but flexible. template ScopedTrace(const char* file, int line, const T& message) { PushTrace(file, line, (Message() << message).GetString()); } // Optimize for some known types. ScopedTrace(const char* file, int line, const char* message) { PushTrace(file, line, message ? message : "(null)"); } ScopedTrace(const char* file, int line, const std::string& message) { PushTrace(file, line, message); } // The d'tor pops the info pushed by the c'tor. // // Note that the d'tor is not virtual in order to be efficient. // Don't inherit from ScopedTrace! ~ScopedTrace(); private: void PushTrace(const char* file, int line, std::string message); GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedTrace); } GTEST_ATTRIBUTE_UNUSED_; // A ScopedTrace object does its job in its // c'tor and d'tor. Therefore it doesn't // need to be used otherwise. // Causes a trace (including the source file path, the current line // number, and the given message) to be included in every test failure // message generated by code in the current scope. The effect is // undone when the control leaves the current scope. // // The message argument can be anything streamable to std::ostream. // // In the implementation, we include the current line number as part // of the dummy variable name, thus allowing multiple SCOPED_TRACE()s // to appear in the same block - as long as they are on different // lines. // // Assuming that each thread maintains its own stack of traces. // Therefore, a SCOPED_TRACE() would (correctly) only affect the // assertions in its own thread. #define SCOPED_TRACE(message) \ ::testing::ScopedTrace GTEST_CONCAT_TOKEN_(gtest_trace_, __LINE__)(\ __FILE__, __LINE__, (message)) // Compile-time assertion for type equality. // StaticAssertTypeEq() compiles if and only if type1 and type2 // are the same type. The value it returns is not interesting. // // Instead of making StaticAssertTypeEq a class template, we make it a // function template that invokes a helper class template. This // prevents a user from misusing StaticAssertTypeEq by // defining objects of that type. // // CAVEAT: // // When used inside a method of a class template, // StaticAssertTypeEq() is effective ONLY IF the method is // instantiated. For example, given: // // template class Foo { // public: // void Bar() { testing::StaticAssertTypeEq(); } // }; // // the code: // // void Test1() { Foo foo; } // // will NOT generate a compiler error, as Foo::Bar() is never // actually instantiated. Instead, you need: // // void Test2() { Foo foo; foo.Bar(); } // // to cause a compiler error. template constexpr bool StaticAssertTypeEq() noexcept { static_assert(std::is_same::value, "type1 and type2 are not the same type"); return true; } // Defines a test. // // The first parameter is the name of the test suite, and the second // parameter is the name of the test within the test suite. // // The convention is to end the test suite name with "Test". For // example, a test suite for the Foo class can be named FooTest. // // Test code should appear between braces after an invocation of // this macro. Example: // // TEST(FooTest, InitializesCorrectly) { // Foo foo; // EXPECT_TRUE(foo.StatusIsOK()); // } // Note that we call GetTestTypeId() instead of GetTypeId< // ::testing::Test>() here to get the type ID of testing::Test. This // is to work around a suspected linker bug when using Google Test as // a framework on Mac OS X. The bug causes GetTypeId< // ::testing::Test>() to return different values depending on whether // the call is from the Google Test framework itself or from user test // code. GetTestTypeId() is guaranteed to always return the same // value, as it always calls GetTypeId<>() from the Google Test // framework. #define GTEST_TEST(test_suite_name, test_name) \ GTEST_TEST_(test_suite_name, test_name, ::testing::Test, \ ::testing::internal::GetTestTypeId()) // Define this macro to 1 to omit the definition of TEST(), which // is a generic name and clashes with some other libraries. #if !GTEST_DONT_DEFINE_TEST #define TEST(test_suite_name, test_name) GTEST_TEST(test_suite_name, test_name) #endif // Defines a test that uses a test fixture. // // The first parameter is the name of the test fixture class, which // also doubles as the test suite name. The second parameter is the // name of the test within the test suite. // // A test fixture class must be declared earlier. The user should put // the test code between braces after using this macro. Example: // // class FooTest : public testing::Test { // protected: // void SetUp() override { b_.AddElement(3); } // // Foo a_; // Foo b_; // }; // // TEST_F(FooTest, InitializesCorrectly) { // EXPECT_TRUE(a_.StatusIsOK()); // } // // TEST_F(FooTest, ReturnsElementCountCorrectly) { // EXPECT_EQ(a_.size(), 0); // EXPECT_EQ(b_.size(), 1); // } // // GOOGLETEST_CM0011 DO NOT DELETE #define TEST_F(test_fixture, test_name)\ GTEST_TEST_(test_fixture, test_name, test_fixture, \ ::testing::internal::GetTypeId()) // Returns a path to temporary directory. // Tries to determine an appropriate directory for the platform. GTEST_API_ std::string TempDir(); #ifdef _MSC_VER # pragma warning(pop) #endif // Dynamically registers a test with the framework. // // This is an advanced API only to be used when the `TEST` macros are // insufficient. The macros should be preferred when possible, as they avoid // most of the complexity of calling this function. // // The `factory` argument is a factory callable (move-constructible) object or // function pointer that creates a new instance of the Test object. It // handles ownership to the caller. The signature of the callable is // `Fixture*()`, where `Fixture` is the test fixture class for the test. All // tests registered with the same `test_suite_name` must return the same // fixture type. This is checked at runtime. // // The framework will infer the fixture class from the factory and will call // the `SetUpTestSuite` and `TearDownTestSuite` for it. // // Must be called before `RUN_ALL_TESTS()` is invoked, otherwise behavior is // undefined. // // Use case example: // // class MyFixture : public ::testing::Test { // public: // // All of these optional, just like in regular macro usage. // static void SetUpTestSuite() { ... } // static void TearDownTestSuite() { ... } // void SetUp() override { ... } // void TearDown() override { ... } // }; // // class MyTest : public MyFixture { // public: // explicit MyTest(int data) : data_(data) {} // void TestBody() override { ... } // // private: // int data_; // }; // // void RegisterMyTests(const std::vector& values) { // for (int v : values) { // ::testing::RegisterTest( // "MyFixture", ("Test" + std::to_string(v)).c_str(), nullptr, // std::to_string(v).c_str(), // __FILE__, __LINE__, // // Important to use the fixture type as the return type here. // [=]() -> MyFixture* { return new MyTest(v); }); // } // } // ... // int main(int argc, char** argv) { // std::vector values_to_test = LoadValuesFromConfig(); // RegisterMyTests(values_to_test); // ... // return RUN_ALL_TESTS(); // } // template TestInfo* RegisterTest(const char* test_suite_name, const char* test_name, const char* type_param, const char* value_param, const char* file, int line, Factory factory) { using TestT = typename std::remove_pointer::type; class FactoryImpl : public internal::TestFactoryBase { public: explicit FactoryImpl(Factory f) : factory_(std::move(f)) {} Test* CreateTest() override { return factory_(); } private: Factory factory_; }; return internal::MakeAndRegisterTestInfo( test_suite_name, test_name, type_param, value_param, internal::CodeLocation(file, line), internal::GetTypeId(), internal::SuiteApiResolver::GetSetUpCaseOrSuite(file, line), internal::SuiteApiResolver::GetTearDownCaseOrSuite(file, line), new FactoryImpl{std::move(factory)}); } } // namespace testing // Use this function in main() to run all tests. It returns 0 if all // tests are successful, or 1 otherwise. // // RUN_ALL_TESTS() should be invoked after the command line has been // parsed by InitGoogleTest(). // // This function was formerly a macro; thus, it is in the global // namespace and has an all-caps name. int RUN_ALL_TESTS() GTEST_MUST_USE_RESULT_; inline int RUN_ALL_TESTS() { return ::testing::UnitTest::GetInstance()->Run(); } GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 #endif // GTEST_INCLUDE_GTEST_GTEST_H_ ucc-1.8.0/test/gtest/common/test_obj_size.cc0000664000175000017500000000102215211535620021220 0ustar alastairalastair/** * Copyright (c) 2001-2019, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Huawei Technologies Co., Ltd. 2020. All rights reserved. * * See file LICENSE for terms. */ #ifdef HAVE_CONFIG_H # include "config.h" #endif #include extern "C" { #include } class test_obj_size : public ucc::test { }; UCC_TEST_F(test_obj_size, size) { /* lets try to keep it within 8 cache lines currently 480b */ EXPECT_LT(sizeof(ucc_coll_task_t), 64 * 8); } ucc-1.8.0/test/gtest/common/test.h0000664000175000017500000001310215211535620017200 0ustar alastairalastair/** * Copyright (c) 2001-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Huawei Technologies Co., Ltd. 2020. All rights reserved. * * See file LICENSE for terms. */ #ifndef UCC_TEST_BASE_H #define UCC_TEST_BASE_H extern "C" { } #include #undef I #define _Imag _Imaginary_I #include "gtest.h" #define UCC_CHECK(_call) EXPECT_EQ(UCC_OK, (_call)) #define UCC_TEST_SKIP_R(_str) GTEST_SKIP_(_str) namespace ucc { class test : public testing::Test { }; #define UCC_TEST_F(...) TEST_F(__VA_ARGS__) #define UCC_TEST_P(...) TEST_P(__VA_ARGS__) } #define ASSERT_FLOAT32_COMPLEX_EQ(expected, actual) \ do { \ static float expected_real = crealf(expected); \ static float expected_imaginary = cimagf(expected); \ static float actual_real = crealf(actual); \ static float actual_imaginary = cimagf(actual); \ ASSERT_PRED_FORMAT2( \ ::testing::internal::CmpHelperFloatingPointEQ, \ expected_real, actual_real); \ ASSERT_PRED_FORMAT2( \ ::testing::internal::CmpHelperFloatingPointEQ, \ expected_imaginary, actual_imaginary); \ } while (0) #define ASSERT_FLOAT64_COMPLEX_EQ(expected, actual) \ do { \ static double expected_real = creal(expected); \ static double expected_imaginary = cimag(expected); \ static double actual_real = creal(actual); \ static double actual_imaginary = cimag(actual); \ ASSERT_PRED_FORMAT2( \ ::testing::internal::CmpHelperFloatingPointEQ, \ expected_real, actual_real); \ ASSERT_PRED_FORMAT2( \ ::testing::internal::CmpHelperFloatingPointEQ, \ expected_imaginary, actual_imaginary); \ } while (0) #define ASSERT_FLOAT128_COMPLEX_EQ(expected, actual) \ do { \ static long double expected_real = creall(expected); \ static long double expected_imaginary = cimagl(expected); \ static long double actual_real = creall(actual); \ static long double actual_imaginary = cimagl(actual); \ ASSERT_PRED_FORMAT2( \ ::testing::internal::CmpHelperFloatingPointEQ, \ expected_real, actual_real); \ ASSERT_PRED_FORMAT2( \ ::testing::internal::CmpHelperFloatingPointEQ, \ expected_imaginary, actual_imaginary); \ } while (0) #define EXPECT_FLOAT32_COMPLEX_EQ(expected, actual) \ static float expected_real = crealf(expected); \ static float expected_imaginary = cimagf(expected); \ static float actual_real = crealf(actual); \ static float actual_imaginary = cimagf(actual); \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ expected_real, actual_real) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ expected_imaginary, actual_imaginary) #define EXPECT_FLOAT64_COMPLEX_EQ(expected, actual) \ static double expected_real = creal(expected); \ static double expected_imaginary = cimag(expected); \ static double actual_real = creal(actual); \ static double actual_imaginary = cimag(actual); \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ expected_real, actual_real) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ expected_imaginary, actual_imaginary) #define EXPECT_FLOAT128_COMPLEX_EQ(expected, actual) \ static long double expected_real = creall(expected); \ static long double expected_imaginary = cimagl(expected); \ static long double actual_real = creall(actual); \ static long double actual_imaginary = cimagl(actual); \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ expected_real, actual_real) \ EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ expected_imaginary, actual_imaginary) #endif ucc-1.8.0/test/gtest/common/test_ucc.cc0000664000175000017500000005327115211535620020203 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_ucc.h" extern "C" { #include "core/ucc_team.h" #include "components/tl/ucc_tl.h" } constexpr ucc_lib_params_t UccProcess::default_lib_params; constexpr ucc_context_params_t UccProcess::default_ctx_params; constexpr int UccJob::staticTeamSizes[]; UccProcess::UccProcess(int _job_rank, const ucc_lib_params_t &lib_params, const ucc_context_params_t &_ctx_params) { ucc_lib_config_h lib_config; ucc_status_t status; std::stringstream err_msg; job_rank = _job_rank; ctx_params = _ctx_params; status = ucc_lib_config_read(NULL, NULL, &lib_config); if (status != UCC_OK) { err_msg << "ucc_lib_config_read failed"; goto exit_err; } status = ucc_init(&lib_params, lib_config, &lib_h); ucc_lib_config_release(lib_config); if (status != UCC_OK) { err_msg << "ucc_init failed"; goto exit_err; } return; exit_err: err_msg << ": "<< ucc_status_string(status) << " (" << status << ")"; throw std::runtime_error(err_msg.str()); } UccProcess::~UccProcess() { EXPECT_EQ(UCC_OK, ucc_context_destroy(ctx_h)); EXPECT_EQ(UCC_OK, ucc_finalize(lib_h)); if (ctx_params.mask & UCC_CONTEXT_PARAM_FIELD_MEM_PARAMS) { for (auto i = 0; i < UCC_TEST_N_MEM_SEGMENTS; i++) { ucc_free(onesided_buf[i]); } } } ucc_status_t UccTeam::allgather(void *src_buf, void *recv_buf, size_t size, void *coll_info, void **request) { UccTeam::allgather_coll_info_t *ci = (UccTeam::allgather_coll_info_t *)coll_info; int my_rank = ci->my_rank; ci->self->ag[my_rank].sbuf = src_buf; ci->self->ag[my_rank].rbuf = recv_buf; ci->self->ag[my_rank].len = size; ci->self->ag[my_rank].phase = UccTeam::AG_READY; *request = (void *)ci; return UCC_OK; } void UccTeam::test_allgather(size_t msglen) { int *sbufs[n_procs]; int *rbufs[n_procs]; size_t count = msglen/sizeof(int); std::vector cis; std::vector reqs; for (int i=0; iself->n_procs; switch (ci->self->ag[ci->my_rank].phase) { case UccTeam::AG_READY: for (int i = 0; i < n_procs; i++) { if ((ci->self->ag[i].phase == UccTeam::AG_INIT) || (ci->self->ag[i].phase == UccTeam::AG_COMPLETE)) { return UCC_INPROGRESS; } } for (int i = 0; i < n_procs; i++) { memcpy((void *)((ptrdiff_t)ci->self->ag[ci->my_rank].rbuf + i * ci->self->ag[i].len), ci->self->ag[i].sbuf, ci->self->ag[i].len); } ci->self->ag[ci->my_rank].phase = UccTeam::AG_COPY_DONE; ; ci->self->copy_complete_count++; break; case UccTeam::AG_COPY_DONE: if (ci->my_rank == 0 && ci->self->copy_complete_count == n_procs) { for (int i = 0; i < n_procs; i++) { ci->self->ag[i].phase = UccTeam::AG_COMPLETE; } ci->self->copy_complete_count = 0; } break; case UccTeam::AG_COMPLETE: return UCC_OK; default: break; } return UCC_INPROGRESS; } ucc_status_t UccTeam::req_free(void *request) { UccTeam::allgather_coll_info_t *ci = (UccTeam::allgather_coll_info_t *)request; ci->self->ag[ci->my_rank].phase = UccTeam::AG_INIT; return UCC_OK; } uint64_t rank_map_cb(uint64_t ep, void *cb_ctx) { UccTeam *team = (UccTeam*)cb_ctx; return (uint64_t)team->procs[(int)ep].p.get()->job_rank; } void UccTeam::init_team(bool use_team_ep_map, bool use_ep_range, bool is_onesided) { ucc_team_params_t team_params; std::vector cis; ucc_status_t status; for (int i = 0; i < n_procs; i++) { cis.push_back(new allgather_coll_info); cis.back()->self = this; cis.back()->my_rank = i; if (use_ep_range) { team_params.ep = i; team_params.ep_range = UCC_COLLECTIVE_EP_RANGE_CONTIG; team_params.mask = UCC_TEAM_PARAM_FIELD_EP | UCC_TEAM_PARAM_FIELD_EP_RANGE; } else { team_params.mask = 0; } if (use_team_ep_map) { team_params.mask |= UCC_TEAM_PARAM_FIELD_EP_MAP; team_params.ep_map.type = UCC_EP_MAP_CB; team_params.ep_map.ep_num = n_procs; team_params.ep_map.cb.cb = rank_map_cb; team_params.ep_map.cb.cb_ctx = (void*)this; } else { team_params.oob.allgather = allgather; team_params.oob.req_test = req_test; team_params.oob.req_free = req_free; team_params.oob.coll_info = (void *)cis.back(); team_params.oob.n_oob_eps = n_procs; team_params.oob.oob_ep = i; team_params.mask |= UCC_TEAM_PARAM_FIELD_OOB; } if (is_onesided) { team_params.mask |= UCC_TEAM_PARAM_FIELD_FLAGS; team_params.flags = UCC_TEAM_FLAG_COLL_WORK_BUFFER; } EXPECT_EQ(UCC_OK, ucc_team_create_post(&(procs[i].p.get()->ctx_h), 1, &team_params, &(procs[i].team))); } int all_done = 0; while (!all_done) { all_done = 1; for (int i = 0; i < n_procs; i++) { ucc_context_progress(procs[i].p.get()->ctx_h); status = ucc_team_create_test(procs[i].team); ASSERT_GE(status, 0); if (UCC_INPROGRESS == status) { all_done = 0; } } } for (auto c : cis) { delete c; } } void UccTeam::destroy_team() { ucc_status_t status; bool all_done; do { all_done = true; for (auto &p : procs) { if (p.team) { status = ucc_team_destroy(p.team); if (UCC_OK == status) { p.team = NULL; } else if (status < 0) { return; } else { all_done = false; } } } } while (!all_done); } void UccTeam::progress() { for (auto &p : procs) { ucc_context_progress(p.p->ctx_h); } } UccTeam::UccTeam(std::vector &_procs, bool use_team_ep_map, bool use_ep_range, bool is_onesided) { n_procs = _procs.size(); ag.resize(n_procs); for (auto &p : _procs) { procs.push_back(proc(p)); } for (auto &a : ag) { a.phase = AG_INIT; } copy_complete_count = 0; init_team(use_team_ep_map, use_ep_range, is_onesided); // test_allgather(128); } UccTeam::~UccTeam() { destroy_team(); } UccJob::UccJob(int _n_procs, ucc_job_ctx_mode_t _ctx_mode, ucc_job_env_t vars) : ta(_n_procs), n_procs(_n_procs), ctx_mode(_ctx_mode) { ucc_job_env_t env_bkp; char *var; /* NCCL TL is disabled since it currently can not support non-blocking team creation. */ vars.push_back({"UCC_TL_NCCL_TUNE", "0"}); vars.push_back({"UCC_TL_RCCL_TUNE", "0"}); /* CUDA TL is disabled since cuda context is not initialized in threads. */ vars.push_back({"UCC_TL_CUDA_TUNE", "0"}); /* GDR is temporarily disabled due to known issue that may result in a hang in the destruction flow */ vars.push_back({"UCX_IB_GPU_DIRECT_RDMA", "no"}); for (auto &v : vars) { var = std::getenv(v.first.c_str()); if (var) { /* found env - back it up for later restore after processes creation */ env_bkp.push_back(ucc_env_var_t(v.first, var)); } setenv(v.first.c_str(), v.second.c_str(), 1); } for (int i = 0; i < n_procs; i++) { procs.push_back(std::make_shared(i)); } create_context(); for (auto &v : env_bkp) { /*restore original env */ setenv(v.first.c_str(), v.second.c_str(), 1); } } void thread_allgather(void *src_buf, void *recv_buf, size_t size, ThreadAllgatherReq *ta_req) { ThreadAllgather *ta = ta_req->ta; while (ta->ready_count > ta->n_procs) { std::this_thread::yield(); } ta->lock.lock(); if (!ta->buffer) { ucc_assert(0 == ta->ready_count); ta->buffer = malloc(size * ta->n_procs); ta->ready_count = 0; } memcpy((void*)((ptrdiff_t)ta->buffer + size * ta_req->rank), src_buf, size); ta->ready_count++; ta->lock.unlock(); while (ta->ready_count < ta->n_procs) { std::this_thread::yield(); } memcpy(recv_buf, ta->buffer, size * ta->n_procs); ta->lock.lock(); ta->ready_count++; if (ta->ready_count == 2 * ta->n_procs) { free(ta->buffer); ta->buffer = NULL; ta->ready_count = 0; } ta->lock.unlock(); ta_req->status = UCC_OK; } ucc_status_t thread_allgather_start(void *src_buf, void *recv_buf, size_t size, void *coll_info, void **request) { ThreadAllgatherReq *ta_req = (ThreadAllgatherReq*)coll_info; *request = coll_info; while (ta_req->status != UCC_OPERATION_INITIALIZED) { std::this_thread::yield(); } ta_req->status = UCC_INPROGRESS; ta_req->t = std::thread(thread_allgather, src_buf, recv_buf, size, ta_req); return UCC_OK; } ucc_status_t thread_allgather_req_test(void *request) { ThreadAllgatherReq *ta_req = (ThreadAllgatherReq*)request; return ta_req->status; } ucc_status_t thread_allgather_req_free(void *request) { ThreadAllgatherReq *ta_req = (ThreadAllgatherReq*)request; ta_req->t.join(); ta_req->status = UCC_OPERATION_INITIALIZED; return UCC_OK; } void proc_context_create(UccProcess_h proc, int id, ThreadAllgather *ta, bool is_global) { const int nnodes = 2; const int nsockets = 2; const int nnumas = 3; ucc_status_t status; ucc_context_config_h ctx_config; std::stringstream err_msg; ucc_proc_info_t proc_info; int node, local_ppn, local_rank, job_size, block; status = ucc_context_config_read(proc->lib_h, NULL, &ctx_config); if (status != UCC_OK) { err_msg << "ucc_context_config_read failed"; goto exit_err; } if (is_global) { proc->ctx_params.mask |= UCC_CONTEXT_PARAM_FIELD_OOB; proc->ctx_params.oob.allgather = thread_allgather_start; proc->ctx_params.oob.req_test = thread_allgather_req_test; proc->ctx_params.oob.req_free = thread_allgather_req_free; proc->ctx_params.oob.coll_info = (void*) &ta->reqs[id]; proc->ctx_params.oob.n_oob_eps = ta->n_procs; proc->ctx_params.oob.oob_ep = id; /* Simulate multi-node topology for larger gtest coverage */ job_size = ta->n_procs; block = ucc_buffer_block_count(job_size, nnodes, 0); node = id / block; local_ppn = ucc_buffer_block_count(job_size, nnodes, node); local_rank = id - ucc_buffer_block_offset(job_size, nnodes, node); proc_info.host_hash = node + 1; block = ucc_buffer_block_count(local_ppn, nsockets, 0); proc_info.socket_id = local_rank / block; block = ucc_buffer_block_count(local_ppn, nnumas, 0); proc_info.numa_id = local_rank / block; proc_info.pid = id + 1; } else { proc_info = ucc_local_proc; } status = ucc_context_create_proc_info(proc->lib_h, &proc->ctx_params, ctx_config, &proc->ctx_h, &proc_info); ucc_context_config_release(ctx_config); if (status != UCC_OK) { err_msg << "ucc_context_create failed"; goto exit_err; } return; exit_err: err_msg << ": "<< ucc_status_string(status) << " (" << status << ")"; throw std::runtime_error(err_msg.str()); } void proc_context_create_mem_params(UccProcess_h proc, int id, ThreadAllgather *ta) { ucc_status_t status; ucc_context_config_h ctx_config; std::stringstream err_msg; ucc_mem_map_t map[UCC_TEST_N_MEM_SEGMENTS]; status = ucc_context_config_read(proc->lib_h, NULL, &ctx_config); if (status != UCC_OK) { err_msg << "ucc_context_config_read failed"; goto exit_err; } for (auto i = 0; i < UCC_TEST_N_MEM_SEGMENTS; i++) { proc->onesided_buf[i] = ucc_calloc(UCC_TEST_MEM_SEGMENT_SIZE, 1, "onesided_buffer"); EXPECT_NE(proc->onesided_buf[i], nullptr); map[i].address = proc->onesided_buf[i]; map[i].len = UCC_TEST_MEM_SEGMENT_SIZE; } proc->ctx_params.mask = UCC_CONTEXT_PARAM_FIELD_OOB; proc->ctx_params.mask |= UCC_CONTEXT_PARAM_FIELD_MEM_PARAMS; proc->ctx_params.oob.allgather = thread_allgather_start; proc->ctx_params.oob.req_test = thread_allgather_req_test; proc->ctx_params.oob.req_free = thread_allgather_req_free; proc->ctx_params.oob.coll_info = (void *)&ta->reqs[id]; proc->ctx_params.oob.n_oob_eps = ta->n_procs; proc->ctx_params.oob.oob_ep = id; proc->ctx_params.mem_params.segments = map; proc->ctx_params.mem_params.n_segments = UCC_TEST_N_MEM_SEGMENTS; status = ucc_context_create(proc->lib_h, &proc->ctx_params, ctx_config, &proc->ctx_h); ucc_context_config_release(ctx_config); if (status != UCC_OK) { err_msg << "ucc_context_create for one-sided context failed"; goto exit_err; } return; exit_err: err_msg << ": " << ucc_status_string(status) << " (" << status << ")"; throw std::runtime_error(err_msg.str()); } void UccJob::create_context() { std::vector workers; for (auto i = 0; i < procs.size(); i++) { if (ctx_mode == UCC_JOB_CTX_GLOBAL_ONESIDED) { workers.push_back( std::thread(proc_context_create_mem_params, procs[i], i, &ta)); } else { workers.push_back(std::thread(proc_context_create, procs[i], i, &ta, ctx_mode == UCC_JOB_CTX_GLOBAL)); } } for (auto i = 0; i < procs.size(); i++) { workers[i].join(); } } void thread_proc_destruct(std::vector *procs, int i) { ucc_assert(true == (*procs)[i].unique()); (*procs)[i] = NULL; } UccJob::~UccJob() { std::vector workers; if (this == UccJob::staticUccJob) { staticTeams.clear(); } for (int i = 0; i < n_procs; i++) { workers.push_back(std::thread(thread_proc_destruct, &procs, i)); } for (int i = 0; i < n_procs; i++) { workers[i].join(); } } UccJob* UccJob::staticUccJob = NULL; UccJob* UccJob::getStaticJob() { if (!staticUccJob) { staticUccJob = new UccJob(UccJob::staticUccJobSize); } return staticUccJob; } std::vector UccJob::staticTeams; const std::vector &UccJob::getStaticTeams() { std::vector teamSizes(std::begin(staticTeamSizes), std::end(staticTeamSizes)); if (0 == staticTeams.size()) { for (auto ts : teamSizes) { if (ts == 1 && !tl_self_available()) { /* don't use team_size = 1 if there is no tl/self. we can't modify nStaticTeams, so just use some other team_size */ ts = 3; } staticTeams.push_back(getStaticJob()->create_team(ts)); } /* Create one more team with reversed ranks order */ std::vector ranks; for (auto r = staticUccJobSize - 1; r >= 0; r--) { ranks.push_back(r); } staticTeams.push_back(getStaticJob()->create_team(ranks, true)); } return staticTeams; } void UccJob::cleanup() { if (staticUccJob) { delete staticUccJob; } } UccTeam_h UccJob::create_team(int _n_procs, bool use_team_ep_map, bool use_ep_range, bool is_onesided) { EXPECT_GE(n_procs, _n_procs); std::vector team_procs; for (int i = 0; i < _n_procs; i++) { team_procs.push_back(procs[i]); } return std::make_shared(team_procs, use_team_ep_map, use_ep_range, is_onesided); } UccTeam_h UccJob::create_team(std::vector &ranks, bool use_team_ep_map, bool use_ep_range, bool is_onesided) { EXPECT_GE(n_procs, ranks.size()); std::vector team_procs; for (int i = 0; i < ranks.size(); i++) { team_procs.push_back(procs[ranks[i]]); } return std::make_shared(team_procs, use_team_ep_map, use_ep_range, is_onesided); } UccReq::UccReq(UccTeam_h _team, ucc_coll_args_t *args) : team(_team) { ucc_coll_req_h req; for (auto &p : team->procs) { if (UCC_OK != ucc_collective_init(args, &req, p.team)) { goto err; } reqs.push_back(req); } return; err: reqs.clear(); } UccReq::UccReq(UccTeam_h _team, UccCollCtxVec ctxs) : team(_team) { std::vector err_st; ucc_coll_req_h req; ucc_status_t st; EXPECT_EQ(team->procs.size(), ctxs.size()); status = UCC_OK; for (auto i = 0; i < team->procs.size(); i++) { if (!ctxs[i]) { continue; } if (UCC_OK !=(st = ucc_collective_init(ctxs[i]->args, &req, team->procs[i].team))) { err_st.push_back(st); } else { reqs.push_back(req); } } if (err_st.size() > 0) { /* All error status should be equal, otherwise it is real fatal error. Only expected error is NOT_SUPPORTED. If collective init returns NOT_SUPPORTED it has to be symmetric for all ranks */ if (!std::equal(err_st.begin() + 1, err_st.end(), err_st.begin()) || err_st.size() != team->procs.size() || err_st[0] != UCC_ERR_NOT_SUPPORTED) { status = UCC_ERR_NO_MESSAGE; } else { ucc_assert(err_st[0] = UCC_ERR_NOT_SUPPORTED); status = err_st[0]; } } } UccReq::~UccReq() { for (auto r : reqs) { EXPECT_EQ(UCC_OK, ucc_collective_finalize(r)); } } void UccReq::start() { ucc_status_t st; for (auto r : reqs) { st = ucc_collective_post(r); ASSERT_EQ(UCC_OK, st); st = ucc_collective_test(r); ASSERT_NE(UCC_OPERATION_INITIALIZED, st); } } ucc_status_t UccReq::test() { ucc_status_t st = UCC_OK; for (auto r : reqs) { st = ucc_collective_test(r); if (UCC_OK != st) { break; } } return st; } ucc_status_t UccReq::wait() { ucc_status_t st; while (UCC_OK != (st = test())) { if (st < 0) { break; } team->progress(); } return st; } void UccReq::waitall(std::vector &reqs) { bool alldone = false; ucc_status_t status; while (!alldone) { alldone = true; for (auto &r : reqs) { if (UCC_OK != (status = r.test())) { if (status < 0) { return; } alldone = false; r.team->progress(); } } } } void UccReq::startall(std::vector &reqs) { for (auto &r : reqs) { r.start(); } } void UccCollArgs::set_mem_type(ucc_memory_type_t _mt) { mem_type = _mt; } void UccCollArgs::set_inplace(gtest_ucc_inplace_t _inplace) { inplace = _inplace; } void UccCollArgs::set_contig(bool _is_contig) { is_contig = _is_contig; } void clear_buffer(void *_buf, size_t size, ucc_memory_type_t mt, uint8_t value) { void *buf = _buf; if (mt != UCC_MEMORY_TYPE_HOST) { buf = ucc_malloc(size, "buf"); ASSERT_NE(0, (uintptr_t)buf); } memset(buf, value, size); if (UCC_MEMORY_TYPE_HOST != mt) { UCC_CHECK(ucc_mc_memcpy(_buf, buf, size, mt, UCC_MEMORY_TYPE_HOST)); ucc_free(buf); } } bool tl_self_available() { ucc_tl_context_t *tl_ctx; ucc_status_t status; status = ucc_tl_context_get(UccJob::getStaticJob()->procs[0]->ctx_h, "self", &tl_ctx); if (UCC_OK != status) { return false; } ucc_tl_context_put(tl_ctx); return true; } ucc-1.8.0/test/gtest/common/main.cc0000664000175000017500000000112015211535620017300 0ustar alastairalastair/** * Copyright (c) 2001-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) UT-Battelle, LLC. 2014. ALL RIGHTS RESERVED. * Copyright (C) Huawei Technologies Co., Ltd. 2020. All rights reserved. * See file LICENSE for terms. */ #ifdef HAVE_CONFIG_H # include "config.h" #endif #ifdef HAVE_CUDA #include #endif #include "test_ucc.h" int main(int argc, char **argv) { int ret; #ifdef HAVE_CUDA cudaSetDevice(0); #endif ::testing::InitGoogleTest(&argc, argv); ret = RUN_ALL_TESTS(); UccJob::cleanup(); return ret; } ucc-1.8.0/test/gtest/common/gtest-all.cc0000664000175000017500000152007715211535620020272 0ustar alastairalastair// Copyright 2008, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Google C++ Testing and Mocking Framework (Google Test) // // Sometimes it's desirable to build Google Test by compiling a single file. // This file serves this purpose. // This line ensures that gtest.h can be compiled on its own, even // when it's fused. #include "gtest.h" // The following lines pull in the real gtest *.cc files. // Copyright 2005, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // The Google C++ Testing and Mocking Framework (Google Test) // Copyright 2007, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Utilities for testing Google Test itself and code that uses Google Test // (e.g. frameworks built on top of Google Test). // GOOGLETEST_CM0004 DO NOT DELETE #ifndef GTEST_INCLUDE_GTEST_GTEST_SPI_H_ #define GTEST_INCLUDE_GTEST_GTEST_SPI_H_ GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) namespace testing { // This helper class can be used to mock out Google Test failure reporting // so that we can test Google Test or code that builds on Google Test. // // An object of this class appends a TestPartResult object to the // TestPartResultArray object given in the constructor whenever a Google Test // failure is reported. It can either intercept only failures that are // generated in the same thread that created this object or it can intercept // all generated failures. The scope of this mock object can be controlled with // the second argument to the two arguments constructor. class GTEST_API_ ScopedFakeTestPartResultReporter : public TestPartResultReporterInterface { public: // The two possible mocking modes of this object. enum InterceptMode { INTERCEPT_ONLY_CURRENT_THREAD, // Intercepts only thread local failures. INTERCEPT_ALL_THREADS // Intercepts all failures. }; // The c'tor sets this object as the test part result reporter used // by Google Test. The 'result' parameter specifies where to report the // results. This reporter will only catch failures generated in the current // thread. DEPRECATED explicit ScopedFakeTestPartResultReporter(TestPartResultArray* result); // Same as above, but you can choose the interception scope of this object. ScopedFakeTestPartResultReporter(InterceptMode intercept_mode, TestPartResultArray* result); // The d'tor restores the previous test part result reporter. ~ScopedFakeTestPartResultReporter() override; // Appends the TestPartResult object to the TestPartResultArray // received in the constructor. // // This method is from the TestPartResultReporterInterface // interface. void ReportTestPartResult(const TestPartResult& result) override; private: void Init(); const InterceptMode intercept_mode_; TestPartResultReporterInterface* old_reporter_; TestPartResultArray* const result_; GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedFakeTestPartResultReporter); }; namespace internal { // A helper class for implementing EXPECT_FATAL_FAILURE() and // EXPECT_NONFATAL_FAILURE(). Its destructor verifies that the given // TestPartResultArray contains exactly one failure that has the given // type and contains the given substring. If that's not the case, a // non-fatal failure will be generated. class GTEST_API_ SingleFailureChecker { public: // The constructor remembers the arguments. SingleFailureChecker(const TestPartResultArray* results, TestPartResult::Type type, const std::string& substr); ~SingleFailureChecker(); private: const TestPartResultArray* const results_; const TestPartResult::Type type_; const std::string substr_; GTEST_DISALLOW_COPY_AND_ASSIGN_(SingleFailureChecker); }; } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 // A set of macros for testing Google Test assertions or code that's expected // to generate Google Test fatal failures. It verifies that the given // statement will cause exactly one fatal Google Test failure with 'substr' // being part of the failure message. // // There are two different versions of this macro. EXPECT_FATAL_FAILURE only // affects and considers failures generated in the current thread and // EXPECT_FATAL_FAILURE_ON_ALL_THREADS does the same but for all threads. // // The verification of the assertion is done correctly even when the statement // throws an exception or aborts the current function. // // Known restrictions: // - 'statement' cannot reference local non-static variables or // non-static members of the current object. // - 'statement' cannot return a value. // - You cannot stream a failure message to this macro. // // Note that even though the implementations of the following two // macros are much alike, we cannot refactor them to use a common // helper macro, due to some peculiarity in how the preprocessor // works. The AcceptsMacroThatExpandsToUnprotectedComma test in // gtest_unittest.cc will fail to compile if we do that. #define EXPECT_FATAL_FAILURE(statement, substr) \ do { \ class GTestExpectFatalFailureHelper {\ public:\ static void Execute() { statement; }\ };\ ::testing::TestPartResultArray gtest_failures;\ ::testing::internal::SingleFailureChecker gtest_checker(\ >est_failures, ::testing::TestPartResult::kFatalFailure, (substr));\ {\ ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ ::testing::ScopedFakeTestPartResultReporter:: \ INTERCEPT_ONLY_CURRENT_THREAD, >est_failures);\ GTestExpectFatalFailureHelper::Execute();\ }\ } while (::testing::internal::AlwaysFalse()) #define EXPECT_FATAL_FAILURE_ON_ALL_THREADS(statement, substr) \ do { \ class GTestExpectFatalFailureHelper {\ public:\ static void Execute() { statement; }\ };\ ::testing::TestPartResultArray gtest_failures;\ ::testing::internal::SingleFailureChecker gtest_checker(\ >est_failures, ::testing::TestPartResult::kFatalFailure, (substr));\ {\ ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ ::testing::ScopedFakeTestPartResultReporter:: \ INTERCEPT_ALL_THREADS, >est_failures);\ GTestExpectFatalFailureHelper::Execute();\ }\ } while (::testing::internal::AlwaysFalse()) // A macro for testing Google Test assertions or code that's expected to // generate Google Test non-fatal failures. It asserts that the given // statement will cause exactly one non-fatal Google Test failure with 'substr' // being part of the failure message. // // There are two different versions of this macro. EXPECT_NONFATAL_FAILURE only // affects and considers failures generated in the current thread and // EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS does the same but for all threads. // // 'statement' is allowed to reference local variables and members of // the current object. // // The verification of the assertion is done correctly even when the statement // throws an exception or aborts the current function. // // Known restrictions: // - You cannot stream a failure message to this macro. // // Note that even though the implementations of the following two // macros are much alike, we cannot refactor them to use a common // helper macro, due to some peculiarity in how the preprocessor // works. If we do that, the code won't compile when the user gives // EXPECT_NONFATAL_FAILURE() a statement that contains a macro that // expands to code containing an unprotected comma. The // AcceptsMacroThatExpandsToUnprotectedComma test in gtest_unittest.cc // catches that. // // For the same reason, we have to write // if (::testing::internal::AlwaysTrue()) { statement; } // instead of // GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) // to avoid an MSVC warning on unreachable code. #define EXPECT_NONFATAL_FAILURE(statement, substr) \ do {\ ::testing::TestPartResultArray gtest_failures;\ ::testing::internal::SingleFailureChecker gtest_checker(\ >est_failures, ::testing::TestPartResult::kNonFatalFailure, \ (substr));\ {\ ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ ::testing::ScopedFakeTestPartResultReporter:: \ INTERCEPT_ONLY_CURRENT_THREAD, >est_failures);\ if (::testing::internal::AlwaysTrue()) { statement; }\ }\ } while (::testing::internal::AlwaysFalse()) #define EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(statement, substr) \ do {\ ::testing::TestPartResultArray gtest_failures;\ ::testing::internal::SingleFailureChecker gtest_checker(\ >est_failures, ::testing::TestPartResult::kNonFatalFailure, \ (substr));\ {\ ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ ::testing::ScopedFakeTestPartResultReporter::INTERCEPT_ALL_THREADS, \ >est_failures);\ if (::testing::internal::AlwaysTrue()) { statement; }\ }\ } while (::testing::internal::AlwaysFalse()) #endif // GTEST_INCLUDE_GTEST_GTEST_SPI_H_ #include #include #include #include #include #include #include #include #include #include #include #include #include #include // NOLINT #include #include #if GTEST_OS_LINUX # define GTEST_HAS_GETTIMEOFDAY_ 1 # include // NOLINT # include // NOLINT # include // NOLINT // Declares vsnprintf(). This header is not available on Windows. # include // NOLINT # include // NOLINT # include // NOLINT # include // NOLINT # include #elif GTEST_OS_ZOS # define GTEST_HAS_GETTIMEOFDAY_ 1 # include // NOLINT // On z/OS we additionally need strings.h for strcasecmp. # include // NOLINT #elif GTEST_OS_WINDOWS_MOBILE // We are on Windows CE. # include // NOLINT # undef min #elif GTEST_OS_WINDOWS // We are on Windows proper. # include // NOLINT # undef min # include // NOLINT # include // NOLINT # include // NOLINT # include // NOLINT # include // NOLINT # include // NOLINT # if GTEST_OS_WINDOWS_MINGW // MinGW has gettimeofday() but not _ftime64(). # define GTEST_HAS_GETTIMEOFDAY_ 1 # include // NOLINT # endif // GTEST_OS_WINDOWS_MINGW #else // Assume other platforms have gettimeofday(). # define GTEST_HAS_GETTIMEOFDAY_ 1 // cpplint thinks that the header is already included, so we want to // silence it. # include // NOLINT # include // NOLINT #endif // GTEST_OS_LINUX #if GTEST_HAS_EXCEPTIONS # include #endif #if GTEST_CAN_STREAM_RESULTS_ # include // NOLINT # include // NOLINT # include // NOLINT # include // NOLINT #endif // Copyright 2005, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Utility functions and classes used by the Google C++ testing framework.// // This file contains purely Google Test's internal implementation. Please // DO NOT #INCLUDE IT IN A USER PROGRAM. #ifndef GTEST_SRC_GTEST_INTERNAL_INL_H_ #define GTEST_SRC_GTEST_INTERNAL_INL_H_ #ifndef _WIN32_WCE # include #endif // !_WIN32_WCE #include #include // For strtoll/_strtoul64/malloc/free. #include // For memmove. #include #include #include #include #if GTEST_CAN_STREAM_RESULTS_ # include // NOLINT # include // NOLINT #endif #if GTEST_OS_WINDOWS # include // NOLINT #endif // GTEST_OS_WINDOWS GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) namespace testing { // Declares the flags. // // We don't want the users to modify this flag in the code, but want // Google Test's own unit tests to be able to access it. Therefore we // declare it here as opposed to in gtest.h. GTEST_DECLARE_bool_(death_test_use_fork); namespace internal { // The value of GetTestTypeId() as seen from within the Google Test // library. This is solely for testing GetTestTypeId(). GTEST_API_ extern const TypeId kTestTypeIdInGoogleTest; // Names of the flags (needed for parsing Google Test flags). const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests"; const char kBreakOnFailureFlag[] = "break_on_failure"; const char kCatchExceptionsFlag[] = "catch_exceptions"; const char kColorFlag[] = "color"; const char kFilterFlag[] = "filter"; const char kListTestsFlag[] = "list_tests"; const char kOutputFlag[] = "output"; const char kPrintTimeFlag[] = "print_time"; const char kPrintUTF8Flag[] = "print_utf8"; const char kRandomSeedFlag[] = "random_seed"; const char kRepeatFlag[] = "repeat"; const char kShuffleFlag[] = "shuffle"; const char kStackTraceDepthFlag[] = "stack_trace_depth"; const char kStreamResultToFlag[] = "stream_result_to"; const char kThrowOnFailureFlag[] = "throw_on_failure"; const char kFlagfileFlag[] = "flagfile"; const char kPrintSkippedFlag[] = "print_skipped"; // A valid random seed must be in [1, kMaxRandomSeed]. const int kMaxRandomSeed = 99999; // g_help_flag is true if and only if the --help flag or an equivalent form // is specified on the command line. GTEST_API_ extern bool g_help_flag; // Returns the current time in milliseconds. GTEST_API_ TimeInMillis GetTimeInMillis(); // Returns true if and only if Google Test should use colors in the output. GTEST_API_ bool ShouldUseColor(bool stdout_is_tty); // Formats the given time in milliseconds as seconds. GTEST_API_ std::string FormatTimeInMillisAsSeconds(TimeInMillis ms); // Converts the given time in milliseconds to a date string in the ISO 8601 // format, without the timezone information. N.B.: due to the use the // non-reentrant localtime() function, this function is not thread safe. Do // not use it in any code that can be called from multiple threads. GTEST_API_ std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms); // Parses a string for an Int32 flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. GTEST_API_ bool ParseInt32Flag( const char* str, const char* flag, Int32* value); // Returns a random seed in range [1, kMaxRandomSeed] based on the // given --gtest_random_seed flag value. inline int GetRandomSeedFromFlag(Int32 random_seed_flag) { const unsigned int raw_seed = (random_seed_flag == 0) ? static_cast(GetTimeInMillis()) : static_cast(random_seed_flag); // Normalizes the actual seed to range [1, kMaxRandomSeed] such that // it's easy to type. const int normalized_seed = static_cast((raw_seed - 1U) % static_cast(kMaxRandomSeed)) + 1; return normalized_seed; } // Returns the first valid random seed after 'seed'. The behavior is // undefined if 'seed' is invalid. The seed after kMaxRandomSeed is // considered to be 1. inline int GetNextRandomSeed(int seed) { GTEST_CHECK_(1 <= seed && seed <= kMaxRandomSeed) << "Invalid random seed " << seed << " - must be in [1, " << kMaxRandomSeed << "]."; const int next_seed = seed + 1; return (next_seed > kMaxRandomSeed) ? 1 : next_seed; } // This class saves the values of all Google Test flags in its c'tor, and // restores them in its d'tor. class GTestFlagSaver { public: // The c'tor. GTestFlagSaver() { also_run_disabled_tests_ = GTEST_FLAG(also_run_disabled_tests); break_on_failure_ = GTEST_FLAG(break_on_failure); catch_exceptions_ = GTEST_FLAG(catch_exceptions); color_ = GTEST_FLAG(color); death_test_style_ = GTEST_FLAG(death_test_style); death_test_use_fork_ = GTEST_FLAG(death_test_use_fork); filter_ = GTEST_FLAG(filter); internal_run_death_test_ = GTEST_FLAG(internal_run_death_test); list_tests_ = GTEST_FLAG(list_tests); output_ = GTEST_FLAG(output); print_time_ = GTEST_FLAG(print_time); print_utf8_ = GTEST_FLAG(print_utf8); random_seed_ = GTEST_FLAG(random_seed); repeat_ = GTEST_FLAG(repeat); shuffle_ = GTEST_FLAG(shuffle); stack_trace_depth_ = GTEST_FLAG(stack_trace_depth); stream_result_to_ = GTEST_FLAG(stream_result_to); throw_on_failure_ = GTEST_FLAG(throw_on_failure); print_skipped_ = GTEST_FLAG(print_skipped); } // The d'tor is not virtual. DO NOT INHERIT FROM THIS CLASS. ~GTestFlagSaver() { GTEST_FLAG(also_run_disabled_tests) = also_run_disabled_tests_; GTEST_FLAG(break_on_failure) = break_on_failure_; GTEST_FLAG(catch_exceptions) = catch_exceptions_; GTEST_FLAG(color) = color_; GTEST_FLAG(death_test_style) = death_test_style_; GTEST_FLAG(death_test_use_fork) = death_test_use_fork_; GTEST_FLAG(filter) = filter_; GTEST_FLAG(internal_run_death_test) = internal_run_death_test_; GTEST_FLAG(list_tests) = list_tests_; GTEST_FLAG(output) = output_; GTEST_FLAG(print_time) = print_time_; GTEST_FLAG(print_utf8) = print_utf8_; GTEST_FLAG(random_seed) = random_seed_; GTEST_FLAG(repeat) = repeat_; GTEST_FLAG(shuffle) = shuffle_; GTEST_FLAG(stack_trace_depth) = stack_trace_depth_; GTEST_FLAG(stream_result_to) = stream_result_to_; GTEST_FLAG(throw_on_failure) = throw_on_failure_; GTEST_FLAG(print_skipped) = print_skipped_; } private: // Fields for saving the original values of flags. bool also_run_disabled_tests_; bool break_on_failure_; bool catch_exceptions_; std::string color_; std::string death_test_style_; bool death_test_use_fork_; std::string filter_; std::string internal_run_death_test_; bool list_tests_; std::string output_; bool print_time_; bool print_utf8_; internal::Int32 random_seed_; internal::Int32 repeat_; bool shuffle_; internal::Int32 stack_trace_depth_; std::string stream_result_to_; bool throw_on_failure_; bool print_skipped_; } GTEST_ATTRIBUTE_UNUSED_; // Converts a Unicode code point to a narrow string in UTF-8 encoding. // code_point parameter is of type UInt32 because wchar_t may not be // wide enough to contain a code point. // If the code_point is not a valid Unicode code point // (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted // to "(Invalid Unicode 0xXXXXXXXX)". GTEST_API_ std::string CodePointToUtf8(UInt32 code_point); // Converts a wide string to a narrow string in UTF-8 encoding. // The wide string is assumed to have the following encoding: // UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin) // UTF-32 if sizeof(wchar_t) == 4 (on Linux) // Parameter str points to a null-terminated wide string. // Parameter num_chars may additionally limit the number // of wchar_t characters processed. -1 is used when the entire string // should be processed. // If the string contains code points that are not valid Unicode code points // (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output // as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding // and contains invalid UTF-16 surrogate pairs, values in those pairs // will be encoded as individual Unicode characters from Basic Normal Plane. GTEST_API_ std::string WideStringToUtf8(const wchar_t* str, int num_chars); // Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file // if the variable is present. If a file already exists at this location, this // function will write over it. If the variable is present, but the file cannot // be created, prints an error and exits. void WriteToShardStatusFileIfNeeded(); // Checks whether sharding is enabled by examining the relevant // environment variable values. If the variables are present, // but inconsistent (e.g., shard_index >= total_shards), prints // an error and exits. If in_subprocess_for_death_test, sharding is // disabled because it must only be applied to the original test // process. Otherwise, we could filter out death tests we intended to execute. GTEST_API_ bool ShouldShard(const char* total_shards_str, const char* shard_index_str, bool in_subprocess_for_death_test); // Parses the environment variable var as an Int32. If it is unset, // returns default_val. If it is not an Int32, prints an error and // and aborts. GTEST_API_ Int32 Int32FromEnvOrDie(const char* env_var, Int32 default_val); // Given the total number of shards, the shard index, and the test id, // returns true if and only if the test should be run on this shard. The test id // is some arbitrary but unique non-negative integer assigned to each test // method. Assumes that 0 <= shard_index < total_shards. GTEST_API_ bool ShouldRunTestOnShard( int total_shards, int shard_index, int test_id); // STL container utilities. // Returns the number of elements in the given container that satisfy // the given predicate. template inline int CountIf(const Container& c, Predicate predicate) { // Implemented as an explicit loop since std::count_if() in libCstd on // Solaris has a non-standard signature. int count = 0; for (typename Container::const_iterator it = c.begin(); it != c.end(); ++it) { if (predicate(*it)) ++count; } return count; } // Applies a function/functor to each element in the container. template void ForEach(const Container& c, Functor functor) { std::for_each(c.begin(), c.end(), functor); } // Returns the i-th element of the vector, or default_value if i is not // in range [0, v.size()). template inline E GetElementOr(const std::vector& v, int i, E default_value) { return (i < 0 || i >= static_cast(v.size())) ? default_value : v[static_cast(i)]; } // Performs an in-place shuffle of a range of the vector's elements. // 'begin' and 'end' are element indices as an STL-style range; // i.e. [begin, end) are shuffled, where 'end' == size() means to // shuffle to the end of the vector. template void ShuffleRange(internal::Random* random, int begin, int end, std::vector* v) { const int size = static_cast(v->size()); GTEST_CHECK_(0 <= begin && begin <= size) << "Invalid shuffle range start " << begin << ": must be in range [0, " << size << "]."; GTEST_CHECK_(begin <= end && end <= size) << "Invalid shuffle range finish " << end << ": must be in range [" << begin << ", " << size << "]."; // Fisher-Yates shuffle, from // http://en.wikipedia.org/wiki/Fisher-Yates_shuffle for (int range_width = end - begin; range_width >= 2; range_width--) { const int last_in_range = begin + range_width - 1; const int selected = begin + static_cast(random->Generate(static_cast(range_width))); std::swap((*v)[static_cast(selected)], (*v)[static_cast(last_in_range)]); } } // Performs an in-place shuffle of the vector's elements. template inline void Shuffle(internal::Random* random, std::vector* v) { ShuffleRange(random, 0, static_cast(v->size()), v); } // A function for deleting an object. Handy for being used as a // functor. template static void Delete(T* x) { delete x; } // A predicate that checks the key of a TestProperty against a known key. // // TestPropertyKeyIs is copyable. class TestPropertyKeyIs { public: // Constructor. // // TestPropertyKeyIs has NO default constructor. explicit TestPropertyKeyIs(const std::string& key) : key_(key) {} // Returns true if and only if the test name of test property matches on key_. bool operator()(const TestProperty& test_property) const { return test_property.key() == key_; } private: std::string key_; }; // Class UnitTestOptions. // // This class contains functions for processing options the user // specifies when running the tests. It has only static members. // // In most cases, the user can specify an option using either an // environment variable or a command line flag. E.g. you can set the // test filter using either GTEST_FILTER or --gtest_filter. If both // the variable and the flag are present, the latter overrides the // former. class GTEST_API_ UnitTestOptions { public: // Functions for processing the gtest_output flag. // Returns the output format, or "" for normal printed output. static std::string GetOutputFormat(); // Returns the absolute path of the requested output file, or the // default (test_detail.xml in the original working directory) if // none was explicitly specified. static std::string GetAbsolutePathToOutputFile(); // Functions for processing the gtest_filter flag. // Returns true if and only if the wildcard pattern matches the string. // The first ':' or '\0' character in pattern marks the end of it. // // This recursive algorithm isn't very efficient, but is clear and // works well enough for matching test names, which are short. static bool PatternMatchesString(const char *pattern, const char *str); // Returns true if and only if the user-specified filter matches the test // suite name and the test name. static bool FilterMatchesTest(const std::string& test_suite_name, const std::string& test_name); #if GTEST_OS_WINDOWS // Function for supporting the gtest_catch_exception flag. // Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the // given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise. // This function is useful as an __except condition. static int GTestShouldProcessSEH(DWORD exception_code); #endif // GTEST_OS_WINDOWS // Returns true if "name" matches the ':' separated list of glob-style // filters in "filter". static bool MatchesFilter(const std::string& name, const char* filter); }; // Returns the current application's name, removing directory path if that // is present. Used by UnitTestOptions::GetOutputFile. GTEST_API_ FilePath GetCurrentExecutableName(); // The role interface for getting the OS stack trace as a string. class OsStackTraceGetterInterface { public: OsStackTraceGetterInterface() {} virtual ~OsStackTraceGetterInterface() {} // Returns the current OS stack trace as an std::string. Parameters: // // max_depth - the maximum number of stack frames to be included // in the trace. // skip_count - the number of top frames to be skipped; doesn't count // against max_depth. virtual std::string CurrentStackTrace(int max_depth, int skip_count) = 0; // UponLeavingGTest() should be called immediately before Google Test calls // user code. It saves some information about the current stack that // CurrentStackTrace() will use to find and hide Google Test stack frames. virtual void UponLeavingGTest() = 0; // This string is inserted in place of stack frames that are part of // Google Test's implementation. static const char* const kElidedFramesMarker; private: GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetterInterface); }; // A working implementation of the OsStackTraceGetterInterface interface. class OsStackTraceGetter : public OsStackTraceGetterInterface { public: OsStackTraceGetter() {} std::string CurrentStackTrace(int max_depth, int skip_count) override; void UponLeavingGTest() override; private: #if GTEST_HAS_ABSL Mutex mutex_; // Protects all internal state. // We save the stack frame below the frame that calls user code. // We do this because the address of the frame immediately below // the user code changes between the call to UponLeavingGTest() // and any calls to the stack trace code from within the user code. void* caller_frame_ = nullptr; #endif // GTEST_HAS_ABSL GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetter); }; // Information about a Google Test trace point. struct TraceInfo { const char* file; int line; std::string message; }; // This is the default global test part result reporter used in UnitTestImpl. // This class should only be used by UnitTestImpl. class DefaultGlobalTestPartResultReporter : public TestPartResultReporterInterface { public: explicit DefaultGlobalTestPartResultReporter(UnitTestImpl* unit_test); // Implements the TestPartResultReporterInterface. Reports the test part // result in the current test. void ReportTestPartResult(const TestPartResult& result) override; private: UnitTestImpl* const unit_test_; GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultGlobalTestPartResultReporter); }; // This is the default per thread test part result reporter used in // UnitTestImpl. This class should only be used by UnitTestImpl. class DefaultPerThreadTestPartResultReporter : public TestPartResultReporterInterface { public: explicit DefaultPerThreadTestPartResultReporter(UnitTestImpl* unit_test); // Implements the TestPartResultReporterInterface. The implementation just // delegates to the current global test part result reporter of *unit_test_. void ReportTestPartResult(const TestPartResult& result) override; private: UnitTestImpl* const unit_test_; GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultPerThreadTestPartResultReporter); }; // The private implementation of the UnitTest class. We don't protect // the methods under a mutex, as this class is not accessible by a // user and the UnitTest class that delegates work to this class does // proper locking. class GTEST_API_ UnitTestImpl { public: explicit UnitTestImpl(UnitTest* parent); virtual ~UnitTestImpl(); // There are two different ways to register your own TestPartResultReporter. // You can register your own repoter to listen either only for test results // from the current thread or for results from all threads. // By default, each per-thread test result repoter just passes a new // TestPartResult to the global test result reporter, which registers the // test part result for the currently running test. // Returns the global test part result reporter. TestPartResultReporterInterface* GetGlobalTestPartResultReporter(); // Sets the global test part result reporter. void SetGlobalTestPartResultReporter( TestPartResultReporterInterface* reporter); // Returns the test part result reporter for the current thread. TestPartResultReporterInterface* GetTestPartResultReporterForCurrentThread(); // Sets the test part result reporter for the current thread. void SetTestPartResultReporterForCurrentThread( TestPartResultReporterInterface* reporter); // Gets the number of successful test suites. int successful_test_suite_count() const; // Gets the number of failed test suites. int failed_test_suite_count() const; // Gets the number of all test suites. int total_test_suite_count() const; // Gets the number of all test suites that contain at least one test // that should run. int test_suite_to_run_count() const; // Gets the number of successful tests. int successful_test_count() const; // Gets the number of skipped tests. int skipped_test_count() const; // Gets the number of failed tests. int failed_test_count() const; // Gets the number of disabled tests that will be reported in the XML report. int reportable_disabled_test_count() const; // Gets the number of disabled tests. int disabled_test_count() const; // Gets the number of tests to be printed in the XML report. int reportable_test_count() const; // Gets the number of all tests. int total_test_count() const; // Gets the number of tests that should run. int test_to_run_count() const; // Gets the time of the test program start, in ms from the start of the // UNIX epoch. TimeInMillis start_timestamp() const { return start_timestamp_; } // Gets the elapsed time, in milliseconds. TimeInMillis elapsed_time() const { return elapsed_time_; } // Returns true if and only if the unit test passed (i.e. all test suites // passed). bool Passed() const { return !Failed(); } // Returns true if and only if the unit test failed (i.e. some test suite // failed or something outside of all tests failed). bool Failed() const { return failed_test_suite_count() > 0 || ad_hoc_test_result()->Failed(); } // Gets the i-th test suite among all the test suites. i can range from 0 to // total_test_suite_count() - 1. If i is not in that range, returns NULL. const TestSuite* GetTestSuite(int i) const { const int index = GetElementOr(test_suite_indices_, i, -1); return index < 0 ? nullptr : test_suites_[static_cast(i)]; } // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ const TestCase* GetTestCase(int i) const { return GetTestSuite(i); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // Gets the i-th test suite among all the test suites. i can range from 0 to // total_test_suite_count() - 1. If i is not in that range, returns NULL. TestSuite* GetMutableSuiteCase(int i) { const int index = GetElementOr(test_suite_indices_, i, -1); return index < 0 ? nullptr : test_suites_[static_cast(index)]; } // Provides access to the event listener list. TestEventListeners* listeners() { return &listeners_; } // Returns the TestResult for the test that's currently running, or // the TestResult for the ad hoc test if no test is running. TestResult* current_test_result(); // Returns the TestResult for the ad hoc test. const TestResult* ad_hoc_test_result() const { return &ad_hoc_test_result_; } // Sets the OS stack trace getter. // // Does nothing if the input and the current OS stack trace getter // are the same; otherwise, deletes the old getter and makes the // input the current getter. void set_os_stack_trace_getter(OsStackTraceGetterInterface* getter); // Returns the current OS stack trace getter if it is not NULL; // otherwise, creates an OsStackTraceGetter, makes it the current // getter, and returns it. OsStackTraceGetterInterface* os_stack_trace_getter(); // Returns the current OS stack trace as an std::string. // // The maximum number of stack frames to be included is specified by // the gtest_stack_trace_depth flag. The skip_count parameter // specifies the number of top frames to be skipped, which doesn't // count against the number of frames to be included. // // For example, if Foo() calls Bar(), which in turn calls // CurrentOsStackTraceExceptTop(1), Foo() will be included in the // trace but Bar() and CurrentOsStackTraceExceptTop() won't. std::string CurrentOsStackTraceExceptTop(int skip_count) GTEST_NO_INLINE_; // Finds and returns a TestSuite with the given name. If one doesn't // exist, creates one and returns it. // // Arguments: // // test_suite_name: name of the test suite // type_param: the name of the test's type parameter, or NULL if // this is not a typed or a type-parameterized test. // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite TestSuite* GetTestSuite(const char* test_suite_name, const char* type_param, internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc); // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ TestCase* GetTestCase(const char* test_case_name, const char* type_param, internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc) { return GetTestSuite(test_case_name, type_param, set_up_tc, tear_down_tc); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // Adds a TestInfo to the unit test. // // Arguments: // // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite // test_info: the TestInfo object void AddTestInfo(internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc, TestInfo* test_info) { // In order to support thread-safe death tests, we need to // remember the original working directory when the test program // was first invoked. We cannot do this in RUN_ALL_TESTS(), as // the user may have changed the current directory before calling // RUN_ALL_TESTS(). Therefore we capture the current directory in // AddTestInfo(), which is called to register a TEST or TEST_F // before main() is reached. if (original_working_dir_.IsEmpty()) { original_working_dir_.Set(FilePath::GetCurrentDir()); GTEST_CHECK_(!original_working_dir_.IsEmpty()) << "Failed to get the current working directory."; } GetTestSuite(test_info->test_suite_name(), test_info->type_param(), set_up_tc, tear_down_tc) ->AddTestInfo(test_info); } // Returns ParameterizedTestSuiteRegistry object used to keep track of // value-parameterized tests and instantiate and register them. internal::ParameterizedTestSuiteRegistry& parameterized_test_registry() { return parameterized_test_registry_; } // Sets the TestSuite object for the test that's currently running. void set_current_test_suite(TestSuite* a_current_test_suite) { current_test_suite_ = a_current_test_suite; } // Sets the TestInfo object for the test that's currently running. If // current_test_info is NULL, the assertion results will be stored in // ad_hoc_test_result_. void set_current_test_info(TestInfo* a_current_test_info) { current_test_info_ = a_current_test_info; } // Registers all parameterized tests defined using TEST_P and // INSTANTIATE_TEST_SUITE_P, creating regular tests for each test/parameter // combination. This method can be called more then once; it has guards // protecting from registering the tests more then once. If // value-parameterized tests are disabled, RegisterParameterizedTests is // present but does nothing. void RegisterParameterizedTests(); // Runs all tests in this UnitTest object, prints the result, and // returns true if all tests are successful. If any exception is // thrown during a test, this test is considered to be failed, but // the rest of the tests will still be run. bool RunAllTests(); // Clears the results of all tests, except the ad hoc tests. void ClearNonAdHocTestResult() { ForEach(test_suites_, TestSuite::ClearTestSuiteResult); } // Clears the results of ad-hoc test assertions. void ClearAdHocTestResult() { ad_hoc_test_result_.Clear(); } // Adds a TestProperty to the current TestResult object when invoked in a // context of a test or a test suite, or to the global property set. If the // result already contains a property with the same key, the value will be // updated. void RecordProperty(const TestProperty& test_property); enum ReactionToSharding { HONOR_SHARDING_PROTOCOL, IGNORE_SHARDING_PROTOCOL }; // Matches the full name of each test against the user-specified // filter to decide whether the test should run, then records the // result in each TestSuite and TestInfo object. // If shard_tests == HONOR_SHARDING_PROTOCOL, further filters tests // based on sharding variables in the environment. // Returns the number of tests that should run. int FilterTests(ReactionToSharding shard_tests); // Prints the names of the tests matching the user-specified filter flag. void ListTestsMatchingFilter(); const TestSuite* current_test_suite() const { return current_test_suite_; } TestInfo* current_test_info() { return current_test_info_; } const TestInfo* current_test_info() const { return current_test_info_; } // Returns the vector of environments that need to be set-up/torn-down // before/after the tests are run. std::vector& environments() { return environments_; } // Getters for the per-thread Google Test trace stack. std::vector& gtest_trace_stack() { return *(gtest_trace_stack_.pointer()); } const std::vector& gtest_trace_stack() const { return gtest_trace_stack_.get(); } #if GTEST_HAS_DEATH_TEST void InitDeathTestSubprocessControlInfo() { internal_run_death_test_flag_.reset(ParseInternalRunDeathTestFlag()); } // Returns a pointer to the parsed --gtest_internal_run_death_test // flag, or NULL if that flag was not specified. // This information is useful only in a death test child process. // Must not be called before a call to InitGoogleTest. const InternalRunDeathTestFlag* internal_run_death_test_flag() const { return internal_run_death_test_flag_.get(); } // Returns a pointer to the current death test factory. internal::DeathTestFactory* death_test_factory() { return death_test_factory_.get(); } void SuppressTestEventsIfInSubprocess(); friend class ReplaceDeathTestFactory; #endif // GTEST_HAS_DEATH_TEST // Initializes the event listener performing XML output as specified by // UnitTestOptions. Must not be called before InitGoogleTest. void ConfigureXmlOutput(); #if GTEST_CAN_STREAM_RESULTS_ // Initializes the event listener for streaming test results to a socket. // Must not be called before InitGoogleTest. void ConfigureStreamingOutput(); #endif // Performs initialization dependent upon flag values obtained in // ParseGoogleTestFlagsOnly. Is called from InitGoogleTest after the call to // ParseGoogleTestFlagsOnly. In case a user neglects to call InitGoogleTest // this function is also called from RunAllTests. Since this function can be // called more than once, it has to be idempotent. void PostFlagParsingInit(); // Gets the random seed used at the start of the current test iteration. int random_seed() const { return random_seed_; } // Gets the random number generator. internal::Random* random() { return &random_; } // Shuffles all test suites, and the tests within each test suite, // making sure that death tests are still run first. void ShuffleTests(); // Restores the test suites and tests to their order before the first shuffle. void UnshuffleTests(); // Returns the value of GTEST_FLAG(catch_exceptions) at the moment // UnitTest::Run() starts. bool catch_exceptions() const { return catch_exceptions_; } private: friend class ::testing::UnitTest; // Used by UnitTest::Run() to capture the state of // GTEST_FLAG(catch_exceptions) at the moment it starts. void set_catch_exceptions(bool value) { catch_exceptions_ = value; } // The UnitTest object that owns this implementation object. UnitTest* const parent_; // The working directory when the first TEST() or TEST_F() was // executed. internal::FilePath original_working_dir_; // The default test part result reporters. DefaultGlobalTestPartResultReporter default_global_test_part_result_reporter_; DefaultPerThreadTestPartResultReporter default_per_thread_test_part_result_reporter_; // Points to (but doesn't own) the global test part result reporter. TestPartResultReporterInterface* global_test_part_result_repoter_; // Protects read and write access to global_test_part_result_reporter_. internal::Mutex global_test_part_result_reporter_mutex_; // Points to (but doesn't own) the per-thread test part result reporter. internal::ThreadLocal per_thread_test_part_result_reporter_; // The vector of environments that need to be set-up/torn-down // before/after the tests are run. std::vector environments_; // The vector of TestSuites in their original order. It owns the // elements in the vector. std::vector test_suites_; // Provides a level of indirection for the test suite list to allow // easy shuffling and restoring the test suite order. The i-th // element of this vector is the index of the i-th test suite in the // shuffled order. std::vector test_suite_indices_; // ParameterizedTestRegistry object used to register value-parameterized // tests. internal::ParameterizedTestSuiteRegistry parameterized_test_registry_; // Indicates whether RegisterParameterizedTests() has been called already. bool parameterized_tests_registered_; // Index of the last death test suite registered. Initially -1. int last_death_test_suite_; // This points to the TestSuite for the currently running test. It // changes as Google Test goes through one test suite after another. // When no test is running, this is set to NULL and Google Test // stores assertion results in ad_hoc_test_result_. Initially NULL. TestSuite* current_test_suite_; // This points to the TestInfo for the currently running test. It // changes as Google Test goes through one test after another. When // no test is running, this is set to NULL and Google Test stores // assertion results in ad_hoc_test_result_. Initially NULL. TestInfo* current_test_info_; // Normally, a user only writes assertions inside a TEST or TEST_F, // or inside a function called by a TEST or TEST_F. Since Google // Test keeps track of which test is current running, it can // associate such an assertion with the test it belongs to. // // If an assertion is encountered when no TEST or TEST_F is running, // Google Test attributes the assertion result to an imaginary "ad hoc" // test, and records the result in ad_hoc_test_result_. TestResult ad_hoc_test_result_; // The list of event listeners that can be used to track events inside // Google Test. TestEventListeners listeners_; // The OS stack trace getter. Will be deleted when the UnitTest // object is destructed. By default, an OsStackTraceGetter is used, // but the user can set this field to use a custom getter if that is // desired. OsStackTraceGetterInterface* os_stack_trace_getter_; // True if and only if PostFlagParsingInit() has been called. bool post_flag_parse_init_performed_; // The random number seed used at the beginning of the test run. int random_seed_; // Our random number generator. internal::Random random_; // The time of the test program start, in ms from the start of the // UNIX epoch. TimeInMillis start_timestamp_; // How long the test took to run, in milliseconds. TimeInMillis elapsed_time_; #if GTEST_HAS_DEATH_TEST // The decomposed components of the gtest_internal_run_death_test flag, // parsed when RUN_ALL_TESTS is called. std::unique_ptr internal_run_death_test_flag_; std::unique_ptr death_test_factory_; #endif // GTEST_HAS_DEATH_TEST // A per-thread stack of traces created by the SCOPED_TRACE() macro. internal::ThreadLocal > gtest_trace_stack_; // The value of GTEST_FLAG(catch_exceptions) at the moment RunAllTests() // starts. bool catch_exceptions_; GTEST_DISALLOW_COPY_AND_ASSIGN_(UnitTestImpl); }; // class UnitTestImpl // Convenience function for accessing the global UnitTest // implementation object. inline UnitTestImpl* GetUnitTestImpl() { return UnitTest::GetInstance()->impl(); } #if GTEST_USES_SIMPLE_RE // Internal helper functions for implementing the simple regular // expression matcher. GTEST_API_ bool IsInSet(char ch, const char* str); GTEST_API_ bool IsAsciiDigit(char ch); GTEST_API_ bool IsAsciiPunct(char ch); GTEST_API_ bool IsRepeat(char ch); GTEST_API_ bool IsAsciiWhiteSpace(char ch); GTEST_API_ bool IsAsciiWordChar(char ch); GTEST_API_ bool IsValidEscape(char ch); GTEST_API_ bool AtomMatchesChar(bool escaped, char pattern, char ch); GTEST_API_ bool ValidateRegex(const char* regex); GTEST_API_ bool MatchRegexAtHead(const char* regex, const char* str); GTEST_API_ bool MatchRepetitionAndRegexAtHead( bool escaped, char ch, char repeat, const char* regex, const char* str); GTEST_API_ bool MatchRegexAnywhere(const char* regex, const char* str); #endif // GTEST_USES_SIMPLE_RE // Parses the command line for Google Test flags, without initializing // other parts of Google Test. GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, char** argv); GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv); #if GTEST_HAS_DEATH_TEST // Returns the message describing the last system error, regardless of the // platform. GTEST_API_ std::string GetLastErrnoDescription(); // Attempts to parse a string into a positive integer pointed to by the // number parameter. Returns true if that is possible. // GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we can use // it here. template bool ParseNaturalNumber(const ::std::string& str, Integer* number) { // Fail fast if the given string does not begin with a digit; // this bypasses strtoXXX's "optional leading whitespace and plus // or minus sign" semantics, which are undesirable here. if (str.empty() || !IsDigit(str[0])) { return false; } errno = 0; char* end; // BiggestConvertible is the largest integer type that system-provided // string-to-number conversion routines can return. # if GTEST_OS_WINDOWS && !defined(__GNUC__) // MSVC and C++ Builder define __int64 instead of the standard long long. typedef unsigned __int64 BiggestConvertible; const BiggestConvertible parsed = _strtoui64(str.c_str(), &end, 10); # else typedef unsigned long long BiggestConvertible; // NOLINT const BiggestConvertible parsed = strtoull(str.c_str(), &end, 10); # endif // GTEST_OS_WINDOWS && !defined(__GNUC__) const bool parse_success = *end == '\0' && errno == 0; GTEST_CHECK_(sizeof(Integer) <= sizeof(parsed)); const Integer result = static_cast(parsed); if (parse_success && static_cast(result) == parsed) { *number = result; return true; } return false; } #endif // GTEST_HAS_DEATH_TEST // TestResult contains some private methods that should be hidden from // Google Test user but are required for testing. This class allow our tests // to access them. // // This class is supplied only for the purpose of testing Google Test's own // constructs. Do not use it in user tests, either directly or indirectly. class TestResultAccessor { public: static void RecordProperty(TestResult* test_result, const std::string& xml_element, const TestProperty& property) { test_result->RecordProperty(xml_element, property); } static void ClearTestPartResults(TestResult* test_result) { test_result->ClearTestPartResults(); } static const std::vector& test_part_results( const TestResult& test_result) { return test_result.test_part_results(); } }; #if GTEST_CAN_STREAM_RESULTS_ // Streams test results to the given port on the given host machine. class StreamingListener : public EmptyTestEventListener { public: // Abstract base class for writing strings to a socket. class AbstractSocketWriter { public: virtual ~AbstractSocketWriter() {} // Sends a string to the socket. virtual void Send(const std::string& message) = 0; // Closes the socket. virtual void CloseConnection() {} // Sends a string and a newline to the socket. void SendLn(const std::string& message) { Send(message + "\n"); } }; // Concrete class for actually writing strings to a socket. class SocketWriter : public AbstractSocketWriter { public: SocketWriter(const std::string& host, const std::string& port) : sockfd_(-1), host_name_(host), port_num_(port) { MakeConnection(); } ~SocketWriter() override { if (sockfd_ != -1) CloseConnection(); } // Sends a string to the socket. void Send(const std::string& message) override { GTEST_CHECK_(sockfd_ != -1) << "Send() can be called only when there is a connection."; const auto len = static_cast(message.length()); if (write(sockfd_, message.c_str(), len) != static_cast(len)) { GTEST_LOG_(WARNING) << "stream_result_to: failed to stream to " << host_name_ << ":" << port_num_; } } private: // Creates a client socket and connects to the server. void MakeConnection(); // Closes the socket. void CloseConnection() override { GTEST_CHECK_(sockfd_ != -1) << "CloseConnection() can be called only when there is a connection."; close(sockfd_); sockfd_ = -1; } int sockfd_; // socket file descriptor const std::string host_name_; const std::string port_num_; GTEST_DISALLOW_COPY_AND_ASSIGN_(SocketWriter); }; // class SocketWriter // Escapes '=', '&', '%', and '\n' characters in str as "%xx". static std::string UrlEncode(const char* str); StreamingListener(const std::string& host, const std::string& port) : socket_writer_(new SocketWriter(host, port)) { Start(); } explicit StreamingListener(AbstractSocketWriter* socket_writer) : socket_writer_(socket_writer) { Start(); } void OnTestProgramStart(const UnitTest& /* unit_test */) override { SendLn("event=TestProgramStart"); } void OnTestProgramEnd(const UnitTest& unit_test) override { // Note that Google Test current only report elapsed time for each // test iteration, not for the entire test program. SendLn("event=TestProgramEnd&passed=" + FormatBool(unit_test.Passed())); // Notify the streaming server to stop. socket_writer_->CloseConnection(); } void OnTestIterationStart(const UnitTest& /* unit_test */, int iteration) override { SendLn("event=TestIterationStart&iteration=" + StreamableToString(iteration)); } void OnTestIterationEnd(const UnitTest& unit_test, int /* iteration */) override { SendLn("event=TestIterationEnd&passed=" + FormatBool(unit_test.Passed()) + "&elapsed_time=" + StreamableToString(unit_test.elapsed_time()) + "ms"); } // Note that "event=TestCaseStart" is a wire format and has to remain // "case" for compatibilty void OnTestCaseStart(const TestCase& test_case) override { SendLn(std::string("event=TestCaseStart&name=") + test_case.name()); } // Note that "event=TestCaseEnd" is a wire format and has to remain // "case" for compatibilty void OnTestCaseEnd(const TestCase& test_case) override { SendLn("event=TestCaseEnd&passed=" + FormatBool(test_case.Passed()) + "&elapsed_time=" + StreamableToString(test_case.elapsed_time()) + "ms"); } void OnTestStart(const TestInfo& test_info) override { SendLn(std::string("event=TestStart&name=") + test_info.name()); } void OnTestEnd(const TestInfo& test_info) override { SendLn("event=TestEnd&passed=" + FormatBool((test_info.result())->Passed()) + "&elapsed_time=" + StreamableToString((test_info.result())->elapsed_time()) + "ms"); } void OnTestPartResult(const TestPartResult& test_part_result) override { const char* file_name = test_part_result.file_name(); if (file_name == nullptr) file_name = ""; SendLn("event=TestPartResult&file=" + UrlEncode(file_name) + "&line=" + StreamableToString(test_part_result.line_number()) + "&message=" + UrlEncode(test_part_result.message())); } private: // Sends the given message and a newline to the socket. void SendLn(const std::string& message) { socket_writer_->SendLn(message); } // Called at the start of streaming to notify the receiver what // protocol we are using. void Start() { SendLn("gtest_streaming_protocol_version=1.0"); } std::string FormatBool(bool value) { return value ? "1" : "0"; } const std::unique_ptr socket_writer_; GTEST_DISALLOW_COPY_AND_ASSIGN_(StreamingListener); }; // class StreamingListener #endif // GTEST_CAN_STREAM_RESULTS_ } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 #endif // GTEST_SRC_GTEST_INTERNAL_INL_H_ #if GTEST_OS_WINDOWS # define vsnprintf _vsnprintf #endif // GTEST_OS_WINDOWS #if GTEST_OS_MAC #ifndef GTEST_OS_IOS #include #endif #endif #if GTEST_HAS_ABSL #include "absl/debugging/failure_signal_handler.h" #include "absl/debugging/stacktrace.h" #include "absl/debugging/symbolize.h" #include "absl/strings/str_cat.h" #endif // GTEST_HAS_ABSL namespace testing { using internal::CountIf; using internal::ForEach; using internal::GetElementOr; using internal::Shuffle; // Constants. // A test whose test suite name or test name matches this filter is // disabled and not run. static const char kDisableTestFilter[] = "DISABLED_*:*/DISABLED_*"; // A test suite whose name matches this filter is considered a death // test suite and will be run before test suites whose name doesn't // match this filter. static const char kDeathTestSuiteFilter[] = "*DeathTest:*DeathTest/*"; // A test filter that matches everything. static const char kUniversalFilter[] = "*"; // The default output format. static const char kDefaultOutputFormat[] = "xml"; // The default output file. static const char kDefaultOutputFile[] = "test_detail"; // The environment variable name for the test shard index. static const char kTestShardIndex[] = "GTEST_SHARD_INDEX"; // The environment variable name for the total number of test shards. static const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS"; // The environment variable name for the test shard status file. static const char kTestShardStatusFile[] = "GTEST_SHARD_STATUS_FILE"; namespace internal { // The text used in failure messages to indicate the start of the // stack trace. const char kStackTraceMarker[] = "\nStack trace:\n"; // g_help_flag is true if and only if the --help flag or an equivalent form // is specified on the command line. bool g_help_flag = false; // Utilty function to Open File for Writing static FILE* OpenFileForWriting(const std::string& output_file) { FILE* fileout = nullptr; FilePath output_file_path(output_file); FilePath output_dir(output_file_path.RemoveFileName()); if (output_dir.CreateDirectoriesRecursively()) { fileout = posix::FOpen(output_file.c_str(), "w"); } if (fileout == nullptr) { GTEST_LOG_(FATAL) << "Unable to open file \"" << output_file << "\""; } return fileout; } } // namespace internal // Bazel passes in the argument to '--test_filter' via the TESTBRIDGE_TEST_ONLY // environment variable. static const char* GetDefaultFilter() { const char* const testbridge_test_only = internal::posix::GetEnv("TESTBRIDGE_TEST_ONLY"); if (testbridge_test_only != nullptr) { return testbridge_test_only; } return kUniversalFilter; } GTEST_DEFINE_bool_( also_run_disabled_tests, internal::BoolFromGTestEnv("also_run_disabled_tests", false), "Run disabled tests too, in addition to the tests normally being run."); GTEST_DEFINE_bool_( break_on_failure, internal::BoolFromGTestEnv("break_on_failure", false), "True if and only if a failed assertion should be a debugger " "break-point."); GTEST_DEFINE_bool_(catch_exceptions, internal::BoolFromGTestEnv("catch_exceptions", true), "True if and only if " GTEST_NAME_ " should catch exceptions and treat them as test failures."); GTEST_DEFINE_string_( color, internal::StringFromGTestEnv("color", "auto"), "Whether to use colors in the output. Valid values: yes, no, " "and auto. 'auto' means to use colors if the output is " "being sent to a terminal and the TERM environment variable " "is set to a terminal type that supports colors."); GTEST_DEFINE_string_( filter, internal::StringFromGTestEnv("filter", GetDefaultFilter()), "A colon-separated list of glob (not regex) patterns " "for filtering the tests to run, optionally followed by a " "'-' and a : separated list of negative patterns (tests to " "exclude). A test is run if it matches one of the positive " "patterns and does not match any of the negative patterns."); GTEST_DEFINE_bool_( install_failure_signal_handler, internal::BoolFromGTestEnv("install_failure_signal_handler", false), "If true and supported on the current platform, " GTEST_NAME_ " should " "install a signal handler that dumps debugging information when fatal " "signals are raised."); GTEST_DEFINE_bool_(list_tests, false, "List all tests without running them."); // The net priority order after flag processing is thus: // --gtest_output command line flag // GTEST_OUTPUT environment variable // XML_OUTPUT_FILE environment variable // '' GTEST_DEFINE_string_( output, internal::StringFromGTestEnv("output", internal::OutputFlagAlsoCheckEnvVar().c_str()), "A format (defaults to \"xml\" but can be specified to be \"json\"), " "optionally followed by a colon and an output file name or directory. " "A directory is indicated by a trailing pathname separator. " "Examples: \"xml:filename.xml\", \"xml::directoryname/\". " "If a directory is specified, output files will be created " "within that directory, with file-names based on the test " "executable's name and, if necessary, made unique by adding " "digits."); GTEST_DEFINE_bool_(print_time, internal::BoolFromGTestEnv("print_time", true), "True if and only if " GTEST_NAME_ " should display elapsed time in text output."); GTEST_DEFINE_bool_(print_utf8, internal::BoolFromGTestEnv("print_utf8", true), "True if and only if " GTEST_NAME_ " prints UTF8 characters as text."); GTEST_DEFINE_int32_( random_seed, internal::Int32FromGTestEnv("random_seed", 0), "Random number seed to use when shuffling test orders. Must be in range " "[1, 99999], or 0 to use a seed based on the current time."); GTEST_DEFINE_int32_( repeat, internal::Int32FromGTestEnv("repeat", 1), "How many times to repeat each test. Specify a negative number " "for repeating forever. Useful for shaking out flaky tests."); GTEST_DEFINE_bool_(show_internal_stack_frames, false, "True if and only if " GTEST_NAME_ " should include internal stack frames when " "printing test failure stack traces."); GTEST_DEFINE_bool_(shuffle, internal::BoolFromGTestEnv("shuffle", false), "True if and only if " GTEST_NAME_ " should randomize tests' order on every run."); GTEST_DEFINE_int32_( stack_trace_depth, internal::Int32FromGTestEnv("stack_trace_depth", kMaxStackTraceDepth), "The maximum number of stack frames to print when an " "assertion fails. The valid range is 0 through 100, inclusive."); GTEST_DEFINE_string_( stream_result_to, internal::StringFromGTestEnv("stream_result_to", ""), "This flag specifies the host name and the port number on which to stream " "test results. Example: \"localhost:555\". The flag is effective only on " "Linux."); GTEST_DEFINE_bool_( throw_on_failure, internal::BoolFromGTestEnv("throw_on_failure", false), "When this flag is specified, a failed assertion will throw an exception " "if exceptions are enabled or exit the program with a non-zero code " "otherwise. For use with an external test framework."); GTEST_DEFINE_bool_( print_skipped, internal::BoolFromGTestEnv("print_skipped", false), "When this flag is specified, list of skipped test names is printed in " "summary"); #if GTEST_USE_OWN_FLAGFILE_FLAG_ GTEST_DEFINE_string_( flagfile, internal::StringFromGTestEnv("flagfile", ""), "This flag specifies the flagfile to read command-line flags from."); #endif // GTEST_USE_OWN_FLAGFILE_FLAG_ namespace internal { // Generates a random number from [0, range), using a Linear // Congruential Generator (LCG). Crashes if 'range' is 0 or greater // than kMaxRange. UInt32 Random::Generate(UInt32 range) { // These constants are the same as are used in glibc's rand(3). // Use wider types than necessary to prevent unsigned overflow diagnostics. state_ = static_cast(1103515245ULL*state_ + 12345U) % kMaxRange; GTEST_CHECK_(range > 0) << "Cannot generate a number in the range [0, 0)."; GTEST_CHECK_(range <= kMaxRange) << "Generation of a number in [0, " << range << ") was requested, " << "but this can only generate numbers in [0, " << kMaxRange << ")."; // Converting via modulus introduces a bit of downward bias, but // it's simple, and a linear congruential generator isn't too good // to begin with. return state_ % range; } // GTestIsInitialized() returns true if and only if the user has initialized // Google Test. Useful for catching the user mistake of not initializing // Google Test before calling RUN_ALL_TESTS(). static bool GTestIsInitialized() { return GetArgvs().size() > 0; } // Iterates over a vector of TestSuites, keeping a running sum of the // results of calling a given int-returning method on each. // Returns the sum. static int SumOverTestSuiteList(const std::vector& case_list, int (TestSuite::*method)() const) { int sum = 0; for (size_t i = 0; i < case_list.size(); i++) { sum += (case_list[i]->*method)(); } return sum; } // Returns true if and only if the test suite passed. static bool TestSuitePassed(const TestSuite* test_suite) { return test_suite->should_run() && test_suite->Passed(); } // Returns true if and only if the test suite failed. static bool TestSuiteFailed(const TestSuite* test_suite) { return test_suite->should_run() && test_suite->Failed(); } // Returns true if and only if test_suite contains at least one test that // should run. static bool ShouldRunTestSuite(const TestSuite* test_suite) { return test_suite->should_run(); } // AssertHelper constructor. AssertHelper::AssertHelper(TestPartResult::Type type, const char* file, int line, const char* message) : data_(new AssertHelperData(type, file, line, message)) { } AssertHelper::~AssertHelper() { delete data_; } // Message assignment, for assertion streaming support. void AssertHelper::operator=(const Message& message) const { UnitTest::GetInstance()-> AddTestPartResult(data_->type, data_->file, data_->line, AppendUserMessage(data_->message, message), UnitTest::GetInstance()->impl() ->CurrentOsStackTraceExceptTop(1) // Skips the stack frame for this function itself. ); // NOLINT } // A copy of all command line arguments. Set by InitGoogleTest(). static ::std::vector g_argvs; ::std::vector GetArgvs() { #if defined(GTEST_CUSTOM_GET_ARGVS_) // GTEST_CUSTOM_GET_ARGVS_() may return a container of std::string or // ::string. This code converts it to the appropriate type. const auto& custom = GTEST_CUSTOM_GET_ARGVS_(); return ::std::vector(custom.begin(), custom.end()); #else // defined(GTEST_CUSTOM_GET_ARGVS_) return g_argvs; #endif // defined(GTEST_CUSTOM_GET_ARGVS_) } // Returns the current application's name, removing directory path if that // is present. FilePath GetCurrentExecutableName() { FilePath result; #if GTEST_OS_WINDOWS || GTEST_OS_OS2 result.Set(FilePath(GetArgvs()[0]).RemoveExtension("exe")); #else result.Set(FilePath(GetArgvs()[0])); #endif // GTEST_OS_WINDOWS return result.RemoveDirectoryName(); } // Functions for processing the gtest_output flag. // Returns the output format, or "" for normal printed output. std::string UnitTestOptions::GetOutputFormat() { const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); const char* const colon = strchr(gtest_output_flag, ':'); return (colon == nullptr) ? std::string(gtest_output_flag) : std::string(gtest_output_flag, static_cast(colon - gtest_output_flag)); } // Returns the name of the requested output file, or the default if none // was explicitly specified. std::string UnitTestOptions::GetAbsolutePathToOutputFile() { const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); std::string format = GetOutputFormat(); if (format.empty()) format = std::string(kDefaultOutputFormat); const char* const colon = strchr(gtest_output_flag, ':'); if (colon == nullptr) return internal::FilePath::MakeFileName( internal::FilePath( UnitTest::GetInstance()->original_working_dir()), internal::FilePath(kDefaultOutputFile), 0, format.c_str()).string(); internal::FilePath output_name(colon + 1); if (!output_name.IsAbsolutePath()) output_name = internal::FilePath::ConcatPaths( internal::FilePath(UnitTest::GetInstance()->original_working_dir()), internal::FilePath(colon + 1)); if (!output_name.IsDirectory()) return output_name.string(); internal::FilePath result(internal::FilePath::GenerateUniqueFileName( output_name, internal::GetCurrentExecutableName(), GetOutputFormat().c_str())); return result.string(); } // Returns true if and only if the wildcard pattern matches the string. // The first ':' or '\0' character in pattern marks the end of it. // // This recursive algorithm isn't very efficient, but is clear and // works well enough for matching test names, which are short. bool UnitTestOptions::PatternMatchesString(const char *pattern, const char *str) { switch (*pattern) { case '\0': case ':': // Either ':' or '\0' marks the end of the pattern. return *str == '\0'; case '?': // Matches any single character. return *str != '\0' && PatternMatchesString(pattern + 1, str + 1); case '*': // Matches any string (possibly empty) of characters. return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || PatternMatchesString(pattern + 1, str); default: // Non-special character. Matches itself. return *pattern == *str && PatternMatchesString(pattern + 1, str + 1); } } bool UnitTestOptions::MatchesFilter( const std::string& name, const char* filter) { const char *cur_pattern = filter; for (;;) { if (PatternMatchesString(cur_pattern, name.c_str())) { return true; } // Finds the next pattern in the filter. cur_pattern = strchr(cur_pattern, ':'); // Returns if no more pattern can be found. if (cur_pattern == nullptr) { return false; } // Skips the pattern separater (the ':' character). cur_pattern++; } } // Returns true if and only if the user-specified filter matches the test // suite name and the test name. bool UnitTestOptions::FilterMatchesTest(const std::string& test_suite_name, const std::string& test_name) { const std::string& full_name = test_suite_name + "." + test_name.c_str(); // Split --gtest_filter at '-', if there is one, to separate into // positive filter and negative filter portions const char* const p = GTEST_FLAG(filter).c_str(); const char* const dash = strchr(p, '-'); std::string positive; std::string negative; if (dash == nullptr) { positive = GTEST_FLAG(filter).c_str(); // Whole string is a positive filter negative = ""; } else { positive = std::string(p, dash); // Everything up to the dash negative = std::string(dash + 1); // Everything after the dash if (positive.empty()) { // Treat '-test1' as the same as '*-test1' positive = kUniversalFilter; } } // A filter is a colon-separated list of patterns. It matches a // test if any pattern in it matches the test. return (MatchesFilter(full_name, positive.c_str()) && !MatchesFilter(full_name, negative.c_str())); } #if GTEST_HAS_SEH // Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the // given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise. // This function is useful as an __except condition. int UnitTestOptions::GTestShouldProcessSEH(DWORD exception_code) { // Google Test should handle a SEH exception if: // 1. the user wants it to, AND // 2. this is not a breakpoint exception, AND // 3. this is not a C++ exception (VC++ implements them via SEH, // apparently). // // SEH exception code for C++ exceptions. // (see http://support.microsoft.com/kb/185294 for more information). const DWORD kCxxExceptionCode = 0xe06d7363; bool should_handle = true; if (!GTEST_FLAG(catch_exceptions)) should_handle = false; else if (exception_code == EXCEPTION_BREAKPOINT) should_handle = false; else if (exception_code == kCxxExceptionCode) should_handle = false; return should_handle ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH; } #endif // GTEST_HAS_SEH } // namespace internal // The c'tor sets this object as the test part result reporter used by // Google Test. The 'result' parameter specifies where to report the // results. Intercepts only failures from the current thread. ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter( TestPartResultArray* result) : intercept_mode_(INTERCEPT_ONLY_CURRENT_THREAD), result_(result) { Init(); } // The c'tor sets this object as the test part result reporter used by // Google Test. The 'result' parameter specifies where to report the // results. ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter( InterceptMode intercept_mode, TestPartResultArray* result) : intercept_mode_(intercept_mode), result_(result) { Init(); } void ScopedFakeTestPartResultReporter::Init() { internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); if (intercept_mode_ == INTERCEPT_ALL_THREADS) { old_reporter_ = impl->GetGlobalTestPartResultReporter(); impl->SetGlobalTestPartResultReporter(this); } else { old_reporter_ = impl->GetTestPartResultReporterForCurrentThread(); impl->SetTestPartResultReporterForCurrentThread(this); } } // The d'tor restores the test part result reporter used by Google Test // before. ScopedFakeTestPartResultReporter::~ScopedFakeTestPartResultReporter() { internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); if (intercept_mode_ == INTERCEPT_ALL_THREADS) { impl->SetGlobalTestPartResultReporter(old_reporter_); } else { impl->SetTestPartResultReporterForCurrentThread(old_reporter_); } } // Increments the test part result count and remembers the result. // This method is from the TestPartResultReporterInterface interface. void ScopedFakeTestPartResultReporter::ReportTestPartResult( const TestPartResult& result) { result_->Append(result); } namespace internal { // Returns the type ID of ::testing::Test. We should always call this // instead of GetTypeId< ::testing::Test>() to get the type ID of // testing::Test. This is to work around a suspected linker bug when // using Google Test as a framework on Mac OS X. The bug causes // GetTypeId< ::testing::Test>() to return different values depending // on whether the call is from the Google Test framework itself or // from user test code. GetTestTypeId() is guaranteed to always // return the same value, as it always calls GetTypeId<>() from the // gtest.cc, which is within the Google Test framework. TypeId GetTestTypeId() { return GetTypeId(); } // The value of GetTestTypeId() as seen from within the Google Test // library. This is solely for testing GetTestTypeId(). extern const TypeId kTestTypeIdInGoogleTest = GetTestTypeId(); // This predicate-formatter checks that 'results' contains a test part // failure of the given type and that the failure message contains the // given substring. static AssertionResult HasOneFailure(const char* /* results_expr */, const char* /* type_expr */, const char* /* substr_expr */, const TestPartResultArray& results, TestPartResult::Type type, const std::string& substr) { const std::string expected(type == TestPartResult::kFatalFailure ? "1 fatal failure" : "1 non-fatal failure"); Message msg; if (results.size() != 1) { msg << "Expected: " << expected << "\n" << " Actual: " << results.size() << " failures"; for (int i = 0; i < results.size(); i++) { msg << "\n" << results.GetTestPartResult(i); } return AssertionFailure() << msg; } const TestPartResult& r = results.GetTestPartResult(0); if (r.type() != type) { return AssertionFailure() << "Expected: " << expected << "\n" << " Actual:\n" << r; } if (strstr(r.message(), substr.c_str()) == nullptr) { return AssertionFailure() << "Expected: " << expected << " containing \"" << substr << "\"\n" << " Actual:\n" << r; } return AssertionSuccess(); } // The constructor of SingleFailureChecker remembers where to look up // test part results, what type of failure we expect, and what // substring the failure message should contain. SingleFailureChecker::SingleFailureChecker(const TestPartResultArray* results, TestPartResult::Type type, const std::string& substr) : results_(results), type_(type), substr_(substr) {} // The destructor of SingleFailureChecker verifies that the given // TestPartResultArray contains exactly one failure that has the given // type and contains the given substring. If that's not the case, a // non-fatal failure will be generated. SingleFailureChecker::~SingleFailureChecker() { EXPECT_PRED_FORMAT3(HasOneFailure, *results_, type_, substr_); } DefaultGlobalTestPartResultReporter::DefaultGlobalTestPartResultReporter( UnitTestImpl* unit_test) : unit_test_(unit_test) {} void DefaultGlobalTestPartResultReporter::ReportTestPartResult( const TestPartResult& result) { unit_test_->current_test_result()->AddTestPartResult(result); unit_test_->listeners()->repeater()->OnTestPartResult(result); } DefaultPerThreadTestPartResultReporter::DefaultPerThreadTestPartResultReporter( UnitTestImpl* unit_test) : unit_test_(unit_test) {} void DefaultPerThreadTestPartResultReporter::ReportTestPartResult( const TestPartResult& result) { unit_test_->GetGlobalTestPartResultReporter()->ReportTestPartResult(result); } // Returns the global test part result reporter. TestPartResultReporterInterface* UnitTestImpl::GetGlobalTestPartResultReporter() { internal::MutexLock lock(&global_test_part_result_reporter_mutex_); return global_test_part_result_repoter_; } // Sets the global test part result reporter. void UnitTestImpl::SetGlobalTestPartResultReporter( TestPartResultReporterInterface* reporter) { internal::MutexLock lock(&global_test_part_result_reporter_mutex_); global_test_part_result_repoter_ = reporter; } // Returns the test part result reporter for the current thread. TestPartResultReporterInterface* UnitTestImpl::GetTestPartResultReporterForCurrentThread() { return per_thread_test_part_result_reporter_.get(); } // Sets the test part result reporter for the current thread. void UnitTestImpl::SetTestPartResultReporterForCurrentThread( TestPartResultReporterInterface* reporter) { per_thread_test_part_result_reporter_.set(reporter); } // Gets the number of successful test suites. int UnitTestImpl::successful_test_suite_count() const { return CountIf(test_suites_, TestSuitePassed); } // Gets the number of failed test suites. int UnitTestImpl::failed_test_suite_count() const { return CountIf(test_suites_, TestSuiteFailed); } // Gets the number of all test suites. int UnitTestImpl::total_test_suite_count() const { return static_cast(test_suites_.size()); } // Gets the number of all test suites that contain at least one test // that should run. int UnitTestImpl::test_suite_to_run_count() const { return CountIf(test_suites_, ShouldRunTestSuite); } // Gets the number of successful tests. int UnitTestImpl::successful_test_count() const { return SumOverTestSuiteList(test_suites_, &TestSuite::successful_test_count); } // Gets the number of skipped tests. int UnitTestImpl::skipped_test_count() const { return SumOverTestSuiteList(test_suites_, &TestSuite::skipped_test_count); } // Gets the number of failed tests. int UnitTestImpl::failed_test_count() const { return SumOverTestSuiteList(test_suites_, &TestSuite::failed_test_count); } // Gets the number of disabled tests that will be reported in the XML report. int UnitTestImpl::reportable_disabled_test_count() const { return SumOverTestSuiteList(test_suites_, &TestSuite::reportable_disabled_test_count); } // Gets the number of disabled tests. int UnitTestImpl::disabled_test_count() const { return SumOverTestSuiteList(test_suites_, &TestSuite::disabled_test_count); } // Gets the number of tests to be printed in the XML report. int UnitTestImpl::reportable_test_count() const { return SumOverTestSuiteList(test_suites_, &TestSuite::reportable_test_count); } // Gets the number of all tests. int UnitTestImpl::total_test_count() const { return SumOverTestSuiteList(test_suites_, &TestSuite::total_test_count); } // Gets the number of tests that should run. int UnitTestImpl::test_to_run_count() const { return SumOverTestSuiteList(test_suites_, &TestSuite::test_to_run_count); } // Returns the current OS stack trace as an std::string. // // The maximum number of stack frames to be included is specified by // the gtest_stack_trace_depth flag. The skip_count parameter // specifies the number of top frames to be skipped, which doesn't // count against the number of frames to be included. // // For example, if Foo() calls Bar(), which in turn calls // CurrentOsStackTraceExceptTop(1), Foo() will be included in the // trace but Bar() and CurrentOsStackTraceExceptTop() won't. std::string UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) { return os_stack_trace_getter()->CurrentStackTrace( static_cast(GTEST_FLAG(stack_trace_depth)), skip_count + 1 // Skips the user-specified number of frames plus this function // itself. ); // NOLINT } // Returns the current time in milliseconds. TimeInMillis GetTimeInMillis() { #if GTEST_OS_WINDOWS_MOBILE || defined(__BORLANDC__) // Difference between 1970-01-01 and 1601-01-01 in milliseconds. // http://analogous.blogspot.com/2005/04/epoch.html const TimeInMillis kJavaEpochToWinFileTimeDelta = static_cast(116444736UL) * 100000UL; const DWORD kTenthMicrosInMilliSecond = 10000; SYSTEMTIME now_systime; FILETIME now_filetime; ULARGE_INTEGER now_int64; GetSystemTime(&now_systime); if (SystemTimeToFileTime(&now_systime, &now_filetime)) { now_int64.LowPart = now_filetime.dwLowDateTime; now_int64.HighPart = now_filetime.dwHighDateTime; now_int64.QuadPart = (now_int64.QuadPart / kTenthMicrosInMilliSecond) - kJavaEpochToWinFileTimeDelta; return now_int64.QuadPart; } return 0; #elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_ __timeb64 now; // MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996 // (deprecated function) there. GTEST_DISABLE_MSC_DEPRECATED_PUSH_() _ftime64(&now); GTEST_DISABLE_MSC_DEPRECATED_POP_() return static_cast(now.time) * 1000 + now.millitm; #elif GTEST_HAS_GETTIMEOFDAY_ struct timeval now; gettimeofday(&now, nullptr); return static_cast(now.tv_sec) * 1000 + now.tv_usec / 1000; #else # error "Don't know how to get the current time on your system." #endif } // Utilities // class String. #if GTEST_OS_WINDOWS_MOBILE // Creates a UTF-16 wide string from the given ANSI string, allocating // memory using new. The caller is responsible for deleting the return // value using delete[]. Returns the wide string, or NULL if the // input is NULL. LPCWSTR String::AnsiToUtf16(const char* ansi) { if (!ansi) return nullptr; const int length = strlen(ansi); const int unicode_length = MultiByteToWideChar(CP_ACP, 0, ansi, length, nullptr, 0); WCHAR* unicode = new WCHAR[unicode_length + 1]; MultiByteToWideChar(CP_ACP, 0, ansi, length, unicode, unicode_length); unicode[unicode_length] = 0; return unicode; } // Creates an ANSI string from the given wide string, allocating // memory using new. The caller is responsible for deleting the return // value using delete[]. Returns the ANSI string, or NULL if the // input is NULL. const char* String::Utf16ToAnsi(LPCWSTR utf16_str) { if (!utf16_str) return nullptr; const int ansi_length = WideCharToMultiByte(CP_ACP, 0, utf16_str, -1, nullptr, 0, nullptr, nullptr); char* ansi = new char[ansi_length + 1]; WideCharToMultiByte(CP_ACP, 0, utf16_str, -1, ansi, ansi_length, nullptr, nullptr); ansi[ansi_length] = 0; return ansi; } #endif // GTEST_OS_WINDOWS_MOBILE // Compares two C strings. Returns true if and only if they have the same // content. // // Unlike strcmp(), this function can handle NULL argument(s). A NULL // C string is considered different to any non-NULL C string, // including the empty string. bool String::CStringEquals(const char * lhs, const char * rhs) { if (lhs == nullptr) return rhs == nullptr; if (rhs == nullptr) return false; return strcmp(lhs, rhs) == 0; } #if GTEST_HAS_STD_WSTRING // Converts an array of wide chars to a narrow string using the UTF-8 // encoding, and streams the result to the given Message object. static void StreamWideCharsToMessage(const wchar_t* wstr, size_t length, Message* msg) { for (size_t i = 0; i != length; ) { // NOLINT if (wstr[i] != L'\0') { *msg << WideStringToUtf8(wstr + i, static_cast(length - i)); while (i != length && wstr[i] != L'\0') i++; } else { *msg << '\0'; i++; } } } #endif // GTEST_HAS_STD_WSTRING void SplitString(const ::std::string& str, char delimiter, ::std::vector< ::std::string>* dest) { ::std::vector< ::std::string> parsed; ::std::string::size_type pos = 0; while (::testing::internal::AlwaysTrue()) { const ::std::string::size_type colon = str.find(delimiter, pos); if (colon == ::std::string::npos) { parsed.push_back(str.substr(pos)); break; } else { parsed.push_back(str.substr(pos, colon - pos)); pos = colon + 1; } } dest->swap(parsed); } } // namespace internal // Constructs an empty Message. // We allocate the stringstream separately because otherwise each use of // ASSERT/EXPECT in a procedure adds over 200 bytes to the procedure's // stack frame leading to huge stack frames in some cases; gcc does not reuse // the stack space. Message::Message() : ss_(new ::std::stringstream) { // By default, we want there to be enough precision when printing // a double to a Message. *ss_ << std::setprecision(std::numeric_limits::digits10 + 2); } // These two overloads allow streaming a wide C string to a Message // using the UTF-8 encoding. Message& Message::operator <<(const wchar_t* wide_c_str) { return *this << internal::String::ShowWideCString(wide_c_str); } Message& Message::operator <<(wchar_t* wide_c_str) { return *this << internal::String::ShowWideCString(wide_c_str); } #if GTEST_HAS_STD_WSTRING // Converts the given wide string to a narrow string using the UTF-8 // encoding, and streams the result to this Message object. Message& Message::operator <<(const ::std::wstring& wstr) { internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this); return *this; } #endif // GTEST_HAS_STD_WSTRING // Gets the text streamed to this object so far as an std::string. // Each '\0' character in the buffer is replaced with "\\0". std::string Message::GetString() const { return internal::StringStreamToString(ss_.get()); } // AssertionResult constructors. // Used in EXPECT_TRUE/FALSE(assertion_result). AssertionResult::AssertionResult(const AssertionResult& other) : success_(other.success_), message_(other.message_.get() != nullptr ? new ::std::string(*other.message_) : static_cast< ::std::string*>(nullptr)) {} // Swaps two AssertionResults. void AssertionResult::swap(AssertionResult& other) { using std::swap; swap(success_, other.success_); swap(message_, other.message_); } // Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. AssertionResult AssertionResult::operator!() const { AssertionResult negation(!success_); if (message_.get() != nullptr) negation << *message_; return negation; } // Makes a successful assertion result. AssertionResult AssertionSuccess() { return AssertionResult(true); } // Makes a failed assertion result. AssertionResult AssertionFailure() { return AssertionResult(false); } // Makes a failed assertion result with the given failure message. // Deprecated; use AssertionFailure() << message. AssertionResult AssertionFailure(const Message& message) { return AssertionFailure() << message; } namespace internal { namespace edit_distance { std::vector CalculateOptimalEdits(const std::vector& left, const std::vector& right) { std::vector > costs( left.size() + 1, std::vector(right.size() + 1)); std::vector > best_move( left.size() + 1, std::vector(right.size() + 1)); // Populate for empty right. for (size_t l_i = 0; l_i < costs.size(); ++l_i) { costs[l_i][0] = static_cast(l_i); best_move[l_i][0] = kRemove; } // Populate for empty left. for (size_t r_i = 1; r_i < costs[0].size(); ++r_i) { costs[0][r_i] = static_cast(r_i); best_move[0][r_i] = kAdd; } for (size_t l_i = 0; l_i < left.size(); ++l_i) { for (size_t r_i = 0; r_i < right.size(); ++r_i) { if (left[l_i] == right[r_i]) { // Found a match. Consume it. costs[l_i + 1][r_i + 1] = costs[l_i][r_i]; best_move[l_i + 1][r_i + 1] = kMatch; continue; } const double add = costs[l_i + 1][r_i]; const double remove = costs[l_i][r_i + 1]; const double replace = costs[l_i][r_i]; if (add < remove && add < replace) { costs[l_i + 1][r_i + 1] = add + 1; best_move[l_i + 1][r_i + 1] = kAdd; } else if (remove < add && remove < replace) { costs[l_i + 1][r_i + 1] = remove + 1; best_move[l_i + 1][r_i + 1] = kRemove; } else { // We make replace a little more expensive than add/remove to lower // their priority. costs[l_i + 1][r_i + 1] = replace + 1.00001; best_move[l_i + 1][r_i + 1] = kReplace; } } } // Reconstruct the best path. We do it in reverse order. std::vector best_path; for (size_t l_i = left.size(), r_i = right.size(); l_i > 0 || r_i > 0;) { EditType move = best_move[l_i][r_i]; best_path.push_back(move); l_i -= move != kAdd; r_i -= move != kRemove; } std::reverse(best_path.begin(), best_path.end()); return best_path; } namespace { // Helper class to convert string into ids with deduplication. class InternalStrings { public: size_t GetId(const std::string& str) { IdMap::iterator it = ids_.find(str); if (it != ids_.end()) return it->second; size_t id = ids_.size(); return ids_[str] = id; } private: typedef std::map IdMap; IdMap ids_; }; } // namespace std::vector CalculateOptimalEdits( const std::vector& left, const std::vector& right) { std::vector left_ids, right_ids; { InternalStrings intern_table; for (size_t i = 0; i < left.size(); ++i) { left_ids.push_back(intern_table.GetId(left[i])); } for (size_t i = 0; i < right.size(); ++i) { right_ids.push_back(intern_table.GetId(right[i])); } } return CalculateOptimalEdits(left_ids, right_ids); } namespace { // Helper class that holds the state for one hunk and prints it out to the // stream. // It reorders adds/removes when possible to group all removes before all // adds. It also adds the hunk header before printint into the stream. class Hunk { public: Hunk(size_t left_start, size_t right_start) : left_start_(left_start), right_start_(right_start), adds_(), removes_(), common_() {} void PushLine(char edit, const char* line) { switch (edit) { case ' ': ++common_; FlushEdits(); hunk_.push_back(std::make_pair(' ', line)); break; case '-': ++removes_; hunk_removes_.push_back(std::make_pair('-', line)); break; case '+': ++adds_; hunk_adds_.push_back(std::make_pair('+', line)); break; } } void PrintTo(std::ostream* os) { PrintHeader(os); FlushEdits(); for (std::list >::const_iterator it = hunk_.begin(); it != hunk_.end(); ++it) { *os << it->first << it->second << "\n"; } } bool has_edits() const { return adds_ || removes_; } private: void FlushEdits() { hunk_.splice(hunk_.end(), hunk_removes_); hunk_.splice(hunk_.end(), hunk_adds_); } // Print a unified diff header for one hunk. // The format is // "@@ -, +, @@" // where the left/right parts are omitted if unnecessary. void PrintHeader(std::ostream* ss) const { *ss << "@@ "; if (removes_) { *ss << "-" << left_start_ << "," << (removes_ + common_); } if (removes_ && adds_) { *ss << " "; } if (adds_) { *ss << "+" << right_start_ << "," << (adds_ + common_); } *ss << " @@\n"; } size_t left_start_, right_start_; size_t adds_, removes_, common_; std::list > hunk_, hunk_adds_, hunk_removes_; }; } // namespace // Create a list of diff hunks in Unified diff format. // Each hunk has a header generated by PrintHeader above plus a body with // lines prefixed with ' ' for no change, '-' for deletion and '+' for // addition. // 'context' represents the desired unchanged prefix/suffix around the diff. // If two hunks are close enough that their contexts overlap, then they are // joined into one hunk. std::string CreateUnifiedDiff(const std::vector& left, const std::vector& right, size_t context) { const std::vector edits = CalculateOptimalEdits(left, right); size_t l_i = 0, r_i = 0, edit_i = 0; std::stringstream ss; while (edit_i < edits.size()) { // Find first edit. while (edit_i < edits.size() && edits[edit_i] == kMatch) { ++l_i; ++r_i; ++edit_i; } // Find the first line to include in the hunk. const size_t prefix_context = std::min(l_i, context); Hunk hunk(l_i - prefix_context + 1, r_i - prefix_context + 1); for (size_t i = prefix_context; i > 0; --i) { hunk.PushLine(' ', left[l_i - i].c_str()); } // Iterate the edits until we found enough suffix for the hunk or the input // is over. size_t n_suffix = 0; for (; edit_i < edits.size(); ++edit_i) { if (n_suffix >= context) { // Continue only if the next hunk is very close. auto it = edits.begin() + static_cast(edit_i); while (it != edits.end() && *it == kMatch) ++it; if (it == edits.end() || static_cast(it - edits.begin()) - edit_i >= context) { // There is no next edit or it is too far away. break; } } EditType edit = edits[edit_i]; // Reset count when a non match is found. n_suffix = edit == kMatch ? n_suffix + 1 : 0; if (edit == kMatch || edit == kRemove || edit == kReplace) { hunk.PushLine(edit == kMatch ? ' ' : '-', left[l_i].c_str()); } if (edit == kAdd || edit == kReplace) { hunk.PushLine('+', right[r_i].c_str()); } // Advance indices, depending on edit type. l_i += edit != kAdd; r_i += edit != kRemove; } if (!hunk.has_edits()) { // We are done. We don't want this hunk. break; } hunk.PrintTo(&ss); } return ss.str(); } } // namespace edit_distance namespace { // The string representation of the values received in EqFailure() are already // escaped. Split them on escaped '\n' boundaries. Leave all other escaped // characters the same. std::vector SplitEscapedString(const std::string& str) { std::vector lines; size_t start = 0, end = str.size(); if (end > 2 && str[0] == '"' && str[end - 1] == '"') { ++start; --end; } bool escaped = false; for (size_t i = start; i + 1 < end; ++i) { if (escaped) { escaped = false; if (str[i] == 'n') { lines.push_back(str.substr(start, i - start - 1)); start = i + 1; } } else { escaped = str[i] == '\\'; } } lines.push_back(str.substr(start, end - start)); return lines; } } // namespace // Constructs and returns the message for an equality assertion // (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. // // The first four parameters are the expressions used in the assertion // and their values, as strings. For example, for ASSERT_EQ(foo, bar) // where foo is 5 and bar is 6, we have: // // lhs_expression: "foo" // rhs_expression: "bar" // lhs_value: "5" // rhs_value: "6" // // The ignoring_case parameter is true if and only if the assertion is a // *_STRCASEEQ*. When it's true, the string "Ignoring case" will // be inserted into the message. AssertionResult EqFailure(const char* lhs_expression, const char* rhs_expression, const std::string& lhs_value, const std::string& rhs_value, bool ignoring_case) { Message msg; msg << "Expected equality of these values:"; msg << "\n " << lhs_expression; if (lhs_value != lhs_expression) { msg << "\n Which is: " << lhs_value; } msg << "\n " << rhs_expression; if (rhs_value != rhs_expression) { msg << "\n Which is: " << rhs_value; } if (ignoring_case) { msg << "\nIgnoring case"; } if (!lhs_value.empty() && !rhs_value.empty()) { const std::vector lhs_lines = SplitEscapedString(lhs_value); const std::vector rhs_lines = SplitEscapedString(rhs_value); if (lhs_lines.size() > 1 || rhs_lines.size() > 1) { msg << "\nWith diff:\n" << edit_distance::CreateUnifiedDiff(lhs_lines, rhs_lines); } } return AssertionFailure() << msg; } // Constructs a failure message for Boolean assertions such as EXPECT_TRUE. std::string GetBoolAssertionFailureMessage( const AssertionResult& assertion_result, const char* expression_text, const char* actual_predicate_value, const char* expected_predicate_value) { const char* actual_message = assertion_result.message(); Message msg; msg << "Value of: " << expression_text << "\n Actual: " << actual_predicate_value; if (actual_message[0] != '\0') msg << " (" << actual_message << ")"; msg << "\nExpected: " << expected_predicate_value; return msg.GetString(); } // Helper function for implementing ASSERT_NEAR. AssertionResult DoubleNearPredFormat(const char* expr1, const char* expr2, const char* abs_error_expr, double val1, double val2, double abs_error) { const double diff = fabs(val1 - val2); if (diff <= abs_error) return AssertionSuccess(); return AssertionFailure() << "The difference between " << expr1 << " and " << expr2 << " is " << diff << ", which exceeds " << abs_error_expr << ", where\n" << expr1 << " evaluates to " << val1 << ",\n" << expr2 << " evaluates to " << val2 << ", and\n" << abs_error_expr << " evaluates to " << abs_error << "."; } // Helper template for implementing FloatLE() and DoubleLE(). template AssertionResult FloatingPointLE(const char* expr1, const char* expr2, RawType val1, RawType val2) { // Returns success if val1 is less than val2, if (val1 < val2) { return AssertionSuccess(); } // or if val1 is almost equal to val2. const FloatingPoint lhs(val1), rhs(val2); if (lhs.AlmostEquals(rhs)) { return AssertionSuccess(); } // Note that the above two checks will both fail if either val1 or // val2 is NaN, as the IEEE floating-point standard requires that // any predicate involving a NaN must return false. ::std::stringstream val1_ss; val1_ss << std::setprecision(std::numeric_limits::digits10 + 2) << val1; ::std::stringstream val2_ss; val2_ss << std::setprecision(std::numeric_limits::digits10 + 2) << val2; return AssertionFailure() << "Expected: (" << expr1 << ") <= (" << expr2 << ")\n" << " Actual: " << StringStreamToString(&val1_ss) << " vs " << StringStreamToString(&val2_ss); } } // namespace internal // Asserts that val1 is less than, or almost equal to, val2. Fails // otherwise. In particular, it fails if either val1 or val2 is NaN. AssertionResult FloatLE(const char* expr1, const char* expr2, float val1, float val2) { return internal::FloatingPointLE(expr1, expr2, val1, val2); } // Asserts that val1 is less than, or almost equal to, val2. Fails // otherwise. In particular, it fails if either val1 or val2 is NaN. AssertionResult DoubleLE(const char* expr1, const char* expr2, double val1, double val2) { return internal::FloatingPointLE(expr1, expr2, val1, val2); } namespace internal { // The helper function for {ASSERT|EXPECT}_EQ with int or enum // arguments. AssertionResult CmpHelperEQ(const char* lhs_expression, const char* rhs_expression, BiggestInt lhs, BiggestInt rhs) { if (lhs == rhs) { return AssertionSuccess(); } return EqFailure(lhs_expression, rhs_expression, FormatForComparisonFailureMessage(lhs, rhs), FormatForComparisonFailureMessage(rhs, lhs), false); } // A macro for implementing the helper functions needed to implement // ASSERT_?? and EXPECT_?? with integer or enum arguments. It is here // just to avoid copy-and-paste of similar code. #define GTEST_IMPL_CMP_HELPER_(op_name, op)\ AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ BiggestInt val1, BiggestInt val2) {\ if (val1 op val2) {\ return AssertionSuccess();\ } else {\ return AssertionFailure() \ << "Expected: (" << expr1 << ") " #op " (" << expr2\ << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\ << " vs " << FormatForComparisonFailureMessage(val2, val1);\ }\ } // Implements the helper function for {ASSERT|EXPECT}_NE with int or // enum arguments. GTEST_IMPL_CMP_HELPER_(NE, !=) // Implements the helper function for {ASSERT|EXPECT}_LE with int or // enum arguments. GTEST_IMPL_CMP_HELPER_(LE, <=) // Implements the helper function for {ASSERT|EXPECT}_LT with int or // enum arguments. GTEST_IMPL_CMP_HELPER_(LT, < ) // Implements the helper function for {ASSERT|EXPECT}_GE with int or // enum arguments. GTEST_IMPL_CMP_HELPER_(GE, >=) // Implements the helper function for {ASSERT|EXPECT}_GT with int or // enum arguments. GTEST_IMPL_CMP_HELPER_(GT, > ) #undef GTEST_IMPL_CMP_HELPER_ // The helper function for {ASSERT|EXPECT}_STREQ. AssertionResult CmpHelperSTREQ(const char* lhs_expression, const char* rhs_expression, const char* lhs, const char* rhs) { if (String::CStringEquals(lhs, rhs)) { return AssertionSuccess(); } return EqFailure(lhs_expression, rhs_expression, PrintToString(lhs), PrintToString(rhs), false); } // The helper function for {ASSERT|EXPECT}_STRCASEEQ. AssertionResult CmpHelperSTRCASEEQ(const char* lhs_expression, const char* rhs_expression, const char* lhs, const char* rhs) { if (String::CaseInsensitiveCStringEquals(lhs, rhs)) { return AssertionSuccess(); } return EqFailure(lhs_expression, rhs_expression, PrintToString(lhs), PrintToString(rhs), true); } // The helper function for {ASSERT|EXPECT}_STRNE. AssertionResult CmpHelperSTRNE(const char* s1_expression, const char* s2_expression, const char* s1, const char* s2) { if (!String::CStringEquals(s1, s2)) { return AssertionSuccess(); } else { return AssertionFailure() << "Expected: (" << s1_expression << ") != (" << s2_expression << "), actual: \"" << s1 << "\" vs \"" << s2 << "\""; } } // The helper function for {ASSERT|EXPECT}_STRCASENE. AssertionResult CmpHelperSTRCASENE(const char* s1_expression, const char* s2_expression, const char* s1, const char* s2) { if (!String::CaseInsensitiveCStringEquals(s1, s2)) { return AssertionSuccess(); } else { return AssertionFailure() << "Expected: (" << s1_expression << ") != (" << s2_expression << ") (ignoring case), actual: \"" << s1 << "\" vs \"" << s2 << "\""; } } } // namespace internal namespace { // Helper functions for implementing IsSubString() and IsNotSubstring(). // This group of overloaded functions return true if and only if needle // is a substring of haystack. NULL is considered a substring of // itself only. bool IsSubstringPred(const char* needle, const char* haystack) { if (needle == nullptr || haystack == nullptr) return needle == haystack; return strstr(haystack, needle) != nullptr; } bool IsSubstringPred(const wchar_t* needle, const wchar_t* haystack) { if (needle == nullptr || haystack == nullptr) return needle == haystack; return wcsstr(haystack, needle) != nullptr; } // StringType here can be either ::std::string or ::std::wstring. template bool IsSubstringPred(const StringType& needle, const StringType& haystack) { return haystack.find(needle) != StringType::npos; } // This function implements either IsSubstring() or IsNotSubstring(), // depending on the value of the expected_to_be_substring parameter. // StringType here can be const char*, const wchar_t*, ::std::string, // or ::std::wstring. template AssertionResult IsSubstringImpl( bool expected_to_be_substring, const char* needle_expr, const char* haystack_expr, const StringType& needle, const StringType& haystack) { if (IsSubstringPred(needle, haystack) == expected_to_be_substring) return AssertionSuccess(); const bool is_wide_string = sizeof(needle[0]) > 1; const char* const begin_string_quote = is_wide_string ? "L\"" : "\""; return AssertionFailure() << "Value of: " << needle_expr << "\n" << " Actual: " << begin_string_quote << needle << "\"\n" << "Expected: " << (expected_to_be_substring ? "" : "not ") << "a substring of " << haystack_expr << "\n" << "Which is: " << begin_string_quote << haystack << "\""; } } // namespace // IsSubstring() and IsNotSubstring() check whether needle is a // substring of haystack (NULL is considered a substring of itself // only), and return an appropriate error message when they fail. AssertionResult IsSubstring( const char* needle_expr, const char* haystack_expr, const char* needle, const char* haystack) { return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); } AssertionResult IsSubstring( const char* needle_expr, const char* haystack_expr, const wchar_t* needle, const wchar_t* haystack) { return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); } AssertionResult IsNotSubstring( const char* needle_expr, const char* haystack_expr, const char* needle, const char* haystack) { return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); } AssertionResult IsNotSubstring( const char* needle_expr, const char* haystack_expr, const wchar_t* needle, const wchar_t* haystack) { return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); } AssertionResult IsSubstring( const char* needle_expr, const char* haystack_expr, const ::std::string& needle, const ::std::string& haystack) { return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); } AssertionResult IsNotSubstring( const char* needle_expr, const char* haystack_expr, const ::std::string& needle, const ::std::string& haystack) { return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); } #if GTEST_HAS_STD_WSTRING AssertionResult IsSubstring( const char* needle_expr, const char* haystack_expr, const ::std::wstring& needle, const ::std::wstring& haystack) { return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); } AssertionResult IsNotSubstring( const char* needle_expr, const char* haystack_expr, const ::std::wstring& needle, const ::std::wstring& haystack) { return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); } #endif // GTEST_HAS_STD_WSTRING namespace internal { #if GTEST_OS_WINDOWS namespace { // Helper function for IsHRESULT{SuccessFailure} predicates AssertionResult HRESULTFailureHelper(const char* expr, const char* expected, long hr) { // NOLINT # if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_TV_TITLE // Windows CE doesn't support FormatMessage. const char error_text[] = ""; # else // Looks up the human-readable system message for the HRESULT code // and since we're not passing any params to FormatMessage, we don't // want inserts expanded. const DWORD kFlags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; const DWORD kBufSize = 4096; // Gets the system's human readable message string for this HRESULT. char error_text[kBufSize] = { '\0' }; DWORD message_length = ::FormatMessageA(kFlags, 0, // no source, we're asking system static_cast(hr), // the error 0, // no line width restrictions error_text, // output buffer kBufSize, // buf size nullptr); // no arguments for inserts // Trims tailing white space (FormatMessage leaves a trailing CR-LF) for (; message_length && IsSpace(error_text[message_length - 1]); --message_length) { error_text[message_length - 1] = '\0'; } # endif // GTEST_OS_WINDOWS_MOBILE const std::string error_hex("0x" + String::FormatHexInt(hr)); return ::testing::AssertionFailure() << "Expected: " << expr << " " << expected << ".\n" << " Actual: " << error_hex << " " << error_text << "\n"; } } // namespace AssertionResult IsHRESULTSuccess(const char* expr, long hr) { // NOLINT if (SUCCEEDED(hr)) { return AssertionSuccess(); } return HRESULTFailureHelper(expr, "succeeds", hr); } AssertionResult IsHRESULTFailure(const char* expr, long hr) { // NOLINT if (FAILED(hr)) { return AssertionSuccess(); } return HRESULTFailureHelper(expr, "fails", hr); } #endif // GTEST_OS_WINDOWS // Utility functions for encoding Unicode text (wide strings) in // UTF-8. // A Unicode code-point can have up to 21 bits, and is encoded in UTF-8 // like this: // // Code-point length Encoding // 0 - 7 bits 0xxxxxxx // 8 - 11 bits 110xxxxx 10xxxxxx // 12 - 16 bits 1110xxxx 10xxxxxx 10xxxxxx // 17 - 21 bits 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx // The maximum code-point a one-byte UTF-8 sequence can represent. const UInt32 kMaxCodePoint1 = (static_cast(1) << 7) - 1; // The maximum code-point a two-byte UTF-8 sequence can represent. const UInt32 kMaxCodePoint2 = (static_cast(1) << (5 + 6)) - 1; // The maximum code-point a three-byte UTF-8 sequence can represent. const UInt32 kMaxCodePoint3 = (static_cast(1) << (4 + 2*6)) - 1; // The maximum code-point a four-byte UTF-8 sequence can represent. const UInt32 kMaxCodePoint4 = (static_cast(1) << (3 + 3*6)) - 1; // Chops off the n lowest bits from a bit pattern. Returns the n // lowest bits. As a side effect, the original bit pattern will be // shifted to the right by n bits. inline UInt32 ChopLowBits(UInt32* bits, int n) { const UInt32 low_bits = *bits & ((static_cast(1) << n) - 1); *bits >>= n; return low_bits; } // Converts a Unicode code point to a narrow string in UTF-8 encoding. // code_point parameter is of type UInt32 because wchar_t may not be // wide enough to contain a code point. // If the code_point is not a valid Unicode code point // (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted // to "(Invalid Unicode 0xXXXXXXXX)". std::string CodePointToUtf8(UInt32 code_point) { if (code_point > kMaxCodePoint4) { return "(Invalid Unicode 0x" + String::FormatHexUInt32(code_point) + ")"; } char str[5]; // Big enough for the largest valid code point. if (code_point <= kMaxCodePoint1) { str[1] = '\0'; str[0] = static_cast(code_point); // 0xxxxxxx } else if (code_point <= kMaxCodePoint2) { str[2] = '\0'; str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[0] = static_cast(0xC0 | code_point); // 110xxxxx } else if (code_point <= kMaxCodePoint3) { str[3] = '\0'; str[2] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[0] = static_cast(0xE0 | code_point); // 1110xxxx } else { // code_point <= kMaxCodePoint4 str[4] = '\0'; str[3] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[2] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[0] = static_cast(0xF0 | code_point); // 11110xxx } return str; } // The following two functions only make sense if the system // uses UTF-16 for wide string encoding. All supported systems // with 16 bit wchar_t (Windows, Cygwin) do use UTF-16. // Determines if the arguments constitute UTF-16 surrogate pair // and thus should be combined into a single Unicode code point // using CreateCodePointFromUtf16SurrogatePair. inline bool IsUtf16SurrogatePair(wchar_t first, wchar_t second) { return sizeof(wchar_t) == 2 && (first & 0xFC00) == 0xD800 && (second & 0xFC00) == 0xDC00; } // Creates a Unicode code point from UTF16 surrogate pair. inline UInt32 CreateCodePointFromUtf16SurrogatePair(wchar_t first, wchar_t second) { const auto first_u = static_cast(first); const auto second_u = static_cast(second); const UInt32 mask = (1 << 10) - 1; return (sizeof(wchar_t) == 2) ? (((first_u & mask) << 10) | (second_u & mask)) + 0x10000 : // This function should not be called when the condition is // false, but we provide a sensible default in case it is. first_u; } // Converts a wide string to a narrow string in UTF-8 encoding. // The wide string is assumed to have the following encoding: // UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin) // UTF-32 if sizeof(wchar_t) == 4 (on Linux) // Parameter str points to a null-terminated wide string. // Parameter num_chars may additionally limit the number // of wchar_t characters processed. -1 is used when the entire string // should be processed. // If the string contains code points that are not valid Unicode code points // (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output // as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding // and contains invalid UTF-16 surrogate pairs, values in those pairs // will be encoded as individual Unicode characters from Basic Normal Plane. std::string WideStringToUtf8(const wchar_t* str, int num_chars) { if (num_chars == -1) num_chars = static_cast(wcslen(str)); ::std::stringstream stream; for (int i = 0; i < num_chars; ++i) { UInt32 unicode_code_point; if (str[i] == L'\0') { break; } else if (i + 1 < num_chars && IsUtf16SurrogatePair(str[i], str[i + 1])) { unicode_code_point = CreateCodePointFromUtf16SurrogatePair(str[i], str[i + 1]); i++; } else { unicode_code_point = static_cast(str[i]); } stream << CodePointToUtf8(unicode_code_point); } return StringStreamToString(&stream); } // Converts a wide C string to an std::string using the UTF-8 encoding. // NULL will be converted to "(null)". std::string String::ShowWideCString(const wchar_t * wide_c_str) { if (wide_c_str == nullptr) return "(null)"; return internal::WideStringToUtf8(wide_c_str, -1); } // Compares two wide C strings. Returns true if and only if they have the // same content. // // Unlike wcscmp(), this function can handle NULL argument(s). A NULL // C string is considered different to any non-NULL C string, // including the empty string. bool String::WideCStringEquals(const wchar_t * lhs, const wchar_t * rhs) { if (lhs == nullptr) return rhs == nullptr; if (rhs == nullptr) return false; return wcscmp(lhs, rhs) == 0; } // Helper function for *_STREQ on wide strings. AssertionResult CmpHelperSTREQ(const char* lhs_expression, const char* rhs_expression, const wchar_t* lhs, const wchar_t* rhs) { if (String::WideCStringEquals(lhs, rhs)) { return AssertionSuccess(); } return EqFailure(lhs_expression, rhs_expression, PrintToString(lhs), PrintToString(rhs), false); } // Helper function for *_STRNE on wide strings. AssertionResult CmpHelperSTRNE(const char* s1_expression, const char* s2_expression, const wchar_t* s1, const wchar_t* s2) { if (!String::WideCStringEquals(s1, s2)) { return AssertionSuccess(); } return AssertionFailure() << "Expected: (" << s1_expression << ") != (" << s2_expression << "), actual: " << PrintToString(s1) << " vs " << PrintToString(s2); } // Compares two C strings, ignoring case. Returns true if and only if they have // the same content. // // Unlike strcasecmp(), this function can handle NULL argument(s). A // NULL C string is considered different to any non-NULL C string, // including the empty string. bool String::CaseInsensitiveCStringEquals(const char * lhs, const char * rhs) { if (lhs == nullptr) return rhs == nullptr; if (rhs == nullptr) return false; return posix::StrCaseCmp(lhs, rhs) == 0; } // Compares two wide C strings, ignoring case. Returns true if and only if they // have the same content. // // Unlike wcscasecmp(), this function can handle NULL argument(s). // A NULL C string is considered different to any non-NULL wide C string, // including the empty string. // NB: The implementations on different platforms slightly differ. // On windows, this method uses _wcsicmp which compares according to LC_CTYPE // environment variable. On GNU platform this method uses wcscasecmp // which compares according to LC_CTYPE category of the current locale. // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the // current locale. bool String::CaseInsensitiveWideCStringEquals(const wchar_t* lhs, const wchar_t* rhs) { if (lhs == nullptr) return rhs == nullptr; if (rhs == nullptr) return false; #if GTEST_OS_WINDOWS return _wcsicmp(lhs, rhs) == 0; #elif GTEST_OS_LINUX && !GTEST_OS_LINUX_ANDROID return wcscasecmp(lhs, rhs) == 0; #else // Android, Mac OS X and Cygwin don't define wcscasecmp. // Other unknown OSes may not define it either. wint_t left, right; do { left = towlower(static_cast(*lhs++)); right = towlower(static_cast(*rhs++)); } while (left && left == right); return left == right; #endif // OS selector } // Returns true if and only if str ends with the given suffix, ignoring case. // Any string is considered to end with an empty suffix. bool String::EndsWithCaseInsensitive( const std::string& str, const std::string& suffix) { const size_t str_len = str.length(); const size_t suffix_len = suffix.length(); return (str_len >= suffix_len) && CaseInsensitiveCStringEquals(str.c_str() + str_len - suffix_len, suffix.c_str()); } // Formats an int value as "%02d". std::string String::FormatIntWidth2(int value) { std::stringstream ss; ss << std::setfill('0') << std::setw(2) << value; return ss.str(); } // Formats an int value as "%X". std::string String::FormatHexUInt32(UInt32 value) { std::stringstream ss; ss << std::hex << std::uppercase << value; return ss.str(); } // Formats an int value as "%X". std::string String::FormatHexInt(int value) { return FormatHexUInt32(static_cast(value)); } // Formats a byte as "%02X". std::string String::FormatByte(unsigned char value) { std::stringstream ss; ss << std::setfill('0') << std::setw(2) << std::hex << std::uppercase << static_cast(value); return ss.str(); } // Converts the buffer in a stringstream to an std::string, converting NUL // bytes to "\\0" along the way. std::string StringStreamToString(::std::stringstream* ss) { const ::std::string& str = ss->str(); const char* const start = str.c_str(); const char* const end = start + str.length(); std::string result; result.reserve(static_cast(2 * (end - start))); for (const char* ch = start; ch != end; ++ch) { if (*ch == '\0') { result += "\\0"; // Replaces NUL with "\\0"; } else { result += *ch; } } return result; } // Appends the user-supplied message to the Google-Test-generated message. std::string AppendUserMessage(const std::string& gtest_msg, const Message& user_msg) { // Appends the user message if it's non-empty. const std::string user_msg_string = user_msg.GetString(); if (user_msg_string.empty()) { return gtest_msg; } return gtest_msg + "\n" + user_msg_string; } } // namespace internal // class TestResult // Creates an empty TestResult. TestResult::TestResult() : death_test_count_(0), start_timestamp_(0), elapsed_time_(0) {} // D'tor. TestResult::~TestResult() { } // Returns the i-th test part result among all the results. i can // range from 0 to total_part_count() - 1. If i is not in that range, // aborts the program. const TestPartResult& TestResult::GetTestPartResult(int i) const { if (i < 0 || i >= total_part_count()) internal::posix::Abort(); return test_part_results_.at(static_cast(i)); } // Returns the i-th test property. i can range from 0 to // test_property_count() - 1. If i is not in that range, aborts the // program. const TestProperty& TestResult::GetTestProperty(int i) const { if (i < 0 || i >= test_property_count()) internal::posix::Abort(); return test_properties_.at(static_cast(i)); } // Clears the test part results. void TestResult::ClearTestPartResults() { test_part_results_.clear(); } // Adds a test part result to the list. void TestResult::AddTestPartResult(const TestPartResult& test_part_result) { test_part_results_.push_back(test_part_result); } // Adds a test property to the list. If a property with the same key as the // supplied property is already represented, the value of this test_property // replaces the old value for that key. void TestResult::RecordProperty(const std::string& xml_element, const TestProperty& test_property) { if (!ValidateTestProperty(xml_element, test_property)) { return; } internal::MutexLock lock(&test_properites_mutex_); const std::vector::iterator property_with_matching_key = std::find_if(test_properties_.begin(), test_properties_.end(), internal::TestPropertyKeyIs(test_property.key())); if (property_with_matching_key == test_properties_.end()) { test_properties_.push_back(test_property); return; } property_with_matching_key->SetValue(test_property.value()); } // The list of reserved attributes used in the element of XML // output. static const char* const kReservedTestSuitesAttributes[] = { "disabled", "errors", "failures", "name", "random_seed", "tests", "time", "timestamp" }; // The list of reserved attributes used in the element of XML // output. static const char* const kReservedTestSuiteAttributes[] = { "disabled", "errors", "failures", "name", "tests", "time", "timestamp"}; // The list of reserved attributes used in the element of XML output. static const char* const kReservedTestCaseAttributes[] = { "classname", "name", "status", "time", "type_param", "value_param", "file", "line"}; // Use a slightly different set for allowed output to ensure existing tests can // still RecordProperty("result") or "RecordProperty(timestamp") static const char* const kReservedOutputTestCaseAttributes[] = { "classname", "name", "status", "time", "type_param", "value_param", "file", "line", "result", "timestamp"}; template std::vector ArrayAsVector(const char* const (&array)[kSize]) { return std::vector(array, array + kSize); } static std::vector GetReservedAttributesForElement( const std::string& xml_element) { if (xml_element == "testsuites") { return ArrayAsVector(kReservedTestSuitesAttributes); } else if (xml_element == "testsuite") { return ArrayAsVector(kReservedTestSuiteAttributes); } else if (xml_element == "testcase") { return ArrayAsVector(kReservedTestCaseAttributes); } else { GTEST_CHECK_(false) << "Unrecognized xml_element provided: " << xml_element; } // This code is unreachable but some compilers may not realizes that. return std::vector(); } // TODO(jdesprez): Merge the two getReserved attributes once skip is improved static std::vector GetReservedOutputAttributesForElement( const std::string& xml_element) { if (xml_element == "testsuites") { return ArrayAsVector(kReservedTestSuitesAttributes); } else if (xml_element == "testsuite") { return ArrayAsVector(kReservedTestSuiteAttributes); } else if (xml_element == "testcase") { return ArrayAsVector(kReservedOutputTestCaseAttributes); } else { GTEST_CHECK_(false) << "Unrecognized xml_element provided: " << xml_element; } // This code is unreachable but some compilers may not realizes that. return std::vector(); } static std::string FormatWordList(const std::vector& words) { Message word_list; for (size_t i = 0; i < words.size(); ++i) { if (i > 0 && words.size() > 2) { word_list << ", "; } if (i == words.size() - 1) { word_list << "and "; } word_list << "'" << words[i] << "'"; } return word_list.GetString(); } static bool ValidateTestPropertyName( const std::string& property_name, const std::vector& reserved_names) { if (std::find(reserved_names.begin(), reserved_names.end(), property_name) != reserved_names.end()) { ADD_FAILURE() << "Reserved key used in RecordProperty(): " << property_name << " (" << FormatWordList(reserved_names) << " are reserved by " << GTEST_NAME_ << ")"; return false; } return true; } // Adds a failure if the key is a reserved attribute of the element named // xml_element. Returns true if the property is valid. bool TestResult::ValidateTestProperty(const std::string& xml_element, const TestProperty& test_property) { return ValidateTestPropertyName(test_property.key(), GetReservedAttributesForElement(xml_element)); } // Clears the object. void TestResult::Clear() { test_part_results_.clear(); test_properties_.clear(); death_test_count_ = 0; elapsed_time_ = 0; } // Returns true off the test part was skipped. static bool TestPartSkipped(const TestPartResult& result) { return result.skipped(); } // Returns true if and only if the test was skipped. bool TestResult::Skipped() const { return !Failed() && CountIf(test_part_results_, TestPartSkipped) > 0; } // Returns true if and only if the test failed. bool TestResult::Failed() const { for (int i = 0; i < total_part_count(); ++i) { if (GetTestPartResult(i).failed()) return true; } return false; } // Returns true if and only if the test part fatally failed. static bool TestPartFatallyFailed(const TestPartResult& result) { return result.fatally_failed(); } // Returns true if and only if the test fatally failed. bool TestResult::HasFatalFailure() const { return CountIf(test_part_results_, TestPartFatallyFailed) > 0; } // Returns true if and only if the test part non-fatally failed. static bool TestPartNonfatallyFailed(const TestPartResult& result) { return result.nonfatally_failed(); } // Returns true if and only if the test has a non-fatal failure. bool TestResult::HasNonfatalFailure() const { return CountIf(test_part_results_, TestPartNonfatallyFailed) > 0; } // Gets the number of all test parts. This is the sum of the number // of successful test parts and the number of failed test parts. int TestResult::total_part_count() const { return static_cast(test_part_results_.size()); } // Returns the number of the test properties. int TestResult::test_property_count() const { return static_cast(test_properties_.size()); } // class Test // Creates a Test object. // The c'tor saves the states of all flags. Test::Test() : gtest_flag_saver_(new GTEST_FLAG_SAVER_) { } // The d'tor restores the states of all flags. The actual work is // done by the d'tor of the gtest_flag_saver_ field, and thus not // visible here. Test::~Test() { } // Sets up the test fixture. // // A sub-class may override this. void Test::SetUp() { } // Tears down the test fixture. // // A sub-class may override this. void Test::TearDown() { } // Allows user supplied key value pairs to be recorded for later output. void Test::RecordProperty(const std::string& key, const std::string& value) { UnitTest::GetInstance()->RecordProperty(key, value); } // Allows user supplied key value pairs to be recorded for later output. void Test::RecordProperty(const std::string& key, int value) { Message value_message; value_message << value; RecordProperty(key, value_message.GetString().c_str()); } namespace internal { void ReportFailureInUnknownLocation(TestPartResult::Type result_type, const std::string& message) { // This function is a friend of UnitTest and as such has access to // AddTestPartResult. UnitTest::GetInstance()->AddTestPartResult( result_type, nullptr, // No info about the source file where the exception occurred. -1, // We have no info on which line caused the exception. message, ""); // No stack trace, either. } } // namespace internal // Google Test requires all tests in the same test suite to use the same test // fixture class. This function checks if the current test has the // same fixture class as the first test in the current test suite. If // yes, it returns true; otherwise it generates a Google Test failure and // returns false. bool Test::HasSameFixtureClass() { internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); const TestSuite* const test_suite = impl->current_test_suite(); // Info about the first test in the current test suite. const TestInfo* const first_test_info = test_suite->test_info_list()[0]; const internal::TypeId first_fixture_id = first_test_info->fixture_class_id_; const char* const first_test_name = first_test_info->name(); // Info about the current test. const TestInfo* const this_test_info = impl->current_test_info(); const internal::TypeId this_fixture_id = this_test_info->fixture_class_id_; const char* const this_test_name = this_test_info->name(); if (this_fixture_id != first_fixture_id) { // Is the first test defined using TEST? const bool first_is_TEST = first_fixture_id == internal::GetTestTypeId(); // Is this test defined using TEST? const bool this_is_TEST = this_fixture_id == internal::GetTestTypeId(); if (first_is_TEST || this_is_TEST) { // Both TEST and TEST_F appear in same test suite, which is incorrect. // Tell the user how to fix this. // Gets the name of the TEST and the name of the TEST_F. Note // that first_is_TEST and this_is_TEST cannot both be true, as // the fixture IDs are different for the two tests. const char* const TEST_name = first_is_TEST ? first_test_name : this_test_name; const char* const TEST_F_name = first_is_TEST ? this_test_name : first_test_name; ADD_FAILURE() << "All tests in the same test suite must use the same test fixture\n" << "class, so mixing TEST_F and TEST in the same test suite is\n" << "illegal. In test suite " << this_test_info->test_suite_name() << ",\n" << "test " << TEST_F_name << " is defined using TEST_F but\n" << "test " << TEST_name << " is defined using TEST. You probably\n" << "want to change the TEST to TEST_F or move it to another test\n" << "case."; } else { // Two fixture classes with the same name appear in two different // namespaces, which is not allowed. Tell the user how to fix this. ADD_FAILURE() << "All tests in the same test suite must use the same test fixture\n" << "class. However, in test suite " << this_test_info->test_suite_name() << ",\n" << "you defined test " << first_test_name << " and test " << this_test_name << "\n" << "using two different test fixture classes. This can happen if\n" << "the two classes are from different namespaces or translation\n" << "units and have the same name. You should probably rename one\n" << "of the classes to put the tests into different test suites."; } return false; } return true; } #if GTEST_HAS_SEH // Adds an "exception thrown" fatal failure to the current test. This // function returns its result via an output parameter pointer because VC++ // prohibits creation of objects with destructors on stack in functions // using __try (see error C2712). static std::string* FormatSehExceptionMessage(DWORD exception_code, const char* location) { Message message; message << "SEH exception with code 0x" << std::setbase(16) << exception_code << std::setbase(10) << " thrown in " << location << "."; return new std::string(message.GetString()); } #endif // GTEST_HAS_SEH namespace internal { #if GTEST_HAS_EXCEPTIONS // Adds an "exception thrown" fatal failure to the current test. static std::string FormatCxxExceptionMessage(const char* description, const char* location) { Message message; if (description != nullptr) { message << "C++ exception with description \"" << description << "\""; } else { message << "Unknown C++ exception"; } message << " thrown in " << location << "."; return message.GetString(); } static std::string PrintTestPartResultToString( const TestPartResult& test_part_result); GoogleTestFailureException::GoogleTestFailureException( const TestPartResult& failure) : ::std::runtime_error(PrintTestPartResultToString(failure).c_str()) {} #endif // GTEST_HAS_EXCEPTIONS // We put these helper functions in the internal namespace as IBM's xlC // compiler rejects the code if they were declared static. // Runs the given method and handles SEH exceptions it throws, when // SEH is supported; returns the 0-value for type Result in case of an // SEH exception. (Microsoft compilers cannot handle SEH and C++ // exceptions in the same function. Therefore, we provide a separate // wrapper function for handling SEH exceptions.) template Result HandleSehExceptionsInMethodIfSupported( T* object, Result (T::*method)(), const char* location) { #if GTEST_HAS_SEH __try { return (object->*method)(); } __except (internal::UnitTestOptions::GTestShouldProcessSEH( // NOLINT GetExceptionCode())) { // We create the exception message on the heap because VC++ prohibits // creation of objects with destructors on stack in functions using __try // (see error C2712). std::string* exception_message = FormatSehExceptionMessage( GetExceptionCode(), location); internal::ReportFailureInUnknownLocation(TestPartResult::kFatalFailure, *exception_message); delete exception_message; return static_cast(0); } #else (void)location; return (object->*method)(); #endif // GTEST_HAS_SEH } // Runs the given method and catches and reports C++ and/or SEH-style // exceptions, if they are supported; returns the 0-value for type // Result in case of an SEH exception. template Result HandleExceptionsInMethodIfSupported( T* object, Result (T::*method)(), const char* location) { // NOTE: The user code can affect the way in which Google Test handles // exceptions by setting GTEST_FLAG(catch_exceptions), but only before // RUN_ALL_TESTS() starts. It is technically possible to check the flag // after the exception is caught and either report or re-throw the // exception based on the flag's value: // // try { // // Perform the test method. // } catch (...) { // if (GTEST_FLAG(catch_exceptions)) // // Report the exception as failure. // else // throw; // Re-throws the original exception. // } // // However, the purpose of this flag is to allow the program to drop into // the debugger when the exception is thrown. On most platforms, once the // control enters the catch block, the exception origin information is // lost and the debugger will stop the program at the point of the // re-throw in this function -- instead of at the point of the original // throw statement in the code under test. For this reason, we perform // the check early, sacrificing the ability to affect Google Test's // exception handling in the method where the exception is thrown. if (internal::GetUnitTestImpl()->catch_exceptions()) { #if GTEST_HAS_EXCEPTIONS try { return HandleSehExceptionsInMethodIfSupported(object, method, location); } catch (const AssertionException&) { // NOLINT // This failure was reported already. } catch (const internal::GoogleTestFailureException&) { // NOLINT // This exception type can only be thrown by a failed Google // Test assertion with the intention of letting another testing // framework catch it. Therefore we just re-throw it. throw; } catch (const std::exception& e) { // NOLINT internal::ReportFailureInUnknownLocation( TestPartResult::kFatalFailure, FormatCxxExceptionMessage(e.what(), location)); } catch (...) { // NOLINT internal::ReportFailureInUnknownLocation( TestPartResult::kFatalFailure, FormatCxxExceptionMessage(nullptr, location)); } return static_cast(0); #else return HandleSehExceptionsInMethodIfSupported(object, method, location); #endif // GTEST_HAS_EXCEPTIONS } else { return (object->*method)(); } } } // namespace internal // Runs the test and updates the test result. void Test::Run() { if (!HasSameFixtureClass()) return; internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported(this, &Test::SetUp, "SetUp()"); // We will run the test only if SetUp() was successful and didn't call // GTEST_SKIP(). if (!HasFatalFailure() && !IsSkipped()) { impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( this, &Test::TestBody, "the test body"); } // However, we want to clean up as much as possible. Hence we will // always call TearDown(), even if SetUp() or the test body has // failed. impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( this, &Test::TearDown, "TearDown()"); } // Returns true if and only if the current test has a fatal failure. bool Test::HasFatalFailure() { return internal::GetUnitTestImpl()->current_test_result()->HasFatalFailure(); } // Returns true if and only if the current test has a non-fatal failure. bool Test::HasNonfatalFailure() { return internal::GetUnitTestImpl()->current_test_result()-> HasNonfatalFailure(); } // Returns true if and only if the current test was skipped. bool Test::IsSkipped() { return internal::GetUnitTestImpl()->current_test_result()->Skipped(); } // class TestInfo // Constructs a TestInfo object. It assumes ownership of the test factory // object. TestInfo::TestInfo(const std::string& a_test_suite_name, const std::string& a_name, const char* a_type_param, const char* a_value_param, internal::CodeLocation a_code_location, internal::TypeId fixture_class_id, internal::TestFactoryBase* factory) : test_suite_name_(a_test_suite_name), name_(a_name), type_param_(a_type_param ? new std::string(a_type_param) : nullptr), value_param_(a_value_param ? new std::string(a_value_param) : nullptr), location_(a_code_location), fixture_class_id_(fixture_class_id), should_run_(false), is_disabled_(false), matches_filter_(false), factory_(factory), result_() {} // Destructs a TestInfo object. TestInfo::~TestInfo() { delete factory_; } namespace internal { // Creates a new TestInfo object and registers it with Google Test; // returns the created object. // // Arguments: // // test_suite_name: name of the test suite // name: name of the test // type_param: the name of the test's type parameter, or NULL if // this is not a typed or a type-parameterized test. // value_param: text representation of the test's value parameter, // or NULL if this is not a value-parameterized test. // code_location: code location where the test is defined // fixture_class_id: ID of the test fixture class // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite // factory: pointer to the factory that creates a test object. // The newly created TestInfo instance will assume // ownership of the factory object. TestInfo* MakeAndRegisterTestInfo( const char* test_suite_name, const char* name, const char* type_param, const char* value_param, CodeLocation code_location, TypeId fixture_class_id, SetUpTestSuiteFunc set_up_tc, TearDownTestSuiteFunc tear_down_tc, TestFactoryBase* factory) { TestInfo* const test_info = new TestInfo(test_suite_name, name, type_param, value_param, code_location, fixture_class_id, factory); GetUnitTestImpl()->AddTestInfo(set_up_tc, tear_down_tc, test_info); return test_info; } void ReportInvalidTestSuiteType(const char* test_suite_name, CodeLocation code_location) { Message errors; errors << "Attempted redefinition of test suite " << test_suite_name << ".\n" << "All tests in the same test suite must use the same test fixture\n" << "class. However, in test suite " << test_suite_name << ", you tried\n" << "to define a test using a fixture class different from the one\n" << "used earlier. This can happen if the two fixture classes are\n" << "from different namespaces and have the same name. You should\n" << "probably rename one of the classes to put the tests into different\n" << "test suites."; GTEST_LOG_(ERROR) << FormatFileLocation(code_location.file.c_str(), code_location.line) << " " << errors.GetString(); } } // namespace internal namespace { // A predicate that checks the test name of a TestInfo against a known // value. // // This is used for implementation of the TestSuite class only. We put // it in the anonymous namespace to prevent polluting the outer // namespace. // // TestNameIs is copyable. class TestNameIs { public: // Constructor. // // TestNameIs has NO default constructor. explicit TestNameIs(const char* name) : name_(name) {} // Returns true if and only if the test name of test_info matches name_. bool operator()(const TestInfo * test_info) const { return test_info && test_info->name() == name_; } private: std::string name_; }; } // namespace namespace internal { // This method expands all parameterized tests registered with macros TEST_P // and INSTANTIATE_TEST_SUITE_P into regular tests and registers those. // This will be done just once during the program runtime. void UnitTestImpl::RegisterParameterizedTests() { if (!parameterized_tests_registered_) { parameterized_test_registry_.RegisterTests(); parameterized_tests_registered_ = true; } } } // namespace internal // Creates the test object, runs it, records its result, and then // deletes it. void TestInfo::Run() { if (!should_run_) return; // Tells UnitTest where to store test result. internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); impl->set_current_test_info(this); TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); // Notifies the unit test event listeners that a test is about to start. repeater->OnTestStart(*this); const TimeInMillis start = internal::GetTimeInMillis(); impl->os_stack_trace_getter()->UponLeavingGTest(); // Creates the test object. Test* const test = internal::HandleExceptionsInMethodIfSupported( factory_, &internal::TestFactoryBase::CreateTest, "the test fixture's constructor"); // Runs the test if the constructor didn't generate a fatal failure or invoke // GTEST_SKIP(). // Note that the object will not be null if (!Test::HasFatalFailure() && !Test::IsSkipped()) { // This doesn't throw as all user code that can throw are wrapped into // exception handling code. test->Run(); } if (test != nullptr) { // Deletes the test object. impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( test, &Test::DeleteSelf_, "the test fixture's destructor"); } result_.set_start_timestamp(start); result_.set_elapsed_time(internal::GetTimeInMillis() - start); // Notifies the unit test event listener that a test has just finished. repeater->OnTestEnd(*this); // Tells UnitTest to stop associating assertion results to this // test. impl->set_current_test_info(nullptr); } // class TestSuite // Gets the number of successful tests in this test suite. int TestSuite::successful_test_count() const { return CountIf(test_info_list_, TestPassed); } // Gets the number of successful tests in this test suite. int TestSuite::skipped_test_count() const { return CountIf(test_info_list_, TestSkipped); } // Gets the number of failed tests in this test suite. int TestSuite::failed_test_count() const { return CountIf(test_info_list_, TestFailed); } // Gets the number of disabled tests that will be reported in the XML report. int TestSuite::reportable_disabled_test_count() const { return CountIf(test_info_list_, TestReportableDisabled); } // Gets the number of disabled tests in this test suite. int TestSuite::disabled_test_count() const { return CountIf(test_info_list_, TestDisabled); } // Gets the number of tests to be printed in the XML report. int TestSuite::reportable_test_count() const { return CountIf(test_info_list_, TestReportable); } // Get the number of tests in this test suite that should run. int TestSuite::test_to_run_count() const { return CountIf(test_info_list_, ShouldRunTest); } // Gets the number of all tests. int TestSuite::total_test_count() const { return static_cast(test_info_list_.size()); } // Creates a TestSuite with the given name. // // Arguments: // // name: name of the test suite // a_type_param: the name of the test suite's type parameter, or NULL if // this is not a typed or a type-parameterized test suite. // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite TestSuite::TestSuite(const char* a_name, const char* a_type_param, internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc) : name_(a_name), type_param_(a_type_param ? new std::string(a_type_param) : nullptr), set_up_tc_(set_up_tc), tear_down_tc_(tear_down_tc), should_run_(false), start_timestamp_(0), elapsed_time_(0) {} // Destructor of TestSuite. TestSuite::~TestSuite() { // Deletes every Test in the collection. ForEach(test_info_list_, internal::Delete); } // Returns the i-th test among all the tests. i can range from 0 to // total_test_count() - 1. If i is not in that range, returns NULL. const TestInfo* TestSuite::GetTestInfo(int i) const { const int index = GetElementOr(test_indices_, i, -1); return index < 0 ? nullptr : test_info_list_[static_cast(index)]; } // Returns the i-th test among all the tests. i can range from 0 to // total_test_count() - 1. If i is not in that range, returns NULL. TestInfo* TestSuite::GetMutableTestInfo(int i) { const int index = GetElementOr(test_indices_, i, -1); return index < 0 ? nullptr : test_info_list_[static_cast(index)]; } // Adds a test to this test suite. Will delete the test upon // destruction of the TestSuite object. void TestSuite::AddTestInfo(TestInfo* test_info) { test_info_list_.push_back(test_info); test_indices_.push_back(static_cast(test_indices_.size())); } // Runs every test in this TestSuite. void TestSuite::Run() { if (!should_run_) return; internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); impl->set_current_test_suite(this); TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); // Call both legacy and the new API repeater->OnTestSuiteStart(*this); // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI repeater->OnTestCaseStart(*this); #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( this, &TestSuite::RunSetUpTestSuite, "SetUpTestSuite()"); start_timestamp_ = internal::GetTimeInMillis(); for (int i = 0; i < total_test_count(); i++) { GetMutableTestInfo(i)->Run(); } elapsed_time_ = internal::GetTimeInMillis() - start_timestamp_; impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( this, &TestSuite::RunTearDownTestSuite, "TearDownTestSuite()"); // Call both legacy and the new API repeater->OnTestSuiteEnd(*this); // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI repeater->OnTestCaseEnd(*this); #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI impl->set_current_test_suite(nullptr); } // Clears the results of all tests in this test suite. void TestSuite::ClearResult() { ad_hoc_test_result_.Clear(); ForEach(test_info_list_, TestInfo::ClearTestResult); } // Shuffles the tests in this test suite. void TestSuite::ShuffleTests(internal::Random* random) { Shuffle(random, &test_indices_); } // Restores the test order to before the first shuffle. void TestSuite::UnshuffleTests() { for (size_t i = 0; i < test_indices_.size(); i++) { test_indices_[i] = static_cast(i); } } // Formats a countable noun. Depending on its quantity, either the // singular form or the plural form is used. e.g. // // FormatCountableNoun(1, "formula", "formuli") returns "1 formula". // FormatCountableNoun(5, "book", "books") returns "5 books". static std::string FormatCountableNoun(int count, const char * singular_form, const char * plural_form) { return internal::StreamableToString(count) + " " + (count == 1 ? singular_form : plural_form); } // Formats the count of tests. static std::string FormatTestCount(int test_count) { return FormatCountableNoun(test_count, "test", "tests"); } // Formats the count of test suites. static std::string FormatTestSuiteCount(int test_suite_count) { return FormatCountableNoun(test_suite_count, "test suite", "test suites"); } // Converts a TestPartResult::Type enum to human-friendly string // representation. Both kNonFatalFailure and kFatalFailure are translated // to "Failure", as the user usually doesn't care about the difference // between the two when viewing the test result. static const char * TestPartResultTypeToString(TestPartResult::Type type) { switch (type) { case TestPartResult::kSkip: return "Skipped"; case TestPartResult::kSuccess: return "Success"; case TestPartResult::kNonFatalFailure: case TestPartResult::kFatalFailure: #ifdef _MSC_VER return "error: "; #else return "Failure\n"; #endif default: return "Unknown result type"; } } namespace internal { // Prints a TestPartResult to an std::string. static std::string PrintTestPartResultToString( const TestPartResult& test_part_result) { return (Message() << internal::FormatFileLocation(test_part_result.file_name(), test_part_result.line_number()) << " " << TestPartResultTypeToString(test_part_result.type()) << test_part_result.message()).GetString(); } // Prints a TestPartResult. static void PrintTestPartResult(const TestPartResult& test_part_result) { const std::string& result = PrintTestPartResultToString(test_part_result); printf("%s\n", result.c_str()); fflush(stdout); // If the test program runs in Visual Studio or a debugger, the // following statements add the test part result message to the Output // window such that the user can double-click on it to jump to the // corresponding source code location; otherwise they do nothing. #if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE // We don't call OutputDebugString*() on Windows Mobile, as printing // to stdout is done by OutputDebugString() there already - we don't // want the same message printed twice. ::OutputDebugStringA(result.c_str()); ::OutputDebugStringA("\n"); #endif } // class PrettyUnitTestResultPrinter #if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \ !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT && !GTEST_OS_WINDOWS_MINGW // Returns the character attribute for the given color. static WORD GetColorAttribute(GTestColor color) { switch (color) { case COLOR_RED: return FOREGROUND_RED; case COLOR_GREEN: return FOREGROUND_GREEN; case COLOR_YELLOW: return FOREGROUND_RED | FOREGROUND_GREEN; default: return 0; } } static int GetBitOffset(WORD color_mask) { if (color_mask == 0) return 0; int bitOffset = 0; while ((color_mask & 1) == 0) { color_mask >>= 1; ++bitOffset; } return bitOffset; } static WORD GetNewColor(GTestColor color, WORD old_color_attrs) { // Let's reuse the BG static const WORD background_mask = BACKGROUND_BLUE | BACKGROUND_GREEN | BACKGROUND_RED | BACKGROUND_INTENSITY; static const WORD foreground_mask = FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_INTENSITY; const WORD existing_bg = old_color_attrs & background_mask; WORD new_color = GetColorAttribute(color) | existing_bg | FOREGROUND_INTENSITY; static const int bg_bitOffset = GetBitOffset(background_mask); static const int fg_bitOffset = GetBitOffset(foreground_mask); if (((new_color & background_mask) >> bg_bitOffset) == ((new_color & foreground_mask) >> fg_bitOffset)) { new_color ^= FOREGROUND_INTENSITY; // invert intensity } return new_color; } #else // Returns the ANSI color code for the given color. COLOR_DEFAULT is // an invalid input. static const char* GetAnsiColorCode(GTestColor color) { switch (color) { case COLOR_RED: return "1"; case COLOR_GREEN: return "2"; case COLOR_YELLOW: return "3"; default: return nullptr; } } #endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE // Returns true if and only if Google Test should use colors in the output. bool ShouldUseColor(bool stdout_is_tty) { const char* const gtest_color = GTEST_FLAG(color).c_str(); if (String::CaseInsensitiveCStringEquals(gtest_color, "auto")) { #if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MINGW // On Windows the TERM variable is usually not set, but the // console there does support colors. return stdout_is_tty; #else // On non-Windows platforms, we rely on the TERM variable. const char* const term = posix::GetEnv("TERM"); const bool term_supports_color = String::CStringEquals(term, "xterm") || String::CStringEquals(term, "xterm-color") || String::CStringEquals(term, "xterm-256color") || String::CStringEquals(term, "screen") || String::CStringEquals(term, "screen-256color") || String::CStringEquals(term, "tmux") || String::CStringEquals(term, "tmux-256color") || String::CStringEquals(term, "rxvt-unicode") || String::CStringEquals(term, "rxvt-unicode-256color") || String::CStringEquals(term, "linux") || String::CStringEquals(term, "cygwin"); return stdout_is_tty && term_supports_color; #endif // GTEST_OS_WINDOWS } return String::CaseInsensitiveCStringEquals(gtest_color, "yes") || String::CaseInsensitiveCStringEquals(gtest_color, "true") || String::CaseInsensitiveCStringEquals(gtest_color, "t") || String::CStringEquals(gtest_color, "1"); // We take "yes", "true", "t", and "1" as meaning "yes". If the // value is neither one of these nor "auto", we treat it as "no" to // be conservative. } // Helpers for printing colored strings to stdout. Note that on Windows, we // cannot simply emit special characters and have the terminal change colors. // This routine must actually emit the characters rather than return a string // that would be colored when printed, as can be done on Linux. void ColoredPrintf(GTestColor color, const char* fmt, ...) { va_list args; va_start(args, fmt); #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_ZOS || GTEST_OS_IOS || \ GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT || defined(ESP_PLATFORM) const bool use_color = AlwaysFalse(); #else static const bool in_color_mode = ShouldUseColor(posix::IsATTY(posix::FileNo(stdout)) != 0); const bool use_color = in_color_mode && (color != COLOR_DEFAULT); #endif // GTEST_OS_WINDOWS_MOBILE || GTEST_OS_ZOS if (!use_color) { vprintf(fmt, args); va_end(args); return; } #if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \ !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT && !GTEST_OS_WINDOWS_MINGW const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); // Gets the current text color. CONSOLE_SCREEN_BUFFER_INFO buffer_info; GetConsoleScreenBufferInfo(stdout_handle, &buffer_info); const WORD old_color_attrs = buffer_info.wAttributes; const WORD new_color = GetNewColor(color, old_color_attrs); // We need to flush the stream buffers into the console before each // SetConsoleTextAttribute call lest it affect the text that is already // printed but has not yet reached the console. fflush(stdout); SetConsoleTextAttribute(stdout_handle, new_color); vprintf(fmt, args); fflush(stdout); // Restores the text color. SetConsoleTextAttribute(stdout_handle, old_color_attrs); #else printf("\033[0;3%sm", GetAnsiColorCode(color)); vprintf(fmt, args); printf("\033[m"); // Resets the terminal to default. #endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE va_end(args); } // Text printed in Google Test's text output and --gtest_list_tests // output to label the type parameter and value parameter for a test. static const char kTypeParamLabel[] = "TypeParam"; static const char kValueParamLabel[] = "GetParam()"; static void PrintFullTestCommentIfPresent(const TestInfo& test_info) { const char* const type_param = test_info.type_param(); const char* const value_param = test_info.value_param(); if (type_param != nullptr || value_param != nullptr) { printf(", where "); if (type_param != nullptr) { printf("%s = %s", kTypeParamLabel, type_param); if (value_param != nullptr) printf(" and "); } if (value_param != nullptr) { printf("%s = %s", kValueParamLabel, value_param); } } } // This class implements the TestEventListener interface. // // Class PrettyUnitTestResultPrinter is copyable. class PrettyUnitTestResultPrinter : public TestEventListener { public: PrettyUnitTestResultPrinter() {} static void PrintTestName(const char* test_suite, const char* test) { printf("%s.%s", test_suite, test); } // The following methods override what's in the TestEventListener class. void OnTestProgramStart(const UnitTest& /*unit_test*/) override {} void OnTestIterationStart(const UnitTest& unit_test, int iteration) override; void OnEnvironmentsSetUpStart(const UnitTest& unit_test) override; void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) override {} #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnTestCaseStart(const TestCase& test_case) override; #else void OnTestSuiteStart(const TestSuite& test_suite) override; #endif // OnTestCaseStart void OnTestStart(const TestInfo& test_info) override; void OnTestPartResult(const TestPartResult& result) override; void OnTestEnd(const TestInfo& test_info) override; #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnTestCaseEnd(const TestCase& test_case) override; #else void OnTestSuiteEnd(const TestSuite& test_suite) override; #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnEnvironmentsTearDownStart(const UnitTest& unit_test) override; void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) override {} void OnTestIterationEnd(const UnitTest& unit_test, int iteration) override; void OnTestProgramEnd(const UnitTest& /*unit_test*/) override {} private: static void PrintFailedTests(const UnitTest& unit_test); static void PrintSkippedTests(const UnitTest& unit_test); }; // Fired before each iteration of tests starts. void PrettyUnitTestResultPrinter::OnTestIterationStart( const UnitTest& unit_test, int iteration) { if (GTEST_FLAG(repeat) != 1) printf("\nRepeating all tests (iteration %d) . . .\n\n", iteration + 1); const char* const filter = GTEST_FLAG(filter).c_str(); // Prints the filter if it's not *. This reminds the user that some // tests may be skipped. if (!String::CStringEquals(filter, kUniversalFilter)) { ColoredPrintf(COLOR_YELLOW, "Note: %s filter = %s\n", GTEST_NAME_, filter); } if (internal::ShouldShard(kTestTotalShards, kTestShardIndex, false)) { const Int32 shard_index = Int32FromEnvOrDie(kTestShardIndex, -1); ColoredPrintf(COLOR_YELLOW, "Note: This is test shard %d of %s.\n", static_cast(shard_index) + 1, internal::posix::GetEnv(kTestTotalShards)); } if (GTEST_FLAG(shuffle)) { ColoredPrintf(COLOR_YELLOW, "Note: Randomizing tests' orders with a seed of %d .\n", unit_test.random_seed()); } ColoredPrintf(COLOR_GREEN, "[==========] "); printf("Running %s from %s.\n", FormatTestCount(unit_test.test_to_run_count()).c_str(), FormatTestSuiteCount(unit_test.test_suite_to_run_count()).c_str()); fflush(stdout); } void PrettyUnitTestResultPrinter::OnEnvironmentsSetUpStart( const UnitTest& /*unit_test*/) { ColoredPrintf(COLOR_GREEN, "[----------] "); printf("Global test environment set-up.\n"); fflush(stdout); } #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void PrettyUnitTestResultPrinter::OnTestCaseStart(const TestCase& test_case) { const std::string counts = FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); ColoredPrintf(COLOR_GREEN, "[----------] "); printf("%s from %s", counts.c_str(), test_case.name()); if (test_case.type_param() == nullptr) { printf("\n"); } else { printf(", where %s = %s\n", kTypeParamLabel, test_case.type_param()); } fflush(stdout); } #else void PrettyUnitTestResultPrinter::OnTestSuiteStart( const TestSuite& test_suite) { const std::string counts = FormatCountableNoun(test_suite.test_to_run_count(), "test", "tests"); ColoredPrintf(COLOR_GREEN, "[----------] "); printf("%s from %s", counts.c_str(), test_suite.name()); if (test_suite.type_param() == nullptr) { printf("\n"); } else { printf(", where %s = %s\n", kTypeParamLabel, test_suite.type_param()); } fflush(stdout); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void PrettyUnitTestResultPrinter::OnTestStart(const TestInfo& test_info) { ColoredPrintf(COLOR_GREEN, "[ RUN ] "); PrintTestName(test_info.test_suite_name(), test_info.name()); printf("\n"); fflush(stdout); } // Called after an assertion failure. void PrettyUnitTestResultPrinter::OnTestPartResult( const TestPartResult& result) { switch (result.type()) { // If the test part succeeded, or was skipped, // we don't need to do anything. case TestPartResult::kSuccess: return; default: // Print failure message from the assertion // (e.g. expected this and got that). PrintTestPartResult(result); fflush(stdout); } } void PrettyUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { if (test_info.result()->Passed()) { ColoredPrintf(COLOR_GREEN, "[ OK ] "); } else if (test_info.result()->Skipped()) { ColoredPrintf(COLOR_GREEN, "[ SKIPPED ] "); } else { ColoredPrintf(COLOR_RED, "[ FAILED ] "); } PrintTestName(test_info.test_suite_name(), test_info.name()); if (test_info.result()->Failed()) PrintFullTestCommentIfPresent(test_info); if (GTEST_FLAG(print_time)) { printf(" (%s ms)\n", internal::StreamableToString( test_info.result()->elapsed_time()).c_str()); } else { printf("\n"); } fflush(stdout); } #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void PrettyUnitTestResultPrinter::OnTestCaseEnd(const TestCase& test_case) { if (!GTEST_FLAG(print_time)) return; const std::string counts = FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); ColoredPrintf(COLOR_GREEN, "[----------] "); printf("%s from %s (%s ms total)\n\n", counts.c_str(), test_case.name(), internal::StreamableToString(test_case.elapsed_time()).c_str()); fflush(stdout); } #else void PrettyUnitTestResultPrinter::OnTestSuiteEnd(const TestSuite& test_suite) { if (!GTEST_FLAG(print_time)) return; const std::string counts = FormatCountableNoun(test_suite.test_to_run_count(), "test", "tests"); ColoredPrintf(COLOR_GREEN, "[----------] "); printf("%s from %s (%s ms total)\n\n", counts.c_str(), test_suite.name(), internal::StreamableToString(test_suite.elapsed_time()).c_str()); fflush(stdout); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void PrettyUnitTestResultPrinter::OnEnvironmentsTearDownStart( const UnitTest& /*unit_test*/) { ColoredPrintf(COLOR_GREEN, "[----------] "); printf("Global test environment tear-down\n"); fflush(stdout); } // Internal helper for printing the list of failed tests. void PrettyUnitTestResultPrinter::PrintFailedTests(const UnitTest& unit_test) { const int failed_test_count = unit_test.failed_test_count(); if (failed_test_count == 0) { return; } for (int i = 0; i < unit_test.total_test_suite_count(); ++i) { const TestSuite& test_suite = *unit_test.GetTestSuite(i); if (!test_suite.should_run() || (test_suite.failed_test_count() == 0)) { continue; } for (int j = 0; j < test_suite.total_test_count(); ++j) { const TestInfo& test_info = *test_suite.GetTestInfo(j); if (!test_info.should_run() || !test_info.result()->Failed()) { continue; } ColoredPrintf(COLOR_RED, "[ FAILED ] "); printf("%s.%s", test_suite.name(), test_info.name()); PrintFullTestCommentIfPresent(test_info); printf("\n"); } } } // Internal helper for printing the list of skipped tests. void PrettyUnitTestResultPrinter::PrintSkippedTests(const UnitTest& unit_test) { const int skipped_test_count = unit_test.skipped_test_count(); if (skipped_test_count == 0) { return; } for (int i = 0; i < unit_test.total_test_suite_count(); ++i) { const TestSuite& test_suite = *unit_test.GetTestSuite(i); if (!test_suite.should_run() || (test_suite.skipped_test_count() == 0)) { continue; } for (int j = 0; j < test_suite.total_test_count(); ++j) { const TestInfo& test_info = *test_suite.GetTestInfo(j); if (!test_info.should_run() || !test_info.result()->Skipped()) { continue; } ColoredPrintf(COLOR_GREEN, "[ SKIPPED ] "); printf("%s.%s", test_suite.name(), test_info.name()); printf("\n"); } } } void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, int /*iteration*/) { ColoredPrintf(COLOR_GREEN, "[==========] "); printf("%s from %s ran.", FormatTestCount(unit_test.test_to_run_count()).c_str(), FormatTestSuiteCount(unit_test.test_suite_to_run_count()).c_str()); if (GTEST_FLAG(print_time)) { printf(" (%s ms total)", internal::StreamableToString(unit_test.elapsed_time()).c_str()); } printf("\n"); ColoredPrintf(COLOR_GREEN, "[ PASSED ] "); printf("%s.\n", FormatTestCount(unit_test.successful_test_count()).c_str()); const int skipped_test_count = unit_test.skipped_test_count(); if (skipped_test_count > 0) { ColoredPrintf(COLOR_GREEN, "[ SKIPPED ] "); if (GTEST_FLAG(print_skipped)) { printf("%s, listed below:\n", FormatTestCount(skipped_test_count).c_str()); PrintSkippedTests(unit_test); } else { printf("%s.\n", FormatTestCount(skipped_test_count).c_str()); } } int num_failures = unit_test.failed_test_count(); if (!unit_test.Passed()) { const int failed_test_count = unit_test.failed_test_count(); ColoredPrintf(COLOR_RED, "[ FAILED ] "); printf("%s, listed below:\n", FormatTestCount(failed_test_count).c_str()); PrintFailedTests(unit_test); printf("\n%2d FAILED %s\n", num_failures, num_failures == 1 ? "TEST" : "TESTS"); } int num_disabled = unit_test.reportable_disabled_test_count(); if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { if (!num_failures) { printf("\n"); // Add a spacer if no FAILURE banner is displayed. } ColoredPrintf(COLOR_YELLOW, " YOU HAVE %d DISABLED %s\n\n", num_disabled, num_disabled == 1 ? "TEST" : "TESTS"); } // Ensure that Google Test output is printed before, e.g., heapchecker output. fflush(stdout); } // End PrettyUnitTestResultPrinter // class TestEventRepeater // // This class forwards events to other event listeners. class TestEventRepeater : public TestEventListener { public: TestEventRepeater() : forwarding_enabled_(true) {} ~TestEventRepeater() override; void Append(TestEventListener *listener); TestEventListener* Release(TestEventListener* listener); // Controls whether events will be forwarded to listeners_. Set to false // in death test child processes. bool forwarding_enabled() const { return forwarding_enabled_; } void set_forwarding_enabled(bool enable) { forwarding_enabled_ = enable; } void OnTestProgramStart(const UnitTest& unit_test) override; void OnTestIterationStart(const UnitTest& unit_test, int iteration) override; void OnEnvironmentsSetUpStart(const UnitTest& unit_test) override; void OnEnvironmentsSetUpEnd(const UnitTest& unit_test) override; // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnTestCaseStart(const TestSuite& parameter) override; #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnTestSuiteStart(const TestSuite& parameter) override; void OnTestStart(const TestInfo& test_info) override; void OnTestPartResult(const TestPartResult& result) override; void OnTestEnd(const TestInfo& test_info) override; // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnTestCaseEnd(const TestCase& parameter) override; #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void OnTestSuiteEnd(const TestSuite& parameter) override; void OnEnvironmentsTearDownStart(const UnitTest& unit_test) override; void OnEnvironmentsTearDownEnd(const UnitTest& unit_test) override; void OnTestIterationEnd(const UnitTest& unit_test, int iteration) override; void OnTestProgramEnd(const UnitTest& unit_test) override; private: // Controls whether events will be forwarded to listeners_. Set to false // in death test child processes. bool forwarding_enabled_; // The list of listeners that receive events. std::vector listeners_; GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventRepeater); }; TestEventRepeater::~TestEventRepeater() { ForEach(listeners_, Delete); } void TestEventRepeater::Append(TestEventListener *listener) { listeners_.push_back(listener); } TestEventListener* TestEventRepeater::Release(TestEventListener *listener) { for (size_t i = 0; i < listeners_.size(); ++i) { if (listeners_[i] == listener) { listeners_.erase(listeners_.begin() + static_cast(i)); return listener; } } return nullptr; } // Since most methods are very similar, use macros to reduce boilerplate. // This defines a member that forwards the call to all listeners. #define GTEST_REPEATER_METHOD_(Name, Type) \ void TestEventRepeater::Name(const Type& parameter) { \ if (forwarding_enabled_) { \ for (size_t i = 0; i < listeners_.size(); i++) { \ listeners_[i]->Name(parameter); \ } \ } \ } // This defines a member that forwards the call to all listeners in reverse // order. #define GTEST_REVERSE_REPEATER_METHOD_(Name, Type) \ void TestEventRepeater::Name(const Type& parameter) { \ if (forwarding_enabled_) { \ for (size_t i = listeners_.size(); i != 0; i--) { \ listeners_[i - 1]->Name(parameter); \ } \ } \ } GTEST_REPEATER_METHOD_(OnTestProgramStart, UnitTest) GTEST_REPEATER_METHOD_(OnEnvironmentsSetUpStart, UnitTest) // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ GTEST_REPEATER_METHOD_(OnTestCaseStart, TestSuite) #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ GTEST_REPEATER_METHOD_(OnTestSuiteStart, TestSuite) GTEST_REPEATER_METHOD_(OnTestStart, TestInfo) GTEST_REPEATER_METHOD_(OnTestPartResult, TestPartResult) GTEST_REPEATER_METHOD_(OnEnvironmentsTearDownStart, UnitTest) GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsSetUpEnd, UnitTest) GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsTearDownEnd, UnitTest) GTEST_REVERSE_REPEATER_METHOD_(OnTestEnd, TestInfo) // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ GTEST_REVERSE_REPEATER_METHOD_(OnTestCaseEnd, TestSuite) #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ GTEST_REVERSE_REPEATER_METHOD_(OnTestSuiteEnd, TestSuite) GTEST_REVERSE_REPEATER_METHOD_(OnTestProgramEnd, UnitTest) #undef GTEST_REPEATER_METHOD_ #undef GTEST_REVERSE_REPEATER_METHOD_ void TestEventRepeater::OnTestIterationStart(const UnitTest& unit_test, int iteration) { if (forwarding_enabled_) { for (size_t i = 0; i < listeners_.size(); i++) { listeners_[i]->OnTestIterationStart(unit_test, iteration); } } } void TestEventRepeater::OnTestIterationEnd(const UnitTest& unit_test, int iteration) { if (forwarding_enabled_) { for (size_t i = listeners_.size(); i > 0; i--) { listeners_[i - 1]->OnTestIterationEnd(unit_test, iteration); } } } // End TestEventRepeater // This class generates an XML output file. class XmlUnitTestResultPrinter : public EmptyTestEventListener { public: explicit XmlUnitTestResultPrinter(const char* output_file); void OnTestIterationEnd(const UnitTest& unit_test, int iteration) override; void ListTestsMatchingFilter(const std::vector& test_suites); // Prints an XML summary of all unit tests. static void PrintXmlTestsList(std::ostream* stream, const std::vector& test_suites); private: // Is c a whitespace character that is normalized to a space character // when it appears in an XML attribute value? static bool IsNormalizableWhitespace(char c) { return c == 0x9 || c == 0xA || c == 0xD; } // May c appear in a well-formed XML document? static bool IsValidXmlCharacter(char c) { return IsNormalizableWhitespace(c) || c >= 0x20; } // Returns an XML-escaped copy of the input string str. If // is_attribute is true, the text is meant to appear as an attribute // value, and normalizable whitespace is preserved by replacing it // with character references. static std::string EscapeXml(const std::string& str, bool is_attribute); // Returns the given string with all characters invalid in XML removed. static std::string RemoveInvalidXmlCharacters(const std::string& str); // Convenience wrapper around EscapeXml when str is an attribute value. static std::string EscapeXmlAttribute(const std::string& str) { return EscapeXml(str, true); } // Convenience wrapper around EscapeXml when str is not an attribute value. static std::string EscapeXmlText(const char* str) { return EscapeXml(str, false); } // Verifies that the given attribute belongs to the given element and // streams the attribute as XML. static void OutputXmlAttribute(std::ostream* stream, const std::string& element_name, const std::string& name, const std::string& value); // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. static void OutputXmlCDataSection(::std::ostream* stream, const char* data); // Streams an XML representation of a TestInfo object. static void OutputXmlTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info); // Prints an XML representation of a TestSuite object static void PrintXmlTestSuite(::std::ostream* stream, const TestSuite& test_suite); // Prints an XML summary of unit_test to output stream out. static void PrintXmlUnitTest(::std::ostream* stream, const UnitTest& unit_test); // Produces a string representing the test properties in a result as space // delimited XML attributes based on the property key="value" pairs. // When the std::string is not empty, it includes a space at the beginning, // to delimit this attribute from prior attributes. static std::string TestPropertiesAsXmlAttributes(const TestResult& result); // Streams an XML representation of the test properties of a TestResult // object. static void OutputXmlTestProperties(std::ostream* stream, const TestResult& result); // The output file. const std::string output_file_; GTEST_DISALLOW_COPY_AND_ASSIGN_(XmlUnitTestResultPrinter); }; // Creates a new XmlUnitTestResultPrinter. XmlUnitTestResultPrinter::XmlUnitTestResultPrinter(const char* output_file) : output_file_(output_file) { if (output_file_.empty()) { GTEST_LOG_(FATAL) << "XML output file may not be null"; } } // Called after the unit test ends. void XmlUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, int /*iteration*/) { FILE* xmlout = OpenFileForWriting(output_file_); std::stringstream stream; PrintXmlUnitTest(&stream, unit_test); fprintf(xmlout, "%s", StringStreamToString(&stream).c_str()); fclose(xmlout); } void XmlUnitTestResultPrinter::ListTestsMatchingFilter( const std::vector& test_suites) { FILE* xmlout = OpenFileForWriting(output_file_); std::stringstream stream; PrintXmlTestsList(&stream, test_suites); fprintf(xmlout, "%s", StringStreamToString(&stream).c_str()); fclose(xmlout); } // Returns an XML-escaped copy of the input string str. If is_attribute // is true, the text is meant to appear as an attribute value, and // normalizable whitespace is preserved by replacing it with character // references. // // Invalid XML characters in str, if any, are stripped from the output. // It is expected that most, if not all, of the text processed by this // module will consist of ordinary English text. // If this module is ever modified to produce version 1.1 XML output, // most invalid characters can be retained using character references. std::string XmlUnitTestResultPrinter::EscapeXml( const std::string& str, bool is_attribute) { Message m; for (size_t i = 0; i < str.size(); ++i) { const char ch = str[i]; switch (ch) { case '<': m << "<"; break; case '>': m << ">"; break; case '&': m << "&"; break; case '\'': if (is_attribute) m << "'"; else m << '\''; break; case '"': if (is_attribute) m << """; else m << '"'; break; default: if (IsValidXmlCharacter(ch)) { if (is_attribute && IsNormalizableWhitespace(ch)) m << "&#x" << String::FormatByte(static_cast(ch)) << ";"; else m << ch; } break; } } return m.GetString(); } // Returns the given string with all characters invalid in XML removed. // Currently invalid characters are dropped from the string. An // alternative is to replace them with certain characters such as . or ?. std::string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters( const std::string& str) { std::string output; output.reserve(str.size()); for (std::string::const_iterator it = str.begin(); it != str.end(); ++it) if (IsValidXmlCharacter(*it)) output.push_back(*it); return output; } // The following routines generate an XML representation of a UnitTest // object. // GOOGLETEST_CM0009 DO NOT DELETE // // This is how Google Test concepts map to the DTD: // // <-- corresponds to a UnitTest object // <-- corresponds to a TestSuite object // <-- corresponds to a TestInfo object // ... // ... // ... // <-- individual assertion failures // // // // Formats the given time in milliseconds as seconds. std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) { ::std::stringstream ss; ss << (static_cast(ms) * 1e-3); return ss.str(); } static bool PortableLocaltime(time_t seconds, struct tm* out) { #if defined(_MSC_VER) return localtime_s(out, &seconds) == 0; #elif defined(__MINGW32__) || defined(__MINGW64__) // MINGW provides neither localtime_r nor localtime_s, but uses // Windows' localtime(), which has a thread-local tm buffer. struct tm* tm_ptr = localtime(&seconds); // NOLINT if (tm_ptr == nullptr) return false; *out = *tm_ptr; return true; #else return localtime_r(&seconds, out) != nullptr; #endif } // Converts the given epoch time in milliseconds to a date string in the ISO // 8601 format, without the timezone information. std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms) { struct tm time_struct; if (!PortableLocaltime(static_cast(ms / 1000), &time_struct)) return ""; // YYYY-MM-DDThh:mm:ss return StreamableToString(time_struct.tm_year + 1900) + "-" + String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" + String::FormatIntWidth2(time_struct.tm_mday) + "T" + String::FormatIntWidth2(time_struct.tm_hour) + ":" + String::FormatIntWidth2(time_struct.tm_min) + ":" + String::FormatIntWidth2(time_struct.tm_sec); } // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. void XmlUnitTestResultPrinter::OutputXmlCDataSection(::std::ostream* stream, const char* data) { const char* segment = data; *stream << ""); if (next_segment != nullptr) { stream->write( segment, static_cast(next_segment - segment)); *stream << "]]>]]>"); } else { *stream << segment; break; } } *stream << "]]>"; } void XmlUnitTestResultPrinter::OutputXmlAttribute( std::ostream* stream, const std::string& element_name, const std::string& name, const std::string& value) { const std::vector& allowed_names = GetReservedOutputAttributesForElement(element_name); GTEST_CHECK_(std::find(allowed_names.begin(), allowed_names.end(), name) != allowed_names.end()) << "Attribute " << name << " is not allowed for element <" << element_name << ">."; *stream << " " << name << "=\"" << EscapeXmlAttribute(value) << "\""; } // Prints an XML representation of a TestInfo object. void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info) { const TestResult& result = *test_info.result(); const std::string kTestsuite = "testcase"; if (test_info.is_in_another_shard()) { return; } *stream << " \n"; return; } OutputXmlAttribute(stream, kTestsuite, "status", test_info.should_run() ? "run" : "notrun"); OutputXmlAttribute(stream, kTestsuite, "result", test_info.should_run() ? (result.Skipped() ? "skipped" : "completed") : "suppressed"); OutputXmlAttribute(stream, kTestsuite, "time", FormatTimeInMillisAsSeconds(result.elapsed_time())); OutputXmlAttribute( stream, kTestsuite, "timestamp", FormatEpochTimeInMillisAsIso8601(result.start_timestamp())); OutputXmlAttribute(stream, kTestsuite, "classname", test_suite_name); int failures = 0; for (int i = 0; i < result.total_part_count(); ++i) { const TestPartResult& part = result.GetTestPartResult(i); if (part.failed()) { if (++failures == 1) { *stream << ">\n"; } const std::string location = internal::FormatCompilerIndependentFileLocation(part.file_name(), part.line_number()); const std::string summary = location + "\n" + part.summary(); *stream << " "; const std::string detail = location + "\n" + part.message(); OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str()); *stream << "\n"; } } if (failures == 0 && result.test_property_count() == 0) { *stream << " />\n"; } else { if (failures == 0) { *stream << ">\n"; } OutputXmlTestProperties(stream, result); *stream << " \n"; } } // Prints an XML representation of a TestSuite object void XmlUnitTestResultPrinter::PrintXmlTestSuite(std::ostream* stream, const TestSuite& test_suite) { const std::string kTestsuite = "testsuite"; *stream << " <" << kTestsuite; OutputXmlAttribute(stream, kTestsuite, "name", test_suite.name()); OutputXmlAttribute(stream, kTestsuite, "tests", StreamableToString(test_suite.reportable_test_count())); if (!GTEST_FLAG(list_tests)) { OutputXmlAttribute(stream, kTestsuite, "failures", StreamableToString(test_suite.failed_test_count())); OutputXmlAttribute( stream, kTestsuite, "disabled", StreamableToString(test_suite.reportable_disabled_test_count())); OutputXmlAttribute(stream, kTestsuite, "errors", "0"); OutputXmlAttribute(stream, kTestsuite, "time", FormatTimeInMillisAsSeconds(test_suite.elapsed_time())); OutputXmlAttribute( stream, kTestsuite, "timestamp", FormatEpochTimeInMillisAsIso8601(test_suite.start_timestamp())); *stream << TestPropertiesAsXmlAttributes(test_suite.ad_hoc_test_result()); } *stream << ">\n"; for (int i = 0; i < test_suite.total_test_count(); ++i) { if (test_suite.GetTestInfo(i)->is_reportable()) OutputXmlTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i)); } *stream << " \n"; } // Prints an XML summary of unit_test to output stream out. void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream, const UnitTest& unit_test) { const std::string kTestsuites = "testsuites"; *stream << "\n"; *stream << "<" << kTestsuites; OutputXmlAttribute(stream, kTestsuites, "tests", StreamableToString(unit_test.reportable_test_count())); OutputXmlAttribute(stream, kTestsuites, "failures", StreamableToString(unit_test.failed_test_count())); OutputXmlAttribute( stream, kTestsuites, "disabled", StreamableToString(unit_test.reportable_disabled_test_count())); OutputXmlAttribute(stream, kTestsuites, "errors", "0"); OutputXmlAttribute(stream, kTestsuites, "time", FormatTimeInMillisAsSeconds(unit_test.elapsed_time())); OutputXmlAttribute( stream, kTestsuites, "timestamp", FormatEpochTimeInMillisAsIso8601(unit_test.start_timestamp())); if (GTEST_FLAG(shuffle)) { OutputXmlAttribute(stream, kTestsuites, "random_seed", StreamableToString(unit_test.random_seed())); } *stream << TestPropertiesAsXmlAttributes(unit_test.ad_hoc_test_result()); OutputXmlAttribute(stream, kTestsuites, "name", "AllTests"); *stream << ">\n"; for (int i = 0; i < unit_test.total_test_suite_count(); ++i) { if (unit_test.GetTestSuite(i)->reportable_test_count() > 0) PrintXmlTestSuite(stream, *unit_test.GetTestSuite(i)); } *stream << "\n"; } void XmlUnitTestResultPrinter::PrintXmlTestsList( std::ostream* stream, const std::vector& test_suites) { const std::string kTestsuites = "testsuites"; *stream << "\n"; *stream << "<" << kTestsuites; int total_tests = 0; for (auto test_suite : test_suites) { total_tests += test_suite->total_test_count(); } OutputXmlAttribute(stream, kTestsuites, "tests", StreamableToString(total_tests)); OutputXmlAttribute(stream, kTestsuites, "name", "AllTests"); *stream << ">\n"; for (auto test_suite : test_suites) { PrintXmlTestSuite(stream, *test_suite); } *stream << "\n"; } // Produces a string representing the test properties in a result as space // delimited XML attributes based on the property key="value" pairs. std::string XmlUnitTestResultPrinter::TestPropertiesAsXmlAttributes( const TestResult& result) { Message attributes; for (int i = 0; i < result.test_property_count(); ++i) { const TestProperty& property = result.GetTestProperty(i); attributes << " " << property.key() << "=" << "\"" << EscapeXmlAttribute(property.value()) << "\""; } return attributes.GetString(); } void XmlUnitTestResultPrinter::OutputXmlTestProperties( std::ostream* stream, const TestResult& result) { const std::string kProperties = "properties"; const std::string kProperty = "property"; if (result.test_property_count() <= 0) { return; } *stream << "<" << kProperties << ">\n"; for (int i = 0; i < result.test_property_count(); ++i) { const TestProperty& property = result.GetTestProperty(i); *stream << "<" << kProperty; *stream << " name=\"" << EscapeXmlAttribute(property.key()) << "\""; *stream << " value=\"" << EscapeXmlAttribute(property.value()) << "\""; *stream << "/>\n"; } *stream << "\n"; } // End XmlUnitTestResultPrinter // This class generates an JSON output file. class JsonUnitTestResultPrinter : public EmptyTestEventListener { public: explicit JsonUnitTestResultPrinter(const char* output_file); void OnTestIterationEnd(const UnitTest& unit_test, int iteration) override; // Prints an JSON summary of all unit tests. static void PrintJsonTestList(::std::ostream* stream, const std::vector& test_suites); private: // Returns an JSON-escaped copy of the input string str. static std::string EscapeJson(const std::string& str); //// Verifies that the given attribute belongs to the given element and //// streams the attribute as JSON. static void OutputJsonKey(std::ostream* stream, const std::string& element_name, const std::string& name, const std::string& value, const std::string& indent, bool comma = true); static void OutputJsonKey(std::ostream* stream, const std::string& element_name, const std::string& name, int value, const std::string& indent, bool comma = true); // Streams a JSON representation of a TestInfo object. static void OutputJsonTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info); // Prints a JSON representation of a TestSuite object static void PrintJsonTestSuite(::std::ostream* stream, const TestSuite& test_suite); // Prints a JSON summary of unit_test to output stream out. static void PrintJsonUnitTest(::std::ostream* stream, const UnitTest& unit_test); // Produces a string representing the test properties in a result as // a JSON dictionary. static std::string TestPropertiesAsJson(const TestResult& result, const std::string& indent); // The output file. const std::string output_file_; GTEST_DISALLOW_COPY_AND_ASSIGN_(JsonUnitTestResultPrinter); }; // Creates a new JsonUnitTestResultPrinter. JsonUnitTestResultPrinter::JsonUnitTestResultPrinter(const char* output_file) : output_file_(output_file) { if (output_file_.empty()) { GTEST_LOG_(FATAL) << "JSON output file may not be null"; } } void JsonUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, int /*iteration*/) { FILE* jsonout = OpenFileForWriting(output_file_); std::stringstream stream; PrintJsonUnitTest(&stream, unit_test); fprintf(jsonout, "%s", StringStreamToString(&stream).c_str()); fclose(jsonout); } // Returns an JSON-escaped copy of the input string str. std::string JsonUnitTestResultPrinter::EscapeJson(const std::string& str) { Message m; for (size_t i = 0; i < str.size(); ++i) { const char ch = str[i]; switch (ch) { case '\\': case '"': case '/': m << '\\' << ch; break; case '\b': m << "\\b"; break; case '\t': m << "\\t"; break; case '\n': m << "\\n"; break; case '\f': m << "\\f"; break; case '\r': m << "\\r"; break; default: if (ch < ' ') { m << "\\u00" << String::FormatByte(static_cast(ch)); } else { m << ch; } break; } } return m.GetString(); } // The following routines generate an JSON representation of a UnitTest // object. // Formats the given time in milliseconds as seconds. static std::string FormatTimeInMillisAsDuration(TimeInMillis ms) { ::std::stringstream ss; ss << (static_cast(ms) * 1e-3) << "s"; return ss.str(); } // Converts the given epoch time in milliseconds to a date string in the // RFC3339 format, without the timezone information. static std::string FormatEpochTimeInMillisAsRFC3339(TimeInMillis ms) { struct tm time_struct; if (!PortableLocaltime(static_cast(ms / 1000), &time_struct)) return ""; // YYYY-MM-DDThh:mm:ss return StreamableToString(time_struct.tm_year + 1900) + "-" + String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" + String::FormatIntWidth2(time_struct.tm_mday) + "T" + String::FormatIntWidth2(time_struct.tm_hour) + ":" + String::FormatIntWidth2(time_struct.tm_min) + ":" + String::FormatIntWidth2(time_struct.tm_sec) + "Z"; } static inline std::string Indent(size_t width) { return std::string(width, ' '); } void JsonUnitTestResultPrinter::OutputJsonKey( std::ostream* stream, const std::string& element_name, const std::string& name, const std::string& value, const std::string& indent, bool comma) { const std::vector& allowed_names = GetReservedOutputAttributesForElement(element_name); GTEST_CHECK_(std::find(allowed_names.begin(), allowed_names.end(), name) != allowed_names.end()) << "Key \"" << name << "\" is not allowed for value \"" << element_name << "\"."; *stream << indent << "\"" << name << "\": \"" << EscapeJson(value) << "\""; if (comma) *stream << ",\n"; } void JsonUnitTestResultPrinter::OutputJsonKey( std::ostream* stream, const std::string& element_name, const std::string& name, int value, const std::string& indent, bool comma) { const std::vector& allowed_names = GetReservedOutputAttributesForElement(element_name); GTEST_CHECK_(std::find(allowed_names.begin(), allowed_names.end(), name) != allowed_names.end()) << "Key \"" << name << "\" is not allowed for value \"" << element_name << "\"."; *stream << indent << "\"" << name << "\": " << StreamableToString(value); if (comma) *stream << ",\n"; } // Prints a JSON representation of a TestInfo object. void JsonUnitTestResultPrinter::OutputJsonTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info) { const TestResult& result = *test_info.result(); const std::string kTestsuite = "testcase"; const std::string kIndent = Indent(10); *stream << Indent(8) << "{\n"; OutputJsonKey(stream, kTestsuite, "name", test_info.name(), kIndent); if (test_info.value_param() != nullptr) { OutputJsonKey(stream, kTestsuite, "value_param", test_info.value_param(), kIndent); } if (test_info.type_param() != nullptr) { OutputJsonKey(stream, kTestsuite, "type_param", test_info.type_param(), kIndent); } if (GTEST_FLAG(list_tests)) { OutputJsonKey(stream, kTestsuite, "file", test_info.file(), kIndent); OutputJsonKey(stream, kTestsuite, "line", test_info.line(), kIndent, false); *stream << "\n" << Indent(8) << "}"; return; } OutputJsonKey(stream, kTestsuite, "status", test_info.should_run() ? "RUN" : "NOTRUN", kIndent); OutputJsonKey(stream, kTestsuite, "result", test_info.should_run() ? (result.Skipped() ? "SKIPPED" : "COMPLETED") : "SUPPRESSED", kIndent); OutputJsonKey(stream, kTestsuite, "timestamp", FormatEpochTimeInMillisAsRFC3339(result.start_timestamp()), kIndent); OutputJsonKey(stream, kTestsuite, "time", FormatTimeInMillisAsDuration(result.elapsed_time()), kIndent); OutputJsonKey(stream, kTestsuite, "classname", test_suite_name, kIndent, false); *stream << TestPropertiesAsJson(result, kIndent); int failures = 0; for (int i = 0; i < result.total_part_count(); ++i) { const TestPartResult& part = result.GetTestPartResult(i); if (part.failed()) { *stream << ",\n"; if (++failures == 1) { *stream << kIndent << "\"" << "failures" << "\": [\n"; } const std::string location = internal::FormatCompilerIndependentFileLocation(part.file_name(), part.line_number()); const std::string message = EscapeJson(location + "\n" + part.message()); *stream << kIndent << " {\n" << kIndent << " \"failure\": \"" << message << "\",\n" << kIndent << " \"type\": \"\"\n" << kIndent << " }"; } } if (failures > 0) *stream << "\n" << kIndent << "]"; *stream << "\n" << Indent(8) << "}"; } // Prints an JSON representation of a TestSuite object void JsonUnitTestResultPrinter::PrintJsonTestSuite( std::ostream* stream, const TestSuite& test_suite) { const std::string kTestsuite = "testsuite"; const std::string kIndent = Indent(6); *stream << Indent(4) << "{\n"; OutputJsonKey(stream, kTestsuite, "name", test_suite.name(), kIndent); OutputJsonKey(stream, kTestsuite, "tests", test_suite.reportable_test_count(), kIndent); if (!GTEST_FLAG(list_tests)) { OutputJsonKey(stream, kTestsuite, "failures", test_suite.failed_test_count(), kIndent); OutputJsonKey(stream, kTestsuite, "disabled", test_suite.reportable_disabled_test_count(), kIndent); OutputJsonKey(stream, kTestsuite, "errors", 0, kIndent); OutputJsonKey( stream, kTestsuite, "timestamp", FormatEpochTimeInMillisAsRFC3339(test_suite.start_timestamp()), kIndent); OutputJsonKey(stream, kTestsuite, "time", FormatTimeInMillisAsDuration(test_suite.elapsed_time()), kIndent, false); *stream << TestPropertiesAsJson(test_suite.ad_hoc_test_result(), kIndent) << ",\n"; } *stream << kIndent << "\"" << kTestsuite << "\": [\n"; bool comma = false; for (int i = 0; i < test_suite.total_test_count(); ++i) { if (test_suite.GetTestInfo(i)->is_reportable()) { if (comma) { *stream << ",\n"; } else { comma = true; } OutputJsonTestInfo(stream, test_suite.name(), *test_suite.GetTestInfo(i)); } } *stream << "\n" << kIndent << "]\n" << Indent(4) << "}"; } // Prints a JSON summary of unit_test to output stream out. void JsonUnitTestResultPrinter::PrintJsonUnitTest(std::ostream* stream, const UnitTest& unit_test) { const std::string kTestsuites = "testsuites"; const std::string kIndent = Indent(2); *stream << "{\n"; OutputJsonKey(stream, kTestsuites, "tests", unit_test.reportable_test_count(), kIndent); OutputJsonKey(stream, kTestsuites, "failures", unit_test.failed_test_count(), kIndent); OutputJsonKey(stream, kTestsuites, "disabled", unit_test.reportable_disabled_test_count(), kIndent); OutputJsonKey(stream, kTestsuites, "errors", 0, kIndent); if (GTEST_FLAG(shuffle)) { OutputJsonKey(stream, kTestsuites, "random_seed", unit_test.random_seed(), kIndent); } OutputJsonKey(stream, kTestsuites, "timestamp", FormatEpochTimeInMillisAsRFC3339(unit_test.start_timestamp()), kIndent); OutputJsonKey(stream, kTestsuites, "time", FormatTimeInMillisAsDuration(unit_test.elapsed_time()), kIndent, false); *stream << TestPropertiesAsJson(unit_test.ad_hoc_test_result(), kIndent) << ",\n"; OutputJsonKey(stream, kTestsuites, "name", "AllTests", kIndent); *stream << kIndent << "\"" << kTestsuites << "\": [\n"; bool comma = false; for (int i = 0; i < unit_test.total_test_suite_count(); ++i) { if (unit_test.GetTestSuite(i)->reportable_test_count() > 0) { if (comma) { *stream << ",\n"; } else { comma = true; } PrintJsonTestSuite(stream, *unit_test.GetTestSuite(i)); } } *stream << "\n" << kIndent << "]\n" << "}\n"; } void JsonUnitTestResultPrinter::PrintJsonTestList( std::ostream* stream, const std::vector& test_suites) { const std::string kTestsuites = "testsuites"; const std::string kIndent = Indent(2); *stream << "{\n"; int total_tests = 0; for (auto test_suite : test_suites) { total_tests += test_suite->total_test_count(); } OutputJsonKey(stream, kTestsuites, "tests", total_tests, kIndent); OutputJsonKey(stream, kTestsuites, "name", "AllTests", kIndent); *stream << kIndent << "\"" << kTestsuites << "\": [\n"; for (size_t i = 0; i < test_suites.size(); ++i) { if (i != 0) { *stream << ",\n"; } PrintJsonTestSuite(stream, *test_suites[i]); } *stream << "\n" << kIndent << "]\n" << "}\n"; } // Produces a string representing the test properties in a result as // a JSON dictionary. std::string JsonUnitTestResultPrinter::TestPropertiesAsJson( const TestResult& result, const std::string& indent) { Message attributes; for (int i = 0; i < result.test_property_count(); ++i) { const TestProperty& property = result.GetTestProperty(i); attributes << ",\n" << indent << "\"" << property.key() << "\": " << "\"" << EscapeJson(property.value()) << "\""; } return attributes.GetString(); } // End JsonUnitTestResultPrinter #if GTEST_CAN_STREAM_RESULTS_ // Checks if str contains '=', '&', '%' or '\n' characters. If yes, // replaces them by "%xx" where xx is their hexadecimal value. For // example, replaces "=" with "%3D". This algorithm is O(strlen(str)) // in both time and space -- important as the input str may contain an // arbitrarily long test failure message and stack trace. std::string StreamingListener::UrlEncode(const char* str) { std::string result; result.reserve(strlen(str) + 1); for (char ch = *str; ch != '\0'; ch = *++str) { switch (ch) { case '%': case '=': case '&': case '\n': result.append("%" + String::FormatByte(static_cast(ch))); break; default: result.push_back(ch); break; } } return result; } void StreamingListener::SocketWriter::MakeConnection() { GTEST_CHECK_(sockfd_ == -1) << "MakeConnection() can't be called when there is already a connection."; addrinfo hints; memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; // To allow both IPv4 and IPv6 addresses. hints.ai_socktype = SOCK_STREAM; addrinfo* servinfo = nullptr; // Use the getaddrinfo() to get a linked list of IP addresses for // the given host name. const int error_num = getaddrinfo( host_name_.c_str(), port_num_.c_str(), &hints, &servinfo); if (error_num != 0) { GTEST_LOG_(WARNING) << "stream_result_to: getaddrinfo() failed: " << gai_strerror(error_num); } // Loop through all the results and connect to the first we can. for (addrinfo* cur_addr = servinfo; sockfd_ == -1 && cur_addr != nullptr; cur_addr = cur_addr->ai_next) { sockfd_ = socket( cur_addr->ai_family, cur_addr->ai_socktype, cur_addr->ai_protocol); if (sockfd_ != -1) { // Connect the client socket to the server socket. if (connect(sockfd_, cur_addr->ai_addr, cur_addr->ai_addrlen) == -1) { close(sockfd_); sockfd_ = -1; } } } freeaddrinfo(servinfo); // all done with this structure if (sockfd_ == -1) { GTEST_LOG_(WARNING) << "stream_result_to: failed to connect to " << host_name_ << ":" << port_num_; } } // End of class Streaming Listener #endif // GTEST_CAN_STREAM_RESULTS__ // class OsStackTraceGetter const char* const OsStackTraceGetterInterface::kElidedFramesMarker = "... " GTEST_NAME_ " internal frames ..."; std::string OsStackTraceGetter::CurrentStackTrace(int max_depth, int skip_count) GTEST_LOCK_EXCLUDED_(mutex_) { #if GTEST_HAS_ABSL std::string result; if (max_depth <= 0) { return result; } max_depth = std::min(max_depth, kMaxStackTraceDepth); std::vector raw_stack(max_depth); // Skips the frames requested by the caller, plus this function. const int raw_stack_size = absl::GetStackTrace(&raw_stack[0], max_depth, skip_count + 1); void* caller_frame = nullptr; { MutexLock lock(&mutex_); caller_frame = caller_frame_; } for (int i = 0; i < raw_stack_size; ++i) { if (raw_stack[i] == caller_frame && !GTEST_FLAG(show_internal_stack_frames)) { // Add a marker to the trace and stop adding frames. absl::StrAppend(&result, kElidedFramesMarker, "\n"); break; } char tmp[1024]; const char* symbol = "(unknown)"; if (absl::Symbolize(raw_stack[i], tmp, sizeof(tmp))) { symbol = tmp; } char line[1024]; snprintf(line, sizeof(line), " %p: %s\n", raw_stack[i], symbol); result += line; } return result; #else // !GTEST_HAS_ABSL static_cast(max_depth); static_cast(skip_count); return ""; #endif // GTEST_HAS_ABSL } void OsStackTraceGetter::UponLeavingGTest() GTEST_LOCK_EXCLUDED_(mutex_) { #if GTEST_HAS_ABSL void* caller_frame = nullptr; if (absl::GetStackTrace(&caller_frame, 1, 3) <= 0) { caller_frame = nullptr; } MutexLock lock(&mutex_); caller_frame_ = caller_frame; #endif // GTEST_HAS_ABSL } // A helper class that creates the premature-exit file in its // constructor and deletes the file in its destructor. class ScopedPrematureExitFile { public: explicit ScopedPrematureExitFile(const char* premature_exit_filepath) : premature_exit_filepath_(premature_exit_filepath ? premature_exit_filepath : "") { // If a path to the premature-exit file is specified... if (!premature_exit_filepath_.empty()) { // create the file with a single "0" character in it. I/O // errors are ignored as there's nothing better we can do and we // don't want to fail the test because of this. FILE* pfile = posix::FOpen(premature_exit_filepath, "w"); fwrite("0", 1, 1, pfile); fclose(pfile); } } ~ScopedPrematureExitFile() { if (!premature_exit_filepath_.empty()) { int retval = remove(premature_exit_filepath_.c_str()); if (retval) { GTEST_LOG_(ERROR) << "Failed to remove premature exit filepath \"" << premature_exit_filepath_ << "\" with error " << retval; } } } private: const std::string premature_exit_filepath_; GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedPrematureExitFile); }; } // namespace internal // class TestEventListeners TestEventListeners::TestEventListeners() : repeater_(new internal::TestEventRepeater()), default_result_printer_(nullptr), default_xml_generator_(nullptr) {} TestEventListeners::~TestEventListeners() { delete repeater_; } // Returns the standard listener responsible for the default console // output. Can be removed from the listeners list to shut down default // console output. Note that removing this object from the listener list // with Release transfers its ownership to the user. void TestEventListeners::Append(TestEventListener* listener) { repeater_->Append(listener); } // Removes the given event listener from the list and returns it. It then // becomes the caller's responsibility to delete the listener. Returns // NULL if the listener is not found in the list. TestEventListener* TestEventListeners::Release(TestEventListener* listener) { if (listener == default_result_printer_) default_result_printer_ = nullptr; else if (listener == default_xml_generator_) default_xml_generator_ = nullptr; return repeater_->Release(listener); } // Returns repeater that broadcasts the TestEventListener events to all // subscribers. TestEventListener* TestEventListeners::repeater() { return repeater_; } // Sets the default_result_printer attribute to the provided listener. // The listener is also added to the listener list and previous // default_result_printer is removed from it and deleted. The listener can // also be NULL in which case it will not be added to the list. Does // nothing if the previous and the current listener objects are the same. void TestEventListeners::SetDefaultResultPrinter(TestEventListener* listener) { if (default_result_printer_ != listener) { // It is an error to pass this method a listener that is already in the // list. delete Release(default_result_printer_); default_result_printer_ = listener; if (listener != nullptr) Append(listener); } } // Sets the default_xml_generator attribute to the provided listener. The // listener is also added to the listener list and previous // default_xml_generator is removed from it and deleted. The listener can // also be NULL in which case it will not be added to the list. Does // nothing if the previous and the current listener objects are the same. void TestEventListeners::SetDefaultXmlGenerator(TestEventListener* listener) { if (default_xml_generator_ != listener) { // It is an error to pass this method a listener that is already in the // list. delete Release(default_xml_generator_); default_xml_generator_ = listener; if (listener != nullptr) Append(listener); } } // Controls whether events will be forwarded by the repeater to the // listeners in the list. bool TestEventListeners::EventForwardingEnabled() const { return repeater_->forwarding_enabled(); } void TestEventListeners::SuppressEventForwarding() { repeater_->set_forwarding_enabled(false); } // class UnitTest // Gets the singleton UnitTest object. The first time this method is // called, a UnitTest object is constructed and returned. Consecutive // calls will return the same object. // // We don't protect this under mutex_ as a user is not supposed to // call this before main() starts, from which point on the return // value will never change. UnitTest* UnitTest::GetInstance() { // CodeGear C++Builder insists on a public destructor for the // default implementation. Use this implementation to keep good OO // design with private destructor. #if defined(__BORLANDC__) static UnitTest* const instance = new UnitTest; return instance; #else static UnitTest instance; return &instance; #endif // defined(__BORLANDC__) } // Gets the number of successful test suites. int UnitTest::successful_test_suite_count() const { return impl()->successful_test_suite_count(); } // Gets the number of failed test suites. int UnitTest::failed_test_suite_count() const { return impl()->failed_test_suite_count(); } // Gets the number of all test suites. int UnitTest::total_test_suite_count() const { return impl()->total_test_suite_count(); } // Gets the number of all test suites that contain at least one test // that should run. int UnitTest::test_suite_to_run_count() const { return impl()->test_suite_to_run_count(); } // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ int UnitTest::successful_test_case_count() const { return impl()->successful_test_suite_count(); } int UnitTest::failed_test_case_count() const { return impl()->failed_test_suite_count(); } int UnitTest::total_test_case_count() const { return impl()->total_test_suite_count(); } int UnitTest::test_case_to_run_count() const { return impl()->test_suite_to_run_count(); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // Gets the number of successful tests. int UnitTest::successful_test_count() const { return impl()->successful_test_count(); } // Gets the number of skipped tests. int UnitTest::skipped_test_count() const { return impl()->skipped_test_count(); } // Gets the number of failed tests. int UnitTest::failed_test_count() const { return impl()->failed_test_count(); } // Gets the number of disabled tests that will be reported in the XML report. int UnitTest::reportable_disabled_test_count() const { return impl()->reportable_disabled_test_count(); } // Gets the number of disabled tests. int UnitTest::disabled_test_count() const { return impl()->disabled_test_count(); } // Gets the number of tests to be printed in the XML report. int UnitTest::reportable_test_count() const { return impl()->reportable_test_count(); } // Gets the number of all tests. int UnitTest::total_test_count() const { return impl()->total_test_count(); } // Gets the number of tests that should run. int UnitTest::test_to_run_count() const { return impl()->test_to_run_count(); } // Gets the time of the test program start, in ms from the start of the // UNIX epoch. internal::TimeInMillis UnitTest::start_timestamp() const { return impl()->start_timestamp(); } // Gets the elapsed time, in milliseconds. internal::TimeInMillis UnitTest::elapsed_time() const { return impl()->elapsed_time(); } // Returns true if and only if the unit test passed (i.e. all test suites // passed). bool UnitTest::Passed() const { return impl()->Passed(); } // Returns true if and only if the unit test failed (i.e. some test suite // failed or something outside of all tests failed). bool UnitTest::Failed() const { return impl()->Failed(); } // Gets the i-th test suite among all the test suites. i can range from 0 to // total_test_suite_count() - 1. If i is not in that range, returns NULL. const TestSuite* UnitTest::GetTestSuite(int i) const { return impl()->GetTestSuite(i); } // Legacy API is deprecated but still available #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ const TestCase* UnitTest::GetTestCase(int i) const { return impl()->GetTestCase(i); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ // Returns the TestResult containing information on test failures and // properties logged outside of individual test suites. const TestResult& UnitTest::ad_hoc_test_result() const { return *impl()->ad_hoc_test_result(); } // Gets the i-th test suite among all the test suites. i can range from 0 to // total_test_suite_count() - 1. If i is not in that range, returns NULL. TestSuite* UnitTest::GetMutableTestSuite(int i) { return impl()->GetMutableSuiteCase(i); } // Returns the list of event listeners that can be used to track events // inside Google Test. TestEventListeners& UnitTest::listeners() { return *impl()->listeners(); } // Registers and returns a global test environment. When a test // program is run, all global test environments will be set-up in the // order they were registered. After all tests in the program have // finished, all global test environments will be torn-down in the // *reverse* order they were registered. // // The UnitTest object takes ownership of the given environment. // // We don't protect this under mutex_, as we only support calling it // from the main thread. Environment* UnitTest::AddEnvironment(Environment* env) { if (env == nullptr) { return nullptr; } impl_->environments().push_back(env); return env; } // Adds a TestPartResult to the current TestResult object. All Google Test // assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) eventually call // this to report their results. The user code should use the // assertion macros instead of calling this directly. void UnitTest::AddTestPartResult( TestPartResult::Type result_type, const char* file_name, int line_number, const std::string& message, const std::string& os_stack_trace) GTEST_LOCK_EXCLUDED_(mutex_) { Message msg; msg << message; internal::MutexLock lock(&mutex_); if (impl_->gtest_trace_stack().size() > 0) { msg << "\n" << GTEST_NAME_ << " trace:"; for (size_t i = impl_->gtest_trace_stack().size(); i > 0; --i) { const internal::TraceInfo& trace = impl_->gtest_trace_stack()[i - 1]; msg << "\n" << internal::FormatFileLocation(trace.file, trace.line) << " " << trace.message; } } if (os_stack_trace.c_str() != nullptr && !os_stack_trace.empty()) { msg << internal::kStackTraceMarker << os_stack_trace; } const TestPartResult result = TestPartResult( result_type, file_name, line_number, msg.GetString().c_str()); impl_->GetTestPartResultReporterForCurrentThread()-> ReportTestPartResult(result); if (result_type != TestPartResult::kSuccess && result_type != TestPartResult::kSkip) { // gtest_break_on_failure takes precedence over // gtest_throw_on_failure. This allows a user to set the latter // in the code (perhaps in order to use Google Test assertions // with another testing framework) and specify the former on the // command line for debugging. if (GTEST_FLAG(break_on_failure)) { #if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT // Using DebugBreak on Windows allows gtest to still break into a debugger // when a failure happens and both the --gtest_break_on_failure and // the --gtest_catch_exceptions flags are specified. DebugBreak(); #elif (!defined(__native_client__)) && \ ((defined(__clang__) || defined(__GNUC__)) && \ (defined(__x86_64__) || defined(__i386__))) // with clang/gcc we can achieve the same effect on x86 by invoking int3 asm("int3"); #else // Dereference nullptr through a volatile pointer to prevent the compiler // from removing. We use this rather than abort() or __builtin_trap() for // portability: some debuggers don't correctly trap abort(). *static_cast(nullptr) = 1; #endif // GTEST_OS_WINDOWS } else if (GTEST_FLAG(throw_on_failure)) { #if GTEST_HAS_EXCEPTIONS throw internal::GoogleTestFailureException(result); #else // We cannot call abort() as it generates a pop-up in debug mode // that cannot be suppressed in VC 7.1 or below. exit(1); #endif } } } // Adds a TestProperty to the current TestResult object when invoked from // inside a test, to current TestSuite's ad_hoc_test_result_ when invoked // from SetUpTestSuite or TearDownTestSuite, or to the global property set // when invoked elsewhere. If the result already contains a property with // the same key, the value will be updated. void UnitTest::RecordProperty(const std::string& key, const std::string& value) { impl_->RecordProperty(TestProperty(key, value)); } // Runs all tests in this UnitTest object and prints the result. // Returns 0 if successful, or 1 otherwise. // // We don't protect this under mutex_, as we only support calling it // from the main thread. int UnitTest::Run() { const bool in_death_test_child_process = internal::GTEST_FLAG(internal_run_death_test).length() > 0; // Google Test implements this protocol for catching that a test // program exits before returning control to Google Test: // // 1. Upon start, Google Test creates a file whose absolute path // is specified by the environment variable // TEST_PREMATURE_EXIT_FILE. // 2. When Google Test has finished its work, it deletes the file. // // This allows a test runner to set TEST_PREMATURE_EXIT_FILE before // running a Google-Test-based test program and check the existence // of the file at the end of the test execution to see if it has // exited prematurely. // If we are in the child process of a death test, don't // create/delete the premature exit file, as doing so is unnecessary // and will confuse the parent process. Otherwise, create/delete // the file upon entering/leaving this function. If the program // somehow exits before this function has a chance to return, the // premature-exit file will be left undeleted, causing a test runner // that understands the premature-exit-file protocol to report the // test as having failed. const internal::ScopedPrematureExitFile premature_exit_file( in_death_test_child_process ? nullptr : internal::posix::GetEnv("TEST_PREMATURE_EXIT_FILE")); // Captures the value of GTEST_FLAG(catch_exceptions). This value will be // used for the duration of the program. impl()->set_catch_exceptions(GTEST_FLAG(catch_exceptions)); #if GTEST_OS_WINDOWS // Either the user wants Google Test to catch exceptions thrown by the // tests or this is executing in the context of death test child // process. In either case the user does not want to see pop-up dialogs // about crashes - they are expected. if (impl()->catch_exceptions() || in_death_test_child_process) { # if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT // SetErrorMode doesn't exist on CE. SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); # endif // !GTEST_OS_WINDOWS_MOBILE # if (defined(_MSC_VER) || GTEST_OS_WINDOWS_MINGW) && !GTEST_OS_WINDOWS_MOBILE // Death test children can be terminated with _abort(). On Windows, // _abort() can show a dialog with a warning message. This forces the // abort message to go to stderr instead. _set_error_mode(_OUT_TO_STDERR); # endif # if defined(_MSC_VER) && !GTEST_OS_WINDOWS_MOBILE // In the debug version, Visual Studio pops up a separate dialog // offering a choice to debug the aborted program. We need to suppress // this dialog or it will pop up for every EXPECT/ASSERT_DEATH statement // executed. Google Test will notify the user of any unexpected // failure via stderr. if (!GTEST_FLAG(break_on_failure)) _set_abort_behavior( 0x0, // Clear the following flags: _WRITE_ABORT_MSG | _CALL_REPORTFAULT); // pop-up window, core dump. # endif // In debug mode, the Windows CRT can crash with an assertion over invalid // input (e.g. passing an invalid file descriptor). The default handling // for these assertions is to pop up a dialog and wait for user input. // Instead ask the CRT to dump such assertions to stderr non-interactively. if (!IsDebuggerPresent()) { (void)_CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG); (void)_CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR); } } #endif // GTEST_OS_WINDOWS return internal::HandleExceptionsInMethodIfSupported( impl(), &internal::UnitTestImpl::RunAllTests, "auxiliary test code (environments or event listeners)") ? 0 : 1; } // Returns the working directory when the first TEST() or TEST_F() was // executed. const char* UnitTest::original_working_dir() const { return impl_->original_working_dir_.c_str(); } // Returns the TestSuite object for the test that's currently running, // or NULL if no test is running. const TestSuite* UnitTest::current_test_suite() const GTEST_LOCK_EXCLUDED_(mutex_) { internal::MutexLock lock(&mutex_); return impl_->current_test_suite(); } // Legacy API is still available but deprecated #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ const TestCase* UnitTest::current_test_case() const GTEST_LOCK_EXCLUDED_(mutex_) { internal::MutexLock lock(&mutex_); return impl_->current_test_suite(); } #endif // Returns the TestInfo object for the test that's currently running, // or NULL if no test is running. const TestInfo* UnitTest::current_test_info() const GTEST_LOCK_EXCLUDED_(mutex_) { internal::MutexLock lock(&mutex_); return impl_->current_test_info(); } // Returns the random seed used at the start of the current test run. int UnitTest::random_seed() const { return impl_->random_seed(); } // Returns ParameterizedTestSuiteRegistry object used to keep track of // value-parameterized tests and instantiate and register them. internal::ParameterizedTestSuiteRegistry& UnitTest::parameterized_test_registry() GTEST_LOCK_EXCLUDED_(mutex_) { return impl_->parameterized_test_registry(); } // Creates an empty UnitTest. UnitTest::UnitTest() { impl_ = new internal::UnitTestImpl(this); } // Destructor of UnitTest. UnitTest::~UnitTest() { delete impl_; } // Pushes a trace defined by SCOPED_TRACE() on to the per-thread // Google Test trace stack. void UnitTest::PushGTestTrace(const internal::TraceInfo& trace) GTEST_LOCK_EXCLUDED_(mutex_) { internal::MutexLock lock(&mutex_); impl_->gtest_trace_stack().push_back(trace); } // Pops a trace from the per-thread Google Test trace stack. void UnitTest::PopGTestTrace() GTEST_LOCK_EXCLUDED_(mutex_) { internal::MutexLock lock(&mutex_); impl_->gtest_trace_stack().pop_back(); } namespace internal { UnitTestImpl::UnitTestImpl(UnitTest* parent) : parent_(parent), GTEST_DISABLE_MSC_WARNINGS_PUSH_(4355 /* using this in initializer */) default_global_test_part_result_reporter_(this), default_per_thread_test_part_result_reporter_(this), GTEST_DISABLE_MSC_WARNINGS_POP_() global_test_part_result_repoter_( &default_global_test_part_result_reporter_), per_thread_test_part_result_reporter_( &default_per_thread_test_part_result_reporter_), parameterized_test_registry_(), parameterized_tests_registered_(false), last_death_test_suite_(-1), current_test_suite_(nullptr), current_test_info_(nullptr), ad_hoc_test_result_(), os_stack_trace_getter_(nullptr), post_flag_parse_init_performed_(false), random_seed_(0), // Will be overridden by the flag before first use. random_(0), // Will be reseeded before first use. start_timestamp_(0), elapsed_time_(0), #if GTEST_HAS_DEATH_TEST death_test_factory_(new DefaultDeathTestFactory), #endif // Will be overridden by the flag before first use. catch_exceptions_(false) { listeners()->SetDefaultResultPrinter(new PrettyUnitTestResultPrinter); } UnitTestImpl::~UnitTestImpl() { // Deletes every TestSuite. ForEach(test_suites_, internal::Delete); // Deletes every Environment. ForEach(environments_, internal::Delete); delete os_stack_trace_getter_; } // Adds a TestProperty to the current TestResult object when invoked in a // context of a test, to current test suite's ad_hoc_test_result when invoke // from SetUpTestSuite/TearDownTestSuite, or to the global property set // otherwise. If the result already contains a property with the same key, // the value will be updated. void UnitTestImpl::RecordProperty(const TestProperty& test_property) { std::string xml_element; TestResult* test_result; // TestResult appropriate for property recording. if (current_test_info_ != nullptr) { xml_element = "testcase"; test_result = &(current_test_info_->result_); } else if (current_test_suite_ != nullptr) { xml_element = "testsuite"; test_result = &(current_test_suite_->ad_hoc_test_result_); } else { xml_element = "testsuites"; test_result = &ad_hoc_test_result_; } test_result->RecordProperty(xml_element, test_property); } #if GTEST_HAS_DEATH_TEST // Disables event forwarding if the control is currently in a death test // subprocess. Must not be called before InitGoogleTest. void UnitTestImpl::SuppressTestEventsIfInSubprocess() { if (internal_run_death_test_flag_.get() != nullptr) listeners()->SuppressEventForwarding(); } #endif // GTEST_HAS_DEATH_TEST // Initializes event listeners performing XML output as specified by // UnitTestOptions. Must not be called before InitGoogleTest. void UnitTestImpl::ConfigureXmlOutput() { const std::string& output_format = UnitTestOptions::GetOutputFormat(); if (output_format == "xml") { listeners()->SetDefaultXmlGenerator(new XmlUnitTestResultPrinter( UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); } else if (output_format == "json") { listeners()->SetDefaultXmlGenerator(new JsonUnitTestResultPrinter( UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); } else if (output_format != "") { GTEST_LOG_(WARNING) << "WARNING: unrecognized output format \"" << output_format << "\" ignored."; } } #if GTEST_CAN_STREAM_RESULTS_ // Initializes event listeners for streaming test results in string form. // Must not be called before InitGoogleTest. void UnitTestImpl::ConfigureStreamingOutput() { const std::string& target = GTEST_FLAG(stream_result_to); if (!target.empty()) { const size_t pos = target.find(':'); if (pos != std::string::npos) { listeners()->Append(new StreamingListener(target.substr(0, pos), target.substr(pos+1))); } else { GTEST_LOG_(WARNING) << "unrecognized streaming target \"" << target << "\" ignored."; } } } #endif // GTEST_CAN_STREAM_RESULTS_ // Performs initialization dependent upon flag values obtained in // ParseGoogleTestFlagsOnly. Is called from InitGoogleTest after the call to // ParseGoogleTestFlagsOnly. In case a user neglects to call InitGoogleTest // this function is also called from RunAllTests. Since this function can be // called more than once, it has to be idempotent. void UnitTestImpl::PostFlagParsingInit() { // Ensures that this function does not execute more than once. if (!post_flag_parse_init_performed_) { post_flag_parse_init_performed_ = true; #if defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_) // Register to send notifications about key process state changes. listeners()->Append(new GTEST_CUSTOM_TEST_EVENT_LISTENER_()); #endif // defined(GTEST_CUSTOM_TEST_EVENT_LISTENER_) #if GTEST_HAS_DEATH_TEST InitDeathTestSubprocessControlInfo(); SuppressTestEventsIfInSubprocess(); #endif // GTEST_HAS_DEATH_TEST // Registers parameterized tests. This makes parameterized tests // available to the UnitTest reflection API without running // RUN_ALL_TESTS. RegisterParameterizedTests(); // Configures listeners for XML output. This makes it possible for users // to shut down the default XML output before invoking RUN_ALL_TESTS. ConfigureXmlOutput(); #if GTEST_CAN_STREAM_RESULTS_ // Configures listeners for streaming test results to the specified server. ConfigureStreamingOutput(); #endif // GTEST_CAN_STREAM_RESULTS_ #if GTEST_HAS_ABSL if (GTEST_FLAG(install_failure_signal_handler)) { absl::FailureSignalHandlerOptions options; absl::InstallFailureSignalHandler(options); } #endif // GTEST_HAS_ABSL } } // A predicate that checks the name of a TestSuite against a known // value. // // This is used for implementation of the UnitTest class only. We put // it in the anonymous namespace to prevent polluting the outer // namespace. // // TestSuiteNameIs is copyable. class TestSuiteNameIs { public: // Constructor. explicit TestSuiteNameIs(const std::string& name) : name_(name) {} // Returns true if and only if the name of test_suite matches name_. bool operator()(const TestSuite* test_suite) const { return test_suite != nullptr && strcmp(test_suite->name(), name_.c_str()) == 0; } private: std::string name_; }; // Finds and returns a TestSuite with the given name. If one doesn't // exist, creates one and returns it. It's the CALLER'S // RESPONSIBILITY to ensure that this function is only called WHEN THE // TESTS ARE NOT SHUFFLED. // // Arguments: // // test_suite_name: name of the test suite // type_param: the name of the test suite's type parameter, or NULL if // this is not a typed or a type-parameterized test suite. // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite TestSuite* UnitTestImpl::GetTestSuite( const char* test_suite_name, const char* type_param, internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc) { // Can we find a TestSuite with the given name? const auto test_suite = std::find_if(test_suites_.rbegin(), test_suites_.rend(), TestSuiteNameIs(test_suite_name)); if (test_suite != test_suites_.rend()) return *test_suite; // No. Let's create one. auto* const new_test_suite = new TestSuite(test_suite_name, type_param, set_up_tc, tear_down_tc); // Is this a death test suite? if (internal::UnitTestOptions::MatchesFilter(test_suite_name, kDeathTestSuiteFilter)) { // Yes. Inserts the test suite after the last death test suite // defined so far. This only works when the test suites haven't // been shuffled. Otherwise we may end up running a death test // after a non-death test. ++last_death_test_suite_; test_suites_.insert(test_suites_.begin() + last_death_test_suite_, new_test_suite); } else { // No. Appends to the end of the list. test_suites_.push_back(new_test_suite); } test_suite_indices_.push_back(static_cast(test_suite_indices_.size())); return new_test_suite; } // Helpers for setting up / tearing down the given environment. They // are for use in the ForEach() function. static void SetUpEnvironment(Environment* env) { env->SetUp(); } static void TearDownEnvironment(Environment* env) { env->TearDown(); } // Runs all tests in this UnitTest object, prints the result, and // returns true if all tests are successful. If any exception is // thrown during a test, the test is considered to be failed, but the // rest of the tests will still be run. // // When parameterized tests are enabled, it expands and registers // parameterized tests first in RegisterParameterizedTests(). // All other functions called from RunAllTests() may safely assume that // parameterized tests are ready to be counted and run. bool UnitTestImpl::RunAllTests() { // True if and only if Google Test is initialized before RUN_ALL_TESTS() is // called. const bool gtest_is_initialized_before_run_all_tests = GTestIsInitialized(); // Do not run any test if the --help flag was specified. if (g_help_flag) return true; // Repeats the call to the post-flag parsing initialization in case the // user didn't call InitGoogleTest. PostFlagParsingInit(); // Even if sharding is not on, test runners may want to use the // GTEST_SHARD_STATUS_FILE to query whether the test supports the sharding // protocol. internal::WriteToShardStatusFileIfNeeded(); // True if and only if we are in a subprocess for running a thread-safe-style // death test. bool in_subprocess_for_death_test = false; #if GTEST_HAS_DEATH_TEST in_subprocess_for_death_test = (internal_run_death_test_flag_.get() != nullptr); # if defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_) if (in_subprocess_for_death_test) { GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_(); } # endif // defined(GTEST_EXTRA_DEATH_TEST_CHILD_SETUP_) #endif // GTEST_HAS_DEATH_TEST const bool should_shard = ShouldShard(kTestTotalShards, kTestShardIndex, in_subprocess_for_death_test); // Compares the full test names with the filter to decide which // tests to run. const bool has_tests_to_run = FilterTests(should_shard ? HONOR_SHARDING_PROTOCOL : IGNORE_SHARDING_PROTOCOL) > 0; // Lists the tests and exits if the --gtest_list_tests flag was specified. if (GTEST_FLAG(list_tests)) { // This must be called *after* FilterTests() has been called. ListTestsMatchingFilter(); return true; } random_seed_ = GTEST_FLAG(shuffle) ? GetRandomSeedFromFlag(GTEST_FLAG(random_seed)) : 0; // True if and only if at least one test has failed. bool failed = false; TestEventListener* repeater = listeners()->repeater(); start_timestamp_ = GetTimeInMillis(); repeater->OnTestProgramStart(*parent_); // How many times to repeat the tests? We don't want to repeat them // when we are inside the subprocess of a death test. const int repeat = in_subprocess_for_death_test ? 1 : GTEST_FLAG(repeat); // Repeats forever if the repeat count is negative. const bool gtest_repeat_forever = repeat < 0; for (int i = 0; gtest_repeat_forever || i != repeat; i++) { // We want to preserve failures generated by ad-hoc test // assertions executed before RUN_ALL_TESTS(). ClearNonAdHocTestResult(); const TimeInMillis start = GetTimeInMillis(); // Shuffles test suites and tests if requested. if (has_tests_to_run && GTEST_FLAG(shuffle)) { random()->Reseed(static_cast(random_seed_)); // This should be done before calling OnTestIterationStart(), // such that a test event listener can see the actual test order // in the event. ShuffleTests(); } // Tells the unit test event listeners that the tests are about to start. repeater->OnTestIterationStart(*parent_, i); // Runs each test suite if there is at least one test to run. if (has_tests_to_run) { // Sets up all environments beforehand. repeater->OnEnvironmentsSetUpStart(*parent_); ForEach(environments_, SetUpEnvironment); repeater->OnEnvironmentsSetUpEnd(*parent_); // Runs the tests only if there was no fatal failure or skip triggered // during global set-up. if (Test::IsSkipped()) { // Emit diagnostics when global set-up calls skip, as it will not be // emitted by default. TestResult& test_result = *internal::GetUnitTestImpl()->current_test_result(); for (int j = 0; j < test_result.total_part_count(); ++j) { const TestPartResult& test_part_result = test_result.GetTestPartResult(j); if (test_part_result.type() == TestPartResult::kSkip) { const std::string& result = test_part_result.message(); printf("%s\n", result.c_str()); } } fflush(stdout); } else if (!Test::HasFatalFailure()) { for (int test_index = 0; test_index < total_test_suite_count(); test_index++) { GetMutableSuiteCase(test_index)->Run(); } } // Tears down all environments in reverse order afterwards. repeater->OnEnvironmentsTearDownStart(*parent_); std::for_each(environments_.rbegin(), environments_.rend(), TearDownEnvironment); repeater->OnEnvironmentsTearDownEnd(*parent_); } elapsed_time_ = GetTimeInMillis() - start; // Tells the unit test event listener that the tests have just finished. repeater->OnTestIterationEnd(*parent_, i); // Gets the result and clears it. if (!Passed()) { failed = true; } // Restores the original test order after the iteration. This // allows the user to quickly repro a failure that happens in the // N-th iteration without repeating the first (N - 1) iterations. // This is not enclosed in "if (GTEST_FLAG(shuffle)) { ... }", in // case the user somehow changes the value of the flag somewhere // (it's always safe to unshuffle the tests). UnshuffleTests(); if (GTEST_FLAG(shuffle)) { // Picks a new random seed for each iteration. random_seed_ = GetNextRandomSeed(random_seed_); } } repeater->OnTestProgramEnd(*parent_); if (!gtest_is_initialized_before_run_all_tests) { ColoredPrintf( COLOR_RED, "\nIMPORTANT NOTICE - DO NOT IGNORE:\n" "This test program did NOT call " GTEST_INIT_GOOGLE_TEST_NAME_ "() before calling RUN_ALL_TESTS(). This is INVALID. Soon " GTEST_NAME_ " will start to enforce the valid usage. " "Please fix it ASAP, or IT WILL START TO FAIL.\n"); // NOLINT #if GTEST_FOR_GOOGLE_ ColoredPrintf(COLOR_RED, "For more details, see http://wiki/Main/ValidGUnitMain.\n"); #endif // GTEST_FOR_GOOGLE_ } return !failed; } // Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file // if the variable is present. If a file already exists at this location, this // function will write over it. If the variable is present, but the file cannot // be created, prints an error and exits. void WriteToShardStatusFileIfNeeded() { const char* const test_shard_file = posix::GetEnv(kTestShardStatusFile); if (test_shard_file != nullptr) { FILE* const file = posix::FOpen(test_shard_file, "w"); if (file == nullptr) { ColoredPrintf(COLOR_RED, "Could not write to the test shard status file \"%s\" " "specified by the %s environment variable.\n", test_shard_file, kTestShardStatusFile); fflush(stdout); exit(EXIT_FAILURE); } fclose(file); } } // Checks whether sharding is enabled by examining the relevant // environment variable values. If the variables are present, // but inconsistent (i.e., shard_index >= total_shards), prints // an error and exits. If in_subprocess_for_death_test, sharding is // disabled because it must only be applied to the original test // process. Otherwise, we could filter out death tests we intended to execute. bool ShouldShard(const char* total_shards_env, const char* shard_index_env, bool in_subprocess_for_death_test) { if (in_subprocess_for_death_test) { return false; } const Int32 total_shards = Int32FromEnvOrDie(total_shards_env, -1); const Int32 shard_index = Int32FromEnvOrDie(shard_index_env, -1); if (total_shards == -1 && shard_index == -1) { return false; } else if (total_shards == -1 && shard_index != -1) { const Message msg = Message() << "Invalid environment variables: you have " << kTestShardIndex << " = " << shard_index << ", but have left " << kTestTotalShards << " unset.\n"; ColoredPrintf(COLOR_RED, "%s", msg.GetString().c_str()); fflush(stdout); exit(EXIT_FAILURE); } else if (total_shards != -1 && shard_index == -1) { const Message msg = Message() << "Invalid environment variables: you have " << kTestTotalShards << " = " << total_shards << ", but have left " << kTestShardIndex << " unset.\n"; ColoredPrintf(COLOR_RED, "%s", msg.GetString().c_str()); fflush(stdout); exit(EXIT_FAILURE); } else if (shard_index < 0 || shard_index >= total_shards) { const Message msg = Message() << "Invalid environment variables: we require 0 <= " << kTestShardIndex << " < " << kTestTotalShards << ", but you have " << kTestShardIndex << "=" << shard_index << ", " << kTestTotalShards << "=" << total_shards << ".\n"; ColoredPrintf(COLOR_RED, "%s", msg.GetString().c_str()); fflush(stdout); exit(EXIT_FAILURE); } return total_shards > 1; } // Parses the environment variable var as an Int32. If it is unset, // returns default_val. If it is not an Int32, prints an error // and aborts. Int32 Int32FromEnvOrDie(const char* var, Int32 default_val) { const char* str_val = posix::GetEnv(var); if (str_val == nullptr) { return default_val; } Int32 result; if (!ParseInt32(Message() << "The value of environment variable " << var, str_val, &result)) { exit(EXIT_FAILURE); } return result; } // Given the total number of shards, the shard index, and the test id, // returns true if and only if the test should be run on this shard. The test id // is some arbitrary but unique non-negative integer assigned to each test // method. Assumes that 0 <= shard_index < total_shards. bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id) { return (test_id % total_shards) == shard_index; } // Compares the name of each test with the user-specified filter to // decide whether the test should be run, then records the result in // each TestSuite and TestInfo object. // If shard_tests == true, further filters tests based on sharding // variables in the environment - see // https://github.com/google/googletest/blob/master/googletest/docs/advanced.md // . Returns the number of tests that should run. int UnitTestImpl::FilterTests(ReactionToSharding shard_tests) { const Int32 total_shards = shard_tests == HONOR_SHARDING_PROTOCOL ? Int32FromEnvOrDie(kTestTotalShards, -1) : -1; const Int32 shard_index = shard_tests == HONOR_SHARDING_PROTOCOL ? Int32FromEnvOrDie(kTestShardIndex, -1) : -1; // num_runnable_tests are the number of tests that will // run across all shards (i.e., match filter and are not disabled). // num_selected_tests are the number of tests to be run on // this shard. int num_runnable_tests = 0; int num_selected_tests = 0; for (auto* test_suite : test_suites_) { const std::string& test_suite_name = test_suite->name(); test_suite->set_should_run(false); for (size_t j = 0; j < test_suite->test_info_list().size(); j++) { TestInfo* const test_info = test_suite->test_info_list()[j]; const std::string test_name(test_info->name()); // A test is disabled if test suite name or test name matches // kDisableTestFilter. const bool is_disabled = internal::UnitTestOptions::MatchesFilter( test_suite_name, kDisableTestFilter) || internal::UnitTestOptions::MatchesFilter( test_name, kDisableTestFilter); test_info->is_disabled_ = is_disabled; const bool matches_filter = internal::UnitTestOptions::FilterMatchesTest( test_suite_name, test_name); test_info->matches_filter_ = matches_filter; const bool is_runnable = (GTEST_FLAG(also_run_disabled_tests) || !is_disabled) && matches_filter; const bool is_in_another_shard = shard_tests != IGNORE_SHARDING_PROTOCOL && !ShouldRunTestOnShard(total_shards, shard_index, num_runnable_tests); test_info->is_in_another_shard_ = is_in_another_shard; const bool is_selected = is_runnable && !is_in_another_shard; num_runnable_tests += is_runnable; num_selected_tests += is_selected; test_info->should_run_ = is_selected; test_suite->set_should_run(test_suite->should_run() || is_selected); } } return num_selected_tests; } // Prints the given C-string on a single line by replacing all '\n' // characters with string "\\n". If the output takes more than // max_length characters, only prints the first max_length characters // and "...". static void PrintOnOneLine(const char* str, int max_length) { if (str != nullptr) { for (int i = 0; *str != '\0'; ++str) { if (i >= max_length) { printf("..."); break; } if (*str == '\n') { printf("\\n"); i += 2; } else { printf("%c", *str); ++i; } } } } // Prints the names of the tests matching the user-specified filter flag. void UnitTestImpl::ListTestsMatchingFilter() { // Print at most this many characters for each type/value parameter. const int kMaxParamLength = 250; for (auto* test_suite : test_suites_) { bool printed_test_suite_name = false; for (size_t j = 0; j < test_suite->test_info_list().size(); j++) { const TestInfo* const test_info = test_suite->test_info_list()[j]; if (test_info->matches_filter_) { if (!printed_test_suite_name) { printed_test_suite_name = true; printf("%s.", test_suite->name()); if (test_suite->type_param() != nullptr) { printf(" # %s = ", kTypeParamLabel); // We print the type parameter on a single line to make // the output easy to parse by a program. PrintOnOneLine(test_suite->type_param(), kMaxParamLength); } printf("\n"); } printf(" %s", test_info->name()); if (test_info->value_param() != nullptr) { printf(" # %s = ", kValueParamLabel); // We print the value parameter on a single line to make the // output easy to parse by a program. PrintOnOneLine(test_info->value_param(), kMaxParamLength); } printf("\n"); } } } fflush(stdout); const std::string& output_format = UnitTestOptions::GetOutputFormat(); if (output_format == "xml" || output_format == "json") { FILE* fileout = OpenFileForWriting( UnitTestOptions::GetAbsolutePathToOutputFile().c_str()); std::stringstream stream; if (output_format == "xml") { XmlUnitTestResultPrinter( UnitTestOptions::GetAbsolutePathToOutputFile().c_str()) .PrintXmlTestsList(&stream, test_suites_); } else if (output_format == "json") { JsonUnitTestResultPrinter( UnitTestOptions::GetAbsolutePathToOutputFile().c_str()) .PrintJsonTestList(&stream, test_suites_); } fprintf(fileout, "%s", StringStreamToString(&stream).c_str()); fclose(fileout); } } // Sets the OS stack trace getter. // // Does nothing if the input and the current OS stack trace getter are // the same; otherwise, deletes the old getter and makes the input the // current getter. void UnitTestImpl::set_os_stack_trace_getter( OsStackTraceGetterInterface* getter) { if (os_stack_trace_getter_ != getter) { delete os_stack_trace_getter_; os_stack_trace_getter_ = getter; } } // Returns the current OS stack trace getter if it is not NULL; // otherwise, creates an OsStackTraceGetter, makes it the current // getter, and returns it. OsStackTraceGetterInterface* UnitTestImpl::os_stack_trace_getter() { if (os_stack_trace_getter_ == nullptr) { #ifdef GTEST_OS_STACK_TRACE_GETTER_ os_stack_trace_getter_ = new GTEST_OS_STACK_TRACE_GETTER_; #else os_stack_trace_getter_ = new OsStackTraceGetter; #endif // GTEST_OS_STACK_TRACE_GETTER_ } return os_stack_trace_getter_; } // Returns the most specific TestResult currently running. TestResult* UnitTestImpl::current_test_result() { if (current_test_info_ != nullptr) { return ¤t_test_info_->result_; } if (current_test_suite_ != nullptr) { return ¤t_test_suite_->ad_hoc_test_result_; } return &ad_hoc_test_result_; } // Shuffles all test suites, and the tests within each test suite, // making sure that death tests are still run first. void UnitTestImpl::ShuffleTests() { // Shuffles the death test suites. ShuffleRange(random(), 0, last_death_test_suite_ + 1, &test_suite_indices_); // Shuffles the non-death test suites. ShuffleRange(random(), last_death_test_suite_ + 1, static_cast(test_suites_.size()), &test_suite_indices_); // Shuffles the tests inside each test suite. for (auto& test_suite : test_suites_) { test_suite->ShuffleTests(random()); } } // Restores the test suites and tests to their order before the first shuffle. void UnitTestImpl::UnshuffleTests() { for (size_t i = 0; i < test_suites_.size(); i++) { // Unshuffles the tests in each test suite. test_suites_[i]->UnshuffleTests(); // Resets the index of each test suite. test_suite_indices_[i] = static_cast(i); } } // Returns the current OS stack trace as an std::string. // // The maximum number of stack frames to be included is specified by // the gtest_stack_trace_depth flag. The skip_count parameter // specifies the number of top frames to be skipped, which doesn't // count against the number of frames to be included. // // For example, if Foo() calls Bar(), which in turn calls // GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in // the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. std::string GetCurrentOsStackTraceExceptTop(UnitTest* /*unit_test*/, int skip_count) { // We pass skip_count + 1 to skip this wrapper function in addition // to what the user really wants to skip. return GetUnitTestImpl()->CurrentOsStackTraceExceptTop(skip_count + 1); } // Used by the GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_ macro to // suppress unreachable code warnings. namespace { class ClassUniqueToAlwaysTrue {}; } bool IsTrue(bool condition) { return condition; } bool AlwaysTrue() { #if GTEST_HAS_EXCEPTIONS // This condition is always false so AlwaysTrue() never actually throws, // but it makes the compiler think that it may throw. if (IsTrue(false)) throw ClassUniqueToAlwaysTrue(); #endif // GTEST_HAS_EXCEPTIONS return true; } // If *pstr starts with the given prefix, modifies *pstr to be right // past the prefix and returns true; otherwise leaves *pstr unchanged // and returns false. None of pstr, *pstr, and prefix can be NULL. bool SkipPrefix(const char* prefix, const char** pstr) { const size_t prefix_len = strlen(prefix); if (strncmp(*pstr, prefix, prefix_len) == 0) { *pstr += prefix_len; return true; } return false; } // Parses a string as a command line flag. The string should have // the format "--flag=value". When def_optional is true, the "=value" // part can be omitted. // // Returns the value of the flag, or NULL if the parsing failed. static const char* ParseFlagValue(const char* str, const char* flag, bool def_optional) { // str and flag must not be NULL. if (str == nullptr || flag == nullptr) return nullptr; // The flag must start with "--" followed by GTEST_FLAG_PREFIX_. const std::string flag_str = std::string("--") + GTEST_FLAG_PREFIX_ + flag; const size_t flag_len = flag_str.length(); if (strncmp(str, flag_str.c_str(), flag_len) != 0) return nullptr; // Skips the flag name. const char* flag_end = str + flag_len; // When def_optional is true, it's OK to not have a "=value" part. if (def_optional && (flag_end[0] == '\0')) { return flag_end; } // If def_optional is true and there are more characters after the // flag name, or if def_optional is false, there must be a '=' after // the flag name. if (flag_end[0] != '=') return nullptr; // Returns the string after "=". return flag_end + 1; } // Parses a string for a bool flag, in the form of either // "--flag=value" or "--flag". // // In the former case, the value is taken as true as long as it does // not start with '0', 'f', or 'F'. // // In the latter case, the value is taken as true. // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. static bool ParseBoolFlag(const char* str, const char* flag, bool* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, true); // Aborts if the parsing failed. if (value_str == nullptr) return false; // Converts the string value to a bool. *value = !(*value_str == '0' || *value_str == 'f' || *value_str == 'F'); return true; } // Parses a string for an Int32 flag, in the form of // "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. bool ParseInt32Flag(const char* str, const char* flag, Int32* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); // Aborts if the parsing failed. if (value_str == nullptr) return false; // Sets *value to the value of the flag. return ParseInt32(Message() << "The value of flag --" << flag, value_str, value); } // Parses a string for a string flag, in the form of // "--flag=value". // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. template static bool ParseStringFlag(const char* str, const char* flag, String* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); // Aborts if the parsing failed. if (value_str == nullptr) return false; // Sets *value to the value of the flag. *value = value_str; return true; } // Determines whether a string has a prefix that Google Test uses for its // flags, i.e., starts with GTEST_FLAG_PREFIX_ or GTEST_FLAG_PREFIX_DASH_. // If Google Test detects that a command line flag has its prefix but is not // recognized, it will print its help message. Flags starting with // GTEST_INTERNAL_PREFIX_ followed by "internal_" are considered Google Test // internal flags and do not trigger the help message. static bool HasGoogleTestFlagPrefix(const char* str) { return (SkipPrefix("--", &str) || SkipPrefix("-", &str) || SkipPrefix("/", &str)) && !SkipPrefix(GTEST_FLAG_PREFIX_ "internal_", &str) && (SkipPrefix(GTEST_FLAG_PREFIX_, &str) || SkipPrefix(GTEST_FLAG_PREFIX_DASH_, &str)); } // Prints a string containing code-encoded text. The following escape // sequences can be used in the string to control the text color: // // @@ prints a single '@' character. // @R changes the color to red. // @G changes the color to green. // @Y changes the color to yellow. // @D changes to the default terminal text color. // static void PrintColorEncoded(const char* str) { GTestColor color = COLOR_DEFAULT; // The current color. // Conceptually, we split the string into segments divided by escape // sequences. Then we print one segment at a time. At the end of // each iteration, the str pointer advances to the beginning of the // next segment. for (;;) { const char* p = strchr(str, '@'); if (p == nullptr) { ColoredPrintf(color, "%s", str); return; } ColoredPrintf(color, "%s", std::string(str, p).c_str()); const char ch = p[1]; str = p + 2; if (ch == '@') { ColoredPrintf(color, "@"); } else if (ch == 'D') { color = COLOR_DEFAULT; } else if (ch == 'R') { color = COLOR_RED; } else if (ch == 'G') { color = COLOR_GREEN; } else if (ch == 'Y') { color = COLOR_YELLOW; } else { --str; } } } static const char kColorEncodedHelpMessage[] = "This program contains tests written using " GTEST_NAME_ ". You can use the\n" "following command line flags to control its behavior:\n" "\n" "Test Selection:\n" " @G--" GTEST_FLAG_PREFIX_ "list_tests@D\n" " List the names of all tests instead of running them. The name of\n" " TEST(Foo, Bar) is \"Foo.Bar\".\n" " @G--" GTEST_FLAG_PREFIX_ "filter=@YPOSTIVE_PATTERNS" "[@G-@YNEGATIVE_PATTERNS]@D\n" " Run only the tests whose name matches one of the positive patterns but\n" " none of the negative patterns. '?' matches any single character; '*'\n" " matches any substring; ':' separates two patterns.\n" " @G--" GTEST_FLAG_PREFIX_ "also_run_disabled_tests@D\n" " Run all disabled tests too.\n" "\n" "Test Execution:\n" " @G--" GTEST_FLAG_PREFIX_ "repeat=@Y[COUNT]@D\n" " Run the tests repeatedly; use a negative count to repeat forever.\n" " @G--" GTEST_FLAG_PREFIX_ "shuffle@D\n" " Randomize tests' orders on every iteration.\n" " @G--" GTEST_FLAG_PREFIX_ "random_seed=@Y[NUMBER]@D\n" " Random number seed to use for shuffling test orders (between 1 and\n" " 99999, or 0 to use a seed based on the current time).\n" "\n" "Test Output:\n" " @G--" GTEST_FLAG_PREFIX_ "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n" " Enable/disable colored output. The default is @Gauto@D.\n" " -@G-" GTEST_FLAG_PREFIX_ "print_time=0@D\n" " Don't print the elapsed time of each test.\n" " @G--" GTEST_FLAG_PREFIX_ "output=@Y(@Gjson@Y|@Gxml@Y)[@G:@YDIRECTORY_PATH@G" GTEST_PATH_SEP_ "@Y|@G:@YFILE_PATH]@D\n" " Generate a JSON or XML report in the given directory or with the given\n" " file name. @YFILE_PATH@D defaults to @Gtest_detail.xml@D.\n" # if GTEST_CAN_STREAM_RESULTS_ " @G--" GTEST_FLAG_PREFIX_ "stream_result_to=@YHOST@G:@YPORT@D\n" " Stream test results to the given server.\n" # endif // GTEST_CAN_STREAM_RESULTS_ " @G--" GTEST_FLAG_PREFIX_ "print_skipped@D\n" " List all the skipped tests names in the summary\n" "\n" "Assertion Behavior:\n" # if GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS " @G--" GTEST_FLAG_PREFIX_ "death_test_style=@Y(@Gfast@Y|@Gthreadsafe@Y)@D\n" " Set the default death test style.\n" # endif // GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS " @G--" GTEST_FLAG_PREFIX_ "break_on_failure@D\n" " Turn assertion failures into debugger break-points.\n" " @G--" GTEST_FLAG_PREFIX_ "throw_on_failure@D\n" " Turn assertion failures into C++ exceptions for use by an external\n" " test framework.\n" " @G--" GTEST_FLAG_PREFIX_ "catch_exceptions=0@D\n" " Do not report exceptions as test failures. Instead, allow them\n" " to crash the program or throw a pop-up (on Windows).\n" "\n" "Except for @G--" GTEST_FLAG_PREFIX_ "list_tests@D, you can alternatively set " "the corresponding\n" "environment variable of a flag (all letters in upper-case). For example, to\n" "disable colored text output, you can either specify @G--" GTEST_FLAG_PREFIX_ "color=no@D or set\n" "the @G" GTEST_FLAG_PREFIX_UPPER_ "COLOR@D environment variable to @Gno@D.\n" "\n" "For more information, please read the " GTEST_NAME_ " documentation at\n" "@G" GTEST_PROJECT_URL_ "@D. If you find a bug in " GTEST_NAME_ "\n" "(not one in your own code or tests), please report it to\n" "@G<" GTEST_DEV_EMAIL_ ">@D.\n"; static bool ParseGoogleTestFlag(const char* const arg) { return ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag, >EST_FLAG(also_run_disabled_tests)) || ParseBoolFlag(arg, kBreakOnFailureFlag, >EST_FLAG(break_on_failure)) || ParseBoolFlag(arg, kCatchExceptionsFlag, >EST_FLAG(catch_exceptions)) || ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) || ParseStringFlag(arg, kDeathTestStyleFlag, >EST_FLAG(death_test_style)) || ParseBoolFlag(arg, kDeathTestUseFork, >EST_FLAG(death_test_use_fork)) || ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) || ParseStringFlag(arg, kInternalRunDeathTestFlag, >EST_FLAG(internal_run_death_test)) || ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) || ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) || ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) || ParseBoolFlag(arg, kPrintUTF8Flag, >EST_FLAG(print_utf8)) || ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) || ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) || ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) || ParseInt32Flag(arg, kStackTraceDepthFlag, >EST_FLAG(stack_trace_depth)) || ParseStringFlag(arg, kStreamResultToFlag, >EST_FLAG(stream_result_to)) || ParseBoolFlag(arg, kThrowOnFailureFlag, >EST_FLAG(throw_on_failure)) || ParseBoolFlag(arg, kPrintSkippedFlag, >EST_FLAG(print_skipped)); } #if GTEST_USE_OWN_FLAGFILE_FLAG_ static void LoadFlagsFromFile(const std::string& path) { FILE* flagfile = posix::FOpen(path.c_str(), "r"); if (!flagfile) { GTEST_LOG_(FATAL) << "Unable to open file \"" << GTEST_FLAG(flagfile) << "\""; } std::string contents(ReadEntireFile(flagfile)); posix::FClose(flagfile); std::vector lines; SplitString(contents, '\n', &lines); for (size_t i = 0; i < lines.size(); ++i) { if (lines[i].empty()) continue; if (!ParseGoogleTestFlag(lines[i].c_str())) g_help_flag = true; } } #endif // GTEST_USE_OWN_FLAGFILE_FLAG_ // Parses the command line for Google Test flags, without initializing // other parts of Google Test. The type parameter CharType can be // instantiated to either char or wchar_t. template void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) { for (int i = 1; i < *argc; i++) { const std::string arg_string = StreamableToString(argv[i]); const char* const arg = arg_string.c_str(); using internal::ParseBoolFlag; using internal::ParseInt32Flag; using internal::ParseStringFlag; bool remove_flag = false; if (ParseGoogleTestFlag(arg)) { remove_flag = true; #if GTEST_USE_OWN_FLAGFILE_FLAG_ } else if (ParseStringFlag(arg, kFlagfileFlag, >EST_FLAG(flagfile))) { LoadFlagsFromFile(GTEST_FLAG(flagfile)); remove_flag = true; #endif // GTEST_USE_OWN_FLAGFILE_FLAG_ } else if (arg_string == "--help" || arg_string == "-h" || arg_string == "-?" || arg_string == "/?" || HasGoogleTestFlagPrefix(arg)) { // Both help flag and unrecognized Google Test flags (excluding // internal ones) trigger help display. g_help_flag = true; } if (remove_flag) { // Shift the remainder of the argv list left by one. Note // that argv has (*argc + 1) elements, the last one always being // NULL. The following loop moves the trailing NULL element as // well. for (int j = i; j != *argc; j++) { argv[j] = argv[j + 1]; } // Decrements the argument count. (*argc)--; // We also need to decrement the iterator as we just removed // an element. i--; } } if (g_help_flag) { // We print the help here instead of in RUN_ALL_TESTS(), as the // latter may not be called at all if the user is using Google // Test with another testing framework. PrintColorEncoded(kColorEncodedHelpMessage); } } // Parses the command line for Google Test flags, without initializing // other parts of Google Test. void ParseGoogleTestFlagsOnly(int* argc, char** argv) { ParseGoogleTestFlagsOnlyImpl(argc, argv); // Fix the value of *_NSGetArgc() on macOS, but if and only if // *_NSGetArgv() == argv // Only applicable to char** version of argv #if GTEST_OS_MAC #ifndef GTEST_OS_IOS if (*_NSGetArgv() == argv) { *_NSGetArgc() = *argc; } #endif #endif } void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv) { ParseGoogleTestFlagsOnlyImpl(argc, argv); } // The internal implementation of InitGoogleTest(). // // The type parameter CharType can be instantiated to either char or // wchar_t. template void InitGoogleTestImpl(int* argc, CharType** argv) { // We don't want to run the initialization code twice. if (GTestIsInitialized()) return; if (*argc <= 0) return; g_argvs.clear(); for (int i = 0; i != *argc; i++) { g_argvs.push_back(StreamableToString(argv[i])); } #if GTEST_HAS_ABSL absl::InitializeSymbolizer(g_argvs[0].c_str()); #endif // GTEST_HAS_ABSL ParseGoogleTestFlagsOnly(argc, argv); GetUnitTestImpl()->PostFlagParsingInit(); } } // namespace internal // Initializes Google Test. This must be called before calling // RUN_ALL_TESTS(). In particular, it parses a command line for the // flags that Google Test recognizes. Whenever a Google Test flag is // seen, it is removed from argv, and *argc is decremented. // // No value is returned. Instead, the Google Test flag variables are // updated. // // Calling the function for the second time has no user-visible effect. void InitGoogleTest(int* argc, char** argv) { #if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv); #else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) internal::InitGoogleTestImpl(argc, argv); #endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) } // This overloaded version can be used in Windows programs compiled in // UNICODE mode. void InitGoogleTest(int* argc, wchar_t** argv) { #if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(argc, argv); #else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) internal::InitGoogleTestImpl(argc, argv); #endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) } // This overloaded version can be used on Arduino/embedded platforms where // there is no argc/argv. void InitGoogleTest() { // Since Arduino doesn't have a command line, fake out the argc/argv arguments int argc = 1; const auto arg0 = "dummy"; char* argv0 = const_cast(arg0); char** argv = &argv0; #if defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_(&argc, argv); #else // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) internal::InitGoogleTestImpl(&argc, argv); #endif // defined(GTEST_CUSTOM_INIT_GOOGLE_TEST_FUNCTION_) } std::string TempDir() { #if defined(GTEST_CUSTOM_TEMPDIR_FUNCTION_) return GTEST_CUSTOM_TEMPDIR_FUNCTION_(); #endif #if GTEST_OS_WINDOWS_MOBILE return "\\temp\\"; #elif GTEST_OS_WINDOWS const char* temp_dir = internal::posix::GetEnv("TEMP"); if (temp_dir == nullptr || temp_dir[0] == '\0') return "\\temp\\"; else if (temp_dir[strlen(temp_dir) - 1] == '\\') return temp_dir; else return std::string(temp_dir) + "\\"; #elif GTEST_OS_LINUX_ANDROID return "/sdcard/"; #else return "/tmp/"; #endif // GTEST_OS_WINDOWS_MOBILE } // Class ScopedTrace // Pushes the given source file location and message onto a per-thread // trace stack maintained by Google Test. void ScopedTrace::PushTrace(const char* file, int line, std::string message) { internal::TraceInfo trace; trace.file = file; trace.line = line; trace.message.swap(message); UnitTest::GetInstance()->PushGTestTrace(trace); } // Pops the info pushed by the c'tor. ScopedTrace::~ScopedTrace() GTEST_LOCK_EXCLUDED_(&UnitTest::mutex_) { UnitTest::GetInstance()->PopGTestTrace(); } } // namespace testing // Copyright 2005, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // This file implements death tests. #include #if GTEST_HAS_DEATH_TEST # if GTEST_OS_MAC # include # endif // GTEST_OS_MAC # include # include # include # if GTEST_OS_LINUX # include # endif // GTEST_OS_LINUX # include # if GTEST_OS_WINDOWS # include # else # include # include # endif // GTEST_OS_WINDOWS # if GTEST_OS_QNX # include # endif // GTEST_OS_QNX # if GTEST_OS_FUCHSIA # include # include # include # include # include # include # include # include # include # include # include # endif // GTEST_OS_FUCHSIA #endif // GTEST_HAS_DEATH_TEST namespace testing { // Constants. // The default death test style. // // This is defined in internal/gtest-port.h as "fast", but can be overridden by // a definition in internal/custom/gtest-port.h. The recommended value, which is // used internally at Google, is "threadsafe". static const char kDefaultDeathTestStyle[] = GTEST_DEFAULT_DEATH_TEST_STYLE; GTEST_DEFINE_string_( death_test_style, internal::StringFromGTestEnv("death_test_style", kDefaultDeathTestStyle), "Indicates how to run a death test in a forked child process: " "\"threadsafe\" (child process re-executes the test binary " "from the beginning, running only the specific death test) or " "\"fast\" (child process runs the death test immediately " "after forking)."); GTEST_DEFINE_bool_( death_test_use_fork, internal::BoolFromGTestEnv("death_test_use_fork", false), "Instructs to use fork()/_exit() instead of clone() in death tests. " "Ignored and always uses fork() on POSIX systems where clone() is not " "implemented. Useful when running under valgrind or similar tools if " "those do not support clone(). Valgrind 3.3.1 will just fail if " "it sees an unsupported combination of clone() flags. " "It is not recommended to use this flag w/o valgrind though it will " "work in 99% of the cases. Once valgrind is fixed, this flag will " "most likely be removed."); namespace internal { GTEST_DEFINE_string_( internal_run_death_test, "", "Indicates the file, line number, temporal index of " "the single death test to run, and a file descriptor to " "which a success code may be sent, all separated by " "the '|' characters. This flag is specified if and only if the " "current process is a sub-process launched for running a thread-safe " "death test. FOR INTERNAL USE ONLY."); } // namespace internal #if GTEST_HAS_DEATH_TEST namespace internal { // Valid only for fast death tests. Indicates the code is running in the // child process of a fast style death test. # if !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA static bool g_in_fast_death_test_child = false; # endif // Returns a Boolean value indicating whether the caller is currently // executing in the context of the death test child process. Tools such as // Valgrind heap checkers may need this to modify their behavior in death // tests. IMPORTANT: This is an internal utility. Using it may break the // implementation of death tests. User code MUST NOT use it. bool InDeathTestChild() { # if GTEST_OS_WINDOWS || GTEST_OS_FUCHSIA // On Windows and Fuchsia, death tests are thread-safe regardless of the value // of the death_test_style flag. return !GTEST_FLAG(internal_run_death_test).empty(); # else if (GTEST_FLAG(death_test_style) == "threadsafe") return !GTEST_FLAG(internal_run_death_test).empty(); else return g_in_fast_death_test_child; #endif } } // namespace internal // ExitedWithCode constructor. ExitedWithCode::ExitedWithCode(int exit_code) : exit_code_(exit_code) { } // ExitedWithCode function-call operator. bool ExitedWithCode::operator()(int exit_status) const { # if GTEST_OS_WINDOWS || GTEST_OS_FUCHSIA return exit_status == exit_code_; # else return WIFEXITED(exit_status) && WEXITSTATUS(exit_status) == exit_code_; # endif // GTEST_OS_WINDOWS || GTEST_OS_FUCHSIA } # if !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA // KilledBySignal constructor. KilledBySignal::KilledBySignal(int signum) : signum_(signum) { } // KilledBySignal function-call operator. bool KilledBySignal::operator()(int exit_status) const { # if defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_) { bool result; if (GTEST_KILLED_BY_SIGNAL_OVERRIDE_(signum_, exit_status, &result)) { return result; } } # endif // defined(GTEST_KILLED_BY_SIGNAL_OVERRIDE_) return WIFSIGNALED(exit_status) && WTERMSIG(exit_status) == signum_; } # endif // !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA namespace internal { // Utilities needed for death tests. // Generates a textual description of a given exit code, in the format // specified by wait(2). static std::string ExitSummary(int exit_code) { Message m; # if GTEST_OS_WINDOWS || GTEST_OS_FUCHSIA m << "Exited with exit status " << exit_code; # else if (WIFEXITED(exit_code)) { m << "Exited with exit status " << WEXITSTATUS(exit_code); } else if (WIFSIGNALED(exit_code)) { m << "Terminated by signal " << WTERMSIG(exit_code); } # ifdef WCOREDUMP if (WCOREDUMP(exit_code)) { m << " (core dumped)"; } # endif # endif // GTEST_OS_WINDOWS || GTEST_OS_FUCHSIA return m.GetString(); } // Returns true if exit_status describes a process that was terminated // by a signal, or exited normally with a nonzero exit code. bool ExitedUnsuccessfully(int exit_status) { return !ExitedWithCode(0)(exit_status); } # if !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA // Generates a textual failure message when a death test finds more than // one thread running, or cannot determine the number of threads, prior // to executing the given statement. It is the responsibility of the // caller not to pass a thread_count of 1. static std::string DeathTestThreadWarning(size_t thread_count) { Message msg; msg << "Death tests use fork(), which is unsafe particularly" << " in a threaded context. For this test, " << GTEST_NAME_ << " "; if (thread_count == 0) { msg << "couldn't detect the number of threads."; } else { msg << "detected " << thread_count << " threads."; } msg << " See " "https://github.com/google/googletest/blob/master/googletest/docs/" "advanced.md#death-tests-and-threads" << " for more explanation and suggested solutions, especially if" << " this is the last message you see before your test times out."; return msg.GetString(); } # endif // !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA // Flag characters for reporting a death test that did not die. static const char kDeathTestLived = 'L'; static const char kDeathTestReturned = 'R'; static const char kDeathTestThrew = 'T'; static const char kDeathTestInternalError = 'I'; #if GTEST_OS_FUCHSIA // File descriptor used for the pipe in the child process. static const int kFuchsiaReadPipeFd = 3; #endif // An enumeration describing all of the possible ways that a death test can // conclude. DIED means that the process died while executing the test // code; LIVED means that process lived beyond the end of the test code; // RETURNED means that the test statement attempted to execute a return // statement, which is not allowed; THREW means that the test statement // returned control by throwing an exception. IN_PROGRESS means the test // has not yet concluded. enum DeathTestOutcome { IN_PROGRESS, DIED, LIVED, RETURNED, THREW }; // Routine for aborting the program which is safe to call from an // exec-style death test child process, in which case the error // message is propagated back to the parent process. Otherwise, the // message is simply printed to stderr. In either case, the program // then exits with status 1. static void DeathTestAbort(const std::string& message) { // On a POSIX system, this function may be called from a threadsafe-style // death test child process, which operates on a very small stack. Use // the heap for any additional non-minuscule memory requirements. const InternalRunDeathTestFlag* const flag = GetUnitTestImpl()->internal_run_death_test_flag(); if (flag != nullptr) { FILE* parent = posix::FDOpen(flag->write_fd(), "w"); fputc(kDeathTestInternalError, parent); fprintf(parent, "%s", message.c_str()); fflush(parent); _exit(1); } else { fprintf(stderr, "%s", message.c_str()); fflush(stderr); posix::Abort(); } } // A replacement for CHECK that calls DeathTestAbort if the assertion // fails. # define GTEST_DEATH_TEST_CHECK_(expression) \ do { \ if (!::testing::internal::IsTrue(expression)) { \ DeathTestAbort( \ ::std::string("CHECK failed: File ") + __FILE__ + ", line " \ + ::testing::internal::StreamableToString(__LINE__) + ": " \ + #expression); \ } \ } while (::testing::internal::AlwaysFalse()) // This macro is similar to GTEST_DEATH_TEST_CHECK_, but it is meant for // evaluating any system call that fulfills two conditions: it must return // -1 on failure, and set errno to EINTR when it is interrupted and // should be tried again. The macro expands to a loop that repeatedly // evaluates the expression as long as it evaluates to -1 and sets // errno to EINTR. If the expression evaluates to -1 but errno is // something other than EINTR, DeathTestAbort is called. # define GTEST_DEATH_TEST_CHECK_SYSCALL_(expression) \ do { \ int gtest_retval; \ do { \ gtest_retval = (expression); \ } while (gtest_retval == -1 && errno == EINTR); \ if (gtest_retval == -1) { \ DeathTestAbort( \ ::std::string("CHECK failed: File ") + __FILE__ + ", line " \ + ::testing::internal::StreamableToString(__LINE__) + ": " \ + #expression + " != -1"); \ } \ } while (::testing::internal::AlwaysFalse()) // Returns the message describing the last system error in errno. std::string GetLastErrnoDescription() { return errno == 0 ? "" : posix::StrError(errno); } // This is called from a death test parent process to read a failure // message from the death test child process and log it with the FATAL // severity. On Windows, the message is read from a pipe handle. On other // platforms, it is read from a file descriptor. static void FailFromInternalError(int fd) { Message error; char buffer[256]; int num_read; do { while ((num_read = posix::Read(fd, buffer, 255)) > 0) { buffer[num_read] = '\0'; error << buffer; } } while (num_read == -1 && errno == EINTR); if (num_read == 0) { GTEST_LOG_(FATAL) << error.GetString(); } else { const int last_error = errno; GTEST_LOG_(FATAL) << "Error while reading death test internal: " << GetLastErrnoDescription() << " [" << last_error << "]"; } } // Death test constructor. Increments the running death test count // for the current test. DeathTest::DeathTest() { TestInfo* const info = GetUnitTestImpl()->current_test_info(); if (info == nullptr) { DeathTestAbort("Cannot run a death test outside of a TEST or " "TEST_F construct"); } } // Creates and returns a death test by dispatching to the current // death test factory. bool DeathTest::Create(const char* statement, Matcher matcher, const char* file, int line, DeathTest** test) { return GetUnitTestImpl()->death_test_factory()->Create( statement, std::move(matcher), file, line, test); } const char* DeathTest::LastMessage() { return last_death_test_message_.c_str(); } void DeathTest::set_last_death_test_message(const std::string& message) { last_death_test_message_ = message; } std::string DeathTest::last_death_test_message_; // Provides cross platform implementation for some death functionality. class DeathTestImpl : public DeathTest { protected: DeathTestImpl(const char* a_statement, Matcher matcher) : statement_(a_statement), matcher_(std::move(matcher)), spawned_(false), status_(-1), outcome_(IN_PROGRESS), read_fd_(-1), write_fd_(-1) {} // read_fd_ is expected to be closed and cleared by a derived class. ~DeathTestImpl() override { GTEST_DEATH_TEST_CHECK_(read_fd_ == -1); } void Abort(AbortReason reason) override; bool Passed(bool status_ok) override; const char* statement() const { return statement_; } bool spawned() const { return spawned_; } void set_spawned(bool is_spawned) { spawned_ = is_spawned; } int status() const { return status_; } void set_status(int a_status) { status_ = a_status; } DeathTestOutcome outcome() const { return outcome_; } void set_outcome(DeathTestOutcome an_outcome) { outcome_ = an_outcome; } int read_fd() const { return read_fd_; } void set_read_fd(int fd) { read_fd_ = fd; } int write_fd() const { return write_fd_; } void set_write_fd(int fd) { write_fd_ = fd; } // Called in the parent process only. Reads the result code of the death // test child process via a pipe, interprets it to set the outcome_ // member, and closes read_fd_. Outputs diagnostics and terminates in // case of unexpected codes. void ReadAndInterpretStatusByte(); // Returns stderr output from the child process. virtual std::string GetErrorLogs(); private: // The textual content of the code this object is testing. This class // doesn't own this string and should not attempt to delete it. const char* const statement_; // A matcher that's expected to match the stderr output by the child process. Matcher matcher_; // True if the death test child process has been successfully spawned. bool spawned_; // The exit status of the child process. int status_; // How the death test concluded. DeathTestOutcome outcome_; // Descriptor to the read end of the pipe to the child process. It is // always -1 in the child process. The child keeps its write end of the // pipe in write_fd_. int read_fd_; // Descriptor to the child's write end of the pipe to the parent process. // It is always -1 in the parent process. The parent keeps its end of the // pipe in read_fd_. int write_fd_; }; // Called in the parent process only. Reads the result code of the death // test child process via a pipe, interprets it to set the outcome_ // member, and closes read_fd_. Outputs diagnostics and terminates in // case of unexpected codes. void DeathTestImpl::ReadAndInterpretStatusByte() { char flag; int bytes_read; // The read() here blocks until data is available (signifying the // failure of the death test) or until the pipe is closed (signifying // its success), so it's okay to call this in the parent before // the child process has exited. do { bytes_read = posix::Read(read_fd(), &flag, 1); } while (bytes_read == -1 && errno == EINTR); if (bytes_read == 0) { set_outcome(DIED); } else if (bytes_read == 1) { switch (flag) { case kDeathTestReturned: set_outcome(RETURNED); break; case kDeathTestThrew: set_outcome(THREW); break; case kDeathTestLived: set_outcome(LIVED); break; case kDeathTestInternalError: FailFromInternalError(read_fd()); // Does not return. break; default: GTEST_LOG_(FATAL) << "Death test child process reported " << "unexpected status byte (" << static_cast(flag) << ")"; } } else { GTEST_LOG_(FATAL) << "Read from death test child process failed: " << GetLastErrnoDescription(); } GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Close(read_fd())); set_read_fd(-1); } std::string DeathTestImpl::GetErrorLogs() { return GetCapturedStderr(); } // Signals that the death test code which should have exited, didn't. // Should be called only in a death test child process. // Writes a status byte to the child's status file descriptor, then // calls _exit(1). void DeathTestImpl::Abort(AbortReason reason) { // The parent process considers the death test to be a failure if // it finds any data in our pipe. So, here we write a single flag byte // to the pipe, then exit. const char status_ch = reason == TEST_DID_NOT_DIE ? kDeathTestLived : reason == TEST_THREW_EXCEPTION ? kDeathTestThrew : kDeathTestReturned; GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Write(write_fd(), &status_ch, 1)); // We are leaking the descriptor here because on some platforms (i.e., // when built as Windows DLL), destructors of global objects will still // run after calling _exit(). On such systems, write_fd_ will be // indirectly closed from the destructor of UnitTestImpl, causing double // close if it is also closed here. On debug configurations, double close // may assert. As there are no in-process buffers to flush here, we are // relying on the OS to close the descriptor after the process terminates // when the destructors are not run. _exit(1); // Exits w/o any normal exit hooks (we were supposed to crash) } // Returns an indented copy of stderr output for a death test. // This makes distinguishing death test output lines from regular log lines // much easier. static ::std::string FormatDeathTestOutput(const ::std::string& output) { ::std::string ret; for (size_t at = 0; ; ) { const size_t line_end = output.find('\n', at); ret += "[ DEATH ] "; if (line_end == ::std::string::npos) { ret += output.substr(at); break; } ret += output.substr(at, line_end + 1 - at); at = line_end + 1; } return ret; } // Assesses the success or failure of a death test, using both private // members which have previously been set, and one argument: // // Private data members: // outcome: An enumeration describing how the death test // concluded: DIED, LIVED, THREW, or RETURNED. The death test // fails in the latter three cases. // status: The exit status of the child process. On *nix, it is in the // in the format specified by wait(2). On Windows, this is the // value supplied to the ExitProcess() API or a numeric code // of the exception that terminated the program. // matcher_: A matcher that's expected to match the stderr output by the child // process. // // Argument: // status_ok: true if exit_status is acceptable in the context of // this particular death test, which fails if it is false // // Returns true if and only if all of the above conditions are met. Otherwise, // the first failing condition, in the order given above, is the one that is // reported. Also sets the last death test message string. bool DeathTestImpl::Passed(bool status_ok) { if (!spawned()) return false; const std::string error_message = GetErrorLogs(); bool success = false; Message buffer; buffer << "Death test: " << statement() << "\n"; switch (outcome()) { case LIVED: buffer << " Result: failed to die.\n" << " Error msg:\n" << FormatDeathTestOutput(error_message); break; case THREW: buffer << " Result: threw an exception.\n" << " Error msg:\n" << FormatDeathTestOutput(error_message); break; case RETURNED: buffer << " Result: illegal return in test statement.\n" << " Error msg:\n" << FormatDeathTestOutput(error_message); break; case DIED: if (status_ok) { if (matcher_.Matches(error_message)) { success = true; } else { std::ostringstream stream; matcher_.DescribeTo(&stream); buffer << " Result: died but not with expected error.\n" << " Expected: " << stream.str() << "\n" << "Actual msg:\n" << FormatDeathTestOutput(error_message); } } else { buffer << " Result: died but not with expected exit code:\n" << " " << ExitSummary(status()) << "\n" << "Actual msg:\n" << FormatDeathTestOutput(error_message); } break; case IN_PROGRESS: default: GTEST_LOG_(FATAL) << "DeathTest::Passed somehow called before conclusion of test"; } DeathTest::set_last_death_test_message(buffer.GetString()); return success; } # if GTEST_OS_WINDOWS // WindowsDeathTest implements death tests on Windows. Due to the // specifics of starting new processes on Windows, death tests there are // always threadsafe, and Google Test considers the // --gtest_death_test_style=fast setting to be equivalent to // --gtest_death_test_style=threadsafe there. // // A few implementation notes: Like the Linux version, the Windows // implementation uses pipes for child-to-parent communication. But due to // the specifics of pipes on Windows, some extra steps are required: // // 1. The parent creates a communication pipe and stores handles to both // ends of it. // 2. The parent starts the child and provides it with the information // necessary to acquire the handle to the write end of the pipe. // 3. The child acquires the write end of the pipe and signals the parent // using a Windows event. // 4. Now the parent can release the write end of the pipe on its side. If // this is done before step 3, the object's reference count goes down to // 0 and it is destroyed, preventing the child from acquiring it. The // parent now has to release it, or read operations on the read end of // the pipe will not return when the child terminates. // 5. The parent reads child's output through the pipe (outcome code and // any possible error messages) from the pipe, and its stderr and then // determines whether to fail the test. // // Note: to distinguish Win32 API calls from the local method and function // calls, the former are explicitly resolved in the global namespace. // class WindowsDeathTest : public DeathTestImpl { public: WindowsDeathTest(const char* a_statement, Matcher matcher, const char* file, int line) : DeathTestImpl(a_statement, std::move(matcher)), file_(file), line_(line) {} // All of these virtual functions are inherited from DeathTest. virtual int Wait(); virtual TestRole AssumeRole(); private: // The name of the file in which the death test is located. const char* const file_; // The line number on which the death test is located. const int line_; // Handle to the write end of the pipe to the child process. AutoHandle write_handle_; // Child process handle. AutoHandle child_handle_; // Event the child process uses to signal the parent that it has // acquired the handle to the write end of the pipe. After seeing this // event the parent can release its own handles to make sure its // ReadFile() calls return when the child terminates. AutoHandle event_handle_; }; // Waits for the child in a death test to exit, returning its exit // status, or 0 if no child process exists. As a side effect, sets the // outcome data member. int WindowsDeathTest::Wait() { if (!spawned()) return 0; // Wait until the child either signals that it has acquired the write end // of the pipe or it dies. const HANDLE wait_handles[2] = { child_handle_.Get(), event_handle_.Get() }; switch (::WaitForMultipleObjects(2, wait_handles, FALSE, // Waits for any of the handles. INFINITE)) { case WAIT_OBJECT_0: case WAIT_OBJECT_0 + 1: break; default: GTEST_DEATH_TEST_CHECK_(false); // Should not get here. } // The child has acquired the write end of the pipe or exited. // We release the handle on our side and continue. write_handle_.Reset(); event_handle_.Reset(); ReadAndInterpretStatusByte(); // Waits for the child process to exit if it haven't already. This // returns immediately if the child has already exited, regardless of // whether previous calls to WaitForMultipleObjects synchronized on this // handle or not. GTEST_DEATH_TEST_CHECK_( WAIT_OBJECT_0 == ::WaitForSingleObject(child_handle_.Get(), INFINITE)); DWORD status_code; GTEST_DEATH_TEST_CHECK_( ::GetExitCodeProcess(child_handle_.Get(), &status_code) != FALSE); child_handle_.Reset(); set_status(static_cast(status_code)); return status(); } // The AssumeRole process for a Windows death test. It creates a child // process with the same executable as the current process to run the // death test. The child process is given the --gtest_filter and // --gtest_internal_run_death_test flags such that it knows to run the // current death test only. DeathTest::TestRole WindowsDeathTest::AssumeRole() { const UnitTestImpl* const impl = GetUnitTestImpl(); const InternalRunDeathTestFlag* const flag = impl->internal_run_death_test_flag(); const TestInfo* const info = impl->current_test_info(); const int death_test_index = info->result()->death_test_count(); if (flag != nullptr) { // ParseInternalRunDeathTestFlag() has performed all the necessary // processing. set_write_fd(flag->write_fd()); return EXECUTE_TEST; } // WindowsDeathTest uses an anonymous pipe to communicate results of // a death test. SECURITY_ATTRIBUTES handles_are_inheritable = {sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE}; HANDLE read_handle, write_handle; GTEST_DEATH_TEST_CHECK_( ::CreatePipe(&read_handle, &write_handle, &handles_are_inheritable, 0) // Default buffer size. != FALSE); set_read_fd(::_open_osfhandle(reinterpret_cast(read_handle), O_RDONLY)); write_handle_.Reset(write_handle); event_handle_.Reset(::CreateEvent( &handles_are_inheritable, TRUE, // The event will automatically reset to non-signaled state. FALSE, // The initial state is non-signalled. nullptr)); // The even is unnamed. GTEST_DEATH_TEST_CHECK_(event_handle_.Get() != nullptr); const std::string filter_flag = std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + info->test_suite_name() + "." + info->name(); const std::string internal_flag = std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + "=" + file_ + "|" + StreamableToString(line_) + "|" + StreamableToString(death_test_index) + "|" + StreamableToString(static_cast(::GetCurrentProcessId())) + // size_t has the same width as pointers on both 32-bit and 64-bit // Windows platforms. // See http://msdn.microsoft.com/en-us/library/tcxf1dw6.aspx. "|" + StreamableToString(reinterpret_cast(write_handle)) + "|" + StreamableToString(reinterpret_cast(event_handle_.Get())); char executable_path[_MAX_PATH + 1]; // NOLINT GTEST_DEATH_TEST_CHECK_(_MAX_PATH + 1 != ::GetModuleFileNameA(nullptr, executable_path, _MAX_PATH)); std::string command_line = std::string(::GetCommandLineA()) + " " + filter_flag + " \"" + internal_flag + "\""; DeathTest::set_last_death_test_message(""); CaptureStderr(); // Flush the log buffers since the log streams are shared with the child. FlushInfoLog(); // The child process will share the standard handles with the parent. STARTUPINFOA startup_info; memset(&startup_info, 0, sizeof(STARTUPINFO)); startup_info.dwFlags = STARTF_USESTDHANDLES; startup_info.hStdInput = ::GetStdHandle(STD_INPUT_HANDLE); startup_info.hStdOutput = ::GetStdHandle(STD_OUTPUT_HANDLE); startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); PROCESS_INFORMATION process_info; GTEST_DEATH_TEST_CHECK_( ::CreateProcessA( executable_path, const_cast(command_line.c_str()), nullptr, // Retuned process handle is not inheritable. nullptr, // Retuned thread handle is not inheritable. TRUE, // Child inherits all inheritable handles (for write_handle_). 0x0, // Default creation flags. nullptr, // Inherit the parent's environment. UnitTest::GetInstance()->original_working_dir(), &startup_info, &process_info) != FALSE); child_handle_.Reset(process_info.hProcess); ::CloseHandle(process_info.hThread); set_spawned(true); return OVERSEE_TEST; } # elif GTEST_OS_FUCHSIA class FuchsiaDeathTest : public DeathTestImpl { public: FuchsiaDeathTest(const char* a_statement, Matcher matcher, const char* file, int line) : DeathTestImpl(a_statement, std::move(matcher)), file_(file), line_(line) {} // All of these virtual functions are inherited from DeathTest. int Wait() override; TestRole AssumeRole() override; std::string GetErrorLogs() override; private: // The name of the file in which the death test is located. const char* const file_; // The line number on which the death test is located. const int line_; // The stderr data captured by the child process. std::string captured_stderr_; zx::process child_process_; zx::channel exception_channel_; zx::socket stderr_socket_; }; // Utility class for accumulating command-line arguments. class Arguments { public: Arguments() { args_.push_back(nullptr); } ~Arguments() { for (std::vector::iterator i = args_.begin(); i != args_.end(); ++i) { free(*i); } } void AddArgument(const char* argument) { args_.insert(args_.end() - 1, posix::StrDup(argument)); } template void AddArguments(const ::std::vector& arguments) { for (typename ::std::vector::const_iterator i = arguments.begin(); i != arguments.end(); ++i) { args_.insert(args_.end() - 1, posix::StrDup(i->c_str())); } } char* const* Argv() { return &args_[0]; } int size() { return args_.size() - 1; } private: std::vector args_; }; // Waits for the child in a death test to exit, returning its exit // status, or 0 if no child process exists. As a side effect, sets the // outcome data member. int FuchsiaDeathTest::Wait() { const int kProcessKey = 0; const int kSocketKey = 1; const int kExceptionKey = 2; if (!spawned()) return 0; // Create a port to wait for socket/task/exception events. zx_status_t status_zx; zx::port port; status_zx = zx::port::create(0, &port); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); // Register to wait for the child process to terminate. status_zx = child_process_.wait_async( port, kProcessKey, ZX_PROCESS_TERMINATED, ZX_WAIT_ASYNC_ONCE); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); // Register to wait for the socket to be readable or closed. status_zx = stderr_socket_.wait_async( port, kSocketKey, ZX_SOCKET_READABLE | ZX_SOCKET_PEER_CLOSED, ZX_WAIT_ASYNC_ONCE); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); // Register to wait for an exception. status_zx = exception_channel_.wait_async( port, kExceptionKey, ZX_CHANNEL_READABLE, ZX_WAIT_ASYNC_ONCE); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); bool process_terminated = false; bool socket_closed = false; do { zx_port_packet_t packet = {}; status_zx = port.wait(zx::time::infinite(), &packet); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); if (packet.key == kExceptionKey) { // Process encountered an exception. Kill it directly rather than // letting other handlers process the event. We will get a kProcessKey // event when the process actually terminates. status_zx = child_process_.kill(); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); } else if (packet.key == kProcessKey) { // Process terminated. GTEST_DEATH_TEST_CHECK_(ZX_PKT_IS_SIGNAL_ONE(packet.type)); GTEST_DEATH_TEST_CHECK_(packet.signal.observed & ZX_PROCESS_TERMINATED); process_terminated = true; } else if (packet.key == kSocketKey) { GTEST_DEATH_TEST_CHECK_(ZX_PKT_IS_SIGNAL_ONE(packet.type)); if (packet.signal.observed & ZX_SOCKET_READABLE) { // Read data from the socket. constexpr size_t kBufferSize = 1024; do { size_t old_length = captured_stderr_.length(); size_t bytes_read = 0; captured_stderr_.resize(old_length + kBufferSize); status_zx = stderr_socket_.read( 0, &captured_stderr_.front() + old_length, kBufferSize, &bytes_read); captured_stderr_.resize(old_length + bytes_read); } while (status_zx == ZX_OK); if (status_zx == ZX_ERR_PEER_CLOSED) { socket_closed = true; } else { GTEST_DEATH_TEST_CHECK_(status_zx == ZX_ERR_SHOULD_WAIT); status_zx = stderr_socket_.wait_async( port, kSocketKey, ZX_SOCKET_READABLE | ZX_SOCKET_PEER_CLOSED, ZX_WAIT_ASYNC_ONCE); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); } } else { GTEST_DEATH_TEST_CHECK_(packet.signal.observed & ZX_SOCKET_PEER_CLOSED); socket_closed = true; } } } while (!process_terminated && !socket_closed); ReadAndInterpretStatusByte(); zx_info_process_t buffer; status_zx = child_process_.get_info( ZX_INFO_PROCESS, &buffer, sizeof(buffer), nullptr, nullptr); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); GTEST_DEATH_TEST_CHECK_(buffer.exited); set_status(buffer.return_code); return status(); } // The AssumeRole process for a Fuchsia death test. It creates a child // process with the same executable as the current process to run the // death test. The child process is given the --gtest_filter and // --gtest_internal_run_death_test flags such that it knows to run the // current death test only. DeathTest::TestRole FuchsiaDeathTest::AssumeRole() { const UnitTestImpl* const impl = GetUnitTestImpl(); const InternalRunDeathTestFlag* const flag = impl->internal_run_death_test_flag(); const TestInfo* const info = impl->current_test_info(); const int death_test_index = info->result()->death_test_count(); if (flag != nullptr) { // ParseInternalRunDeathTestFlag() has performed all the necessary // processing. set_write_fd(kFuchsiaReadPipeFd); return EXECUTE_TEST; } // Flush the log buffers since the log streams are shared with the child. FlushInfoLog(); // Build the child process command line. const std::string filter_flag = std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + info->test_suite_name() + "." + info->name(); const std::string internal_flag = std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + "=" + file_ + "|" + StreamableToString(line_) + "|" + StreamableToString(death_test_index); Arguments args; args.AddArguments(GetInjectableArgvs()); args.AddArgument(filter_flag.c_str()); args.AddArgument(internal_flag.c_str()); // Build the pipe for communication with the child. zx_status_t status; zx_handle_t child_pipe_handle; int child_pipe_fd; status = fdio_pipe_half(&child_pipe_fd, &child_pipe_handle); GTEST_DEATH_TEST_CHECK_(status == ZX_OK); set_read_fd(child_pipe_fd); // Set the pipe handle for the child. fdio_spawn_action_t spawn_actions[2] = {}; fdio_spawn_action_t* add_handle_action = &spawn_actions[0]; add_handle_action->action = FDIO_SPAWN_ACTION_ADD_HANDLE; add_handle_action->h.id = PA_HND(PA_FD, kFuchsiaReadPipeFd); add_handle_action->h.handle = child_pipe_handle; // Create a socket pair will be used to receive the child process' stderr. zx::socket stderr_producer_socket; status = zx::socket::create(0, &stderr_producer_socket, &stderr_socket_); GTEST_DEATH_TEST_CHECK_(status >= 0); int stderr_producer_fd = -1; status = fdio_fd_create(stderr_producer_socket.release(), &stderr_producer_fd); GTEST_DEATH_TEST_CHECK_(status >= 0); // Make the stderr socket nonblocking. GTEST_DEATH_TEST_CHECK_(fcntl(stderr_producer_fd, F_SETFL, 0) == 0); fdio_spawn_action_t* add_stderr_action = &spawn_actions[1]; add_stderr_action->action = FDIO_SPAWN_ACTION_CLONE_FD; add_stderr_action->fd.local_fd = stderr_producer_fd; add_stderr_action->fd.target_fd = STDERR_FILENO; // Create a child job. zx_handle_t child_job = ZX_HANDLE_INVALID; status = zx_job_create(zx_job_default(), 0, & child_job); GTEST_DEATH_TEST_CHECK_(status == ZX_OK); zx_policy_basic_t policy; policy.condition = ZX_POL_NEW_ANY; policy.policy = ZX_POL_ACTION_ALLOW; status = zx_job_set_policy( child_job, ZX_JOB_POL_RELATIVE, ZX_JOB_POL_BASIC, &policy, 1); GTEST_DEATH_TEST_CHECK_(status == ZX_OK); // Create an exception channel attached to the |child_job|, to allow // us to suppress the system default exception handler from firing. status = zx_task_create_exception_channel( child_job, 0, exception_channel_.reset_and_get_address()); GTEST_DEATH_TEST_CHECK_(status == ZX_OK); // Spawn the child process. status = fdio_spawn_etc( child_job, FDIO_SPAWN_CLONE_ALL, args.Argv()[0], args.Argv(), nullptr, 2, spawn_actions, child_process_.reset_and_get_address(), nullptr); GTEST_DEATH_TEST_CHECK_(status == ZX_OK); set_spawned(true); return OVERSEE_TEST; } std::string FuchsiaDeathTest::GetErrorLogs() { return captured_stderr_; } #else // We are neither on Windows, nor on Fuchsia. // ForkingDeathTest provides implementations for most of the abstract // methods of the DeathTest interface. Only the AssumeRole method is // left undefined. class ForkingDeathTest : public DeathTestImpl { public: ForkingDeathTest(const char* statement, Matcher matcher); // All of these virtual functions are inherited from DeathTest. int Wait() override; protected: void set_child_pid(pid_t child_pid) { child_pid_ = child_pid; } private: // PID of child process during death test; 0 in the child process itself. pid_t child_pid_; }; // Constructs a ForkingDeathTest. ForkingDeathTest::ForkingDeathTest(const char* a_statement, Matcher matcher) : DeathTestImpl(a_statement, std::move(matcher)), child_pid_(-1) {} // Waits for the child in a death test to exit, returning its exit // status, or 0 if no child process exists. As a side effect, sets the // outcome data member. int ForkingDeathTest::Wait() { if (!spawned()) return 0; ReadAndInterpretStatusByte(); int status_value; GTEST_DEATH_TEST_CHECK_SYSCALL_(waitpid(child_pid_, &status_value, 0)); set_status(status_value); return status_value; } // A concrete death test class that forks, then immediately runs the test // in the child process. class NoExecDeathTest : public ForkingDeathTest { public: NoExecDeathTest(const char* a_statement, Matcher matcher) : ForkingDeathTest(a_statement, std::move(matcher)) {} TestRole AssumeRole() override; }; // The AssumeRole process for a fork-and-run death test. It implements a // straightforward fork, with a simple pipe to transmit the status byte. DeathTest::TestRole NoExecDeathTest::AssumeRole() { const size_t thread_count = GetThreadCount(); if (thread_count != 1) { GTEST_LOG_(WARNING) << DeathTestThreadWarning(thread_count); } int pipe_fd[2]; GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1); DeathTest::set_last_death_test_message(""); CaptureStderr(); // When we fork the process below, the log file buffers are copied, but the // file descriptors are shared. We flush all log files here so that closing // the file descriptors in the child process doesn't throw off the // synchronization between descriptors and buffers in the parent process. // This is as close to the fork as possible to avoid a race condition in case // there are multiple threads running before the death test, and another // thread writes to the log file. FlushInfoLog(); const pid_t child_pid = fork(); GTEST_DEATH_TEST_CHECK_(child_pid != -1); set_child_pid(child_pid); if (child_pid == 0) { GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[0])); set_write_fd(pipe_fd[1]); // Redirects all logging to stderr in the child process to prevent // concurrent writes to the log files. We capture stderr in the parent // process and append the child process' output to a log. LogToStderr(); // Event forwarding to the listeners of event listener API mush be shut // down in death test subprocesses. GetUnitTestImpl()->listeners()->SuppressEventForwarding(); g_in_fast_death_test_child = true; return EXECUTE_TEST; } else { GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); set_read_fd(pipe_fd[0]); set_spawned(true); return OVERSEE_TEST; } } // A concrete death test class that forks and re-executes the main // program from the beginning, with command-line flags set that cause // only this specific death test to be run. class ExecDeathTest : public ForkingDeathTest { public: ExecDeathTest(const char* a_statement, Matcher matcher, const char* file, int line) : ForkingDeathTest(a_statement, std::move(matcher)), file_(file), line_(line) {} TestRole AssumeRole() override; private: static ::std::vector GetArgvsForDeathTestChildProcess() { ::std::vector args = GetInjectableArgvs(); # if defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_) ::std::vector extra_args = GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_(); args.insert(args.end(), extra_args.begin(), extra_args.end()); # endif // defined(GTEST_EXTRA_DEATH_TEST_COMMAND_LINE_ARGS_) return args; } // The name of the file in which the death test is located. const char* const file_; // The line number on which the death test is located. const int line_; }; // Utility class for accumulating command-line arguments. class Arguments { public: Arguments() { args_.push_back(nullptr); } ~Arguments() { for (std::vector::iterator i = args_.begin(); i != args_.end(); ++i) { free(*i); } } void AddArgument(const char* argument) { args_.insert(args_.end() - 1, posix::StrDup(argument)); } template void AddArguments(const ::std::vector& arguments) { for (typename ::std::vector::const_iterator i = arguments.begin(); i != arguments.end(); ++i) { args_.insert(args_.end() - 1, posix::StrDup(i->c_str())); } } char* const* Argv() { return &args_[0]; } private: std::vector args_; }; // A struct that encompasses the arguments to the child process of a // threadsafe-style death test process. struct ExecDeathTestArgs { char* const* argv; // Command-line arguments for the child's call to exec int close_fd; // File descriptor to close; the read end of a pipe }; # if GTEST_OS_MAC inline char** GetEnviron() { // When Google Test is built as a framework on MacOS X, the environ variable // is unavailable. Apple's documentation (man environ) recommends using // _NSGetEnviron() instead. return *_NSGetEnviron(); } # else // Some POSIX platforms expect you to declare environ. extern "C" makes // it reside in the global namespace. extern "C" char** environ; inline char** GetEnviron() { return environ; } # endif // GTEST_OS_MAC # if !GTEST_OS_QNX // The main function for a threadsafe-style death test child process. // This function is called in a clone()-ed process and thus must avoid // any potentially unsafe operations like malloc or libc functions. static int ExecDeathTestChildMain(void* child_arg) { ExecDeathTestArgs* const args = static_cast(child_arg); GTEST_DEATH_TEST_CHECK_SYSCALL_(close(args->close_fd)); // We need to execute the test program in the same environment where // it was originally invoked. Therefore we change to the original // working directory first. const char* const original_dir = UnitTest::GetInstance()->original_working_dir(); // We can safely call chdir() as it's a direct system call. if (chdir(original_dir) != 0) { DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + GetLastErrnoDescription()); return EXIT_FAILURE; } // We can safely call execve() as it's a direct system call. We // cannot use execvp() as it's a libc function and thus potentially // unsafe. Since execve() doesn't search the PATH, the user must // invoke the test program via a valid path that contains at least // one path separator. execve(args->argv[0], args->argv, GetEnviron()); DeathTestAbort(std::string("execve(") + args->argv[0] + ", ...) in " + original_dir + " failed: " + GetLastErrnoDescription()); return EXIT_FAILURE; } # endif // !GTEST_OS_QNX # if GTEST_HAS_CLONE // Two utility routines that together determine the direction the stack // grows. // This could be accomplished more elegantly by a single recursive // function, but we want to guard against the unlikely possibility of // a smart compiler optimizing the recursion away. // // GTEST_NO_INLINE_ is required to prevent GCC 4.6 from inlining // StackLowerThanAddress into StackGrowsDown, which then doesn't give // correct answer. static void StackLowerThanAddress(const void* ptr, bool* result) GTEST_NO_INLINE_; // HWAddressSanitizer add a random tag to the MSB of the local variable address, // making comparison result unpredictable. GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ static void StackLowerThanAddress(const void* ptr, bool* result) { int dummy; *result = (&dummy < ptr); } // Make sure AddressSanitizer does not tamper with the stack here. GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ static bool StackGrowsDown() { int dummy = 0; bool result = 0; StackLowerThanAddress(&dummy, &result); return result; } # endif // GTEST_HAS_CLONE // Spawns a child process with the same executable as the current process in // a thread-safe manner and instructs it to run the death test. The // implementation uses fork(2) + exec. On systems where clone(2) is // available, it is used instead, being slightly more thread-safe. On QNX, // fork supports only single-threaded environments, so this function uses // spawn(2) there instead. The function dies with an error message if // anything goes wrong. static pid_t ExecDeathTestSpawnChild(char* const* argv, int close_fd) { ExecDeathTestArgs args = { argv, close_fd }; pid_t child_pid = -1; # if GTEST_OS_QNX // Obtains the current directory and sets it to be closed in the child // process. const int cwd_fd = open(".", O_RDONLY); GTEST_DEATH_TEST_CHECK_(cwd_fd != -1); GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(cwd_fd, F_SETFD, FD_CLOEXEC)); // We need to execute the test program in the same environment where // it was originally invoked. Therefore we change to the original // working directory first. const char* const original_dir = UnitTest::GetInstance()->original_working_dir(); // We can safely call chdir() as it's a direct system call. if (chdir(original_dir) != 0) { DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + GetLastErrnoDescription()); return EXIT_FAILURE; } int fd_flags; // Set close_fd to be closed after spawn. GTEST_DEATH_TEST_CHECK_SYSCALL_(fd_flags = fcntl(close_fd, F_GETFD)); GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(close_fd, F_SETFD, fd_flags | FD_CLOEXEC)); struct inheritance inherit = {0}; // spawn is a system call. child_pid = spawn(args.argv[0], 0, nullptr, &inherit, args.argv, GetEnviron()); // Restores the current working directory. GTEST_DEATH_TEST_CHECK_(fchdir(cwd_fd) != -1); GTEST_DEATH_TEST_CHECK_SYSCALL_(close(cwd_fd)); # else // GTEST_OS_QNX # if GTEST_OS_LINUX // When a SIGPROF signal is received while fork() or clone() are executing, // the process may hang. To avoid this, we ignore SIGPROF here and re-enable // it after the call to fork()/clone() is complete. struct sigaction saved_sigprof_action; struct sigaction ignore_sigprof_action; memset(&ignore_sigprof_action, 0, sizeof(ignore_sigprof_action)); sigemptyset(&ignore_sigprof_action.sa_mask); ignore_sigprof_action.sa_handler = SIG_IGN; GTEST_DEATH_TEST_CHECK_SYSCALL_(sigaction( SIGPROF, &ignore_sigprof_action, &saved_sigprof_action)); # endif // GTEST_OS_LINUX # if GTEST_HAS_CLONE const bool use_fork = GTEST_FLAG(death_test_use_fork); if (!use_fork) { static const bool stack_grows_down = StackGrowsDown(); const auto stack_size = static_cast(getpagesize()); // MMAP_ANONYMOUS is not defined on Mac, so we use MAP_ANON instead. void* const stack = mmap(nullptr, stack_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); GTEST_DEATH_TEST_CHECK_(stack != MAP_FAILED); // Maximum stack alignment in bytes: For a downward-growing stack, this // amount is subtracted from size of the stack space to get an address // that is within the stack space and is aligned on all systems we care // about. As far as I know there is no ABI with stack alignment greater // than 64. We assume stack and stack_size already have alignment of // kMaxStackAlignment. const size_t kMaxStackAlignment = 64; void* const stack_top = static_cast(stack) + (stack_grows_down ? stack_size - kMaxStackAlignment : 0); GTEST_DEATH_TEST_CHECK_( static_cast(stack_size) > kMaxStackAlignment && reinterpret_cast(stack_top) % kMaxStackAlignment == 0); child_pid = clone(&ExecDeathTestChildMain, stack_top, SIGCHLD, &args); GTEST_DEATH_TEST_CHECK_(munmap(stack, stack_size) != -1); } # else const bool use_fork = true; # endif // GTEST_HAS_CLONE if (use_fork && (child_pid = fork()) == 0) { ExecDeathTestChildMain(&args); _exit(0); } # endif // GTEST_OS_QNX # if GTEST_OS_LINUX GTEST_DEATH_TEST_CHECK_SYSCALL_( sigaction(SIGPROF, &saved_sigprof_action, nullptr)); # endif // GTEST_OS_LINUX GTEST_DEATH_TEST_CHECK_(child_pid != -1); return child_pid; } // The AssumeRole process for a fork-and-exec death test. It re-executes the // main program from the beginning, setting the --gtest_filter // and --gtest_internal_run_death_test flags to cause only the current // death test to be re-run. DeathTest::TestRole ExecDeathTest::AssumeRole() { const UnitTestImpl* const impl = GetUnitTestImpl(); const InternalRunDeathTestFlag* const flag = impl->internal_run_death_test_flag(); const TestInfo* const info = impl->current_test_info(); const int death_test_index = info->result()->death_test_count(); if (flag != nullptr) { set_write_fd(flag->write_fd()); return EXECUTE_TEST; } int pipe_fd[2]; GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1); // Clear the close-on-exec flag on the write end of the pipe, lest // it be closed when the child process does an exec: GTEST_DEATH_TEST_CHECK_(fcntl(pipe_fd[1], F_SETFD, 0) != -1); const std::string filter_flag = std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + info->test_suite_name() + "." + info->name(); const std::string internal_flag = std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + "=" + file_ + "|" + StreamableToString(line_) + "|" + StreamableToString(death_test_index) + "|" + StreamableToString(pipe_fd[1]); Arguments args; args.AddArguments(GetArgvsForDeathTestChildProcess()); args.AddArgument(filter_flag.c_str()); args.AddArgument(internal_flag.c_str()); DeathTest::set_last_death_test_message(""); CaptureStderr(); // See the comment in NoExecDeathTest::AssumeRole for why the next line // is necessary. FlushInfoLog(); const pid_t child_pid = ExecDeathTestSpawnChild(args.Argv(), pipe_fd[0]); GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); set_child_pid(child_pid); set_read_fd(pipe_fd[0]); set_spawned(true); return OVERSEE_TEST; } # endif // !GTEST_OS_WINDOWS // Creates a concrete DeathTest-derived class that depends on the // --gtest_death_test_style flag, and sets the pointer pointed to // by the "test" argument to its address. If the test should be // skipped, sets that pointer to NULL. Returns true, unless the // flag is set to an invalid value. bool DefaultDeathTestFactory::Create(const char* statement, Matcher matcher, const char* file, int line, DeathTest** test) { UnitTestImpl* const impl = GetUnitTestImpl(); const InternalRunDeathTestFlag* const flag = impl->internal_run_death_test_flag(); const int death_test_index = impl->current_test_info() ->increment_death_test_count(); if (flag != nullptr) { if (death_test_index > flag->index()) { DeathTest::set_last_death_test_message( "Death test count (" + StreamableToString(death_test_index) + ") somehow exceeded expected maximum (" + StreamableToString(flag->index()) + ")"); return false; } if (!(flag->file() == file && flag->line() == line && flag->index() == death_test_index)) { *test = nullptr; return true; } } # if GTEST_OS_WINDOWS if (GTEST_FLAG(death_test_style) == "threadsafe" || GTEST_FLAG(death_test_style) == "fast") { *test = new WindowsDeathTest(statement, std::move(matcher), file, line); } # elif GTEST_OS_FUCHSIA if (GTEST_FLAG(death_test_style) == "threadsafe" || GTEST_FLAG(death_test_style) == "fast") { *test = new FuchsiaDeathTest(statement, std::move(matcher), file, line); } # else if (GTEST_FLAG(death_test_style) == "threadsafe") { *test = new ExecDeathTest(statement, std::move(matcher), file, line); } else if (GTEST_FLAG(death_test_style) == "fast") { *test = new NoExecDeathTest(statement, std::move(matcher)); } # endif // GTEST_OS_WINDOWS else { // NOLINT - this is more readable than unbalanced brackets inside #if. DeathTest::set_last_death_test_message( "Unknown death test style \"" + GTEST_FLAG(death_test_style) + "\" encountered"); return false; } return true; } # if GTEST_OS_WINDOWS // Recreates the pipe and event handles from the provided parameters, // signals the event, and returns a file descriptor wrapped around the pipe // handle. This function is called in the child process only. static int GetStatusFileDescriptor(unsigned int parent_process_id, size_t write_handle_as_size_t, size_t event_handle_as_size_t) { AutoHandle parent_process_handle(::OpenProcess(PROCESS_DUP_HANDLE, FALSE, // Non-inheritable. parent_process_id)); if (parent_process_handle.Get() == INVALID_HANDLE_VALUE) { DeathTestAbort("Unable to open parent process " + StreamableToString(parent_process_id)); } GTEST_CHECK_(sizeof(HANDLE) <= sizeof(size_t)); const HANDLE write_handle = reinterpret_cast(write_handle_as_size_t); HANDLE dup_write_handle; // The newly initialized handle is accessible only in the parent // process. To obtain one accessible within the child, we need to use // DuplicateHandle. if (!::DuplicateHandle(parent_process_handle.Get(), write_handle, ::GetCurrentProcess(), &dup_write_handle, 0x0, // Requested privileges ignored since // DUPLICATE_SAME_ACCESS is used. FALSE, // Request non-inheritable handler. DUPLICATE_SAME_ACCESS)) { DeathTestAbort("Unable to duplicate the pipe handle " + StreamableToString(write_handle_as_size_t) + " from the parent process " + StreamableToString(parent_process_id)); } const HANDLE event_handle = reinterpret_cast(event_handle_as_size_t); HANDLE dup_event_handle; if (!::DuplicateHandle(parent_process_handle.Get(), event_handle, ::GetCurrentProcess(), &dup_event_handle, 0x0, FALSE, DUPLICATE_SAME_ACCESS)) { DeathTestAbort("Unable to duplicate the event handle " + StreamableToString(event_handle_as_size_t) + " from the parent process " + StreamableToString(parent_process_id)); } const int write_fd = ::_open_osfhandle(reinterpret_cast(dup_write_handle), O_APPEND); if (write_fd == -1) { DeathTestAbort("Unable to convert pipe handle " + StreamableToString(write_handle_as_size_t) + " to a file descriptor"); } // Signals the parent that the write end of the pipe has been acquired // so the parent can release its own write end. ::SetEvent(dup_event_handle); return write_fd; } # endif // GTEST_OS_WINDOWS // Returns a newly created InternalRunDeathTestFlag object with fields // initialized from the GTEST_FLAG(internal_run_death_test) flag if // the flag is specified; otherwise returns NULL. InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag() { if (GTEST_FLAG(internal_run_death_test) == "") return nullptr; // GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we // can use it here. int line = -1; int index = -1; ::std::vector< ::std::string> fields; SplitString(GTEST_FLAG(internal_run_death_test).c_str(), '|', &fields); int write_fd = -1; # if GTEST_OS_WINDOWS unsigned int parent_process_id = 0; size_t write_handle_as_size_t = 0; size_t event_handle_as_size_t = 0; if (fields.size() != 6 || !ParseNaturalNumber(fields[1], &line) || !ParseNaturalNumber(fields[2], &index) || !ParseNaturalNumber(fields[3], &parent_process_id) || !ParseNaturalNumber(fields[4], &write_handle_as_size_t) || !ParseNaturalNumber(fields[5], &event_handle_as_size_t)) { DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + GTEST_FLAG(internal_run_death_test)); } write_fd = GetStatusFileDescriptor(parent_process_id, write_handle_as_size_t, event_handle_as_size_t); # elif GTEST_OS_FUCHSIA if (fields.size() != 3 || !ParseNaturalNumber(fields[1], &line) || !ParseNaturalNumber(fields[2], &index)) { DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + GTEST_FLAG(internal_run_death_test)); } # else if (fields.size() != 4 || !ParseNaturalNumber(fields[1], &line) || !ParseNaturalNumber(fields[2], &index) || !ParseNaturalNumber(fields[3], &write_fd)) { DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + GTEST_FLAG(internal_run_death_test)); } # endif // GTEST_OS_WINDOWS return new InternalRunDeathTestFlag(fields[0], line, index, write_fd); } } // namespace internal #endif // GTEST_HAS_DEATH_TEST } // namespace testing // Copyright 2008, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #if GTEST_OS_WINDOWS_MOBILE # include #elif GTEST_OS_WINDOWS # include # include #else # include # include // Some Linux distributions define PATH_MAX here. #endif // GTEST_OS_WINDOWS_MOBILE #if GTEST_OS_WINDOWS # define GTEST_PATH_MAX_ _MAX_PATH #elif defined(PATH_MAX) # define GTEST_PATH_MAX_ PATH_MAX #elif defined(_XOPEN_PATH_MAX) # define GTEST_PATH_MAX_ _XOPEN_PATH_MAX #else # define GTEST_PATH_MAX_ _POSIX_PATH_MAX #endif // GTEST_OS_WINDOWS namespace testing { namespace internal { #if GTEST_OS_WINDOWS // On Windows, '\\' is the standard path separator, but many tools and the // Windows API also accept '/' as an alternate path separator. Unless otherwise // noted, a file path can contain either kind of path separators, or a mixture // of them. const char kPathSeparator = '\\'; const char kAlternatePathSeparator = '/'; const char kAlternatePathSeparatorString[] = "/"; # if GTEST_OS_WINDOWS_MOBILE // Windows CE doesn't have a current directory. You should not use // the current directory in tests on Windows CE, but this at least // provides a reasonable fallback. const char kCurrentDirectoryString[] = "\\"; // Windows CE doesn't define INVALID_FILE_ATTRIBUTES const DWORD kInvalidFileAttributes = 0xffffffff; # else const char kCurrentDirectoryString[] = ".\\"; # endif // GTEST_OS_WINDOWS_MOBILE #else const char kPathSeparator = '/'; const char kCurrentDirectoryString[] = "./"; #endif // GTEST_OS_WINDOWS // Returns whether the given character is a valid path separator. static bool IsPathSeparator(char c) { #if GTEST_HAS_ALT_PATH_SEP_ return (c == kPathSeparator) || (c == kAlternatePathSeparator); #else return c == kPathSeparator; #endif } // Returns the current working directory, or "" if unsuccessful. FilePath FilePath::GetCurrentDir() { #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || \ GTEST_OS_WINDOWS_RT || ARDUINO || defined(ESP_PLATFORM) // These platforms do not have a current directory, so we just return // something reasonable. return FilePath(kCurrentDirectoryString); #elif GTEST_OS_WINDOWS char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; return FilePath(_getcwd(cwd, sizeof(cwd)) == nullptr ? "" : cwd); #else char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; char* result = getcwd(cwd, sizeof(cwd)); # if GTEST_OS_NACL // getcwd will likely fail in NaCl due to the sandbox, so return something // reasonable. The user may have provided a shim implementation for getcwd, // however, so fallback only when failure is detected. return FilePath(result == nullptr ? kCurrentDirectoryString : cwd); # endif // GTEST_OS_NACL return FilePath(result == nullptr ? "" : cwd); #endif // GTEST_OS_WINDOWS_MOBILE } // Returns a copy of the FilePath with the case-insensitive extension removed. // Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns // FilePath("dir/file"). If a case-insensitive extension is not // found, returns a copy of the original FilePath. FilePath FilePath::RemoveExtension(const char* extension) const { const std::string dot_extension = std::string(".") + extension; if (String::EndsWithCaseInsensitive(pathname_, dot_extension)) { return FilePath(pathname_.substr( 0, pathname_.length() - dot_extension.length())); } return *this; } // Returns a pointer to the last occurrence of a valid path separator in // the FilePath. On Windows, for example, both '/' and '\' are valid path // separators. Returns NULL if no path separator was found. const char* FilePath::FindLastPathSeparator() const { const char* const last_sep = strrchr(c_str(), kPathSeparator); #if GTEST_HAS_ALT_PATH_SEP_ const char* const last_alt_sep = strrchr(c_str(), kAlternatePathSeparator); // Comparing two pointers of which only one is NULL is undefined. if (last_alt_sep != nullptr && (last_sep == nullptr || last_alt_sep > last_sep)) { return last_alt_sep; } #endif return last_sep; } // Returns a copy of the FilePath with the directory part removed. // Example: FilePath("path/to/file").RemoveDirectoryName() returns // FilePath("file"). If there is no directory part ("just_a_file"), it returns // the FilePath unmodified. If there is no file part ("just_a_dir/") it // returns an empty FilePath (""). // On Windows platform, '\' is the path separator, otherwise it is '/'. FilePath FilePath::RemoveDirectoryName() const { const char* const last_sep = FindLastPathSeparator(); return last_sep ? FilePath(last_sep + 1) : *this; } // RemoveFileName returns the directory path with the filename removed. // Example: FilePath("path/to/file").RemoveFileName() returns "path/to/". // If the FilePath is "a_file" or "/a_file", RemoveFileName returns // FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does // not have a file, like "just/a/dir/", it returns the FilePath unmodified. // On Windows platform, '\' is the path separator, otherwise it is '/'. FilePath FilePath::RemoveFileName() const { const char* const last_sep = FindLastPathSeparator(); std::string dir; if (last_sep) { dir = std::string(c_str(), static_cast(last_sep + 1 - c_str())); } else { dir = kCurrentDirectoryString; } return FilePath(dir); } // Helper functions for naming files in a directory for xml output. // Given directory = "dir", base_name = "test", number = 0, // extension = "xml", returns "dir/test.xml". If number is greater // than zero (e.g., 12), returns "dir/test_12.xml". // On Windows platform, uses \ as the separator rather than /. FilePath FilePath::MakeFileName(const FilePath& directory, const FilePath& base_name, int number, const char* extension) { std::string file; if (number == 0) { file = base_name.string() + "." + extension; } else { file = base_name.string() + "_" + StreamableToString(number) + "." + extension; } return ConcatPaths(directory, FilePath(file)); } // Given directory = "dir", relative_path = "test.xml", returns "dir/test.xml". // On Windows, uses \ as the separator rather than /. FilePath FilePath::ConcatPaths(const FilePath& directory, const FilePath& relative_path) { if (directory.IsEmpty()) return relative_path; const FilePath dir(directory.RemoveTrailingPathSeparator()); return FilePath(dir.string() + kPathSeparator + relative_path.string()); } // Returns true if pathname describes something findable in the file-system, // either a file, directory, or whatever. bool FilePath::FileOrDirectoryExists() const { #if GTEST_OS_WINDOWS_MOBILE LPCWSTR unicode = String::AnsiToUtf16(pathname_.c_str()); const DWORD attributes = GetFileAttributes(unicode); delete [] unicode; return attributes != kInvalidFileAttributes; #else posix::StatStruct file_stat; return posix::Stat(pathname_.c_str(), &file_stat) == 0; #endif // GTEST_OS_WINDOWS_MOBILE } // Returns true if pathname describes a directory in the file-system // that exists. bool FilePath::DirectoryExists() const { bool result = false; #if GTEST_OS_WINDOWS // Don't strip off trailing separator if path is a root directory on // Windows (like "C:\\"). const FilePath& path(IsRootDirectory() ? *this : RemoveTrailingPathSeparator()); #else const FilePath& path(*this); #endif #if GTEST_OS_WINDOWS_MOBILE LPCWSTR unicode = String::AnsiToUtf16(path.c_str()); const DWORD attributes = GetFileAttributes(unicode); delete [] unicode; if ((attributes != kInvalidFileAttributes) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) { result = true; } #else posix::StatStruct file_stat; result = posix::Stat(path.c_str(), &file_stat) == 0 && posix::IsDir(file_stat); #endif // GTEST_OS_WINDOWS_MOBILE return result; } // Returns true if pathname describes a root directory. (Windows has one // root directory per disk drive.) bool FilePath::IsRootDirectory() const { #if GTEST_OS_WINDOWS return pathname_.length() == 3 && IsAbsolutePath(); #else return pathname_.length() == 1 && IsPathSeparator(pathname_.c_str()[0]); #endif } // Returns true if pathname describes an absolute path. bool FilePath::IsAbsolutePath() const { const char* const name = pathname_.c_str(); #if GTEST_OS_WINDOWS return pathname_.length() >= 3 && ((name[0] >= 'a' && name[0] <= 'z') || (name[0] >= 'A' && name[0] <= 'Z')) && name[1] == ':' && IsPathSeparator(name[2]); #else return IsPathSeparator(name[0]); #endif } // Returns a pathname for a file that does not currently exist. The pathname // will be directory/base_name.extension or // directory/base_name_.extension if directory/base_name.extension // already exists. The number will be incremented until a pathname is found // that does not already exist. // Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'. // There could be a race condition if two or more processes are calling this // function at the same time -- they could both pick the same filename. FilePath FilePath::GenerateUniqueFileName(const FilePath& directory, const FilePath& base_name, const char* extension) { FilePath full_pathname; int number = 0; do { full_pathname.Set(MakeFileName(directory, base_name, number++, extension)); } while (full_pathname.FileOrDirectoryExists()); return full_pathname; } // Returns true if FilePath ends with a path separator, which indicates that // it is intended to represent a directory. Returns false otherwise. // This does NOT check that a directory (or file) actually exists. bool FilePath::IsDirectory() const { return !pathname_.empty() && IsPathSeparator(pathname_.c_str()[pathname_.length() - 1]); } // Create directories so that path exists. Returns true if successful or if // the directories already exist; returns false if unable to create directories // for any reason. bool FilePath::CreateDirectoriesRecursively() const { if (!this->IsDirectory()) { return false; } if (pathname_.length() == 0 || this->DirectoryExists()) { return true; } const FilePath parent(this->RemoveTrailingPathSeparator().RemoveFileName()); return parent.CreateDirectoriesRecursively() && this->CreateFolder(); } // Create the directory so that path exists. Returns true if successful or // if the directory already exists; returns false if unable to create the // directory for any reason, including if the parent directory does not // exist. Not named "CreateDirectory" because that's a macro on Windows. bool FilePath::CreateFolder() const { #if GTEST_OS_WINDOWS_MOBILE FilePath removed_sep(this->RemoveTrailingPathSeparator()); LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str()); int result = CreateDirectory(unicode, nullptr) ? 0 : -1; delete [] unicode; #elif GTEST_OS_WINDOWS int result = _mkdir(pathname_.c_str()); #else int result = mkdir(pathname_.c_str(), 0777); #endif // GTEST_OS_WINDOWS_MOBILE if (result == -1) { return this->DirectoryExists(); // An error is OK if the directory exists. } return true; // No error. } // If input name has a trailing separator character, remove it and return the // name, otherwise return the name string unmodified. // On Windows platform, uses \ as the separator, other platforms use /. FilePath FilePath::RemoveTrailingPathSeparator() const { return IsDirectory() ? FilePath(pathname_.substr(0, pathname_.length() - 1)) : *this; } // Removes any redundant separators that might be in the pathname. // For example, "bar///foo" becomes "bar/foo". Does not eliminate other // redundancies that might be in a pathname involving "." or "..". void FilePath::Normalize() { if (pathname_.c_str() == nullptr) { pathname_ = ""; return; } const char* src = pathname_.c_str(); char* const dest = new char[pathname_.length() + 1]; char* dest_ptr = dest; memset(dest_ptr, 0, pathname_.length() + 1); while (*src != '\0') { *dest_ptr = *src; if (!IsPathSeparator(*src)) { src++; } else { #if GTEST_HAS_ALT_PATH_SEP_ if (*dest_ptr == kAlternatePathSeparator) { *dest_ptr = kPathSeparator; } #endif while (IsPathSeparator(*src)) src++; } dest_ptr++; } *dest_ptr = '\0'; pathname_ = dest; delete[] dest; } } // namespace internal } // namespace testing // Copyright 2007, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // The Google C++ Testing and Mocking Framework (Google Test) // // This file implements just enough of the matcher interface to allow // EXPECT_DEATH and friends to accept a matcher argument. #include namespace testing { // Constructs a matcher that matches a const std::string& whose value is // equal to s. Matcher::Matcher(const std::string& s) { *this = Eq(s); } // Constructs a matcher that matches a const std::string& whose value is // equal to s. Matcher::Matcher(const char* s) { *this = Eq(std::string(s)); } // Constructs a matcher that matches a std::string whose value is equal to // s. Matcher::Matcher(const std::string& s) { *this = Eq(s); } // Constructs a matcher that matches a std::string whose value is equal to // s. Matcher::Matcher(const char* s) { *this = Eq(std::string(s)); } #if GTEST_HAS_ABSL // Constructs a matcher that matches a const absl::string_view& whose value is // equal to s. Matcher::Matcher(const std::string& s) { *this = Eq(s); } // Constructs a matcher that matches a const absl::string_view& whose value is // equal to s. Matcher::Matcher(const char* s) { *this = Eq(std::string(s)); } // Constructs a matcher that matches a const absl::string_view& whose value is // equal to s. Matcher::Matcher(absl::string_view s) { *this = Eq(std::string(s)); } // Constructs a matcher that matches a absl::string_view whose value is equal to // s. Matcher::Matcher(const std::string& s) { *this = Eq(s); } // Constructs a matcher that matches a absl::string_view whose value is equal to // s. Matcher::Matcher(const char* s) { *this = Eq(std::string(s)); } // Constructs a matcher that matches a absl::string_view whose value is equal to // s. Matcher::Matcher(absl::string_view s) { *this = Eq(std::string(s)); } #endif // GTEST_HAS_ABSL } // namespace testing // Copyright 2008, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include #include #include #include #if GTEST_OS_WINDOWS # include # include # include # include // Used in ThreadLocal. # ifdef _MSC_VER # include # endif // _MSC_VER #else # include #endif // GTEST_OS_WINDOWS #if GTEST_OS_MAC # include # include # include #endif // GTEST_OS_MAC #if GTEST_OS_DRAGONFLY || GTEST_OS_FREEBSD || GTEST_OS_GNU_KFREEBSD || \ GTEST_OS_NETBSD || GTEST_OS_OPENBSD # include # if GTEST_OS_DRAGONFLY || GTEST_OS_FREEBSD || GTEST_OS_GNU_KFREEBSD # include # endif #endif #if GTEST_OS_QNX # include # include # include #endif // GTEST_OS_QNX #if GTEST_OS_AIX # include # include #endif // GTEST_OS_AIX #if GTEST_OS_FUCHSIA # include # include #endif // GTEST_OS_FUCHSIA namespace testing { namespace internal { #if defined(_MSC_VER) || defined(__BORLANDC__) // MSVC and C++Builder do not provide a definition of STDERR_FILENO. const int kStdOutFileno = 1; const int kStdErrFileno = 2; #else const int kStdOutFileno = STDOUT_FILENO; const int kStdErrFileno = STDERR_FILENO; #endif // _MSC_VER #if GTEST_OS_LINUX namespace { template T ReadProcFileField(const std::string& filename, int field) { std::string dummy; std::ifstream file(filename.c_str()); while (field-- > 0) { file >> dummy; } T output = 0; file >> output; return output; } } // namespace // Returns the number of active threads, or 0 when there is an error. size_t GetThreadCount() { const std::string filename = (Message() << "/proc/" << getpid() << "/stat").GetString(); return ReadProcFileField(filename, 19); } #elif GTEST_OS_MAC size_t GetThreadCount() { const task_t task = mach_task_self(); mach_msg_type_number_t thread_count; thread_act_array_t thread_list; const kern_return_t status = task_threads(task, &thread_list, &thread_count); if (status == KERN_SUCCESS) { // task_threads allocates resources in thread_list and we need to free them // to avoid leaks. vm_deallocate(task, reinterpret_cast(thread_list), sizeof(thread_t) * thread_count); return static_cast(thread_count); } else { return 0; } } #elif GTEST_OS_DRAGONFLY || GTEST_OS_FREEBSD || GTEST_OS_GNU_KFREEBSD || \ GTEST_OS_NETBSD #if GTEST_OS_NETBSD #undef KERN_PROC #define KERN_PROC KERN_PROC2 #define kinfo_proc kinfo_proc2 #endif #if GTEST_OS_DRAGONFLY #define KP_NLWP(kp) (kp.kp_nthreads) #elif GTEST_OS_FREEBSD || GTEST_OS_GNU_KFREEBSD #define KP_NLWP(kp) (kp.ki_numthreads) #elif GTEST_OS_NETBSD #define KP_NLWP(kp) (kp.p_nlwps) #endif // Returns the number of threads running in the process, or 0 to indicate that // we cannot detect it. size_t GetThreadCount() { int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid(), #if GTEST_OS_NETBSD sizeof(struct kinfo_proc), 1, #endif }; u_int miblen = sizeof(mib) / sizeof(mib[0]); struct kinfo_proc info; size_t size = sizeof(info); if (sysctl(mib, miblen, &info, &size, NULL, 0)) { return 0; } return static_cast(KP_NLWP(info)); } #elif GTEST_OS_OPENBSD // Returns the number of threads running in the process, or 0 to indicate that // we cannot detect it. size_t GetThreadCount() { int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID | KERN_PROC_SHOW_THREADS, getpid(), sizeof(struct kinfo_proc), 0, }; u_int miblen = sizeof(mib) / sizeof(mib[0]); // get number of structs size_t size; if (sysctl(mib, miblen, NULL, &size, NULL, 0)) { return 0; } mib[5] = size / mib[4]; // populate array of structs struct kinfo_proc info[mib[5]]; if (sysctl(mib, miblen, &info, &size, NULL, 0)) { return 0; } // exclude empty members int nthreads = 0; for (int i = 0; i < size / mib[4]; i++) { if (info[i].p_tid != -1) nthreads++; } return nthreads; } #elif GTEST_OS_QNX // Returns the number of threads running in the process, or 0 to indicate that // we cannot detect it. size_t GetThreadCount() { const int fd = open("/proc/self/as", O_RDONLY); if (fd < 0) { return 0; } procfs_info process_info; const int status = devctl(fd, DCMD_PROC_INFO, &process_info, sizeof(process_info), nullptr); close(fd); if (status == EOK) { return static_cast(process_info.num_threads); } else { return 0; } } #elif GTEST_OS_AIX size_t GetThreadCount() { struct procentry64 entry; pid_t pid = getpid(); int status = getprocs64(&entry, sizeof(entry), nullptr, 0, &pid, 1); if (status == 1) { return entry.pi_thcount; } else { return 0; } } #elif GTEST_OS_FUCHSIA size_t GetThreadCount() { int dummy_buffer; size_t avail; zx_status_t status = zx_object_get_info( zx_process_self(), ZX_INFO_PROCESS_THREADS, &dummy_buffer, 0, nullptr, &avail); if (status == ZX_OK) { return avail; } else { return 0; } } #else size_t GetThreadCount() { // There's no portable way to detect the number of threads, so we just // return 0 to indicate that we cannot detect it. return 0; } #endif // GTEST_OS_LINUX #if GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS void SleepMilliseconds(int n) { ::Sleep(static_cast(n)); } AutoHandle::AutoHandle() : handle_(INVALID_HANDLE_VALUE) {} AutoHandle::AutoHandle(Handle handle) : handle_(handle) {} AutoHandle::~AutoHandle() { Reset(); } AutoHandle::Handle AutoHandle::Get() const { return handle_; } void AutoHandle::Reset() { Reset(INVALID_HANDLE_VALUE); } void AutoHandle::Reset(HANDLE handle) { // Resetting with the same handle we already own is invalid. if (handle_ != handle) { if (IsCloseable()) { ::CloseHandle(handle_); } handle_ = handle; } else { GTEST_CHECK_(!IsCloseable()) << "Resetting a valid handle to itself is likely a programmer error " "and thus not allowed."; } } bool AutoHandle::IsCloseable() const { // Different Windows APIs may use either of these values to represent an // invalid handle. return handle_ != nullptr && handle_ != INVALID_HANDLE_VALUE; } Notification::Notification() : event_(::CreateEvent(nullptr, // Default security attributes. TRUE, // Do not reset automatically. FALSE, // Initially unset. nullptr)) { // Anonymous event. GTEST_CHECK_(event_.Get() != nullptr); } void Notification::Notify() { GTEST_CHECK_(::SetEvent(event_.Get()) != FALSE); } void Notification::WaitForNotification() { GTEST_CHECK_( ::WaitForSingleObject(event_.Get(), INFINITE) == WAIT_OBJECT_0); } Mutex::Mutex() : owner_thread_id_(0), type_(kDynamic), critical_section_init_phase_(0), critical_section_(new CRITICAL_SECTION) { ::InitializeCriticalSection(critical_section_); } Mutex::~Mutex() { // Static mutexes are leaked intentionally. It is not thread-safe to try // to clean them up. if (type_ == kDynamic) { ::DeleteCriticalSection(critical_section_); delete critical_section_; critical_section_ = nullptr; } } void Mutex::Lock() { ThreadSafeLazyInit(); ::EnterCriticalSection(critical_section_); owner_thread_id_ = ::GetCurrentThreadId(); } void Mutex::Unlock() { ThreadSafeLazyInit(); // We don't protect writing to owner_thread_id_ here, as it's the // caller's responsibility to ensure that the current thread holds the // mutex when this is called. owner_thread_id_ = 0; ::LeaveCriticalSection(critical_section_); } // Does nothing if the current thread holds the mutex. Otherwise, crashes // with high probability. void Mutex::AssertHeld() { ThreadSafeLazyInit(); GTEST_CHECK_(owner_thread_id_ == ::GetCurrentThreadId()) << "The current thread is not holding the mutex @" << this; } namespace { #ifdef _MSC_VER // Use the RAII idiom to flag mem allocs that are intentionally never // deallocated. The motivation is to silence the false positive mem leaks // that are reported by the debug version of MS's CRT which can only detect // if an alloc is missing a matching deallocation. // Example: // MemoryIsNotDeallocated memory_is_not_deallocated; // critical_section_ = new CRITICAL_SECTION; // class MemoryIsNotDeallocated { public: MemoryIsNotDeallocated() : old_crtdbg_flag_(0) { old_crtdbg_flag_ = _CrtSetDbgFlag(_CRTDBG_REPORT_FLAG); // Set heap allocation block type to _IGNORE_BLOCK so that MS debug CRT // doesn't report mem leak if there's no matching deallocation. _CrtSetDbgFlag(old_crtdbg_flag_ & ~_CRTDBG_ALLOC_MEM_DF); } ~MemoryIsNotDeallocated() { // Restore the original _CRTDBG_ALLOC_MEM_DF flag _CrtSetDbgFlag(old_crtdbg_flag_); } private: int old_crtdbg_flag_; GTEST_DISALLOW_COPY_AND_ASSIGN_(MemoryIsNotDeallocated); }; #endif // _MSC_VER } // namespace // Initializes owner_thread_id_ and critical_section_ in static mutexes. void Mutex::ThreadSafeLazyInit() { // Dynamic mutexes are initialized in the constructor. if (type_ == kStatic) { switch ( ::InterlockedCompareExchange(&critical_section_init_phase_, 1L, 0L)) { case 0: // If critical_section_init_phase_ was 0 before the exchange, we // are the first to test it and need to perform the initialization. owner_thread_id_ = 0; { // Use RAII to flag that following mem alloc is never deallocated. #ifdef _MSC_VER MemoryIsNotDeallocated memory_is_not_deallocated; #endif // _MSC_VER critical_section_ = new CRITICAL_SECTION; } ::InitializeCriticalSection(critical_section_); // Updates the critical_section_init_phase_ to 2 to signal // initialization complete. GTEST_CHECK_(::InterlockedCompareExchange( &critical_section_init_phase_, 2L, 1L) == 1L); break; case 1: // Somebody else is already initializing the mutex; spin until they // are done. while (::InterlockedCompareExchange(&critical_section_init_phase_, 2L, 2L) != 2L) { // Possibly yields the rest of the thread's time slice to other // threads. ::Sleep(0); } break; case 2: break; // The mutex is already initialized and ready for use. default: GTEST_CHECK_(false) << "Unexpected value of critical_section_init_phase_ " << "while initializing a static mutex."; } } } namespace { class ThreadWithParamSupport : public ThreadWithParamBase { public: static HANDLE CreateThread(Runnable* runnable, Notification* thread_can_start) { ThreadMainParam* param = new ThreadMainParam(runnable, thread_can_start); DWORD thread_id; HANDLE thread_handle = ::CreateThread( nullptr, // Default security. 0, // Default stack size. &ThreadWithParamSupport::ThreadMain, param, // Parameter to ThreadMainStatic 0x0, // Default creation flags. &thread_id); // Need a valid pointer for the call to work under Win98. GTEST_CHECK_(thread_handle != nullptr) << "CreateThread failed with error " << ::GetLastError() << "."; if (thread_handle == nullptr) { delete param; } return thread_handle; } private: struct ThreadMainParam { ThreadMainParam(Runnable* runnable, Notification* thread_can_start) : runnable_(runnable), thread_can_start_(thread_can_start) { } std::unique_ptr runnable_; // Does not own. Notification* thread_can_start_; }; static DWORD WINAPI ThreadMain(void* ptr) { // Transfers ownership. std::unique_ptr param(static_cast(ptr)); if (param->thread_can_start_ != nullptr) param->thread_can_start_->WaitForNotification(); param->runnable_->Run(); return 0; } // Prohibit instantiation. ThreadWithParamSupport(); GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParamSupport); }; } // namespace ThreadWithParamBase::ThreadWithParamBase(Runnable *runnable, Notification* thread_can_start) : thread_(ThreadWithParamSupport::CreateThread(runnable, thread_can_start)) { } ThreadWithParamBase::~ThreadWithParamBase() { Join(); } void ThreadWithParamBase::Join() { GTEST_CHECK_(::WaitForSingleObject(thread_.Get(), INFINITE) == WAIT_OBJECT_0) << "Failed to join the thread with error " << ::GetLastError() << "."; } // Maps a thread to a set of ThreadIdToThreadLocals that have values // instantiated on that thread and notifies them when the thread exits. A // ThreadLocal instance is expected to persist until all threads it has // values on have terminated. class ThreadLocalRegistryImpl { public: // Registers thread_local_instance as having value on the current thread. // Returns a value that can be used to identify the thread from other threads. static ThreadLocalValueHolderBase* GetValueOnCurrentThread( const ThreadLocalBase* thread_local_instance) { DWORD current_thread = ::GetCurrentThreadId(); MutexLock lock(&mutex_); ThreadIdToThreadLocals* const thread_to_thread_locals = GetThreadLocalsMapLocked(); ThreadIdToThreadLocals::iterator thread_local_pos = thread_to_thread_locals->find(current_thread); if (thread_local_pos == thread_to_thread_locals->end()) { thread_local_pos = thread_to_thread_locals->insert( std::make_pair(current_thread, ThreadLocalValues())).first; StartWatcherThreadFor(current_thread); } ThreadLocalValues& thread_local_values = thread_local_pos->second; ThreadLocalValues::iterator value_pos = thread_local_values.find(thread_local_instance); if (value_pos == thread_local_values.end()) { value_pos = thread_local_values .insert(std::make_pair( thread_local_instance, std::shared_ptr( thread_local_instance->NewValueForCurrentThread()))) .first; } return value_pos->second.get(); } static void OnThreadLocalDestroyed( const ThreadLocalBase* thread_local_instance) { std::vector > value_holders; // Clean up the ThreadLocalValues data structure while holding the lock, but // defer the destruction of the ThreadLocalValueHolderBases. { MutexLock lock(&mutex_); ThreadIdToThreadLocals* const thread_to_thread_locals = GetThreadLocalsMapLocked(); for (ThreadIdToThreadLocals::iterator it = thread_to_thread_locals->begin(); it != thread_to_thread_locals->end(); ++it) { ThreadLocalValues& thread_local_values = it->second; ThreadLocalValues::iterator value_pos = thread_local_values.find(thread_local_instance); if (value_pos != thread_local_values.end()) { value_holders.push_back(value_pos->second); thread_local_values.erase(value_pos); // This 'if' can only be successful at most once, so theoretically we // could break out of the loop here, but we don't bother doing so. } } } // Outside the lock, let the destructor for 'value_holders' deallocate the // ThreadLocalValueHolderBases. } static void OnThreadExit(DWORD thread_id) { GTEST_CHECK_(thread_id != 0) << ::GetLastError(); std::vector > value_holders; // Clean up the ThreadIdToThreadLocals data structure while holding the // lock, but defer the destruction of the ThreadLocalValueHolderBases. { MutexLock lock(&mutex_); ThreadIdToThreadLocals* const thread_to_thread_locals = GetThreadLocalsMapLocked(); ThreadIdToThreadLocals::iterator thread_local_pos = thread_to_thread_locals->find(thread_id); if (thread_local_pos != thread_to_thread_locals->end()) { ThreadLocalValues& thread_local_values = thread_local_pos->second; for (ThreadLocalValues::iterator value_pos = thread_local_values.begin(); value_pos != thread_local_values.end(); ++value_pos) { value_holders.push_back(value_pos->second); } thread_to_thread_locals->erase(thread_local_pos); } } // Outside the lock, let the destructor for 'value_holders' deallocate the // ThreadLocalValueHolderBases. } private: // In a particular thread, maps a ThreadLocal object to its value. typedef std::map > ThreadLocalValues; // Stores all ThreadIdToThreadLocals having values in a thread, indexed by // thread's ID. typedef std::map ThreadIdToThreadLocals; // Holds the thread id and thread handle that we pass from // StartWatcherThreadFor to WatcherThreadFunc. typedef std::pair ThreadIdAndHandle; static void StartWatcherThreadFor(DWORD thread_id) { // The returned handle will be kept in thread_map and closed by // watcher_thread in WatcherThreadFunc. HANDLE thread = ::OpenThread(SYNCHRONIZE | THREAD_QUERY_INFORMATION, FALSE, thread_id); GTEST_CHECK_(thread != nullptr); // We need to pass a valid thread ID pointer into CreateThread for it // to work correctly under Win98. DWORD watcher_thread_id; HANDLE watcher_thread = ::CreateThread( nullptr, // Default security. 0, // Default stack size &ThreadLocalRegistryImpl::WatcherThreadFunc, reinterpret_cast(new ThreadIdAndHandle(thread_id, thread)), CREATE_SUSPENDED, &watcher_thread_id); GTEST_CHECK_(watcher_thread != nullptr); // Give the watcher thread the same priority as ours to avoid being // blocked by it. ::SetThreadPriority(watcher_thread, ::GetThreadPriority(::GetCurrentThread())); ::ResumeThread(watcher_thread); ::CloseHandle(watcher_thread); } // Monitors exit from a given thread and notifies those // ThreadIdToThreadLocals about thread termination. static DWORD WINAPI WatcherThreadFunc(LPVOID param) { const ThreadIdAndHandle* tah = reinterpret_cast(param); GTEST_CHECK_( ::WaitForSingleObject(tah->second, INFINITE) == WAIT_OBJECT_0); OnThreadExit(tah->first); ::CloseHandle(tah->second); delete tah; return 0; } // Returns map of thread local instances. static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() { mutex_.AssertHeld(); #ifdef _MSC_VER MemoryIsNotDeallocated memory_is_not_deallocated; #endif // _MSC_VER static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals(); return map; } // Protects access to GetThreadLocalsMapLocked() and its return value. static Mutex mutex_; // Protects access to GetThreadMapLocked() and its return value. static Mutex thread_map_mutex_; }; Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex); Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex); ThreadLocalValueHolderBase* ThreadLocalRegistry::GetValueOnCurrentThread( const ThreadLocalBase* thread_local_instance) { return ThreadLocalRegistryImpl::GetValueOnCurrentThread( thread_local_instance); } void ThreadLocalRegistry::OnThreadLocalDestroyed( const ThreadLocalBase* thread_local_instance) { ThreadLocalRegistryImpl::OnThreadLocalDestroyed(thread_local_instance); } #endif // GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS #if GTEST_USES_POSIX_RE // Implements RE. Currently only needed for death tests. RE::~RE() { if (is_valid_) { // regfree'ing an invalid regex might crash because the content // of the regex is undefined. Since the regex's are essentially // the same, one cannot be valid (or invalid) without the other // being so too. regfree(&partial_regex_); regfree(&full_regex_); } free(const_cast(pattern_)); } // Returns true if and only if regular expression re matches the entire str. bool RE::FullMatch(const char* str, const RE& re) { if (!re.is_valid_) return false; regmatch_t match; return regexec(&re.full_regex_, str, 1, &match, 0) == 0; } // Returns true if and only if regular expression re matches a substring of // str (including str itself). bool RE::PartialMatch(const char* str, const RE& re) { if (!re.is_valid_) return false; regmatch_t match; return regexec(&re.partial_regex_, str, 1, &match, 0) == 0; } // Initializes an RE from its string representation. void RE::Init(const char* regex) { pattern_ = posix::StrDup(regex); // Reserves enough bytes to hold the regular expression used for a // full match. const size_t full_regex_len = strlen(regex) + 10; char* const full_pattern = new char[full_regex_len]; snprintf(full_pattern, full_regex_len, "^(%s)$", regex); is_valid_ = regcomp(&full_regex_, full_pattern, REG_EXTENDED) == 0; // We want to call regcomp(&partial_regex_, ...) even if the // previous expression returns false. Otherwise partial_regex_ may // not be properly initialized can may cause trouble when it's // freed. // // Some implementation of POSIX regex (e.g. on at least some // versions of Cygwin) doesn't accept the empty string as a valid // regex. We change it to an equivalent form "()" to be safe. if (is_valid_) { const char* const partial_regex = (*regex == '\0') ? "()" : regex; is_valid_ = regcomp(&partial_regex_, partial_regex, REG_EXTENDED) == 0; } EXPECT_TRUE(is_valid_) << "Regular expression \"" << regex << "\" is not a valid POSIX Extended regular expression."; delete[] full_pattern; } #elif GTEST_USES_SIMPLE_RE // Returns true if and only if ch appears anywhere in str (excluding the // terminating '\0' character). bool IsInSet(char ch, const char* str) { return ch != '\0' && strchr(str, ch) != nullptr; } // Returns true if and only if ch belongs to the given classification. // Unlike similar functions in , these aren't affected by the // current locale. bool IsAsciiDigit(char ch) { return '0' <= ch && ch <= '9'; } bool IsAsciiPunct(char ch) { return IsInSet(ch, "^-!\"#$%&'()*+,./:;<=>?@[\\]_`{|}~"); } bool IsRepeat(char ch) { return IsInSet(ch, "?*+"); } bool IsAsciiWhiteSpace(char ch) { return IsInSet(ch, " \f\n\r\t\v"); } bool IsAsciiWordChar(char ch) { return ('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z') || ('0' <= ch && ch <= '9') || ch == '_'; } // Returns true if and only if "\\c" is a supported escape sequence. bool IsValidEscape(char c) { return (IsAsciiPunct(c) || IsInSet(c, "dDfnrsStvwW")); } // Returns true if and only if the given atom (specified by escaped and // pattern) matches ch. The result is undefined if the atom is invalid. bool AtomMatchesChar(bool escaped, char pattern_char, char ch) { if (escaped) { // "\\p" where p is pattern_char. switch (pattern_char) { case 'd': return IsAsciiDigit(ch); case 'D': return !IsAsciiDigit(ch); case 'f': return ch == '\f'; case 'n': return ch == '\n'; case 'r': return ch == '\r'; case 's': return IsAsciiWhiteSpace(ch); case 'S': return !IsAsciiWhiteSpace(ch); case 't': return ch == '\t'; case 'v': return ch == '\v'; case 'w': return IsAsciiWordChar(ch); case 'W': return !IsAsciiWordChar(ch); } return IsAsciiPunct(pattern_char) && pattern_char == ch; } return (pattern_char == '.' && ch != '\n') || pattern_char == ch; } // Helper function used by ValidateRegex() to format error messages. static std::string FormatRegexSyntaxError(const char* regex, int index) { return (Message() << "Syntax error at index " << index << " in simple regular expression \"" << regex << "\": ").GetString(); } // Generates non-fatal failures and returns false if regex is invalid; // otherwise returns true. bool ValidateRegex(const char* regex) { if (regex == nullptr) { ADD_FAILURE() << "NULL is not a valid simple regular expression."; return false; } bool is_valid = true; // True if and only if ?, *, or + can follow the previous atom. bool prev_repeatable = false; for (int i = 0; regex[i]; i++) { if (regex[i] == '\\') { // An escape sequence i++; if (regex[i] == '\0') { ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) << "'\\' cannot appear at the end."; return false; } if (!IsValidEscape(regex[i])) { ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) << "invalid escape sequence \"\\" << regex[i] << "\"."; is_valid = false; } prev_repeatable = true; } else { // Not an escape sequence. const char ch = regex[i]; if (ch == '^' && i > 0) { ADD_FAILURE() << FormatRegexSyntaxError(regex, i) << "'^' can only appear at the beginning."; is_valid = false; } else if (ch == '$' && regex[i + 1] != '\0') { ADD_FAILURE() << FormatRegexSyntaxError(regex, i) << "'$' can only appear at the end."; is_valid = false; } else if (IsInSet(ch, "()[]{}|")) { ADD_FAILURE() << FormatRegexSyntaxError(regex, i) << "'" << ch << "' is unsupported."; is_valid = false; } else if (IsRepeat(ch) && !prev_repeatable) { ADD_FAILURE() << FormatRegexSyntaxError(regex, i) << "'" << ch << "' can only follow a repeatable token."; is_valid = false; } prev_repeatable = !IsInSet(ch, "^$?*+"); } } return is_valid; } // Matches a repeated regex atom followed by a valid simple regular // expression. The regex atom is defined as c if escaped is false, // or \c otherwise. repeat is the repetition meta character (?, *, // or +). The behavior is undefined if str contains too many // characters to be indexable by size_t, in which case the test will // probably time out anyway. We are fine with this limitation as // std::string has it too. bool MatchRepetitionAndRegexAtHead( bool escaped, char c, char repeat, const char* regex, const char* str) { const size_t min_count = (repeat == '+') ? 1 : 0; const size_t max_count = (repeat == '?') ? 1 : static_cast(-1) - 1; // We cannot call numeric_limits::max() as it conflicts with the // max() macro on Windows. for (size_t i = 0; i <= max_count; ++i) { // We know that the atom matches each of the first i characters in str. if (i >= min_count && MatchRegexAtHead(regex, str + i)) { // We have enough matches at the head, and the tail matches too. // Since we only care about *whether* the pattern matches str // (as opposed to *how* it matches), there is no need to find a // greedy match. return true; } if (str[i] == '\0' || !AtomMatchesChar(escaped, c, str[i])) return false; } return false; } // Returns true if and only if regex matches a prefix of str. regex must // be a valid simple regular expression and not start with "^", or the // result is undefined. bool MatchRegexAtHead(const char* regex, const char* str) { if (*regex == '\0') // An empty regex matches a prefix of anything. return true; // "$" only matches the end of a string. Note that regex being // valid guarantees that there's nothing after "$" in it. if (*regex == '$') return *str == '\0'; // Is the first thing in regex an escape sequence? const bool escaped = *regex == '\\'; if (escaped) ++regex; if (IsRepeat(regex[1])) { // MatchRepetitionAndRegexAtHead() calls MatchRegexAtHead(), so // here's an indirect recursion. It terminates as the regex gets // shorter in each recursion. return MatchRepetitionAndRegexAtHead( escaped, regex[0], regex[1], regex + 2, str); } else { // regex isn't empty, isn't "$", and doesn't start with a // repetition. We match the first atom of regex with the first // character of str and recurse. return (*str != '\0') && AtomMatchesChar(escaped, *regex, *str) && MatchRegexAtHead(regex + 1, str + 1); } } // Returns true if and only if regex matches any substring of str. regex must // be a valid simple regular expression, or the result is undefined. // // The algorithm is recursive, but the recursion depth doesn't exceed // the regex length, so we won't need to worry about running out of // stack space normally. In rare cases the time complexity can be // exponential with respect to the regex length + the string length, // but usually it's must faster (often close to linear). bool MatchRegexAnywhere(const char* regex, const char* str) { if (regex == nullptr || str == nullptr) return false; if (*regex == '^') return MatchRegexAtHead(regex + 1, str); // A successful match can be anywhere in str. do { if (MatchRegexAtHead(regex, str)) return true; } while (*str++ != '\0'); return false; } // Implements the RE class. RE::~RE() { free(const_cast(pattern_)); free(const_cast(full_pattern_)); } // Returns true if and only if regular expression re matches the entire str. bool RE::FullMatch(const char* str, const RE& re) { return re.is_valid_ && MatchRegexAnywhere(re.full_pattern_, str); } // Returns true if and only if regular expression re matches a substring of // str (including str itself). bool RE::PartialMatch(const char* str, const RE& re) { return re.is_valid_ && MatchRegexAnywhere(re.pattern_, str); } // Initializes an RE from its string representation. void RE::Init(const char* regex) { pattern_ = full_pattern_ = nullptr; if (regex != nullptr) { pattern_ = posix::StrDup(regex); } is_valid_ = ValidateRegex(regex); if (!is_valid_) { // No need to calculate the full pattern when the regex is invalid. return; } const size_t len = strlen(regex); // Reserves enough bytes to hold the regular expression used for a // full match: we need space to prepend a '^', append a '$', and // terminate the string with '\0'. char* buffer = static_cast(malloc(len + 3)); full_pattern_ = buffer; if (*regex != '^') *buffer++ = '^'; // Makes sure full_pattern_ starts with '^'. // We don't use snprintf or strncpy, as they trigger a warning when // compiled with VC++ 8.0. memcpy(buffer, regex, len); buffer += len; if (len == 0 || regex[len - 1] != '$') *buffer++ = '$'; // Makes sure full_pattern_ ends with '$'. *buffer = '\0'; } #endif // GTEST_USES_POSIX_RE const char kUnknownFile[] = "unknown file"; // Formats a source file path and a line number as they would appear // in an error message from the compiler used to compile this code. GTEST_API_ ::std::string FormatFileLocation(const char* file, int line) { const std::string file_name(file == nullptr ? kUnknownFile : file); if (line < 0) { return file_name + ":"; } #ifdef _MSC_VER return file_name + "(" + StreamableToString(line) + "):"; #else return file_name + ":" + StreamableToString(line) + ":"; #endif // _MSC_VER } // Formats a file location for compiler-independent XML output. // Although this function is not platform dependent, we put it next to // FormatFileLocation in order to contrast the two functions. // Note that FormatCompilerIndependentFileLocation() does NOT append colon // to the file location it produces, unlike FormatFileLocation(). GTEST_API_ ::std::string FormatCompilerIndependentFileLocation( const char* file, int line) { const std::string file_name(file == nullptr ? kUnknownFile : file); if (line < 0) return file_name; else return file_name + ":" + StreamableToString(line); } GTestLog::GTestLog(GTestLogSeverity severity, const char* file, int line) : severity_(severity) { const char* const marker = severity == GTEST_INFO ? "[ INFO ]" : severity == GTEST_WARNING ? "[WARNING]" : severity == GTEST_ERROR ? "[ ERROR ]" : "[ FATAL ]"; GetStream() << ::std::endl << marker << " " << FormatFileLocation(file, line).c_str() << ": "; } // Flushes the buffers and, if severity is GTEST_FATAL, aborts the program. GTestLog::~GTestLog() { GetStream() << ::std::endl; if (severity_ == GTEST_FATAL) { fflush(stderr); posix::Abort(); } } // Disable Microsoft deprecation warnings for POSIX functions called from // this class (creat, dup, dup2, and close) GTEST_DISABLE_MSC_DEPRECATED_PUSH_() #if GTEST_HAS_STREAM_REDIRECTION // Object that captures an output stream (stdout/stderr). class CapturedStream { public: // The ctor redirects the stream to a temporary file. explicit CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) { # if GTEST_OS_WINDOWS char temp_dir_path[MAX_PATH + 1] = { '\0' }; // NOLINT char temp_file_path[MAX_PATH + 1] = { '\0' }; // NOLINT ::GetTempPathA(sizeof(temp_dir_path), temp_dir_path); const UINT success = ::GetTempFileNameA(temp_dir_path, "gtest_redir", 0, // Generate unique file name. temp_file_path); GTEST_CHECK_(success != 0) << "Unable to create a temporary file in " << temp_dir_path; const int captured_fd = creat(temp_file_path, _S_IREAD | _S_IWRITE); GTEST_CHECK_(captured_fd != -1) << "Unable to open temporary file " << temp_file_path; filename_ = temp_file_path; # else // There's no guarantee that a test has write access to the current // directory, so we create the temporary file in the /tmp directory // instead. We use /tmp on most systems, and /sdcard on Android. // That's because Android doesn't have /tmp. # if GTEST_OS_LINUX_ANDROID // Note: Android applications are expected to call the framework's // Context.getExternalStorageDirectory() method through JNI to get // the location of the world-writable SD Card directory. However, // this requires a Context handle, which cannot be retrieved // globally from native code. Doing so also precludes running the // code as part of a regular standalone executable, which doesn't // run in a Dalvik process (e.g. when running it through 'adb shell'). // // The location /data/local/tmp is directly accessible from native code. // '/sdcard' and other variants cannot be relied on, as they are not // guaranteed to be mounted, or may have a delay in mounting. char name_template[] = "/data/local/tmp/gtest_captured_stream.XXXXXX"; # else char name_template[] = "/tmp/captured_stream.XXXXXX"; # endif // GTEST_OS_LINUX_ANDROID const int captured_fd = mkstemp(name_template); if (captured_fd == -1) { GTEST_LOG_(WARNING) << "Failed to create tmp file " << name_template << " for test; does the test have access to the /tmp directory?"; } filename_ = name_template; # endif // GTEST_OS_WINDOWS fflush(nullptr); dup2(captured_fd, fd_); close(captured_fd); } ~CapturedStream() { remove(filename_.c_str()); } std::string GetCapturedString() { if (uncaptured_fd_ != -1) { // Restores the original stream. fflush(nullptr); dup2(uncaptured_fd_, fd_); close(uncaptured_fd_); uncaptured_fd_ = -1; } FILE* const file = posix::FOpen(filename_.c_str(), "r"); if (file == nullptr) { GTEST_LOG_(FATAL) << "Failed to open tmp file " << filename_ << " for capturing stream."; } const std::string content = ReadEntireFile(file); posix::FClose(file); return content; } private: const int fd_; // A stream to capture. int uncaptured_fd_; // Name of the temporary file holding the stderr output. ::std::string filename_; GTEST_DISALLOW_COPY_AND_ASSIGN_(CapturedStream); }; GTEST_DISABLE_MSC_DEPRECATED_POP_() static CapturedStream* g_captured_stderr = nullptr; static CapturedStream* g_captured_stdout = nullptr; // Starts capturing an output stream (stdout/stderr). static void CaptureStream(int fd, const char* stream_name, CapturedStream** stream) { if (*stream != nullptr) { GTEST_LOG_(FATAL) << "Only one " << stream_name << " capturer can exist at a time."; } *stream = new CapturedStream(fd); } // Stops capturing the output stream and returns the captured string. static std::string GetCapturedStream(CapturedStream** captured_stream) { const std::string content = (*captured_stream)->GetCapturedString(); delete *captured_stream; *captured_stream = nullptr; return content; } // Starts capturing stdout. void CaptureStdout() { CaptureStream(kStdOutFileno, "stdout", &g_captured_stdout); } // Starts capturing stderr. void CaptureStderr() { CaptureStream(kStdErrFileno, "stderr", &g_captured_stderr); } // Stops capturing stdout and returns the captured string. std::string GetCapturedStdout() { return GetCapturedStream(&g_captured_stdout); } // Stops capturing stderr and returns the captured string. std::string GetCapturedStderr() { return GetCapturedStream(&g_captured_stderr); } #endif // GTEST_HAS_STREAM_REDIRECTION size_t GetFileSize(FILE* file) { fseek(file, 0, SEEK_END); return static_cast(ftell(file)); } std::string ReadEntireFile(FILE* file) { const size_t file_size = GetFileSize(file); char* const buffer = new char[file_size]; size_t bytes_last_read = 0; // # of bytes read in the last fread() size_t bytes_read = 0; // # of bytes read so far fseek(file, 0, SEEK_SET); // Keeps reading the file until we cannot read further or the // pre-determined file size is reached. do { bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file); bytes_read += bytes_last_read; } while (bytes_last_read > 0 && bytes_read < file_size); const std::string content(buffer, bytes_read); delete[] buffer; return content; } #if GTEST_HAS_DEATH_TEST static const std::vector* g_injected_test_argvs = nullptr; // Owned. std::vector GetInjectableArgvs() { if (g_injected_test_argvs != nullptr) { return *g_injected_test_argvs; } return GetArgvs(); } void SetInjectableArgvs(const std::vector* new_argvs) { if (g_injected_test_argvs != new_argvs) delete g_injected_test_argvs; g_injected_test_argvs = new_argvs; } void SetInjectableArgvs(const std::vector& new_argvs) { SetInjectableArgvs( new std::vector(new_argvs.begin(), new_argvs.end())); } void ClearInjectableArgvs() { delete g_injected_test_argvs; g_injected_test_argvs = nullptr; } #endif // GTEST_HAS_DEATH_TEST #if GTEST_OS_WINDOWS_MOBILE namespace posix { void Abort() { DebugBreak(); TerminateProcess(GetCurrentProcess(), 1); } } // namespace posix #endif // GTEST_OS_WINDOWS_MOBILE // Returns the name of the environment variable corresponding to the // given flag. For example, FlagToEnvVar("foo") will return // "GTEST_FOO" in the open-source version. static std::string FlagToEnvVar(const char* flag) { const std::string full_flag = (Message() << GTEST_FLAG_PREFIX_ << flag).GetString(); Message env_var; for (size_t i = 0; i != full_flag.length(); i++) { env_var << ToUpper(full_flag.c_str()[i]); } return env_var.GetString(); } // Parses 'str' for a 32-bit signed integer. If successful, writes // the result to *value and returns true; otherwise leaves *value // unchanged and returns false. bool ParseInt32(const Message& src_text, const char* str, Int32* value) { // Parses the environment variable as a decimal integer. char* end = nullptr; const long long_value = strtol(str, &end, 10); // NOLINT // Has strtol() consumed all characters in the string? if (*end != '\0') { // No - an invalid character was encountered. Message msg; msg << "WARNING: " << src_text << " is expected to be a 32-bit integer, but actually" << " has value \"" << str << "\".\n"; printf("%s", msg.GetString().c_str()); fflush(stdout); return false; } // Is the parsed value in the range of an Int32? const Int32 result = static_cast(long_value); if (long_value == LONG_MAX || long_value == LONG_MIN || // The parsed value overflows as a long. (strtol() returns // LONG_MAX or LONG_MIN when the input overflows.) result != long_value // The parsed value overflows as an Int32. ) { Message msg; msg << "WARNING: " << src_text << " is expected to be a 32-bit integer, but actually" << " has value " << str << ", which overflows.\n"; printf("%s", msg.GetString().c_str()); fflush(stdout); return false; } *value = result; return true; } // Reads and returns the Boolean environment variable corresponding to // the given flag; if it's not set, returns default_value. // // The value is considered true if and only if it's not "0". bool BoolFromGTestEnv(const char* flag, bool default_value) { #if defined(GTEST_GET_BOOL_FROM_ENV_) return GTEST_GET_BOOL_FROM_ENV_(flag, default_value); #else const std::string env_var = FlagToEnvVar(flag); const char* const string_value = posix::GetEnv(env_var.c_str()); return string_value == nullptr ? default_value : strcmp(string_value, "0") != 0; #endif // defined(GTEST_GET_BOOL_FROM_ENV_) } // Reads and returns a 32-bit integer stored in the environment // variable corresponding to the given flag; if it isn't set or // doesn't represent a valid 32-bit integer, returns default_value. Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) { #if defined(GTEST_GET_INT32_FROM_ENV_) return GTEST_GET_INT32_FROM_ENV_(flag, default_value); #else const std::string env_var = FlagToEnvVar(flag); const char* const string_value = posix::GetEnv(env_var.c_str()); if (string_value == nullptr) { // The environment variable is not set. return default_value; } Int32 result = default_value; if (!ParseInt32(Message() << "Environment variable " << env_var, string_value, &result)) { printf("The default value %s is used.\n", (Message() << default_value).GetString().c_str()); fflush(stdout); return default_value; } return result; #endif // defined(GTEST_GET_INT32_FROM_ENV_) } // As a special case for the 'output' flag, if GTEST_OUTPUT is not // set, we look for XML_OUTPUT_FILE, which is set by the Bazel build // system. The value of XML_OUTPUT_FILE is a filename without the // "xml:" prefix of GTEST_OUTPUT. // Note that this is meant to be called at the call site so it does // not check that the flag is 'output' // In essence this checks an env variable called XML_OUTPUT_FILE // and if it is set we prepend "xml:" to its value, if it not set we return "" std::string OutputFlagAlsoCheckEnvVar(){ std::string default_value_for_output_flag = ""; const char* xml_output_file_env = posix::GetEnv("XML_OUTPUT_FILE"); if (nullptr != xml_output_file_env) { default_value_for_output_flag = std::string("xml:") + xml_output_file_env; } return default_value_for_output_flag; } // Reads and returns the string environment variable corresponding to // the given flag; if it's not set, returns default_value. const char* StringFromGTestEnv(const char* flag, const char* default_value) { #if defined(GTEST_GET_STRING_FROM_ENV_) return GTEST_GET_STRING_FROM_ENV_(flag, default_value); #else const std::string env_var = FlagToEnvVar(flag); const char* const value = posix::GetEnv(env_var.c_str()); return value == nullptr ? default_value : value; #endif // defined(GTEST_GET_STRING_FROM_ENV_) } } // namespace internal } // namespace testing // Copyright 2007, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // Google Test - The Google C++ Testing and Mocking Framework // // This file implements a universal value printer that can print a // value of any type T: // // void ::testing::internal::UniversalPrinter::Print(value, ostream_ptr); // // It uses the << operator when possible, and prints the bytes in the // object otherwise. A user can override its behavior for a class // type Foo by defining either operator<<(::std::ostream&, const Foo&) // or void PrintTo(const Foo&, ::std::ostream*) in the namespace that // defines Foo. #include #include #include #include // NOLINT #include namespace testing { namespace { using ::std::ostream; // Prints a segment of bytes in the given object. GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ void PrintByteSegmentInObjectTo(const unsigned char* obj_bytes, size_t start, size_t count, ostream* os) { char text[5] = ""; for (size_t i = 0; i != count; i++) { const size_t j = start + i; if (i != 0) { // Organizes the bytes into groups of 2 for easy parsing by // human. if ((j % 2) == 0) *os << ' '; else *os << '-'; } GTEST_SNPRINTF_(text, sizeof(text), "%02X", obj_bytes[j]); *os << text; } } // Prints the bytes in the given value to the given ostream. void PrintBytesInObjectToImpl(const unsigned char* obj_bytes, size_t count, ostream* os) { // Tells the user how big the object is. *os << count << "-byte object <"; const size_t kThreshold = 132; const size_t kChunkSize = 64; // If the object size is bigger than kThreshold, we'll have to omit // some details by printing only the first and the last kChunkSize // bytes. if (count < kThreshold) { PrintByteSegmentInObjectTo(obj_bytes, 0, count, os); } else { PrintByteSegmentInObjectTo(obj_bytes, 0, kChunkSize, os); *os << " ... "; // Rounds up to 2-byte boundary. const size_t resume_pos = (count - kChunkSize + 1)/2*2; PrintByteSegmentInObjectTo(obj_bytes, resume_pos, count - resume_pos, os); } *os << ">"; } } // namespace namespace internal2 { // Delegates to PrintBytesInObjectToImpl() to print the bytes in the // given object. The delegation simplifies the implementation, which // uses the << operator and thus is easier done outside of the // ::testing::internal namespace, which contains a << operator that // sometimes conflicts with the one in STL. void PrintBytesInObjectTo(const unsigned char* obj_bytes, size_t count, ostream* os) { PrintBytesInObjectToImpl(obj_bytes, count, os); } } // namespace internal2 namespace internal { // Depending on the value of a char (or wchar_t), we print it in one // of three formats: // - as is if it's a printable ASCII (e.g. 'a', '2', ' '), // - as a hexadecimal escape sequence (e.g. '\x7F'), or // - as a special escape sequence (e.g. '\r', '\n'). enum CharFormat { kAsIs, kHexEscape, kSpecialEscape }; // Returns true if c is a printable ASCII character. We test the // value of c directly instead of calling isprint(), which is buggy on // Windows Mobile. inline bool IsPrintableAscii(wchar_t c) { return 0x20 <= c && c <= 0x7E; } // Prints a wide or narrow char c as a character literal without the // quotes, escaping it when necessary; returns how c was formatted. // The template argument UnsignedChar is the unsigned version of Char, // which is the type of c. template static CharFormat PrintAsCharLiteralTo(Char c, ostream* os) { wchar_t w_c = static_cast(c); switch (w_c) { case L'\0': *os << "\\0"; break; case L'\'': *os << "\\'"; break; case L'\\': *os << "\\\\"; break; case L'\a': *os << "\\a"; break; case L'\b': *os << "\\b"; break; case L'\f': *os << "\\f"; break; case L'\n': *os << "\\n"; break; case L'\r': *os << "\\r"; break; case L'\t': *os << "\\t"; break; case L'\v': *os << "\\v"; break; default: if (IsPrintableAscii(w_c)) { *os << static_cast(c); return kAsIs; } else { ostream::fmtflags flags = os->flags(); *os << "\\x" << std::hex << std::uppercase << static_cast(static_cast(c)); os->flags(flags); return kHexEscape; } } return kSpecialEscape; } // Prints a wchar_t c as if it's part of a string literal, escaping it when // necessary; returns how c was formatted. static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) { switch (c) { case L'\'': *os << "'"; return kAsIs; case L'"': *os << "\\\""; return kSpecialEscape; default: return PrintAsCharLiteralTo(c, os); } } // Prints a char c as if it's part of a string literal, escaping it when // necessary; returns how c was formatted. static CharFormat PrintAsStringLiteralTo(char c, ostream* os) { return PrintAsStringLiteralTo( static_cast(static_cast(c)), os); } // Prints a wide or narrow character c and its code. '\0' is printed // as "'\\0'", other unprintable characters are also properly escaped // using the standard C++ escape sequence. The template argument // UnsignedChar is the unsigned version of Char, which is the type of c. template void PrintCharAndCodeTo(Char c, ostream* os) { // First, print c as a literal in the most readable form we can find. *os << ((sizeof(c) > 1) ? "L'" : "'"); const CharFormat format = PrintAsCharLiteralTo(c, os); *os << "'"; // To aid user debugging, we also print c's code in decimal, unless // it's 0 (in which case c was printed as '\\0', making the code // obvious). if (c == 0) return; *os << " (" << static_cast(c); // For more convenience, we print c's code again in hexadecimal, // unless c was already printed in the form '\x##' or the code is in // [1, 9]. if (format == kHexEscape || (1 <= c && c <= 9)) { // Do nothing. } else { *os << ", 0x" << String::FormatHexInt(static_cast(c)); } *os << ")"; } void PrintTo(unsigned char c, ::std::ostream* os) { PrintCharAndCodeTo(c, os); } void PrintTo(signed char c, ::std::ostream* os) { PrintCharAndCodeTo(c, os); } // Prints a wchar_t as a symbol if it is printable or as its internal // code otherwise and also as its code. L'\0' is printed as "L'\\0'". void PrintTo(wchar_t wc, ostream* os) { PrintCharAndCodeTo(wc, os); } // Prints the given array of characters to the ostream. CharType must be either // char or wchar_t. // The array starts at begin, the length is len, it may include '\0' characters // and may not be NUL-terminated. template GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ static CharFormat PrintCharsAsStringTo( const CharType* begin, size_t len, ostream* os) { const char* const kQuoteBegin = sizeof(CharType) == 1 ? "\"" : "L\""; *os << kQuoteBegin; bool is_previous_hex = false; CharFormat print_format = kAsIs; for (size_t index = 0; index < len; ++index) { const CharType cur = begin[index]; if (is_previous_hex && IsXDigit(cur)) { // Previous character is of '\x..' form and this character can be // interpreted as another hexadecimal digit in its number. Break string to // disambiguate. *os << "\" " << kQuoteBegin; } is_previous_hex = PrintAsStringLiteralTo(cur, os) == kHexEscape; // Remember if any characters required hex escaping. if (is_previous_hex) { print_format = kHexEscape; } } *os << "\""; return print_format; } // Prints a (const) char/wchar_t array of 'len' elements, starting at address // 'begin'. CharType must be either char or wchar_t. template GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ static void UniversalPrintCharArray( const CharType* begin, size_t len, ostream* os) { // The code // const char kFoo[] = "foo"; // generates an array of 4, not 3, elements, with the last one being '\0'. // // Therefore when printing a char array, we don't print the last element if // it's '\0', such that the output matches the string literal as it's // written in the source code. if (len > 0 && begin[len - 1] == '\0') { PrintCharsAsStringTo(begin, len - 1, os); return; } // If, however, the last element in the array is not '\0', e.g. // const char kFoo[] = { 'f', 'o', 'o' }; // we must print the entire array. We also print a message to indicate // that the array is not NUL-terminated. PrintCharsAsStringTo(begin, len, os); *os << " (no terminating NUL)"; } // Prints a (const) char array of 'len' elements, starting at address 'begin'. void UniversalPrintArray(const char* begin, size_t len, ostream* os) { UniversalPrintCharArray(begin, len, os); } // Prints a (const) wchar_t array of 'len' elements, starting at address // 'begin'. void UniversalPrintArray(const wchar_t* begin, size_t len, ostream* os) { UniversalPrintCharArray(begin, len, os); } // Prints the given C string to the ostream. void PrintTo(const char* s, ostream* os) { if (s == nullptr) { *os << "NULL"; } else { *os << ImplicitCast_(s) << " pointing to "; PrintCharsAsStringTo(s, strlen(s), os); } } // MSVC compiler can be configured to define whar_t as a typedef // of unsigned short. Defining an overload for const wchar_t* in that case // would cause pointers to unsigned shorts be printed as wide strings, // possibly accessing more memory than intended and causing invalid // memory accesses. MSVC defines _NATIVE_WCHAR_T_DEFINED symbol when // wchar_t is implemented as a native type. #if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) // Prints the given wide C string to the ostream. void PrintTo(const wchar_t* s, ostream* os) { if (s == nullptr) { *os << "NULL"; } else { *os << ImplicitCast_(s) << " pointing to "; PrintCharsAsStringTo(s, wcslen(s), os); } } #endif // wchar_t is native namespace { bool ContainsUnprintableControlCodes(const char* str, size_t length) { const unsigned char *s = reinterpret_cast(str); for (size_t i = 0; i < length; i++) { unsigned char ch = *s++; if (std::iscntrl(ch)) { switch (ch) { case '\t': case '\n': case '\r': break; default: return true; } } } return false; } bool IsUTF8TrailByte(unsigned char t) { return 0x80 <= t && t<= 0xbf; } bool IsValidUTF8(const char* str, size_t length) { const unsigned char *s = reinterpret_cast(str); for (size_t i = 0; i < length;) { unsigned char lead = s[i++]; if (lead <= 0x7f) { continue; // single-byte character (ASCII) 0..7F } if (lead < 0xc2) { return false; // trail byte or non-shortest form } else if (lead <= 0xdf && (i + 1) <= length && IsUTF8TrailByte(s[i])) { ++i; // 2-byte character } else if (0xe0 <= lead && lead <= 0xef && (i + 2) <= length && IsUTF8TrailByte(s[i]) && IsUTF8TrailByte(s[i + 1]) && // check for non-shortest form and surrogate (lead != 0xe0 || s[i] >= 0xa0) && (lead != 0xed || s[i] < 0xa0)) { i += 2; // 3-byte character } else if (0xf0 <= lead && lead <= 0xf4 && (i + 3) <= length && IsUTF8TrailByte(s[i]) && IsUTF8TrailByte(s[i + 1]) && IsUTF8TrailByte(s[i + 2]) && // check for non-shortest form (lead != 0xf0 || s[i] >= 0x90) && (lead != 0xf4 || s[i] < 0x90)) { i += 3; // 4-byte character } else { return false; } } return true; } void ConditionalPrintAsText(const char* str, size_t length, ostream* os) { if (!ContainsUnprintableControlCodes(str, length) && IsValidUTF8(str, length)) { *os << "\n As Text: \"" << str << "\""; } } } // anonymous namespace void PrintStringTo(const ::std::string& s, ostream* os) { if (PrintCharsAsStringTo(s.data(), s.size(), os) == kHexEscape) { if (GTEST_FLAG(print_utf8)) { ConditionalPrintAsText(s.data(), s.size(), os); } } } #if GTEST_HAS_STD_WSTRING void PrintWideStringTo(const ::std::wstring& s, ostream* os) { PrintCharsAsStringTo(s.data(), s.size(), os); } #endif // GTEST_HAS_STD_WSTRING } // namespace internal } // namespace testing // Copyright 2008, Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // The Google C++ Testing and Mocking Framework (Google Test) namespace testing { using internal::GetUnitTestImpl; // Gets the summary of the failure message by omitting the stack trace // in it. std::string TestPartResult::ExtractSummary(const char* message) { const char* const stack_trace = strstr(message, internal::kStackTraceMarker); return stack_trace == nullptr ? message : std::string(message, stack_trace); } // Prints a TestPartResult object. std::ostream& operator<<(std::ostream& os, const TestPartResult& result) { return os << result.file_name() << ":" << result.line_number() << ": " << (result.type() == TestPartResult::kSuccess ? "Success" : result.type() == TestPartResult::kSkip ? "Skipped" : result.type() == TestPartResult::kFatalFailure ? "Fatal failure" : "Non-fatal failure") << ":\n" << result.message() << std::endl; } // Appends a TestPartResult to the array. void TestPartResultArray::Append(const TestPartResult& result) { array_.push_back(result); } // Returns the TestPartResult at the given index (0-based). const TestPartResult& TestPartResultArray::GetTestPartResult(int index) const { if (index < 0 || index >= size()) { printf("\nInvalid index (%d) into TestPartResultArray.\n", index); internal::posix::Abort(); } return array_[static_cast(index)]; } // Returns the number of TestPartResult objects in the array. int TestPartResultArray::size() const { return static_cast(array_.size()); } namespace internal { HasNewFatalFailureHelper::HasNewFatalFailureHelper() : has_new_fatal_failure_(false), original_reporter_(GetUnitTestImpl()-> GetTestPartResultReporterForCurrentThread()) { GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread(this); } HasNewFatalFailureHelper::~HasNewFatalFailureHelper() { GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread( original_reporter_); } void HasNewFatalFailureHelper::ReportTestPartResult( const TestPartResult& result) { if (result.fatally_failed()) has_new_fatal_failure_ = true; original_reporter_->ReportTestPartResult(result); } } // namespace internal } // namespace testing // Copyright 2008 Google Inc. // All Rights Reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. namespace testing { namespace internal { #if GTEST_HAS_TYPED_TEST_P // Skips to the first non-space char in str. Returns an empty string if str // contains only whitespace characters. static const char* SkipSpaces(const char* str) { while (IsSpace(*str)) str++; return str; } static std::vector SplitIntoTestNames(const char* src) { std::vector name_vec; src = SkipSpaces(src); for (; src != nullptr; src = SkipComma(src)) { name_vec.push_back(StripTrailingSpaces(GetPrefixUntilComma(src))); } return name_vec; } // Verifies that registered_tests match the test names in // registered_tests_; returns registered_tests if successful, or // aborts the program otherwise. const char* TypedTestSuitePState::VerifyRegisteredTestNames( const char* file, int line, const char* registered_tests) { typedef RegisteredTestsMap::const_iterator RegisteredTestIter; registered_ = true; std::vector name_vec = SplitIntoTestNames(registered_tests); Message errors; std::set tests; for (std::vector::const_iterator name_it = name_vec.begin(); name_it != name_vec.end(); ++name_it) { const std::string& name = *name_it; if (tests.count(name) != 0) { errors << "Test " << name << " is listed more than once.\n"; continue; } bool found = false; for (RegisteredTestIter it = registered_tests_.begin(); it != registered_tests_.end(); ++it) { if (name == it->first) { found = true; break; } } if (found) { tests.insert(name); } else { errors << "No test named " << name << " can be found in this test suite.\n"; } } for (RegisteredTestIter it = registered_tests_.begin(); it != registered_tests_.end(); ++it) { if (tests.count(it->first) == 0) { errors << "You forgot to list test " << it->first << ".\n"; } } const std::string& errors_str = errors.GetString(); if (errors_str != "") { fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(), errors_str.c_str()); fflush(stderr); posix::Abort(); } return registered_tests; } #endif // GTEST_HAS_TYPED_TEST_P } // namespace internal } // namespace testing ucc-1.8.0/test/gtest/active_set/0000775000175000017500000000000015211535620016711 5ustar alastairalastairucc-1.8.0/test/gtest/active_set/test_active_set.cc0000664000175000017500000002052615211535620022412 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "common/test_ucc.h" #include typedef std::tuple op_t; using Param = std::tuple, ucc_job_env_t, int>; #define OP_T(_root, _aset_start, _aset_stride, _aset_size, _msg_size, _mt) ({ \ op_t _op(_root, _aset_start, _aset_stride, _aset_size, \ _msg_size, UCC_MEMORY_TYPE_ ## _mt); \ _op; \ }) class test_active_set : public ucc::test, public ::testing::WithParamInterface { public: std::vector ctxs; void data_init(std::vector &ops, UccTeam_h team) { ucc_rank_t tsize = team->procs.size(); ctxs.resize(ops.size()); for (int i = 0; i < ops.size(); i++) { uint64_t aset_root = std::get<0>(ops[i]); uint64_t aset_start = std::get<1>(ops[i]); int64_t aset_stride = std::get<2>(ops[i]); uint64_t aset_size = std::get<3>(ops[i]); size_t msglen = std::get<4>(ops[i]); ucc_memory_type_t mt = std::get<5>(ops[i]); // aset contains ranks of the active_set in terms of the original // team uint64_t to_add = aset_start; std::unordered_set aset{to_add}; while(aset.size() != aset_size) { to_add = (to_add + aset_stride) % tsize; // the following assertion makes sure the active set // start/stride/size combo doesnt select the same rank twice EXPECT_EQ(aset.find(to_add), aset.end()); aset.insert(to_add); } ctxs[i].resize(tsize); for (int j = 0; j < tsize; j++) { if (aset.find(j) == aset.end()) { ctxs[i][j] = NULL; continue; } ctxs[i][j] = (gtest_ucc_coll_ctx_t*) calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ucc_coll_args_t *coll = (ucc_coll_args_t*) calloc(1, sizeof(ucc_coll_args_t)); ctxs[i][j]->args = coll; coll->mask = UCC_COLL_ARGS_FIELD_ACTIVE_SET | UCC_COLL_ARGS_FIELD_TAG; coll->coll_type = UCC_COLL_TYPE_BCAST; coll->src.info.mem_type = mt; coll->src.info.count = (ucc_count_t)msglen; coll->src.info.datatype = UCC_DT_INT8; coll->root = aset_root; coll->active_set.size = aset_size; coll->active_set.start = aset_start; coll->active_set.stride = aset_stride; coll->tag = i; ctxs[i][j]->rbuf_size = msglen; UCC_CHECK(ucc_mc_alloc(&ctxs[i][j]->src_mc_header, ctxs[i][j]->rbuf_size, mt)); coll->src.info.buffer = ctxs[i][j]->src_mc_header->addr; for (int k = 0; k < ctxs[i][j]->rbuf_size; k++) { ((uint8_t *)coll->src.info.buffer)[k] = (uint8_t) 0; } if (j == aset_root) { ctxs[i][j]->init_buf = ucc_malloc(ctxs[i][j]->rbuf_size, "init buf"); EXPECT_NE(ctxs[i][j]->init_buf, nullptr); uint8_t *sbuf = (uint8_t*)ctxs[i][j]->init_buf; for (int k = 0; k < ctxs[i][j]->rbuf_size; k++) { sbuf[k] = (uint8_t) aset_root; } UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[i][j]->init_buf, ctxs[i][j]->rbuf_size, mt, UCC_MEMORY_TYPE_HOST)); } } } } void data_fini(UccTeam_h team) { for (int i = 0; i < ctxs.size(); i++) { for (int j = 0; j < ctxs[i].size(); j++ ) { gtest_ucc_coll_ctx_t *ctx = ctxs[i][j]; if (!ctx) { continue; } ucc_coll_args_t* coll = ctx->args; UCC_CHECK(ucc_mc_free(ctx->src_mc_header)); if (j == coll->root) { ucc_free(ctx->init_buf); } free(coll); free(ctx); } ctxs[i].clear(); } ctxs.clear(); } bool data_validate_one(UccCollCtxVec ctx, UccTeam_h team) { bool ret = true; int root = 0; ucc_memory_type_t mem_type; uint8_t *rst; for (int i = 0; i < ctx.size(); i++) { if (!ctx[i]) { continue; } root = ctx[i]->args->root; mem_type = ctx[i]->args->src.info.mem_type; rst = (uint8_t*) ucc_malloc(ctx[i]->rbuf_size, "dsts buf"); EXPECT_NE(rst, nullptr); UCC_CHECK(ucc_mc_memcpy(rst, ctx[i]->args->src.info.buffer, ctx[i]->rbuf_size, UCC_MEMORY_TYPE_HOST, mem_type)); for (int j = 0; j < ctx[root]->rbuf_size; j++) { if ((uint8_t) root != rst[j]) { ret = false; break; } } ucc_free(rst); } return ret; } bool data_validate(UccTeam_h team) { for (auto &c : ctxs) { if (true != data_validate_one(c, team)) { return false; } } return true; } }; UCC_TEST_P(test_active_set, single) { auto ops = std::get<0>(GetParam()); const ucc_job_env_t env = std::get<1>(GetParam()); const int n_procs = std::get<2>(GetParam()); UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); data_init(ops, team); for (auto &c : ctxs) { UccReq req(team, c); req.start(); req.wait(); } EXPECT_EQ(true, data_validate(team)); data_fini(team); } UCC_TEST_P(test_active_set, multiple) { auto ops = std::get<0>(GetParam()); const ucc_job_env_t env = std::get<1>(GetParam()); const int n_procs = std::get<2>(GetParam()); UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); std::vector reqs; data_init(ops, team); for (auto &c : ctxs) { reqs.push_back(UccReq(team, c)); } UccReq::startall(reqs); UccReq::waitall(reqs); EXPECT_EQ(true, data_validate(team)); data_fini(team); } ucc_job_env_t knomial_env = {{"UCC_TL_UCP_TUNE", "bcast:@knomial:0-inf:inf"}, {"UCC_CLS", "basic"}}; extern ucc_job_env_t dbt_env; // test_bcast.cc INSTANTIATE_TEST_CASE_P ( , test_active_set, ::testing::Combine ( ::testing::Values ( std::vector ({ // root, start, stride, size, msglen, mt OP_T(0, 0, 1, 16, 8, HOST), // subset == full set OP_T(0, 0, 2, 8, 6, HOST), // even ranks in full set OP_T(3, 3, 8, 2, 65530, HOST), // pt2pt OP_T(2, 1, 1, 2, 65531, HOST), // pt2pt OP_T(0, 0, 1, 2, 8, HOST), // pt2pt OP_T(3, 3, 4, 2, 1024, HOST), // pt2pt OP_T(7, 3, 4, 2, 1023, HOST), // pt2pt OP_T(3, 11, -8, 2, 65530, HOST), // pt2pt OP_T(11, 11, -8, 2, 65531, HOST), // pt2pt OP_T(5, 7, -2, 2, 123456, HOST), // pt2pt OP_T(7, 7, -2, 2, 123455, HOST), // pt2pt OP_T(0, 0, 1, 4, 1337, HOST), OP_T(2, 0, 2, 4, 64, HOST), OP_T(6, 0, 3, 3, 1335, HOST), OP_T(4, 7, -1, 6, 18, HOST) }) ), ::testing::Values(knomial_env, dbt_env), // env ::testing::Values(16) // n_procs ) ); ucc-1.8.0/test/gtest/coll/0000775000175000017500000000000015211535620015514 5ustar alastairalastairucc-1.8.0/test/gtest/coll/test_reduce_scatter.cc0000664000175000017500000003617515211535620022072 0ustar alastairalastair/** * Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "core/test_mc_reduce.h" #include "common/test_ucc.h" #include "utils/ucc_math.h" #include template class test_reduce_scatter : public UccCollArgs, public testing::Test { public: virtual void TestBody(){}; void data_init(int nprocs, ucc_datatype_t dt, size_t count, UccCollCtxVec &ctxs, bool persistent) { size_t rcount; ctxs.resize(nprocs); if (count < nprocs) { count = nprocs; } count = count - (count % nprocs); rcount = count / nprocs; if (TEST_INPLACE == inplace) { rcount = count; } for (int r = 0; r < nprocs; r++) { ucc_coll_args_t *coll = (ucc_coll_args_t *)calloc(1, sizeof(ucc_coll_args_t)); ctxs[r] = (gtest_ucc_coll_ctx_t *)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->mask = 0; coll->coll_type = UCC_COLL_TYPE_REDUCE_SCATTER; coll->op = T::redop; ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(dt) * count, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); for (int i = 0; i < count; i++) { typename T::type *ptr; ptr = (typename T::type *)ctxs[r]->init_buf; /* need to limit the init value so that "prod" operation would not grow too large. We have teams up to 16 procs in gtest, this would result in prod ~2**48 */ ptr[i] = (typename T::type)((i + r + 1) % 8); } UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ucc_dt_size(dt) * rcount, mem_type)); coll->dst.info.buffer = ctxs[r]->dst_mc_header->addr; coll->src.info.buffer = NULL; if (TEST_INPLACE == inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; UCC_CHECK(ucc_mc_memcpy( coll->dst.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dt) * rcount, mem_type, UCC_MEMORY_TYPE_HOST)); coll->src.info.mem_type = UCC_MEMORY_TYPE_UNKNOWN; coll->src.info.count = SIZE_MAX; coll->src.info.datatype = (ucc_datatype_t)-1; } else { UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(dt) * count, mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy( coll->src.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dt) * count, mem_type, UCC_MEMORY_TYPE_HOST)); coll->src.info.mem_type = mem_type; coll->src.info.count = (ucc_count_t)count; coll->src.info.datatype = dt; } coll->dst.info.mem_type = mem_type; coll->dst.info.count = (ucc_count_t)rcount; coll->dst.info.datatype = dt; if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void data_fini(UccCollCtxVec ctxs) { for (gtest_ucc_coll_ctx_t *ctx : ctxs) { ucc_coll_args_t *coll = ctx->args; if (coll->src.info.buffer) { /* no inplace */ UCC_CHECK(ucc_mc_free(ctx->src_mc_header)); } UCC_CHECK(ucc_mc_free(ctx->dst_mc_header)); ucc_free(ctx->init_buf); free(coll); free(ctx); } ctxs.clear(); } void reset(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; size_t count = coll->dst.info.count; ucc_datatype_t dtype = coll->dst.info.datatype; clear_buffer(coll->dst.info.buffer, count * ucc_dt_size(dtype), mem_type, 0); if (TEST_INPLACE == inplace) { UCC_CHECK(ucc_mc_memcpy(coll->dst.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dtype) * count, mem_type, UCC_MEMORY_TYPE_HOST)); } } } bool data_validate(UccCollCtxVec ctxs) { size_t total_count, rcount, offset; typename T::type *dst, *dst_p; if (TEST_INPLACE != inplace) { total_count = (ctxs[0])->args->src.info.count; rcount = (ctxs[0])->args->dst.info.count; } else { total_count = (ctxs[0])->args->dst.info.count; rcount = total_count / ctxs.size(); } ucc_assert(rcount * ctxs.size() == total_count); dst = (typename T::type *)ucc_malloc( total_count * sizeof(typename T::type), "dst buf"); dst_p = dst; for (int r = 0; r < ctxs.size(); r++) { offset = (TEST_INPLACE == inplace) ? rcount * r * sizeof(typename T::type) : 0; UCC_CHECK(ucc_mc_memcpy( dst_p, PTR_OFFSET(ctxs[r]->args->dst.info.buffer, offset), rcount * sizeof(typename T::type), UCC_MEMORY_TYPE_HOST, mem_type)); dst_p += rcount; } for (int i = 0; i < total_count; i++) { typename T::type res = ((typename T::type *)((ctxs[0])->init_buf))[i]; for (int r = 1; r < ctxs.size(); r++) { res = T::do_op(res, ((typename T::type *)((ctxs[r])->init_buf))[i]); } if (T::redop == UCC_OP_AVG) { res = res / (typename T::type)ctxs.size(); } T::assert_equal(res, dst[i]); } ucc_free(dst); return true; } }; template class test_reduce_scatter_host : public test_reduce_scatter {}; template class test_reduce_scatter_cuda : public test_reduce_scatter {}; TYPED_TEST_CASE(test_reduce_scatter_host, CollReduceTypeOpsHost); TYPED_TEST_CASE(test_reduce_scatter_cuda, CollReduceTypeOpsCuda); #define TEST_DECLARE(_mem_type, _inplace, _repeat, _persistent) \ { \ std::array counts{123}; \ CHECK_TYPE_OP_SKIP(TypeParam::dt, TypeParam::redop, _mem_type); \ for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { \ for (int count : counts) { \ UccTeam_h team = UccJob::getStaticTeams()[tid]; \ int size = team->procs.size(); \ UccCollCtxVec ctxs; \ SET_MEM_TYPE(_mem_type); \ this->set_inplace(_inplace); \ this->data_init(size, TypeParam::dt, count, ctxs, _persistent);\ UccReq req(team, ctxs); \ CHECK_REQ_NOT_SUPPORTED_SKIP(req, this->data_fini(ctxs)); \ for (auto i = 0; i < _repeat; i++) { \ req.start(); \ req.wait(); \ EXPECT_EQ(true, this->data_validate(ctxs)); \ this->reset(ctxs); \ } \ this->data_fini(ctxs); \ } \ } \ } TYPED_TEST(test_reduce_scatter_host, single) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatter_host, single_persistent) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_reduce_scatter_host, single_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatter_host, single_persistent_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE, 3, 1); } #ifdef HAVE_CUDA TYPED_TEST(test_reduce_scatter_cuda, single) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatter_cuda, single_persistent) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_reduce_scatter_cuda, single_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatter_cuda, single_persistent_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE, 3, 1); } TYPED_TEST(test_reduce_scatter_cuda, single_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatter_cuda, single_persistent_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_reduce_scatter_cuda, single_inplace_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatter_cuda, single_persistent_inplace_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE, 3, 1); } #endif #define TEST_DECLARE_MULTIPLE(_mem_type, _inplace) \ { \ std::array counts{4, 256, 65536}; \ CHECK_TYPE_OP_SKIP(TypeParam::dt, TypeParam::redop, _mem_type); \ for (int count : counts) { \ std::vector reqs; \ std::vector ctxs; \ for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { \ UccTeam_h team = UccJob::getStaticTeams()[tid]; \ int size = team->procs.size(); \ UccCollCtxVec ctx; \ this->set_inplace(_inplace); \ SET_MEM_TYPE(_mem_type); \ this->data_init(size, TypeParam::dt, count, ctx, false); \ ctxs.push_back(ctx); \ reqs.push_back(UccReq(team, ctx)); \ CHECK_REQ_NOT_SUPPORTED_SKIP(reqs.back(), \ DATA_FINI_ALL(this, ctxs)); \ } \ UccReq::startall(reqs); \ UccReq::waitall(reqs); \ for (auto ctx : ctxs) { \ EXPECT_EQ(true, this->data_validate(ctx)); \ } \ DATA_FINI_ALL(this, ctxs); \ } \ } TYPED_TEST(test_reduce_scatter_host, multiple) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE); } TYPED_TEST(test_reduce_scatter_host, multiple_inplace) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE); } #ifdef HAVE_CUDA TYPED_TEST(test_reduce_scatter_cuda, multiple) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE); } TYPED_TEST(test_reduce_scatter_cuda, multiple_inplace) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE); } TYPED_TEST(test_reduce_scatter_cuda, multiple_managed) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE); } TYPED_TEST(test_reduce_scatter_cuda, multiple_inplace_managed) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE); } #endif using Param_0 = std::tuple; class test_reduce_scatter_alg : public ucc::test, public ::testing::WithParamInterface { }; UCC_TEST_P(test_reduce_scatter_alg,) { test_reduce_scatter> rs_test; int n_procs = 15; const ucc_job_env_t env = std::get<0>(GetParam()); UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); int repeat = 3; UccCollCtxVec ctxs; std::vector mt = {UCC_MEMORY_TYPE_HOST}; if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { mt.push_back(UCC_MEMORY_TYPE_CUDA); } if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA_MANAGED)) { mt.push_back(UCC_MEMORY_TYPE_CUDA_MANAGED); } for (auto count : {65536, 123567}) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { for (auto m : mt) { rs_test.set_mem_type(m); rs_test.set_inplace(inplace); rs_test.data_init(n_procs, UCC_DT_INT32, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < repeat; i++) { req.start(); req.wait(); EXPECT_EQ(true, rs_test.data_validate(ctxs)); rs_test.reset(ctxs); } rs_test.data_fini(ctxs); } } } } ucc_job_env_t ring_unidir_env = {{"name", "ring_unidirectional"}, {"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", "reduce_scatter:@ring:inf"}, {"UCC_TL_UCP_REDUCE_SCATTER_RING_BIDIRECTIONAL", "n"}}; ucc_job_env_t ring_bidir_env = {{"name", "ring_bidirectional"}, {"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", "reduce_scatter:@ring:inf"}, {"UCC_TL_UCP_REDUCE_SCATTER_RING_BIDIRECTIONAL", "y"}}; ucc_job_env_t knomial = {{"name", "knomial"}, {"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", "reduce_scatter:@knomial:inf"}}; INSTANTIATE_TEST_CASE_P( , test_reduce_scatter_alg, ::testing::Combine( ::testing::Values(ring_unidir_env, ring_bidir_env, knomial)), [](const testing::TestParamInfo& info) { const ucc_job_env_t env = std::get<0>(info.param); return env[0].second;}); ucc-1.8.0/test/gtest/coll/test_gatherv.cc0000664000175000017500000002621715211535620020532 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "common/test_ucc.h" #include "utils/ucc_math.h" using Param_0 = std::tuple; using Param_1 = std::tuple; class test_gatherv : public UccCollArgs, public ucc::test { private: int root; public: void data_init(int nprocs, ucc_datatype_t dtype, size_t count, UccCollCtxVec &ctxs, bool persistent) { ucc_coll_args_t *coll; int *counts, *displs; size_t my_count, all_counts; ctxs.resize(nprocs); for (auto r = 0; r < nprocs; r++) { coll = (ucc_coll_args_t *)calloc(1, sizeof(ucc_coll_args_t)); my_count = (nprocs - r) * count; ctxs[r] = (gtest_ucc_coll_ctx_t *)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->mask = 0; coll->flags = 0; coll->coll_type = UCC_COLL_TYPE_GATHERV; coll->root = root; coll->src.info.mem_type = mem_type; coll->src.info.count = (ucc_count_t)my_count; coll->src.info.datatype = dtype; ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(dtype) * my_count, "init buf"); ASSERT_NE(ctxs[r]->init_buf, nullptr); for (int i = 0; i < my_count * ucc_dt_size(dtype); i++) { uint8_t *sbuf = (uint8_t *)ctxs[r]->init_buf; sbuf[i] = ((i + r) % 256); } if (r == root) { all_counts = 0; counts = (int*)malloc(sizeof(int) * nprocs); ASSERT_NE(counts, nullptr); displs = (int*)malloc(sizeof(int) * nprocs); ASSERT_NE(displs, nullptr); for (int i = 0; i < nprocs; i++) { counts[i] = (nprocs - i) * count; displs[i] = all_counts; all_counts += counts[i]; } coll->dst.info_v.mem_type = mem_type; coll->dst.info_v.counts = (ucc_count_t *)counts; coll->dst.info_v.displacements = (ucc_aint_t *)displs; coll->dst.info_v.datatype = dtype; ctxs[r]->rbuf_size = ucc_dt_size(dtype) * all_counts; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ctxs[r]->rbuf_size, mem_type)); coll->dst.info_v.buffer = ctxs[r]->dst_mc_header->addr; if (inplace) { UCC_CHECK(ucc_mc_memcpy( (void *)((ptrdiff_t)coll->dst.info_v.buffer + displs[r] * ucc_dt_size(dtype)), ctxs[r]->init_buf, ucc_dt_size(dtype) * my_count, mem_type, UCC_MEMORY_TYPE_HOST)); } } if (r != root || !inplace) { UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(dtype) * my_count, mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dtype) * my_count, mem_type, UCC_MEMORY_TYPE_HOST)); } if (inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; } if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void data_fini(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; if (r == root) { UCC_CHECK(ucc_mc_free(ctxs[r]->dst_mc_header)); free(coll->dst.info_v.counts); free(coll->dst.info_v.displacements); } if (r != root || !inplace) { UCC_CHECK(ucc_mc_free(ctxs[r]->src_mc_header)); } ucc_free(ctxs[r]->init_buf); free(coll); free(ctxs[r]); } ctxs.clear(); } void reset(UccCollCtxVec ctxs) { ucc_coll_args_t *coll = ctxs[root]->args; size_t my_count = coll->src.info.count; ucc_datatype_t dtype = coll->dst.info_v.datatype; int * displs = (int *)coll->dst.info_v.displacements; clear_buffer(coll->dst.info_v.buffer, ctxs[root]->rbuf_size, mem_type, 0); if (TEST_INPLACE == inplace) { UCC_CHECK(ucc_mc_memcpy( (void *)((ptrdiff_t)coll->dst.info_v.buffer + displs[root] * ucc_dt_size(dtype)), ctxs[root]->init_buf, ucc_dt_size(dtype) * my_count, mem_type, UCC_MEMORY_TYPE_HOST)); } } bool data_validate(UccCollCtxVec ctxs) { bool ret = true; int root = ctxs[0]->args->root; int *displs = (int*)ctxs[root]->args->dst.info_v.displacements; size_t dt_size = ucc_dt_size(ctxs[root]->args->src.info.datatype); ucc_count_t my_count; uint8_t *dsts; if (UCC_MEMORY_TYPE_HOST != mem_type) { dsts = (uint8_t *)ucc_malloc(ctxs[root]->rbuf_size, "dsts buf"); ucc_assert(dsts != nullptr); UCC_CHECK(ucc_mc_memcpy(dsts, ctxs[root]->args->dst.info_v.buffer, ctxs[root]->rbuf_size, UCC_MEMORY_TYPE_HOST, mem_type)); } else { dsts = (uint8_t *)ctxs[root]->args->dst.info_v.buffer; } for (int r = 0; r < ctxs.size(); r++) { my_count = ctxs[r]->args->src.info.count; for (int i = 0; i < my_count * dt_size; i++) { if ((uint8_t)((i + r) % 256) != dsts[(displs[r] * dt_size + i)]) { ret = false; break; } } } if (UCC_MEMORY_TYPE_HOST != mem_type) { ucc_free(dsts); } return ret; } void set_root(int _root) { root = _root; } }; class test_gatherv_0 : public test_gatherv, public ::testing::WithParamInterface { }; UCC_TEST_P(test_gatherv_0, single) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const int root = std::get<4>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<5>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); UccCollCtxVec ctxs; if (size <= root) { GTEST_SKIP(); } set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } UCC_TEST_P(test_gatherv_0, single_persistent) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const int root = std::get<4>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<5>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); const int n_calls = 3; UccCollCtxVec ctxs; if (size <= root) { GTEST_SKIP(); } set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < n_calls; i++) { req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); reset(ctxs); } data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( , test_gatherv_0, ::testing::Combine(::testing::Range(1, UccJob::nStaticTeams), // team_ids PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1, 3, 8192), // count ::testing::Values(0, 1), // root ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE))); class test_gatherv_1 : public test_gatherv, public ::testing::WithParamInterface { }; UCC_TEST_P(test_gatherv_1, multiple_host) { const ucc_datatype_t dtype = std::get<0>(GetParam()); const ucc_memory_type_t mem_type = std::get<1>(GetParam()); const int count = std::get<2>(GetParam()); const int root = std::get<3>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<4>(GetParam()); std::vector reqs; std::vector ctxs; for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { UccTeam_h team = UccJob::getStaticTeams()[tid]; int size = team->procs.size(); UccCollCtxVec ctx; if (size <= root) { /* skip invalid */ continue; } this->set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctx, false); reqs.push_back(UccReq(team, ctx)); ctxs.push_back(ctx); } UccReq::startall(reqs); UccReq::waitall(reqs); for (auto ctx : ctxs) { EXPECT_EQ(true, data_validate(ctx)); data_fini(ctx); } } INSTANTIATE_TEST_CASE_P( , test_gatherv_1, ::testing::Combine(PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1, 3, 8192), // count ::testing::Values(0, 1), // root ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE))); ucc-1.8.0/test/gtest/coll/test_reduce_scatterv.cc0000664000175000017500000004024515211535620022251 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "core/test_mc_reduce.h" #include "common/test_ucc.h" #include "utils/ucc_math.h" #include "utils/ucc_coll_utils.h" #include #include #include template class test_reduce_scatterv : public UccCollArgs, public testing::Test { public: virtual void TestBody(){}; /* generates uniform random array of counts that sum up to total_count */ std::vector generate_counts(int nprocs, size_t total) { std::default_random_engine eng; std::vector counts, tmp; eng.seed(123); std::uniform_int_distribution urd(1, total - 1); for (int i = 0; i < nprocs - 1; i++) { tmp.push_back(urd(eng)); } tmp.push_back(0); tmp.push_back(total); std::sort(tmp.begin(), tmp.end()); for (int i = 1; i < tmp.size(); i++) { counts.push_back(tmp[i] - tmp[i - 1]); total -= counts.back(); } ucc_assert(total == 0); return counts; } void data_init(int nprocs, ucc_datatype_t dt, size_t total_count, UccCollCtxVec &ctxs, bool persistent) { size_t rcount = total_count; ctxs.resize(nprocs); auto counts = generate_counts(nprocs, total_count); for (int r = 0; r < nprocs; r++) { if (TEST_INPLACE != inplace) { rcount = counts[r]; } ucc_coll_args_t *coll = (ucc_coll_args_t *)calloc(1, sizeof(ucc_coll_args_t)); ctxs[r] = (gtest_ucc_coll_ctx_t *)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->mask = 0; coll->coll_type = UCC_COLL_TYPE_REDUCE_SCATTERV; coll->op = T::redop; ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(dt) * total_count, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); for (int i = 0; i < total_count; i++) { typename T::type *ptr; ptr = (typename T::type *)ctxs[r]->init_buf; /* need to limit the init value so that "prod" operation would not grow too large. We have teams up to 16 procs in gtest, this would result in prod ~2**48 */ ptr[i] = (typename T::type)((i + r + 1) % 8); } UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ucc_dt_size(dt) * rcount, mem_type)); coll->dst.info_v.buffer = ctxs[r]->dst_mc_header->addr; coll->src.info.buffer = NULL; if (TEST_INPLACE == inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; UCC_CHECK(ucc_mc_memcpy( coll->dst.info_v.buffer, ctxs[r]->init_buf, ucc_dt_size(dt) * rcount, mem_type, UCC_MEMORY_TYPE_HOST)); coll->src.info.mem_type = UCC_MEMORY_TYPE_UNKNOWN; coll->src.info.count = SIZE_MAX; coll->src.info.datatype = (ucc_datatype_t)-1; } else { UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(dt) * total_count, mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dt) * total_count, mem_type, UCC_MEMORY_TYPE_HOST)); coll->src.info.mem_type = mem_type; coll->src.info.count = (ucc_count_t)total_count; coll->src.info.datatype = dt; } coll->dst.info_v.mem_type = mem_type; coll->dst.info_v.counts = (ucc_count_t *)ucc_malloc(nprocs * sizeof(uint32_t), "counts"); memcpy(coll->dst.info_v.counts, counts.data(), sizeof(uint32_t) * nprocs); coll->dst.info_v.datatype = dt; if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void data_fini(UccCollCtxVec ctxs) { for (gtest_ucc_coll_ctx_t *ctx : ctxs) { ucc_coll_args_t *coll = ctx->args; if (coll->src.info.buffer) { /* no inplace */ UCC_CHECK(ucc_mc_free(ctx->src_mc_header)); } UCC_CHECK(ucc_mc_free(ctx->dst_mc_header)); ucc_free(ctx->init_buf); ucc_free(coll->dst.info_v.counts); free(coll); free(ctx); } ctxs.clear(); } size_t get_total_count(UccCollCtxVec ctxs) { ucc_coll_args_t *coll = ctxs[0]->args; return ucc_coll_args_get_total_count(coll, coll->dst.info_v.counts, ctxs.size()); } void reset(UccCollCtxVec ctxs) { size_t total_count = get_total_count(ctxs); uint32_t *counts = (uint32_t *)ctxs[0]->args->dst.info_v.counts; for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; ucc_datatype_t dtype = coll->dst.info_v.datatype; size_t rcount = (TEST_INPLACE == inplace) ? total_count : counts[r]; clear_buffer(coll->dst.info_v.buffer, rcount * ucc_dt_size(dtype), mem_type, 0); if (TEST_INPLACE == inplace) { UCC_CHECK(ucc_mc_memcpy(coll->dst.info_v.buffer, ctxs[r]->init_buf, ucc_dt_size(dtype) * total_count, mem_type, UCC_MEMORY_TYPE_HOST)); } } } size_t check_offset(UccCollCtxVec ctxs, int rank) { uint32_t *counts = (uint32_t *)ctxs[0]->args->dst.info_v.counts; size_t offset = 0; if (TEST_INPLACE == inplace) { for (int i = 0; i < rank; i++) { offset += counts[i]; } } return offset * sizeof(typename T::type); } bool data_validate(UccCollCtxVec ctxs) { size_t total_count = get_total_count(ctxs); uint32_t * counts = (uint32_t *)ctxs[0]->args->dst.info_v.counts; size_t rcount; typename T::type *dst, *dst_p; dst = (typename T::type *)ucc_malloc( total_count * sizeof(typename T::type), "dst buf"); dst_p = dst; for (int r = 0; r < ctxs.size(); r++) { rcount = counts[r]; UCC_CHECK(ucc_mc_memcpy(dst_p, PTR_OFFSET(ctxs[r]->args->dst.info_v.buffer, check_offset(ctxs, r)), rcount * sizeof(typename T::type), UCC_MEMORY_TYPE_HOST, mem_type)); dst_p += rcount; } for (int i = 0; i < total_count; i++) { typename T::type res = ((typename T::type *)((ctxs[0])->init_buf))[i]; for (int r = 1; r < ctxs.size(); r++) { res = T::do_op(res, ((typename T::type *)((ctxs[r])->init_buf))[i]); } if (T::redop == UCC_OP_AVG) { res = res / (typename T::type)ctxs.size(); } T::assert_equal(res, dst[i]); } ucc_free(dst); return true; } }; template class test_reduce_scatterv_host : public test_reduce_scatterv {}; template class test_reduce_scatterv_cuda : public test_reduce_scatterv {}; TYPED_TEST_CASE(test_reduce_scatterv_host, CollReduceTypeOpsHost); TYPED_TEST_CASE(test_reduce_scatterv_cuda, CollReduceTypeOpsCuda); #define TEST_DECLARE(_mem_type, _inplace, _repeat, _persistent) \ { \ std::array counts{4, 123, 65536}; \ CHECK_TYPE_OP_SKIP(TypeParam::dt, TypeParam::redop, _mem_type); \ for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { \ for (int count : counts) { \ UccTeam_h team = UccJob::getStaticTeams()[tid]; \ int size = team->procs.size(); \ UccCollCtxVec ctxs; \ SET_MEM_TYPE(_mem_type); \ this->set_inplace(_inplace); \ this->data_init(size, TypeParam::dt, count, ctxs, \ _persistent); \ UccReq req(team, ctxs); \ CHECK_REQ_NOT_SUPPORTED_SKIP(req, this->data_fini(ctxs)); \ for (auto i = 0; i < _repeat; i++) { \ req.start(); \ req.wait(); \ EXPECT_EQ(true, this->data_validate(ctxs)); \ this->reset(ctxs); \ } \ this->data_fini(ctxs); \ } \ } \ } TYPED_TEST(test_reduce_scatterv_host, single) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatterv_host, single_persistent) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_reduce_scatterv_host, single_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatterv_host, single_persistent_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE, 3, 1); } #ifdef HAVE_CUDA TYPED_TEST(test_reduce_scatterv_cuda, single) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatterv_cuda, single_persistent) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_reduce_scatterv_cuda, single_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatterv_cuda, single_persistent_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE, 3, 1); } TYPED_TEST(test_reduce_scatterv_cuda, single_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatterv_cuda, single_persistent_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_reduce_scatterv_cuda, single_inplace_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE, 1, 0); } TYPED_TEST(test_reduce_scatterv_cuda, single_persistent_inplace_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE, 3, 1); } #endif #define TEST_DECLARE_MULTIPLE(_mem_type, _inplace) \ { \ std::array counts{4, 123, 65536}; \ CHECK_TYPE_OP_SKIP(TypeParam::dt, TypeParam::redop, _mem_type); \ for (int count : counts) { \ std::vector reqs; \ std::vector ctxs; \ for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { \ UccTeam_h team = UccJob::getStaticTeams()[tid]; \ int size = team->procs.size(); \ UccCollCtxVec ctx; \ this->set_inplace(_inplace); \ SET_MEM_TYPE(_mem_type); \ this->data_init(size, TypeParam::dt, count, ctx, false); \ ctxs.push_back(ctx); \ reqs.push_back(UccReq(team, ctx)); \ CHECK_REQ_NOT_SUPPORTED_SKIP(reqs.back(), \ DATA_FINI_ALL(this, ctxs)); \ } \ UccReq::startall(reqs); \ UccReq::waitall(reqs); \ for (auto ctx : ctxs) { \ EXPECT_EQ(true, this->data_validate(ctx)); \ } \ DATA_FINI_ALL(this, ctxs); \ } \ } TYPED_TEST(test_reduce_scatterv_host, multiple) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE); } TYPED_TEST(test_reduce_scatterv_host, multiple_inplace) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE); } #ifdef HAVE_CUDA TYPED_TEST(test_reduce_scatterv_cuda, multiple) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE); } TYPED_TEST(test_reduce_scatterv_cuda, multiple_inplace) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE); } TYPED_TEST(test_reduce_scatterv_cuda, multiple_managed) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE); } TYPED_TEST(test_reduce_scatterv_cuda, multiple_inplace_managed) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE); } #endif class test_reduce_scatterv_alg : public ucc::test, public ::testing::WithParamInterface { }; UCC_TEST_P(test_reduce_scatterv_alg, ring) { test_reduce_scatterv> rsv_test; int n_procs = 15; std::string bidir = GetParam(); ucc_job_env_t env = {{"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", "reduce_scatterv:@ring:inf"}, {"REDUCE_SCATTERV_RING_BIDIRECTIONAL", bidir == "bidirectional" ? "y" : "n"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); int repeat = 3; UccCollCtxVec ctxs; std::vector mt = {UCC_MEMORY_TYPE_HOST}; if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { mt.push_back(UCC_MEMORY_TYPE_CUDA); } if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA_MANAGED)) { mt.push_back(UCC_MEMORY_TYPE_CUDA_MANAGED); } for (auto count : {65536, 123567}) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { for (auto m : mt) { rsv_test.set_mem_type(m); rsv_test.set_inplace(inplace); rsv_test.data_init(n_procs, UCC_DT_INT32, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < repeat; i++) { req.start(); req.wait(); EXPECT_EQ(true, rsv_test.data_validate(ctxs)); rsv_test.reset(ctxs); } rsv_test.data_fini(ctxs); } } } } INSTANTIATE_TEST_CASE_P(, test_reduce_scatterv_alg, ::testing::Values("bidirectional", "unidirectional")); ucc-1.8.0/test/gtest/coll/test_bcast.cc0000664000175000017500000002743415211535620020170 0ustar alastairalastair/** * Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "common/test_ucc.h" #include "utils/ucc_math.h" using Param_0 = std::tuple; using Param_1 = std::tuple; using Param_2 = std::tuple; class test_bcast : public UccCollArgs, public ucc::test { private: int root; public: void data_init(int nprocs, ucc_datatype_t dtype, size_t count, UccCollCtxVec &ctxs, bool persistent) { ctxs.resize(nprocs); for (auto r = 0; r < nprocs; r++) { ucc_coll_args_t *coll = (ucc_coll_args_t*) calloc(1, sizeof(ucc_coll_args_t)); ctxs[r] = (gtest_ucc_coll_ctx_t*)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->mask = 0; coll->coll_type = UCC_COLL_TYPE_BCAST; coll->src.info.mem_type = mem_type; coll->src.info.count = (ucc_count_t)count; coll->src.info.datatype = dtype; coll->root = root; ctxs[r]->rbuf_size = ucc_dt_size(dtype) * count; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ctxs[r]->rbuf_size, mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; if (r == root) { ctxs[r]->init_buf = ucc_malloc(ctxs[r]->rbuf_size, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); uint8_t *sbuf = (uint8_t*)ctxs[r]->init_buf; for (int i = 0; i < ctxs[r]->rbuf_size; i++) { sbuf[i] = (uint8_t)i; } UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ctxs[r]->rbuf_size, mem_type, UCC_MEMORY_TYPE_HOST)); } if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void reset(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; size_t count = coll->src.info.count; ucc_datatype_t dtype = coll->src.info.datatype; if (r != root) { clear_buffer(coll->src.info.buffer, count * ucc_dt_size(dtype), mem_type, 0); } else { UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ctxs[r]->rbuf_size, mem_type, UCC_MEMORY_TYPE_HOST)); } } } void data_fini(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { gtest_ucc_coll_ctx_t *ctx = ctxs[r]; ucc_coll_args_t* coll = ctx->args; UCC_CHECK(ucc_mc_free(ctx->src_mc_header)); if (r == coll->root) { ucc_free(ctx->init_buf); } free(coll); free(ctx); } ctxs.clear(); } bool data_validate(UccCollCtxVec ctxs) { bool ret = true; int root = ctxs[0]->args->root; uint8_t *dsts; if (UCC_MEMORY_TYPE_HOST != mem_type) { dsts = (uint8_t*) ucc_malloc(ctxs[root]->rbuf_size, "dsts buf"); EXPECT_NE(dsts, nullptr); UCC_CHECK(ucc_mc_memcpy(dsts, ctxs[root]->args->src.info.buffer, ctxs[root]->rbuf_size, UCC_MEMORY_TYPE_HOST, mem_type)); } else { dsts = (uint8_t*)ctxs[root]->args->src.info.buffer; } for (int r = 0; r < ctxs.size(); r++) { ucc_coll_args_t* coll = ctxs[r]->args; if (coll->root == r) { continue; } for (int i = 0; i < ctxs[r]->rbuf_size; i++) { if ((uint8_t)i != dsts[i]) { ret = false; break; } } } if (UCC_MEMORY_TYPE_HOST != mem_type) { ucc_free(dsts); } return ret; } void set_root(int _root) { root = _root; } }; class test_bcast_0 : public test_bcast, public ::testing::WithParamInterface {}; UCC_TEST_P(test_bcast_0, single) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const int root = std::get<4>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); UccCollCtxVec ctxs; SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } UCC_TEST_P(test_bcast_0, single_persistent) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const int root = std::get<4>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); const int n_calls = 3; UccCollCtxVec ctxs; SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < n_calls; i++) { req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); reset(ctxs); } data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( , test_bcast_0, ::testing::Combine( ::testing::Range(1, UccJob::nStaticTeams), // team_ids PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1,3,65536), // count ::testing::Values(0,1))); // root class test_bcast_1 : public test_bcast, public ::testing::WithParamInterface {}; UCC_TEST_P(test_bcast_1, multiple) { const ucc_datatype_t dtype = std::get<0>(GetParam()); const ucc_memory_type_t mem_type = std::get<1>(GetParam()); const int count = std::get<2>(GetParam()); const int root = std::get<3>(GetParam()); std::vector reqs; std::vector ctxs; for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { UccTeam_h team = UccJob::getStaticTeams()[tid]; int size = team->procs.size(); UccCollCtxVec ctx; if (size == 1 && root > 0) { /* skip team size 1 and root > 0, which are invalid */ continue; } SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctx, false); reqs.push_back(UccReq(team, ctx)); ctxs.push_back(ctx); } UccReq::startall(reqs); UccReq::waitall(reqs); for (auto ctx : ctxs) { EXPECT_EQ(true, data_validate(ctx)); data_fini(ctx); } } INSTANTIATE_TEST_CASE_P( , test_bcast_1, ::testing::Combine( PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1,3,65536), // count ::testing::Values(0,1))); // root class test_bcast_alg : public test_bcast, public ::testing::WithParamInterface {}; UCC_TEST_P(test_bcast_alg,) { const ucc_memory_type_t mt = std::get<0>(GetParam()); const ucc_job_env_t env = std::get<1>(GetParam()); const int count = std::get<2>(GetParam()); const int n_procs = std::get<3>(GetParam()); UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); int repeat = 1; UccCollCtxVec ctxs; SET_MEM_TYPE(mt); for (int root = 0; root < n_procs; root++) { this->set_root(root); this->data_init(n_procs, UCC_DT_INT8, count, ctxs, false); UccReq req(team, ctxs); for (auto i = 0; i < repeat; i++) { req.start(); req.wait(); EXPECT_EQ(true, this->data_validate(ctxs)); this->reset(ctxs); } this->data_fini(ctxs); } } ucc_job_env_t two_step_env = {{"UCC_CL_HIER_TUNE", "bcast:@2step:0-inf:inf"}, {"UCC_CLS", "all"}}; ucc_job_env_t dbt_env = {{"UCC_TL_UCP_TUNE", "bcast:@dbt:0-inf:inf"}, {"UCC_CLS", "basic"}}; ucc_job_env_t cuda_env = {{"UCC_TL_CUDA_TUNE", "bcast:cuda:@0:0-inf:inf"}, {"UCC_CLS", "basic"}}; ucc_job_env_t host_mcast_env = {{"UCC_TLS", "ucp,mlx5"}, {"UCC_TL_MLX5_TUNE", "inf"}, {"UCC_TL_MLX5_MCAST_ENABLE", "1"}, {"UCC_TL_MLX5_MCAST_NET_DEVICE", "mlx5_0"}, {"UCC_TL_MLX5_MCAST_CUDA_MEM_ENABLE", "0"}, {"UCC_TL_MLX5_MIN_TEAM_SIZE", "2"}}; ucc_job_env_t host_mcast_rel_env = {{"UCC_TLS", "ucp,mlx5"}, {"UCC_TL_MLX5_TUNE", "inf"}, {"UCC_TL_MLX5_MCAST_ENABLE", "1"}, {"UCC_TL_MLX5_MCAST_NET_DEVICE", "mlx5_0"}, {"UCC_TL_MLX5_MCAST_CUDA_MEM_ENABLE", "0"}, {"UCC_TL_MLX5_MCAST_TIMEOUT", "0"}, {"UCC_TL_MLX5_MIN_TEAM_SIZE", "2"}}; ucc_job_env_t cuda_mcast_env = {{"UCC_TLS", "ucp,mlx5"}, {"UCC_TL_MLX5_TUNE", "inf"}, {"UCC_TL_MLX5_MCAST_ENABLE", "1"}, {"UCC_TL_MLX5_MCAST_NET_DEVICE", "mlx5_0"}, {"UCC_TL_MLX5_MCAST_CUDA_MEM_ENABLE", "1"}, {"UCC_TL_MLX5_MIN_TEAM_SIZE", "2"}}; ucc_job_env_t cuda_mcast_rel_env = {{"UCC_TLS", "ucp,mlx5"}, {"UCC_TL_MLX5_TUNE", "inf"}, {"UCC_TL_MLX5_MCAST_ENABLE", "1"}, {"UCC_TL_MLX5_MCAST_NET_DEVICE", "mlx5_0"}, {"UCC_TL_MLX5_MCAST_CUDA_MEM_ENABLE", "1"}, {"UCC_TL_MLX5_MCAST_TIMEOUT", "0"}, {"UCC_TL_MLX5_MIN_TEAM_SIZE", "2"}}; INSTANTIATE_TEST_CASE_P( , test_bcast_alg, ::testing::Combine( #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), ::testing::Values(two_step_env, dbt_env, cuda_env, host_mcast_env, host_mcast_rel_env, cuda_mcast_env, cuda_mcast_rel_env), //env #else ::testing::Values(UCC_MEMORY_TYPE_HOST), ::testing::Values(two_step_env, dbt_env, host_mcast_env, host_mcast_rel_env), //env #endif ::testing::Values(8, 65536), // count ::testing::Values(15, 16))); // n_procs ucc-1.8.0/test/gtest/coll/test_reduce.cc0000664000175000017500000004017115211535620020334 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "core/test_mc_reduce.h" #include "common/test_ucc.h" #include "utils/ucc_math.h" #include template class test_reduce : public UccCollArgs, public testing::Test { private: int root = 0; public: void data_init(int nprocs, ucc_datatype_t dt, size_t count, UccCollCtxVec &ctxs, bool persistent) { ctxs.resize(nprocs); for (int r = 0; r < nprocs; r++) { ucc_coll_args_t *coll = (ucc_coll_args_t*) calloc(1, sizeof(ucc_coll_args_t)); ctxs[r] = (gtest_ucc_coll_ctx_t*)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(dt) * count, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); for (int i = 0; i < count; i++) { typename T::type * ptr; ptr = (typename T::type *)ctxs[r]->init_buf; /* need to limit the init value so that "prod" operation would not grow too large. We have teams up to 16 procs in gtest, this would result in prod ~2**48 */ /* bFloat16 will be assigned with the floats matching the uint16_t bit pattern*/ ptr[i] = (typename T::type)((i + r + 1) % 8); } coll->coll_type = UCC_COLL_TYPE_REDUCE; coll->op = T::redop; coll->root = root; if (r != root || !inplace) { coll->src.info.mem_type = mem_type; coll->src.info.count = (ucc_count_t)count; coll->src.info.datatype = dt; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(dt) * count, mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dt) * count, mem_type, UCC_MEMORY_TYPE_HOST)); } if (r == root) { coll->dst.info.mem_type = mem_type; coll->dst.info.count = (ucc_count_t)count; coll->dst.info.datatype = dt; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ucc_dt_size(dt) * count, mem_type)); coll->dst.info.buffer = ctxs[r]->dst_mc_header->addr; if (inplace) { UCC_CHECK(ucc_mc_memcpy(coll->dst.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dt) * count, mem_type, UCC_MEMORY_TYPE_HOST)); } } if (inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; } if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void data_fini(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t* coll = ctxs[r]->args; if (r == root) { UCC_CHECK(ucc_mc_free(ctxs[r]->dst_mc_header)); } if (r != root || !inplace) { UCC_CHECK(ucc_mc_free(ctxs[r]->src_mc_header)); } ucc_free(ctxs[r]->init_buf); free(coll); free(ctxs[r]); } ctxs.clear(); } void reset(UccCollCtxVec ctxs) { ucc_coll_args_t *coll = ctxs[root]->args; size_t count = coll->dst.info.count; ucc_datatype_t dtype = coll->dst.info.datatype; clear_buffer(coll->dst.info.buffer, count * ucc_dt_size(dtype), mem_type, 0); if (TEST_INPLACE == inplace) { UCC_CHECK(ucc_mc_memcpy(coll->dst.info.buffer, ctxs[root]->init_buf, ucc_dt_size(dtype) * count, mem_type, UCC_MEMORY_TYPE_HOST)); } } bool data_validate(UccCollCtxVec ctxs) { size_t count = (ctxs[0])->args->src.info.count; typename T::type * dsts; if (UCC_MEMORY_TYPE_HOST != mem_type) { dsts = (typename T::type *) ucc_malloc(count * sizeof(typename T::type), "dsts buf"); EXPECT_NE(dsts, nullptr); UCC_CHECK(ucc_mc_memcpy(dsts, ctxs[root]->args->dst.info.buffer, count * sizeof(typename T::type), UCC_MEMORY_TYPE_HOST, mem_type)); } else { dsts = (typename T::type *)ctxs[root]->args->dst.info.buffer; } for (int i = 0; i < count; i++) { typename T::type res = ((typename T::type *)((ctxs[0])->init_buf))[i]; for (int r = 1; r < ctxs.size(); r++) { res = T::do_op(res, ((typename T::type *)((ctxs[r])->init_buf))[i]); } if (T::redop == UCC_OP_AVG) { if (T::dt == UCC_DT_BFLOAT16){ float32tobfloat16(bfloat16tofloat32(&res) / (float)ctxs.size(), &res); } else { res = res / (typename T::type)ctxs.size(); } } T::assert_equal(res, dsts[i]); } if (UCC_MEMORY_TYPE_HOST != mem_type) { ucc_free(dsts); } return true; } }; template class test_reduce_host : public test_reduce {}; template class test_reduce_cuda : public test_reduce {}; TYPED_TEST_CASE(test_reduce_host, CollReduceTypeOpsHost); TYPED_TEST_CASE(test_reduce_cuda, CollReduceTypeOpsCuda); #define TEST_DECLARE(_mem_type, _inplace, _repeat, _persistent) \ { \ std::array counts{4, 256, 65536}; \ CHECK_TYPE_OP_SKIP(TypeParam::dt, TypeParam::redop, _mem_type); \ for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { \ for (int count : counts) { \ UccTeam_h team = UccJob::getStaticTeams()[tid]; \ int size = team->procs.size(); \ UccCollCtxVec ctxs; \ SET_MEM_TYPE(_mem_type); \ this->set_inplace(_inplace); \ this->data_init(size, TypeParam::dt, count, ctxs, _persistent);\ UccReq req(team, ctxs); \ CHECK_REQ_NOT_SUPPORTED_SKIP(req, this->data_fini(ctxs)); \ for (auto i = 0; i < _repeat; i++) { \ req.start(); \ req.wait(); \ EXPECT_EQ(true, this->data_validate(ctxs)); \ this->reset(ctxs); \ } \ this->data_fini(ctxs); \ } \ } \ } TYPED_TEST(test_reduce_host, single) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_reduce_host, single_persistent) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_reduce_host, single_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE, 1, 0); } TYPED_TEST(test_reduce_host, single_persistent_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE, 3, 1); } #ifdef HAVE_CUDA TYPED_TEST(test_reduce_cuda, single) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_reduce_cuda, single_persistent) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_reduce_cuda, single_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE, 1, 0); } TYPED_TEST(test_reduce_cuda, single_persistent_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE, 3, 1); } TYPED_TEST(test_reduce_cuda, single_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_reduce_cuda, single_persistent_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_reduce_cuda, single_inplace_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE, 1, 0); } TYPED_TEST(test_reduce_cuda, single_persistent_inplace_managed) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE, 3, 1); } #endif #define TEST_DECLARE_MULTIPLE(_mem_type, _inplace) \ { \ std::array counts{4, 256, 65536}; \ CHECK_TYPE_OP_SKIP(TypeParam::dt, TypeParam::redop, _mem_type); \ for (int count : counts) { \ std::vector reqs; \ std::vector ctxs; \ for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { \ UccTeam_h team = UccJob::getStaticTeams()[tid]; \ int size = team->procs.size(); \ UccCollCtxVec ctx; \ this->set_inplace(_inplace); \ SET_MEM_TYPE(_mem_type); \ this->data_init(size, TypeParam::dt, count, ctx, false); \ reqs.push_back(UccReq(team, ctx)); \ CHECK_REQ_NOT_SUPPORTED_SKIP(reqs.back(), \ DATA_FINI_ALL(this, ctxs)); \ ctxs.push_back(ctx); \ } \ UccReq::startall(reqs); \ UccReq::waitall(reqs); \ for (auto ctx : ctxs) { \ EXPECT_EQ(true, this->data_validate(ctx)); \ this->data_fini(ctx); \ } \ } \ } TYPED_TEST(test_reduce_host, multiple) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE); } TYPED_TEST(test_reduce_host, multiple_inplace) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE); } #ifdef HAVE_CUDA TYPED_TEST(test_reduce_cuda, multiple) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE); } TYPED_TEST(test_reduce_cuda, multiple_inplace) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE); } TYPED_TEST(test_reduce_cuda, multiple_managed) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE); } TYPED_TEST(test_reduce_cuda, multiple_inplace_managed) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE); } #endif template class test_reduce_avg_order : public test_reduce { }; template class test_reduce_dbt : public test_reduce { }; template class test_reduce_2step : public test_reduce { }; template class test_reduce_srg : public test_reduce { }; #define TEST_DECLARE_WITH_ENV(_env, _n_procs, _persistent) \ { \ UccJob job(_n_procs, UccJob::UCC_JOB_CTX_GLOBAL, _env); \ UccTeam_h team = job.create_team(_n_procs); \ int repeat = _persistent ? 3 : 1; \ UccCollCtxVec ctxs; \ std::vector mt = {UCC_MEMORY_TYPE_HOST}; \ if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { \ mt.push_back(UCC_MEMORY_TYPE_CUDA); \ } \ if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA_MANAGED)) { \ mt.push_back(UCC_MEMORY_TYPE_CUDA_MANAGED); \ } \ for (auto count : {5, 256, 65536}) { \ for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { \ for (auto m : mt) { \ CHECK_TYPE_OP_SKIP(TypeParam::dt, TypeParam::redop, m); \ SET_MEM_TYPE(m); \ this->set_inplace(inplace); \ this->data_init(_n_procs, TypeParam::dt, count, ctxs, \ _persistent); \ UccReq req(team, ctxs); \ CHECK_REQ_NOT_SUPPORTED_SKIP(req, this->data_fini(ctxs)); \ for (auto i = 0; i < repeat; i++) { \ req.start(); \ req.wait(); \ EXPECT_EQ(true, this->data_validate(ctxs)); \ this->reset(ctxs); \ } \ this->data_fini(ctxs); \ } \ } \ } \ } TYPED_TEST_CASE(test_reduce_avg_order, CollReduceTypeOpsAvg); TYPED_TEST_CASE(test_reduce_dbt, CollReduceTypeOpsHost); TYPED_TEST_CASE(test_reduce_2step, CollReduceTypeOpsHost); TYPED_TEST_CASE(test_reduce_srg, CollReduceTypeOpsHost); ucc_job_env_t post_op_env = {{"UCC_TL_UCP_REDUCE_AVG_PRE_OP", "0"}}; ucc_job_env_t reduce_dbt_env = {{"UCC_TL_UCP_TUNE", "reduce:@dbt:0-inf:inf"}, {"UCC_CLS", "basic"}}; ucc_job_env_t reduce_2step_env = {{"UCC_CL_HIER_TUNE", "reduce:@2step:0-inf:inf"}, {"UCC_CLS", "all"}}; ucc_job_env_t reduce_srg_env = {{"UCC_TL_UCP_TUNE", "reduce:@srg:0-inf:inf"}, {"UCC_CLS", "basic"}}; TYPED_TEST(test_reduce_avg_order, avg_post_op) { TEST_DECLARE_WITH_ENV(post_op_env, 15, true); } TYPED_TEST(test_reduce_dbt, reduce_dbt_shift) { TEST_DECLARE_WITH_ENV(reduce_dbt_env, 15, true); } TYPED_TEST(test_reduce_dbt, reduce_dbt_mirror) { TEST_DECLARE_WITH_ENV(reduce_dbt_env, 16, true); } TYPED_TEST(test_reduce_2step, 2step) { TEST_DECLARE_WITH_ENV(reduce_2step_env, 16, false); } TYPED_TEST(test_reduce_srg, srg) { TEST_DECLARE_WITH_ENV(reduce_srg_env, 15, false); } ucc-1.8.0/test/gtest/coll/test_allreduce_sliding_window.h0000664000175000017500000000152515211535620023767 0ustar alastairalastair#ifndef TEST_ALLREDUCE_SW_H #define TEST_ALLREDUCE_SW_H #include "common/test_ucc.h" #ifdef HAVE_UCX #include typedef struct global_work_buf_info { void *packed_src_memh; void *packed_dst_memh; } global_work_buf_info; struct export_buf { ucp_context_h ucp_context; ucp_mem_h memh; void * packed_memh; size_t packed_memh_len; uint64_t memh_id; }; typedef struct test_ucp_info_t { ucp_context_h ucp_ctx; ucp_config_t *ucp_config; struct export_buf src_ebuf; struct export_buf dst_ebuf; } test_ucp_info_t; void free_gwbi(int n_procs, UccCollCtxVec &ctxs, test_ucp_info_t *ucp_infos, bool inplace); ucs_status_t setup_gwbi(int n_procs, UccCollCtxVec &ctxs, test_ucp_info_t **ucp_infos_p /* out */, bool inplace); #endif #endif ucc-1.8.0/test/gtest/coll/test_scatterv.cc0000664000175000017500000002536115211535620020724 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "common/test_ucc.h" #include "utils/ucc_math.h" using Param_0 = std::tuple; using Param_1 = std::tuple; class test_scatterv : public UccCollArgs, public ucc::test { private: int root; public: void data_init(int nprocs, ucc_datatype_t dtype, size_t count, UccCollCtxVec &ctxs, bool persistent) { ucc_coll_args_t *coll; int *counts, *displs; size_t my_count, all_counts; ctxs.resize(nprocs); for (auto r = 0; r < nprocs; r++) { coll = (ucc_coll_args_t *)calloc(1, sizeof(ucc_coll_args_t)); my_count = (nprocs - r) * count; ctxs[r] = (gtest_ucc_coll_ctx_t *)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->mask = 0; coll->flags = 0; coll->coll_type = UCC_COLL_TYPE_SCATTERV; coll->root = root; coll->dst.info.mem_type = mem_type; coll->dst.info.count = (ucc_count_t)my_count; coll->dst.info.datatype = dtype; if (r == root) { all_counts = 0; counts = (int*)malloc(sizeof(int) * nprocs); ASSERT_NE(counts, nullptr); displs = (int*)malloc(sizeof(int) * nprocs); ASSERT_NE(displs, nullptr); for (int i = 0; i < nprocs; i++) { counts[i] = (nprocs - i) * count; displs[i] = all_counts; all_counts += counts[i]; } ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(dtype) * all_counts, "init buf"); ASSERT_NE(ctxs[r]->init_buf, nullptr); uint8_t *sbuf = (uint8_t*)ctxs[r]->init_buf; for (int p = 0; p < nprocs; p++) { for (int i = 0; i < ucc_dt_size(dtype) * counts[p]; i++) { sbuf[(displs[p] * ucc_dt_size(dtype) + i)] = (uint8_t)((i + p) % 256); } } coll->src.info_v.mem_type = mem_type; coll->src.info_v.counts = (ucc_count_t *)counts; coll->src.info_v.displacements = (ucc_aint_t *)displs; coll->src.info_v.datatype = dtype; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(dtype) * all_counts, mem_type)); coll->src.info_v.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info_v.buffer, ctxs[r]->init_buf, ucc_dt_size(dtype) * all_counts, mem_type, UCC_MEMORY_TYPE_HOST)); } if (r != root || !inplace) { UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ucc_dt_size(dtype) * my_count, mem_type)); coll->dst.info.buffer = ctxs[r]->dst_mc_header->addr; } if (inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; } if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void data_fini(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; if (r == root) { UCC_CHECK(ucc_mc_free(ctxs[r]->src_mc_header)); free(coll->src.info_v.counts); free(coll->src.info_v.displacements); } if (r != root || !inplace) { UCC_CHECK(ucc_mc_free(ctxs[r]->dst_mc_header)); } ucc_free(ctxs[r]->init_buf); free(coll); free(ctxs[r]); } ctxs.clear(); } void reset(UccCollCtxVec ctxs) { ucc_coll_args_t *coll; ucc_datatype_t dtype = (ctxs[0])->args->dst.info.datatype; size_t my_count; for (auto r = 0; r < ctxs.size(); r++) { if (r != root || !inplace) { coll = ctxs[r]->args; my_count = coll->dst.info.count; clear_buffer(coll->dst.info.buffer, my_count * ucc_dt_size(dtype), mem_type, 0); } } } bool data_validate(UccCollCtxVec ctxs) { bool ret = true; int root = ctxs[0]->args->root; size_t dt_size; size_t my_count; uint8_t *dsts; for (auto r = 0; r < ctxs.size(); r++) { if (r == root && inplace) { continue; } dt_size = ucc_dt_size((ctxs[r])->args->dst.info.datatype); my_count = (ctxs[r])->args->dst.info.count; if (UCC_MEMORY_TYPE_HOST != mem_type) { dsts = (uint8_t *)ucc_malloc(my_count * dt_size, "dsts buf"); ucc_assert(dsts != nullptr); UCC_CHECK(ucc_mc_memcpy(dsts, ctxs[r]->args->dst.info.buffer, my_count * dt_size, UCC_MEMORY_TYPE_HOST, mem_type)); } else { dsts = (uint8_t *)ctxs[r]->args->dst.info.buffer; } for (int i = 0; i < my_count * dt_size; i++) { if ((uint8_t)((i + r) % 256) != dsts[i]) { ret = false; break; } } if (UCC_MEMORY_TYPE_HOST != mem_type) { ucc_free(dsts); if (!ret) { break; } } } return ret; } void set_root(int _root) { root = _root; } }; class test_scatterv_0 : public test_scatterv, public ::testing::WithParamInterface { }; UCC_TEST_P(test_scatterv_0, single) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const int root = std::get<4>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<5>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); UccCollCtxVec ctxs; if (size <= root) { GTEST_SKIP(); } set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } UCC_TEST_P(test_scatterv_0, single_persistent) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const int root = std::get<4>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<5>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); const int n_calls = 3; UccCollCtxVec ctxs; if (size <= root) { GTEST_SKIP(); } set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < n_calls; i++) { req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); reset(ctxs); } data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( , test_scatterv_0, ::testing::Combine(::testing::Range(1, UccJob::nStaticTeams), // team_ids PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1, 3, 8192), // count ::testing::Values(0, 1), // root ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE))); class test_scatterv_1 : public test_scatterv, public ::testing::WithParamInterface { }; UCC_TEST_P(test_scatterv_1, multiple_host) { const ucc_datatype_t dtype = std::get<0>(GetParam()); const ucc_memory_type_t mem_type = std::get<1>(GetParam()); const int count = std::get<2>(GetParam()); const int root = std::get<3>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<4>(GetParam()); std::vector reqs; std::vector ctxs; for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { UccTeam_h team = UccJob::getStaticTeams()[tid]; int size = team->procs.size(); UccCollCtxVec ctx; if (size <= root) { /* skip invalid */ continue; } this->set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctx, false); reqs.push_back(UccReq(team, ctx)); ctxs.push_back(ctx); } UccReq::startall(reqs); UccReq::waitall(reqs); for (auto ctx : ctxs) { EXPECT_EQ(true, data_validate(ctx)); data_fini(ctx); } } INSTANTIATE_TEST_CASE_P( , test_scatterv_1, ::testing::Combine(PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1, 3, 8192), // count ::testing::Values(0, 1), // root ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE))); ucc-1.8.0/test/gtest/coll/test_allreduce.cc0000664000175000017500000006374715211535620021043 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "core/test_mc_reduce.h" #include "common/test_ucc.h" #include "utils/ucc_math.h" // For sliding window allreduce #include "test_allreduce_sliding_window.h" #include template class test_allreduce : public UccCollArgs, public testing::Test { public: void data_init(int nprocs, ucc_datatype_t dt, size_t count, UccCollCtxVec &ctxs, bool persistent) { ctxs.resize(nprocs); for (int r = 0; r < nprocs; r++) { ucc_coll_args_t *coll = (ucc_coll_args_t*) calloc(1, sizeof(ucc_coll_args_t)); ctxs[r] = (gtest_ucc_coll_ctx_t*)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->coll_type = UCC_COLL_TYPE_ALLREDUCE; coll->op = T::redop; coll->global_work_buffer = NULL; ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(dt) * count, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); for (int i = 0; i < count; i++) { typename T::type * ptr; ptr = (typename T::type *)ctxs[r]->init_buf; /* need to limit the init value so that "prod" operation would not grow too large. We have teams up to 16 procs in gtest, this would result in prod ~2**48 */ /* bFloat16 will be assigned with the floats matching the uint16_t bit pattern*/ ptr[i] = (typename T::type)((i + r + 1) % 8); } UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ucc_dt_size(dt) * count, mem_type)); coll->dst.info.buffer = ctxs[r]->dst_mc_header->addr; if (TEST_INPLACE == inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; UCC_CHECK(ucc_mc_memcpy(coll->dst.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dt) * count, mem_type, UCC_MEMORY_TYPE_HOST)); coll->src.info.mem_type = UCC_MEMORY_TYPE_UNKNOWN; coll->src.info.count = SIZE_MAX; coll->src.info.datatype = (ucc_datatype_t)-1; } else { UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(dt) * count, mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dt) * count, mem_type, UCC_MEMORY_TYPE_HOST)); coll->src.info.mem_type = mem_type; coll->src.info.count = (ucc_count_t)count; coll->src.info.datatype = dt; } coll->dst.info.mem_type = mem_type; coll->dst.info.count = (ucc_count_t)count; coll->dst.info.datatype = dt; if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void data_fini(UccCollCtxVec ctxs) { for (gtest_ucc_coll_ctx_t* ctx : ctxs) { ucc_coll_args_t* coll = ctx->args; if (coll->src.info.buffer) { /* no inplace */ UCC_CHECK(ucc_mc_free(ctx->src_mc_header)); } UCC_CHECK(ucc_mc_free(ctx->dst_mc_header)); ucc_free(ctx->init_buf); free(coll); free(ctx); } ctxs.clear(); } void reset(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; size_t count = coll->dst.info.count; ucc_datatype_t dtype = coll->dst.info.datatype; clear_buffer(coll->dst.info.buffer, count * ucc_dt_size(dtype), mem_type, 0); if (TEST_INPLACE == inplace) { UCC_CHECK(ucc_mc_memcpy(coll->dst.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dtype) * count, mem_type, UCC_MEMORY_TYPE_HOST)); } } } bool data_validate(UccCollCtxVec ctxs) { size_t count = (ctxs[0])->args->dst.info.count; std::vector dsts(ctxs.size()); if (UCC_MEMORY_TYPE_HOST != mem_type) { for (int r = 0; r < ctxs.size(); r++) { dsts[r] = (typename T::type *) ucc_malloc(count * sizeof(typename T::type), "dsts buf"); EXPECT_NE(dsts[r], nullptr); UCC_CHECK(ucc_mc_memcpy(dsts[r], ctxs[r]->args->dst.info.buffer, count * sizeof(typename T::type), UCC_MEMORY_TYPE_HOST, mem_type)); } } else { for (int r = 0; r < ctxs.size(); r++) { dsts[r] = (typename T::type *)(ctxs[r]->args->dst.info.buffer); } } for (int i = 0; i < count; i++) { typename T::type res = ((typename T::type *)((ctxs[0])->init_buf))[i]; for (int r = 1; r < ctxs.size(); r++) { res = T::do_op(res, ((typename T::type *)((ctxs[r])->init_buf))[i]); } if (T::redop == UCC_OP_AVG) { if (T::dt == UCC_DT_BFLOAT16){ float32tobfloat16(bfloat16tofloat32(&res) / (float)ctxs.size(), &res); } else { res = res / (typename T::type)ctxs.size(); } } for (int r = 0; r < ctxs.size(); r++) { T::assert_equal(res, dsts[r][i]); } } if (UCC_MEMORY_TYPE_HOST != mem_type) { for (int r = 0; r < ctxs.size(); r++) { ucc_free(dsts[r]); } } return true; } }; template class test_allreduce_host : public test_allreduce {}; template class test_allreduce_cuda : public test_allreduce {}; TYPED_TEST_CASE(test_allreduce_host, CollReduceTypeOpsHost); TYPED_TEST_CASE(test_allreduce_cuda, CollReduceTypeOpsCuda); #define TEST_DECLARE(_mem_type, _inplace, _repeat, _persistent) \ { \ std::array counts{4, 256, 65536}; \ for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { \ for (int count : counts) { \ UccTeam_h team = UccJob::getStaticTeams()[tid]; \ int size = team->procs.size(); \ UccCollCtxVec ctxs; \ SET_MEM_TYPE(_mem_type); \ this->set_inplace(_inplace); \ this->data_init(size, TypeParam::dt, count, ctxs, _persistent);\ UccReq req(team, ctxs); \ for (auto i = 0; i < _repeat; i++) { \ req.start(); \ req.wait(); \ EXPECT_EQ(true, this->data_validate(ctxs)); \ this->reset(ctxs); \ } \ this->data_fini(ctxs); \ } \ } \ } TYPED_TEST(test_allreduce_host, single) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_allreduce_host, single_persistent) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_allreduce_host, single_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE, 1, 0); } TYPED_TEST(test_allreduce_host, single_persistent_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE, 3, 1); } #ifdef HAVE_CUDA TYPED_TEST(test_allreduce_cuda, single) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_allreduce_cuda, single_persistent) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_allreduce_cuda, single_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE, 1, 0); } TYPED_TEST(test_allreduce_cuda, single_persistent_inplace) { TEST_DECLARE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE, 3, 1); } TYPED_TEST(test_allreduce_cuda, single_managed) { TEST_DECLARE( UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE, 1, 0); } TYPED_TEST(test_allreduce_cuda, single_persistent_managed) { TEST_DECLARE( UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE, 3, 1); } TYPED_TEST(test_allreduce_cuda, single_inplace_managed) { TEST_DECLARE( UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE, 1, 0); } TYPED_TEST(test_allreduce_cuda, single_persistent_inplace_managed) { TEST_DECLARE( UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE, 3, 1); } #endif #define TEST_DECLARE_MULTIPLE(_mem_type, _inplace) \ { \ std::array counts{4, 256, 65536}; \ for (int count : counts) { \ std::vector reqs; \ std::vector ctxs; \ for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { \ UccTeam_h team = UccJob::getStaticTeams()[tid]; \ int size = team->procs.size(); \ UccCollCtxVec ctx; \ this->set_inplace(_inplace); \ SET_MEM_TYPE(_mem_type); \ this->data_init(size, TypeParam::dt, count, ctx, false); \ reqs.push_back(UccReq(team, ctx)); \ ctxs.push_back(ctx); \ } \ UccReq::startall(reqs); \ UccReq::waitall(reqs); \ for (auto ctx : ctxs) { \ EXPECT_EQ(true, this->data_validate(ctx)); \ this->data_fini(ctx); \ } \ } \ } TYPED_TEST(test_allreduce_host, multiple) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_HOST, TEST_NO_INPLACE); } TYPED_TEST(test_allreduce_host, multiple_inplace) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_HOST, TEST_INPLACE); } #ifdef HAVE_CUDA TYPED_TEST(test_allreduce_cuda, multiple) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA, TEST_NO_INPLACE); } TYPED_TEST(test_allreduce_cuda, multiple_inplace) { TEST_DECLARE_MULTIPLE(UCC_MEMORY_TYPE_CUDA, TEST_INPLACE); } TYPED_TEST(test_allreduce_cuda, multiple_managed) { TEST_DECLARE_MULTIPLE( UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_NO_INPLACE); } TYPED_TEST(test_allreduce_cuda, multiple_inplace_managed) { TEST_DECLARE_MULTIPLE( UCC_MEMORY_TYPE_CUDA_MANAGED, TEST_INPLACE); } #endif template class test_allreduce_alg : public test_allreduce {}; // Expanded type list for allreduce algorithm tests to cover more data types and operations using test_allreduce_alg_type = ::testing::Types< TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair >; TYPED_TEST_CASE(test_allreduce_alg, test_allreduce_alg_type); TYPED_TEST(test_allreduce_alg, sra_knomial_pipelined) { int n_procs = 15; ucc_job_env_t env = {{"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", "allreduce:@sra_knomial:inf"}, {"UCC_TL_UCP_ALLREDUCE_SRA_KN_PIPELINE", "thresh=1024:nfrags=11"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); int repeat = 3; UccCollCtxVec ctxs; std::vector mt = {UCC_MEMORY_TYPE_HOST}; if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { mt.push_back(UCC_MEMORY_TYPE_CUDA); } if (UCC_OK == ucc_mc_available( UCC_MEMORY_TYPE_CUDA_MANAGED)) { mt.push_back( UCC_MEMORY_TYPE_CUDA_MANAGED); } for (auto count : {65536, 123567}) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { for (auto m : mt) { SET_MEM_TYPE(m); this->set_inplace(inplace); this->data_init(n_procs, TypeParam::dt, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < repeat; i++) { req.start(); req.wait(); EXPECT_EQ(true, this->data_validate(ctxs)); this->reset(ctxs); } this->data_fini(ctxs); } } } } TYPED_TEST(test_allreduce_alg, dbt) { int n_procs = 15; ucc_job_env_t env = {{"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", "allreduce:@dbt:inf"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); int repeat = 3; UccCollCtxVec ctxs; std::vector mt = {UCC_MEMORY_TYPE_HOST}; if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { mt.push_back(UCC_MEMORY_TYPE_CUDA); } if (UCC_OK == ucc_mc_available( UCC_MEMORY_TYPE_CUDA_MANAGED)) { mt.push_back( UCC_MEMORY_TYPE_CUDA_MANAGED); } for (auto count : {65536, 123567}) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { for (auto m : mt) { SET_MEM_TYPE(m); this->set_inplace(inplace); this->data_init(n_procs, TypeParam::dt, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < repeat; i++) { req.start(); req.wait(); EXPECT_EQ(true, this->data_validate(ctxs)); this->reset(ctxs); } this->data_fini(ctxs); } } } } TYPED_TEST(test_allreduce_alg, ring) { int n_procs = 15; /* "0-inf:@ring" forces ring for all message sizes; ring returns * UCC_ERR_NOT_SUPPORTED when count % team_size != 0 (see * ring_count_not_divisible). All counts below are chosen to be * divisible by n_procs so that this tune is valid here. */ ucc_job_env_t env = {{"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", "allreduce:0-inf:@ring"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); int repeat = 3; UccCollCtxVec ctxs; std::vector mt = {UCC_MEMORY_TYPE_HOST}; if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { mt.push_back(UCC_MEMORY_TYPE_CUDA); } if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA_MANAGED)) { mt.push_back(UCC_MEMORY_TYPE_CUDA_MANAGED); } // Test with various data sizes: small, medium, large for (auto count : {15, 65535, 123570}) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { for (auto m : mt) { SET_MEM_TYPE(m); this->set_inplace(inplace); this->data_init(n_procs, TypeParam::dt, count, ctxs, true); UccReq req(team, ctxs); ASSERT_EQ(UCC_OK, req.status); for (auto i = 0; i < repeat; i++) { req.start(); req.wait(); EXPECT_EQ(true, this->data_validate(ctxs)); this->reset(ctxs); } this->data_fini(ctxs); } } } } TYPED_TEST(test_allreduce_alg, ring_edge_cases) { // Test with non-power-of-two team sizes and edge cases for (auto team_size : {3, 7, 13}) { ucc_job_env_t env = {{"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", "allreduce:0-inf:@ring"}}; UccJob job(team_size, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(team_size); UccCollCtxVec ctxs; std::vector mt = {UCC_MEMORY_TYPE_HOST}; if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { mt.push_back(UCC_MEMORY_TYPE_CUDA); } if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA_MANAGED)) { mt.push_back(UCC_MEMORY_TYPE_CUDA_MANAGED); } // counts are multiples of LCM(3,7,13)=273, valid for all team sizes for (auto count : {273, 546, 819}) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { for (auto m : mt) { SET_MEM_TYPE(m); this->set_inplace(inplace); this->data_init(team_size, TypeParam::dt, count, ctxs, false); UccReq req(team, ctxs); ASSERT_EQ(UCC_OK, req.status); req.start(); req.wait(); EXPECT_EQ(true, this->data_validate(ctxs)); this->data_fini(ctxs); } } } } } TYPED_TEST(test_allreduce_alg, ring_persistent) { // Test persistent operation - results should be consistent across multiple calls int n_procs = 8; ucc_job_env_t env = {{"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", "allreduce:0-inf:@ring"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); UccCollCtxVec ctxs; size_t count = 1024; std::vector mt = {UCC_MEMORY_TYPE_HOST}; if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { mt.push_back(UCC_MEMORY_TYPE_CUDA); } if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA_MANAGED)) { mt.push_back(UCC_MEMORY_TYPE_CUDA_MANAGED); } for (auto m : mt) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { SET_MEM_TYPE(m); this->set_inplace(inplace); this->data_init(n_procs, TypeParam::dt, count, ctxs, true); UccReq req(team, ctxs); CHECK_REQ_NOT_SUPPORTED_SKIP(req, this->data_fini(ctxs)); for (int i = 0; i < 5; i++) { req.start(); req.wait(); EXPECT_EQ(true, this->data_validate(ctxs)); this->reset(ctxs); } this->data_fini(ctxs); } } } #ifdef HAVE_UCX TYPED_TEST(test_allreduce_alg, sliding_window) { int n_procs = 8; ucc_job_env_t env = {{"UCC_TL_UCP_TUNE", "allreduce:@sliding_window"}, {"UCC_CLS", "all"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL_ONESIDED, env); UccTeam_h team = job.create_team(n_procs); int repeat = 3; test_ucp_info_t *ucp_infos = NULL; std::vector mt = {UCC_MEMORY_TYPE_HOST}; ucs_status_t ucs_status = UCS_OK; UccCollCtxVec ctxs; if (UCC_OK == ucc_mc_available( UCC_MEMORY_TYPE_CUDA)) { //add cuda_managed for cl hier? mt.push_back(UCC_MEMORY_TYPE_CUDA); } for (auto count : {65536, 123567}) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { for (auto m : mt) { SET_MEM_TYPE(m); this->set_inplace(inplace); this->data_init(n_procs, TypeParam::dt, count, ctxs, true); // set args->global_work_buffer on each ctx ucs_status = setup_gwbi(n_procs, ctxs, &ucp_infos, inplace == TEST_INPLACE); if (ucs_status != UCS_OK) { free_gwbi(n_procs, ctxs, ucp_infos, inplace == TEST_INPLACE); this->data_fini(ctxs); if (ucs_status == UCS_ERR_UNSUPPORTED) { GTEST_SKIP() << "Exported memory key not supported"; } else { GTEST_FAIL() << ucs_status_string(ucs_status); } } for (auto i = 0; i < repeat; i++) { this->reset(ctxs); } free_gwbi(n_procs, ctxs, ucp_infos, inplace == TEST_INPLACE); ucp_infos = NULL; this->data_fini(ctxs); } } } } #endif TYPED_TEST(test_allreduce_alg, rab) { int n_procs = 15; ucc_job_env_t env = {{"UCC_CL_HIER_TUNE", "allreduce:@rab:0-inf:inf"}, {"UCC_CLS", "all"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); int repeat = 3; UccCollCtxVec ctxs; std::vector mt = {UCC_MEMORY_TYPE_HOST}; if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { //add cuda_managed for cl hier? mt.push_back(UCC_MEMORY_TYPE_CUDA); } for (auto count : {8, 65536, 123567}) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { for (auto m : mt) { SET_MEM_TYPE(m); this->set_inplace(inplace); this->data_init(n_procs, TypeParam::dt, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < repeat; i++) { req.start(); req.wait(); EXPECT_EQ(true, this->data_validate(ctxs)); this->reset(ctxs); } this->data_fini(ctxs); } } } } TYPED_TEST(test_allreduce_alg, rab_pipelined) { int n_procs = 15; ucc_job_env_t env = {{"UCC_CL_HIER_TUNE", "allreduce:@rab:0-inf:inf"}, {"UCC_CL_HIER_ALLREDUCE_RAB_PIPELINE", "thresh=1024:nfrags=11"}, {"UCC_CLS", "all"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); int repeat = 3; UccCollCtxVec ctxs; std::vector mt = {UCC_MEMORY_TYPE_HOST}; if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { //add cuda_managed for cl hier? mt.push_back(UCC_MEMORY_TYPE_CUDA); } for (auto count : {65536, 123567}) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { for (auto m : mt) { SET_MEM_TYPE(m); this->set_inplace(inplace); this->data_init(n_procs, TypeParam::dt, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < repeat; i++) { req.start(); req.wait(); EXPECT_EQ(true, this->data_validate(ctxs)); this->reset(ctxs); } this->data_fini(ctxs); } } } } template class test_allreduce_avg_order : public test_allreduce { }; using test_allreduce_avg_order_type = ::testing::Types< TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair>; TYPED_TEST_CASE(test_allreduce_avg_order, test_allreduce_avg_order_type); TYPED_TEST(test_allreduce_avg_order, avg_post_op) { int n_procs = 15; ucc_job_env_t env = {{"UCC_TL_UCP_REDUCE_AVG_PRE_OP", "0"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); int repeat = 3; UccCollCtxVec ctxs; std::vector mt = {UCC_MEMORY_TYPE_HOST}; if (UCC_OK == ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { mt.push_back(UCC_MEMORY_TYPE_CUDA); } if (UCC_OK == ucc_mc_available( UCC_MEMORY_TYPE_CUDA_MANAGED)) { mt.push_back( UCC_MEMORY_TYPE_CUDA_MANAGED); } for (auto count : {4, 256, 65536}) { for (auto inplace : {TEST_NO_INPLACE, TEST_INPLACE}) { for (auto m : mt) { CHECK_TYPE_OP_SKIP(TypeParam::dt, TypeParam::redop, m); SET_MEM_TYPE(m); this->set_inplace(inplace); this->data_init(n_procs, TypeParam::dt, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < repeat; i++) { req.start(); req.wait(); EXPECT_EQ(true, this->data_validate(ctxs)); this->reset(ctxs); } this->data_fini(ctxs); } } } } ucc-1.8.0/test/gtest/coll/test_alltoallv.cc0000664000175000017500000003342615211535620021064 0ustar alastairalastair/** * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "common/test_ucc.h" #include "utils/ucc_math.h" using Param_0 = std::tuple; using Param_1 = std::tuple; template class test_alltoallv : public UccCollArgs, public ucc::test { public: uint64_t coll_mask; uint64_t coll_flags; test_alltoallv() : coll_mask(0), coll_flags(0) {} void data_init(int nprocs, ucc_datatype_t dtype, size_t count, UccCollCtxVec &ctxs, bool persistent = false) { int buf_count; ctxs.resize(nprocs); for (auto r = 0; r < nprocs; r++) { ucc_coll_args_t *coll = (ucc_coll_args_t*) calloc(1, sizeof(ucc_coll_args_t)); ctxs[r] = (gtest_ucc_coll_ctx_t*)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->coll_type = UCC_COLL_TYPE_ALLTOALLV; coll->mask = coll_mask; coll->flags = coll_flags; coll->src.info_v.mem_type = mem_type; coll->src.info_v.counts = (ucc_count_t*)malloc(sizeof(T) * nprocs); coll->src.info_v.datatype = dtype; coll->src.info_v.displacements = (ucc_aint_t*)malloc(sizeof(T) * nprocs); coll->dst.info_v.mem_type = mem_type; coll->dst.info_v.counts = (ucc_count_t*)malloc(sizeof(T) * nprocs); coll->dst.info_v.datatype = dtype; coll->dst.info_v.displacements = (ucc_aint_t*)malloc(sizeof(T) * nprocs); buf_count = 0; for (int i = 0; i < nprocs; i++) { int rank_count = (nprocs + r - i) * count; ((T*)coll->src.info_v.counts)[i] = rank_count; ((T*)coll->src.info_v.displacements)[i] = buf_count; buf_count += rank_count; } /* Force at least 1 zero count for bigger coverage of corner cases */ ((T*)coll->src.info_v.counts)[(r + 1) % nprocs] = 0; ctxs[r]->init_buf = ucc_malloc(buf_count * ucc_dt_size(dtype), "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); for (int i = 0; i < nprocs; i++) { alltoallx_init_buf(r, i, (uint8_t*)ctxs[r]->init_buf + ((T*)coll->src.info_v.displacements)[i] * ucc_dt_size(dtype), ((T*)coll->src.info_v.counts)[i] * ucc_dt_size(dtype)); } UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, buf_count * ucc_dt_size(dtype), mem_type)); coll->src.info_v.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info_v.buffer, ctxs[r]->init_buf, buf_count * ucc_dt_size(dtype), mem_type, UCC_MEMORY_TYPE_HOST)); /* TODO: inplace support */ buf_count = 0; for (int i = 0; i < nprocs; i++) { int rank_count = (nprocs - r + i) * count; ((T*)coll->dst.info_v.counts)[i] = rank_count; ((T*)coll->dst.info_v.displacements)[i] = buf_count; buf_count += rank_count; } ((T*)coll->dst.info_v.counts)[(r - 1 + nprocs) % nprocs] = 0; ctxs[r]->rbuf_size = buf_count * ucc_dt_size(dtype); UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, buf_count * ucc_dt_size(dtype), mem_type)); coll->dst.info_v.buffer = ctxs[r]->dst_mc_header->addr; if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void reset(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; clear_buffer(coll->dst.info_v.buffer, ctxs[r]->rbuf_size, mem_type, 0); } } bool data_validate(UccCollCtxVec ctxs) { bool ret = true; std::vector dsts(ctxs.size()); if (UCC_MEMORY_TYPE_HOST != mem_type) { for (int r = 0; r < ctxs.size(); r++) { dsts[r] = (uint8_t *) ucc_malloc(ctxs[r]->rbuf_size, "dsts buf"); EXPECT_NE(dsts[r], nullptr); UCC_CHECK(ucc_mc_memcpy(dsts[r], ctxs[r]->args->dst.info_v.buffer, ctxs[r]->rbuf_size, UCC_MEMORY_TYPE_HOST, mem_type)); } } else { for (int r = 0; r < ctxs.size(); r++) { dsts[r] = (uint8_t *)(ctxs[r]->args->dst.info_v.buffer); } } for (int r = 0; r < ctxs.size(); r++) { ucc_coll_args_t* coll = ctxs[r]->args; for (int i = 0; i < ctxs.size(); i++) { size_t rank_size = ucc_dt_size(coll->dst.info_v.datatype) * (size_t)((T*)coll->dst.info_v.counts)[i]; size_t rank_offs = ucc_dt_size(coll->dst.info_v.datatype) * (size_t)((T*)coll->dst.info_v.displacements)[i]; if (0 != alltoallx_validate_buf(r, i, (uint8_t*)dsts[r] + rank_offs, rank_size)) { ret = false; break; } } } if (UCC_MEMORY_TYPE_HOST != mem_type) { for (int r = 0; r < ctxs.size(); r++) { ucc_free(dsts[r]); } } return ret; } void data_fini(UccCollCtxVec ctxs) { for (gtest_ucc_coll_ctx_t* ctx : ctxs) { ucc_coll_args_t* coll = ctx->args; UCC_CHECK(ucc_mc_free(ctx->src_mc_header)); free(coll->src.info_v.counts); free(coll->src.info_v.displacements); UCC_CHECK(ucc_mc_free(ctx->dst_mc_header)); free(coll->dst.info_v.counts); free(coll->dst.info_v.displacements); ucc_free(ctx->init_buf); free(coll); free(ctx); } ctxs.clear(); } }; class test_alltoallv_0 : public test_alltoallv , public ::testing::WithParamInterface {}; UCC_TEST_P(test_alltoallv_0, single) { const int team_id = std::get<0>(GetParam()); ucc_memory_type_t mem_type = std::get<1>(GetParam()); gtest_ucc_inplace_t inplace = std::get<2>(GetParam()); const ucc_datatype_t dtype = std::get<3>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); UccCollCtxVec ctxs; coll_mask = UCC_COLL_ARGS_FIELD_FLAGS; coll_flags = UCC_COLL_ARGS_FLAG_COUNT_64BIT | UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT; set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, 1, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } UCC_TEST_P(test_alltoallv_0, single_persistent) { const int team_id = std::get<0>(GetParam()); ucc_memory_type_t mem_type = std::get<1>(GetParam()); gtest_ucc_inplace_t inplace = std::get<2>(GetParam()); const ucc_datatype_t dtype = std::get<3>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); const int n_calls = 3; UccCollCtxVec ctxs; coll_mask = UCC_COLL_ARGS_FIELD_FLAGS; coll_flags = UCC_COLL_ARGS_FLAG_COUNT_64BIT | UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT; set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, 1, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < n_calls; i++) { req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); reset(ctxs); } data_fini(ctxs); } class test_alltoallv_1 : public test_alltoallv , public ::testing::WithParamInterface {}; UCC_TEST_P(test_alltoallv_1, single) { const int team_id = std::get<0>(GetParam()); ucc_memory_type_t mem_type = std::get<1>(GetParam()); gtest_ucc_inplace_t inplace = std::get<2>(GetParam()); const ucc_datatype_t dtype = std::get<3>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); UccCollCtxVec ctxs; set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, 1, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( 64, test_alltoallv_0, ::testing::Combine( ::testing::Range(1, UccJob::nStaticTeams), // team_ids #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(/*TEST_INPLACE,*/ TEST_NO_INPLACE), PREDEFINED_DTYPES)); // dtype INSTANTIATE_TEST_CASE_P( 32, test_alltoallv_1, ::testing::Combine( ::testing::Range(1, UccJob::nStaticTeams), // team_ids #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(/*TEST_INPLACE,*/ TEST_NO_INPLACE), PREDEFINED_DTYPES)); // dtype class test_alltoallv_2 : public test_alltoallv, public ::testing::WithParamInterface {}; class test_alltoallv_3 : public test_alltoallv, public ::testing::WithParamInterface {}; class test_alltoallv_alg : public test_alltoallv, public ::testing::WithParamInterface {}; UCC_TEST_P(test_alltoallv_alg, hybrid) { int n_procs = 15; ucc_memory_type_t mem_type = std::get<0>(GetParam()); gtest_ucc_inplace_t inplace = std::get<1>(GetParam()); const ucc_datatype_t dtype = std::get<2>(GetParam()); ASSERT_NE(inplace, TEST_INPLACE); ucc_job_env_t env = {{"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", "alltoallv:@hybrid:inf"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); UccCollCtxVec ctxs; SET_MEM_TYPE(mem_type); data_init(n_procs, dtype, 16, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } UCC_TEST_P(test_alltoallv_2, multiple) { ucc_memory_type_t mem_type = std::get<0>(GetParam()); gtest_ucc_inplace_t inplace = std::get<1>(GetParam()); const ucc_datatype_t dtype = std::get<2>(GetParam()); std::vector reqs; std::vector ctxs; coll_mask = UCC_COLL_ARGS_FIELD_FLAGS; coll_flags = UCC_COLL_ARGS_FLAG_COUNT_64BIT | UCC_COLL_ARGS_FLAG_DISPLACEMENTS_64BIT; for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { UccTeam_h team = UccJob::getStaticTeams()[tid]; int size = team->procs.size(); UccCollCtxVec ctx; this->set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, 1, ctx, false); reqs.push_back(UccReq(team, ctx)); ctxs.push_back(ctx); } UccReq::startall(reqs); UccReq::waitall(reqs); for (auto ctx : ctxs) { EXPECT_EQ(true, data_validate(ctx)); data_fini(ctx); } } UCC_TEST_P(test_alltoallv_3, multiple) { ucc_memory_type_t mem_type = std::get<0>(GetParam()); gtest_ucc_inplace_t inplace = std::get<1>(GetParam()); const ucc_datatype_t dtype = std::get<2>(GetParam()); std::vector reqs; std::vector ctxs; for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { UccTeam_h team = UccJob::getStaticTeams()[tid]; int size = team->procs.size(); UccCollCtxVec ctx; this->set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, 1, ctx, false); reqs.push_back(UccReq(team, ctx)); ctxs.push_back(ctx); } UccReq::startall(reqs); UccReq::waitall(reqs); for (auto ctx : ctxs) { EXPECT_EQ(true, data_validate(ctx)); data_fini(ctx); } } INSTANTIATE_TEST_CASE_P( alltoallv_algs, test_alltoallv_alg, ::testing::Combine( ::testing::Values(UCC_MEMORY_TYPE_HOST), ::testing::Values(TEST_NO_INPLACE), PREDEFINED_DTYPES)); INSTANTIATE_TEST_CASE_P( 64, test_alltoallv_2, ::testing::Combine( #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(/*TEST_INPLACE,*/ TEST_NO_INPLACE), PREDEFINED_DTYPES)); // dtype INSTANTIATE_TEST_CASE_P( 32, test_alltoallv_3, ::testing::Combine( #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(/*TEST_INPLACE,*/ TEST_NO_INPLACE), PREDEFINED_DTYPES)); // dtype ucc-1.8.0/test/gtest/coll/test_barrier.cc0000664000175000017500000000157315211535620020516 0ustar alastairalastair /** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "common/test_ucc.h" class test_barrier : public ucc::test { public: ucc_coll_args_t coll; test_barrier() { coll.mask = 0; coll.coll_type = UCC_COLL_TYPE_BARRIER; } }; UCC_TEST_F(test_barrier, single_2proc) { UccTeam_h team = UccJob::getStaticJob()->create_team(2); UccReq req(team, &coll); req.start(); req.wait(); } UCC_TEST_F(test_barrier, single_max_procs) { UccTeam_h team = UccJob::getStaticTeams().back(); UccReq req(team, &coll); req.start(); req.wait(); } UCC_TEST_F(test_barrier, multiple) { std::vector reqs; for (auto &team : UccJob::getStaticTeams()) { reqs.push_back(UccReq(team, &coll)); } UccReq::startall(reqs); UccReq::waitall(reqs); } ucc-1.8.0/test/gtest/coll/test_allgatherv.cc0000664000175000017500000003302015211535620021211 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "common/test_ucc.h" #include "utils/ucc_math.h" #include "utils/ucc_log.h" #include "utils/ucc_coll_utils.h" using Param_0 = std::tuple; using Param_1 = std::tuple; using Param_2 = std::tuple; size_t noncontig_padding = 1; // # elements worth of space in between each rank's contribution to the dst buf class test_allgatherv : public UccCollArgs, public ucc::test { public: void data_init(int nprocs, ucc_datatype_t dtype, size_t count, UccCollCtxVec &ctxs, bool persistent) { ctxs.resize(nprocs); for (auto r = 0; r < nprocs; r++) { int *counts; int *displs; size_t my_count = (nprocs - r) * count; size_t disp_counter = 0; ucc_coll_args_t *coll = (ucc_coll_args_t*)calloc(1, sizeof(ucc_coll_args_t)); ctxs[r] = (gtest_ucc_coll_ctx_t*)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; counts = (int*)malloc(sizeof(int) * nprocs); displs = (int*)malloc(sizeof(int) * nprocs); if (is_contig) { for (int i = 0; i < nprocs; i++) { counts[i] = (nprocs - i) * count; displs[i] = disp_counter; disp_counter += counts[i]; } coll->flags = UCC_COLL_ARGS_FLAG_CONTIG_DST_BUFFER; } else { for (int i = 0; i < nprocs; i++) { counts[i] = (nprocs - i) * count; displs[i] = disp_counter; disp_counter += counts[i] + noncontig_padding; // Add noncontig_padding elemnts of space between the bufs } } coll->mask = UCC_COLL_ARGS_FIELD_FLAGS; coll->coll_type = UCC_COLL_TYPE_ALLGATHERV; coll->src.info.mem_type = mem_type; coll->src.info.count = my_count; coll->src.info.datatype = dtype; coll->dst.info_v.mem_type = mem_type; coll->dst.info_v.counts = (ucc_count_t*)counts; coll->dst.info_v.displacements = (ucc_aint_t*)displs; coll->dst.info_v.datatype = dtype; ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(dtype) * my_count, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); for (int i = 0; i < (ucc_dt_size(dtype) * my_count); i++) { uint8_t *sbuf = (uint8_t*)ctxs[r]->init_buf; sbuf[i] = r; } ctxs[r]->rbuf_size = ucc_dt_size(dtype) * disp_counter; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ctxs[r]->rbuf_size, mem_type)); coll->dst.info_v.buffer = ctxs[r]->dst_mc_header->addr; if (TEST_INPLACE == inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; UCC_CHECK(ucc_mc_memcpy((void*)((ptrdiff_t)coll->dst.info_v.buffer + displs[r] * ucc_dt_size(dtype)), ctxs[r]->init_buf, ucc_dt_size(dtype) * my_count, mem_type, UCC_MEMORY_TYPE_HOST)); } else { UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(dtype) * my_count, mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dtype) * my_count, mem_type, UCC_MEMORY_TYPE_HOST)); } if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void reset(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; size_t my_count = coll->src.info.count; ucc_datatype_t dtype = coll->dst.info_v.datatype; int * displs = (int *)coll->dst.info_v.displacements; clear_buffer(coll->dst.info_v.buffer, ctxs[r]->rbuf_size, mem_type, 0); if (TEST_INPLACE == inplace) { UCC_CHECK(ucc_mc_memcpy( (void *)((ptrdiff_t)coll->dst.info_v.buffer + displs[r] * ucc_dt_size(dtype)), ctxs[r]->init_buf, ucc_dt_size(dtype) * my_count, mem_type, UCC_MEMORY_TYPE_HOST)); } } } void data_fini(UccCollCtxVec ctxs) { for (gtest_ucc_coll_ctx_t* ctx : ctxs) { ucc_coll_args_t* coll = ctx->args; if (coll->src.info.buffer) { /* no inplace */ UCC_CHECK(ucc_mc_free(ctx->src_mc_header)); } UCC_CHECK(ucc_mc_free(ctx->dst_mc_header)); free(coll->dst.info_v.displacements); free(coll->dst.info_v.counts); ucc_free(ctx->init_buf); free(coll); free(ctx); } ctxs.clear(); } bool data_validate(UccCollCtxVec ctxs) { bool ret = true; std::vector dsts(ctxs.size()); if (UCC_MEMORY_TYPE_HOST != mem_type) { for (int r = 0; r < ctxs.size(); r++) { dsts[r] = (uint8_t *) ucc_malloc(ctxs[r]->rbuf_size, "ctxs buf"); EXPECT_NE(dsts[r], nullptr); UCC_CHECK(ucc_mc_memcpy(dsts[r], ctxs[r]->args->dst.info_v.buffer, ctxs[r]->rbuf_size, UCC_MEMORY_TYPE_HOST, mem_type)); } } else { for (int r = 0; r < ctxs.size(); r++) { dsts[r] = (uint8_t *)(ctxs[r]->args->dst.info_v.buffer); } } for (int i = 0; i < ctxs.size(); i++) { size_t rank_size = 0; uint8_t *rbuf = dsts[i]; int is_contig = UCC_COLL_IS_DST_CONTIG(ctxs[i]->args); for (int r = 0; r < ctxs.size(); r++) { rbuf += rank_size; rank_size = ucc_dt_size((ctxs[r])->args->src.info.datatype) * (ctxs[r])->args->src.info.count; for (int j = 0; j < rank_size; j++) { if (r != rbuf[j]) { ret = false; break; } } if (!is_contig) { rbuf += noncontig_padding * ucc_dt_size((ctxs[r])->args->src.info.datatype); } } } if (UCC_MEMORY_TYPE_HOST != mem_type) { for (int r = 0; r < ctxs.size(); r++) { ucc_free(dsts[r]); } } return ret; } }; class test_allgatherv_0 : public test_allgatherv, public ::testing::WithParamInterface {}; UCC_TEST_P(test_allgatherv_0, single) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<4>(GetParam()); const bool contig = std::get<5>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); UccCollCtxVec ctxs; set_inplace(inplace); set_contig(contig); SET_MEM_TYPE(mem_type); data_init(size, dtype, count, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs));; data_fini(ctxs); } UCC_TEST_P(test_allgatherv_0, single_persistent) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<4>(GetParam()); const bool contig = std::get<5>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); const int n_calls = 3; UccCollCtxVec ctxs; set_inplace(inplace); set_contig(contig); SET_MEM_TYPE(mem_type); data_init(size, dtype, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < n_calls; i++) { req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); reset(ctxs); } data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( , test_allgatherv_0, ::testing::Combine( ::testing::Range(1, UccJob::nStaticTeams), // team_ids PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), // mem type #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1,3,8192), // count ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE), // inplace ::testing::Bool() // contig dst buf displacements )); class test_allgatherv_1 : public test_allgatherv, public ::testing::WithParamInterface {}; UCC_TEST_P(test_allgatherv_1, multiple) { const ucc_datatype_t dtype = std::get<0>(GetParam()); const ucc_memory_type_t mem_type = std::get<1>(GetParam()); const int count = std::get<2>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<3>(GetParam()); const bool contig = std::get<4>(GetParam()); std::vector reqs; std::vector ctxs; for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { UccTeam_h team = UccJob::getStaticTeams()[tid]; int size = team->procs.size(); UccCollCtxVec ctx; this->set_inplace(inplace); this->set_contig(contig); SET_MEM_TYPE(mem_type); data_init(size, dtype, count, ctx, false); reqs.push_back(UccReq(team, ctx)); ctxs.push_back(ctx); } UccReq::startall(reqs); UccReq::waitall(reqs); for (auto ctx : ctxs) { EXPECT_EQ(true, data_validate(ctx)); data_fini(ctx); } } INSTANTIATE_TEST_CASE_P( , test_allgatherv_1, ::testing::Combine( PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1,3,8192), // count ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE), ::testing::Bool()) // dst buf contig ); class test_allgatherv_alg : public test_allgatherv, public ::testing::WithParamInterface {}; UCC_TEST_P(test_allgatherv_alg, alg) { const ucc_datatype_t dtype = std::get<0>(GetParam()); const ucc_memory_type_t mem_type = std::get<1>(GetParam()); const int count = std::get<2>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<3>(GetParam()); const bool contig = std::get<5>(GetParam()); int n_procs = 5; char tune[32]; sprintf(tune, "allgatherv:@%s:inf", std::get<4>(GetParam()).c_str()); ucc_job_env_t env = {{"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", tune}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); UccCollCtxVec ctxs; set_inplace(inplace); set_contig(contig); SET_MEM_TYPE(mem_type); data_init(n_procs, dtype, count, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( , test_allgatherv_alg, ::testing::Combine( PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1,3,8192), // count ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE), ::testing::Values("knomial", "ring"), ::testing::Bool()), // dst buf contig [](const testing::TestParamInfo& info) { std::string name; name += ucc_datatype_str(std::get<0>(info.param)); name += std::string("_") + std::string(ucc_mem_type_str(std::get<1>(info.param))); name += std::string("_count_")+std::to_string(std::get<2>(info.param)); name += std::string("_inplace_")+std::to_string(std::get<3>(info.param)); name += std::string("_contig_")+std::to_string(std::get<5>(info.param)); name += std::string("_")+std::get<4>(info.param); return name; } ); ucc-1.8.0/test/gtest/coll/test_scatter.cc0000664000175000017500000002426615211535620020541 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "common/test_ucc.h" #include "utils/ucc_math.h" using Param_0 = std::tuple; using Param_1 = std::tuple; class test_scatter : public UccCollArgs, public ucc::test { private: int root; public: void data_init(int nprocs, ucc_datatype_t dtype, size_t single_rank_count, UccCollCtxVec &ctxs, bool persistent) { ucc_coll_args_t *coll; ucc_count_t count; ctxs.resize(nprocs); count = single_rank_count * nprocs; for (auto r = 0; r < nprocs; r++) { coll = (ucc_coll_args_t *)calloc(1, sizeof(ucc_coll_args_t)); ctxs[r] = (gtest_ucc_coll_ctx_t *)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->mask = 0; coll->flags = 0; coll->coll_type = UCC_COLL_TYPE_SCATTER; coll->root = root; coll->dst.info.mem_type = mem_type; coll->dst.info.count = (ucc_count_t)single_rank_count; coll->dst.info.datatype = dtype; if (r == root) { ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(dtype) * count, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); uint8_t *sbuf = (uint8_t*)ctxs[r]->init_buf; for (int p = 0; p < nprocs; p++) { for (int i = 0; i < ucc_dt_size(dtype) * single_rank_count; i++) { sbuf[(p * ucc_dt_size(dtype) * single_rank_count + i)] = (uint8_t)((i+p) % 256); } } coll->src.info.mem_type = mem_type; coll->src.info.count = (ucc_count_t)count; coll->src.info.datatype = dtype; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(dtype) * count, mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dtype) * count, mem_type, UCC_MEMORY_TYPE_HOST)); } if (r != root || !inplace) { UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ucc_dt_size(dtype) * single_rank_count, mem_type)); coll->dst.info.buffer = ctxs[r]->dst_mc_header->addr; } if (inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; } if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void data_fini(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; if (r == root) { UCC_CHECK(ucc_mc_free(ctxs[r]->src_mc_header)); } if (r != root || !inplace) { UCC_CHECK(ucc_mc_free(ctxs[r]->dst_mc_header)); } ucc_free(ctxs[r]->init_buf); free(coll); free(ctxs[r]); } ctxs.clear(); } void reset(UccCollCtxVec ctxs) { ucc_coll_args_t *coll; size_t single_rank_count = (ctxs[0])->args->dst.info.count; ucc_datatype_t dtype = (ctxs[0])->args->dst.info.datatype; for (auto r = 0; r < ctxs.size(); r++) { coll = ctxs[r]->args; if (r != root || !inplace) { clear_buffer(coll->dst.info.buffer, single_rank_count * ucc_dt_size(dtype), mem_type, 0); } } } bool data_validate(UccCollCtxVec ctxs) { bool ret = true; int root = ctxs[0]->args->root; size_t single_rank_count = (ctxs[0])->args->dst.info.count; size_t dt_size = ucc_dt_size((ctxs[0])->args->dst.info.datatype); uint8_t *dsts = nullptr; if (UCC_MEMORY_TYPE_HOST != mem_type) { dsts = (uint8_t *)ucc_malloc(single_rank_count * dt_size, "dsts buf"); EXPECT_NE(dsts, nullptr); } for (auto r = 0; r < ctxs.size(); r++) { if (r == root && inplace) { continue; } if (UCC_MEMORY_TYPE_HOST != mem_type) { UCC_CHECK(ucc_mc_memcpy(dsts, ctxs[r]->args->dst.info.buffer, single_rank_count * dt_size, UCC_MEMORY_TYPE_HOST, mem_type)); } else { dsts = (uint8_t *)ctxs[r]->args->dst.info.buffer; } for (int i = 0; i < single_rank_count * dt_size; i++) { if ((uint8_t)((i + r) % 256) != dsts[i]) { ret = false; break; } } } if (UCC_MEMORY_TYPE_HOST != mem_type) { ucc_free(dsts); } return ret; } void set_root(int _root) { root = _root; } }; class test_scatter_0 : public test_scatter, public ::testing::WithParamInterface { }; UCC_TEST_P(test_scatter_0, single) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const int root = std::get<4>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<5>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); UccCollCtxVec ctxs; GTEST_SKIP(); //scatter not implemented as stand alone in TL/UCP set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } UCC_TEST_P(test_scatter_0, single_persistent) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const int root = std::get<4>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<5>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); const int n_calls = 3; UccCollCtxVec ctxs; GTEST_SKIP(); //scatter not implemented as stand alone in TL/UCP set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < n_calls; i++) { req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); reset(ctxs); } data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( , test_scatter_0, ::testing::Combine(::testing::Range(1, UccJob::nStaticTeams), // team_ids PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1, 3, 8192), // count ::testing::Values(0, 1), // root ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE))); class test_scatter_1 : public test_scatter, public ::testing::WithParamInterface { }; UCC_TEST_P(test_scatter_1, multiple_host) { const ucc_datatype_t dtype = std::get<0>(GetParam()); const ucc_memory_type_t mem_type = std::get<1>(GetParam()); const int count = std::get<2>(GetParam()); const int root = std::get<3>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<4>(GetParam()); std::vector reqs; std::vector ctxs; GTEST_SKIP(); //scatter not implemented as stand alone in TL/UCP for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { UccTeam_h team = UccJob::getStaticTeams()[tid]; int size = team->procs.size(); UccCollCtxVec ctx; if (size == 1 && root > 0) { /* skip team size 1 and root > 0, which are invalid */ continue; } this->set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctx, false); reqs.push_back(UccReq(team, ctx)); ctxs.push_back(ctx); } UccReq::startall(reqs); UccReq::waitall(reqs); for (auto ctx : ctxs) { EXPECT_EQ(true, data_validate(ctx)); data_fini(ctx); } } INSTANTIATE_TEST_CASE_P( , test_scatter_1, ::testing::Combine(PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1, 3, 8192), // count ::testing::Values(0, 1), // root ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE))); ucc-1.8.0/test/gtest/coll/test_gather.cc0000664000175000017500000002464215211535620020344 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "common/test_ucc.h" #include "utils/ucc_math.h" using Param_0 = std::tuple; using Param_1 = std::tuple; class test_gather : public UccCollArgs, public ucc::test { private: int root; public: void data_init(int nprocs, ucc_datatype_t dtype, size_t single_rank_count, UccCollCtxVec &ctxs, bool persistent) { ctxs.resize(nprocs); for (auto r = 0; r < nprocs; r++) { ucc_coll_args_t *coll = (ucc_coll_args_t *)calloc(1, sizeof(ucc_coll_args_t)); ctxs[r] = (gtest_ucc_coll_ctx_t *)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->mask = 0; coll->flags = 0; coll->coll_type = UCC_COLL_TYPE_GATHER; coll->root = root; coll->src.info.mem_type = mem_type; coll->src.info.count = (ucc_count_t)single_rank_count; coll->src.info.datatype = dtype; ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(dtype) * single_rank_count, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); for (int i = 0; i < single_rank_count * ucc_dt_size(dtype); i++) { uint8_t *ptr = (uint8_t *)ctxs[r]->init_buf; ptr[i] = ((i + r) % 256); } if (r == root) { coll->dst.info.mem_type = mem_type; coll->dst.info.count = (ucc_count_t)single_rank_count * nprocs; coll->dst.info.datatype = dtype; ctxs[r]->rbuf_size = ucc_dt_size(dtype) * single_rank_count * nprocs; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ctxs[r]->rbuf_size, mem_type)); coll->dst.info.buffer = ctxs[r]->dst_mc_header->addr; if (inplace) { UCC_CHECK(ucc_mc_memcpy( (void *)((ptrdiff_t)coll->dst.info.buffer + r * single_rank_count * ucc_dt_size(dtype)), ctxs[r]->init_buf, ucc_dt_size(dtype) * single_rank_count, mem_type, UCC_MEMORY_TYPE_HOST)); } } if (r != root || !inplace) { UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(dtype) * single_rank_count, mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dtype) * single_rank_count, mem_type, UCC_MEMORY_TYPE_HOST)); } if (inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; } if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void data_fini(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; if (r == root) { UCC_CHECK(ucc_mc_free(ctxs[r]->dst_mc_header)); } if (r != root || !inplace) { UCC_CHECK(ucc_mc_free(ctxs[r]->src_mc_header)); } ucc_free(ctxs[r]->init_buf); free(coll); free(ctxs[r]); } ctxs.clear(); } void reset(UccCollCtxVec ctxs) { ucc_coll_args_t *coll = ctxs[root]->args; size_t single_rank_count = coll->dst.info.count / ctxs.size(); ucc_datatype_t dtype = coll->dst.info.datatype; clear_buffer(coll->dst.info.buffer, single_rank_count * ucc_dt_size(dtype) * ctxs.size(), mem_type, 0); if (TEST_INPLACE == inplace) { UCC_CHECK(ucc_mc_memcpy( (void *)((ptrdiff_t)coll->dst.info.buffer + root * single_rank_count * ucc_dt_size(dtype)), ctxs[root]->init_buf, ucc_dt_size(dtype) * single_rank_count, mem_type, UCC_MEMORY_TYPE_HOST)); } } bool data_validate(UccCollCtxVec ctxs) { bool ret = true; int root = ctxs[0]->args->root; size_t count = (ctxs[root])->args->dst.info.count; size_t single_rank_count = count / ctxs.size(); size_t dt_size = ucc_dt_size(ctxs[0]->args->src.info.datatype); uint8_t *dsts; if (UCC_MEMORY_TYPE_HOST != mem_type) { dsts = (uint8_t *)ucc_malloc(count * dt_size, "dsts buf"); EXPECT_NE(dsts, nullptr); UCC_CHECK(ucc_mc_memcpy(dsts, ctxs[root]->args->dst.info.buffer, count * dt_size, UCC_MEMORY_TYPE_HOST, mem_type)); } else { dsts = (uint8_t *)ctxs[root]->args->dst.info.buffer; } for (int r = 0; r < ctxs.size(); r++) { for (int i = 0; i < single_rank_count * dt_size; i++) { if ((uint8_t)((i + r) % 256) != dsts[(r * single_rank_count * dt_size + i)]) { ret = false; break; } } } if (UCC_MEMORY_TYPE_HOST != mem_type) { ucc_free(dsts); } return ret; } void set_root(int _root) { root = _root; } }; class test_gather_0 : public test_gather, public ::testing::WithParamInterface { }; UCC_TEST_P(test_gather_0, single) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const int root = std::get<4>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<5>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); UccCollCtxVec ctxs; set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } UCC_TEST_P(test_gather_0, single_persistent) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const int root = std::get<4>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<5>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); const int n_calls = 3; UccCollCtxVec ctxs; set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < n_calls; i++) { req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); reset(ctxs); } data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( , test_gather_0, ::testing::Combine(::testing::Range(1, UccJob::nStaticTeams), // team_ids PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1, 3, 8192), // count ::testing::Values(0, 1), // root ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE))); class test_gather_1 : public test_gather, public ::testing::WithParamInterface { }; UCC_TEST_P(test_gather_1, multiple_host) { const ucc_datatype_t dtype = std::get<0>(GetParam()); const ucc_memory_type_t mem_type = std::get<1>(GetParam()); const int count = std::get<2>(GetParam()); const int root = std::get<3>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<4>(GetParam()); std::vector reqs; std::vector ctxs; for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { UccTeam_h team = UccJob::getStaticTeams()[tid]; int size = team->procs.size(); UccCollCtxVec ctx; if (size == 1 && root > 0) { /* skip team size 1 and root > 0, which are invalid */ continue; } this->set_inplace(inplace); SET_MEM_TYPE(mem_type); set_root(root); data_init(size, dtype, count, ctx, false); reqs.push_back(UccReq(team, ctx)); ctxs.push_back(ctx); } UccReq::startall(reqs); UccReq::waitall(reqs); for (auto ctx : ctxs) { EXPECT_EQ(true, data_validate(ctx)); data_fini(ctx); } } INSTANTIATE_TEST_CASE_P( , test_gather_1, ::testing::Combine(PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1, 3, 8192), // count ::testing::Values(0, 1), // root ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE))); ucc-1.8.0/test/gtest/coll/test_allreduce_sliding_window.cc0000664000175000017500000001452315211535620024127 0ustar alastairalastair/** * Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ /* This file is for setting up the global work buffer for sliding window allreduce. This entails allocating ucp workers, registering memory, exchanging rkeys, and allocating the pipeline datastructure the algorithm uses. */ #include "common/test_ucc.h" #ifdef HAVE_UCX #include "core/test_mc_reduce.h" #include "utils/ucc_math.h" #include #include "test_allreduce_sliding_window.h" void test_init_ucp(ucp_context_h *ucp_ctx, ucp_config_t **ucp_config_p) { ucs_status_t ucs_status; ucp_config_t *ucp_config; ucp_params_t ucp_params; ucp_context_h ucp_context; ucs_status = ucp_config_read(NULL, NULL, &ucp_config); EXPECT_EQ(UCS_OK, ucs_status) << "ucp_config_read() returned error: " << ucs_status_string(ucs_status); ucp_params.field_mask = UCP_PARAM_FIELD_FEATURES; ucp_params.features = UCP_FEATURE_TAG | UCP_FEATURE_RMA | UCP_FEATURE_AMO64 | UCP_FEATURE_EXPORTED_MEMH; ucs_status = ucp_init(&ucp_params, ucp_config, &ucp_context); EXPECT_EQ(UCS_OK, ucs_status) << "ucp_init() returned error: " << ucs_status_string(ucs_status); *ucp_ctx = ucp_context; *ucp_config_p = ucp_config; } ucs_status_t buffer_export_ucc(ucp_context_h ucp_context, void *buf, size_t len, struct export_buf *ebuf) { ucs_status_t ucs_status = UCS_OK; ucp_mem_map_params_t params; ucp_memh_pack_params_t pack_params; ebuf->ucp_context = ucp_context; params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH; params.address = buf; params.length = len; ucs_status = ucp_mem_map(ucp_context, ¶ms, &ebuf->memh); if (ucs_status != UCS_OK) { printf("ucp_mem_map() returned error: %s\n", ucs_status_string(ucs_status)); return ucs_status; } pack_params.field_mask = UCP_MEMH_PACK_PARAM_FIELD_FLAGS; pack_params.flags = UCP_MEMH_PACK_FLAG_EXPORT; ucs_status = ucp_memh_pack(ebuf->memh, &pack_params, &ebuf->packed_memh, &ebuf->packed_memh_len); if (ucs_status != UCS_OK) { printf("ucp_memh_pack() returned error: %s\n", ucs_status_string(ucs_status)); return ucs_status; } return ucs_status; } ucs_status_t setup_gwbi(int n_procs, UccCollCtxVec &ctxs, test_ucp_info_t **ucp_infos_p /* out */, bool inplace) { int i; ucs_status_t ucs_status = UCS_OK; test_ucp_info_t *ucp_infos = (test_ucp_info_t *)ucc_malloc(sizeof(test_ucp_info_t) * n_procs); EXPECT_NE(ucp_infos, nullptr); *ucp_infos_p = ucp_infos; // allocate gwbi for (auto ctx : ctxs) { global_work_buf_info *gwbi = (global_work_buf_info *)ucc_malloc( sizeof(global_work_buf_info), "global work buf info"); EXPECT_NE(gwbi, nullptr); ctx->args->global_work_buffer = gwbi; } // setup ucp contexts and workers for (i = 0; i < n_procs; i++) { test_init_ucp(&ucp_infos[i].ucp_ctx, &ucp_infos[i].ucp_config); ucp_infos[i].src_ebuf = {0}; ucp_infos[i].dst_ebuf = {0}; } // set up packed src/dst memh for (i = 0; i < n_procs; i++) { // my proc's gwbi global_work_buf_info *gwbi = (global_work_buf_info *)ctxs[i] ->args->global_work_buffer; // my proc's ucp_info test_ucp_info_t * ucp_info = &ucp_infos[i]; struct export_buf *dst_ebuf = &ucp_info->dst_ebuf; size_t dst_len = ctxs[i]->args->dst.info.count * ucc_dt_size(ctxs[i]->args->dst.info.datatype); ucs_status = buffer_export_ucc( ucp_info->ucp_ctx, ctxs[i]->args->dst.info.buffer, dst_len, dst_ebuf); if (ucs_status != UCS_OK) return ucs_status; gwbi->packed_dst_memh = dst_ebuf->packed_memh; if (!inplace) { size_t src_len = ctxs[i]->args->src.info.count * ucc_dt_size(ctxs[i]->args->src.info.datatype); struct export_buf *src_ebuf = &ucp_info->src_ebuf; ucs_status = buffer_export_ucc( ucp_info->ucp_ctx, ctxs[i]->args->src.info.buffer, src_len, src_ebuf); if (ucs_status != UCS_OK) return ucs_status; gwbi->packed_src_memh = src_ebuf->packed_memh; } } // set the flag that indicates the global work buffer was passed for (auto ctx : ctxs) { ctx->args->mask |= UCC_COLL_ARGS_FIELD_FLAGS | UCC_COLL_ARGS_FIELD_GLOBAL_WORK_BUFFER; ctx->args->flags |= UCC_COLL_ARGS_FLAG_MEM_MAPPED_BUFFERS; } return ucs_status; } void free_gwbi(int n_procs, UccCollCtxVec &ctxs, test_ucp_info_t *ucp_infos, bool inplace) { int i, k; ucs_status_t ucs_status; // free sbufs, rbufs, src_rkeys, and dst_rkeys for (i = 0; i < n_procs; i++) { // my proc's ucp_info test_ucp_info_t *ucp_info = &ucp_infos[i]; if (!inplace) { struct export_buf *src_ebuf = &ucp_info->src_ebuf; if (src_ebuf->memh != 0) { ucs_status = ucp_mem_unmap(ucp_info->ucp_ctx, src_ebuf->memh); ASSERT_EQ(UCS_OK, ucs_status) << "ucp_mem_unmap() returned error: " << ucs_status_string(ucs_status); } } struct export_buf *dst_ebuf = &ucp_info->dst_ebuf; if (dst_ebuf->memh != 0) { ucs_status = ucp_mem_unmap(ucp_info->ucp_ctx, dst_ebuf->memh); ASSERT_EQ(UCS_OK, ucs_status) << "ucp_mem_unmap() returned error: " << ucs_status_string(ucs_status); } } // free ucp contexts for (i = 0; i < n_procs; i++) { ucp_config_release(ucp_infos[i].ucp_config); ucp_cleanup(ucp_infos[i].ucp_ctx); } // free gwbi and each gwbi's set of pipes for (k = 0; k < n_procs; k++) { global_work_buf_info *gwbi = (global_work_buf_info *) ctxs[k]->args->global_work_buffer; ucc_free(gwbi); } ucc_free(ucp_infos); } #endif ucc-1.8.0/test/gtest/coll/test_alltoall.cc0000664000175000017500000003045115211535620020671 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "common/test_ucc.h" #include "utils/ucc_math.h" using Param_0 = std::tuple; using Param_1 = std::tuple; class test_alltoall : public UccCollArgs, public ucc::test { public: void data_init(int nprocs, ucc_datatype_t dtype, size_t single_rank_count, UccCollCtxVec &ctxs, UccTeam_h team, bool persistent) { bool is_onesided = (NULL != team); void *sbuf; void *rbuf; long *work_buf; ctxs.resize(nprocs); for (auto i = 0; i < nprocs; i++) { ucc_coll_args_t *coll = (ucc_coll_args_t *)calloc(1, sizeof(ucc_coll_args_t)); ctxs[i] = (gtest_ucc_coll_ctx_t *)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[i]->args = coll; coll->mask = 0; coll->coll_type = UCC_COLL_TYPE_ALLTOALL; coll->src.info.mem_type = mem_type; coll->src.info.count = (ucc_count_t)single_rank_count * nprocs; coll->src.info.datatype = dtype; coll->dst.info.mem_type = mem_type; coll->dst.info.count = (ucc_count_t)single_rank_count * nprocs; coll->dst.info.datatype = dtype; ctxs[i]->init_buf = ucc_malloc( ucc_dt_size(dtype) * single_rank_count * nprocs, "init buf"); EXPECT_NE(ctxs[i]->init_buf, nullptr); for (int r = 0; r < nprocs; r++) { size_t rank_size = ucc_dt_size(dtype) * single_rank_count; alltoallx_init_buf(r, i, (uint8_t *)ctxs[i]->init_buf + r * rank_size, rank_size); } if (is_onesided) { sbuf = team->procs[i].p->onesided_buf[0]; rbuf = team->procs[i].p->onesided_buf[1]; work_buf = (long *)team->procs[i].p->onesided_buf[2]; coll->mask = UCC_COLL_ARGS_FIELD_FLAGS | UCC_COLL_ARGS_FIELD_GLOBAL_WORK_BUFFER; coll->flags = UCC_COLL_ARGS_FLAG_MEM_MAPPED_BUFFERS; coll->src.info.buffer = sbuf; coll->src.info.mem_type = UCC_MEMORY_TYPE_HOST; coll->dst.info.buffer = rbuf; coll->dst.info.mem_type = UCC_MEMORY_TYPE_HOST; coll->global_work_buffer = work_buf; } else { UCC_CHECK(ucc_mc_alloc( &ctxs[i]->dst_mc_header, ucc_dt_size(dtype) * single_rank_count * nprocs, mem_type)); coll->dst.info.buffer = ctxs[i]->dst_mc_header->addr; } if (TEST_INPLACE == inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; UCC_CHECK(ucc_mc_memcpy( coll->dst.info.buffer, ctxs[i]->init_buf, ucc_dt_size(dtype) * single_rank_count * nprocs, mem_type, UCC_MEMORY_TYPE_HOST)); } else { if (!is_onesided) { UCC_CHECK(ucc_mc_alloc(&ctxs[i]->src_mc_header, ucc_dt_size(dtype) * single_rank_count * nprocs, mem_type)); coll->src.info.buffer = ctxs[i]->src_mc_header->addr; } UCC_CHECK(ucc_mc_memcpy( coll->src.info.buffer, ctxs[i]->init_buf, ucc_dt_size(dtype) * single_rank_count * nprocs, mem_type, UCC_MEMORY_TYPE_HOST)); } if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void data_init(int nprocs, ucc_datatype_t dtype, size_t single_rank_count, UccCollCtxVec &ctxs, bool persistent = false) { data_init(nprocs, dtype, single_rank_count, ctxs, NULL, persistent); } void reset(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; size_t single_rank_count = coll->dst.info.count / ctxs.size(); ucc_datatype_t dtype = coll->dst.info.datatype; clear_buffer(coll->dst.info.buffer, single_rank_count * ucc_dt_size(dtype), mem_type, 0); if (TEST_INPLACE == inplace) { UCC_CHECK(ucc_mc_memcpy( (void *)(ptrdiff_t)coll->dst.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dtype) * single_rank_count, mem_type, UCC_MEMORY_TYPE_HOST)); } } } void data_fini(UccCollCtxVec ctxs) { for (gtest_ucc_coll_ctx_t* ctx : ctxs) { ucc_coll_args_t* coll = ctx->args; if (coll->src.info.buffer) { /* no inplace */ UCC_CHECK(ucc_mc_free(ctx->src_mc_header)); } UCC_CHECK(ucc_mc_free(ctx->dst_mc_header)); ucc_free(ctx->init_buf); free(coll); free(ctx); } ctxs.clear(); } void data_fini_onesided(UccCollCtxVec ctxs) { for (gtest_ucc_coll_ctx_t *ctx : ctxs) { ucc_coll_args_t *coll = ctx->args; ucc_free(ctx->init_buf); free(coll); free(ctx); } ctxs.clear(); } bool data_validate(UccCollCtxVec ctxs) { bool ret = true; std::vector dsts(ctxs.size()); if (UCC_MEMORY_TYPE_HOST != mem_type) { for (int r = 0; r < ctxs.size(); r++) { size_t buf_size = ucc_dt_size(ctxs[r]->args->dst.info.datatype) * (size_t)ctxs[r]->args->dst.info.count; dsts[r] = (uint8_t *) ucc_malloc(buf_size, "dsts buf"); EXPECT_NE(dsts[r], nullptr); UCC_CHECK(ucc_mc_memcpy(dsts[r], ctxs[r]->args->dst.info.buffer, buf_size, UCC_MEMORY_TYPE_HOST, mem_type)); } } else { for (int r = 0; r < ctxs.size(); r++) { dsts[r] = (uint8_t *)(ctxs[r]->args->dst.info.buffer); } } for (int r = 0; r < ctxs.size(); r++) { ucc_coll_args_t* coll = ctxs[r]->args; for (int i = 0; i < ctxs.size(); i++) { size_t rank_size = ucc_dt_size(coll->dst.info.datatype) * (size_t)(coll->dst.info.count / ctxs.size()); if (0 != alltoallx_validate_buf(i, r, (uint8_t*)dsts[r] + rank_size * i, rank_size)) { ret = false; break; } } } if (UCC_MEMORY_TYPE_HOST != mem_type) { for (int r = 0; r < ctxs.size(); r++) { ucc_free(dsts[r]); } } return ret; } }; class test_alltoall_0 : public test_alltoall, public ::testing::WithParamInterface {}; UCC_TEST_P(test_alltoall_0, single) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); ucc_memory_type_t mem_type = std::get<2>(GetParam()); gtest_ucc_inplace_t inplace = std::get<3>(GetParam()); const int count = std::get<4>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); UccCollCtxVec ctxs; this->set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, count, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } UCC_TEST_P(test_alltoall_0, single_onesided) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); ucc_memory_type_t mem_type = std::get<2>(GetParam()); gtest_ucc_inplace_t inplace = std::get<3>(GetParam()); const int count = std::get<4>(GetParam()); UccTeam_h reference_team = UccJob::getStaticTeams()[team_id]; int size = reference_team->procs.size(); ucc_job_env_t env = {{"UCC_TL_UCP_TUNE", "alltoall:0-inf:@1"}}; bool is_contig = true; UccJob job(size, UccJob::UCC_JOB_CTX_GLOBAL_ONESIDED, env); UccTeam_h team; std::vector reference_ranks; UccCollCtxVec ctxs; for (auto i = 0; i < reference_team->n_procs; i++) { int rank = reference_team->procs[i].p->job_rank; reference_ranks.push_back(rank); if (is_contig && i > 0 && (rank - reference_ranks[i - 1] > 1 || reference_ranks[i - 1] - rank > 1)) { is_contig = false; } } team = job.create_team(reference_ranks, true, is_contig, true); this->set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, count, ctxs, team, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini_onesided(ctxs); } UCC_TEST_P(test_alltoall_0, single_persistent) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); ucc_memory_type_t mem_type = std::get<2>(GetParam()); gtest_ucc_inplace_t inplace = std::get<3>(GetParam()); const int count = std::get<4>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); const int n_calls = 3; UccCollCtxVec ctxs; this->set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < n_calls; i++) { req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); reset(ctxs); } data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( , test_alltoall_0, ::testing::Combine( ::testing::Range(1, UccJob::nStaticTeams), // team_ids PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(/*TEST_INPLACE,*/ TEST_NO_INPLACE), ::testing::Values(1,3))); class test_alltoall_1 : public test_alltoall, public ::testing::WithParamInterface {}; UCC_TEST_P(test_alltoall_1, multiple) { const ucc_datatype_t dtype = std::get<0>(GetParam()); ucc_memory_type_t mem_type = std::get<1>(GetParam()); gtest_ucc_inplace_t inplace = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); std::vector reqs; std::vector ctxs; for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { UccTeam_h team = UccJob::getStaticTeams()[tid]; int size = team->procs.size(); UccCollCtxVec ctx; this->set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, count, ctx, false); reqs.push_back(UccReq(team, ctx)); ctxs.push_back(ctx); } UccReq::startall(reqs); UccReq::waitall(reqs); for (auto ctx : ctxs) { EXPECT_EQ(true, data_validate(ctx)); data_fini(ctx); } } INSTANTIATE_TEST_CASE_P( , test_alltoall_1, ::testing::Combine( PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(/*TEST_INPLACE,*/ TEST_NO_INPLACE), ::testing::Values(1,3,8192))); // count ucc-1.8.0/test/gtest/coll/test_allgather.cc0000664000175000017500000002725115211535620021034 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ #include "common/test_ucc.h" #include "utils/ucc_math.h" #include "utils/ucc_log.h" using Param_0 = std::tuple; using Param_1 = std::tuple; using Param_2 = std::tuple; class test_allgather : public UccCollArgs, public ucc::test { public: void data_init(int nprocs, ucc_datatype_t dtype, size_t single_rank_count, UccCollCtxVec &ctxs, bool persistent) { ctxs.resize(nprocs); for (auto r = 0; r < nprocs; r++) { ucc_coll_args_t *coll = (ucc_coll_args_t*) calloc(1, sizeof(ucc_coll_args_t)); ctxs[r] = (gtest_ucc_coll_ctx_t*)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->mask = 0; coll->flags = 0; coll->coll_type = UCC_COLL_TYPE_ALLGATHER; coll->src.info.mem_type = mem_type; coll->src.info.count = (ucc_count_t)single_rank_count; coll->src.info.datatype = dtype; coll->dst.info.mem_type = mem_type; coll->dst.info.count = (ucc_count_t)single_rank_count * nprocs; coll->dst.info.datatype = dtype; ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(dtype) * single_rank_count, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); uint8_t *sbuf = (uint8_t *)ctxs[r]->init_buf; for (int i = 0; i < ucc_dt_size(dtype) * single_rank_count; i++) { sbuf[i] = r; } ctxs[r]->rbuf_size = ucc_dt_size(dtype) * single_rank_count * nprocs; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ctxs[r]->rbuf_size, mem_type)); coll->dst.info.buffer = ctxs[r]->dst_mc_header->addr; if (TEST_INPLACE == inplace) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_IN_PLACE; UCC_CHECK(ucc_mc_memcpy( (void *)((ptrdiff_t)coll->dst.info.buffer + r * single_rank_count * ucc_dt_size(dtype)), ctxs[r]->init_buf, ucc_dt_size(dtype) * single_rank_count, mem_type, UCC_MEMORY_TYPE_HOST)); } else { UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(dtype) * single_rank_count, mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ucc_dt_size(dtype) * single_rank_count, mem_type, UCC_MEMORY_TYPE_HOST)); } if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } } } void data_fini(UccCollCtxVec ctxs) { for (gtest_ucc_coll_ctx_t* ctx : ctxs) { ucc_coll_args_t* coll = ctx->args; if (coll->src.info.buffer) { /* no inplace */ UCC_CHECK(ucc_mc_free(ctx->src_mc_header)); } UCC_CHECK(ucc_mc_free(ctx->dst_mc_header)); ucc_free(ctx->init_buf); free(coll); free(ctx); } ctxs.clear(); } void reset(UccCollCtxVec ctxs) { for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; size_t single_rank_count = coll->dst.info.count / ctxs.size(); ucc_datatype_t dtype = coll->dst.info.datatype; clear_buffer(coll->dst.info.buffer, single_rank_count * ucc_dt_size(dtype) * ctxs.size(), mem_type, 0); if (TEST_INPLACE == inplace) { UCC_CHECK(ucc_mc_memcpy( (void *)((ptrdiff_t)coll->dst.info.buffer + r * single_rank_count * ucc_dt_size(dtype)), ctxs[r]->init_buf, ucc_dt_size(dtype) * single_rank_count, mem_type, UCC_MEMORY_TYPE_HOST)); } } } bool data_validate(UccCollCtxVec ctxs) { bool ret = true; std::vector dsts(ctxs.size()); if (UCC_MEMORY_TYPE_HOST != mem_type) { for (int r = 0; r < ctxs.size(); r++) { dsts[r] = (uint8_t *) ucc_malloc(ctxs[r]->rbuf_size, "dsts buf"); EXPECT_NE(dsts[r], nullptr); UCC_CHECK(ucc_mc_memcpy(dsts[r], ctxs[r]->args->dst.info.buffer, ctxs[r]->rbuf_size, UCC_MEMORY_TYPE_HOST, mem_type)); } } else { for (int r = 0; r < ctxs.size(); r++) { dsts[r] = (uint8_t *)(ctxs[r]->args->dst.info.buffer); } } for (int i = 0; i < ctxs.size(); i++) { uint8_t *rbuf = dsts[i]; for (int r = 0; r < ctxs.size(); r++) { size_t rank_size = ucc_dt_size((ctxs[r])->args->src.info.datatype) * (ctxs[r])->args->src.info.count; for (int j = 0; j < rank_size; j++) { if (r != rbuf[r*rank_size + j]) { ret = false; break; } } } } if (UCC_MEMORY_TYPE_HOST != mem_type) { for (int r = 0; r < ctxs.size(); r++) { ucc_free(dsts[r]); } } return ret; } }; class test_allgather_0 : public test_allgather, public ::testing::WithParamInterface {}; UCC_TEST_P(test_allgather_0, single) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<4>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); UccCollCtxVec ctxs; set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, count, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } UCC_TEST_P(test_allgather_0, single_persistent) { const int team_id = std::get<0>(GetParam()); const ucc_datatype_t dtype = std::get<1>(GetParam()); const ucc_memory_type_t mem_type = std::get<2>(GetParam()); const int count = std::get<3>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<4>(GetParam()); UccTeam_h team = UccJob::getStaticTeams()[team_id]; int size = team->procs.size(); const int n_calls = 3; UccCollCtxVec ctxs; set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, count, ctxs, true); UccReq req(team, ctxs); for (auto i = 0; i < n_calls; i++) { req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); reset(ctxs); } data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( , test_allgather_0, ::testing::Combine( ::testing::Range(1, UccJob::nStaticTeams), // team_ids PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1,3,8192), // count ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE))); class test_allgather_1 : public test_allgather, public ::testing::WithParamInterface {}; UCC_TEST_P(test_allgather_1, multiple_host) { const ucc_datatype_t dtype = std::get<0>(GetParam()); const ucc_memory_type_t mem_type = std::get<1>(GetParam()); const int count = std::get<2>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<3>(GetParam()); std::vector reqs; std::vector ctxs; for (int tid = 0; tid < UccJob::nStaticTeams; tid++) { UccTeam_h team = UccJob::getStaticTeams()[tid]; int size = team->procs.size(); UccCollCtxVec ctx; this->set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(size, dtype, count, ctx, false); reqs.push_back(UccReq(team, ctx)); ctxs.push_back(ctx); } UccReq::startall(reqs); UccReq::waitall(reqs); for (auto ctx : ctxs) { EXPECT_EQ(true, data_validate(ctx)); data_fini(ctx); } } INSTANTIATE_TEST_CASE_P( , test_allgather_1, ::testing::Combine( PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1,3,8192), // count ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE))); class test_allgather_alg : public test_allgather, public ::testing::WithParamInterface {}; UCC_TEST_P(test_allgather_alg, alg) { const ucc_datatype_t dtype = std::get<0>(GetParam()); const ucc_memory_type_t mem_type = std::get<1>(GetParam()); const int count = std::get<2>(GetParam()); const gtest_ucc_inplace_t inplace = std::get<3>(GetParam()); int n_procs = 5; char tune[32]; sprintf(tune, "allgather:@%s:inf", std::get<4>(GetParam()).c_str()); ucc_job_env_t env = {{"UCC_CL_BASIC_TUNE", "inf"}, {"UCC_TL_UCP_TUNE", tune}, {"UCC_TL_UCP_ALLGATHER_BATCHED_NUM_POSTS", "1"}}; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL, env); UccTeam_h team = job.create_team(n_procs); UccCollCtxVec ctxs; set_inplace(inplace); SET_MEM_TYPE(mem_type); data_init(n_procs, dtype, count, ctxs, false); UccReq req(team, ctxs); req.start(); req.wait(); EXPECT_EQ(true, data_validate(ctxs)); data_fini(ctxs); } INSTANTIATE_TEST_CASE_P( , test_allgather_alg, ::testing::Combine( PREDEFINED_DTYPES, #ifdef HAVE_CUDA ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA, UCC_MEMORY_TYPE_CUDA_MANAGED), #else ::testing::Values(UCC_MEMORY_TYPE_HOST), #endif ::testing::Values(1,3,8192), // count ::testing::Values(TEST_INPLACE, TEST_NO_INPLACE), ::testing::Values("knomial", "ring", "neighbor", "bruck", "sparbit", "linear", "batched")), [](const testing::TestParamInfo& info) { std::string name; name += ucc_datatype_str(std::get<0>(info.param)); name += std::string("_") + std::string(ucc_mem_type_str(std::get<1>(info.param))); name += std::string("_count_")+std::to_string(std::get<2>(info.param)); name += std::string("_inplace_")+std::to_string(std::get<3>(info.param)); name += std::string("_")+std::get<4>(info.param); return name; }); ucc-1.8.0/test/gtest/asym_mem/0000775000175000017500000000000015211535620016372 5ustar alastairalastairucc-1.8.0/test/gtest/asym_mem/test_asymmetric_memory.cc0000664000175000017500000004645715211535620023525 0ustar alastairalastair/** * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "common/test_ucc.h" #ifdef HAVE_CUDA using Param = std::tuple; class test_asymmetric_memory : public ucc::test, public ::testing::WithParamInterface { public: UccCollCtxVec ctxs; void data_init(ucc_coll_type_t coll_type, ucc_memory_type_t src_mem_type, ucc_memory_type_t dst_mem_type, UccTeam_h team, bool persistent = false) { ucc_rank_t tsize = team->procs.size(); int root = 0; size_t msglen = 2048; size_t src_modifier = 1; size_t dst_modifier = 1; ctxs.resize(tsize); if (coll_type == UCC_COLL_TYPE_GATHER) { dst_modifier = tsize; } else if (coll_type == UCC_COLL_TYPE_SCATTER) { src_modifier = tsize; } for (int i = 0; i < tsize; i++) { ctxs[i] = (gtest_ucc_coll_ctx_t*) calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ucc_coll_args_t *coll = (ucc_coll_args_t*) calloc(1, sizeof(ucc_coll_args_t)); ctxs[i]->args = coll; coll->coll_type = coll_type; coll->src.info.mem_type = src_mem_type; coll->src.info.count = (ucc_count_t)msglen * src_modifier; coll->src.info.datatype = UCC_DT_INT8; coll->root = root; if (persistent) { coll->mask |= UCC_COLL_ARGS_FIELD_FLAGS; coll->flags |= UCC_COLL_ARGS_FLAG_PERSISTENT; } if (i == root || coll_type != UCC_COLL_TYPE_SCATTER) { UCC_CHECK(ucc_mc_alloc(&ctxs[i]->src_mc_header, msglen * src_modifier, src_mem_type)); coll->src.info.buffer = ctxs[i]->src_mc_header->addr; ctxs[i]->init_buf = ucc_malloc(msglen * src_modifier, "init buf"); EXPECT_NE(ctxs[i]->init_buf, nullptr); uint8_t *sbuf = (uint8_t*)ctxs[i]->init_buf; for (int j = 0; j < msglen * src_modifier; j++) { sbuf[j] = (uint8_t) 1; } UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[i]->init_buf, msglen * src_modifier, src_mem_type, UCC_MEMORY_TYPE_HOST)); } ctxs[i]->rbuf_size = msglen * dst_modifier; if (i == root || coll_type == UCC_COLL_TYPE_SCATTER) { UCC_CHECK(ucc_mc_alloc(&ctxs[i]->dst_mc_header, ctxs[i]->rbuf_size, dst_mem_type)); coll->dst.info.buffer = ctxs[i]->dst_mc_header->addr; coll->dst.info.count = (ucc_count_t)ctxs[i]->rbuf_size; coll->dst.info.datatype = UCC_DT_INT8; coll->dst.info.mem_type = dst_mem_type; } } } void data_fini() { for (int i = 0; i < ctxs.size(); i++) { gtest_ucc_coll_ctx_t *ctx = ctxs[i]; if (!ctx) { continue; } ucc_coll_args_t* coll = ctx->args; if (i == coll->root || coll->coll_type != UCC_COLL_TYPE_SCATTER) { ucc_free(ctx->init_buf); UCC_CHECK(ucc_mc_free(ctx->src_mc_header)); } if (i == coll->root || coll->coll_type == UCC_COLL_TYPE_SCATTER) { UCC_CHECK(ucc_mc_free(ctx->dst_mc_header)); } free(coll); free(ctx); } ctxs.clear(); } bool data_validate(uint8_t data = 1) { bool ret = true; int root = 0; uint8_t result = data; ucc_memory_type_t dst_mem_type; uint8_t *rst; if (ctxs[0]->args->coll_type == UCC_COLL_TYPE_REDUCE) { result *= (uint8_t) ctxs.size(); } for (int i = 0; i < ctxs.size(); i++) { if (!ctxs[i]) { continue; } root = ctxs[i]->args->root; if (i == root || ctxs[i]->args->coll_type == UCC_COLL_TYPE_SCATTER) { dst_mem_type = ctxs[i]->args->dst.info.mem_type; rst = (uint8_t*) ucc_malloc(ctxs[i]->rbuf_size, "validation buf"); EXPECT_NE(rst, nullptr); UCC_CHECK(ucc_mc_memcpy(rst, ctxs[i]->args->dst.info.buffer, ctxs[i]->rbuf_size, UCC_MEMORY_TYPE_HOST, dst_mem_type)); for (int j = 0; j < ctxs[i]->rbuf_size; j++) { if (result != rst[j]) { ret = false; break; } } ucc_free(rst); } } return ret; } void data_update(uint8_t data) { ucc_rank_t tsize = ctxs.size(); size_t msglen = 2048; size_t src_modifier = 1; ucc_coll_type_t coll_type = ctxs[0]->args->coll_type; int root = ctxs[0]->args->root; ucc_memory_type_t src_mem_type = ctxs[0]->args->src.info.mem_type; if (coll_type == UCC_COLL_TYPE_SCATTER) { src_modifier = tsize; } for (int i = 0; i < tsize; i++) { if (i == root || coll_type != UCC_COLL_TYPE_SCATTER) { ucc_coll_args_t *coll = ctxs[i]->args; uint8_t *sbuf = (uint8_t*)ctxs[i]->init_buf; for (int j = 0; j < msglen * src_modifier; j++) { sbuf[j] = (uint8_t) data; } UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[i]->init_buf, msglen * src_modifier, src_mem_type, UCC_MEMORY_TYPE_HOST)); } } } }; class test_asymmetric_memory_v : public ucc::test, public ::testing::WithParamInterface { public: UccCollCtxVec ctxs; void data_init(ucc_coll_type_t coll_type, ucc_memory_type_t src_mem_type, ucc_memory_type_t dst_mem_type, UccTeam_h team) { int nprocs = team->n_procs; size_t count = 2048; ucc_rank_t root = 0; ucc_coll_args_t *coll; int *counts, *displs; size_t my_count, all_counts; ctxs.resize(nprocs); for (auto r = 0; r < nprocs; r++) { coll = (ucc_coll_args_t *)calloc(1, sizeof(ucc_coll_args_t)); my_count = (nprocs - r) * count; ctxs[r] = (gtest_ucc_coll_ctx_t *)calloc(1, sizeof(gtest_ucc_coll_ctx_t)); ctxs[r]->args = coll; coll->mask = 0; coll->flags = 0; coll->coll_type = coll_type; coll->root = root; if (coll_type == UCC_COLL_TYPE_GATHERV) { coll->src.info.mem_type = src_mem_type; coll->src.info.count = (ucc_count_t)my_count; coll->src.info.datatype = UCC_DT_INT8; ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(UCC_DT_INT8) * my_count, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); for (int i = 0; i < my_count * ucc_dt_size(UCC_DT_INT8); i++) { uint8_t *sbuf = (uint8_t *)ctxs[r]->init_buf; sbuf[i] = ((i + r) % 256); } if (r == root) { all_counts = 0; counts = (int*)malloc(sizeof(int) * nprocs); EXPECT_NE(counts, nullptr); displs = (int*)malloc(sizeof(int) * nprocs); EXPECT_NE(displs, nullptr); for (int i = 0; i < nprocs; i++) { counts[i] = (nprocs - i) * count; displs[i] = all_counts; all_counts += counts[i]; } coll->dst.info_v.mem_type = dst_mem_type; coll->dst.info_v.counts = (ucc_count_t *)counts; coll->dst.info_v.displacements = (ucc_aint_t *)displs; coll->dst.info_v.datatype = UCC_DT_INT8; ctxs[r]->rbuf_size = ucc_dt_size(UCC_DT_INT8) * all_counts; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ctxs[r]->rbuf_size, dst_mem_type)); coll->dst.info_v.buffer = ctxs[r]->dst_mc_header->addr; } UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(UCC_DT_INT8) * my_count, src_mem_type)); coll->src.info.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info.buffer, ctxs[r]->init_buf, ucc_dt_size(UCC_DT_INT8) * my_count, src_mem_type, UCC_MEMORY_TYPE_HOST)); } else { // scatterv coll->dst.info.mem_type = dst_mem_type; coll->dst.info.count = (ucc_count_t)my_count; coll->dst.info.datatype = UCC_DT_INT8; if (r == root) { all_counts = 0; counts = (int*)malloc(sizeof(int) * nprocs); EXPECT_NE(counts, nullptr); displs = (int*)malloc(sizeof(int) * nprocs); EXPECT_NE(displs, nullptr); for (int i = 0; i < nprocs; i++) { counts[i] = (nprocs - i) * count; displs[i] = all_counts; all_counts += counts[i]; } ctxs[r]->init_buf = ucc_malloc(ucc_dt_size(UCC_DT_INT8) * all_counts, "init buf"); EXPECT_NE(ctxs[r]->init_buf, nullptr); uint8_t *sbuf = (uint8_t*)ctxs[r]->init_buf; for (int p = 0; p < nprocs; p++) { for (int i = 0; i < ucc_dt_size(UCC_DT_INT8) * counts[p]; i++) { sbuf[(displs[p] * ucc_dt_size(UCC_DT_INT8) + i)] = (uint8_t)((i + p) % 256); } } coll->src.info_v.mem_type = src_mem_type; coll->src.info_v.counts = (ucc_count_t *)counts; coll->src.info_v.displacements = (ucc_aint_t *)displs; coll->src.info_v.datatype = UCC_DT_INT8; UCC_CHECK(ucc_mc_alloc(&ctxs[r]->src_mc_header, ucc_dt_size(UCC_DT_INT8) * all_counts, src_mem_type)); coll->src.info_v.buffer = ctxs[r]->src_mc_header->addr; UCC_CHECK(ucc_mc_memcpy(coll->src.info_v.buffer, ctxs[r]->init_buf, ucc_dt_size(UCC_DT_INT8) * all_counts, src_mem_type, UCC_MEMORY_TYPE_HOST)); } UCC_CHECK(ucc_mc_alloc(&ctxs[r]->dst_mc_header, ucc_dt_size(UCC_DT_INT8) * my_count, dst_mem_type)); coll->dst.info.buffer = ctxs[r]->dst_mc_header->addr; } } } bool data_validate() { bool ret = true; int root = ctxs[0]->args->root; int *displs = (int*)ctxs[root]->args->dst.info_v.displacements; size_t dt_size; ucc_memory_type_t dst_mem_type; ucc_count_t my_count; uint8_t *dsts; if (ctxs[root]->args->coll_type == UCC_COLL_TYPE_GATHERV) { dt_size = ucc_dt_size(ctxs[root]->args->src.info.datatype); dst_mem_type = ctxs[root]->args->dst.info_v.mem_type; if (UCC_MEMORY_TYPE_HOST != dst_mem_type) { dsts = (uint8_t *)ucc_malloc(ctxs[root]->rbuf_size, "dsts buf"); ucc_assert(dsts != nullptr); UCC_CHECK(ucc_mc_memcpy(dsts, ctxs[root]->args->dst.info_v.buffer, ctxs[root]->rbuf_size, UCC_MEMORY_TYPE_HOST, dst_mem_type)); } else { dsts = (uint8_t *)ctxs[root]->args->dst.info_v.buffer; } for (int r = 0; r < ctxs.size(); r++) { my_count = ctxs[r]->args->src.info.count; for (int i = 0; i < my_count * dt_size; i++) { if ((uint8_t)((i + r) % 256) != dsts[(displs[r] * dt_size + i)]) { ret = false; break; } } } if (UCC_MEMORY_TYPE_HOST != dst_mem_type) { ucc_free(dsts); } } else { // scatterv dst_mem_type = ctxs[root]->args->dst.info.mem_type; for (auto r = 0; r < ctxs.size(); r++) { dt_size = ucc_dt_size((ctxs[r])->args->dst.info.datatype); my_count = (ctxs[r])->args->dst.info.count; if (UCC_MEMORY_TYPE_HOST != dst_mem_type) { dsts = (uint8_t *)ucc_malloc(my_count * dt_size, "dsts buf"); ucc_assert(dsts != nullptr); UCC_CHECK(ucc_mc_memcpy(dsts, ctxs[r]->args->dst.info.buffer, my_count * dt_size, UCC_MEMORY_TYPE_HOST, dst_mem_type)); } else { dsts = (uint8_t *)ctxs[r]->args->dst.info.buffer; } for (int i = 0; i < my_count * dt_size; i++) { if ((uint8_t)((i + r) % 256) != dsts[i]) { ret = false; break; } } if (UCC_MEMORY_TYPE_HOST != dst_mem_type) { ucc_free(dsts); if (!ret) { break; } } } } return ret; } void data_fini() { int root = ctxs[0]->args->root; for (auto r = 0; r < ctxs.size(); r++) { ucc_coll_args_t *coll = ctxs[r]->args; if (coll->coll_type == UCC_COLL_TYPE_GATHERV) { if (r == root) { UCC_CHECK(ucc_mc_free(ctxs[r]->dst_mc_header)); free(coll->dst.info_v.counts); free(coll->dst.info_v.displacements); } UCC_CHECK(ucc_mc_free(ctxs[r]->src_mc_header)); } else { // scatterv if (r == root) { UCC_CHECK(ucc_mc_free(ctxs[r]->src_mc_header)); free(coll->src.info_v.counts); free(coll->src.info_v.displacements); } UCC_CHECK(ucc_mc_free(ctxs[r]->dst_mc_header)); } ucc_free(ctxs[r]->init_buf); free(coll); free(ctxs[r]); } ctxs.clear(); } }; #define TEST_ASYM_DECLARE \ const ucc_coll_type_t coll_type = std::get<0>(GetParam()); \ const ucc_memory_type_t src_mem_type = std::get<1>(GetParam()); \ const ucc_memory_type_t dst_mem_type = std::get<2>(GetParam()); \ const int n_procs = std::get<3>(GetParam()); \ \ UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL); \ UccTeam_h team = job.create_team(n_procs); \ \ data_init(coll_type, src_mem_type, dst_mem_type, team); \ UccReq req(team, ctxs); \ if (req.status != UCC_OK) { \ data_fini(); \ GTEST_SKIP() << "ucc_collective_init returned " \ << ucc_status_string(req.status); \ } \ req.start(); \ req.wait(); \ EXPECT_EQ(true, data_validate()); \ data_fini(); UCC_TEST_P(test_asymmetric_memory, single) { TEST_ASYM_DECLARE } UCC_TEST_P(test_asymmetric_memory, persistent) { const ucc_coll_type_t coll_type = std::get<0>(GetParam()); const ucc_memory_type_t src_mem_type = std::get<1>(GetParam()); const ucc_memory_type_t dst_mem_type = std::get<2>(GetParam()); const int n_procs = std::get<3>(GetParam()); int times = 3; UccJob job(n_procs, UccJob::UCC_JOB_CTX_GLOBAL); UccTeam_h team = job.create_team(n_procs); data_init(coll_type, src_mem_type, dst_mem_type, team, /*persistent*/true); UccReq req(team, ctxs); if (req.status != UCC_OK) { data_fini(); GTEST_SKIP() << "ucc_collective_init returned " << ucc_status_string(req.status); } for (; times > 0; times--) { data_update(times); // Set each element in src to times req.start(); req.wait(); EXPECT_EQ(true, data_validate(times)); // Check that the dst was correct based on times } data_fini(); } INSTANTIATE_TEST_CASE_P ( , test_asymmetric_memory, ::testing::Combine ( ::testing::Values(UCC_COLL_TYPE_REDUCE, UCC_COLL_TYPE_GATHER, UCC_COLL_TYPE_SCATTER), // coll type (scatter may be skipped because tl/ucp does not support scatter) ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA), // src mem type ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA), // dst mem type ::testing::Values(8) // n_procs ) ); UCC_TEST_P(test_asymmetric_memory_v, single_v) { TEST_ASYM_DECLARE } INSTANTIATE_TEST_CASE_P ( , test_asymmetric_memory_v, ::testing::Combine ( ::testing::Values(UCC_COLL_TYPE_GATHERV, UCC_COLL_TYPE_SCATTERV), // coll type ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA), // src mem type ::testing::Values(UCC_MEMORY_TYPE_HOST, UCC_MEMORY_TYPE_CUDA), // dst mem type ::testing::Values(8) // n_procs ) ); #endif ucc-1.8.0/test/gtest/tl/0000775000175000017500000000000015211535620015202 5ustar alastairalastairucc-1.8.0/test/gtest/tl/tl_test.cc0000664000175000017500000000034115211535620017165 0ustar alastairalastair/** * Copyright (C) Huawei Technologies Co., Ltd. 2020. All rights reserved. * See file LICENSE for terms. */ #include class test_tl : public ucc::test {}; UCC_TEST_F(test_tl, dummy_test) { } ucc-1.8.0/test/gtest/tl/mlx5/0000775000175000017500000000000015211535620016067 5ustar alastairalastairucc-1.8.0/test/gtest/tl/mlx5/test_tl_mlx5_wqe.h0000664000175000017500000001641415211535620021545 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_tl_mlx5.h" #include "test_tl_mlx5_qps.h" #include "components/tl/mlx5/tl_mlx5_wqe.h" #define DT uint8_t typedef ucc_status_t (*ucc_tl_mlx5_post_rdma_fn_t)( struct ibv_qp *qp, uint32_t qpn, struct ibv_ah *ah, uintptr_t src_mkey_addr, size_t len, uint32_t src_mr_lkey, uintptr_t dst_addr, uint32_t dst_mr_key, int send_flags, uint64_t wr_id); typedef ucc_status_t (*ucc_tl_mlx5_post_transpose_fn_t)( struct ibv_qp *qp, uint32_t src_mr_lkey, uint32_t dst_mr_key, uintptr_t src_mkey_addr, uintptr_t dst_addr, uint32_t element_size, uint16_t ncols, uint16_t nrows, int send_flags); typedef ucc_status_t (*ucc_tl_mlx5_post_wait_on_data_fn_t)(struct ibv_qp *qp, uint64_t value, uint32_t lkey, uintptr_t addr, void *task_ptr); typedef ucc_status_t (*ucc_tl_mlx5_post_umr_fn_t)( struct ibv_qp *qp, struct mlx5dv_mkey *dv_mkey, uint32_t access_flags, uint32_t repeat_count, uint16_t num_entries, struct mlx5dv_mr_interleaved *data, uint32_t ptr_mkey, void *ptr_address); typedef ucc_status_t (*ucc_tl_mlx5_dm_alloc_reg_fn_t)( struct ibv_context *ib_ctx, struct ibv_pd *pd, int dm_host, size_t buf_size, size_t *buf_num_p, struct ibv_dm **ptr, struct ibv_mr **mr, ucc_base_lib_t *lib); // (msgsize) using RdmaWriteParams = int; // (buf_size) using DmParams = int; // (nrows, ncols, element_size) using TransposeParams = std::tuple; // (nbr_srcs, bytes_count, repeat_count, bytes_skip) using UmrParams = std::tuple; // (buffer_size, buffer_nums) using AllocDmParams = std::tuple; // (wait_on_value, init_ctrl_value) using WaitOnDataParams = std::tuple; class test_tl_mlx5_wqe : public test_tl_mlx5_rc_qp { public: ucc_tl_mlx5_post_rdma_fn_t post_rdma_write; ucc_tl_mlx5_post_transpose_fn_t post_transpose; ucc_tl_mlx5_post_wait_on_data_fn_t post_wait_on_data; ucc_tl_mlx5_post_umr_fn_t post_umr; void SetUp() { test_tl_mlx5_rc_qp::SetUp(); CHECK_TEST_STATUS(); post_rdma_write = (ucc_tl_mlx5_post_rdma_fn_t)dlsym( tl_mlx5_so_handle, "ucc_tl_mlx5_post_rdma"); ASSERT_EQ(nullptr, dlerror()); post_transpose = (ucc_tl_mlx5_post_transpose_fn_t)dlsym( tl_mlx5_so_handle, "ucc_tl_mlx5_post_transpose"); ASSERT_EQ(nullptr, dlerror()); post_wait_on_data = (ucc_tl_mlx5_post_wait_on_data_fn_t)dlsym( tl_mlx5_so_handle, "ucc_tl_mlx5_post_wait_on_data"); ASSERT_EQ(nullptr, dlerror()); post_umr = (ucc_tl_mlx5_post_umr_fn_t)dlsym(tl_mlx5_so_handle, "ucc_tl_mlx5_post_umr"); ASSERT_EQ(nullptr, dlerror()); create_qp(); CHECK_TEST_STATUS(); connect_qp_loopback(); CHECK_TEST_STATUS(); create_umr_qp(); } }; class test_tl_mlx5_transpose : public test_tl_mlx5_wqe, public ::testing::WithParamInterface { }; class test_tl_mlx5_wait_on_data : public test_tl_mlx5_wqe, public ::testing::WithParamInterface { }; class test_tl_mlx5_umr_wqe : public test_tl_mlx5_wqe, public ::testing::WithParamInterface { }; class test_tl_mlx5_rdma_write : public test_tl_mlx5_wqe, public ::testing::WithParamInterface { public: DT *src = nullptr; DT *dst = nullptr; struct ibv_mr *src_mr = nullptr; struct ibv_mr *dst_mr = nullptr; int bufsize; void buffers_init() { src = (DT *)malloc(bufsize); GTEST_ASSERT_NE(src, nullptr); dst = (DT *)malloc(bufsize); GTEST_ASSERT_NE(dst, nullptr); for (int i = 0; i < bufsize; i++) { src[i] = i % 256; dst[i] = 0; } src_mr = ibv_reg_mr(pd, src, bufsize, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); GTEST_ASSERT_NE(nullptr, src_mr); dst_mr = ibv_reg_mr(pd, dst, bufsize, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); GTEST_ASSERT_NE(nullptr, dst_mr); } void wait_for_completion() { int completions_num = 0; struct ibv_wc wcs[1]; while (!completions_num) { completions_num = ibv_poll_cq(cq, 1, wcs); } GTEST_ASSERT_EQ(completions_num, 1); GTEST_ASSERT_EQ(wcs[0].status, IBV_WC_SUCCESS); } void validate_buffers() { for (int i = 0; i < bufsize; i++) { GTEST_ASSERT_EQ(src[i], dst[i]); } } void TearDown() { if (src_mr != nullptr) { GTEST_ASSERT_EQ(ibv_dereg_mr(src_mr), UCC_OK); } if (dst_mr != nullptr) { GTEST_ASSERT_EQ(ibv_dereg_mr(dst_mr), UCC_OK); } if (src != nullptr) { free(src); } if (dst != nullptr) { free(dst); } } }; class test_tl_mlx5_dm : public test_tl_mlx5_rdma_write { public: struct ibv_dm *dm_ptr = nullptr; struct ibv_mr *dm_mr = nullptr; struct ibv_alloc_dm_attr dm_attr; void buffers_init() { test_tl_mlx5_rdma_write::buffers_init(); CHECK_TEST_STATUS(); struct ibv_device_attr_ex attr; memset(&attr, 0, sizeof(attr)); GTEST_ASSERT_EQ(ibv_query_device_ex(ctx, NULL, &attr), 0); if (attr.max_dm_size < bufsize) { if (!attr.max_dm_size) { GTEST_SKIP() << "device doesn't support dm allocation"; } else { GTEST_SKIP() << "the requested buffer size (=" << bufsize << ") for device memory should be less than " << attr.max_dm_size; } } memset(&dm_attr, 0, sizeof(dm_attr)); dm_attr.length = bufsize; dm_ptr = ibv_alloc_dm(ctx, &dm_attr); if (!dm_ptr) { GTEST_SKIP() << "device cannot allocate a buffer of size " << bufsize; } dm_mr = ibv_reg_dm_mr(pd, dm_ptr, 0, dm_attr.length, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_ZERO_BASED); GTEST_ASSERT_NE(dm_mr, nullptr); } void TearDown() { if (dm_mr) { ibv_dereg_mr(dm_mr); } if (dm_ptr) { ibv_free_dm(dm_ptr); } test_tl_mlx5_rdma_write::TearDown(); } }; class test_tl_mlx5_dm_alloc_reg : public test_tl_mlx5_wqe, public ::testing::WithParamInterface { public: ucc_tl_mlx5_dm_alloc_reg_fn_t dm_alloc_reg; void SetUp() { test_tl_mlx5_wqe::SetUp(); CHECK_TEST_STATUS(); dm_alloc_reg = (ucc_tl_mlx5_dm_alloc_reg_fn_t)dlsym( tl_mlx5_so_handle, "ucc_tl_mlx5_dm_alloc_reg"); ASSERT_EQ(nullptr, dlerror()); } }; ucc-1.8.0/test/gtest/tl/mlx5/test_tl_mlx5.h0000664000175000017500000000276615211535620020676 0ustar alastairalastair/** * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef TEST_TL_MLX5_H #define TEST_TL_MLX5_H #include #include "common/test_ucc.h" #include "components/tl/mlx5/tl_mlx5.h" #include "components/tl/mlx5/tl_mlx5_dm.h" #include "components/tl/mlx5/tl_mlx5_ib.h" #define CHECK_TEST_STATUS() \ if (Test::HasFatalFailure() || Test::IsSkipped()) { \ return; \ } typedef ucc_status_t (*ucc_tl_mlx5_create_ibv_ctx_fn_t)( char **ib_devname, struct ibv_context **ctx, ucc_base_lib_t *lib); typedef int (*ucc_tl_mlx5_get_active_port_fn_t)(struct ibv_context *ctx); class test_tl_mlx5 : public ucc::test { protected: void *tl_mlx5_so_handle; public: ucc_base_lib_t lib; ucc_tl_mlx5_create_ibv_ctx_fn_t create_ibv_ctx; ucc_tl_mlx5_get_active_port_fn_t get_active_port; struct ibv_port_attr port_attr; struct ibv_context * ctx; struct ibv_pd * pd; struct ibv_cq * cq; int port; test_tl_mlx5(); virtual ~test_tl_mlx5(); virtual void SetUp() override; // Check for Mellanox/NVIDIA vendor ID (0x02c9) and CX7 (MT4129) vendor_part_id bool is_cx7_vendor_id() const { struct ibv_device_attr device_attr; ibv_query_device(ctx, &device_attr); return device_attr.vendor_id == 0x02c9 && device_attr.vendor_part_id == 4129; } }; #endif ucc-1.8.0/test/gtest/tl/mlx5/test_tl_mlx5.cc0000664000175000017500000000342715211535620021027 0ustar alastairalastair/** * Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_tl_mlx5.h" test_tl_mlx5::test_tl_mlx5() { tl_mlx5_so_handle = NULL; ctx = NULL; pd = NULL; cq = NULL; } void test_tl_mlx5::SetUp() { char * devname = NULL; ucc_status_t status; ASSERT_EQ(UCC_OK, ucc_constructor()); ucc_strncpy_safe(lib.log_component.name, "GTEST_MLX5", sizeof(lib.log_component.name)); lib.log_component.log_level = UCC_LOG_LEVEL_ERROR; std::string path = std::string(ucc_global_config.component_path) + "/libucc_tl_mlx5.so"; tl_mlx5_so_handle = dlopen(path.c_str(), RTLD_NOW); if (!tl_mlx5_so_handle) { GTEST_SKIP() << "cannot open ucc_tl_mlx5 library" ; } create_ibv_ctx = (ucc_tl_mlx5_create_ibv_ctx_fn_t)dlsym( tl_mlx5_so_handle, "ucc_tl_mlx5_create_ibv_ctx"); ASSERT_EQ(nullptr, dlerror()); get_active_port = (ucc_tl_mlx5_get_active_port_fn_t)dlsym( tl_mlx5_so_handle, "ucc_tl_mlx5_get_active_port"); ASSERT_EQ(nullptr, dlerror()); status = create_ibv_ctx(&devname, &ctx, &lib); if (UCC_OK != status) { GTEST_SKIP() << "no ib devices"; } port = get_active_port(ctx); ASSERT_GE(port, 0); ASSERT_EQ(ibv_query_port(ctx, port, &port_attr), 0); pd = ibv_alloc_pd(ctx); ASSERT_NE(nullptr, pd); cq = ibv_create_cq(ctx, 8, NULL, NULL, 0); ASSERT_NE(nullptr, cq); } test_tl_mlx5::~test_tl_mlx5() { if (cq) { ibv_destroy_cq(cq); } if (pd) { ibv_dealloc_pd(pd); } if (ctx) { ibv_close_device(ctx); } if (tl_mlx5_so_handle) { dlclose(tl_mlx5_so_handle); } } ucc-1.8.0/test/gtest/tl/mlx5/test_tl_mlx5_wqe.cc0000664000175000017500000003636615211535620021713 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_tl_mlx5_wqe.h" #include "utils/arch/cpu.h" #include #include // Rounds up a given integer to the closet power of two static int roundUpToPowerOfTwo(int a) { int b = 1; while (b < a) { b *= 2; } return b; } // Returns whether a matrix fits ConnectX-7 HW limitations static bool doesMatrixFit(int nrows, int ncols, int elem_size) { // Compute the matrix size as is done by ConnectX-7 HW int matrix_size = nrows * std::max(128, roundUpToPowerOfTwo(ncols) * roundUpToPowerOfTwo(std::max(elem_size, 8))); return matrix_size <= pow(2, 13) //= 8Kb && elem_size <= 128 && nrows <= 64 && ncols <= 64; } UCC_TEST_P(test_tl_mlx5_transpose, transposeWqe) { int nrows = std::get<0>(GetParam()); int ncols = std::get<1>(GetParam()); int elem_size = sizeof(DT) * std::get<2>(GetParam()); int completions_num = 0; DT src[nrows][ncols * elem_size], dst[ncols][nrows * elem_size]; struct ibv_wc wcs[1]; struct ibv_mr *src_mr, *dst_mr; int i, j, k; if (!is_cx7_vendor_id()) { GTEST_SKIP() << "The test needs CX7"; } // Skips if do not match HW limitations if (!doesMatrixFit(nrows, ncols, elem_size)) { GTEST_SKIP(); } for (i = 0; i < nrows; i++) { for (j = 0; j < ncols; j++) { for (k = 0; k < elem_size; k++) { src[i][j * elem_size + k] = (i * nrows * elem_size + j * elem_size + k) % 256; dst[j][i * elem_size + k] = 0; } } } src_mr = ibv_reg_mr(pd, src, nrows * ncols * elem_size, IBV_ACCESS_LOCAL_WRITE); GTEST_ASSERT_NE(nullptr, src_mr); dst_mr = ibv_reg_mr(pd, dst, nrows * ncols * elem_size, IBV_ACCESS_LOCAL_WRITE); GTEST_ASSERT_NE(nullptr, dst_mr); ibv_wr_start(qp.qp_ex); post_transpose(qp.qp, src_mr->lkey, dst_mr->rkey, (uintptr_t)src, (uintptr_t)dst, elem_size, ncols, nrows, IBV_SEND_SIGNALED); GTEST_ASSERT_EQ(ibv_wr_complete(qp.qp_ex), 0); while (!completions_num) { completions_num = ibv_poll_cq(cq, 1, wcs); } GTEST_ASSERT_EQ(completions_num, 1); GTEST_ASSERT_EQ(wcs[0].status, IBV_WC_SUCCESS); for (i = 0; i < nrows; i++) { for (j = 0; j < ncols; j++) { for (k = 0; k < elem_size; k++) { GTEST_ASSERT_EQ(src[i][j * elem_size + k], dst[j][i * elem_size + k]); } } } GTEST_ASSERT_EQ(ibv_dereg_mr(src_mr), UCC_OK); GTEST_ASSERT_EQ(ibv_dereg_mr(dst_mr), UCC_OK); } INSTANTIATE_TEST_SUITE_P(, test_tl_mlx5_transpose, ::testing::Combine(::testing::Values(1, 7, 32, 64), ::testing::Values(1, 5, 32, 64), ::testing::Values(1, 3, 8, 128))); UCC_TEST_P(test_tl_mlx5_rdma_write, RdmaWriteWqe) { struct ibv_sge sg; struct ibv_send_wr wr; bufsize = GetParam(); buffers_init(); CHECK_TEST_STATUS(); memset(&sg, 0, sizeof(sg)); sg.addr = (uintptr_t)src; sg.length = bufsize; sg.lkey = src_mr->lkey; memset(&wr, 0, sizeof(wr)); wr.wr_id = 0; wr.sg_list = &sg; wr.num_sge = 1; wr.opcode = IBV_WR_RDMA_WRITE; wr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_FENCE; wr.next = NULL; wr.wr.rdma.remote_addr = (uintptr_t)dst; wr.wr.rdma.rkey = dst_mr->rkey; // This work request is posted with wr_id = 0 GTEST_ASSERT_EQ(ibv_post_send(qp.qp, &wr, NULL), 0); wait_for_completion(); CHECK_TEST_STATUS(); validate_buffers(); } UCC_TEST_P(test_tl_mlx5_rdma_write, CustomRdmaWriteWqe) { bufsize = GetParam(); buffers_init(); CHECK_TEST_STATUS(); ibv_wr_start(qp.qp_ex); post_rdma_write(qp.qp, qpn, nullptr, (uintptr_t)src, bufsize, src_mr->lkey, (uintptr_t)dst, dst_mr->rkey, IBV_SEND_SIGNALED | IBV_SEND_FENCE, 0); GTEST_ASSERT_EQ(ibv_wr_complete(qp.qp_ex), 0); wait_for_completion(); CHECK_TEST_STATUS(); validate_buffers(); } INSTANTIATE_TEST_SUITE_P(, test_tl_mlx5_rdma_write, ::testing::Values(1, 31, 128, 1024)); UCC_TEST_P(test_tl_mlx5_dm, MemcpyToDeviceMemory) { bufsize = GetParam(); buffers_init(); CHECK_TEST_STATUS(); if (!dm_ptr) { return; } if (bufsize % 4 != 0) { GTEST_SKIP() << "for memcpy involving device memory, buffer size " << "must be a multiple of 4"; } GTEST_ASSERT_EQ(ibv_memcpy_to_dm(dm_ptr, 0, (void *)src, bufsize), 0); GTEST_ASSERT_EQ(ibv_memcpy_from_dm((void *)dst, dm_ptr, 0, bufsize), 0); validate_buffers(); } UCC_TEST_P(test_tl_mlx5_dm, RdmaToDeviceMemory) { struct ibv_sge sg; struct ibv_send_wr wr; bufsize = GetParam(); buffers_init(); CHECK_TEST_STATUS(); if (!dm_ptr) { return; } // RDMA write from host source to device memory memset(&sg, 0, sizeof(sg)); sg.addr = (uintptr_t)src; sg.length = bufsize; sg.lkey = src_mr->lkey; memset(&wr, 0, sizeof(wr)); wr.wr_id = 0; wr.sg_list = &sg; wr.num_sge = 1; wr.opcode = IBV_WR_RDMA_WRITE; wr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_FENCE; wr.next = NULL; wr.wr.rdma.remote_addr = (uintptr_t)0; wr.wr.rdma.rkey = dm_mr->rkey; GTEST_ASSERT_EQ(ibv_post_send(qp.qp, &wr, NULL), 0); wait_for_completion(); CHECK_TEST_STATUS(); // RDMA write from device memory to host destination memset(&sg, 0, sizeof(sg)); sg.addr = (uintptr_t)0; sg.length = bufsize; sg.lkey = dm_mr->lkey; memset(&wr, 0, sizeof(wr)); wr.wr_id = 0; wr.sg_list = &sg; wr.num_sge = 1; wr.opcode = IBV_WR_RDMA_WRITE; wr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_FENCE; wr.next = NULL; wr.wr.rdma.remote_addr = (uintptr_t)dst; wr.wr.rdma.rkey = dst_mr->rkey; GTEST_ASSERT_EQ(ibv_post_send(qp.qp, &wr, NULL), 0); wait_for_completion(); CHECK_TEST_STATUS(); validate_buffers(); } UCC_TEST_P(test_tl_mlx5_dm, CustomRdmaToDeviceMemory) { bufsize = GetParam(); buffers_init(); CHECK_TEST_STATUS(); if (!dm_ptr) { return; } // RDMA write from host source to device memory ibv_wr_start(qp.qp_ex); post_rdma_write(qp.qp, qpn, nullptr, (uintptr_t)src, bufsize, src_mr->lkey, (uintptr_t)0, dm_mr->rkey, IBV_SEND_SIGNALED | IBV_SEND_FENCE, 0); GTEST_ASSERT_EQ(ibv_wr_complete(qp.qp_ex), 0); wait_for_completion(); CHECK_TEST_STATUS(); // RDMA write from device memory to host destination ibv_wr_start(qp.qp_ex); post_rdma_write(qp.qp, qpn, nullptr, (uintptr_t)0, bufsize, dm_mr->lkey, (uintptr_t)dst, dst_mr->rkey, IBV_SEND_SIGNALED | IBV_SEND_FENCE, 0); GTEST_ASSERT_EQ(ibv_wr_complete(qp.qp_ex), 0); wait_for_completion(); CHECK_TEST_STATUS(); validate_buffers(); } INSTANTIATE_TEST_SUITE_P(, test_tl_mlx5_dm, ::testing::Values(1, 12, 31, 32, 8192, 8193, 32768, 65536)); UCC_TEST_P(test_tl_mlx5_wait_on_data, waitOnDataWqe) { uint64_t wait_on_value = std::get<0>(GetParam()); uint64_t init_ctrl_value = std::get<1>(GetParam()); uint64_t buffer[3]; volatile uint64_t *ctrl, *src, *dst; int completions_num; struct ibv_wc wcs[1]; struct ibv_mr * buffer_mr; struct ibv_sge sg; struct ibv_send_wr wr; if (!is_cx7_vendor_id()) { GTEST_SKIP() << "The test needs CX7"; } memset(buffer, 0, 3 * sizeof(uint64_t)); buffer_mr = ibv_reg_mr(pd, buffer, 3 * sizeof(uint64_t), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); GTEST_ASSERT_NE(nullptr, buffer_mr); ctrl = &buffer[0]; src = &buffer[1]; dst = &buffer[2]; *ctrl = init_ctrl_value; memset(&sg, 0, sizeof(sg)); sg.addr = (uintptr_t)src; sg.length = sizeof(uint64_t); sg.lkey = buffer_mr->lkey; memset(&wr, 0, sizeof(wr)); wr.wr_id = 0; wr.sg_list = &sg; wr.num_sge = 1; wr.opcode = IBV_WR_RDMA_WRITE; wr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_FENCE; wr.next = NULL; wr.wr.rdma.remote_addr = (uintptr_t)dst; wr.wr.rdma.rkey = buffer_mr->rkey; // This work request is posted with wr_id = 1 GTEST_ASSERT_EQ(post_wait_on_data(qp.qp, wait_on_value, buffer_mr->lkey, (uintptr_t)ctrl, nullptr), UCC_OK); // This work request is posted with wr_id = 0 GTEST_ASSERT_EQ(ibv_post_send(qp.qp, &wr, NULL), 0); sleep(1); *src = 0xdeadbeef; //memory barrier ucc_memory_cpu_fence(); *ctrl = wait_on_value; while (1) { completions_num = ibv_poll_cq(cq, 1, wcs); if (completions_num != 0) { GTEST_ASSERT_EQ(completions_num, 1); GTEST_ASSERT_EQ(wcs[0].status, IBV_WC_SUCCESS); if (wcs[0].wr_id == 0) { break; } } } //validation GTEST_ASSERT_EQ(*dst, *src); GTEST_ASSERT_EQ(ibv_dereg_mr(buffer_mr), UCC_OK); } INSTANTIATE_TEST_SUITE_P( , test_tl_mlx5_wait_on_data, ::testing::Combine(::testing::Values(1, 1024, 1025, 0xF0F30F00, 0xFFFFFFFF), ::testing::Values(0, 0xF0F30F01))); UCC_TEST_P(test_tl_mlx5_umr_wqe, umrWqe) { uint16_t nbr_srcs = std::get<0>(GetParam()); uint32_t bytes_count = std::get<1>(GetParam()); uint32_t repeat_count = std::get<2>(GetParam()); uint32_t bytes_skip = std::get<3>(GetParam()); int src_size = (bytes_count + bytes_skip) * repeat_count; int dst_size = bytes_count * nbr_srcs * repeat_count; int send_mem_access_flags = 0; void *umr_entries_buf = nullptr; int recv_mem_access_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE; DT src[nbr_srcs][src_size], dst[dst_size]; struct ibv_mr * src_mr[nbr_srcs], *dst_mr, *umr_entries_mr; struct mlx5dv_mkey * umr_mkey; struct mlx5dv_mkey_init_attr umr_mkey_init_attr; size_t umr_buf_size; struct mlx5dv_mr_interleaved mkey_entries[nbr_srcs]; struct ibv_wc wcs[1]; int i, j, completions_num, src_index, repetition_count, offset; // Setup src and dst buffers for (i = 0; i < nbr_srcs; i++) { for (j = 0; j < src_size; j++) { src[i][j] = (i * (src_size + 7) + j) % 255; } src_mr[i] = ibv_reg_mr(pd, src[i], src_size, send_mem_access_flags); GTEST_ASSERT_NE(nullptr, src_mr[i]); } memset(dst, 0, dst_size); dst_mr = ibv_reg_mr(pd, dst, dst_size, recv_mem_access_flags); GTEST_ASSERT_NE(nullptr, dst_mr); // UMR umr_buf_size = ucc_align_up( sizeof(struct mlx5_wqe_umr_repeat_ent_seg) * (nbr_srcs + 1), 64); GTEST_ASSERT_EQ(ucc_posix_memalign(&umr_entries_buf, 2048, umr_buf_size), 0); umr_entries_mr = ibv_reg_mr(pd, umr_entries_buf, umr_buf_size, send_mem_access_flags); GTEST_ASSERT_NE(nullptr, umr_entries_mr); memset(&umr_mkey_init_attr, 0, sizeof(umr_mkey_init_attr)); umr_mkey_init_attr.pd = pd; umr_mkey_init_attr.create_flags = MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT; umr_mkey_init_attr.max_entries = nbr_srcs + 1; //+1 for the "repeat block" umr_mkey = mlx5dv_create_mkey(&umr_mkey_init_attr); GTEST_ASSERT_NE(nullptr, umr_mkey); GTEST_ASSERT_GE(umr_mkey_init_attr.max_entries, nbr_srcs + 1); for (i = 0; i < nbr_srcs; i++) { mkey_entries[i].addr = (uintptr_t)src[i]; mkey_entries[i].bytes_count = bytes_count; mkey_entries[i].bytes_skip = bytes_skip; mkey_entries[i].lkey = src_mr[i]->lkey; } post_umr(umr_qp.qp, umr_mkey, send_mem_access_flags, repeat_count, nbr_srcs, mkey_entries, (uint32_t)umr_entries_mr->lkey, umr_entries_buf); completions_num = 0; while (!completions_num) { completions_num = ibv_poll_cq(cq, 1, wcs); } GTEST_ASSERT_EQ(completions_num, 1); GTEST_ASSERT_EQ(wcs[0].status, IBV_WC_SUCCESS); GTEST_ASSERT_EQ(wcs[0].wr_id, 0); // RDMA Write ibv_wr_start(qp.qp_ex); post_rdma_write(qp.qp, qpn, nullptr, (uintptr_t)0, dst_size, umr_mkey->lkey, (uintptr_t)dst, dst_mr->rkey, IBV_SEND_SIGNALED | IBV_SEND_FENCE, 0); GTEST_ASSERT_EQ(ibv_wr_complete(qp.qp_ex), 0); completions_num = 0; while (!completions_num) { completions_num = ibv_poll_cq(cq, 1, wcs); } GTEST_ASSERT_EQ(completions_num, 1); GTEST_ASSERT_EQ(wcs[0].status, IBV_WC_SUCCESS); GTEST_ASSERT_EQ(wcs[0].wr_id, 0); // Verification for (i = 0; i < dst_size; i++) { src_index = (i / bytes_count) % nbr_srcs; repetition_count = (i / bytes_count) / nbr_srcs; offset = repetition_count * (bytes_count + bytes_skip) + (i % bytes_count); GTEST_ASSERT_EQ(dst[i], src[src_index][offset]); } // Tear down GTEST_ASSERT_EQ(0, mlx5dv_destroy_mkey(umr_mkey)); GTEST_ASSERT_EQ(ibv_dereg_mr(umr_entries_mr), UCC_OK); ucc_free(umr_entries_buf); for (i = 0; i < nbr_srcs; i++) { GTEST_ASSERT_EQ(ibv_dereg_mr(src_mr[i]), UCC_OK); } GTEST_ASSERT_EQ(ibv_dereg_mr(dst_mr), UCC_OK); } INSTANTIATE_TEST_SUITE_P(, test_tl_mlx5_umr_wqe, ::testing::Combine(::testing::Values(1, 129, 1024), ::testing::Values(5, 64), ::testing::Values(1, 3, 16), ::testing::Values(0, 7))); UCC_TEST_P(test_tl_mlx5_dm_alloc_reg, DeviceMemoryAllocation) { size_t buf_size = std::get<0>(GetParam()); size_t buf_num = std::get<1>(GetParam()); struct ibv_dm *ptr = nullptr; struct ibv_mr *mr = nullptr; ucc_status_t status; status = dm_alloc_reg(ctx, pd, 0, buf_size, &buf_num, &ptr, &mr, &lib); if (status == UCC_ERR_NO_MEMORY || status == UCC_ERR_NO_RESOURCE) { GTEST_SKIP() << "cannot allocate " << buf_num << " chunk(s) of size " << buf_size << " in device memory"; } GTEST_ASSERT_EQ(status, UCC_OK); ibv_dereg_mr(mr); ibv_free_dm(ptr); } INSTANTIATE_TEST_SUITE_P( , test_tl_mlx5_dm_alloc_reg, ::testing::Combine(::testing::Values(1, 2, 1024, 8191, 8192, 8193, 32768, 65536, 262144), ::testing::Values(UCC_ULUNITS_AUTO, 1, 3, 8))); ucc-1.8.0/test/gtest/tl/mlx5/test_tl_mlx5_qps.cc0000664000175000017500000000165015211535620021706 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_tl_mlx5_qps.h" UCC_TEST_F(test_tl_mlx5_rc_qp, create) { create_qp(); } UCC_TEST_F(test_tl_mlx5_rc_qp, create_umr) { create_umr_qp(); } UCC_TEST_F(test_tl_mlx5_rc_qp, connect_loopback) { create_qp(); CHECK_TEST_STATUS(); connect_qp_loopback(); } UCC_TEST_F(test_tl_mlx5_dc, init_dct) { ucc_status_t status; status = init_dct(pd, ctx, cq, srq, port, &dct_qp, &dct_qpn, &qp_conf, &lib); GTEST_ASSERT_EQ(UCC_OK, status); } UCC_TEST_F(test_tl_mlx5_dc, init_dci) { ucc_status_t status; status = init_dci(&dci, pd, ctx, cq, port, 4, &qp_conf, &lib); GTEST_ASSERT_EQ(UCC_OK, status); } UCC_TEST_F(test_tl_mlx5_dc, create_ah) { ucc_status_t status; status = create_ah(pd, port_attr.lid, port, &ah, &lib); GTEST_ASSERT_EQ(UCC_OK, status); } ucc-1.8.0/test/gtest/tl/mlx5/test_tl_mlx5_qps.h0000664000175000017500000001256115211535620021553 0ustar alastairalastair/** * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_tl_mlx5.h" class test_tl_mlx5_qp : public test_tl_mlx5 { public: ucc_tl_mlx5_ib_qp_conf_t qp_conf; virtual void SetUp() { test_tl_mlx5::SetUp(); CHECK_TEST_STATUS(); qp_conf.qp_rnr_retry = 7; qp_conf.qp_rnr_timer = 20; qp_conf.qp_retry_cnt = 7; qp_conf.qp_timeout = 18; qp_conf.qp_max_atomic = 1; qp_conf.qp_sl = 1; } }; typedef ucc_status_t (*ucc_tl_mlx5_create_umr_qp_fn_t)( struct ibv_context *ctx, struct ibv_pd *pd, struct ibv_cq *cq, int ib_port, struct ibv_qp **qp, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib); typedef ucc_status_t (*ucc_tl_mlx5_qp_connect_fn_t)( struct ibv_qp *qp, uint32_t qp_num, uint16_t lid, int port, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib); typedef ucc_status_t (*ucc_tl_mlx5_create_rc_qp_fn_t)( struct ibv_context *ctx, struct ibv_pd *pd, struct ibv_cq *cq, int tx_depth, ucc_tl_mlx5_qp_t *qp, uint32_t *qpn, ucc_base_lib_t *lib); class test_tl_mlx5_rc_qp : public test_tl_mlx5_qp { public: ucc_tl_mlx5_qp_t qp = {}; ucc_tl_mlx5_qp_t umr_qp = {}; uint32_t qpn; ucc_tl_mlx5_ib_qp_conf_t umr_qp_conf; int tx_depth; ucc_tl_mlx5_create_rc_qp_fn_t create_rc_qp; ucc_tl_mlx5_qp_connect_fn_t qp_connect; ucc_tl_mlx5_create_umr_qp_fn_t create_rc_umr_qp; virtual void SetUp() { qp.qp = NULL; umr_qp.qp = NULL; tx_depth = 4; test_tl_mlx5_qp::SetUp(); CHECK_TEST_STATUS(); umr_qp_conf = qp_conf; create_rc_qp = (ucc_tl_mlx5_create_rc_qp_fn_t)dlsym( tl_mlx5_so_handle, "ucc_tl_mlx5_create_rc_qp"); ASSERT_EQ(nullptr, dlerror()); qp_connect = (ucc_tl_mlx5_qp_connect_fn_t)dlsym( tl_mlx5_so_handle, "ucc_tl_mlx5_qp_connect"); ASSERT_EQ(nullptr, dlerror()); create_rc_umr_qp = (ucc_tl_mlx5_create_umr_qp_fn_t)dlsym( tl_mlx5_so_handle, "ucc_tl_mlx5_create_umr_qp"); ASSERT_EQ(nullptr, dlerror()); } ~test_tl_mlx5_rc_qp() { if (qp.qp) { ibv_destroy_qp(qp.qp); } if (umr_qp.qp) { ibv_destroy_qp(umr_qp.qp); } } void create_qp() { GTEST_ASSERT_EQ(create_rc_qp(ctx, pd, cq, tx_depth, &qp, &qpn, &lib), UCC_OK); } void create_umr_qp() { GTEST_ASSERT_EQ( create_rc_umr_qp(ctx, pd, cq, port, &umr_qp.qp, &umr_qp_conf, &lib), UCC_OK); } void connect_qp_loopback() { GTEST_ASSERT_EQ( qp_connect(qp.qp, qpn, port_attr.lid, port, &qp_conf, &lib), UCC_OK); }; }; typedef ucc_status_t (*ucc_tl_mlx5_init_dct_fn_t)( struct ibv_pd *pd, struct ibv_context *ctx, struct ibv_cq *cq, struct ibv_srq *srq, uint8_t port_num, struct ibv_qp **dct_qp, uint32_t *qpn, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib); typedef ucc_status_t (*ucc_tl_mlx5_init_dci_fn_t)( ucc_tl_mlx5_dci_t *dci, struct ibv_pd *pd, struct ibv_context *ctx, struct ibv_cq *cq, uint8_t port_num, int tx_depth, ucc_tl_mlx5_ib_qp_conf_t *qp_conf, ucc_base_lib_t *lib); typedef ucc_status_t (*ucc_tl_mlx5_create_ah_fn_t)(struct ibv_pd * pd, uint16_t lid, uint8_t port_num, struct ibv_ah **ah_ptr, ucc_base_lib_t *lib); class test_tl_mlx5_dc : public test_tl_mlx5_qp { public: struct ibv_qp *dct_qp = nullptr; struct ibv_ah *ah = nullptr; struct ibv_srq *srq = nullptr; ucc_tl_mlx5_dci_t dci = {}; uint32_t dct_qpn; ucc_tl_mlx5_init_dct_fn_t init_dct; ucc_tl_mlx5_init_dci_fn_t init_dci; ucc_tl_mlx5_create_ah_fn_t create_ah; virtual void SetUp() { struct ibv_srq_init_attr srq_attr; test_tl_mlx5_qp::SetUp(); CHECK_TEST_STATUS(); init_dct = (ucc_tl_mlx5_init_dct_fn_t)dlsym(tl_mlx5_so_handle, "ucc_tl_mlx5_init_dct"); ASSERT_EQ(nullptr, dlerror()); init_dci = (ucc_tl_mlx5_init_dci_fn_t)dlsym(tl_mlx5_so_handle, "ucc_tl_mlx5_init_dci"); ASSERT_EQ(nullptr, dlerror()); create_ah = (ucc_tl_mlx5_create_ah_fn_t)dlsym(tl_mlx5_so_handle, "ucc_tl_mlx5_create_ah"); ASSERT_EQ(nullptr, dlerror()); memset(&srq_attr, 0, sizeof(struct ibv_srq_init_attr)); srq_attr.attr.max_wr = 1; srq_attr.attr.max_sge = 1; srq = ibv_create_srq(pd, &srq_attr); EXPECT_NE(nullptr, srq); dct_qp = NULL; } ~test_tl_mlx5_dc() { if (ah) { ibv_destroy_ah(ah); } if (dct_qp) { ibv_destroy_qp(dct_qp); } if (dci.dci_qp) { ibv_destroy_qp(dci.dci_qp); } if (srq) { ibv_destroy_srq(srq); } } }; ucc-1.8.0/test/gtest/utils/0000775000175000017500000000000015211535620015723 5ustar alastairalastairucc-1.8.0/test/gtest/utils/ucc_test.conf0000664000175000017500000000032515211535620020403 0ustar alastairalastair# Some random comment UCC_FOO = 10 ## Tests setting global variable GTEST_UCC_CFG_BAR = 20 ;; test setting var with ENV_PREFIX = GTEST #variable with undefined ENV_PREFIX - will be ignored GGGTEST_UCC_BOO = 30 ucc-1.8.0/test/gtest/utils/test_lock_free_queue.cc0000664000175000017500000000622615211535620022434 0ustar alastairalastair /** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ extern "C" { #include "utils/ucc_lock_free_queue.h" #include "utils/ucc_atomic.h" #include "utils/ucc_malloc.h" #include #include } #include #include #define NUM_ITERS 5000000 typedef struct ucc_test_queue { ucc_lf_queue_t lf_queue; int64_t test_sum; uint32_t elems_num; uint32_t active_producers_threads; uint32_t memory_err; } ucc_test_queue_t; void *producer_thread(void *arg) { ucc_test_queue_t *test = (ucc_test_queue_t *)arg; for (int j = 0; j < NUM_ITERS; j++) { ucc_lf_queue_elem_t *elem = (ucc_lf_queue_elem_t *)ucc_malloc(sizeof(ucc_lf_queue_elem_t)); ucc_lf_queue_init_elem(elem); if (!elem) { ucc_atomic_add32(&test->memory_err, 1); goto exit; } ucc_lf_queue_enqueue(&test->lf_queue, elem); ucc_atomic_add64((uint64_t *)&test->test_sum, (uint64_t)elem); ucc_atomic_add32(&test->elems_num, 1); } exit: ucc_atomic_sub32(&test->active_producers_threads,1); return 0; } void *consumer_thread(void *arg) { ucc_test_queue_t *test = (ucc_test_queue_t *)arg; while(test->active_producers_threads || test->elems_num){ ucc_lf_queue_elem_t *elem = ucc_lf_queue_dequeue(&test->lf_queue, 1); if (elem) { ucc_atomic_sub64((uint64_t *)&test->test_sum, (uint64_t)elem); ucc_atomic_sub32(&test->elems_num, 1); ucc_free(elem); } } return 0; } class test_lf_queue : public ucc::test { public: ucc_test_queue_t test; int i; std::vector producers_threads; std::vector consumers_threads; int lf_test(int num_of_producers, int num_of_consumers); }; int test_lf_queue::lf_test(int num_of_producers, int num_of_consumers){ producers_threads.resize(num_of_producers); consumers_threads.resize(num_of_consumers); memset(&test, 0, sizeof(ucc_test_queue_t)); ucc_lf_queue_init(&test.lf_queue); for (i = 0; i < num_of_producers; i++) { ucc_atomic_add32(&test.active_producers_threads, 1); pthread_create(&producers_threads[i], NULL, &producer_thread, (void *)&test); } for (i = 0; i < num_of_consumers; i++) { pthread_create(&consumers_threads[i], NULL, &consumer_thread, (void *)&test); } for (i = 0; i < num_of_producers; i++) { pthread_join(producers_threads[i], NULL); } for (i = 0; i < num_of_consumers; i++) { pthread_join(consumers_threads[i], NULL); } ucc_lf_queue_destroy(&test.lf_queue); if (test.memory_err) { return 1; } if (test.test_sum) { return 1; } return 0; } UCC_TEST_F(test_lf_queue, oneProducerOneConsumer) { EXPECT_EQ(lf_test(1, 1), 0); } UCC_TEST_F(test_lf_queue, oneProducerManyConsumers) { EXPECT_EQ(lf_test(1, 7), 0); } UCC_TEST_F(test_lf_queue, manyProducersManyConsumers) { EXPECT_EQ(lf_test(7, 7), 0); } ucc-1.8.0/test/gtest/utils/test_math.cc0000664000175000017500000000233415211535620020224 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ extern "C" { #include "utils/ucc_math.h" } #include using floatParams = float; class test_floats_cast : public ucc::test, public ::testing::WithParamInterface { }; UCC_TEST_P(test_floats_cast, start_with_float) { float p = GetParam(); uint16_t bfloat16Val; float32tobfloat16(p, &bfloat16Val); // All values in test_floats_cast use only first 16 bits of the 32. rest are 0. EXPECT_EQ(p, bfloat16tofloat32(&bfloat16Val)); } INSTANTIATE_TEST_CASE_P(, test_floats_cast, ::testing::Values(-4.26941244675e+18, -6.95441104568e+13, 2.015625)); using bfloat16Params = uint16_t; class test_bfloats16_cast : public ucc::test, public ::testing::WithParamInterface { }; UCC_TEST_P(test_bfloats16_cast, start_with_bfloat16) { uint16_t p = GetParam(); uint16_t res; float32tobfloat16(bfloat16tofloat32(&p), &res); EXPECT_EQ(p, res); } INSTANTIATE_TEST_CASE_P(, test_bfloats16_cast, ::testing::Values(31000, 400, 17, 13569, 0)); ucc-1.8.0/test/gtest/utils/test_parser.cc0000664000175000017500000000460615211535620020573 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ extern "C" { #include "utils/ucc_parser.h" #include "utils/ucc_datastruct.h" } #include #include class test_parse_mrange : public ucc::test { public: ucc_mrange_uint_t *p; test_parse_mrange() { p = (ucc_mrange_uint_t *) ucc_malloc(sizeof(ucc_mrange_uint_t)); } ~test_parse_mrange() { ucc_free(p); } }; UCC_TEST_F(test_parse_mrange, check_valid) { std::string str = "0-4K:host:8,auto"; size_t msgsize1 = 1024, msgsize2 = 8192; EXPECT_EQ(1, ucc_config_sscanf_uint_ranged(str.c_str(), p, NULL)); EXPECT_EQ(8, ucc_mrange_uint_get(p, msgsize1, UCC_MEMORY_TYPE_HOST)); EXPECT_EQ(UCC_UUNITS_AUTO, ucc_mrange_uint_get(p, msgsize2, UCC_MEMORY_TYPE_HOST)); ucc_mrange_uint_destroy(p); } UCC_TEST_F(test_parse_mrange, check_invalid) { std::string str = "0-4K:host:8:8"; EXPECT_EQ(0, ucc_config_sscanf_uint_ranged(str.c_str(), p, NULL)); ucc_mrange_uint_destroy(p); str = "0-4K:host:a"; EXPECT_EQ(0, ucc_config_sscanf_uint_ranged(str.c_str(), p, NULL)); ucc_mrange_uint_destroy(p); str = "0-4K:gpu:8"; EXPECT_EQ(0, ucc_config_sscanf_uint_ranged(str.c_str(), p, NULL)); ucc_mrange_uint_destroy(p); str = "0-f:host:8"; EXPECT_EQ(0, ucc_config_sscanf_uint_ranged(str.c_str(), p, NULL)); ucc_mrange_uint_destroy(p); } UCC_TEST_F(test_parse_mrange, check_range_multiple) { std::string str = "0-4K:host:8,4k-inf:host:10,0-4k:cuda:7,0-4k:cuda_managed:6,auto"; size_t msgsize1 = 1024, msgsize2 = 8192; EXPECT_EQ(1, ucc_config_sscanf_uint_ranged(str.c_str(), p, NULL)); EXPECT_EQ(8, ucc_mrange_uint_get(p, msgsize1, UCC_MEMORY_TYPE_HOST)); EXPECT_EQ(10, ucc_mrange_uint_get(p, msgsize2, UCC_MEMORY_TYPE_HOST)); EXPECT_EQ(7, ucc_mrange_uint_get(p, msgsize1, UCC_MEMORY_TYPE_CUDA)); EXPECT_EQ(UCC_UUNITS_AUTO, ucc_mrange_uint_get(p, msgsize2, UCC_MEMORY_TYPE_CUDA)); EXPECT_EQ(6, ucc_mrange_uint_get(p, msgsize1, UCC_MEMORY_TYPE_CUDA_MANAGED)); EXPECT_EQ(UCC_UUNITS_AUTO, ucc_mrange_uint_get(p, msgsize2, UCC_MEMORY_TYPE_CUDA_MANAGED)); ucc_mrange_uint_destroy(p); } ucc-1.8.0/test/gtest/utils/test_ep_map.cc0000664000175000017500000001063715211535620020541 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ extern "C" { #include "utils/ucc_coll_utils.h" } #include #include #include class EpMap { public: ucc_ep_map_t map; ucc_rank_t * array; EpMap(){}; EpMap(ucc_ep_map_t _map) { array = NULL; map = _map; }; EpMap(uint64_t full, bool reverse = false) { array = NULL; if (reverse) { map = ucc_ep_map_create_reverse(full); } else { map.type = UCC_EP_MAP_FULL; map.ep_num = full; } }; EpMap(uint64_t start, int64_t stride, uint64_t num, uint64_t full) { map.type = ((num == full) && (stride == 1)) ? UCC_EP_MAP_FULL : UCC_EP_MAP_STRIDED; map.ep_num = num; map.strided.start = start; map.strided.stride = stride; array = NULL; }; EpMap(const std::vector &ranks, uint64_t full, int need_free = 0) { array = (ucc_rank_t*)malloc(sizeof(ucc_rank_t) * ranks.size()); memcpy(array, ranks.data(), ranks.size() * sizeof(ucc_rank_t)); map = ucc_ep_map_from_array(&array, ranks.size(), full, need_free); } ~EpMap() { if (array) { free(array); } } friend bool operator==(const EpMap &lhs, const EpMap &rhs) { if ((lhs.map.type != rhs.map.type) || (lhs.map.ep_num != rhs.map.ep_num)) { return false; } switch(lhs.map.type) { case UCC_EP_MAP_FULL: return true; case UCC_EP_MAP_STRIDED: return (lhs.map.strided.start == rhs.map.strided.start) && (lhs.map.strided.stride == rhs.map.strided.stride); default: break; } return false; } }; class test_ep_map : public ucc::test {}; UCC_TEST_F(test_ep_map, from_array) { // Full contiguous map EXPECT_EQ(EpMap({1,2,3,4,5,6,7,8,9,10}, 10), EpMap(10)); // Strided contiguous map EXPECT_EQ(EpMap({1,31,61,91,121}, 150), EpMap(1, 30, 5, 150)); // Strided negative EXPECT_EQ(EpMap({100,90,80,70,60}, 150), EpMap(100, -10, 5, 150)); } UCC_TEST_F(test_ep_map, from_array_free) { /* strided pattern not found - array is not released */ EXPECT_NE((void*)NULL, EpMap({1, 5, 6, 8, 11}, 10, 1).array); /* strided pattern found - array is released */ EXPECT_EQ((void*)NULL, EpMap({2, 4, 6, 8, 10}, 10, 1).array); /* FULL pattern found - array is released */ EXPECT_EQ((void*)NULL, EpMap({1, 2, 3, 4, 5}, 5, 1).array); } UCC_TEST_F(test_ep_map, reverse) { const int size = 10; ucc_ep_map_t map = ucc_ep_map_create_reverse(size); for (int i = 0; i < size; i++) { EXPECT_EQ(size - 1 - i, ucc_ep_map_eval(map, i)); } } UCC_TEST_F(test_ep_map, nested) { auto map1 = EpMap(100); //full map, size 100 auto map2 = EpMap(0, 2, 50, 100); // submap even only auto map3 = EpMap(1, 2, 25, 50); // submap odd only from 50 ucc_ep_map_t nested1, nested2; EXPECT_EQ(UCC_OK, ucc_ep_map_create_nested(&map1.map, &map2.map, &nested1)); EXPECT_EQ(50, nested1.ep_num); for (int i = 0; i < nested1.ep_num; i++) { EXPECT_EQ(0 + i * 2, ucc_ep_map_eval(nested1, i)); } EXPECT_EQ(UCC_OK, ucc_ep_map_create_nested(&nested1, &map3.map, &nested2)); EXPECT_EQ(25, nested2.ep_num); for (int i = 0; i < nested2.ep_num; i++) { EXPECT_EQ(2 + i * 4, ucc_ep_map_eval(nested2, i)); } ucc_ep_map_destroy_nested(&nested1); ucc_ep_map_destroy_nested(&nested2); } class test_ep_map_inv : public test_ep_map { public: void check_inv(EpMap map) { ucc_ep_map_t inv; EXPECT_EQ(UCC_OK, ucc_ep_map_create_inverse(map.map, &inv, 0)); for (int i = 0; i < map.map.ep_num; i++) { EXPECT_EQ(i, ucc_ep_map_eval(inv, ucc_ep_map_eval(map.map, i))); } ucc_ep_map_destroy(&inv); }; }; UCC_TEST_F(test_ep_map_inv, contig) { /* reverse of FULL */ check_inv(EpMap(10)); /* reverse of INVERSE */ check_inv(EpMap(10, true)); } UCC_TEST_F(test_ep_map_inv, strided) { /* stride positive */ check_inv(EpMap(1, 30, 5, 150)); /* stride negative */ check_inv(EpMap(100, -10, 5, 150)); } UCC_TEST_F(test_ep_map_inv, random) { check_inv(EpMap({4, 0, 1, 2, 3}, 5)); } ucc-1.8.0/test/gtest/utils/test_string.cc0000664000175000017500000000543415211535620020605 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ extern "C" { #include "utils/ucc_string.h" #include "utils/ucc_malloc.h" } #include #include #include using Param = std::vector; class test_string : public ucc::test, public ::testing::WithParamInterface { public: std::string str; void make_str(Param &p); }; void test_string::make_str(Param &p) { unsigned i; str = ""; for (i = 1; i < p.size() - 1; i++) { str += p[i] + p[0]; } str += p[i]; } UCC_TEST_P(test_string, split) { auto p = GetParam(); const char *delim = p[0].c_str(); make_str(p); char **split = ucc_str_split(str.c_str(), delim); EXPECT_NE(nullptr, split); EXPECT_EQ(ucc_str_split_count(split), p.size() - 1); for (auto i = 0; i < ucc_str_split_count(split); i++) { EXPECT_EQ(p[i + 1], split[i]); } ucc_str_split_free(split); } INSTANTIATE_TEST_CASE_P( , test_string, ::testing::Values(std::vector({",", "aaa", "bbb", "ccc"}), std::vector({":", "a", "b", "c", "d", "e"}), std::vector({" ", "a", "bb"}), std::vector({"...", "aaaaaa", "b", "ccccc"}))); UCC_TEST_F(test_string, split_no_delim) { std::string delim = ":"; std::string s = "string with no delimiter"; char ** split = ucc_str_split(s.c_str(), delim.c_str()); EXPECT_NE(nullptr, split); EXPECT_EQ(ucc_str_split_count(split), 1); EXPECT_EQ(s, split[0]); ucc_str_split_free(split); } UCC_TEST_F(test_string, find_last) { const char* str1 = "aaa bbb aaa ccc aaa ddd"; const char* str2 = "/tmp/build/ucc_tl_ucp/install/lib/ucc_tl_ucp.so"; const char *s; s = ucc_strstr_last(str1, "aaa"); EXPECT_EQ(16, (ptrdiff_t)s - (ptrdiff_t)str1); s = ucc_strstr_last(str2, "ucc_tl_ucp"); EXPECT_EQ(strlen(str2) - strlen("ucc_tl_ucp.so"), (ptrdiff_t)s - (ptrdiff_t)str2); s = ucc_strstr_last(str1, "fff"); EXPECT_EQ(NULL, s); s = ucc_strstr_last(str2, "/tmp"); EXPECT_EQ(str2, s); } UCC_TEST_F(test_string, concat) { char *rst; EXPECT_EQ(UCC_OK, ucc_str_concat("aaa", "bbb", &rst)); EXPECT_EQ("aaabbb", std::string(rst)); ucc_free(rst); EXPECT_EQ(UCC_OK, ucc_str_concat("aaabbbccc", "d", &rst)); EXPECT_EQ("aaabbbcccd", std::string(rst)); ucc_free(rst); EXPECT_EQ(UCC_OK, ucc_str_concat("aaa", "", &rst)); EXPECT_EQ("aaa", std::string(rst)); ucc_free(rst); EXPECT_EQ(UCC_OK, ucc_str_concat("", "aaa", &rst)); EXPECT_EQ("aaa", std::string(rst)); ucc_free(rst); } ucc-1.8.0/test/gtest/utils/test_cfg_file.cc0000664000175000017500000000633015211535620021031 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * * See file LICENSE for terms. */ extern "C" { #include "utils/ucc_parser.h" #include "core/ucc_global_opts.h" } #include typedef struct ucc_gtest_config_base { int foo; } ucc_gtest_config_base_t; typedef struct ucc_gtest_config { ucc_gtest_config_base_t super; int bar; int boo; } ucc_gtest_config_t; static ucc_config_field_t ucc_gtest_config_table_base[] = { {"FOO", "1", "gtest base config variable", ucc_offsetof(ucc_gtest_config_base_t, foo), UCC_CONFIG_TYPE_INT}, {NULL} }; static ucc_config_field_t ucc_gtest_config_table[] = { {"", "", NULL, ucc_offsetof(ucc_gtest_config_t, super), UCC_CONFIG_TYPE_TABLE(ucc_gtest_config_table_base)}, {"BAR", "1", "gtest config variable", ucc_offsetof(ucc_gtest_config_t, bar), UCC_CONFIG_TYPE_INT}, {"BOO", "1", "gtest config variable", ucc_offsetof(ucc_gtest_config_t, boo), UCC_CONFIG_TYPE_INT}, {NULL} }; UCC_CONFIG_DECLARE_TABLE(ucc_gtest_config_table, "GTEST TABLE", "CFG_", ucc_gtest_config_t); class test_cfg_file : public ucc::test { public: ucc_file_config_t *file_cfg; ucc_gtest_config_t cfg; std::string test_dir; test_cfg_file() { file_cfg = NULL; test_dir = std::string(GTEST_UCC_TOP_SRCDIR) + "/test/gtest/utils/"; } ~test_cfg_file() { if (file_cfg) { ucc_release_file_config(file_cfg); } ucc_config_parser_release_opts(&cfg, ucc_gtest_config_table); } void init_cfg() { ucc_status_t status; std::swap(ucc_global_config.file_cfg, file_cfg); status = ucc_config_parser_fill_opts( &cfg, UCC_CONFIG_GET_TABLE(ucc_gtest_config_table), "GTEST_UCC_", 1); std::swap(ucc_global_config.file_cfg, file_cfg); EXPECT_EQ(UCC_OK, status); }; }; UCC_TEST_F(test_cfg_file, parse_existing) { std::string filename = test_dir + "ucc_test.conf"; EXPECT_EQ(UCC_OK, ucc_parse_file_config(filename.c_str(), &file_cfg)); } UCC_TEST_F(test_cfg_file, parse_non_existing) { std::string filename = test_dir + "ucc_test_nonexisting.conf"; EXPECT_EQ(UCC_ERR_NOT_FOUND, ucc_parse_file_config(filename.c_str(), &file_cfg)); } /* Checks options are applied from cfg file */ UCC_TEST_F(test_cfg_file, opts_applied) { std::string filename = test_dir + "ucc_test.conf"; EXPECT_EQ(UCC_OK, ucc_parse_file_config(filename.c_str(), &file_cfg)); init_cfg(); EXPECT_EQ(10, cfg.super.foo); EXPECT_EQ(20, cfg.bar); EXPECT_EQ(1, cfg.boo); } /* Checks that options set via env var have preference over cfg file */ UCC_TEST_F(test_cfg_file, env_preference) { std::string filename = test_dir + "ucc_test.conf"; setenv("GTEST_UCC_CFG_BAR", "123", 1); EXPECT_EQ(UCC_OK, ucc_parse_file_config(filename.c_str(), &file_cfg)); init_cfg(); unsetenv("GTEST_UCC_CFG_BAR"); EXPECT_EQ(10, cfg.super.foo); /* Expected value is 123 from env rather than 20 from file */ EXPECT_EQ(123, cfg.bar); EXPECT_EQ(1, cfg.boo); } ucc-1.8.0/test/gtest/Makefile.am0000664000175000017500000001414215211535620016621 0ustar alastairalastair# # Copyright (c) 2001-2014, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Copyright (C) UT-Battelle, LLC. 2015. ALL RIGHTS RESERVED. # Copyright (C) The University of Tennessee and the University of Tennessee Research Foundation. 2016. ALL RIGHTS RESERVED. # Copyright (C) Los Alamos National Security, LLC. 2018 ALL RIGHTS RESERVED. # Copyright (C) Advanced Micro Devices, Inc. 2019. ALL RIGHTS RESERVED. # Copyright (C) Huawei Technologies Co., Ltd. 2020. All rights reserved. # # See file LICENSE for terms. # if HAVE_GTEST # Set default configuration for running tests UCC_HANDLE_ERRORS ?= freeze UCC_LOG_LEVEL ?= warn UCC_LOG_PRINT_ENABLE ?= y GTEST_FILTER ?= * GTEST_EXTRA_ARGS ?= LAUNCHER ?= VALGRIND_EXTRA_ARGS ?= export UCC_HANDLE_ERRORS export UCC_LOG_LEVEL export UCC_LOG_PRINT_ENABLE GTEST_ARGS = \ --gtest_filter=$(GTEST_FILTER) \ $(GTEST_EXTRA_ARGS) VALGRIND_ARGS = \ --tool=memcheck \ --leak-check=full \ --track-origins=yes \ --fair-sched=try \ --num-callers=25 \ --error-exitcode=1 \ --child-silent-after-fork=yes \ --suppressions=$(top_srcdir)/contrib/valgrind.supp \ $(VALGRIND_EXTRA_ARGS) bin_PROGRAMS = gtest gtestdir = $(includedir) gtest_LDADD = \ $(top_builddir)/src/libucc.la \ $(GTEST_LIBS) gtest_CPPFLAGS = \ $(BASE_CPPFLAGS) \ -I$(top_srcdir)/src \ -I$(top_srcdir)/test \ -I$(top_builddir)/src \ -I$(top_srcdir)/test/gtest \ $(GTEST_CPPFLAGS) gtest_LDFLAGS = $(GTEST_LDFLAGS) -pthread -no-install -Wl,-dynamic-list-data \ -Wl,--rpath-link=${UCS_LIBDIR} gtest_CFLAGS = $(BASE_CFLAGS) $(AM_CPPFLAGS) gtest_CXXFLAGS = -std=gnu++11 \ $(BASE_CXXFLAGS) $(GTEST_CXXFLAGS) \ -DGTEST_UCM_HOOK_LIB_DIR="\"${abs_builddir}/ucm/test_dlopen/.libs\"" \ -DGTEST_UCC_TOP_SRCDIR="\"${UCC_TOP_SRCDIR}\"" gtest_SOURCES = \ common/gtest-all.cc \ common/test_obj_size.cc \ common/main.cc \ common/test_ucc.cc \ tl/tl_test.cc \ core/test_lib_config.cc \ core/test_lib.cc \ core/test_context_config.cc \ core/test_context.cc \ core/test_mc.cc \ core/test_mc_reduce.cc \ core/test_team.cc \ core/test_schedule.cc \ core/test_topo.cc \ core/test_service_coll.cc \ core/test_timeout.cc \ core/test_utils.cc \ core/test_mem_map.cc \ coll/test_barrier.cc \ coll/test_alltoall.cc \ coll/test_alltoallv.cc \ coll/test_allgather.cc \ coll/test_allgatherv.cc \ coll/test_gather.cc \ coll/test_gatherv.cc \ coll/test_bcast.cc \ coll/test_reduce.cc \ coll/test_allreduce_sliding_window.cc \ coll/test_allreduce.cc \ coll/test_reduce_scatter.cc \ coll/test_reduce_scatterv.cc \ coll/test_scatter.cc \ coll/test_scatterv.cc \ utils/test_string.cc \ utils/test_ep_map.cc \ utils/test_lock_free_queue.cc \ utils/test_math.cc \ utils/test_cfg_file.cc \ utils/test_parser.cc \ coll_score/test_score.cc \ coll_score/test_score_str.cc \ coll_score/test_score_update.cc \ active_set/test_active_set.cc \ asym_mem/test_asymmetric_memory.cc if TL_MLX5_ENABLED gtest_SOURCES += tl/mlx5/test_tl_mlx5.cc \ tl/mlx5/test_tl_mlx5_qps.cc\ tl/mlx5/test_tl_mlx5_wqe.cc gtest_LDADD += $(IBVERBS_LIBADD) $(MLX5DV_LIBADD) endif if HAVE_CUDA gtest_SOURCES += \ core/test_mc_cuda.cc \ core/test_ec_cuda.cc gtest_CPPFLAGS += \ $(CUDA_CPPFLAGS) gtest_LDFLAGS += \ $(CUDA_LDFLAGS) gtest_LDADD += \ $(CUDA_LIBS) endif if HAVE_ROCM gtest_SOURCES += core/test_mc_rocm.cc gtest_CXXFLAGS += $(HIP_CXXFLAGS) gtest_CPPFLAGS += $(HIP_CPPFLAGS) gtest_LDFLAGS += $(HIP_LDFLAGS) gtest_LDADD += $(HIP_LIBS) endif if HAVE_UCX gtest_CXXFLAGS += $(UCX_CXXFLAGS) gtest_CPPFLAGS += $(UCX_CPPFLAGS) gtest_LDFLAGS += $(UCX_LDFLAGS) gtest_LDADD += $(UCX_LIBS) $(UCX_LIBADD) endif noinst_HEADERS = \ common/gtest.h \ common/test.h \ common/test_ucc.h \ core/test_context.h \ core/test_mem_map.h \ core/test_mc_reduce.h \ coll/test_allreduce_sliding_window.h \ coll_score/test_score.h .PHONY: test test gdb valgrind fix_rpath ucc all-local: gtest ucc: $(MAKE) -C $(top_builddir) help: @echo @echo "Targets:" @echo " list : List unit tests." @echo " test : Run unit tests." @echo " test_gdb : Run unit tests with GDB." @echo " test_valgrind : Run unit tests with Valgrind." @echo @echo "Environment variables:" @echo " GTEST_FILTER : Unit tests filter (\"$(GTEST_FILTER)\")" @echo " GTEST_EXTRA_ARGS : Additional arguments for gtest (\"$(GTEST_EXTRA_ARGS)\")" @echo " LAUNCHER : Custom launcher for gtest executable (\"$(LAUNCHER)\")" @echo " VALGRIND_EXTRA_ARGS : Additional arguments for Valgrind (\"$(VALGRIND_EXTRA_ARGS)\")" @echo # # List unit tests # list: gtest $(abs_builddir)/gtest --gtest_list_tests $(GTEST_ARGS) # # Run unit tests # test: ucc gtest @rm -f core.* $(LAUNCHER) stdbuf -e0 -o0 $(abs_builddir)/gtest $(GTEST_ARGS) # # Run unit tests with GDB # test_gdb: ucc gtest echo -e 'r\ninit-if-undefined $$_exitcode=-1\nif $$_exitcode>=0\n\tq\nend' > .gdbcommands $(LAUNCHER) env UCC_HANDLE_ERRORS=none \ gdb -x .gdbcommands --args $(GDB_ARGS) \ $(abs_builddir)/gtest $(GTEST_ARGS) # # Run unit tests with valgrind # test_valgrind: ucc gtest $(LAUNCHER) env LD_LIBRARY_PATH="$(VALGRIND_LIBPATH):${LD_LIBRARY_PATH}" \ stdbuf -e0 -o0 valgrind $(VALGRIND_ARGS) $(abs_builddir)/gtest $(GTEST_ARGS) endif ucc-1.8.0/test/gtest/coll_score/0000775000175000017500000000000015211535620016707 5ustar alastairalastairucc-1.8.0/test/gtest/coll_score/test_score_update.cc0000664000175000017500000002132415211535620022734 0ustar alastairalastair/** * Copyright (c) 2021 - 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * RESERVED. See file LICENSE for terms. */ #include "test_score.h" class test_score_update : public test_score { public: ucc_coll_score_t *score; ucc_coll_score_t *update; test_score_update() { EXPECT_EQ(UCC_OK, ucc_coll_score_alloc(&score)); EXPECT_EQ(UCC_OK, ucc_coll_score_alloc(&update)); } ~test_score_update() { ucc_coll_score_free(score); ucc_coll_score_free(update); } void reset() { ucc_coll_score_free(score); ucc_coll_score_free(update); EXPECT_EQ(UCC_OK, ucc_coll_score_alloc(&score)); EXPECT_EQ(UCC_OK, ucc_coll_score_alloc(&update)); } }; UCC_TEST_F(test_score_update, non_overlap) { init_score(score, RLIST({RANGE(0, 10, 10), RANGE(40, 50, 5)}), UCC_COLL_TYPE_BARRIER); init_score(update, RLIST({RANGE(10, 20, 100), RANGE(30, 35, 1)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 10, 10), RANGE(40, 50, 5)}))); } UCC_TEST_F(test_score_update, overlap_single) { init_score(score, RLIST({RANGE(0, 100, 10)}), UCC_COLL_TYPE_BARRIER); init_score(update, RLIST({RANGE(50, 150, 100)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 50, 10), RANGE(50, 100, 100)}))); reset(); init_score(score, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER); init_score(update, RLIST({RANGE(50, 150, 10)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 50, 100), RANGE(50, 100, 10)}))); } UCC_TEST_F(test_score_update, inclusive) { init_score(score, RLIST({RANGE(0, 90, 100)}), UCC_COLL_TYPE_BARRIER); init_score(update, RLIST({RANGE(30, 60, 10)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 30, 100), RANGE(30, 60, 10), RANGE(60, 90, 100)}))); reset(); init_score(score, RLIST({RANGE(0, 90, 10)}), UCC_COLL_TYPE_BARRIER); init_score(update, RLIST({RANGE(30, 60, 100)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 30, 10), RANGE(30, 60, 100), RANGE(60, 90, 10)}))); } UCC_TEST_F(test_score_update, same_start) { init_score(score, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER); init_score(update, RLIST({RANGE(0, 50, 10)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 50, 10), RANGE(50, 100, 100)}))); reset(); init_score(score, RLIST({RANGE(0, 100, 10)}), UCC_COLL_TYPE_BARRIER); init_score(update, RLIST({RANGE(0, 50, 100)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 50, 100), RANGE(50, 100, 10)}))); reset(); init_score(score, RLIST({RANGE(1, 100, 10)}), UCC_COLL_TYPE_BARRIER); init_score(update, RLIST({RANGE(1, 50, 100)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(1, 50, 100), RANGE(50, 100, 10)}))); } UCC_TEST_F(test_score_update, 1_overlaps_many) { init_score(score, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER); init_score(update, RLIST({RANGE(10, 20, 10), RANGE(30, 40, 20), RANGE(60, 70, 30)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 10, 100), RANGE(10, 20, 10), RANGE(20, 30, 100), RANGE(30, 40, 20), RANGE(40, 60, 100), RANGE(60, 70, 30), RANGE(70, 100, 100)}))); reset(); init_score(score, RLIST({RANGE(0, 100, 10)}), UCC_COLL_TYPE_BARRIER); init_score( update, RLIST({RANGE(10, 20, 100), RANGE(30, 40, 100), RANGE(60, 70, 5)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 10, 10), RANGE(10, 20, 100), RANGE(20, 30, 10), RANGE(30, 40, 100), RANGE(40, 60, 10), RANGE(60, 70, 5), RANGE(70, 100, 10)}))); } UCC_TEST_F(test_score_update, same_score) { init_score(score, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER); init_score(update, RLIST({RANGE(100, 200, 100)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 100, 100)}))); } UCC_TEST_F(test_score_update, non_overlap_2) { init_score(score, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER, 0x1, 0x1); init_score(update, RLIST({RANGE(300, 400, 100)}), UCC_COLL_TYPE_BARRIER, 0x2, 0x2); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 100, 100), RANGE(300, 400, 100)}))); } UCC_TEST_F(test_score_update, init_reset) { init_score(score, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER, 0x1, 0x1); init_score(update, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER, 0x2, 0x2); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 100, 100)}))); EXPECT_EQ(0x2, (uint64_t)(FIRST_RANGE(score, BARRIER, HOST)->super.init)); reset(); init_score(score, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER, 0x1, 0x1); init_score(update, RLIST({RANGE(50, 150, 50)}), UCC_COLL_TYPE_BARRIER, 0x2, 0x2); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 50, 100), RANGE(50, 100, 50), RANGE(100, 150, 50)}))); EXPECT_EQ(0x1, (uint64_t)(FIRST_RANGE(score, BARRIER, HOST)->super.init)); reset(); init_score(score, RLIST({RANGE(50, 150, 100)}), UCC_COLL_TYPE_BARRIER, 0x1, 0x1); init_score(update, RLIST({RANGE(0, 100, 50)}), UCC_COLL_TYPE_BARRIER, 0x2, 0x2); EXPECT_EQ(UCC_OK, ucc_coll_score_update(score, update, 0, NULL, 0, UCC_COLL_TYPE_ALL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 50, 50), RANGE(50, 100, 50), RANGE(100, 150, 100)}))); EXPECT_EQ(0x2, (uint64_t)(FIRST_RANGE(score, BARRIER, HOST)->super.init)); } ucc-1.8.0/test/gtest/coll_score/test_score_str.cc0000664000175000017500000001200615211535620022257 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_score.h" class test_score_str : public test_score { }; #define SCORE(_score, _ct, _mt) \ ({ \ ucc_list_link_t *l = \ _score->scores[ucc_ilog2(UCC_COLL_TYPE_##_ct)] \ [UCC_MEMORY_TYPE_##_mt] \ .next; \ ucc_msg_range_t *range = \ ucc_container_of(l, ucc_msg_range_t, super.list_elem); \ range->super.score; \ }) UCC_TEST_F(test_score_str, check_valid) { std::string str = "alltoall:cuda:10"; ucc_coll_score_t *score; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc_from_str(str.c_str(), &score, 0, NULL, NULL, NULL)); EXPECT_EQ(10, SCORE(score, ALLTOALL, CUDA)); ucc_coll_score_free(score); str = "host,Cuda:Bcast,SCATTER:10"; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc_from_str(str.c_str(), &score, 0, NULL, NULL, NULL)); EXPECT_EQ(10, SCORE(score, BCAST, CUDA)); EXPECT_EQ(10, SCORE(score, BCAST, HOST)); EXPECT_EQ(10, SCORE(score, SCATTER, CUDA)); EXPECT_EQ(10, SCORE(score, SCATTER, HOST)); ucc_coll_score_free(score); str = "inf:gatHerv"; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc_from_str(str.c_str(), &score, 0, NULL, NULL, NULL)); EXPECT_EQ(UCC_SCORE_MAX, SCORE(score, GATHERV, ROCM)); EXPECT_EQ(UCC_SCORE_MAX, SCORE(score, GATHERV, HOST)); ucc_coll_score_free(score); str = "alltoall,bCAst:hOst:10#scatter:inf#reduce:1"; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc_from_str(str.c_str(), &score, 0, NULL, NULL, NULL)); EXPECT_EQ(10, SCORE(score, BCAST, HOST)); EXPECT_EQ(UCC_SCORE_MAX, SCORE(score, SCATTER, HOST)); EXPECT_EQ(UCC_SCORE_MAX, SCORE(score, SCATTER, CUDA_MANAGED)); EXPECT_EQ(1, SCORE(score, REDUCE, HOST)); ucc_coll_score_free(score); } UCC_TEST_F(test_score_str, check_invalid) { std::string str = "alltoallll:cuda:10"; ucc_coll_score_t *score; testing::internal::CaptureStdout(); EXPECT_NE(UCC_OK, ucc_coll_score_alloc_from_str(str.c_str(), &score, 0, NULL, NULL, NULL)); testing::internal::GetCapturedStdout(); } UCC_TEST_F(test_score_str, check_range_1) { std::string str = "alltoall:64-256:cuda:10"; ucc_coll_score_t *score; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc_from_str(str.c_str(), &score, 0, NULL, NULL, NULL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_ALLTOALL, UCC_MEMORY_TYPE_CUDA, RLIST({RANGE(64, 256, 10)}))); EXPECT_NE(UCC_OK, check_range(score, UCC_COLL_TYPE_ALLTOALL, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(64, 256, 10)}))); ucc_coll_score_free(score); str = "10-20:scatter:host:99"; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc_from_str(str.c_str(), &score, 0, NULL, NULL, NULL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_SCATTER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(10, 20, 99)}))); ucc_coll_score_free(score); } UCC_TEST_F(test_score_str, check_range_multiple) { std::string str = "alltoall:1k-2k,64-256,4096-5000:cuda:10"; ucc_coll_score_t *score; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc_from_str(str.c_str(), &score, 0, NULL, NULL, NULL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_ALLTOALL, UCC_MEMORY_TYPE_CUDA, RLIST({RANGE(64, 256, 10), RANGE(1024,2048,10), RANGE(4096,5000,10)}))); ucc_coll_score_free(score); str = "alltoall,barrier:1k-4K,64-256:cuda:10#20:99-12M:bcast"; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc_from_str(str.c_str(), &score, 0, NULL, NULL, NULL)); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_ALLTOALL, UCC_MEMORY_TYPE_CUDA, RLIST({RANGE(64, 256, 10), RANGE(1024,4096,10)}))); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_CUDA, RLIST({RANGE(64, 256, 10), RANGE(1024,4096,10)}))); EXPECT_EQ(UCC_OK, check_range(score, UCC_COLL_TYPE_BCAST, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(99, 12*1024*1024, 20)}))); ucc_coll_score_free(score); } ucc-1.8.0/test/gtest/coll_score/test_update_score.cc0000664000175000017500000000044215211535620022732 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * RESERVED. See file LICENSE for terms. */ extern "C" { #include "coll_select/coll_select.h" } #include #include #include class test_score_update : public ucc::test { }; ucc-1.8.0/test/gtest/coll_score/test_score.h0000664000175000017500000000400615211535620021232 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_TEST_SCORE_H #define UCC_TEST_SCORE_H extern "C" { #include "coll_score/ucc_coll_score.h" } #include #include #include typedef std::tuple range_t; typedef std::tuple fallback_t; #define RANGE(_start, _end, _score) std::make_tuple(_start, _end, _score) #define RLIST(...) std::vector(__VA_ARGS__) #define FB(_score, _init) std::make_tuple(_score, _init) #define FB_LIST(...) std::vector(__VA_ARGS__) #define FB_LLIST(...) std::vector>(__VA_ARGS__) #define FIRST_RANGE(_score, _ct, _mt) \ ({ \ ucc_list_link_t *l = \ _score->scores[ucc_ilog2(UCC_COLL_TYPE_##_ct)] \ [UCC_MEMORY_TYPE_##_mt] \ .next; \ ucc_msg_range_t *range = \ ucc_container_of(l, ucc_msg_range_t, super.list_elem); \ range; \ }) class test_score : public ucc::test { public: ucc_status_t check_range(ucc_coll_score_t *score, ucc_coll_type_t c, ucc_memory_type_t m, std::vector check); ucc_status_t check_fallback(ucc_coll_score_t *score, ucc_coll_type_t c, ucc_memory_type_t m, std::vector> check); void init_score(ucc_coll_score_t *score, std::vector v, ucc_coll_type_t c, uint64_t init_fn = 0, uint64_t team = 0); }; #endif ucc-1.8.0/test/gtest/coll_score/test_score.cc0000664000175000017500000003116015211535620021371 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_score.h" UCC_TEST_F(test_score, alloc_free) { ucc_coll_score_t *score; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc(&score)); ucc_coll_score_free(score); } UCC_TEST_F(test_score, add_range) { ucc_coll_type_t c = UCC_COLL_TYPE_BARRIER; ucc_memory_type_t m = UCC_MEMORY_TYPE_HOST; ucc_score_t value; ucc_coll_score_t *score; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc(&score)); /* adding range with 0 score value - should be skipped */ value = 0; EXPECT_EQ(UCC_OK, ucc_coll_score_add_range(score, c, m, 16, 65536, value, NULL, NULL)); EXPECT_EQ(0, ucc_list_length(&score->scores[ucc_ilog2(c)][m])); value = 10; EXPECT_EQ(UCC_OK, ucc_coll_score_add_range(score, c, m, 16, 65536, value, NULL, NULL)); EXPECT_EQ(1, ucc_list_length(&score->scores[ucc_ilog2(c)][m])); ucc_coll_score_free(score); } UCC_TEST_F(test_score, add_range_sorted) { ucc_coll_type_t c = UCC_COLL_TYPE_BARRIER; ucc_memory_type_t m = UCC_MEMORY_TYPE_HOST; ucc_score_t value; ucc_coll_score_t *score; EXPECT_EQ(UCC_OK, ucc_coll_score_alloc(&score)); value = 10; EXPECT_EQ(UCC_OK, ucc_coll_score_add_range(score, c, m, 50, 60, value, NULL, NULL)); EXPECT_EQ(UCC_OK, ucc_coll_score_add_range(score, c, m, 0, 10, value, NULL, NULL)); EXPECT_EQ(UCC_OK, ucc_coll_score_add_range(score, c, m, 100, 1000, value, NULL, NULL)); EXPECT_EQ(UCC_OK, ucc_coll_score_add_range(score, c, m, 20, 40, value, NULL, NULL)); ucc_list_link_t *list = &score->scores[ucc_ilog2(c)][m]; EXPECT_EQ(4, ucc_list_length(list)); ucc_msg_range_t *range; size_t expected[4] = {0, 20, 50, 100}; size_t * e = expected; ucc_list_for_each(range, list, super.list_elem) { EXPECT_EQ(*e, range->start); e++; } ucc_coll_score_free(score); } class test_score_merge : public test_score { public: ucc_coll_score_t *score1; ucc_coll_score_t *score2; ucc_coll_score_t *merge; test_score_merge() { EXPECT_EQ(UCC_OK, ucc_coll_score_alloc(&score1)); EXPECT_EQ(UCC_OK, ucc_coll_score_alloc(&score2)); } ~test_score_merge() { ucc_coll_score_free(merge); } }; ucc_status_t test_score::check_range(ucc_coll_score_t * score, ucc_coll_type_t c, ucc_memory_type_t m, std::vector check) { ucc_msg_range_t *range; ucc_list_link_t *list = &score->scores[ucc_ilog2(c)][m]; auto r = check.begin(); if (check.size() != ucc_list_length(list)) { return UCC_ERR_NO_MESSAGE; } ucc_list_for_each(range, list, super.list_elem) { if (range->start != std::get<0>(*r) || range->end != std::get<1>(*r) || range->super.score != std::get<2>(*r)) { return UCC_ERR_NO_MESSAGE; } r++; } return UCC_OK; } ucc_status_t test_score::check_fallback(ucc_coll_score_t * score, ucc_coll_type_t c, ucc_memory_type_t m, std::vector > check) { ucc_list_link_t *list = &score->scores[ucc_ilog2(c)][m]; auto fb = check.begin(); ucc_list_link_t *fallback; ucc_msg_range_t *range; ucc_coll_entry_t *fb_r; if (check.size() != ucc_list_length(list)) { return UCC_ERR_NO_MESSAGE; } ucc_list_for_each(range, list, super.list_elem) { fallback = &range->fallback; if (fb->size() != ucc_list_length(fallback)) { return UCC_ERR_NO_MESSAGE; } auto f = fb->begin(); ucc_list_for_each(fb_r, fallback, list_elem) { if (fb_r->score != std::get<0>(*f) || (uint64_t)fb_r->init != std::get<1>(*f)) { return UCC_ERR_NO_MESSAGE; } f++; } fb++; } return UCC_OK; } void test_score::init_score(ucc_coll_score_t *score, std::vector v, ucc_coll_type_t c, uint64_t init_fn, uint64_t team) { for (auto &r : v) { EXPECT_EQ(UCC_OK, ucc_coll_score_add_range( score, c, UCC_MEMORY_TYPE_HOST, std::get<0>(r), std::get<1>(r), std::get<2>(r), (ucc_base_coll_init_fn_t)init_fn, (ucc_base_team_t *)team)); } } UCC_TEST_F(test_score_merge, non_overlap) { init_score(score1, RLIST({RANGE(0, 10, 10), RANGE(40, 50, 5)}), UCC_COLL_TYPE_BARRIER); init_score(score2, RLIST({RANGE(10, 20, 100), RANGE(30, 35, 1)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 10, 10), RANGE(10, 20, 100), RANGE(30, 35, 1), RANGE(40, 50, 5)}))); } UCC_TEST_F(test_score_merge, overlap_single) { init_score(score1, RLIST({RANGE(0, 100, 10)}), UCC_COLL_TYPE_BARRIER); init_score(score2, RLIST({RANGE(50, 150, 100)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 50, 10), RANGE(50, 150, 100)}))); ucc_coll_score_free(merge); ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score1)); ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score2)); init_score(score1, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER); init_score(score2, RLIST({RANGE(50, 150, 10)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 100, 100), RANGE(100, 150, 10)}))); } UCC_TEST_F(test_score_merge, inclusive) { init_score(score1, RLIST({RANGE(0, 90, 100)}), UCC_COLL_TYPE_BARRIER); init_score(score2, RLIST({RANGE(30, 60, 10)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 90, 100)}))); ucc_coll_score_free(merge); ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score1)); ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score2)); init_score(score1, RLIST({RANGE(0, 90, 10)}), UCC_COLL_TYPE_BARRIER); init_score(score2, RLIST({RANGE(30, 60, 100)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 30, 10), RANGE(30, 60, 100), RANGE(60, 90, 10)}))); } UCC_TEST_F(test_score_merge, same_start) { init_score(score1, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER); init_score(score2, RLIST({RANGE(0, 50, 10)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 100, 100)}))); ucc_coll_score_free(merge); ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score1)); ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score2)); init_score(score1, RLIST({RANGE(0, 100, 10)}), UCC_COLL_TYPE_BARRIER); init_score(score2, RLIST({RANGE(0, 50, 100)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 50, 100), RANGE(50, 100, 10)}))); ucc_coll_score_free(merge); ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score1)); ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score2)); init_score(score1, RLIST({RANGE(1, 100, 10)}), UCC_COLL_TYPE_BARRIER); init_score(score2, RLIST({RANGE(1, 50, 100)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(1, 50, 100), RANGE(50, 100, 10)}))); } UCC_TEST_F(test_score_merge, 1_overlaps_many) { init_score(score1, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER); init_score(score2, RLIST({RANGE(10, 20, 10), RANGE(30, 40, 10), RANGE(60, 70, 10)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 100, 100)}))); ucc_coll_score_free(merge); ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score1)); ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score2)); init_score(score1, RLIST({RANGE(0, 100, 10)}), UCC_COLL_TYPE_BARRIER); init_score( score2, RLIST({RANGE(10, 20, 100), RANGE(30, 40, 100), RANGE(60, 70, 5)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 10, 10), RANGE(10, 20, 100), RANGE(20, 30, 10), RANGE(30, 40, 100), RANGE(40, 100, 10)}))); } UCC_TEST_F(test_score_merge, same_score) { init_score(score1, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER); init_score(score2, RLIST({RANGE(100, 200, 100)}), UCC_COLL_TYPE_BARRIER); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_range(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, RLIST({RANGE(0, 200, 100)}))); } UCC_TEST_F(test_score_merge, fallback_single) { init_score(score1, RLIST({RANGE(0, 100, 100), RANGE(200, 300, 5)}), UCC_COLL_TYPE_BARRIER, 0x1, 0x2); init_score(score2, RLIST({RANGE(0, 100, 200), RANGE(250, 350, 3)}), UCC_COLL_TYPE_BARRIER, 0x3, 0x4); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); /* First range overlaps intirely with 200 being higer score: fallback must be init=0x1. Second range overlaps into 3 pieces: 2 w/o fallback and 1 with fallback init=0x3 since its score (3) is smaller */ EXPECT_EQ(UCC_OK, check_fallback(merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, FB_LLIST({FB_LIST({FB(100, 0x1)}), FB_LIST({}), FB_LIST({FB(3, 0x3)}), FB_LIST({})}))); } UCC_TEST_F(test_score_merge, fallback_multiple) { /* Same range has 5 different scores. There must be 4 fallbask ordered by score */ init_score(score1, RLIST({RANGE(0, 100, 100)}), UCC_COLL_TYPE_BARRIER, 0x1, 0x2); init_score(score2, RLIST({RANGE(0, 100, 300)}), UCC_COLL_TYPE_BARRIER, 0x3, 0x4); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); score1 = merge; ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score2)); init_score(score2, RLIST({RANGE(0, 100, 500)}), UCC_COLL_TYPE_BARRIER, 0x5, 0x6); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); score1 = merge; ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score2)); init_score(score2, RLIST({RANGE(0, 100, 400)}), UCC_COLL_TYPE_BARRIER, 0x7, 0x8); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); score1 = merge; ASSERT_EQ(UCC_OK, ucc_coll_score_alloc(&score2)); init_score(score2, RLIST({RANGE(0, 100, 600)}), UCC_COLL_TYPE_BARRIER, 0x9, 0x10); EXPECT_EQ(UCC_OK, ucc_coll_score_merge(score1, score2, &merge, 1)); EXPECT_EQ(UCC_OK, check_fallback( merge, UCC_COLL_TYPE_BARRIER, UCC_MEMORY_TYPE_HOST, FB_LLIST({FB_LIST({FB(500, 0x5), FB(400, 0x7), FB(300, 0x3), FB(100, 0x1)})}))); } ucc-1.8.0/test/gtest/core/0000775000175000017500000000000015211535620015513 5ustar alastairalastairucc-1.8.0/test/gtest/core/test_service_coll.cc0000664000175000017500000001461415211535620021540 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include #include #include extern "C" { #include "core/ucc_team.h" #include "core/ucc_service_coll.h" } class test_service_coll { public: int * array; UccTeam_h team; std::vector subsets; std::vector reqs; test_service_coll(std::vector _subset, UccTeam_h _team) { team = _team; array = new int[_subset.size()]; memcpy(array, _subset.data(), sizeof(int) * _subset.size()); subsets.resize(_subset.size()); reqs.resize(_subset.size()); for (auto i = 0; i < _subset.size(); i++) { subsets[i].myrank = i; subsets[i].map.type = UCC_EP_MAP_ARRAY; subsets[i].map.array.map = (void *)array; subsets[i].map.array.elem_size = sizeof(int); subsets[i].map.ep_num = _subset.size(); } } ~test_service_coll() { delete[] array; } void progress() { for (auto i = 0; i < reqs.size(); i++) { ucc_context_progress(team.get()->procs[array[i]].p->ctx_h); } } void wait() { ucc_status_t status; bool ready; do { ready = true; progress(); for (auto &r : reqs) { status = ucc_service_coll_test(r); EXPECT_GE(status, 0); if (UCC_INPROGRESS == status) { ready = false; } } } while (!ready); for (auto &r : reqs) { ucc_service_coll_finalize(r); } } }; class test_service_allreduce : public test_service_coll { std::vector> sbuf; std::vector> rbuf; public: test_service_allreduce(std::vector _subset, size_t count, UccTeam_h _team) : test_service_coll(_subset, _team) { sbuf.resize(_subset.size()); rbuf.resize(_subset.size()); for (auto i = 0; i < _subset.size(); i++) { sbuf[i].resize(count); rbuf[i].resize(count); for (auto j = 0; j < count; j++) { sbuf[i][j] = i + j + 1; rbuf[i][j] = 0; } } } void start() { ucc_status_t status; for (auto i = 0; i < reqs.size(); i++) { auto r = array[i]; status = ucc_service_allreduce( team.get()->procs[r].team, sbuf[i].data(), rbuf[i].data(), UCC_DT_INT32, sbuf[i].size(), UCC_OP_SUM, subsets[i], &reqs[i]); EXPECT_EQ(UCC_OK, status); } } void check() { int size = reqs.size(); for (auto i = 0; i < size; i++) { for (auto j = 0; j < sbuf[i].size(); j++) { int check = size * (size + 1) / 2 + j * size; EXPECT_EQ(check, rbuf[i][j]); } } } }; class test_scoll_allreduce : public ucc::test, public ::testing::WithParamInterface> { }; UCC_TEST_P(test_scoll_allreduce, allreduce) { /* Reversed team of size staticUccJobSize - last one in static teawms array */ auto team = UccJob::getStaticTeams().back(); ASSERT_EQ(team.get()->procs.size(), 16); std::vector subset = GetParam(); test_service_allreduce t(subset, 4, team); t.start(); t.wait(); t.check(); } INSTANTIATE_TEST_CASE_P( , test_scoll_allreduce, ::testing::Values(std::vector({1, 0}), std::vector({2, 3}), std::vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}), std::vector({0, 1, 2, 3, 4, 5, 6, 7, 8}), std::vector({15, 14, 13, 12, 11, 10, 9}), std::vector({0, 2, 4, 6, 8}))); class test_service_allgather : public test_service_coll { std::vector> sbuf; std::vector> rbuf; public: test_service_allgather(std::vector _subset, size_t count, UccTeam_h _team) : test_service_coll(_subset, _team) { sbuf.resize(_subset.size()); rbuf.resize(_subset.size()); for (auto i = 0; i < _subset.size(); i++) { sbuf[i].resize(count); rbuf[i].resize(count * _subset.size()); for (auto j = 0; j < count; j++) { sbuf[i][j] = i + j + 1; } for (auto j = 0; j < count * _subset.size(); j++) { rbuf[i][j] = 0; } } } void start() { ucc_status_t status; for (auto i = 0; i < reqs.size(); i++) { auto r = array[i]; status = ucc_service_allgather( team.get()->procs[r].team, sbuf[i].data(), rbuf[i].data(), sbuf[i].size() * sizeof(int), subsets[i], &reqs[i]); EXPECT_EQ(UCC_OK, status); } } void check() { int size = reqs.size(); int count = sbuf[0].size(); for (auto i = 0; i < size; i++) { for (auto j = 0; j < rbuf[i].size(); j++) { int check = (j % count) + 1 + (j / count); EXPECT_EQ(check, rbuf[i][j]); } } } }; class test_scoll_allgather : public ucc::test, public ::testing::WithParamInterface> { }; UCC_TEST_P(test_scoll_allgather, allgather) { /* Reversed team of size staticUccJobSize - last one in static teawms array */ auto team = UccJob::getStaticTeams().back(); ASSERT_EQ(team.get()->procs.size(), 16); std::vector subset = GetParam(); test_service_allgather t(subset, 4, team); t.start(); t.wait(); t.check(); } INSTANTIATE_TEST_CASE_P( , test_scoll_allgather, ::testing::Values(std::vector({1, 0}), std::vector({2, 3}), std::vector({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}), std::vector({0, 1, 2, 3, 4, 5, 6, 7, 8}), std::vector({15, 14, 13, 12, 11, 10, 9}), std::vector({0, 2, 4, 6, 8}))); ucc-1.8.0/test/gtest/core/test_ec_cuda.cc0000664000175000017500000001057715211535620020456 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ extern "C" { #include #include } #include #include #include class test_ec_cuda : public ucc::test { void TestECCudaSetUp(ucc_ec_params_t ec_params) { ucc::test::SetUp(); ucc_constructor(); ucc_ec_init(&ec_params); } virtual void SetUp() override { ucc_ec_params_t ec_params = { .thread_mode = UCC_THREAD_SINGLE, }; TestECCudaSetUp(ec_params); if (UCC_OK != ucc_ec_available(UCC_EE_CUDA_STREAM)) { GTEST_SKIP(); } } virtual void TearDown() override { ucc_ec_finalize(); ucc::test::TearDown(); } public: ucc_status_t get_cuda_executor(ucc_ee_executor_t **executor) { ucc_ee_executor_params_t eparams; eparams.mask = UCC_EE_EXECUTOR_PARAM_FIELD_TYPE; eparams.ee_type = UCC_EE_CUDA_STREAM; return ucc_ee_executor_init(&eparams, executor); } ucc_status_t put_cuda_executor(ucc_ee_executor_t *executor) { return ucc_ee_executor_finalize(executor); } }; UCC_TEST_F(test_ec_cuda, ec_cuda_load) { ASSERT_EQ(UCC_OK, ucc_ec_available(UCC_EE_CUDA_STREAM)); } UCC_TEST_F(test_ec_cuda, ec_cuda_executor_init_finalize) { ucc_ee_executor_t *executor; ASSERT_EQ(UCC_OK, get_cuda_executor(&executor)); ASSERT_EQ(UCC_OK, put_cuda_executor(executor)); } UCC_TEST_F(test_ec_cuda, ec_cuda_executor_interruptible_start) { ucc_ee_executor_t *executor; ucc_status_t status; ASSERT_EQ(UCC_OK, get_cuda_executor(&executor)); status = ucc_ee_executor_start(executor, nullptr); EXPECT_EQ(UCC_OK, status); if (status == UCC_OK) { EXPECT_EQ(UCC_OK, ucc_ee_executor_stop(executor)); } ASSERT_EQ(UCC_OK, put_cuda_executor(executor)); } UCC_TEST_F(test_ec_cuda, ec_cuda_executor_interruptible_copy) { ucc_ee_executor_t *executor; ucc_status_t status; cudaError_t cuda_st; int *src_host, *dst_host, *src, *dst; const size_t size = 4850; ucc_ee_executor_task_t *task; ASSERT_EQ(UCC_OK, get_cuda_executor(&executor)); status = ucc_ee_executor_start(executor, nullptr); EXPECT_EQ(UCC_OK, status); if (status != UCC_OK) { goto exit; } src_host = (int*)malloc(size * sizeof(int)); EXPECT_NE(nullptr, src_host); if (!src_host) { goto exit; } dst_host = (int*)malloc(size * sizeof(int)); EXPECT_NE(nullptr, dst_host); if (!dst_host) { goto exit; } cuda_st = cudaMalloc(&src, size * sizeof(int)); EXPECT_EQ(cudaSuccess, cuda_st); if (cuda_st != cudaSuccess) { goto exit; } cuda_st = cudaMalloc(&dst, size * sizeof(int)); EXPECT_EQ(cudaSuccess, cuda_st); if (cuda_st != cudaSuccess) { goto exit; } for (int i = 0; i < size; i++) { src_host[i] = i; } cuda_st = cudaMemcpy(src, src_host, size * sizeof(int), cudaMemcpyDefault); EXPECT_EQ(cudaSuccess, cuda_st); if (cuda_st != cudaSuccess) { goto exit; } ucc_ee_executor_task_args_t args; args.task_type = UCC_EE_EXECUTOR_TASK_COPY; args.copy.src = src; args.copy.dst = dst; args.copy.len = size * sizeof(int); status = ucc_ee_executor_task_post(executor, &args, &task); EXPECT_EQ(UCC_OK, status); if (status != UCC_OK) { goto exit; } do { status = ucc_ee_executor_task_test(task); } while (status > 0); EXPECT_EQ(UCC_OK, status); if (status != UCC_OK) { goto exit; } cuda_st = cudaMemcpy(dst_host, dst, size * sizeof(int), cudaMemcpyDefault); EXPECT_EQ(cudaSuccess, cuda_st); if (cuda_st != cudaSuccess) { goto exit; } for (int i = 0; i < size; i++) { if (dst_host[i] != src_host[i]) { EXPECT_EQ(dst_host[i], src_host[i]); goto exit; } } exit: if (task) { EXPECT_EQ(UCC_OK, ucc_ee_executor_task_finalize(task)); } if (src_host) { free(src_host); } if (dst_host) { free(dst_host); } if (src) { cudaFree(src); } if (dst) { cudaFree(dst); } EXPECT_EQ(UCC_OK, ucc_ee_executor_stop(executor)); ASSERT_EQ(UCC_OK, put_cuda_executor(executor)); } ucc-1.8.0/test/gtest/core/test_mc.cc0000664000175000017500000001007315211535620017461 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ extern "C" { #include #include } #include #include void *mt_ucc_mc_cpu_allocs(void *args) { // Final size will be: // size * (quantifier^(num_of_allocs/2)) // and should be larger than mpool buffer size which is 1MB by default, // to assure testing both fast and slow ucc_mc_alloc path. // if num_of_allocs is changed, change quantifier accordingly. size_t size = 4; int quantifier = 2; int num_of_allocs = 40; std::vector headers; std::vector pointers; headers.resize(num_of_allocs); pointers.resize(num_of_allocs); for (int i = 0; i < num_of_allocs; i++) { EXPECT_EQ(UCC_OK, ucc_mc_alloc(&headers[i], size, UCC_MEMORY_TYPE_HOST)); pointers[i] = headers[i]->addr; memset(pointers[i], 0, size); EXPECT_EQ(UCC_OK, ucc_mc_free(headers[i])); if (i % 2) { size *= quantifier; } } return 0; } class test_mc : public ucc::test { }; UCC_TEST_F(test_mc, init_finalize) { EXPECT_EQ(UCC_OK, ucc_constructor()); ucc_mc_params_t mc_params = { .thread_mode = UCC_THREAD_SINGLE, }; EXPECT_EQ(UCC_OK, ucc_mc_init(&mc_params)); EXPECT_EQ(UCC_OK, ucc_mc_finalize()); } UCC_TEST_F(test_mc, init_is_required) { ASSERT_EQ(UCC_OK, ucc_constructor()); EXPECT_NE(UCC_OK, ucc_mc_available(UCC_MEMORY_TYPE_HOST)); ucc_mc_params_t mc_params = { .thread_mode = UCC_THREAD_SINGLE, }; EXPECT_EQ(UCC_OK, ucc_mc_init(&mc_params)); EXPECT_EQ(UCC_OK, ucc_mc_available(UCC_MEMORY_TYPE_HOST)); ucc_mc_finalize(); } UCC_TEST_F(test_mc, can_alloc_and_free_host_mem) { // mpool will be used only if size is smaller than UCC_MC_CPU_ELEM_SIZE, which by default set to 1MB and is configurable at runtime. size_t size = 4096; ucc_mc_buffer_header_t *h; void *ptr = NULL; ASSERT_EQ(UCC_OK, ucc_constructor()); ucc_mc_params_t mc_params = { .thread_mode = UCC_THREAD_SINGLE, }; ASSERT_EQ(UCC_OK, ucc_mc_init(&mc_params)); EXPECT_EQ(UCC_OK, ucc_mc_alloc(&h, size, UCC_MEMORY_TYPE_HOST)); ptr = h->addr; memset(ptr, 0, size); EXPECT_EQ(UCC_OK, ucc_mc_free(h)); ucc_mc_finalize(); } // Disabled because can't reinit mc with different thread mode UCC_TEST_F(test_mc, DISABLED_can_alloc_and_free_host_mem_mt) { // mpool will be used only if size is smaller than UCC_MC_CPU_ELEM_SIZE, which by default set to 1MB and is configurable at runtime. int num_of_threads = 10; std::vector threads; threads.resize(num_of_threads); ASSERT_EQ(UCC_OK, ucc_constructor()); ucc_mc_params_t mc_params = { .thread_mode = UCC_THREAD_MULTIPLE, }; ASSERT_EQ(UCC_OK, ucc_mc_init(&mc_params)); for (int i = 0; i < num_of_threads; i++) { pthread_create(&threads[i], NULL, &mt_ucc_mc_cpu_allocs, NULL); } for (int i = 0; i < num_of_threads; i++) { pthread_join(threads[i], NULL); } ucc_mc_finalize(); } UCC_TEST_F(test_mc, init_twice) { ucc_lib_config_h cfg; ucc_lib_params_t lib_params; ucc_lib_h lib1, lib2; ucc_mc_base_t *mc; ASSERT_EQ(UCC_OK, ucc_lib_config_read(NULL, NULL, &cfg)); lib_params.mask = UCC_LIB_PARAM_FIELD_THREAD_MODE; lib_params.thread_mode = UCC_THREAD_SINGLE; ASSERT_EQ(UCC_OK, ucc_init(&lib_params, cfg, &lib1)); mc = ucc_derived_of(ucc_global_config.mc_framework.components[0], ucc_mc_base_t); EXPECT_EQ(1, mc->ref_cnt); EXPECT_EQ(UCC_OK, ucc_init(&lib_params, cfg, &lib2)); EXPECT_EQ(2, mc->ref_cnt); EXPECT_EQ(UCC_OK, ucc_finalize(lib1)); EXPECT_EQ(1, mc->ref_cnt); EXPECT_EQ(UCC_OK, ucc_finalize(lib2)); EXPECT_EQ(0, mc->ref_cnt); ucc_lib_config_release(cfg); } ucc-1.8.0/test/gtest/core/test_mc_cuda.cc0000664000175000017500000001457615211535620020471 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ extern "C" { #include #include } #include #include #include #include void *mt_ucc_mc_cuda_allocs(void *args) { // Final size will be: // size * (quantifier^(num_of_allocs/2)) // and should be larger than mpool buffer size which is 1MB by default, // to assure testing both fast and slow ucc_mc_alloc path. // if num_of_allocs is changed, change quantifier accordingly. int quantifier = 2; size_t size = 4; int num_of_allocs = 40; std::vector headers; std::vector pointers; headers.resize(num_of_allocs); pointers.resize(num_of_allocs); for (int i = 0; i < num_of_allocs; i++) { EXPECT_EQ(UCC_OK, ucc_mc_alloc(&headers[i], size, UCC_MEMORY_TYPE_CUDA)); pointers[i] = headers[i]->addr; EXPECT_EQ(cudaSuccess, cudaMemset(pointers[i], 0, size)); EXPECT_EQ(UCC_OK, ucc_mc_free(headers[i])); if (i % 2) { size *= quantifier; } } return 0; } class test_mc_cuda : public ucc::test { protected: const int TEST_ALLOC_SIZE = 1024; ucc_mc_buffer_header_t *mc_header; void * test_ptr; ucc_memory_type_t test_mtype; void TestMCCudaSetUp(ucc_mc_params_t mc_params) { ucc::test::SetUp(); ucc_constructor(); ucc_mc_init(&mc_params); test_ptr = NULL; test_mtype = UCC_MEMORY_TYPE_UNKNOWN; } virtual void SetUp() override { ucc_mc_params_t mc_params = { .thread_mode = UCC_THREAD_SINGLE, }; if (UCC_OK != ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { GTEST_SKIP(); } TestMCCudaSetUp(mc_params); } virtual void TearDown() override { ucc_mc_finalize(); ucc::test::TearDown(); } }; class test_mc_cuda_mt : public test_mc_cuda { protected: virtual void SetUp() override { ucc_mc_params_t mc_params = { .thread_mode = UCC_THREAD_MULTIPLE, }; if (UCC_OK != ucc_mc_available(UCC_MEMORY_TYPE_CUDA)) { GTEST_SKIP(); } TestMCCudaSetUp(mc_params); } }; UCC_TEST_F(test_mc_cuda, mc_cuda_load) { EXPECT_EQ(UCC_OK, ucc_mc_available(UCC_MEMORY_TYPE_CUDA)); } UCC_TEST_F(test_mc_cuda, can_alloc_and_free_mem) { // mpool will be used only if size is smaller than UCC_MC_CPU_ELEM_SIZE, which by default set to 1MB and is configurable at runtime. EXPECT_EQ(UCC_OK, ucc_mc_alloc(&mc_header, TEST_ALLOC_SIZE, UCC_MEMORY_TYPE_CUDA)); test_ptr = mc_header->addr; EXPECT_EQ(cudaSuccess, cudaMemset(test_ptr, 0, TEST_ALLOC_SIZE)); EXPECT_EQ(UCC_OK, ucc_mc_free(mc_header)); } // Disabled because can't reinit mc with different thread mode UCC_TEST_F(test_mc_cuda_mt, DISABLED_can_alloc_and_free_mem_mt) { // mpool will be used only if size is smaller than UCC_MC_CPU_ELEM_SIZE, which by default set to 1MB and is configurable at runtime. int num_of_threads = 10; std::vector threads; threads.resize(num_of_threads); for (int i = 0; i < num_of_threads; i++) { pthread_create(&threads[i], NULL, &mt_ucc_mc_cuda_allocs, NULL); } for (int i = 0; i < num_of_threads; i++) { pthread_join(threads[i], NULL); } } UCC_TEST_F(test_mc_cuda, can_detect_host_mem) { ucc_mem_attr_t mem_attr; mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; test_ptr = malloc(TEST_ALLOC_SIZE); if (test_ptr == NULL) { ADD_FAILURE() << "failed to allocate host memory"; } EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(test_ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_HOST, mem_attr.mem_type); free(test_ptr); } UCC_TEST_F(test_mc_cuda, can_detect_cuda_mem) { ucc_mem_attr_t mem_attr; cudaError_t st; mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; st = cudaMalloc(&test_ptr, TEST_ALLOC_SIZE); if (st != cudaSuccess) { ADD_FAILURE() << "failed to allocate device memory"; } EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(test_ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_CUDA, mem_attr.mem_type); cudaFree(test_ptr); ucc_mc_finalize(); } UCC_TEST_F(test_mc_cuda, can_query_cuda_mem) { ucc_mem_attr_t mem_attr; cudaError_t st; void *ptr; st = cudaMalloc(&test_ptr, TEST_ALLOC_SIZE); if (st != cudaSuccess) { ADD_FAILURE() << "failed to allocate device memory"; } mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(test_ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_CUDA, mem_attr.mem_type); /* query base addr and length */ mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE | UCC_MEM_ATTR_FIELD_BASE_ADDRESS | UCC_MEM_ATTR_FIELD_ALLOC_LENGTH; mem_attr.alloc_length = 1; ptr = (char *)test_ptr + TEST_ALLOC_SIZE/2; EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_CUDA, mem_attr.mem_type); EXPECT_EQ(test_ptr, mem_attr.base_address); EXPECT_EQ(TEST_ALLOC_SIZE, mem_attr.alloc_length); cudaFree(test_ptr); ucc_mc_finalize(); } UCC_TEST_F(test_mc_cuda, can_detect_managed_mem) { cudaError_t st; ucc_mem_attr_t mem_attr; st = cudaMallocManaged(&test_ptr, TEST_ALLOC_SIZE); if (st != cudaSuccess) { ADD_FAILURE() << "failed to allocate managed memory"; } mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(test_ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_CUDA_MANAGED, mem_attr.mem_type); cudaFree(test_ptr); } UCC_TEST_F(test_mc_cuda, can_detect_host_alloc_mem) { cudaError_t st; ucc_mem_attr_t mem_attr; st = cudaHostAlloc(&test_ptr, TEST_ALLOC_SIZE, cudaHostAllocDefault); if (st != cudaSuccess) { ADD_FAILURE() << "failed to allocate host mapped memory"; } mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(test_ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_HOST, mem_attr.mem_type); cudaFreeHost(test_ptr); } ucc-1.8.0/test/gtest/core/test_lib.cc0000664000175000017500000000254015211535620017630 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include #include #include class test_lib : public ucc::test { }; UCC_TEST_F(test_lib, init_finalize) { ucc_lib_config_h cfg; ucc_lib_params_t lib_params; ucc_lib_h lib; EXPECT_EQ(UCC_OK, ucc_lib_config_read(NULL, NULL, &cfg)); lib_params.mask = UCC_LIB_PARAM_FIELD_THREAD_MODE; lib_params.thread_mode = UCC_THREAD_SINGLE; EXPECT_EQ(UCC_OK, ucc_init(&lib_params, cfg, &lib)); ucc_lib_config_release(cfg); EXPECT_EQ(UCC_OK, ucc_finalize(lib)); } UCC_TEST_F(test_lib, init_multiple) { const int n_libs = 8; ucc_lib_config_h cfg; ucc_lib_params_t lib_params; ucc_lib_h lib_h; std::vector libs; EXPECT_EQ(UCC_OK, ucc_lib_config_read(NULL, NULL, &cfg)); lib_params.mask = UCC_LIB_PARAM_FIELD_THREAD_MODE; lib_params.thread_mode = UCC_THREAD_SINGLE; for (int i = 0; i < n_libs; i++) { EXPECT_EQ(UCC_OK, ucc_init(&lib_params, cfg, &lib_h)); libs.push_back(lib_h); } ucc_lib_config_release(cfg); std::shuffle(libs.begin(), libs.end(), std::default_random_engine()); for (auto lib_h : libs) { EXPECT_EQ(UCC_OK, ucc_finalize(lib_h)); } } ucc-1.8.0/test/gtest/core/test_mc_rocm.cc0000664000175000017500000001512315211535620020502 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * Copyright (C) Advanced Micro Devices, Inc. 2022. ALL RIGHTS RESERVED. * See file LICENSE for terms. */ extern "C" { #include #include } #include #include #include void *mt_ucc_mc_rocm_allocs(void *args) { // Final size will be: // size * (quantifier^(num_of_allocs/2)) // and should be larger than mpool buffer size which is 1MB by default, // to assure testing both fast and slow ucc_mc_alloc path. // if num_of_allocs is changed, change quantifier accordingly. int quantifier = 2; size_t size = 4; int num_of_allocs = 40; std::vector headers; std::vector pointers; headers.resize(num_of_allocs); pointers.resize(num_of_allocs); for (int i = 0; i < num_of_allocs; i++) { EXPECT_EQ(UCC_OK, ucc_mc_alloc(&headers[i], size, UCC_MEMORY_TYPE_ROCM)); pointers[i] = headers[i]->addr; EXPECT_EQ(hipSuccess, hipMemset(pointers[i], 0, size)); EXPECT_EQ(UCC_OK, ucc_mc_free(headers[i])); if (i % 2) { size *= quantifier; } } return 0; } class test_mc_rocm : public ucc::test { protected: const int TEST_ALLOC_SIZE = 1024; ucc_mc_buffer_header_t *mc_header; void * test_ptr; ucc_memory_type_t test_mtype; void TestMCRocmSetUp(ucc_mc_params_t mc_params) { ucc::test::SetUp(); ucc_constructor(); ucc_mc_init(&mc_params); test_ptr = NULL; test_mtype = UCC_MEMORY_TYPE_UNKNOWN; } virtual void SetUp() override { ucc_mc_params_t mc_params = { .thread_mode = UCC_THREAD_SINGLE, }; TestMCRocmSetUp(mc_params); } virtual void TearDown() override { ucc_mc_finalize(); ucc::test::TearDown(); } }; class test_mc_rocm_mt : public test_mc_rocm { protected: virtual void SetUp() override { ucc_mc_params_t mc_params = { .thread_mode = UCC_THREAD_MULTIPLE, }; TestMCRocmSetUp(mc_params); } }; UCC_TEST_F(test_mc_rocm, mc_rocm_load) { EXPECT_EQ(UCC_OK, ucc_mc_available(UCC_MEMORY_TYPE_ROCM)); } UCC_TEST_F(test_mc_rocm, can_alloc_and_free_mem) { // mpool will be used only if size is smaller than UCC_MC_CPU_ELEM_SIZE, which by default set to 1MB and is configurable at runtime. EXPECT_EQ(UCC_OK, ucc_mc_alloc(&mc_header, TEST_ALLOC_SIZE, UCC_MEMORY_TYPE_ROCM)); test_ptr = mc_header->addr; EXPECT_EQ(hipSuccess, hipMemset(test_ptr, 0, TEST_ALLOC_SIZE)); EXPECT_EQ(UCC_OK, ucc_mc_free(mc_header)); } // Disabled because can't reinit mc with different thread mode UCC_TEST_F(test_mc_rocm_mt, DISABLED_can_alloc_and_free_mem_mt) { // mpool will be used only if size is smaller than UCC_MC_CPU_ELEM_SIZE, which by default set to 1MB and is configurable at runtime. int num_of_threads = 10; std::vector threads; threads.resize(num_of_threads); for (int i = 0; i < num_of_threads; i++) { pthread_create(&threads[i], NULL, &mt_ucc_mc_rocm_allocs, NULL); } for (int i = 0; i < num_of_threads; i++) { pthread_join(threads[i], NULL); } } UCC_TEST_F(test_mc_rocm, can_detect_host_mem) { ucc_mem_attr_t mem_attr; mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; test_ptr = malloc(TEST_ALLOC_SIZE); if (test_ptr == NULL) { ADD_FAILURE() << "failed to allocate host memory"; } EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(test_ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_HOST, mem_attr.mem_type); free(test_ptr); } UCC_TEST_F(test_mc_rocm, can_detect_rocm_mem) { ucc_mem_attr_t mem_attr; hipError_t st; mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; st = hipMalloc(&test_ptr, TEST_ALLOC_SIZE); if (st != hipSuccess) { ADD_FAILURE() << "failed to allocate device memory"; } EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(test_ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_ROCM, mem_attr.mem_type); st = hipFree(test_ptr); if (st != hipSuccess) { ADD_FAILURE() << "failed to free device memory"; } ucc_mc_finalize(); } UCC_TEST_F(test_mc_rocm, can_query_rocm_mem) { ucc_mem_attr_t mem_attr; hipError_t st; void *ptr; st = hipMalloc(&test_ptr, TEST_ALLOC_SIZE); if (st != hipSuccess) { ADD_FAILURE() << "failed to allocate device memory"; } mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(test_ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_ROCM, mem_attr.mem_type); /* query base addr and length */ mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE | UCC_MEM_ATTR_FIELD_BASE_ADDRESS | UCC_MEM_ATTR_FIELD_ALLOC_LENGTH; mem_attr.alloc_length = 1; ptr = (char *)test_ptr + TEST_ALLOC_SIZE/2; EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_ROCM, mem_attr.mem_type); EXPECT_EQ(test_ptr, mem_attr.base_address); EXPECT_EQ(TEST_ALLOC_SIZE, mem_attr.alloc_length); st = hipFree(test_ptr); if (st != hipSuccess) { ADD_FAILURE() << "failed to free device memory"; } ucc_mc_finalize(); } UCC_TEST_F(test_mc_rocm, can_detect_managed_mem) { hipError_t st; ucc_mem_attr_t mem_attr; st = hipMallocManaged(&test_ptr, TEST_ALLOC_SIZE); if (st != hipSuccess) { ADD_FAILURE() << "failed to allocate managed memory"; } mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(test_ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_ROCM_MANAGED, mem_attr.mem_type); st = hipFree(test_ptr); if (st != hipSuccess) { ADD_FAILURE() << "failed to free managed memory"; } } UCC_TEST_F(test_mc_rocm, can_detect_host_alloc_mem) { hipError_t st; ucc_mem_attr_t mem_attr; st = hipHostMalloc(&test_ptr, TEST_ALLOC_SIZE, hipHostMallocDefault); if (st != hipSuccess) { ADD_FAILURE() << "failed to allocate host mapped memory"; } mem_attr.field_mask = UCC_MEM_ATTR_FIELD_MEM_TYPE; EXPECT_EQ(UCC_OK, ucc_mc_get_mem_attr(test_ptr, &mem_attr)); EXPECT_EQ(UCC_MEMORY_TYPE_HOST, mem_attr.mem_type); st = hipFreeHost(test_ptr); if (st != hipSuccess) { ADD_FAILURE() << "failed to free host mapped memory"; } } ucc-1.8.0/test/gtest/core/test_context.h0000664000175000017500000000130115211535620020402 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef UCC_TEST_CONTEXT_H #define UCC_TEST_CONTEXT_H #include class test_context_config : public ucc::test { public: test_context_config(); ~test_context_config(); ucc_lib_config_h lib_config; ucc_lib_params_t lib_params; ucc_lib_h lib_h; }; class test_context : public test_context_config { public: ucc_context_config_h ctx_config; test_context(); ~test_context(); }; class test_context_get_attr : public test_context { public: ucc_context_h ctx_h; test_context_get_attr(); ~test_context_get_attr(); }; #endif ucc-1.8.0/test/gtest/core/test_lib_config.cc0000664000175000017500000000362215211535620021157 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include class test_lib_config : public ucc::test { }; UCC_TEST_F(test_lib_config, read_release) { ucc_lib_config_h cfg; EXPECT_EQ(UCC_OK, ucc_lib_config_read(NULL, NULL, &cfg)); ucc_lib_config_release(cfg); } UCC_TEST_F(test_lib_config, print) { ucc_lib_config_h cfg; unsigned flags; testing::internal::CaptureStdout(); EXPECT_EQ(UCC_OK, ucc_lib_config_read(NULL, NULL, &cfg)); flags = UCC_CONFIG_PRINT_CONFIG | UCC_CONFIG_PRINT_HEADER | UCC_CONFIG_PRINT_DOC | UCC_CONFIG_PRINT_HIDDEN; ucc_lib_config_print(cfg, stdout, "TEST_TITLE", (ucc_config_print_flags_t)flags); ucc_lib_config_release(cfg); std::string output = testing::internal::GetCapturedStdout(); EXPECT_NE(std::string::npos, output.find("# TEST_TITLE")); EXPECT_NE(std::string::npos, output.find("UCC_CLS=")); } UCC_TEST_F(test_lib_config, modify) { ucc_lib_config_h cfg; unsigned flags; testing::internal::CaptureStdout(); EXPECT_EQ(UCC_OK, ucc_lib_config_read(NULL, NULL, &cfg)); /* modify known field to expected value */ EXPECT_EQ(UCC_OK, ucc_lib_config_modify(cfg, "CLS", "basic")); flags = UCC_CONFIG_PRINT_CONFIG; ucc_lib_config_print(cfg, stdout, "", (ucc_config_print_flags_t)flags); std::string output = testing::internal::GetCapturedStdout(); EXPECT_NE(std::string::npos, output.find("UCC_CLS=basic")); /* modify known field to unexpected value */ /* temporarily commented out due to a bug in UCS which results in segv */ // EXPECT_NE(UCC_OK, ucc_lib_config_modify(cfg, "CLS", "_unknown_value")); /* modify uknown field */ EXPECT_NE(UCC_OK, ucc_lib_config_modify(cfg, "_UNKNOWN_FIELD", "_unknown_value")); ucc_lib_config_release(cfg); } ucc-1.8.0/test/gtest/core/test_context.cc0000664000175000017500000000546415211535620020556 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_context.h" #include "../common/test_ucc.h" #include #include #include test_context::test_context() { EXPECT_EQ(UCC_OK, ucc_context_config_read(lib_h, NULL, &ctx_config)); } test_context::~test_context() { ucc_context_config_release(ctx_config); } UCC_TEST_F(test_context, create_destroy) { ucc_context_params_t ctx_params; ucc_context_h ctx_h; ctx_params.mask = UCC_CONTEXT_PARAM_FIELD_TYPE; ctx_params.type = UCC_CONTEXT_EXCLUSIVE; EXPECT_EQ(UCC_OK, ucc_context_create(lib_h, &ctx_params, ctx_config, &ctx_h)); EXPECT_EQ(UCC_OK, ucc_context_destroy(ctx_h)); } UCC_TEST_F(test_context, init_multiple) { const int n_ctxs = 8; ucc_context_params_t ctx_params; ucc_context_h ctx_h; std::vector ctxs; ctx_params.mask = UCC_CONTEXT_PARAM_FIELD_TYPE; ctx_params.type = UCC_CONTEXT_EXCLUSIVE; for (int i = 0; i < n_ctxs; i++) { EXPECT_EQ(UCC_OK, ucc_context_create(lib_h, &ctx_params, ctx_config, &ctx_h)); ctxs.push_back(ctx_h); } std::shuffle(ctxs.begin(), ctxs.end(), std::default_random_engine()); for (auto ctx_h : ctxs) { EXPECT_EQ(UCC_OK, ucc_context_destroy(ctx_h)); } } test_context_get_attr::test_context_get_attr() { ucc_context_params_t ctx_params; ctx_params.mask = UCC_CONTEXT_PARAM_FIELD_TYPE; ctx_params.type = UCC_CONTEXT_EXCLUSIVE; EXPECT_EQ(UCC_OK, ucc_context_create(lib_h, &ctx_params, ctx_config, &ctx_h)); } test_context_get_attr::~test_context_get_attr() { EXPECT_EQ(UCC_OK, ucc_context_destroy(ctx_h)); } UCC_TEST_F(test_context_get_attr, addr_len) { ucc_context_attr_t attr; attr.mask = UCC_CONTEXT_ATTR_FIELD_CTX_ADDR_LEN; EXPECT_EQ(UCC_OK, ucc_context_get_attr(ctx_h, &attr)); } UCC_TEST_F(test_context_get_attr, addr) { ucc_context_attr_t attr; attr.mask = UCC_CONTEXT_ATTR_FIELD_CTX_ADDR; EXPECT_EQ(UCC_OK, ucc_context_get_attr(ctx_h, &attr)); EXPECT_EQ(true, ((attr.ctx_addr_len == 0) || (NULL != attr.ctx_addr))); } UCC_TEST_F(test_context_get_attr, work_buffer_size) { ucc_context_attr_t attr; attr.mask = UCC_CONTEXT_ATTR_FIELD_WORK_BUFFER_SIZE; EXPECT_EQ(UCC_OK, ucc_context_get_attr(ctx_h, &attr)); EXPECT_EQ(5, attr.global_work_buffer_size); } UCC_TEST_F(test_context, global) { /* Create and cleanup several Jobs (ucc contextss) with OOB */ UccJob job1(1, UccJob::UCC_JOB_CTX_GLOBAL); job1.cleanup(); UccJob job3(3, UccJob::UCC_JOB_CTX_GLOBAL); job3.cleanup(); UccJob job11(11, UccJob::UCC_JOB_CTX_GLOBAL); job11.cleanup(); UccJob job16(16, UccJob::UCC_JOB_CTX_GLOBAL); job16.cleanup(); } ucc-1.8.0/test/gtest/core/test_schedule.cc0000664000175000017500000000702715211535620020663 0ustar alastairalastair/** * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include extern "C" { #include "schedule/ucc_schedule.h" } class test_coll_task : public ucc_coll_task_t { public: test_coll_task() { ucc_coll_task_construct(this); EXPECT_EQ(UCC_OK, ucc_coll_task_init((ucc_coll_task_t *)this, NULL, NULL)); } ~test_coll_task() { ucc_coll_task_destruct(this); } }; typedef std::tuple rst_t; class test_schedule : public test_coll_task, public ucc::test { public: std::vector rst; static ucc_status_t handler_1(ucc_coll_task_t *parent, ucc_coll_task_t *task) { test_schedule *ts = (test_schedule*)task; ts->rst.push_back(rst_t((test_coll_task*)parent, 1)); return UCC_OK; } static ucc_status_t handler_2(ucc_coll_task_t *parent, ucc_coll_task_t *task) { test_schedule *ts = (test_schedule*)task; ts->rst.push_back(rst_t((test_coll_task*)parent, 2)); return UCC_OK; } }; /* Tasks subscribes on 2 tasks to EVENT_COMPLETED with the same handler */ UCC_TEST_F(test_schedule, single_handler) { std::vector tasks(2); for (auto &t : tasks) { ucc_event_manager_subscribe(&t, UCC_EVENT_COMPLETED, (ucc_coll_task_t*)this, test_schedule::handler_1); } for (auto &t : tasks) { EXPECT_EQ(UCC_OK, ucc_event_manager_notify(&t, UCC_EVENT_COMPLETED)); } EXPECT_EQ(2, rst.size()); EXPECT_EQ(true, (std::get<0>(rst[0]) == &tasks[0]) && (std::get<1>(rst[0]) == 1)); EXPECT_EQ(true, (std::get<0>(rst[1]) == &tasks[1]) && (std::get<1>(rst[1]) == 1)); } /* Tasks subscribes on 2 tasks to EVENT_COMPLETED with 2 different handlers */ UCC_TEST_F(test_schedule, different_handlers) { std::vector tasks(2); ucc_event_manager_subscribe(&tasks[0], UCC_EVENT_COMPLETED, (ucc_coll_task_t*)this, test_schedule::handler_1); ucc_event_manager_subscribe(&tasks[1], UCC_EVENT_COMPLETED, (ucc_coll_task_t*)this, test_schedule::handler_2); for (auto &t : tasks) { EXPECT_EQ(UCC_OK, ucc_event_manager_notify(&t, UCC_EVENT_COMPLETED)); } EXPECT_EQ(2, rst.size()); EXPECT_EQ(true, (std::get<0>(rst[0]) == &tasks[0]) && (std::get<1>(rst[0]) == 1)); EXPECT_EQ(true, (std::get<0>(rst[1]) == &tasks[1]) && (std::get<1>(rst[1]) == 2)); } /* Tasks subscribes to multiple tasks exceeding MAX_LISTENERS */ UCC_TEST_F(test_schedule, multiple) { const int n_subscribers = 16; std::vector tasks(n_subscribers); for (int i = 0; i < n_subscribers; i++) { ucc_event_manager_subscribe(&tasks[i], UCC_EVENT_COMPLETED, (ucc_coll_task_t*)this, ((i % 2) == 0 ? test_schedule::handler_1 : test_schedule::handler_2)); } for (auto &t : tasks) { EXPECT_EQ(UCC_OK, ucc_event_manager_notify(&t, UCC_EVENT_COMPLETED)); } EXPECT_EQ(n_subscribers, rst.size()); for (int i = 0; i < n_subscribers; i++) { EXPECT_EQ(true, (std::get<0>(rst[i]) == &tasks[i]) && (std::get<1>(rst[i]) == ((i % 2) + 1))); } } ucc-1.8.0/test/gtest/core/test_mem_map.cc0000664000175000017500000003022515211535620020476 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_mem_map.h" #include #include test_mem_map::test_mem_map() : ctx_h(nullptr), ctx_config(nullptr) { memset(&ctx_params, 0, sizeof(ctx_params)); } test_mem_map::~test_mem_map() { } void test_mem_map::SetUp() { test_context_config::SetUp(); EXPECT_EQ(UCC_OK, ucc_context_config_read(lib_h, NULL, &ctx_config)); ctx_params.mask = UCC_CONTEXT_PARAM_FIELD_TYPE; ctx_params.type = UCC_CONTEXT_EXCLUSIVE; EXPECT_EQ(UCC_OK, ucc_context_create(lib_h, &ctx_params, ctx_config, &ctx_h)); } void test_mem_map::TearDown() { if (ctx_h) { EXPECT_EQ(UCC_OK, ucc_context_destroy(ctx_h)); } if (ctx_config) { ucc_context_config_release(ctx_config); } test_context_config::TearDown(); } test_mem_map_export::test_mem_map_export() : test_buffer(nullptr), buffer_size(0) { memset(&map_params, 0, sizeof(map_params)); memset(&segment, 0, sizeof(segment)); } test_mem_map_export::~test_mem_map_export() { } void test_mem_map_export::SetUp() { test_mem_map::SetUp(); /* Allocate test buffer */ buffer_size = 1024 * 1024; /* 1MB */ test_buffer = malloc(buffer_size); ASSERT_NE(nullptr, test_buffer); /* Initialize buffer with test data */ memset(test_buffer, 0xAA, buffer_size); /* Set up memory map parameters */ segment.address = test_buffer; segment.len = buffer_size; map_params.segments = &segment; map_params.n_segments = 1; } void test_mem_map_export::TearDown() { if (test_buffer) { free(test_buffer); test_buffer = nullptr; } test_mem_map::TearDown(); } test_mem_map_import::test_mem_map_import() : test_buffer(nullptr), buffer_size(0), memh(nullptr), memh_size(0) { } test_mem_map_import::~test_mem_map_import() { } void test_mem_map_import::SetUp() { test_mem_map::SetUp(); /* Allocate test buffer */ buffer_size = 1024 * 1024; /* 1MB */ test_buffer = malloc(buffer_size); ASSERT_NE(nullptr, test_buffer); /* Initialize buffer with test data */ memset(test_buffer, 0xBB, buffer_size); } void test_mem_map_import::TearDown() { if (test_buffer) { free(test_buffer); test_buffer = nullptr; } test_mem_map::TearDown(); } /* Test basic memory map export functionality */ UCC_TEST_F(test_mem_map_export, basic_export) { ucc_mem_map_mem_h memh = nullptr; size_t memh_size = 0; EXPECT_EQ(UCC_OK, ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_EXPORT, &map_params, &memh_size, &memh)); EXPECT_NE(nullptr, memh); EXPECT_GT(memh_size, 0); /* Test unmap */ EXPECT_EQ(UCC_OK, ucc_mem_unmap(&memh)); /* Note: ucc_mem_unmap doesn't set memh to nullptr, it only frees the memory */ } /* Test memory map export with different buffer sizes */ UCC_TEST_F(test_mem_map_export, different_sizes) { std::vector sizes = {1024, 4096, 65536, 1024 * 1024}; for (auto size : sizes) { /* Reallocate buffer with new size */ if (test_buffer) { free(test_buffer); } test_buffer = malloc(size); ASSERT_NE(nullptr, test_buffer); memset(test_buffer, 0xCC, size); segment.address = test_buffer; segment.len = size; ucc_mem_map_mem_h memh = nullptr; size_t memh_size = 0; EXPECT_EQ(UCC_OK, ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_EXPORT, &map_params, &memh_size, &memh)); EXPECT_NE(nullptr, memh); EXPECT_GT(memh_size, 0); EXPECT_EQ(UCC_OK, ucc_mem_unmap(&memh)); } } /* Test memory map export with multiple segments (should fail as UCC only supports one segment) */ UCC_TEST_F(test_mem_map_export, multiple_segments) { ucc_mem_map_mem_h memh = nullptr; size_t memh_size = 0; ucc_mem_map_t segments[2]; ucc_mem_map_params_t multi_params; /* Create two segments */ segments[0].address = test_buffer; segments[0].len = buffer_size / 2; segments[1].address = (char *)test_buffer + buffer_size / 2; segments[1].len = buffer_size / 2; multi_params.segments = segments; multi_params.n_segments = 2; /* This should fail as UCC only supports one segment per call */ EXPECT_EQ(UCC_ERR_INVALID_PARAM, ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_EXPORT, &multi_params, &memh_size, &memh)); EXPECT_EQ(nullptr, memh); } /* Test memory map export with invalid parameters */ UCC_TEST_F(test_mem_map_export, invalid_params) { ucc_mem_map_mem_h memh = nullptr; size_t memh_size = 0; /* Test with NULL params */ EXPECT_EQ(UCC_ERR_INVALID_PARAM, ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_EXPORT, nullptr, &memh_size, &memh)); /* Test with invalid mode */ ucc_mem_map_mode_t invalid_mode = UCC_MEM_MAP_MODE_LAST; EXPECT_EQ(UCC_ERR_INVALID_PARAM, ucc_mem_map(ctx_h, invalid_mode, &map_params, &memh_size, &memh)); } /* Test memory map export with zero length buffer */ UCC_TEST_F(test_mem_map_export, zero_length) { ucc_mem_map_mem_h memh = nullptr; size_t memh_size = 0; segment.len = 0; /* This might succeed or fail depending on implementation */ ucc_status_t status = ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_EXPORT, &map_params, &memh_size, &memh); if (status == UCC_OK) { EXPECT_NE(nullptr, memh); EXPECT_EQ(UCC_OK, ucc_mem_unmap(&memh)); } } /* Test memory map import functionality */ UCC_TEST_F(test_mem_map_import, basic_import) { ucc_mem_map_mem_h export_memh = nullptr; size_t export_memh_size = 0; ucc_mem_map_mem_h import_memh = nullptr; size_t import_memh_size = 0; ucc_mem_map_t export_segment; ucc_mem_map_params_t export_params; ucc_status_t export_status; ucc_status_t import_status; export_segment.address = test_buffer; export_segment.len = buffer_size; export_params.segments = &export_segment; export_params.n_segments = 1; /* Export the memory handle */ export_status = ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_EXPORT, &export_params, &export_memh_size, &export_memh); if (export_status != UCC_OK) { /* If export fails, skip the test */ GTEST_SKIP() << "Export failed, skipping import test"; return; } EXPECT_NE(nullptr, export_memh); EXPECT_GT(export_memh_size, 0); /* For import, we need to create a new memory handle with the exported data */ /* The import function expects the memh to be pre-allocated with the exported data */ import_memh = (ucc_mem_map_mem_h)malloc(export_memh_size); ASSERT_NE(nullptr, import_memh); memcpy(import_memh, export_memh, export_memh_size); import_status = ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_IMPORT, &export_params, &import_memh_size, &import_memh); if (import_status == UCC_OK) { EXPECT_NE(nullptr, import_memh); /* Cleanup import */ ucc_mem_unmap(&import_memh); } else { /* Import might not be supported, which is acceptable */ EXPECT_TRUE(import_status == UCC_ERR_NOT_SUPPORTED || import_status == UCC_ERR_NOT_IMPLEMENTED); /* Clean up the allocated memory if import failed */ free(import_memh); } /* Cleanup export */ ucc_mem_unmap(&export_memh); } /* Test memory map import with different buffer sizes */ UCC_TEST_F(test_mem_map_import, import_different_sizes) { std::vector sizes = {1024, 4096, 65536, 1024 * 1024}; ucc_mem_map_mem_h export_memh = nullptr; size_t export_memh_size = 0; ucc_mem_map_t export_segment; ucc_mem_map_params_t export_params; ucc_mem_map_mem_h import_memh; size_t import_memh_size; ucc_status_t import_status; for (auto size : sizes) { /* Reallocate buffer with new size */ if (test_buffer) { free(test_buffer); } test_buffer = malloc(size); ASSERT_NE(nullptr, test_buffer); memset(test_buffer, 0xDD, size); export_segment.address = test_buffer; export_segment.len = size; export_params.segments = &export_segment; export_params.n_segments = 1; /* Export the memory handle */ ucc_status_t export_status = ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_EXPORT, &export_params, &export_memh_size, &export_memh); if (export_status != UCC_OK) { continue; /* Skip this size if export fails */ } EXPECT_NE(nullptr, export_memh); EXPECT_GT(export_memh_size, 0); /* Test import */ import_memh = (ucc_mem_map_mem_h)malloc(export_memh_size); ASSERT_NE(nullptr, import_memh); memcpy(import_memh, export_memh, export_memh_size); import_status = ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_IMPORT, &export_params, &import_memh_size, &import_memh); if (import_status == UCC_OK) { EXPECT_NE(nullptr, import_memh); ucc_mem_unmap(&import_memh); } else { /* Import might not be supported for all sizes */ EXPECT_TRUE(import_status == UCC_ERR_NOT_SUPPORTED || import_status == UCC_ERR_NOT_IMPLEMENTED); /* Clean up the allocated memory if import failed */ free(import_memh); } /* Cleanup export */ ucc_mem_unmap(&export_memh); } } /* Test memory map import with invalid parameters */ UCC_TEST_F(test_mem_map_import, import_invalid_params) { /* Test import with NULL params */ ucc_mem_map_mem_h memh = nullptr; size_t memh_size = 0; ucc_mem_map_params_t params; ucc_mem_map_t segment; EXPECT_EQ(UCC_ERR_INVALID_PARAM, ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_IMPORT, nullptr, &memh_size, &memh)); /* Test import with NULL memh */ segment.address = test_buffer; segment.len = buffer_size; params.segments = &segment; params.n_segments = 1; EXPECT_EQ(UCC_ERR_INVALID_PARAM, ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_IMPORT, ¶ms, &memh_size, nullptr)); } /* Test memory map unmap with NULL handle */ UCC_TEST_F(test_mem_map_export, unmap_null) { ucc_mem_map_mem_h memh = nullptr; /* Should handle NULL gracefully */ EXPECT_EQ(UCC_ERR_INVALID_PARAM, ucc_mem_unmap(&memh)); } /* Test memory map with different modes */ UCC_TEST_F(test_mem_map_export, different_modes) { ucc_mem_map_mem_h memh1 = nullptr; size_t memh_size = 0; /* Test EXPORT mode */ EXPECT_EQ(UCC_OK, ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_EXPORT, &map_params, &memh_size, &memh1)); EXPECT_NE(nullptr, memh1); EXPECT_EQ(UCC_OK, ucc_mem_unmap(&memh1)); } /* Test memory map stress test */ UCC_TEST_F(test_mem_map_export, stress_test) { const int num_iterations = 100; ucc_mem_map_mem_h memh; size_t memh_size; for (int i = 0; i < num_iterations; i++) { memh = nullptr; memh_size = 0; /* Fill buffer with iteration-specific pattern */ memset(test_buffer, i % 256, buffer_size); EXPECT_EQ(UCC_OK, ucc_mem_map(ctx_h, UCC_MEM_MAP_MODE_EXPORT, &map_params, &memh_size, &memh)); EXPECT_NE(nullptr, memh); EXPECT_EQ(UCC_OK, ucc_mem_unmap(&memh)); } } ucc-1.8.0/test/gtest/core/test_mc_reduce.cc0000664000175000017500000003765015211535620021022 0ustar alastairalastair/** * Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #include "test_mc_reduce.h" extern "C" { #include "components/ec/ucc_ec.h" } #ifdef HAVE_CUDA #include #endif template class test_mc_reduce : public testing::Test { protected: const int COUNT = 1024; ucc_memory_type_t mem_type; ucc_ee_executor_t *executor; void *ee_context = NULL; virtual void SetUp() override { ucc_constructor(); ucc_mc_params_t mc_params = { .thread_mode = UCC_THREAD_SINGLE, }; ucc_ec_params_t ec_params = { .thread_mode = UCC_THREAD_SINGLE, }; ucc_mc_init(&mc_params); ucc_ec_init(&ec_params); buf1_h = buf2_h = res_h = nullptr; buf1_d = buf2_d = res_d = nullptr; executor = nullptr; } ucc_status_t alloc_executor(ucc_memory_type_t mtype) { ucc_ee_executor_params_t params; ucc_ee_type_t coll_ee_type; ucc_status_t status; switch (mtype) { case UCC_MEMORY_TYPE_CUDA: coll_ee_type = UCC_EE_CUDA_STREAM; #ifdef HAVE_CUDA if (triggered) { cudaStream_t stream; if (cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking) != cudaSuccess) { std::cerr << "failed to create cuda stream" << std::endl; return UCC_ERR_NO_RESOURCE; } ee_context = (void *)stream; } #endif break; case UCC_MEMORY_TYPE_CUDA_MANAGED: coll_ee_type = UCC_EE_CUDA_STREAM; break; case UCC_MEMORY_TYPE_HOST: coll_ee_type = UCC_EE_CPU_THREAD; break; default: std::cerr << "invalid executor mem type\n"; return UCC_ERR_INVALID_PARAM; break; } params.mask = UCC_EE_EXECUTOR_PARAM_FIELD_TYPE; params.ee_type = coll_ee_type; status = ucc_ee_executor_init(¶ms, &executor); if (UCC_OK != status) { std::cerr << "failed to init executor: " << ucc_status_string(status) << std::endl; return status; } status = ucc_ee_executor_start(executor, ee_context); if (UCC_OK != status) { std::cerr << "failed to start executor: " << ucc_status_string(status) << std::endl; ucc_ee_executor_finalize(executor); } return status; } ucc_status_t free_executor() { ucc_status_t status; status = ucc_ee_executor_stop(executor); if (UCC_OK != status) { std::cerr << "failed to stop executor: " << ucc_status_string(status) << std::endl; } ucc_ee_executor_finalize(executor); #ifdef HAVE_CUDA if (triggered) { if (cudaStreamDestroy((cudaStream_t)ee_context) != cudaSuccess) { std::cerr << "failed to destory cuda stream" << std::endl; return UCC_ERR_NO_MESSAGE; } ee_context = NULL; } #endif return status; } ucc_status_t setup(ucc_memory_type_t mtype, size_t n) { ucc_status_t status; status = alloc_bufs(mtype, n); if (UCC_OK != status) { return status; } return alloc_executor(mtype); } ucc_status_t alloc_bufs(ucc_memory_type_t mtype, size_t n) { size_t n_bytes = COUNT*sizeof(typename T::type); mem_type = mtype; ucc_mc_alloc(&res_h_mc_header, n_bytes, UCC_MEMORY_TYPE_HOST); res_h = (typename T::type *)res_h_mc_header->addr; ucc_mc_alloc(&buf1_h_mc_header, n_bytes, UCC_MEMORY_TYPE_HOST); buf1_h = (typename T::type *)buf1_h_mc_header->addr; ucc_mc_alloc(&buf2_h_mc_header, n * n_bytes, UCC_MEMORY_TYPE_HOST); buf2_h = (typename T::type *)buf2_h_mc_header->addr; for (int i = 0; i < COUNT; i++) { res_h[i] = (typename T::type)(0); } for (int i = 0; i < COUNT; i++) { /* bFloat16 will be assigned with the floats matching the uint16_t bit pattern*/ buf1_h[i] = (typename T::type)(i + 1); } for (int j = 0; j < n; j++) { for (int i = 0; i < COUNT; i++) { buf2_h[i + j * COUNT] = (typename T::type)(2 * i + j + 1); } } if (mtype != UCC_MEMORY_TYPE_HOST) { ucc_mc_alloc(&res_d_mc_header, n_bytes, mtype); res_d = (typename T::type *)res_d_mc_header->addr; ucc_mc_alloc(&buf1_d_mc_header, n_bytes, mtype); buf1_d = (typename T::type *)buf1_d_mc_header->addr; ucc_mc_alloc(&buf2_d_mc_header, n * n_bytes, mtype); buf2_d = (typename T::type *)buf2_d_mc_header->addr; ucc_mc_memcpy(res_d, res_h, n_bytes, mtype, UCC_MEMORY_TYPE_HOST); ucc_mc_memcpy(buf1_d, buf1_h, n_bytes, mtype, UCC_MEMORY_TYPE_HOST); ucc_mc_memcpy(buf2_d, buf2_h, n * n_bytes, mtype, UCC_MEMORY_TYPE_HOST); buf1 = buf1_d; buf2 = buf2_d; res = res_d; } else { buf1 = buf1_h; buf2 = buf2_h; res = res_h; } return UCC_OK; } ucc_status_t free_bufs(ucc_memory_type_t mtype) { if (buf1_h != nullptr) { ucc_mc_free(buf1_h_mc_header); } if (buf2_h != nullptr) { ucc_mc_free(buf2_h_mc_header); } if (res_h != nullptr) { ucc_mc_free(res_h_mc_header); } if (buf1_d != nullptr) { ucc_mc_free(buf1_d_mc_header); } if (buf2_d != nullptr) { ucc_mc_free(buf2_d_mc_header); } if (res_d != nullptr) { ucc_mc_free(res_d_mc_header); } return UCC_OK; } virtual void TearDown() override { free_bufs(mem_type); ucc_mc_finalize(); } ucc_status_t do_reduce(void *src1, void *src2, void *dst, size_t count, uint16_t n_src2, size_t stride, ucc_datatype_t dt, ucc_reduction_op_t op, bool with_alpha, double alpha) { ucc_ee_executor_task_args_t eargs; ucc_status_t status; ucc_ee_executor_task_t * task; eargs.flags = with_alpha ? UCC_EEE_TASK_FLAG_REDUCE_WITH_ALPHA : 0; eargs.task_type = UCC_EE_EXECUTOR_TASK_REDUCE_STRIDED; eargs.reduce_strided.count = count; eargs.reduce_strided.dt = dt; eargs.reduce_strided.op = op; eargs.reduce_strided.n_src2 = n_src2; eargs.reduce_strided.dst = dst; eargs.reduce_strided.src1 = src1; eargs.reduce_strided.src2 = src2; eargs.reduce_strided.stride = stride; eargs.reduce_strided.alpha = alpha; status = ucc_ee_executor_task_post(executor, &eargs, &task); if (UCC_OK != status) { std::cerr << "failed to post executor task: " << ucc_status_string(status) << std::endl; return status; } while (0 < (status = ucc_ee_executor_task_test(task))) { ; } ucc_ee_executor_task_finalize(task); return status; } void test_reduce(ucc_memory_type_t mt) { ucc_status_t status; if (UCC_OK != ucc_mc_available(mt)) { GTEST_SKIP(); } ASSERT_EQ(this->setup(mt, 1), UCC_OK); status = do_reduce(this->buf1, this->buf2, this->res, this->COUNT, 1, 0, T::dt, T::redop, false, 0); if (UCC_ERR_NOT_SUPPORTED == status) { GTEST_SKIP(); } ASSERT_EQ(status, UCC_OK); if (executor) { free_executor(); } if (mt != UCC_MEMORY_TYPE_HOST) { ucc_mc_memcpy(this->res_h, this->res_d, this->COUNT * sizeof(*this->res_d), UCC_MEMORY_TYPE_HOST, mt); } for (int i = 0; i < this->COUNT; i++) { T::assert_equal(T::do_op(this->buf1_h[i], this->buf2_h[i]), this->res_h[i]); } }; void test_reduce_multi(ucc_memory_type_t mt) { const int num_vec = 3; ucc_status_t status; if (UCC_OK != ucc_mc_available(mt)) { GTEST_SKIP(); } ASSERT_EQ(this->setup(mt, num_vec), UCC_OK); status = do_reduce(this->buf1, this->buf2, this->res, this->COUNT, num_vec, this->COUNT * sizeof(*this->buf2), T::dt, T::redop, false, 0); if (UCC_ERR_NOT_SUPPORTED == status) { GTEST_SKIP(); } ASSERT_EQ(status, UCC_OK); if (executor) { free_executor(); } if (mt != UCC_MEMORY_TYPE_HOST) { ucc_mc_memcpy(this->res_h, this->res_d, this->COUNT * sizeof(*this->res_d), UCC_MEMORY_TYPE_HOST, mt); } for (int i = 0; i < this->COUNT; i++) { typename T::type res = T::do_op(this->buf1_h[i], this->buf2_h[i]); for (int j = 1; j < num_vec; j++) { res = T::do_op(this->buf2_h[i + j * this->COUNT], res); } T::assert_equal(res, this->res_h[i]); } }; void test_reduce_multi_alpha(ucc_memory_type_t mt) { const int num_vec = 20; const double alpha = 0.7; ucc_status_t status; if (UCC_OK != ucc_mc_available(mt)) { GTEST_SKIP(); } ASSERT_EQ(UCC_OK, this->setup(mt, num_vec)); status = do_reduce(this->buf1, this->buf2, this->res, this->COUNT, num_vec, this->COUNT * sizeof(*this->buf2), T::dt, T::redop, true, alpha); if (UCC_ERR_NOT_SUPPORTED == status) { GTEST_SKIP(); } ASSERT_EQ(status, UCC_OK); if (executor) { free_executor(); } if (mt != UCC_MEMORY_TYPE_HOST) { ucc_mc_memcpy(this->res_h, this->res_d, this->COUNT * sizeof(*this->res_d), UCC_MEMORY_TYPE_HOST, mt); } for (int i = 0; i < this->COUNT; i++) { typename T::type res = T::do_op(this->buf1_h[i], this->buf2_h[i]); for (int j = 1; j < num_vec; j++) { res = T::do_op(this->buf2_h[i + j * this->COUNT], res); } if (T::dt == UCC_DT_BFLOAT16) { float32tobfloat16(bfloat16tofloat32(&res)*(float)alpha, &res); } else { res *= (typename T::type)alpha; } T::assert_equal(res, this->res_h[i]); } } ucc_mc_buffer_header_t *buf1_h_mc_header, *buf2_h_mc_header, *res_h_mc_header, *buf1_d_mc_header, *buf2_d_mc_header, *res_d_mc_header; typename T::type *buf1_h; typename T::type *buf2_h; typename T::type *res_h; typename T::type *buf1_d; typename T::type *buf2_d; typename T::type *res_d; typename T::type *buf1; typename T::type *buf2; typename T::type *res; }; #define INT_OP_PAIRS(_TYPE) ARITHMETIC_OP_PAIRS(_TYPE), \ TypeOpPair, \ TypeOpPair, \ TypeOpPair, \ TypeOpPair, \ TypeOpPair, \ TypeOpPair using TypeOpPairsInt = ::testing::Types; using TypeOpPairsUint = ::testing::Types; using TypeOpPairsFloat = ::testing::Types, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair>; using TypeOpPairsFloatCuda = ::testing::Types< ARITHMETIC_OP_PAIRS(FLOAT32), ARITHMETIC_OP_PAIRS(FLOAT64), ARITHMETIC_OP_PAIRS(BFLOAT16), TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair, TypeOpPair>; template class test_mc_reduce_int : public test_mc_reduce {}; TYPED_TEST_CASE(test_mc_reduce_int, TypeOpPairsInt); template class test_mc_reduce_uint : public test_mc_reduce {}; TYPED_TEST_CASE(test_mc_reduce_uint, TypeOpPairsUint); template class test_mc_reduce_float : public test_mc_reduce {}; TYPED_TEST_CASE(test_mc_reduce_float, TypeOpPairsFloat); #define DECLARE_REDUCE_TEST(_type, _mt) \ TYPED_TEST(test_mc_reduce_ ## _type, _mt) { \ this->test_reduce(UCC_MEMORY_TYPE_ ## _mt); \ } \ #define DECLARE_REDUCE_MULTI_TEST(_type, _mt) \ TYPED_TEST(test_mc_reduce_ ## _type, multi_ ## _mt) { \ this->test_reduce_multi(UCC_MEMORY_TYPE_ ## _mt); \ } \ #define DECLARE_REDUCE_MULTI_ALPHA_TEST(_type, _mt) \ TYPED_TEST(test_mc_reduce_ ## _type, multi_alpha_ ## _mt) { \ this->test_reduce_multi_alpha(UCC_MEMORY_TYPE_ ## _mt); \ } \ DECLARE_REDUCE_TEST(int, HOST); DECLARE_REDUCE_TEST(uint, HOST); DECLARE_REDUCE_TEST(float, HOST); DECLARE_REDUCE_MULTI_TEST(int, HOST); DECLARE_REDUCE_MULTI_TEST(uint, HOST); DECLARE_REDUCE_MULTI_TEST(float, HOST); DECLARE_REDUCE_MULTI_ALPHA_TEST(float, HOST); #ifdef HAVE_CUDA DECLARE_REDUCE_TEST(int, CUDA); DECLARE_REDUCE_TEST(uint, CUDA); DECLARE_REDUCE_TEST(float, CUDA); DECLARE_REDUCE_MULTI_TEST(int, CUDA); DECLARE_REDUCE_MULTI_TEST(uint, CUDA); DECLARE_REDUCE_MULTI_TEST(float, CUDA); DECLARE_REDUCE_MULTI_ALPHA_TEST(float, CUDA); template class test_mc_reduce_int_triggered : public test_mc_reduce {}; TYPED_TEST_CASE(test_mc_reduce_int_triggered, TypeOpPairsInt); template class test_mc_reduce_uint_triggered : public test_mc_reduce {}; TYPED_TEST_CASE(test_mc_reduce_uint_triggered, TypeOpPairsUint); template class test_mc_reduce_float_triggered : public test_mc_reduce {}; TYPED_TEST_CASE(test_mc_reduce_float_triggered, TypeOpPairsFloatCuda); DECLARE_REDUCE_TEST(int_triggered, CUDA); DECLARE_REDUCE_TEST(uint_triggered, CUDA); DECLARE_REDUCE_TEST(float_triggered, CUDA); DECLARE_REDUCE_MULTI_TEST(int_triggered, CUDA); DECLARE_REDUCE_MULTI_TEST(uint_triggered, CUDA); DECLARE_REDUCE_MULTI_TEST(float_triggered, CUDA); DECLARE_REDUCE_MULTI_ALPHA_TEST(float_triggered, CUDA); #endif ucc-1.8.0/test/gtest/core/test_topo.cc0000664000175000017500000006040615211535620020050 0ustar alastairalastair/** * Copyright (c) 2020, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ extern "C" { #include #include #include } #include #include #include #include class addr_storage { public: ucc_addr_storage_t storage; std::vector h; addr_storage(int size) { h.resize(size); storage.storage = h.data(); storage.size = size; storage.addr_len = sizeof(ucc_context_addr_header_t); } }; class test_topo : public ucc::test { public: ucc_context_topo_t *ctx_topo; ucc_topo_t * topo; test_topo() { ucc_constructor(); } ~test_topo() { ucc_topo_cleanup(topo); ucc_context_topo_cleanup(ctx_topo); } bool check_sbgp(ucc_sbgp_t *sbgp, std::vector r) { EXPECT_EQ(sbgp->group_size, r.size()); for (int i = 0; i < r.size(); i++) { if (ucc_ep_map_eval(sbgp->map, i) != r[i]) { return false; } } return true; } }; #define SET_PI(_s, _i, _host, _sock, _pid) \ _s.h[_i].ctx_id.pi.host_hash = _host; \ _s.h[_i].ctx_id.pi.socket_id = _sock; \ _s.h[_i].ctx_id.pi.pid = _pid; UCC_TEST_F(test_topo, single_node) { const ucc_rank_t ctx_size = 4; addr_storage s(ctx_size); ucc_sbgp_t * sbgp; ucc_subset_t set; /* simulates world proc array */ SET_PI(s, 0, 0xabcd, 0, 0); SET_PI(s, 1, 0xabcd, 0, 1); SET_PI(s, 2, 0xabcd, 0, 2); SET_PI(s, 3, 0xabcd, 0, 3); /* subset from the world */ set.map.ep_num = ctx_size; set.myrank = 0; set.map.type = UCC_EP_MAP_FULL; /* Init topo for such subset */ EXPECT_EQ(UCC_OK, ucc_context_topo_init(&s.storage, &ctx_topo)); EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); /* Check subgroups */ /* NODE subgroup - ALL on the same node*/ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(sbgp->group_size, ctx_size); EXPECT_EQ(sbgp->group_rank, 0); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_FULL); /* NODE_LEADERS subgroup */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); EXPECT_EQ(UCC_SBGP_NOT_EXISTS, sbgp->status); /* SOCKET subgroup - ALL on the same socket */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(sbgp->group_size, ctx_size); EXPECT_EQ(sbgp->group_rank, 0); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_FULL); /* SOCKET_LEADERS subgroup - just 1 socket - no socket_leaders group*/ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET_LEADERS); EXPECT_EQ(UCC_SBGP_NOT_EXISTS, sbgp->status); } UCC_TEST_F(test_topo, node_reordered) { const ucc_rank_t ctx_size = 4; const ucc_rank_t team_size = 3; ucc_rank_t team_ranks[team_size] = {2, 3, 1}; addr_storage s(ctx_size); ucc_sbgp_t * sbgp; ucc_subset_t set; /* simulates world proc array */ SET_PI(s, 0, 0xabcd, 0, 0); SET_PI(s, 1, 0xabcd, 0, 1); SET_PI(s, 2, 0xabcd, 0, 2); SET_PI(s, 3, 0xabcd, 0, 3); /* subset from the world */ set.map.ep_num = team_size; set.myrank = 2; //will build subgroups from rank 2 perspective set.map.type = UCC_EP_MAP_ARRAY; set.map.array.map = team_ranks; set.map.array.elem_size = sizeof(ucc_rank_t); /* Init topo for such subset */ EXPECT_EQ(UCC_OK, ucc_context_topo_init(&s.storage, &ctx_topo)); EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); /* Check subgroups */ /* NODE subgroup - ALL on the same node*/ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(team_size, sbgp->group_size); EXPECT_EQ(2, sbgp->group_rank); } UCC_TEST_F(test_topo, 1node_2sockets) { const ucc_rank_t ctx_size = 6; const ucc_rank_t team_size = 6; addr_storage s(ctx_size); ucc_sbgp_t * sbgp; ucc_subset_t set; /* simulates world proc array */ SET_PI(s, 0, 0xabcd, 0, 0); SET_PI(s, 1, 0xabcd, 1, 1); SET_PI(s, 2, 0xabcd, 0, 2); SET_PI(s, 3, 0xabcd, 1, 3); SET_PI(s, 4, 0xabcd, 0, 4); SET_PI(s, 5, 0xabcd, 1, 5); /* subset from the world */ set.map.ep_num = team_size; set.myrank = 3; // from rank 1 perspective set.map.type = UCC_EP_MAP_FULL; /* Init topo for such subset */ EXPECT_EQ(UCC_OK, ucc_context_topo_init(&s.storage, &ctx_topo)); EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); /* Check subgroups */ /* NODE subgroup - ALL on the same node*/ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(team_size, sbgp->group_size); EXPECT_EQ(3, sbgp->group_rank); /* SOCKET subgroup - must contain ranks 1, 3, 5 */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(sbgp->group_size, ctx_size / 2); EXPECT_EQ(sbgp->group_rank, 1); //rank 3 is rank 1 in subgroup 1, 3, 5 EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 1); EXPECT_EQ(sbgp->map.strided.stride, 2); /* SOCKET_LEADERS subgroup - ranks 0 and 1. Rank 3 does not participate, so the SBGP is disabled for him */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET_LEADERS); EXPECT_EQ(UCC_SBGP_DISABLED, sbgp->status); ucc_topo_cleanup(topo); set.myrank = 1; EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); /* SOCKET_LEADERS subgroup - ranks 0 and 1. Rank 1 is also rank 1 in the SBGP*/ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET_LEADERS); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(sbgp->group_size, 2); EXPECT_EQ(sbgp->group_rank, 1); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 0); EXPECT_EQ(sbgp->map.strided.stride, 1); } UCC_TEST_F(test_topo, 2nodes) { const ucc_rank_t ctx_size = 8; const ucc_rank_t team_size = 8; addr_storage s(ctx_size); ucc_sbgp_t * sbgp; ucc_subset_t set; /* simulates world proc array : 2 nodes, 5 ranks on 1st node and 3 on 2nd*/ SET_PI(s, 0, 0xaaa, 0, 0); SET_PI(s, 1, 0xaaa, 1, 1); SET_PI(s, 2, 0xaaa, 0, 2); SET_PI(s, 3, 0xaaa, 1, 3); SET_PI(s, 4, 0xaaa, 0, 4); SET_PI(s, 5, 0xbbb, 0, 5); SET_PI(s, 6, 0xbbb, 1, 6); SET_PI(s, 7, 0xbbb, 0, 7); /* subset from the world */ set.map.ep_num = team_size; set.map.type = UCC_EP_MAP_FULL; set.myrank = 3; // from rank 1 perspective /* Init topo for such subset */ EXPECT_EQ(UCC_OK, ucc_context_topo_init(&s.storage, &ctx_topo)); EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); /* NODE subgroup - ALL on the same node*/ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(5, sbgp->group_size); EXPECT_EQ(3, sbgp->group_rank); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 0); EXPECT_EQ(sbgp->map.strided.stride, 1); /* SOCKET subgroup - must contain ranks 1, 3, 5 */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(sbgp->group_size, 2); EXPECT_EQ(sbgp->group_rank, 1); //rank 3 is rank 1 in subgroup 1, 3, 5 EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 1); EXPECT_EQ(sbgp->map.strided.stride, 2); /* SOCKET_LEADERS subgroup - ranks 0 and 1. Rank 3 does not participate, so the SBGP is disabled for him */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET_LEADERS); EXPECT_EQ(UCC_SBGP_DISABLED, sbgp->status); /* NODE LEADERS subgroup - ranks 0 and 5. Rank 3 does not participate, so the SBGP is disabled for him */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); EXPECT_EQ(UCC_SBGP_DISABLED, sbgp->status); /* NET subgroup - there is no process with local rank 3 on 2nd node */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NET); EXPECT_EQ(UCC_SBGP_NOT_EXISTS, sbgp->status); /* RANK 6 perspective */ ucc_topo_cleanup(topo); set.myrank = 6; EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); /* NODE subgroup - ALL on the same node*/ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(3, sbgp->group_size); EXPECT_EQ(1, sbgp->group_rank); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 5); EXPECT_EQ(sbgp->map.strided.stride, 1); /* SOCKET subgroup - has only 1 rank on socket 1, so SBGP NOT EXISTS */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET); EXPECT_EQ(UCC_SBGP_NOT_EXISTS, sbgp->status); /* SOCKET_LEADERS subgroup - ranks 5 and 6*/ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET_LEADERS); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(2, sbgp->group_size); EXPECT_EQ(1, sbgp->group_rank); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 5); EXPECT_EQ(sbgp->map.strided.stride, 1); /* NODE LEADERS subgroup - ranks 0 and 5. Rank 6 does not participate, so the SBGP is disabled for him */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); EXPECT_EQ(UCC_SBGP_DISABLED, sbgp->status); /* NET subgroup - ranks 1 and 6 (local ranks 0 on nodes) */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NET); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(2, sbgp->group_size); EXPECT_EQ(1, sbgp->group_rank); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 1); EXPECT_EQ(sbgp->map.strided.stride, 5); } UCC_TEST_F(test_topo, 4nodes_half) { const ucc_rank_t ctx_size = 8; addr_storage s(ctx_size); ucc_sbgp_t * sbgp; ucc_subset_t set; /* simulates world proc array : 4 nodes, 2 ranks per node*/ SET_PI(s, 0, 0xaaa, 0, 0); SET_PI(s, 1, 0xaaa, 1, 1); SET_PI(s, 2, 0xbbb, 0, 2); SET_PI(s, 3, 0xbbb, 1, 3); SET_PI(s, 4, 0xccc, 0, 4); SET_PI(s, 5, 0xccc, 1, 5); SET_PI(s, 6, 0xddd, 0, 6); SET_PI(s, 7, 0xddd, 1, 7); /* subset from the world */ set.map.ep_num = 4; set.map.type = UCC_EP_MAP_STRIDED; set.map.strided.start = 0; set.map.strided.stride = 1; set.myrank = 1; // from rank 1 perspective /* Init topo for such subset */ EXPECT_EQ(UCC_OK, ucc_context_topo_init(&s.storage, &ctx_topo)); EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); /* NODE subgroup - 2 ranks on the same node*/ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(2, sbgp->group_size); EXPECT_EQ(1, sbgp->group_rank); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 0); EXPECT_EQ(sbgp->map.strided.stride, 1); /* NODE LEADERS subgroup - ranks 0 and 2. Rank 1 does not participate, so the SBGP is disabled for him */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); EXPECT_EQ(UCC_SBGP_DISABLED, sbgp->status); /* RANK 2 perspective */ ucc_topo_cleanup(topo); set.myrank = 2; EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE_LEADERS); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(2, sbgp->group_size); EXPECT_EQ(1, sbgp->group_rank); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 0); EXPECT_EQ(sbgp->map.strided.stride, 2); } UCC_TEST_F(test_topo, 4sockets_half) { const ucc_rank_t ctx_size = 8; addr_storage s(ctx_size); ucc_sbgp_t * sbgp; ucc_subset_t set; /* simulates world proc array : 4 sockets, 2 ranks per socket*/ SET_PI(s, 0, 0xaaa, 0, 0); SET_PI(s, 1, 0xaaa, 0, 1); SET_PI(s, 2, 0xaaa, 2, 2); SET_PI(s, 3, 0xaaa, 2, 3); SET_PI(s, 4, 0xaaa, 3, 4); SET_PI(s, 5, 0xaaa, 3, 5); SET_PI(s, 6, 0xaaa, 4, 6); SET_PI(s, 7, 0xaaa, 4, 7); /* team from the world */ set.map.ep_num = 4; set.map.type = UCC_EP_MAP_STRIDED; set.map.strided.start = 0; set.map.strided.stride = 1; set.myrank = 1; // from rank 1 perspective /* Init topo for such team */ EXPECT_EQ(UCC_OK, ucc_context_topo_init(&s.storage, &ctx_topo)); EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); /* SOCKET subgroup - 2 ranks on the same socket*/ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(2, sbgp->group_size); EXPECT_EQ(1, sbgp->group_rank); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 0); EXPECT_EQ(sbgp->map.strided.stride, 1); /* SOCKET LEADERS subgroup - ranks 0 and 2. Rank 1 does not participate, so the SBGP is disabled for him */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET_LEADERS); EXPECT_EQ(UCC_SBGP_DISABLED, sbgp->status); /* RANK 2 perspective */ ucc_topo_cleanup(topo); set.myrank = 2; EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_SOCKET_LEADERS); EXPECT_EQ(UCC_SBGP_ENABLED, sbgp->status); EXPECT_EQ(2, sbgp->group_size); EXPECT_EQ(1, sbgp->group_rank); EXPECT_EQ(sbgp->map.type, UCC_EP_MAP_STRIDED); EXPECT_EQ(sbgp->map.strided.start, 0); EXPECT_EQ(sbgp->map.strided.stride, 2); } UCC_TEST_F(test_topo, 4sockets_all) { const ucc_rank_t ctx_size = 16; addr_storage s(ctx_size); ucc_sbgp_t * sbgps; ucc_subset_t set; int n_sbgps; /* simulates world proc array : 4 sockets, 2 ranks per socket*/ SET_PI(s, 0, 0xaaa, 0, 0); SET_PI(s, 1, 0xaaa, 0, 1); SET_PI(s, 2, 0xaaa, 2, 2); SET_PI(s, 3, 0xaaa, 2, 3); SET_PI(s, 4, 0xaaa, 3, 4); SET_PI(s, 5, 0xaaa, 3, 5); SET_PI(s, 6, 0xaaa, 4, 6); SET_PI(s, 7, 0xaaa, 4, 7); SET_PI(s, 8, 0xaaa, 0, 8); SET_PI(s, 9, 0xaaa, 0, 9); SET_PI(s, 10, 0xaaa, 2, 10); SET_PI(s, 11, 0xaaa, 2, 11); SET_PI(s, 12, 0xaaa, 3, 12); SET_PI(s, 13, 0xaaa, 3, 13); SET_PI(s, 14, 0xaaa, 4, 14); SET_PI(s, 15, 0xaaa, 4, 15); EXPECT_EQ(UCC_OK, ucc_context_topo_init(&s.storage, &ctx_topo)); /* world */ set.map.ep_num = 16; set.map.type = UCC_EP_MAP_FULL; set.myrank = 1; // from rank 1 perspective /* Init topo for such subset */ EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); EXPECT_EQ(UCC_OK, ucc_topo_get_all_sockets(topo, &sbgps, &n_sbgps)); EXPECT_EQ(4, n_sbgps); EXPECT_EQ(true, check_sbgp(&sbgps[0], {0, 1, 8, 9})); EXPECT_EQ(true, check_sbgp(&sbgps[1], {2, 3, 10, 11})); EXPECT_EQ(true, check_sbgp(&sbgps[2], {4, 5, 12, 13})); EXPECT_EQ(true, check_sbgp(&sbgps[3], {6, 7, 14, 15})); /* world subset, 3 procs from each socket re-ordered*/ ucc_topo_cleanup(topo); ucc_rank_t ranks[] = {1, 9, 8, 3, 2, 11, 4, 13, 12, 7, 14, 15}; set.map.ep_num = 12; set.map.type = UCC_EP_MAP_ARRAY; set.map.array.map = (void *)ranks; set.map.array.elem_size = sizeof(ucc_rank_t); set.myrank = 1; // from rank 1 perspective /* Init topo for such subset */ EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); /* SOCKET subgroup - 2 ranks on the same socket*/ EXPECT_EQ(UCC_OK, ucc_topo_get_all_sockets(topo, &sbgps, &n_sbgps)); EXPECT_EQ(4, n_sbgps); EXPECT_EQ(true, check_sbgp(&sbgps[0], {0, 1, 2})); EXPECT_EQ(true, check_sbgp(&sbgps[1], {3, 4, 5})); EXPECT_EQ(true, check_sbgp(&sbgps[2], {6, 7, 8})); EXPECT_EQ(true, check_sbgp(&sbgps[3], {9, 10, 11})); /* world subset, 1 proc from each socket*/ ucc_topo_cleanup(topo); ucc_rank_t ranks2[] = {1, 2, 13, 7}; set.map.ep_num = 4; set.map.type = UCC_EP_MAP_ARRAY; set.map.array.map = (void *)ranks2; set.map.array.elem_size = sizeof(ucc_rank_t); set.myrank = 1; // from rank 1 perspective /* Init topo for such subset */ EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); EXPECT_EQ(UCC_OK, ucc_topo_get_all_sockets(topo, &sbgps, &n_sbgps)); EXPECT_EQ(4, n_sbgps); EXPECT_EQ(true, check_sbgp(&sbgps[0], {0})); EXPECT_EQ(true, check_sbgp(&sbgps[1], {1})); EXPECT_EQ(true, check_sbgp(&sbgps[2], {2})); EXPECT_EQ(true, check_sbgp(&sbgps[3], {3})); /* world subset, 1 full socket + 1 proc from another socket */ ucc_topo_cleanup(topo); ucc_rank_t ranks3[] = {0, 1, 2, 8, 9}; set.map.ep_num = 5; set.map.type = UCC_EP_MAP_ARRAY; set.map.array.map = (void *)ranks3; set.map.array.elem_size = sizeof(ucc_rank_t); set.myrank = 1; // from rank 1 perspective /* Init topo for such subset */ EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); EXPECT_EQ(UCC_OK, ucc_topo_get_all_sockets(topo, &sbgps, &n_sbgps)); EXPECT_EQ(2, n_sbgps); EXPECT_EQ(true, check_sbgp(&sbgps[0], {0, 1, 3, 4})); EXPECT_EQ(true, check_sbgp(&sbgps[1], {2})); } UCC_TEST_F(test_topo, all_nodes) { const ucc_rank_t ctx_size = 16; addr_storage s(ctx_size); ucc_sbgp_t *sbgps, *sbgp; ucc_subset_t set; int n_sbgps; /* simulates world proc array: 4 nodes, different number of ranks per node */ /* Node 0: ranks 0,1,2,3 */ SET_PI(s, 0, 0xaaa, 0, 0); SET_PI(s, 1, 0xaaa, 1, 1); SET_PI(s, 2, 0xaaa, 0, 2); SET_PI(s, 3, 0xaaa, 1, 3); /* Node 1: ranks 4,5,6 */ SET_PI(s, 4, 0xbbb, 0, 4); SET_PI(s, 5, 0xbbb, 1, 5); SET_PI(s, 6, 0xbbb, 0, 6); /* Node 2: ranks 7,8,9,10 */ SET_PI(s, 7, 0xccc, 0, 7); SET_PI(s, 8, 0xccc, 1, 8); SET_PI(s, 9, 0xccc, 0, 9); SET_PI(s, 10, 0xccc, 1, 10); /* Node 3: ranks 11,12,13,14,15 */ SET_PI(s, 11, 0xddd, 0, 11); SET_PI(s, 12, 0xddd, 1, 12); SET_PI(s, 13, 0xddd, 0, 13); SET_PI(s, 14, 0xddd, 1, 14); SET_PI(s, 15, 0xddd, 0, 15); EXPECT_EQ(UCC_OK, ucc_context_topo_init(&s.storage, &ctx_topo)); /* Test full world */ set.map.ep_num = 16; set.map.type = UCC_EP_MAP_FULL; set.myrank = 1; EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); EXPECT_EQ(UCC_OK, ucc_topo_get_all_nodes(topo, &sbgps, &n_sbgps)); EXPECT_EQ(4, n_sbgps); EXPECT_EQ(true, check_sbgp(&sbgps[0], {0, 1, 2, 3})); EXPECT_EQ(true, check_sbgp(&sbgps[1], {4, 5, 6})); EXPECT_EQ(true, check_sbgp(&sbgps[2], {7, 8, 9, 10})); EXPECT_EQ(true, check_sbgp(&sbgps[3], {11, 12, 13, 14, 15})); /* Compare with the node sbgp on rank 1. Skip checking group_rank because it isn't set in ucc_topo_get_all_nodes */ sbgp = ucc_topo_get_sbgp(topo, UCC_SBGP_NODE); EXPECT_EQ(sbgp->status, sbgps[0].status); EXPECT_EQ(sbgp->group_size, sbgps[0].group_size); EXPECT_EQ(sbgp->map.type, sbgps[0].map.type); /* Test subset with ranks from different nodes */ ucc_topo_cleanup(topo); ucc_rank_t ranks[] = {9, 12, 13, 14}; set.map.ep_num = 4; set.map.type = UCC_EP_MAP_ARRAY; set.map.array.map = (void*)ranks; set.map.array.elem_size = sizeof(ucc_rank_t); set.myrank = 2; EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); EXPECT_EQ(UCC_OK, ucc_topo_get_all_nodes(topo, &sbgps, &n_sbgps)); EXPECT_EQ(2, n_sbgps); EXPECT_EQ(UCC_SBGP_NOT_EXISTS, sbgps[0].status); // 1 rank node sbgp doesnt exist EXPECT_EQ(UCC_SBGP_ENABLED, sbgps[1].status); // 3 rank node sbgp should be enabled /* Test subset with ranks from different nodes */ ucc_topo_cleanup(topo); ucc_rank_t ranks1[] = {2, 0, 5, 6, 7, 10, 14, 12}; set.map.ep_num = 8; set.map.type = UCC_EP_MAP_ARRAY; set.map.array.map = (void*)ranks1; set.map.array.elem_size = sizeof(ucc_rank_t); set.myrank = 0; EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); EXPECT_EQ(UCC_OK, ucc_topo_get_all_nodes(topo, &sbgps, &n_sbgps)); EXPECT_EQ(4, n_sbgps); EXPECT_EQ(true, check_sbgp(&sbgps[0], {0, 1})); EXPECT_EQ(true, check_sbgp(&sbgps[1], {2, 3})); EXPECT_EQ(true, check_sbgp(&sbgps[2], {4, 5})); EXPECT_EQ(true, check_sbgp(&sbgps[3], {6, 7})); /* Test subset with multiple ranks from same node--should err because node leader subgroup not exists */ ucc_topo_cleanup(topo); ucc_rank_t ranks2[] = {11, 12, 13, 14, 15}; set.map.ep_num = 5; set.map.type = UCC_EP_MAP_ARRAY; set.map.array.map = (void*)ranks2; set.map.array.elem_size = sizeof(ucc_rank_t); set.myrank = 0; EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); EXPECT_EQ(UCC_ERR_INVALID_PARAM, ucc_topo_get_all_nodes(topo, &sbgps, &n_sbgps)); } UCC_TEST_F(test_topo, node_leaders) { const ucc_rank_t ctx_size = 8; addr_storage s(ctx_size); ucc_subset_t set; ucc_rank_t i; ucc_rank_t *node_leaders; /* simulates world proc array: 2 nodes, 4 ranks per node */ SET_PI(s, 0, 0xaaa, 0, 0); // Node 0, rank 0 SET_PI(s, 1, 0xaaa, 1, 1); // Node 0, rank 1 SET_PI(s, 2, 0xaaa, 0, 2); // Node 0, rank 2 SET_PI(s, 3, 0xaaa, 1, 3); // Node 0, rank 3 SET_PI(s, 4, 0xbbb, 0, 4); // Node 1, rank 0 SET_PI(s, 5, 0xbbb, 1, 5); // Node 1, rank 1 SET_PI(s, 6, 0xbbb, 0, 6); // Node 1, rank 2 SET_PI(s, 7, 0xbbb, 1, 7); // Node 1, rank 3 /* subset from the world */ set.map.ep_num = ctx_size; set.map.type = UCC_EP_MAP_FULL; set.myrank = 0; /* Test with node_leader_rank_id = 0 (default) */ EXPECT_EQ(UCC_OK, ucc_context_topo_init(&s.storage, &ctx_topo)); EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); topo->node_leader_rank_id = 0; EXPECT_EQ(2, topo->topo->nnodes); EXPECT_EQ(UCC_OK, ucc_topo_get_node_leaders(topo, &node_leaders)); /* Verify node leaders array */ // Node 0 ranks should point to rank 0 (first rank on node 0) for (i = 0; i < 4; i++) { EXPECT_EQ(0, node_leaders[i]); } // Node 1 ranks should point to rank 4 (first rank on node 1) for (i = 4; i < 8; i++) { EXPECT_EQ(4, node_leaders[i]); } /* Test with node_leader_rank_id = 1 */ ucc_topo_cleanup(topo); EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); topo->node_leader_rank_id = 1; EXPECT_EQ(UCC_OK, ucc_topo_get_node_leaders(topo, &node_leaders)); /* Verify node leaders array */ // Node 0 ranks should point to rank 1 (second rank on node 0) for (i = 0; i < 4; i++) { EXPECT_EQ(1, node_leaders[i]); } // Node 1 ranks should point to rank 5 (second rank on node 1) for (i = 4; i < 8; i++) { EXPECT_EQ(5, node_leaders[i]); } /* Test with a subset of ranks */ ucc_topo_cleanup(topo); ucc_rank_t ranks[] = {1, 2, 5, 6}; // Mix of ranks from both nodes set.map.ep_num = 4; set.map.type = UCC_EP_MAP_ARRAY; set.map.array.map = (void*)ranks; set.map.array.elem_size = sizeof(ucc_rank_t); set.myrank = 0; /* Test subset with node_leader_rank_id = 0 */ EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); topo->node_leader_rank_id = 0; EXPECT_EQ(UCC_OK, ucc_topo_get_node_leaders(topo, &node_leaders)); /* Verify node leaders array for subset */ // Ranks 0,1 (from node 0) should point to rank 0 (first rank on node 0) EXPECT_EQ(0, node_leaders[0]); EXPECT_EQ(0, node_leaders[1]); // Ranks 2,3 (from node 1) should point to rank 2 (first rank on node 1) EXPECT_EQ(2, node_leaders[2]); EXPECT_EQ(2, node_leaders[3]); /* Test subset with node_leader_rank_id = 1 */ ucc_topo_cleanup(topo); EXPECT_EQ(UCC_OK, ucc_topo_init(set, ctx_topo, &topo)); topo->node_leader_rank_id = 1; EXPECT_EQ(UCC_OK, ucc_topo_get_node_leaders(topo, &node_leaders)); /* Verify node leaders array for subset */ // Ranks 0,1 (from node 0) should point to rank 1 (second rank on node 0) EXPECT_EQ(1, node_leaders[0]); EXPECT_EQ(1, node_leaders[1]); // Ranks 2,3 (from node 1) should point to rank 3 (second rank on node 1) EXPECT_EQ(3, node_leaders[2]); EXPECT_EQ(3, node_leaders[3]); } ucc-1.8.0/test/gtest/core/test_mem_map.h0000664000175000017500000000231615211535620020340 0ustar alastairalastair/** * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ #ifndef TEST_MEM_MAP_H #define TEST_MEM_MAP_H #include "../common/test_ucc.h" #include "test_context.h" #include #include class test_mem_map : public test_context_config { protected: ucc_context_h ctx_h; ucc_context_params_t ctx_params; ucc_context_config_h ctx_config; public: test_mem_map(); ~test_mem_map(); void SetUp() override; void TearDown() override; }; class test_mem_map_export : public test_mem_map { protected: void * test_buffer; size_t buffer_size; ucc_mem_map_params_t map_params; ucc_mem_map_t segment; public: test_mem_map_export(); ~test_mem_map_export(); void SetUp() override; void TearDown() override; }; class test_mem_map_import : public test_mem_map { protected: void * test_buffer; size_t buffer_size; ucc_mem_map_mem_h memh; size_t memh_size; public: test_mem_map_import(); ~test_mem_map_import(); void SetUp() override; void TearDown() override; }; #endif /* TEST_MEM_MAP_H */ ucc-1.8.0/test/gtest/core/test_mc_reduce.h0000664000175000017500000001674415211535620020665 0ustar alastairalastair/** * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * See file LICENSE for terms. */ extern "C" { #include #include } #include template class op> struct TypeOpPair; #define DECLARE_TYPE_OP_PAIR(_type, _TYPE, _EQ) \ template