pax_global_header00006660000000000000000000000064151503710220014506gustar00rootroot0000000000000052 comment=b91894bd5b190c874d98a017f93f5daa515b65d0 nccl-2.29.7-1/000077500000000000000000000000001515037102200126645ustar00rootroot00000000000000nccl-2.29.7-1/.github/000077500000000000000000000000001515037102200142245ustar00rootroot00000000000000nccl-2.29.7-1/.github/ISSUE_TEMPLATE/000077500000000000000000000000001515037102200164075ustar00rootroot00000000000000nccl-2.29.7-1/.github/ISSUE_TEMPLATE/ISSUE.yaml000066400000000000000000000061071515037102200201670ustar00rootroot00000000000000name: NCCL issue or bug description: Report an issue or failure when running NCCL code title: "[Issue]: " labels: ["triage"] body: - type: markdown attributes: value: | Thanks for reaching out! Before reporting a new issue, please feel free to search for the behavior in the existing issues. If you found an issue which is already closed or you are unsure, open a new issue and reference the old one from it. You can also check out the [troubleshooting section](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/troubleshooting.html) in our user guide. --- To ensure we can assist you quickly and accurately, we often need the following information: - type: dropdown id: type attributes: label: How is this issue impacting you? description: What best describes your issue? options: - Lower performance than expected - Application crash - Data corruption - Application hang validations: required: true - type: textarea id: log attributes: label: Share Your Debug Logs description: | The logs and topo-files are a great tool to pin down issues. You can create them by setting these environment variables before the run. * `NCCL_DEBUG=INFO` and `NCCL_DEBUG_FILE=ncclDebug.%h.%p` to produce one file per rank * `NCCL_TOPO_DUMP_FILE=ncclSystem.txt` - type: textarea id: repro attributes: label: Steps to Reproduce the Issue description: | * **Minimal Steps**: Please provide a simple way to recreate the issue (see [Minimal Bug Reports](https://matthewrocklin.com/minimal-bug-reports) for inspiration). * **Environment Details**: Include software versions and relevant settings. * **Intermittency**: Is this a sporadic issue? If so, how often does it occur? * **Previous Success**: Did this work with an older NCCL version? The easier we can reproduce on our side the more likely we are to be able to solve it in a timely manner. - type: input id: nccl_version attributes: label: NCCL Version description: | NCCL reports its version string in the debug logs. You can also determine the version if you know which library was used by running `strings libnccl.so | grep 'NCCL version'`. placeholder: "e.g. 2.27.1+cuda12.8" validations: required: true - type: textarea id: platform attributes: label: Your platform details description: | * **GPU & Network**: Share your architecture and topology (e.g., from `nvidia-smi`, `nvidia-smi topo -m`, `ibstatus`). * **Environment**: Bare-metal, containers, or cloud? * **Scalability**: Does this issue occur with a specific number of ranks/nodes? - type: textarea id: issue-description attributes: label: Error Message & Behavior description: | * **First Error**: What was the initial `NCCL WARN` message in your logs? * **Expected vs. Actual**: Briefly describe the anticipated behavior versus what you're seeing. nccl-2.29.7-1/.github/ISSUE_TEMPLATE/QUESTION.yaml000066400000000000000000000010541515037102200205420ustar00rootroot00000000000000name: NCCL question description: Ask the NCCL team a question title: "[Question]: " labels: ["question"] body: - type: markdown attributes: value: | Thanks for reaching out! To solve your problem, feel free to check out the [user guide](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/index.html), in particular the troubleshooting section, and also the [release notes](https://docs.nvidia.com/deeplearning/nccl/release-notes/index.html). --- - type: textarea id: question attributes: label: Question nccl-2.29.7-1/.github/ISSUE_TEMPLATE/RFE.yaml000066400000000000000000000016011515037102200177050ustar00rootroot00000000000000name: NCCL request for enhancement description: Request for enhancement title: "[RFE]: " labels: ["enhancement"] body: - type: markdown attributes: value: | Thanks for your feedback! Before reporting a new RFE you could quickly check if this already exists in our [existing requests](https://github.com/NVIDIA/nccl/issues?q=sort%3Aupdated-desc%20is%3Aissue%20is%3Aopen%20label%3Aenhancement). --- - type: textarea id: rfe-description attributes: label: Please provide the below details to ensure we understand your needs description: | * What is the goal of this request? * Who will benefit from this feature? * Is this request for a specific GPU architecture or network infrastructure? * How will this feature improve current workflows or processes? * What is the priority level of this request? nccl-2.29.7-1/.github/ISSUE_TEMPLATE/config.yml000066400000000000000000000000341515037102200203740ustar00rootroot00000000000000blank_issues_enabled: false nccl-2.29.7-1/.github/pull_request_template.md000066400000000000000000000005271515037102200211710ustar00rootroot00000000000000## Description ## Related Issues ## Changes & Impact ## Performance Impact nccl-2.29.7-1/.gitignore000066400000000000000000000001351515037102200146530ustar00rootroot00000000000000# Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved. /build *.gcov /coverage/ nccl-2.29.7-1/CMakeLists.txt000066400000000000000000000252341515037102200154320ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.25) # Version information # Read makefiles/version.mk file file(READ ${CMAKE_SOURCE_DIR}/makefiles/version.mk VERSION_CONTENT) string(REGEX REPLACE ".*NCCL_MAJOR[ ]*:=[ ]*([0-9]+).*" "\\1" NCCL_MAJOR "${VERSION_CONTENT}") string(REGEX REPLACE ".*NCCL_MINOR[ ]*:=[ ]*([0-9]+).*" "\\1" NCCL_MINOR "${VERSION_CONTENT}") string(REGEX REPLACE ".*NCCL_PATCH[ ]*:=[ ]*([0-9]+).*" "\\1" NCCL_PATCH "${VERSION_CONTENT}") string(REGEX REPLACE ".*NCCL_SUFFIX[ ]*:=[ ]*([a-zA-Z0-9]*).*" "\\1" NCCL_SUFFIX "${VERSION_CONTENT}") string(REGEX REPLACE ".*PKG_REVISION[ ]*:=[ ]*([0-9]+).*" "\\1" PKG_REVISION "${VERSION_CONTENT}") math(EXPR NCCL_VERSION_CODE "(${NCCL_MAJOR} * 10000) + (${NCCL_MINOR} * 100) + ${NCCL_PATCH}") # Make version information available to C++ source files add_compile_definitions( NCCL_USE_CMAKE NCCL_MAJOR=${NCCL_MAJOR} NCCL_MINOR=${NCCL_MINOR} NCCL_PATCH=${NCCL_PATCH} NCCL_VERSION_CODE=${NCCL_VERSION_CODE} ) set(ENV{NCCL_USE_CMAKE} "1") project(NCCL VERSION ${NCCL_MAJOR}.${NCCL_MINOR}.${NCCL_PATCH} LANGUAGES CUDA CXX C) include(GNUInstallDirs) # Make CMAKE_BUILD_TYPE to release by default if not set if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "Release") endif() # Package building options option(BUILD_DEBIAN_PACKAGE "Build Debian package" OFF) option(BUILD_REDHAT_PACKAGE "Build Redhat package" OFF) option(BUILD_TXZ_PACKAGE "Build TXZ package" OFF) option(BUILD_CONDA_PACKAGE "Build Conda package" OFF) option(BUILD_SRCTXZ_PACKAGE "Build source tarball package" OFF) option(BUILD_DOC_PACKAGE "Build documentation package" OFF) option(BUILD_PACKAGES "Build all packages" OFF) option(VERBOSE "Enable verbose output" OFF) option(KEEP "Keep intermediate files" OFF) option(DEBUG "Enable debug build" OFF) option(ASAN "Enable Address Sanitizer" OFF) option(UBSAN "Enable Undefined Behavior Sanitizer" OFF) option(TRACE "Enable tracing" OFF) option(WERROR "Treat warnings as errors" OFF) option(PROFAPI "Enable profiling API" ON) option(NVTX "Enable NVTX" ON) option(RDMA_CORE "Enable RDMA core" OFF) option(NET_PROFILER "Enable network profiler" OFF) option(MLX5DV "Enable MLX5DV" OFF) option(MAX_EXT_NET_PLUGINS "Maximum external network plugins" 0) option(EMIT_LLVM_IR "Generate LLVM IR for device APIs" OFF) option(BUILD_NCCL4PY "Enable nccl4py Python bindings targets" OFF) # Detect OS if(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(NCCL_OS_WINDOWS ON) add_definitions(-DNCCL_OS_WINDOWS) elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(NCCL_OS_LINUX ON) add_definitions(-DNCCL_OS_LINUX) else() message(FATAL_ERROR "Unsupported OS: ${CMAKE_SYSTEM_NAME}") endif() # Detect compiler if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") set(NCCL_COMPILER_MSVC ON) add_definitions(-DNCCL_COMPILER_MSVC) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set(NCCL_COMPILER_GCC ON) add_definitions(-DNCCL_COMPILER_GCC) else() message(FATAL_ERROR "Unsupported compiler: ${CMAKE_CXX_COMPILER_ID}") endif() find_package(CUDAToolkit REQUIRED) find_package(Threads REQUIRED) find_package(Python3 REQUIRED COMPONENTS Interpreter) # CUDA version detection string(REGEX MATCH "([0-9]+\\.[0-9]+)" CUDA_VERSION "${CUDAToolkit_VERSION}") # Extract major and minor version numbers string(REGEX MATCH "([0-9]+)" CUDA_MAJOR "${CUDA_VERSION}") string(REGEX MATCH "([0-9]+)$" CUDA_MINOR "${CUDA_VERSION}") string(REGEX REPLACE ".*\\.([0-9]+)$" "\\1" CUDA_MINOR "${CUDA_VERSION}") # Add CUDA version definitions after find_package add_compile_definitions( CUDA_MAJOR=${CUDA_MAJOR} CUDA_MINOR=${CUDA_MINOR} ) # CUDA 13.0 requires C++17 if(${CUDA_MAJOR} GREATER_EQUAL 13) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CUDA_STANDARD 17) else() set(CMAKE_CXX_STANDARD 14) set(CMAKE_CUDA_STANDARD 14) endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CUDA_STANDARD_REQUIRED ON) # CUDA architecture flags if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "") message(STATUS "CMAKE_CUDA_ARCHITECTURES not defined or empty, setting default values based on CUDA version") if(${CUDA_MAJOR} LESS 9) set(CMAKE_CUDA_ARCHITECTURES "35;50;60;61") elseif(${CUDA_MAJOR} EQUAL 9) set(CMAKE_CUDA_ARCHITECTURES "35;50;60;61;70") elseif(${CUDA_MAJOR} EQUAL 10) set(CMAKE_CUDA_ARCHITECTURES "35;50;60;61;70") elseif(${CUDA_MAJOR} EQUAL 11) if(${CUDA_MINOR} LESS 8) set(CMAKE_CUDA_ARCHITECTURES "35;50;60;61;70;80") else() set(CMAKE_CUDA_ARCHITECTURES "35;50;60;61;70;80;90") endif() elseif(${CUDA_MAJOR} EQUAL 12) if(${CUDA_MINOR} LESS 8) set(CMAKE_CUDA_ARCHITECTURES "50;60;61;70;80;90") else() set(CMAKE_CUDA_ARCHITECTURES "50;60;61;70;80;90;100;120") endif() elseif(${CUDA_MAJOR} EQUAL 13) set(CMAKE_CUDA_ARCHITECTURES "50;60;61;70;80;90;100;110;120") else() # For future CUDA versions, include all architectures up to the latest known set(CMAKE_CUDA_ARCHITECTURES "50;60;61;70;80;90;100;110;120") endif() endif() message(STATUS "Using CUDA_ARCHITECTURES: ${CMAKE_CUDA_ARCHITECTURES}") # Compiler-specific flags if(NCCL_COMPILER_GCC) # GCC/Clang-specific flags set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fvisibility=hidden -Wall -Wno-unused-function -Wno-sign-compare -Wvla -g") set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-extended-lambda -Xptxas -maxrregcount=96 -Xfatbin -compress-all -fPIC") elseif(NCCL_COMPILER_MSVC) # MSVC-specific flags # /Zc:preprocessor enables conformant preprocessor for proper variadic macro handling (needed for NVTX) # /wd4146 disables "unary minus on unsigned" warning - intentional for two's complement and alignment masks # /wd4197 disables "top-level volatile in cast ignored" - benign when using atomic intrinsics # /wd5105 disables "macro expansion producing 'defined'" - triggered by Windows SDK headers (winbase.h) # /wd4805 disables "unsafe mix of type 'bool' and type 'int'" - intentional bool/int mixing for flags # /wd4018 disables "signed/unsigned mismatch" - safe comparisons where signed value is always positive set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W3 /wd4267 /wd4244 /wd4996 /wd4146 /wd4197 /wd5105 /wd4805 /wd4018 /FS /Zc:preprocessor") set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-extended-lambda -Xptxas -maxrregcount=96 --compress-mode=balance") endif() # Sanitizer options (GCC/Clang only) if(NCCL_COMPILER_GCC) if(ASAN) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -static-libasan") endif() if(UBSAN) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -static-libubsan") endif() endif() # Additional options if(TRACE) add_definitions(-DENABLE_TRACE) endif() if(NOT NVTX) add_definitions(-DNVTX_DISABLE) endif() if(WERROR) if(NCCL_COMPILER_MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") endif() endif() if(PROFAPI) add_definitions(-DPROFAPI) endif() set(EXTRA_LIBS) # RDMA and MLX5DV are Linux-specific features if(RDMA_CORE) add_definitions(-DNCCL_BUILD_RDMA_CORE=1) find_library(VERBS_LIBRARY NAMES verbs) if(VERBS_LIBRARY) list(APPEND EXTRA_LIBS verbs) endif() endif() if(MLX5DV) add_definitions(-DNCCL_BUILD_MLX5DV=1) find_library(MLX5_LIBRARY NAMES mlx5) if(MLX5_LIBRARY) list(APPEND EXTRA_LIBS mlx5) endif() endif() if(NET_PROFILER) add_definitions(-DNCCL_ENABLE_NET_PROFILING=1) endif() if(MAX_EXT_NET_PLUGINS GREATER 0) add_definitions(-DNCCL_NET_MAX_PLUGINS=${MAX_EXT_NET_PLUGINS}) endif() add_definitions(-DDOCA_VERBS_USE_CUDA_WRAPPER) add_definitions(-DDOCA_VERBS_USE_NET_WRAPPER) add_definitions(-DNCCL_GIN_PROXY_ENABLE=1) if(EMIT_LLVM_IR) add_definitions(-DEMIT_LLVM_IR=1) endif() # Library dependencies find_library(RT_LIBRARY NAMES rt) if(RT_LIBRARY) list(APPEND EXTRA_LIBS rt) endif() # Debug/Release specific flags if(NCCL_COMPILER_GCC) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -O0") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -O3") elseif(NCCL_COMPILER_MSVC) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} /Od /Zi /FS") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} /O2 /FS") # Maximum available optimization for MSVC endif() set(CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS} -O0 -G -g") set(CMAKE_CUDA_FLAGS_RELEASE "${CMAKE_CUDA_FLAGS} -O3") add_subdirectory(plugins/mixed/example) add_subdirectory(plugins/net) add_subdirectory(plugins/profiler/example) add_subdirectory(plugins/tuner/example) add_subdirectory(plugins/env/example) add_subdirectory(src) # Add package building subdirectories if(BUILD_DEBIAN_PACKAGE OR BUILD_PACKAGES) add_subdirectory(pkg/debian) endif() if(BUILD_REDHAT_PACKAGE OR BUILD_PACKAGES) add_subdirectory(pkg/redhat) endif() if(BUILD_TXZ_PACKAGE OR BUILD_PACKAGES) add_subdirectory(pkg/txz) endif() if(BUILD_CONDA_PACKAGE OR BUILD_PACKAGES) add_subdirectory(pkg/conda) endif() if(BUILD_SRCTXZ_PACKAGE OR BUILD_PACKAGES) add_subdirectory(pkg/srctxz) endif() if(BUILD_DOC_PACKAGE OR BUILD_PACKAGES) add_subdirectory(pkg/doc) endif() if(EMIT_LLVM_IR) add_subdirectory(bindings/ir) add_dependencies(llvm_ir nccl_header) add_custom_target(nccl_with_ir ALL DEPENDS nccl llvm_ir) message(STATUS "LLVM IR generation will be included in default build") endif() if(BUILD_NCCL4PY) add_subdirectory(bindings/nccl4py) endif() ###################### CMake package config for find_package(NCCL) ###################### include(CMakePackageConfigHelpers) set(NCCL_CMAKE_CONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/NCCL") # Export installed targets (NCCL::nccl, NCCL::nccl_static) install(EXPORT NCCLTargets FILE NCCLTargets.cmake NAMESPACE NCCL:: DESTINATION "${NCCL_CMAKE_CONFIG_DIR}" ) # Generate & install NCCLConfig.cmake / NCCLConfigVersion.cmake configure_package_config_file( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/NCCLConfig.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/NCCLConfig.cmake" INSTALL_DESTINATION "${NCCL_CMAKE_CONFIG_DIR}" ) write_basic_package_version_file( "${CMAKE_CURRENT_BINARY_DIR}/NCCLConfigVersion.cmake" VERSION "${PROJECT_VERSION}" COMPATIBILITY SameMajorVersion ) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/NCCLConfig.cmake" "${CMAKE_CURRENT_BINARY_DIR}/NCCLConfigVersion.cmake" DESTINATION "${NCCL_CMAKE_CONFIG_DIR}" ) # Also export targets for build-tree usage (optional convenience). export(EXPORT NCCLTargets FILE "${CMAKE_CURRENT_BINARY_DIR}/NCCLTargets.cmake" NAMESPACE NCCL:: ) nccl-2.29.7-1/CONTRIBUTING.md000066400000000000000000000211451515037102200151200ustar00rootroot00000000000000# Contributing to NCCL Thank you for your interest in contributing to NCCL! We appreciate the time and effort you're putting into helping improve the library. This document will help guide you through the contribution process. ## Before You Start To help ensure your contribution can be accepted smoothly: - **Open an issue first** for significant changes or new features. This allows us to discuss the approach before you invest significant time. - **Check existing issues** to see if someone else is already working on something similar. - **Ask questions** if you're unsure about anything. We're happy to help! ## Getting Started 1. **Fork the Repository**: Fork [https://github.com/NVIDIA/nccl](https://github.com/NVIDIA/nccl) and clone your fork locally. 2. **Create a Branch**: Create a feature branch for your work. ```bash git checkout -b ``` 3. **Commit your Changes**: Commit and push into feature branch ```bash git add git commit --message git push origin --set-upstream ``` ## What We're Looking For We welcome contributions in the following areas: - **Bug fixes**: Reproducible bugs with clear fixes are always appreciated - **Performance improvements**: Targeted optimizations, guarded by appropriate checks - **Platform support**: Extending NCCL to new platforms, network fabrics, or PCI bridges - **Documentation**: Improvements to README, comments, or usage examples ## Making Your Contribution Successful To help us review and integrate your contribution efficiently: ### Scope and Focus - Keep changes focused on a single issue or feature - Break large changes into smaller, reviewable pieces when possible - Avoid mixing unrelated changes (e.g., bug fix + formatting changes) in one PR ### Quality Standards - Ensure your code compiles without warnings - Test thoroughly on relevant hardware configurations - Performance claims should be backed by measurements - Public API changes require discussion (see below) ### Common Challenges Some types of contributions require extra discussion before we can accept them: - **Public API changes**: NCCL's API stability is important to users. If your contribution changes public APIs, please open an issue to discuss the design first. We need to ensure backward compatibility and consistency. - **Architecture-specific code**: NCCL supports specific GPU architectures and network configurations. Contributions targeting unsupported architectures may not be accepted unless there's a clear plan for ongoing maintenance. - **Workarounds for external issues**: If a change works around a problem in another component (drivers, libraries, frameworks), we may need to address it differently. Let's discuss the root cause first. - **Incomplete implementations**: Partial features or fixes that don't fully address the problem are difficult to maintain. If you need help completing an implementation, let us know - we're happy to assist! ## New Features If you're proposing a substantial new feature (e.g., new collective operations, transport mechanisms, algorithms, or significant architectural changes), we follow a more structured process: 1. **Open an issue** describing the feature and use case 2. **Discussion phase**: We'll discuss whether it fits NCCL's direction 3. **Document Design**: Document the proposed architecture, a testing and validation plan as well as any known performance impact. 4. **Review**: The design will be reviewed by NCCL maintainers 5. **Implementation**: We recommend that you wait with the implementation until the review has concluded. Once there's a mutual agreement on the approach, proceed with implementation 6. **Pull request**: Submit your PR referencing the original issue and design doc ## Code Style NCCL follows these coding conventions: ### General Guidelines - Use 2 spaces for indentation (no tabs) - Maximum line length: 100 characters - Follow K&R brace style for C/CUDA code - Use clear, descriptive variable names ### Naming Conventions - Functions and variables: `ncclCamelCase` - Macros and constants: `UPPER_CASE_WITH_UNDERSCORES` - Struct/type names: `ncclFooBar` ### CUDA-Specific - Minimize register usage in performance-critical kernels - Avoid warp divergence where possible - Document kernel launch configurations and occupancy considerations - Prefer `cudaLaunchKernel` over `<<<>>>` syntax ### Return Codes - NCCL functions should return a `ncclResult_t` - All function calls should be guarded by `NCCLCHECK` or similar macro - All external function calls should be guarded with `CUDACHECK`, `SYSCHECK`, `PTHREADCHECK`, etc. ### Memory Management - Always allocate memory through NCCL alloc functions, e.g. `ncclCalloc()` - Use appropriate memory fences for synchronization - Free resources in reverse order of allocation ### Comments and Documentation - Write clear comments for complex algorithms - Document assumptions and invariants - Explain "why" not just "what" for non-obvious code ### Code Formatting - Avoid trailing white-spaces - Try to match the existing style in files you're modifying ## Pull Request Guidelines ### Before Submitting 1. Rebase your branch on the latest master branch 2. Run `make` to ensure clean build with no warnings ### Commit Messages - Use imperative mood: "Add feature" not "Added feature" - First line: brief summary (50 chars or less) - Second line blank - Detailed explanation including the Problem, Solution, and any Limitations - Reference issue numbers: "Fixes #123" An example commit message is: ```text Comment | Commit message -----------------|-------------------------------------------------------- Title | Fix crash in proxy on systems with more than 2 GPUs Blank line | Problem | When a system has more than 2 GPUs, the table we use to | store addresses overflows. Solution | This fix increases the size of the table to the maximum | number of GPUs we can have within a node. Limitations and | We may want to make the table size dynamic to save Caveats | memory, but since it is a part of a struct, it is easier | for now to keep the code simple. ``` ### Signed Commits All commits must be signed off to certify you have the right to submit the code: ```bash git commit -s -m "Your commit message" ``` This adds: `Signed-off-by: Your Name ` This certifies your agreement with the Developer Certificate of Origin (DCO). ``` Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. ``` ## Code Review Process After you submit a PR: 1. **Maintainer review**: NCCL engineers will review your code 2. **Discussion**: We may ask questions or request changes 3. **Iteration**: Address feedback and update your PR 4. **Approval**: Once approved, we'll merge your contribution Please be patient during review. We aim to provide initial feedback within a week. Feel free to ping us if we take much longer than that. ## Communication - **GitHub Issues**: For bug reports and feature requests - **Pull Requests**: For code contributions and discussion ## Getting Help If you need help at any stage: - Comment on your issue or PR - Ask questions in GitHub Issues - Refer to NCCL documentation and examples We're committed to helping contributors succeed! ## Thank You We truly appreciate your time and effort in contributing to NCCL. Happy coding! nccl-2.29.7-1/LICENSE.txt000066400000000000000000000317151515037102200145160ustar00rootroot00000000000000 Most of this project is licensed under the Apache-2 license (right below). Parts of the project retain their original BSD license (see at the end). Files borrowed from other projects include their own license text. ------------------------------------------------------------------------------- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ------------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of NVIDIA CORPORATION, Lawrence Berkeley National Laboratory, the U.S. Department of Energy, nor the names of their contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. nccl-2.29.7-1/Makefile000066400000000000000000000020231515037102200143210ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # .PHONY: all clean default: src.build default: ir.build install: src.install BUILDDIR ?= $(abspath ./build) ABSBUILDDIR := $(abspath $(BUILDDIR)) TARGETS := src pkg nccl4py ir clean: ${TARGETS:%=%.clean} examples.build: src.build ir.build: src.build LICENSE_FILES := LICENSE.txt LICENSE_TARGETS := $(LICENSE_FILES:%=$(BUILDDIR)/%) lic: $(LICENSE_TARGETS) ${BUILDDIR}/%.txt: %.txt @printf "Copying %-35s > %s\n" $< $@ mkdir -p ${BUILDDIR} install -m 644 $< $@ src.%: ${MAKE} -C src $* BUILDDIR=${ABSBUILDDIR} examples: src.build ${MAKE} -C docs/examples NCCL_HOME=${ABSBUILDDIR} pkg.%: ${MAKE} -C pkg $* BUILDDIR=${ABSBUILDDIR} nccl4py.%: ${MAKE} -C bindings/nccl4py $* BUILDDIR=${ABSBUILDDIR} # IR generation requires src.build first ir.%: ${MAKE} -C bindings/ir $* BUILDDIR=${ABSBUILDDIR} pkg.debian.prep: lic pkg.txz.prep: lic nccl-2.29.7-1/README.md000066400000000000000000000047561515037102200141570ustar00rootroot00000000000000# NCCL Optimized primitives for inter-GPU communication. ## Introduction NCCL (pronounced "Nickel") is a stand-alone library of standard communication routines for GPUs, implementing all-reduce, all-gather, reduce, broadcast, reduce-scatter, as well as any send/receive based communication pattern. It has been optimized to achieve high bandwidth on platforms using PCIe, NVLink, NVswitch, as well as networking using InfiniBand Verbs or TCP/IP sockets. NCCL supports an arbitrary number of GPUs installed in a single node or across multiple nodes, and can be used in either single- or multi-process (e.g., MPI) applications. For more information on NCCL usage, please refer to the [NCCL documentation](https://docs.nvidia.com/deeplearning/sdk/nccl-developer-guide/index.html). ## Build Note: the official and tested builds of NCCL can be downloaded from: https://developer.nvidia.com/nccl. You can skip the following build steps if you choose to use the official builds. To build the library : ```shell $ cd nccl $ make -j src.build ``` If CUDA is not installed in the default /usr/local/cuda path, you can define the CUDA path with : ```shell $ make src.build CUDA_HOME= ``` NCCL will be compiled and installed in `build/` unless `BUILDDIR` is set. By default, NCCL is compiled for all supported architectures. To accelerate the compilation and reduce the binary size, consider redefining `NVCC_GENCODE` (defined in `makefiles/common.mk`) to only include the architecture of the target platform : ```shell $ make -j src.build NVCC_GENCODE="-gencode=arch=compute_90,code=sm_90" ``` ## Install To install NCCL on the system, create a package then install it as root. Debian/Ubuntu : ```shell $ # Install tools to create debian packages $ sudo apt install build-essential devscripts debhelper fakeroot $ # Build NCCL deb package $ make pkg.debian.build $ ls build/pkg/deb/ ``` RedHat/CentOS : ```shell $ # Install tools to create rpm packages $ sudo yum install rpm-build rpmdevtools $ # Build NCCL rpm package $ make pkg.redhat.build $ ls build/pkg/rpm/ ``` OS-agnostic tarball : ```shell $ make pkg.txz.build $ ls build/pkg/txz/ ``` ## Tests Tests for NCCL are maintained separately at https://github.com/nvidia/nccl-tests. ```shell $ git clone https://github.com/NVIDIA/nccl-tests.git $ cd nccl-tests $ make $ ./build/all_reduce_perf -b 8 -e 256M -f 2 -g ``` ## Copyright All source code and accompanying documentation is copyright (c) 2015-2020, NVIDIA CORPORATION. All rights reserved. nccl-2.29.7-1/ThirdPartyNotices.txt000066400000000000000000000303021515037102200170420ustar00rootroot00000000000000============================================================================== NVTX is under the Apache License v2.0 with LLVM Exceptions: ============================================================================== Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ---- LLVM Exceptions to the Apache 2.0 License ---- As an exception, if, as a result of your compiling your source code, portions of this Software are embedded into an Object form of such source code, you may redistribute such embedded portions in such Object form without complying with the conditions of Sections 4(a), 4(b) and 4(d) of the License. In addition, if you combine or link compiled forms of this Software with software that is licensed under the GPLv2 ("Combined Software") and if a court of competent jurisdiction determines that the patent provision (Section 3), the indemnity provision (Section 9) or other Section of the License conflicts with the conditions of the GPLv2, you may retroactively and prospectively choose to deem waived or otherwise exclude such Section(s) of the License, but only in their entirety and only with respect to the Combined Software. nccl-2.29.7-1/bindings/000077500000000000000000000000001515037102200144615ustar00rootroot00000000000000nccl-2.29.7-1/bindings/ir/000077500000000000000000000000001515037102200150735ustar00rootroot00000000000000nccl-2.29.7-1/bindings/ir/CMakeLists.txt000066400000000000000000000114661515037102200176430ustar00rootroot00000000000000# LLVM IR generation for NCCL device APIs # Source and output configuration set(LLVM_SRC ${CMAKE_CURRENT_SOURCE_DIR}/nccl_device_wrapper__impl.h) set(OBJDIR ${CMAKE_BINARY_DIR}/obj/llvm_ir) set(LIBDIR ${CMAKE_BINARY_DIR}/lib) set(INCLUDEDIR ${CMAKE_BINARY_DIR}/include) # Output files (all aux files in obj/llvm_ir) set(UNOPTIMIZED_BC ${OBJDIR}/libnccl_device.bc.unoptimized) set(OPTIMIZED_BC ${OBJDIR}/libnccl_device.bc.optimized) set(LLVM_IR_FILE ${OBJDIR}/libnccl_device.ll) set(FINAL_BC ${LIBDIR}/libnccl_device.bc) set(WRAPPER_HEADER ${INCLUDEDIR}/nccl_device_wrapper.h) # Build configuration # Select GPU arch and C++ standard based on CUDA Toolkit version if(CUDAToolkit_VERSION_MAJOR LESS 12) set(BITCODE_LIB_ARCH sm_70 CACHE STRING "CUDA architecture for LLVM IR") else() set(BITCODE_LIB_ARCH sm_90 CACHE STRING "CUDA architecture for LLVM IR") endif() set(BITCODE_CXX_STD c++17 CACHE STRING "C++ standard for LLVM IR") # Find required tools find_program(CLANG_EXECUTABLE clang REQUIRED) find_program(OPT_EXECUTABLE opt REQUIRED) find_program(LLVM_DIS_EXECUTABLE llvm-dis REQUIRED) find_program(LLVM_AS_EXECUTABLE llvm-as REQUIRED) # Include paths set(NCCL_INCLUDES -I${CMAKE_BINARY_DIR}/include -I${CMAKE_SOURCE_DIR}/src/include -I${CMAKE_SOURCE_DIR}/src/include/nccl_device -I${CMAKE_SOURCE_DIR}/src/device ) set(CUDA_INCLUDES -I${CUDAToolkit_INCLUDE_DIRS} -I${CUDAToolkit_INCLUDE_DIRS}/cccl ) # Common clang flags used for both preprocessing and IR generation set(COMMON_CLANG_FLAGS -std=${BITCODE_CXX_STD} -x cuda --cuda-path=${CUDAToolkit_ROOT_DIR} --cuda-device-only --cuda-gpu-arch=${BITCODE_LIB_ARCH} ${NCCL_INCLUDES} ${CUDA_INCLUDES} -D__clang_llvm_bitcode_lib__ -DCUDA_MAJOR=${CUDA_MAJOR} -DCUDA_MINOR=${CUDA_MINOR} ) # Clang flags for LLVM IR generation set(CLANG_FLAGS -c -emit-llvm -O1 ${COMMON_CLANG_FLAGS} ) # Copy the preauthored wrapper header to the build include directory add_custom_command( OUTPUT ${WRAPPER_HEADER} COMMAND ${CMAKE_COMMAND} -E make_directory ${INCLUDEDIR} COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/nccl_device_wrapper.h ${WRAPPER_HEADER} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/nccl_device_wrapper.h COMMENT "Copying nccl_device_wrapper.h to build include..." VERBATIM ) # Generate unoptimized LLVM bitcode add_custom_command( OUTPUT ${UNOPTIMIZED_BC} COMMAND ${CMAKE_COMMAND} -E make_directory ${OBJDIR} COMMAND ${CLANG_EXECUTABLE} ${CLANG_FLAGS} ${LLVM_SRC} -o ${UNOPTIMIZED_BC} DEPENDS ${LLVM_SRC} COMMENT "Generating unoptimized LLVM bitcode..." VERBATIM ) # Generate optimized LLVM bitcode add_custom_command( OUTPUT ${OPTIMIZED_BC} COMMAND ${OPT_EXECUTABLE} --passes=internalize,inline,globaldce -internalize-public-api-list=nccl* ${UNOPTIMIZED_BC} -o ${OPTIMIZED_BC} DEPENDS ${UNOPTIMIZED_BC} COMMENT "Optimizing LLVM bitcode..." VERBATIM ) # Generate LLVM IR text file add_custom_command( OUTPUT ${LLVM_IR_FILE} COMMAND ${LLVM_DIS_EXECUTABLE} ${UNOPTIMIZED_BC} -o ${LLVM_IR_FILE}.tmp COMMAND ${CMAKE_COMMAND} -E echo "Cleaning LLVM IR (removing nvvm-reflect-ftz)..." COMMAND bash -c "myVar=$$(cat ${LLVM_IR_FILE}.tmp | grep -E '!([0-9]+) = !\\{[^\"]*\"nvvm-reflect-ftz\"' | cut -d ' ' -f 1)\; awk '!/nvvm-reflect-ftz/' ${LLVM_IR_FILE}.tmp | sed \"/^!llvm\\.module\\.flags = /s/$$myVar, //\" > ${LLVM_IR_FILE}" COMMAND ${CMAKE_COMMAND} -E remove ${LLVM_IR_FILE}.tmp DEPENDS ${UNOPTIMIZED_BC} COMMENT "Generating LLVM IR..." VERBATIM ) # Generate final bitcode from cleaned LLVM IR add_custom_command( OUTPUT ${FINAL_BC} COMMAND ${CMAKE_COMMAND} -E make_directory ${LIBDIR} COMMAND ${CMAKE_COMMAND} -E echo "Generating final bitcode from cleaned LLVM IR..." COMMAND ${LLVM_AS_EXECUTABLE} ${LLVM_IR_FILE} -o ${FINAL_BC} DEPENDS ${LLVM_IR_FILE} COMMENT "Generating final bitcode..." VERBATIM ) # Custom target for LLVM IR generation add_custom_target(llvm_ir DEPENDS ${FINAL_BC} ${OPTIMIZED_BC} ${WRAPPER_HEADER} COMMENT "LLVM IR and bitcode generated successfully" ) # Print generated files on completion add_custom_command(TARGET llvm_ir POST_BUILD COMMAND ${CMAKE_COMMAND} -E echo "LLVM IR and bitcode generated successfully:" COMMAND ${CMAKE_COMMAND} -E echo " C++ Standard: ${BITCODE_CXX_STD}" COMMAND ${CMAKE_COMMAND} -E echo " GPU Architecture: ${BITCODE_LIB_ARCH}" COMMAND ${CMAKE_COMMAND} -E echo " Unoptimized: ${UNOPTIMIZED_BC}" COMMAND ${CMAKE_COMMAND} -E echo " Optimized: ${OPTIMIZED_BC}" COMMAND ${CMAKE_COMMAND} -E echo " LLVM IR: ${LLVM_IR_FILE}" COMMAND ${CMAKE_COMMAND} -E echo " Final BC: ${FINAL_BC}" COMMAND ${CMAKE_COMMAND} -E echo " Wrapper Header: ${FINAL_BC}" VERBATIM ) nccl-2.29.7-1/bindings/ir/Makefile000066400000000000000000000064471515037102200165460ustar00rootroot00000000000000# # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. # # See LICENSE.txt for license information # include ../../makefiles/common.mk # LLVM IR generation ifneq ($(EMIT_LLVM_IR), 0) # Source and target files LLVM_SRC := nccl_device_wrapper__impl.h BUILDDIR ?= $(abspath ../../build) OBJDIR := $(BUILDDIR)/obj/llvm_ir LIBDIR := $(BUILDDIR)/lib # LLVM IR output files (all aux files in obj/llvm_ir) UNOPTIMIZED_BC := $(OBJDIR)/libnccl_device.bc.unoptimized OPTIMIZED_BC := $(OBJDIR)/libnccl_device.bc.optimized LLVM_IR_FILE := $(OBJDIR)/libnccl_device.ll FINAL_BC := $(LIBDIR)/libnccl_device.bc # Build configuration ifeq ($(shell test "0$(CUDA_MAJOR)" -lt 12; echo $$?),0) BITCODE_LIB_ARCH ?= sm_70 else BITCODE_LIB_ARCH ?= sm_90 endif # Notes: # Device API code follows C++17 # Error in GIN code while compiling with c++17 BITCODE_CXX_STD ?= gnu++17 CLANG ?= clang OPT ?= opt LLVM_DIS ?= llvm-dis LLVM_AS ?= llvm-as PYTHON ?= python3 # Include paths NCCL_INCLUDES := -I$(BUILDDIR)/include -I../../src/include -I../../src/include/nccl_device -I../../src/device CUDA_INCLUDES := -I$(CUDA_INC) -I$(CUDA_INC)/cccl # Clang flags for LLVM IR generation COMMON_CLANG_FLAGS := -std=$(BITCODE_CXX_STD) -x cuda \ --cuda-path=$(CUDA_HOME) --cuda-device-only \ --cuda-gpu-arch=$(BITCODE_LIB_ARCH) \ $(NCCL_INCLUDES) $(CUDA_INCLUDES) \ -D__clang_llvm_bitcode_lib__ \ -DCUDA_MAJOR=$(CUDA_MAJOR) -DCUDA_MINOR=$(CUDA_MINOR) CLANG_FLAGS := -c -emit-llvm -O1 $(COMMON_CLANG_FLAGS) # Optimization passes OPT_PASSES := --passes='internalize,inline,globaldce' \ -internalize-public-api-list='*nccl*' # Build rules WRAPPER_HEADER := $(BUILDDIR)/include/nccl_device_wrapper.h $(WRAPPER_HEADER): nccl_device_wrapper.h @mkdir -p $(BUILDDIR)/include @echo "Copying nccl_device_wrapper.h to build include..." cp -f $< $@ $(UNOPTIMIZED_BC): $(LLVM_SRC) @mkdir -p $(OBJDIR) @echo "Generating unoptimized LLVM bitcode..." $(CLANG) $(CLANG_FLAGS) $(LLVM_SRC) -o $@ $(OPTIMIZED_BC): $(UNOPTIMIZED_BC) @echo "Optimizing LLVM bitcode..." $(OPT) $(OPT_PASSES) $< -o $@ # The LLVM bc file by default forces flush subnormal to 1. # Removing the flag so that the subnormal handling is preserved. $(LLVM_IR_FILE): $(OPTIMIZED_BC) @echo "Generating LLVM IR..." $(LLVM_DIS) $< -o $@.tmp @echo "Cleaning LLVM IR (removing nvvm-reflect-ftz)..." @myVar="$$(cat $@.tmp | grep -E '!([0-9]+) = !\{[^"]*"nvvm-reflect-ftz"' | cut -d ' ' -f 1)"; \ awk '!/nvvm-reflect-ftz/' $@.tmp | sed "/^!llvm\.module\.flags = /s/$$myVar, //" > $@ @rm -f $@.tmp $(FINAL_BC): $(LLVM_IR_FILE) @mkdir -p $(LIBDIR) @echo "Generating final bitcode from cleaned LLVM IR..." $(LLVM_AS) $< -o $@ llvm_ir: $(FINAL_BC) $(OPTIMIZED_BC) $(WRAPPER_HEADER) @echo "LLVM IR and bitcode generated successfully:" @echo " C++ Standard: $(BITCODE_CXX_STD)" @echo " GPU Architecture: $(BITCODE_LIB_ARCH)" @echo " Unoptimized: $(UNOPTIMIZED_BC)" @echo " Optimized: $(OPTIMIZED_BC)" @echo " LLVM IR (Human-readable): $(LLVM_IR_FILE)" @echo " Final BC: $(FINAL_BC)" @echo " Wrapper Header: $(WRAPPER_HEADER)" else llvm_ir: @echo "LLVM IR generation disabled (EMIT_LLVM_IR=0)" endif build: llvm_ir clean: rm -rf $(OBJDIR) $(FINAL_BC) $(WRAPPER_HEADER) .PHONY: llvm_ir build clean nccl-2.29.7-1/bindings/ir/nccl_device_wrapper.h000066400000000000000000000064051515037102200212470ustar00rootroot00000000000000/************************************************************************* * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ #ifndef _NCCL_DEVICE_WRAPPER_H_ #define _NCCL_DEVICE_WRAPPER_H_ /* * NCCL Device API C-style wrapper functions */ #include "nccl_device.h" /* Struct definitions */ struct ncclLsaBarrierSession_C { ncclLsaBarrierSession bar; }; struct ncclGinBarrierSession_C { ncclGinBarrierSession bar; }; struct ncclBarrierSession_C { ncclBarrierSession bar; }; /* Peer pointer API */ NCCL_IR_EXTERN_C __device__ void* ncclGetPeerPointerTeam(ncclWindow_t w, size_t offset, ncclTeam tm, int peer); /* Coop initialization and utility functions */ NCCL_IR_EXTERN_C __device__ void ncclCoopAnyInitThread(ncclCoopAny* coop); NCCL_IR_EXTERN_C __device__ void ncclCoopAnyInitWarp(ncclCoopAny* coop); NCCL_IR_EXTERN_C __device__ void ncclCoopAnyInitLanes(ncclCoopAny* coop, uint32_t lane_mask); NCCL_IR_EXTERN_C __device__ void ncclCoopAnyInitWarpSpan(ncclCoopAny* coop, int warp0, int nWarps, int id); NCCL_IR_EXTERN_C __device__ void ncclCoopAnyInitCta(ncclCoopAny* coop); NCCL_IR_EXTERN_C __device__ int ncclCoopThreadRank(const ncclCoopAny* coop); NCCL_IR_EXTERN_C __device__ int ncclCoopSize(const ncclCoopAny* coop); NCCL_IR_EXTERN_C __device__ int ncclCoopNumThreads(const ncclCoopAny* coop); NCCL_IR_EXTERN_C __device__ void ncclCoopSync(const ncclCoopAny* coop); /* LSA Barrier Session APIs */ NCCL_IR_EXTERN_C __device__ void ncclLsaBarrierSessionInit( ncclLsaBarrierSession_C* session, ncclCoopAny coop, ncclDevComm const& comm, ncclTeam team, ncclLsaBarrierHandle handle, uint32_t index, bool multimem = false, ncclMultimemHandle mmHandle = {}); NCCL_IR_EXTERN_C __device__ void ncclLsaBarrierSessionArrive(ncclLsaBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order); NCCL_IR_EXTERN_C __device__ void ncclLsaBarrierSessionWait(ncclLsaBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order); NCCL_IR_EXTERN_C __device__ void ncclLsaBarrierSessionSync(ncclLsaBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order); /* GIN Barrier Session APIs */ NCCL_IR_EXTERN_C __device__ void ncclGinBarrierSessionInit( ncclGinBarrierSession_C* session, ncclCoopAny coop, ncclGin_C net, ncclTeam team, ncclGinBarrierHandle handle, uint32_t index); NCCL_IR_EXTERN_C __device__ void ncclGinBarrierSessionSync( ncclGinBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order, ncclGinFenceLevel fence); /* Barrier Session APIs */ NCCL_IR_EXTERN_C __device__ void ncclBarrierSessionInit( ncclBarrierSession_C* session, ncclCoopAny coop, ncclTeam innerTeam, ncclTeam outerTeam, ncclGin_C net, ncclLsaBarrierHandle const innerBarHandle, ncclGinBarrierHandle const outerBarHandle, uint32_t index, bool multimem=false, ncclMultimemHandle const innerMmHandle={}); NCCL_IR_EXTERN_C __device__ void ncclBarrierSessionSync( ncclBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order, ncclGinFenceLevel fence); #endif // _NCCL_DEVICE_WRAPPER_H_ nccl-2.29.7-1/bindings/ir/nccl_device_wrapper__impl.h000077500000000000000000000104171515037102200224300ustar00rootroot00000000000000/************************************************************************* * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ #ifndef _NCCL_DEVICE_WRAPPER__IMPL_H_ #define _NCCL_DEVICE_WRAPPER__IMPL_H_ /* * NCCL Device API force instantiation and C style APIs for LLVM IR generation */ #include "nccl_device_wrapper.h" #include #if NCCL_CHECK_CUDACC NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetPeerPointerTeam(ncclWindow_t w, size_t offset, ncclTeam tm, int peer) { return ncclGetPeerPointer(w, offset, tm, peer); } /* coop */ NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclCoopAnyInitThread(ncclCoopAny* coop) { ::new (coop) ncclCoopAny(ncclCoopThread()); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclCoopAnyInitWarp(ncclCoopAny* coop) { ::new (coop) ncclCoopAny(ncclCoopWarp()); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclCoopAnyInitLanes(ncclCoopAny* coop, uint32_t lane_mask) { ::new (coop) ncclCoopAny(ncclCoopLanes(lane_mask)); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclCoopAnyInitWarpSpan(ncclCoopAny* coop, int warp0, int nWarps, int id) { ::new (coop) ncclCoopAny(ncclCoopWarpSpan(warp0, nWarps, id)); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclCoopAnyInitCta(ncclCoopAny* coop) { ::new (coop) ncclCoopAny(ncclCoopCta()); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE int ncclCoopThreadRank(const ncclCoopAny* coop) { return coop->thread_rank(); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE int ncclCoopSize(const ncclCoopAny* coop) { return coop->size(); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE int ncclCoopNumThreads(const ncclCoopAny* coop) { return coop->num_threads(); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclCoopSync(const ncclCoopAny* coop) { const_cast(coop)->sync(); } /* lsa barrier session */ NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclLsaBarrierSessionInit( ncclLsaBarrierSession_C* session, ncclCoopAny coop, ncclDevComm const& comm, ncclTeam team, ncclLsaBarrierHandle handle, uint32_t index, bool multimem, ncclMultimemHandle mmHandle) { ::new (&(session->bar)) ncclLsaBarrierSession(coop, comm, team, handle, index, multimem, mmHandle); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclLsaBarrierSessionArrive(ncclLsaBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order) { session->bar.arrive(coop, order); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclLsaBarrierSessionWait(ncclLsaBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order) { session->bar.wait(coop, order); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclLsaBarrierSessionSync(ncclLsaBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order) { session->bar.sync(coop, order); } /* GIN barrier session */ NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinBarrierSessionInit( ncclGinBarrierSession_C* session, ncclCoopAny coop, ncclGin_C net, ncclTeam team, ncclGinBarrierHandle handle, uint32_t index) { ::new (&(session->bar)) ncclGinBarrierSession(coop, reinterpret_cast(net), team, handle, index); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinBarrierSessionSync( ncclGinBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order, ncclGinFenceLevel fence) { session->bar.sync(coop, order, fence); } /* Barrier Session*/ NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclBarrierSessionInit( ncclBarrierSession_C* session, ncclCoopAny coop, ncclTeam innerTeam, ncclTeam outerTeam, ncclGin_C net, ncclLsaBarrierHandle const innerBarHandle, ncclGinBarrierHandle const outerBarHandle, uint32_t index, bool multimem, ncclMultimemHandle const innerMmHandle) { ::new (&(session->bar)) ncclBarrierSession(coop, innerTeam, outerTeam, reinterpret_cast(net), innerBarHandle, outerBarHandle, index, multimem, innerMmHandle); } NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclBarrierSessionSync( ncclBarrierSession_C* session, ncclCoopAny coop, cuda::memory_order order, ncclGinFenceLevel fence) { session->bar.sync(coop, order, fence); } #endif #endif // _NCCL_DEVICE_WRAPPER__IMPL_H_ nccl-2.29.7-1/bindings/nccl4py/000077500000000000000000000000001515037102200160355ustar00rootroot00000000000000nccl-2.29.7-1/bindings/nccl4py/.gitignore000066400000000000000000000002701515037102200200240ustar00rootroot00000000000000# Build artifacts build/ dist/ *.egg-info/ # Cython compiled outputs nccl/bindings/**/*.cpp nccl/bindings/**/*.so __pycache__ # Tool caches .pytest_cache/ .ruff_cache/ .mypy_cache/ nccl-2.29.7-1/bindings/nccl4py/CMakeLists.txt000066400000000000000000000026541515037102200206040ustar00rootroot00000000000000find_program(UV_EXECUTABLE uv DOC "Path to uv executable" REQUIRED) set( UV_ENV UV_PYTHON_PREFERENCE=only-managed UV_PYTHON=3.13 ) set(NCCL4PY_SUPPORTED_CUDA_MAJOR 12 13) set(NCCL4PY_DIR "${CMAKE_CURRENT_SOURCE_DIR}") set(NCCL4PY_DIST_DIR "${CMAKE_BINARY_DIR}/dist") if(NOT "${CUDAToolkit_VERSION_MAJOR}" IN_LIST NCCL4PY_SUPPORTED_CUDA_MAJOR) message( FATAL_ERROR "Only CUDA ${NCCL4PY_SUPPORTED_CUDA_MAJOR} are supported. Found CUDA ${CUDAToolkit_VERSION_MAJOR}" ) endif() add_custom_target( nccl4py_dev COMMAND ${CMAKE_COMMAND} -E env ${UV_ENV} "${UV_EXECUTABLE}" sync --group test --extra cu${CUDAToolkit_VERSION_MAJOR} COMMAND ${CMAKE_COMMAND} -E env ${UV_ENV} "${UV_EXECUTABLE}" pip install -i "https://download.pytorch.org/whl/cu${CUDAToolkit_VERSION_MAJOR}${CUDAToolkit_VERSION_MINOR}" torch COMMAND ${CMAKE_COMMAND} -E env ${UV_ENV} "${UV_EXECUTABLE}" pip install cupy-cuda${CUDAToolkit_VERSION_MAJOR}x~=13.6 WORKING_DIRECTORY "${NCCL4PY_DIR}" COMMENT "Installing nccl4py with cu${CUDAToolkit_VERSION_MAJOR} extra and test dependencies" ) add_custom_target( nccl4py COMMAND ${CMAKE_COMMAND} -E env ${UV_ENV} "${UV_EXECUTABLE}" run python --version COMMAND ${CMAKE_COMMAND} -E env ${UV_ENV} "${UV_EXECUTABLE}" build --out-dir "${NCCL4PY_DIST_DIR}" "${NCCL4PY_DIR}" WORKING_DIRECTORY "${NCCL4PY_DIR}" COMMENT "Building nccl4py wheel into ${NCCL4PY_DIST_DIR}" ) nccl-2.29.7-1/bindings/nccl4py/DESCRIPTION.rst000066400000000000000000000024021515037102200203500ustar00rootroot00000000000000***************************************************** nccl4py: Pythonic NCCL Communication for GPU Clusters ***************************************************** .. image:: https://img.shields.io/badge/NVIDIA-black?logo=nvidia :target: https://www.nvidia.com/ :alt: NVIDIA `nccl4py `_ bridges Python's simplicity with the performance of NVIDIA Collective Communications Library (NCCL), and provides a Pythonic interface to NCCL library's functionality. It enables Python applications to leverage NCCL's GPU-accelerated multi-GPU and multi-node communication capabilities for distributed computing workloads. nccl4py follows the NCCL SLA. The details of the NCCL SLA is available `here `_. * `Homepage `_ * `Repository `_ * `Documentation `_ * `Issue tracker `_ ``nccl4py`` is under active development. Feedback and suggestions are welcome! Installation ============ For CUDA 12.x: .. code-block:: bash pip install nccl4py[cu12] For CUDA 13.x: .. code-block:: bash pip install nccl4py[cu13] nccl-2.29.7-1/bindings/nccl4py/LICENSE.txt000066400000000000000000000261351515037102200176670ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. nccl-2.29.7-1/bindings/nccl4py/MANIFEST.in000066400000000000000000000005441515037102200175760ustar00rootroot00000000000000# Include nccl package with all source files recursive-include nccl *.pyx *.pxd # Exclude directories not needed in sdist prune examples prune *.egg-info # Exclude development files from root and subdirectoriese x exclude Makefile exclude uv.lock global-exclude .gitignore .gitattributes # Exclude compiled/generated files global-exclude *.so *.cpp *.c nccl-2.29.7-1/bindings/nccl4py/Makefile000066400000000000000000000043721515037102200175030ustar00rootroot00000000000000UV ?= uv # Ensure only UV-managed Python versions are used export UV_PYTHON_PREFERENCE := only-managed export UV_PYTHON := 3.13 # Root Makefile passes BUILDDIR; default to ../../build if not set MKFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) BUILDDIR ?= $(abspath $(MKFILE_DIR)/../../build) NCCL4PY_DIR := $(abspath $(MKFILE_DIR)) NCCL4PY_BINDINGS_DIR := $(NCCL4PY_DIR)/nccl/bindings DIST_DIR := $(BUILDDIR)/dist ifdef CUDA_HOME CUDA_VERSION := $(shell $(CUDA_HOME)/bin/nvcc --version 2>/dev/null | sed -n 's/.*release \([0-9]\+\.[0-9]\+\).*/\1/p') CUDA_MAJOR := $(firstword $(subst ., ,$(CUDA_VERSION))) CUDA_MINOR := $(word 2,$(subst ., ,$(CUDA_VERSION))) PYTORCH_INDEX_URL := https://download.pytorch.org/whl/cu$(CUDA_MAJOR)$(CUDA_MINOR) endif # Supported CUDA major versions SUPPORTED_CUDA_VERSIONS := 12 13 .DEFAULT_GOAL := build .PHONY: all dev build clean check-cuda all: build check-cuda: ifndef CUDA_HOME $(error CUDA_HOME is not set. Please set CUDA_HOME to your CUDA installation directory, e.g. export CUDA_HOME=/usr/local/cuda) endif ifeq ($(CUDA_VERSION),) $(error nvcc not found at $(CUDA_HOME)/bin/nvcc or unable to detect CUDA version) endif ifeq ($(filter $(CUDA_MAJOR),$(SUPPORTED_CUDA_VERSIONS)),) $(error Only CUDA $(SUPPORTED_CUDA_VERSIONS) are supported. Found CUDA $(CUDA_MAJOR)) endif @echo "Using CUDA $(CUDA_VERSION) at $(CUDA_HOME)" dev: check-cuda @echo "Installing nccl4py with cu$(CUDA_MAJOR) extra and test dependencies..." cd $(NCCL4PY_DIR) && $(UV) sync --group test --extra cu$(CUDA_MAJOR) cd $(NCCL4PY_DIR) && $(UV) pip install -i $(PYTORCH_INDEX_URL) torch cd $(NCCL4PY_DIR) && $(UV) pip install cupy-cuda$(CUDA_MAJOR)x~=13.6 build: check-cuda @echo "Using Python: $$($(UV) run python --version)" $(UV) build --out-dir "$(DIST_DIR)" "$(NCCL4PY_DIR)" @echo "Wheel built successfully in $(DIST_DIR)" clean: @rm -rf $(DIST_DIR) \ $(NCCL4PY_DIR)/*.egg-info \ $(NCCL4PY_DIR)/build \ $(NCCL4PY_DIR)/dist @find $(NCCL4PY_DIR) -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true @find $(NCCL4PY_DIR) -type f -name "*.pyc" -delete 2>/dev/null || true @find $(NCCL4PY_BINDINGS_DIR) -type f \( -name "*.so" -o -name "*.cpp" -o -name "*.c" \) -delete 2>/dev/null || true nccl-2.29.7-1/bindings/nccl4py/README.md000066400000000000000000000041311515037102200173130ustar00rootroot00000000000000# nccl4py: Python Bindings for NCCL Python bindings for NVIDIA Collective Communications Library (NCCL), providing high-performance multi-GPU and multi-node communication primitives. This package provides both low-level Cython bindings and a high-level Pythonic API for NCCL collective operations. ### Experimental Cython Support ```python from nccl.bindings cimport cynccl ``` This allows Cython code to call NCCL functions with minimal overhead. The `cynccl.pxd` file is included in the package distribution for direct Cython integration. ## Requirements - **CUDA Toolkit**: CUDA 12.x or 13.x - **NCCL Library**: Matching CUDA version (nvidia-nccl-cu12 or nvidia-nccl-cu13) - **Python**: 3.10 or later ## Development Setup ### Prerequisites **Set CUDA_HOME environment variable:** ```bash export CUDA_HOME=/usr/local/cuda # Or your CUDA installation path ``` ### Building from Source (using Makefile) The easiest way to build is using the Makefile, which requires [uv](https://docs.astral.sh/uv/): ```bash # Install uv curl -LsSf https://astral.sh/uv/install.sh | sh cd nccl4py # Create development environment make dev # Build package (sdist and wheel) make build # Clean build artifacts make clean ``` The Makefile automatically: - Detects CUDA version from `$CUDA_HOME` - Installs appropriate CUDA-specific dependencies (cu12/cu13) - Builds Cython extensions ### Manual Build (without uv) If you prefer not to use `uv`, you can build manually with standard Python tools: ```bash # Set CUDA_HOME export CUDA_HOME=/usr/local/cuda cd nccl4py # Create virtual environment python -m venv .venv source .venv/bin/activate # Install in editable mode with CUDA dependencies pip install -e .[cu12] # For CUDA 12.x # OR pip install -e .[cu13] # For CUDA 13.x # Build distribution packages pip install build python -m build ``` ## Building Distribution Packages ### Build for current Python version: ```bash make build # Output: build/dist/nccl4py-*.tar.gz and nccl4py-*.whl ``` ## References - [NCCL Documentation](https://docs.nvidia.com/deeplearning/nccl/) - [NCCL Repository](https://github.com/NVIDIA/nccl) nccl-2.29.7-1/bindings/nccl4py/examples/000077500000000000000000000000001515037102200176535ustar00rootroot00000000000000nccl-2.29.7-1/bindings/nccl4py/examples/01_basic/000077500000000000000000000000001515037102200212345ustar00rootroot00000000000000nccl-2.29.7-1/bindings/nccl4py/examples/01_basic/01_allreduce.py000066400000000000000000000043201515037102200240450ustar00rootroot00000000000000#!/usr/bin/env python3 # # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # """ NCCL4Py Basic Example: AllReduce ================================== The simplest possible example showing how to use NCCL4Py with AllReduce. Each rank contributes its rank number, and all ranks receive the sum. USAGE: mpirun -np 4 python 01_allreduce.py """ import sys try: from mpi4py import MPI except ImportError: print("ERROR: mpi4py required. Install with: pip install mpi4py") sys.exit(1) try: import torch except ImportError: print("ERROR: PyTorch required. Install with: pip install torch") sys.exit(1) import nccl.core as nccl def main(): # Initialize MPI comm_mpi = MPI.COMM_WORLD rank = comm_mpi.Get_rank() nranks = comm_mpi.Get_size() # Set rank 0 as the root root = 0 # Assign GPU to each process device = torch.device(f"cuda:{rank % torch.cuda.device_count()}") torch.cuda.set_device(device) # [NCCL4Py] Generate unique ID on the root rank unique_id = nccl.get_unique_id() if rank == root else None # Broadcast unique ID to all ranks unique_id = comm_mpi.bcast(unique_id, root=root) # [NCCL4Py] Initialize NCCL communicator nccl_comm = nccl.Communicator.init(nranks=nranks, rank=rank, unique_id=unique_id) if rank == root: print(f"Running AllReduce with {nranks} ranks...") # Create PyTorch tensor with rank value data = torch.tensor([float(rank)], dtype=torch.float32, device=device) # [NCCL4Py] AllReduce: Sum all rank values nccl_comm.reduce(data, data, nccl.SUM) torch.cuda.synchronize() # Verify result expected = float(nranks * (nranks - 1) // 2) actual = float(data[0].item()) print(f"Rank {rank}: AllReduce result = {actual:.0f} (expected {expected:.0f})") # [NCCL4Py] Destroy NCCL communicator (collective call) nccl_comm.destroy() if rank == root: if actual == expected: print("SUCCESS!") else: print("FAILED!") return 0 if actual == expected else 1 if __name__ == "__main__": sys.exit(main()) nccl-2.29.7-1/bindings/nccl4py/examples/01_basic/02_send_recv.py000066400000000000000000000060431515037102200240620ustar00rootroot00000000000000#!/usr/bin/env python3 # # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # """ NCCL4Py Basic Example: Send/Recv ================================== Simple point-to-point communication example showing two ranks exchanging data. Demonstrates the use of group() to avoid deadlocks. USAGE: mpirun -np 2 python 02_send_recv.py """ import sys try: from mpi4py import MPI except ImportError: print("ERROR: mpi4py required. Install with: pip install mpi4py") sys.exit(1) try: import torch except ImportError: print("ERROR: PyTorch required. Install with: pip install torch") sys.exit(1) import nccl.core as nccl def main(): # Initialize MPI comm_mpi = MPI.COMM_WORLD rank = comm_mpi.Get_rank() nranks = comm_mpi.Get_size() # Set rank 0 as the root root = 0 if nranks != 2: if rank == root: print("ERROR: This example requires exactly 2 ranks") print("Usage: mpirun -np 2 python 02_send_recv.py") sys.exit(1) # Assign GPU to each process device = torch.device(f"cuda:{rank % torch.cuda.device_count()}") torch.cuda.set_device(device) # [NCCL4Py] Generate unique ID on the root rank unique_id = nccl.get_unique_id() if rank == root else None # Broadcast unique ID to all ranks unique_id = comm_mpi.bcast(unique_id, root=root) # [NCCL4Py] Initialize NCCL communicator nccl_comm = nccl.Communicator.init(nranks=nranks, rank=rank, unique_id=unique_id) if rank == root: print("Running Send/Recv between 2 ranks...") # Create tensors send_data = torch.tensor([float(100 + rank)], dtype=torch.float32, device=device) recv_data = torch.zeros(1, dtype=torch.float32, device=device) other_rank = 1 - rank # Rank 0 <-> Rank 1 # Exchange data using group() to avoid deadlock # IMPORTANT: Using group() allows send() and recv() to be called in any order. # Without group(), you must ensure send() and recv() are ordered carefully to avoid deadlock. # For example: # - All ranks call send() first, then all call recv(), OR # - Even ranks send then recv, odd ranks recv then send # [NCCL4Py] Use group() to avoid deadlock with nccl.group(): # [NCCL4Py] Send data to the other rank nccl_comm.send(send_data, peer=other_rank) # [NCCL4Py] Receive data from the other rank nccl_comm.recv(recv_data, peer=other_rank) torch.cuda.synchronize() # Verify result expected = float(100 + (1 - rank)) actual = float(recv_data[0].item()) print(f"Rank {rank}: Sent {100 + rank:.0f}, Received {actual:.0f} (expected {expected:.0f})") # [NCCL4Py] Destroy NCCL communicator (collective call) nccl_comm.destroy() if rank == root: if actual == expected: print("SUCCESS!") else: print("FAILED!") return 0 if actual == expected else 1 if __name__ == "__main__": sys.exit(main()) nccl-2.29.7-1/bindings/nccl4py/nccl/000077500000000000000000000000001515037102200167545ustar00rootroot00000000000000nccl-2.29.7-1/bindings/nccl4py/nccl/__init__.py000066400000000000000000000011341515037102200210640ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # """ NCCL4Py: Python bindings for NVIDIA Collective Communications Library (NCCL). NCCL4Py provides Pythonic access to NCCL for efficient multi-GPU and multi-node communication. It supports all NCCL collective operations, point-to-point communication, and advanced features like buffer registration and custom reduction operators. """ from nccl._version import __version__ __all__ = [ "__version__", ] nccl-2.29.7-1/bindings/nccl4py/nccl/_version.py000066400000000000000000000003321515037102200211500ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # __version__ = "0.1.1" nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/000077500000000000000000000000001515037102200205515ustar00rootroot00000000000000nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/__init__.py000066400000000000000000000000241515037102200226560ustar00rootroot00000000000000from .nccl import * nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/_internal/000077500000000000000000000000001515037102200225245ustar00rootroot00000000000000nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/_internal/__init__.py000066400000000000000000000025341515037102200246410ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # """ Internal bindings implementation. This module preloads the NCCL library using cuda-pathfinder if available, which provides better library discovery across different environments (conda, system installs, custom CUDA paths, etc.). If cuda-pathfinder is not available or fails to find NCCL, the Cython bindings will fall back to direct dlopen("libnccl.so.2") which works if the library is in standard system paths. See documentation for cuda-pathfinder including the search order at: https://nvidia.github.io/cuda-python/cuda-pathfinder/latest/generated/cuda.pathfinder.load_nvidia_dynamic_lib.html#cuda.pathfinder.load_nvidia_dynamic_lib """ # Optional: Preload NCCL library for better discovery # This runs before the Cython extensions are loaded, allowing # dlsym(RTLD_DEFAULT, ...) to find the already-loaded library try: from cuda.pathfinder import load_nvidia_dynamic_lib load_nvidia_dynamic_lib("nccl") except ImportError: # cuda-python not installed, fall back to Cython's dlopen pass except Exception: # Library not found by pathfinder or other error # Fall back to Cython's dlopen - it will provide the error message if needed pass nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/_internal/nccl.pxd000066400000000000000000000145671515037102200241750ustar00rootroot00000000000000# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. # # SPDX-License-Identifier: Apache-2.0 # # This code was automatically generated with version 2.28.0. Do not modify it directly. from ..cynccl cimport * ############################################################################### # Wrapper functions ############################################################################### cdef ncclResult_t _ncclMemAlloc(void** ptr, size_t size) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclMemFree(void* ptr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGetVersion(int* version) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGetUniqueId(ncclUniqueId* uniqueId) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommInitRankConfig(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommInitRank(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommInitAll(ncclComm_t* comm, int ndev, const int* devlist) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommFinalize(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommDestroy(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommAbort(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommSplit(ncclComm_t comm, int color, int key, ncclComm_t* newcomm, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommShrink(ncclComm_t comm, int* excludeRanksList, int excludeRanksCount, ncclComm_t* newcomm, ncclConfig_t* config, int shrinkFlags) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommInitRankScalable(ncclComm_t* newcomm, int nranks, int myrank, int nId, ncclUniqueId* commIds, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef const char* _ncclGetErrorString(ncclResult_t result) except?NULL nogil cdef const char* _ncclGetLastError(ncclComm_t comm) except?NULL nogil cdef ncclResult_t _ncclCommGetAsyncError(ncclComm_t comm, ncclResult_t* asyncError) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommCount(const ncclComm_t comm, int* count) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommCuDevice(const ncclComm_t comm, int* device) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommUserRank(const ncclComm_t comm, int* rank) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommRegister(const ncclComm_t comm, void* buff, size_t size, void** handle) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommDeregister(const ncclComm_t comm, void* handle) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommWindowRegister(ncclComm_t comm, void* buff, size_t size, ncclWindow_t* win, int winFlags) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclCommWindowDeregister(ncclComm_t comm, ncclWindow_t win) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclRedOpCreatePreMulSum(ncclRedOp_t* op, void* scalar, ncclDataType_t datatype, ncclScalarResidence_t residence, ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclRedOpDestroy(ncclRedOp_t op, ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclReduceScatter(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclAlltoAll(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGather(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclScatter(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclRecv(void* recvbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclSignal(int peer, int sigIdx, int ctx, unsigned int flags, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclWaitSignal(int nDesc, ncclWaitSignalDesc_t* signalDescs, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGroupStart() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGroupEnd() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t _ncclGroupSimulateEnd(ncclSimInfo_t* simInfo) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/_internal/nccl_linux.pyx000066400000000000000000001144661515037102200254400ustar00rootroot00000000000000# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. # # SPDX-License-Identifier: Apache-2.0 # # This code was automatically generated with version 2.28.0. Do not modify it directly. from libc.stdint cimport intptr_t import threading from .utils import FunctionNotFoundError, NotSupportedError ############################################################################### # Extern ############################################################################### # You must 'from .utils import NotSupportedError' before using this template cdef extern from "" nogil: void* dlopen(const char*, int) char* dlerror() void* dlsym(void*, const char*) int dlclose(void*) enum: RTLD_LAZY RTLD_NOW RTLD_GLOBAL RTLD_LOCAL const void* RTLD_DEFAULT 'RTLD_DEFAULT' cdef int get_cuda_version(): cdef void* handle = NULL cdef int err, driver_ver = 0 # Load driver to check version handle = dlopen('libcuda.so.1', RTLD_NOW | RTLD_GLOBAL) if handle == NULL: err_msg = dlerror() raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') cuDriverGetVersion = dlsym(handle, "cuDriverGetVersion") if cuDriverGetVersion == NULL: raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') err = (cuDriverGetVersion)(&driver_ver) if err != 0: raise RuntimeError(f'cuDriverGetVersion returned error code {err}') return driver_ver ############################################################################### # Wrapper init ############################################################################### cdef object __symbol_lock = threading.Lock() cdef bint __py_nccl_init = False cdef void* __ncclMemAlloc = NULL cdef void* __ncclMemFree = NULL cdef void* __ncclGetVersion = NULL cdef void* __ncclGetUniqueId = NULL cdef void* __ncclCommInitRankConfig = NULL cdef void* __ncclCommInitRank = NULL cdef void* __ncclCommInitAll = NULL cdef void* __ncclCommFinalize = NULL cdef void* __ncclCommDestroy = NULL cdef void* __ncclCommAbort = NULL cdef void* __ncclCommSplit = NULL cdef void* __ncclCommShrink = NULL cdef void* __ncclCommInitRankScalable = NULL cdef void* __ncclGetErrorString = NULL cdef void* __ncclGetLastError = NULL cdef void* __ncclCommGetAsyncError = NULL cdef void* __ncclCommCount = NULL cdef void* __ncclCommCuDevice = NULL cdef void* __ncclCommUserRank = NULL cdef void* __ncclCommRegister = NULL cdef void* __ncclCommDeregister = NULL cdef void* __ncclCommWindowRegister = NULL cdef void* __ncclCommWindowDeregister = NULL cdef void* __ncclRedOpCreatePreMulSum = NULL cdef void* __ncclRedOpDestroy = NULL cdef void* __ncclReduce = NULL cdef void* __ncclBcast = NULL cdef void* __ncclBroadcast = NULL cdef void* __ncclAllReduce = NULL cdef void* __ncclReduceScatter = NULL cdef void* __ncclAllGather = NULL cdef void* __ncclAlltoAll = NULL cdef void* __ncclGather = NULL cdef void* __ncclScatter = NULL cdef void* __ncclSend = NULL cdef void* __ncclRecv = NULL cdef void* __ncclSignal = NULL cdef void* __ncclWaitSignal = NULL cdef void* __ncclGroupStart = NULL cdef void* __ncclGroupEnd = NULL cdef void* __ncclGroupSimulateEnd = NULL cdef void* load_library() except* nogil: cdef void* handle handle = dlopen("libnccl.so.2", RTLD_NOW | RTLD_GLOBAL) if handle == NULL: with gil: err_msg = dlerror() raise RuntimeError(f'Failed to dlopen libnccl ({err_msg.decode()})') return handle cdef int _check_or_init_nccl() except -1 nogil: global __py_nccl_init if __py_nccl_init: return 0 cdef void* handle = NULL with gil, __symbol_lock: # Recheck the flag after obtaining the locks if __py_nccl_init: return 0 # Load function global __ncclMemAlloc __ncclMemAlloc = dlsym(RTLD_DEFAULT, 'ncclMemAlloc') if __ncclMemAlloc == NULL: if handle == NULL: handle = load_library() __ncclMemAlloc = dlsym(handle, 'ncclMemAlloc') global __ncclMemFree __ncclMemFree = dlsym(RTLD_DEFAULT, 'ncclMemFree') if __ncclMemFree == NULL: if handle == NULL: handle = load_library() __ncclMemFree = dlsym(handle, 'ncclMemFree') global __ncclGetVersion __ncclGetVersion = dlsym(RTLD_DEFAULT, 'ncclGetVersion') if __ncclGetVersion == NULL: if handle == NULL: handle = load_library() __ncclGetVersion = dlsym(handle, 'ncclGetVersion') global __ncclGetUniqueId __ncclGetUniqueId = dlsym(RTLD_DEFAULT, 'ncclGetUniqueId') if __ncclGetUniqueId == NULL: if handle == NULL: handle = load_library() __ncclGetUniqueId = dlsym(handle, 'ncclGetUniqueId') global __ncclCommInitRankConfig __ncclCommInitRankConfig = dlsym(RTLD_DEFAULT, 'ncclCommInitRankConfig') if __ncclCommInitRankConfig == NULL: if handle == NULL: handle = load_library() __ncclCommInitRankConfig = dlsym(handle, 'ncclCommInitRankConfig') global __ncclCommInitRank __ncclCommInitRank = dlsym(RTLD_DEFAULT, 'ncclCommInitRank') if __ncclCommInitRank == NULL: if handle == NULL: handle = load_library() __ncclCommInitRank = dlsym(handle, 'ncclCommInitRank') global __ncclCommInitAll __ncclCommInitAll = dlsym(RTLD_DEFAULT, 'ncclCommInitAll') if __ncclCommInitAll == NULL: if handle == NULL: handle = load_library() __ncclCommInitAll = dlsym(handle, 'ncclCommInitAll') global __ncclCommFinalize __ncclCommFinalize = dlsym(RTLD_DEFAULT, 'ncclCommFinalize') if __ncclCommFinalize == NULL: if handle == NULL: handle = load_library() __ncclCommFinalize = dlsym(handle, 'ncclCommFinalize') global __ncclCommDestroy __ncclCommDestroy = dlsym(RTLD_DEFAULT, 'ncclCommDestroy') if __ncclCommDestroy == NULL: if handle == NULL: handle = load_library() __ncclCommDestroy = dlsym(handle, 'ncclCommDestroy') global __ncclCommAbort __ncclCommAbort = dlsym(RTLD_DEFAULT, 'ncclCommAbort') if __ncclCommAbort == NULL: if handle == NULL: handle = load_library() __ncclCommAbort = dlsym(handle, 'ncclCommAbort') global __ncclCommSplit __ncclCommSplit = dlsym(RTLD_DEFAULT, 'ncclCommSplit') if __ncclCommSplit == NULL: if handle == NULL: handle = load_library() __ncclCommSplit = dlsym(handle, 'ncclCommSplit') global __ncclCommShrink __ncclCommShrink = dlsym(RTLD_DEFAULT, 'ncclCommShrink') if __ncclCommShrink == NULL: if handle == NULL: handle = load_library() __ncclCommShrink = dlsym(handle, 'ncclCommShrink') global __ncclCommInitRankScalable __ncclCommInitRankScalable = dlsym(RTLD_DEFAULT, 'ncclCommInitRankScalable') if __ncclCommInitRankScalable == NULL: if handle == NULL: handle = load_library() __ncclCommInitRankScalable = dlsym(handle, 'ncclCommInitRankScalable') global __ncclGetErrorString __ncclGetErrorString = dlsym(RTLD_DEFAULT, 'ncclGetErrorString') if __ncclGetErrorString == NULL: if handle == NULL: handle = load_library() __ncclGetErrorString = dlsym(handle, 'ncclGetErrorString') global __ncclGetLastError __ncclGetLastError = dlsym(RTLD_DEFAULT, 'ncclGetLastError') if __ncclGetLastError == NULL: if handle == NULL: handle = load_library() __ncclGetLastError = dlsym(handle, 'ncclGetLastError') global __ncclCommGetAsyncError __ncclCommGetAsyncError = dlsym(RTLD_DEFAULT, 'ncclCommGetAsyncError') if __ncclCommGetAsyncError == NULL: if handle == NULL: handle = load_library() __ncclCommGetAsyncError = dlsym(handle, 'ncclCommGetAsyncError') global __ncclCommCount __ncclCommCount = dlsym(RTLD_DEFAULT, 'ncclCommCount') if __ncclCommCount == NULL: if handle == NULL: handle = load_library() __ncclCommCount = dlsym(handle, 'ncclCommCount') global __ncclCommCuDevice __ncclCommCuDevice = dlsym(RTLD_DEFAULT, 'ncclCommCuDevice') if __ncclCommCuDevice == NULL: if handle == NULL: handle = load_library() __ncclCommCuDevice = dlsym(handle, 'ncclCommCuDevice') global __ncclCommUserRank __ncclCommUserRank = dlsym(RTLD_DEFAULT, 'ncclCommUserRank') if __ncclCommUserRank == NULL: if handle == NULL: handle = load_library() __ncclCommUserRank = dlsym(handle, 'ncclCommUserRank') global __ncclCommRegister __ncclCommRegister = dlsym(RTLD_DEFAULT, 'ncclCommRegister') if __ncclCommRegister == NULL: if handle == NULL: handle = load_library() __ncclCommRegister = dlsym(handle, 'ncclCommRegister') global __ncclCommDeregister __ncclCommDeregister = dlsym(RTLD_DEFAULT, 'ncclCommDeregister') if __ncclCommDeregister == NULL: if handle == NULL: handle = load_library() __ncclCommDeregister = dlsym(handle, 'ncclCommDeregister') global __ncclCommWindowRegister __ncclCommWindowRegister = dlsym(RTLD_DEFAULT, 'ncclCommWindowRegister') if __ncclCommWindowRegister == NULL: if handle == NULL: handle = load_library() __ncclCommWindowRegister = dlsym(handle, 'ncclCommWindowRegister') global __ncclCommWindowDeregister __ncclCommWindowDeregister = dlsym(RTLD_DEFAULT, 'ncclCommWindowDeregister') if __ncclCommWindowDeregister == NULL: if handle == NULL: handle = load_library() __ncclCommWindowDeregister = dlsym(handle, 'ncclCommWindowDeregister') global __ncclRedOpCreatePreMulSum __ncclRedOpCreatePreMulSum = dlsym(RTLD_DEFAULT, 'ncclRedOpCreatePreMulSum') if __ncclRedOpCreatePreMulSum == NULL: if handle == NULL: handle = load_library() __ncclRedOpCreatePreMulSum = dlsym(handle, 'ncclRedOpCreatePreMulSum') global __ncclRedOpDestroy __ncclRedOpDestroy = dlsym(RTLD_DEFAULT, 'ncclRedOpDestroy') if __ncclRedOpDestroy == NULL: if handle == NULL: handle = load_library() __ncclRedOpDestroy = dlsym(handle, 'ncclRedOpDestroy') global __ncclReduce __ncclReduce = dlsym(RTLD_DEFAULT, 'ncclReduce') if __ncclReduce == NULL: if handle == NULL: handle = load_library() __ncclReduce = dlsym(handle, 'ncclReduce') global __ncclBcast __ncclBcast = dlsym(RTLD_DEFAULT, 'ncclBcast') if __ncclBcast == NULL: if handle == NULL: handle = load_library() __ncclBcast = dlsym(handle, 'ncclBcast') global __ncclBroadcast __ncclBroadcast = dlsym(RTLD_DEFAULT, 'ncclBroadcast') if __ncclBroadcast == NULL: if handle == NULL: handle = load_library() __ncclBroadcast = dlsym(handle, 'ncclBroadcast') global __ncclAllReduce __ncclAllReduce = dlsym(RTLD_DEFAULT, 'ncclAllReduce') if __ncclAllReduce == NULL: if handle == NULL: handle = load_library() __ncclAllReduce = dlsym(handle, 'ncclAllReduce') global __ncclReduceScatter __ncclReduceScatter = dlsym(RTLD_DEFAULT, 'ncclReduceScatter') if __ncclReduceScatter == NULL: if handle == NULL: handle = load_library() __ncclReduceScatter = dlsym(handle, 'ncclReduceScatter') global __ncclAllGather __ncclAllGather = dlsym(RTLD_DEFAULT, 'ncclAllGather') if __ncclAllGather == NULL: if handle == NULL: handle = load_library() __ncclAllGather = dlsym(handle, 'ncclAllGather') global __ncclAlltoAll __ncclAlltoAll = dlsym(RTLD_DEFAULT, 'ncclAlltoAll') if __ncclAlltoAll == NULL: if handle == NULL: handle = load_library() __ncclAlltoAll = dlsym(handle, 'ncclAlltoAll') global __ncclGather __ncclGather = dlsym(RTLD_DEFAULT, 'ncclGather') if __ncclGather == NULL: if handle == NULL: handle = load_library() __ncclGather = dlsym(handle, 'ncclGather') global __ncclScatter __ncclScatter = dlsym(RTLD_DEFAULT, 'ncclScatter') if __ncclScatter == NULL: if handle == NULL: handle = load_library() __ncclScatter = dlsym(handle, 'ncclScatter') global __ncclSend __ncclSend = dlsym(RTLD_DEFAULT, 'ncclSend') if __ncclSend == NULL: if handle == NULL: handle = load_library() __ncclSend = dlsym(handle, 'ncclSend') global __ncclRecv __ncclRecv = dlsym(RTLD_DEFAULT, 'ncclRecv') if __ncclRecv == NULL: if handle == NULL: handle = load_library() __ncclRecv = dlsym(handle, 'ncclRecv') global __ncclSignal __ncclSignal = dlsym(RTLD_DEFAULT, 'ncclSignal') if __ncclSignal == NULL: if handle == NULL: handle = load_library() __ncclSignal = dlsym(handle, 'ncclSignal') global __ncclWaitSignal __ncclWaitSignal = dlsym(RTLD_DEFAULT, 'ncclWaitSignal') if __ncclWaitSignal == NULL: if handle == NULL: handle = load_library() __ncclWaitSignal = dlsym(handle, 'ncclWaitSignal') global __ncclGroupStart __ncclGroupStart = dlsym(RTLD_DEFAULT, 'ncclGroupStart') if __ncclGroupStart == NULL: if handle == NULL: handle = load_library() __ncclGroupStart = dlsym(handle, 'ncclGroupStart') global __ncclGroupEnd __ncclGroupEnd = dlsym(RTLD_DEFAULT, 'ncclGroupEnd') if __ncclGroupEnd == NULL: if handle == NULL: handle = load_library() __ncclGroupEnd = dlsym(handle, 'ncclGroupEnd') global __ncclGroupSimulateEnd __ncclGroupSimulateEnd = dlsym(RTLD_DEFAULT, 'ncclGroupSimulateEnd') if __ncclGroupSimulateEnd == NULL: if handle == NULL: handle = load_library() __ncclGroupSimulateEnd = dlsym(handle, 'ncclGroupSimulateEnd') __py_nccl_init = True return 0 cdef dict func_ptrs = None cpdef dict _inspect_function_pointers(): global func_ptrs if func_ptrs is not None: return func_ptrs _check_or_init_nccl() cdef dict data = {} global __ncclMemAlloc data["__ncclMemAlloc"] = __ncclMemAlloc global __ncclMemFree data["__ncclMemFree"] = __ncclMemFree global __ncclGetVersion data["__ncclGetVersion"] = __ncclGetVersion global __ncclGetUniqueId data["__ncclGetUniqueId"] = __ncclGetUniqueId global __ncclCommInitRankConfig data["__ncclCommInitRankConfig"] = __ncclCommInitRankConfig global __ncclCommInitRank data["__ncclCommInitRank"] = __ncclCommInitRank global __ncclCommInitAll data["__ncclCommInitAll"] = __ncclCommInitAll global __ncclCommFinalize data["__ncclCommFinalize"] = __ncclCommFinalize global __ncclCommDestroy data["__ncclCommDestroy"] = __ncclCommDestroy global __ncclCommAbort data["__ncclCommAbort"] = __ncclCommAbort global __ncclCommSplit data["__ncclCommSplit"] = __ncclCommSplit global __ncclCommShrink data["__ncclCommShrink"] = __ncclCommShrink global __ncclCommInitRankScalable data["__ncclCommInitRankScalable"] = __ncclCommInitRankScalable global __ncclGetErrorString data["__ncclGetErrorString"] = __ncclGetErrorString global __ncclGetLastError data["__ncclGetLastError"] = __ncclGetLastError global __ncclCommGetAsyncError data["__ncclCommGetAsyncError"] = __ncclCommGetAsyncError global __ncclCommCount data["__ncclCommCount"] = __ncclCommCount global __ncclCommCuDevice data["__ncclCommCuDevice"] = __ncclCommCuDevice global __ncclCommUserRank data["__ncclCommUserRank"] = __ncclCommUserRank global __ncclCommRegister data["__ncclCommRegister"] = __ncclCommRegister global __ncclCommDeregister data["__ncclCommDeregister"] = __ncclCommDeregister global __ncclCommWindowRegister data["__ncclCommWindowRegister"] = __ncclCommWindowRegister global __ncclCommWindowDeregister data["__ncclCommWindowDeregister"] = __ncclCommWindowDeregister global __ncclRedOpCreatePreMulSum data["__ncclRedOpCreatePreMulSum"] = __ncclRedOpCreatePreMulSum global __ncclRedOpDestroy data["__ncclRedOpDestroy"] = __ncclRedOpDestroy global __ncclReduce data["__ncclReduce"] = __ncclReduce global __ncclBcast data["__ncclBcast"] = __ncclBcast global __ncclBroadcast data["__ncclBroadcast"] = __ncclBroadcast global __ncclAllReduce data["__ncclAllReduce"] = __ncclAllReduce global __ncclReduceScatter data["__ncclReduceScatter"] = __ncclReduceScatter global __ncclAllGather data["__ncclAllGather"] = __ncclAllGather global __ncclAlltoAll data["__ncclAlltoAll"] = __ncclAlltoAll global __ncclGather data["__ncclGather"] = __ncclGather global __ncclScatter data["__ncclScatter"] = __ncclScatter global __ncclSend data["__ncclSend"] = __ncclSend global __ncclRecv data["__ncclRecv"] = __ncclRecv global __ncclSignal data["__ncclSignal"] = __ncclSignal global __ncclWaitSignal data["__ncclWaitSignal"] = __ncclWaitSignal global __ncclGroupStart data["__ncclGroupStart"] = __ncclGroupStart global __ncclGroupEnd data["__ncclGroupEnd"] = __ncclGroupEnd global __ncclGroupSimulateEnd data["__ncclGroupSimulateEnd"] = __ncclGroupSimulateEnd func_ptrs = data return data cpdef _inspect_function_pointer(str name): global func_ptrs if func_ptrs is None: func_ptrs = _inspect_function_pointers() return func_ptrs[name] ############################################################################### # Wrapper functions ############################################################################### cdef ncclResult_t _ncclMemAlloc(void** ptr, size_t size) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclMemAlloc _check_or_init_nccl() if __ncclMemAlloc == NULL: with gil: raise FunctionNotFoundError("function ncclMemAlloc is not found") return (__ncclMemAlloc)( ptr, size) cdef ncclResult_t _ncclMemFree(void* ptr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclMemFree _check_or_init_nccl() if __ncclMemFree == NULL: with gil: raise FunctionNotFoundError("function ncclMemFree is not found") return (__ncclMemFree)( ptr) cdef ncclResult_t _ncclGetVersion(int* version) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclGetVersion _check_or_init_nccl() if __ncclGetVersion == NULL: with gil: raise FunctionNotFoundError("function ncclGetVersion is not found") return (__ncclGetVersion)( version) cdef ncclResult_t _ncclGetUniqueId(ncclUniqueId* uniqueId) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclGetUniqueId _check_or_init_nccl() if __ncclGetUniqueId == NULL: with gil: raise FunctionNotFoundError("function ncclGetUniqueId is not found") return (__ncclGetUniqueId)( uniqueId) cdef ncclResult_t _ncclCommInitRankConfig(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommInitRankConfig _check_or_init_nccl() if __ncclCommInitRankConfig == NULL: with gil: raise FunctionNotFoundError("function ncclCommInitRankConfig is not found") return (__ncclCommInitRankConfig)( comm, nranks, commId, rank, config) cdef ncclResult_t _ncclCommInitRank(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommInitRank _check_or_init_nccl() if __ncclCommInitRank == NULL: with gil: raise FunctionNotFoundError("function ncclCommInitRank is not found") return (__ncclCommInitRank)( comm, nranks, commId, rank) cdef ncclResult_t _ncclCommInitAll(ncclComm_t* comm, int ndev, const int* devlist) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommInitAll _check_or_init_nccl() if __ncclCommInitAll == NULL: with gil: raise FunctionNotFoundError("function ncclCommInitAll is not found") return (__ncclCommInitAll)( comm, ndev, devlist) cdef ncclResult_t _ncclCommFinalize(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommFinalize _check_or_init_nccl() if __ncclCommFinalize == NULL: with gil: raise FunctionNotFoundError("function ncclCommFinalize is not found") return (__ncclCommFinalize)( comm) cdef ncclResult_t _ncclCommDestroy(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommDestroy _check_or_init_nccl() if __ncclCommDestroy == NULL: with gil: raise FunctionNotFoundError("function ncclCommDestroy is not found") return (__ncclCommDestroy)( comm) cdef ncclResult_t _ncclCommAbort(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommAbort _check_or_init_nccl() if __ncclCommAbort == NULL: with gil: raise FunctionNotFoundError("function ncclCommAbort is not found") return (__ncclCommAbort)( comm) cdef ncclResult_t _ncclCommSplit(ncclComm_t comm, int color, int key, ncclComm_t* newcomm, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommSplit _check_or_init_nccl() if __ncclCommSplit == NULL: with gil: raise FunctionNotFoundError("function ncclCommSplit is not found") return (__ncclCommSplit)( comm, color, key, newcomm, config) cdef ncclResult_t _ncclCommShrink(ncclComm_t comm, int* excludeRanksList, int excludeRanksCount, ncclComm_t* newcomm, ncclConfig_t* config, int shrinkFlags) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommShrink _check_or_init_nccl() if __ncclCommShrink == NULL: with gil: raise FunctionNotFoundError("function ncclCommShrink is not found") return (__ncclCommShrink)( comm, excludeRanksList, excludeRanksCount, newcomm, config, shrinkFlags) cdef ncclResult_t _ncclCommInitRankScalable(ncclComm_t* newcomm, int nranks, int myrank, int nId, ncclUniqueId* commIds, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommInitRankScalable _check_or_init_nccl() if __ncclCommInitRankScalable == NULL: with gil: raise FunctionNotFoundError("function ncclCommInitRankScalable is not found") return (__ncclCommInitRankScalable)( newcomm, nranks, myrank, nId, commIds, config) cdef const char* _ncclGetErrorString(ncclResult_t result) except?NULL nogil: global __ncclGetErrorString _check_or_init_nccl() if __ncclGetErrorString == NULL: with gil: raise FunctionNotFoundError("function ncclGetErrorString is not found") return (__ncclGetErrorString)( result) cdef const char* _ncclGetLastError(ncclComm_t comm) except?NULL nogil: global __ncclGetLastError _check_or_init_nccl() if __ncclGetLastError == NULL: with gil: raise FunctionNotFoundError("function ncclGetLastError is not found") return (__ncclGetLastError)( comm) cdef ncclResult_t _ncclCommGetAsyncError(ncclComm_t comm, ncclResult_t* asyncError) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommGetAsyncError _check_or_init_nccl() if __ncclCommGetAsyncError == NULL: with gil: raise FunctionNotFoundError("function ncclCommGetAsyncError is not found") return (__ncclCommGetAsyncError)( comm, asyncError) cdef ncclResult_t _ncclCommCount(const ncclComm_t comm, int* count) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommCount _check_or_init_nccl() if __ncclCommCount == NULL: with gil: raise FunctionNotFoundError("function ncclCommCount is not found") return (__ncclCommCount)( comm, count) cdef ncclResult_t _ncclCommCuDevice(const ncclComm_t comm, int* device) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommCuDevice _check_or_init_nccl() if __ncclCommCuDevice == NULL: with gil: raise FunctionNotFoundError("function ncclCommCuDevice is not found") return (__ncclCommCuDevice)( comm, device) cdef ncclResult_t _ncclCommUserRank(const ncclComm_t comm, int* rank) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommUserRank _check_or_init_nccl() if __ncclCommUserRank == NULL: with gil: raise FunctionNotFoundError("function ncclCommUserRank is not found") return (__ncclCommUserRank)( comm, rank) cdef ncclResult_t _ncclCommRegister(const ncclComm_t comm, void* buff, size_t size, void** handle) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommRegister _check_or_init_nccl() if __ncclCommRegister == NULL: with gil: raise FunctionNotFoundError("function ncclCommRegister is not found") return (__ncclCommRegister)( comm, buff, size, handle) cdef ncclResult_t _ncclCommDeregister(const ncclComm_t comm, void* handle) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommDeregister _check_or_init_nccl() if __ncclCommDeregister == NULL: with gil: raise FunctionNotFoundError("function ncclCommDeregister is not found") return (__ncclCommDeregister)( comm, handle) cdef ncclResult_t _ncclCommWindowRegister(ncclComm_t comm, void* buff, size_t size, ncclWindow_t* win, int winFlags) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommWindowRegister _check_or_init_nccl() if __ncclCommWindowRegister == NULL: with gil: raise FunctionNotFoundError("function ncclCommWindowRegister is not found") return (__ncclCommWindowRegister)( comm, buff, size, win, winFlags) cdef ncclResult_t _ncclCommWindowDeregister(ncclComm_t comm, ncclWindow_t win) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclCommWindowDeregister _check_or_init_nccl() if __ncclCommWindowDeregister == NULL: with gil: raise FunctionNotFoundError("function ncclCommWindowDeregister is not found") return (__ncclCommWindowDeregister)( comm, win) cdef ncclResult_t _ncclRedOpCreatePreMulSum(ncclRedOp_t* op, void* scalar, ncclDataType_t datatype, ncclScalarResidence_t residence, ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclRedOpCreatePreMulSum _check_or_init_nccl() if __ncclRedOpCreatePreMulSum == NULL: with gil: raise FunctionNotFoundError("function ncclRedOpCreatePreMulSum is not found") return (__ncclRedOpCreatePreMulSum)( op, scalar, datatype, residence, comm) cdef ncclResult_t _ncclRedOpDestroy(ncclRedOp_t op, ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclRedOpDestroy _check_or_init_nccl() if __ncclRedOpDestroy == NULL: with gil: raise FunctionNotFoundError("function ncclRedOpDestroy is not found") return (__ncclRedOpDestroy)( op, comm) cdef ncclResult_t _ncclReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclReduce _check_or_init_nccl() if __ncclReduce == NULL: with gil: raise FunctionNotFoundError("function ncclReduce is not found") return (__ncclReduce)( sendbuff, recvbuff, count, datatype, op, root, comm, stream) cdef ncclResult_t _ncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclBcast _check_or_init_nccl() if __ncclBcast == NULL: with gil: raise FunctionNotFoundError("function ncclBcast is not found") return (__ncclBcast)( buff, count, datatype, root, comm, stream) cdef ncclResult_t _ncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclBroadcast _check_or_init_nccl() if __ncclBroadcast == NULL: with gil: raise FunctionNotFoundError("function ncclBroadcast is not found") return (__ncclBroadcast)( sendbuff, recvbuff, count, datatype, root, comm, stream) cdef ncclResult_t _ncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclAllReduce _check_or_init_nccl() if __ncclAllReduce == NULL: with gil: raise FunctionNotFoundError("function ncclAllReduce is not found") return (__ncclAllReduce)( sendbuff, recvbuff, count, datatype, op, comm, stream) cdef ncclResult_t _ncclReduceScatter(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclReduceScatter _check_or_init_nccl() if __ncclReduceScatter == NULL: with gil: raise FunctionNotFoundError("function ncclReduceScatter is not found") return (__ncclReduceScatter)( sendbuff, recvbuff, recvcount, datatype, op, comm, stream) cdef ncclResult_t _ncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclAllGather _check_or_init_nccl() if __ncclAllGather == NULL: with gil: raise FunctionNotFoundError("function ncclAllGather is not found") return (__ncclAllGather)( sendbuff, recvbuff, sendcount, datatype, comm, stream) cdef ncclResult_t _ncclAlltoAll(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclAlltoAll _check_or_init_nccl() if __ncclAlltoAll == NULL: with gil: raise FunctionNotFoundError("function ncclAlltoAll is not found") return (__ncclAlltoAll)( sendbuff, recvbuff, count, datatype, comm, stream) cdef ncclResult_t _ncclGather(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclGather _check_or_init_nccl() if __ncclGather == NULL: with gil: raise FunctionNotFoundError("function ncclGather is not found") return (__ncclGather)( sendbuff, recvbuff, count, datatype, root, comm, stream) cdef ncclResult_t _ncclScatter(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclScatter _check_or_init_nccl() if __ncclScatter == NULL: with gil: raise FunctionNotFoundError("function ncclScatter is not found") return (__ncclScatter)( sendbuff, recvbuff, count, datatype, root, comm, stream) cdef ncclResult_t _ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclSend _check_or_init_nccl() if __ncclSend == NULL: with gil: raise FunctionNotFoundError("function ncclSend is not found") return (__ncclSend)( sendbuff, count, datatype, peer, comm, stream) cdef ncclResult_t _ncclRecv(void* recvbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclRecv _check_or_init_nccl() if __ncclRecv == NULL: with gil: raise FunctionNotFoundError("function ncclRecv is not found") return (__ncclRecv)( recvbuff, count, datatype, peer, comm, stream) cdef ncclResult_t _ncclSignal(int peer, int sigIdx, int ctx, unsigned int flags, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclSignal _check_or_init_nccl() if __ncclSignal == NULL: with gil: raise FunctionNotFoundError("function ncclSignal is not found") return (__ncclSignal)( peer, sigIdx, ctx, flags, comm, stream) cdef ncclResult_t _ncclWaitSignal(int nDesc, ncclWaitSignalDesc_t* signalDescs, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclWaitSignal _check_or_init_nccl() if __ncclWaitSignal == NULL: with gil: raise FunctionNotFoundError("function ncclWaitSignal is not found") return (__ncclWaitSignal)( nDesc, signalDescs, comm, stream) cdef ncclResult_t _ncclGroupStart() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclGroupStart _check_or_init_nccl() if __ncclGroupStart == NULL: with gil: raise FunctionNotFoundError("function ncclGroupStart is not found") return (__ncclGroupStart)( ) cdef ncclResult_t _ncclGroupEnd() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclGroupEnd _check_or_init_nccl() if __ncclGroupEnd == NULL: with gil: raise FunctionNotFoundError("function ncclGroupEnd is not found") return (__ncclGroupEnd)( ) cdef ncclResult_t _ncclGroupSimulateEnd(ncclSimInfo_t* simInfo) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: global __ncclGroupSimulateEnd _check_or_init_nccl() if __ncclGroupSimulateEnd == NULL: with gil: raise FunctionNotFoundError("function ncclGroupSimulateEnd is not found") return (__ncclGroupSimulateEnd)( simInfo) nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/_internal/utils.pxd000066400000000000000000000111031515037102200243750ustar00rootroot00000000000000# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. # # SPDX-License-Identifier: Apache-2.0 from libc.stdint cimport int32_t, int64_t, intptr_t from libcpp.vector cimport vector from libcpp cimport bool as cppbool from libcpp cimport nullptr_t, nullptr from libcpp.memory cimport unique_ptr from ..cynccl cimport ncclUniqueId cdef extern from * nogil: """ template class nullable_unique_ptr { public: nullable_unique_ptr() noexcept = default; nullable_unique_ptr(std::nullptr_t) noexcept = delete; explicit nullable_unique_ptr(T* data, bool own_data): own_data_(own_data) { if (own_data) manager_.reset(data); else raw_data_ = data; } nullable_unique_ptr(const nullable_unique_ptr&) = delete; nullable_unique_ptr& operator=(const nullable_unique_ptr&) = delete; nullable_unique_ptr(nullable_unique_ptr&& other) noexcept { own_data_ = other.own_data_; other.own_data_ = false; // ownership is transferred if (own_data_) { manager_ = std::move(other.manager_); raw_data_ = nullptr; // just in case } else { manager_.reset(nullptr); // just in case raw_data_ = other.raw_data_; } } nullable_unique_ptr& operator=(nullable_unique_ptr&& other) noexcept { own_data_ = other.own_data_; other.own_data_ = false; // ownership is transferred if (own_data_) { manager_ = std::move(other.manager_); raw_data_ = nullptr; // just in case } else { manager_.reset(nullptr); // just in case raw_data_ = other.raw_data_; } return *this; } ~nullable_unique_ptr() = default; void reset(T* data, bool own_data) { own_data_ = own_data; if (own_data_) { manager_.reset(data); raw_data_ = nullptr; } else { manager_.reset(nullptr); raw_data_ = data; } } void swap(nullable_unique_ptr& other) noexcept { std::swap(manager_, other.manager_); std::swap(raw_data_, other.raw_data_); std::swap(own_data_, other.own_data_); } /* * Get the pointer to the underlying object (this is different from data()!). */ T* get() const noexcept { if (own_data_) return manager_.get(); else return raw_data_; } /* * Get the pointer to the underlying buffer (this is different from get()!). */ void* data() noexcept { if (own_data_) return manager_.get()->data(); else return raw_data_; } T& operator*() { if (own_data_) return *manager_; else return *raw_data_; } private: std::unique_ptr manager_{}; T* raw_data_{nullptr}; bool own_data_{false}; }; """ # xref: cython/Cython/Includes/libcpp/memory.pxd cdef cppclass nullable_unique_ptr[T]: nullable_unique_ptr() nullable_unique_ptr(T*, cppbool) nullable_unique_ptr(nullable_unique_ptr[T]&) # Modifiers void reset(T*, cppbool) void swap(nullable_unique_ptr&) # Observers T* get() T& operator*() void* data() ctypedef fused ResT: int int32_t int64_t char float double ncclUniqueId ctypedef fused PtrT: void cdef cppclass nested_resource[T]: nullable_unique_ptr[ vector[intptr_t] ] ptrs nullable_unique_ptr[ vector[vector[T]] ] nested_resource_ptr # accepts the output pointer as input to use the return value for exception propagation cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1 cdef int get_resource_ptrs(nullable_unique_ptr[ vector[PtrT*] ] &in_out_ptr, object obj, PtrT* __unused) except 1 cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1 cdef bint is_nested_sequence(data) cdef void* get_buffer_pointer(buf, Py_ssize_t size, readonly=*) except* nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/_internal/utils.pyx000066400000000000000000000115041515037102200244270ustar00rootroot00000000000000# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. # # SPDX-License-Identifier: Apache-2.0 cimport cpython from libc.stdint cimport intptr_t from libcpp.utility cimport move from cython.operator cimport dereference as deref cdef bint is_nested_sequence(data): if not cpython.PySequence_Check(data): return False else: for i in data: if not cpython.PySequence_Check(i): return False else: return True cdef void* get_buffer_pointer(buf, Py_ssize_t size, readonly=True) except*: """The caller must ensure ``buf`` is alive when the returned pointer is in use.""" cdef void* bufPtr cdef int flags = cpython.PyBUF_ANY_CONTIGUOUS if not readonly: flags |= cpython.PyBUF_WRITABLE cdef int status = -1 cdef cpython.Py_buffer view if isinstance(buf, int): bufPtr = buf else: # try buffer protocol try: status = cpython.PyObject_GetBuffer(buf, &view, flags) # when the caller does not provide a size, it is set to -1 at generate-time by cybind if size != -1: assert view.len == size assert view.ndim == 1 except Exception as e: adj = "writable " if not readonly else "" raise ValueError( "buf must be either a Python int representing the pointer " f"address to a valid buffer, or a 1D contiguous {adj}" "buffer, of size bytes") from e else: bufPtr = view.buf finally: if status == 0: cpython.PyBuffer_Release(&view) return bufPtr # Cython can't infer the ResT overload when it is wrapped in nullable_unique_ptr, # so we need a dummy (__unused) input argument to help it cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: if cpython.PySequence_Check(obj): vec = new vector[ResT](len(obj)) # set the ownership immediately to avoid leaking the `vec` memory in # case of exception in the following loop in_out_ptr.reset(vec, True) for i in range(len(obj)): deref(vec)[i] = obj[i] else: in_out_ptr.reset(obj, False) return 0 cdef int get_resource_ptrs(nullable_unique_ptr[ vector[PtrT*] ] &in_out_ptr, object obj, PtrT* __unused) except 1: if cpython.PySequence_Check(obj): vec = new vector[PtrT*](len(obj)) # set the ownership immediately to avoid leaking the `vec` memory in # case of exception in the following loop in_out_ptr.reset(vec, True) for i in range(len(obj)): deref(vec)[i] = (obj[i]) else: in_out_ptr.reset(obj, False) return 0 cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1: cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr cdef vector[intptr_t]* nested_vec = NULL cdef vector[vector[ResT]]* nested_res_vec = NULL cdef size_t i = 0, length = 0 cdef intptr_t addr if is_nested_sequence(obj): length = len(obj) nested_res_vec = new vector[vector[ResT]](length) nested_vec = new vector[intptr_t](length) # set the ownership immediately to avoid leaking memory in case of # exception in the following loop nested_res_ptr.reset(nested_res_vec, True) nested_ptr.reset(nested_vec, True) for i, obj_i in enumerate(obj): if ResT is char: obj_i_bytes = ((obj_i)).encode() str_len = (len(obj_i_bytes)) + 1 # including null termination deref(nested_res_vec)[i].resize(str_len) obj_i_ptr = (obj_i_bytes) # cast to size_t explicitly to work around a potentially Cython bug deref(nested_res_vec)[i].assign(obj_i_ptr, obj_i_ptr + str_len) else: deref(nested_res_vec)[i] = obj_i deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) elif cpython.PySequence_Check(obj): length = len(obj) nested_vec = new vector[intptr_t](length) nested_ptr.reset(nested_vec, True) for i, addr in enumerate(obj): deref(nested_vec)[i] = addr nested_res_ptr.reset(NULL, False) else: # obj is an int (ResT**) nested_res_ptr.reset(NULL, False) nested_ptr.reset(obj, False) in_out_ptr.ptrs = move(nested_ptr) in_out_ptr.nested_resource_ptr = move(nested_res_ptr) return 0 class FunctionNotFoundError(RuntimeError): pass class NotSupportedError(RuntimeError): pass nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/cynccl.pxd000066400000000000000000000227201515037102200225440ustar00rootroot00000000000000# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. # # SPDX-License-Identifier: Apache-2.0 # # This code was automatically generated with version 2.28.0. Do not modify it directly. from libc.stdint cimport int64_t ############################################################################### # Types (structs, enums, ...) ############################################################################### # enums ctypedef enum ncclResult_t "ncclResult_t": ncclSuccess "ncclSuccess" = 0 ncclUnhandledCudaError "ncclUnhandledCudaError" = 1 ncclSystemError "ncclSystemError" = 2 ncclInternalError "ncclInternalError" = 3 ncclInvalidArgument "ncclInvalidArgument" = 4 ncclInvalidUsage "ncclInvalidUsage" = 5 ncclRemoteError "ncclRemoteError" = 6 ncclInProgress "ncclInProgress" = 7 ncclNumResults "ncclNumResults" = 8 _NCCLRESULT_T_INTERNAL_LOADING_ERROR "_NCCLRESULT_T_INTERNAL_LOADING_ERROR" = -42 ctypedef enum ncclCommMemStat_t "ncclCommMemStat_t": ncclStatGpuMemSuspend "ncclStatGpuMemSuspend" = 0 ncclStatGpuMemSuspended "ncclStatGpuMemSuspended" = 1 ncclStatGpuMemPersist "ncclStatGpuMemPersist" = 2 ncclStatGpuMemTotal "ncclStatGpuMemTotal" = 3 ctypedef enum ncclRedOp_dummy_t "ncclRedOp_dummy_t": ncclNumOps_dummy "ncclNumOps_dummy" = 5 ctypedef enum ncclRedOp_t "ncclRedOp_t": ncclSum "ncclSum" = 0 ncclProd "ncclProd" = 1 ncclMax "ncclMax" = 2 ncclMin "ncclMin" = 3 ncclAvg "ncclAvg" = 4 ncclNumOps "ncclNumOps" = 5 ncclMaxRedOp "ncclMaxRedOp" = (0x7fffffff >> (32 - (8 * sizeof(ncclRedOp_dummy_t)))) ctypedef enum ncclDataType_t "ncclDataType_t": ncclInt8 "ncclInt8" = 0 ncclChar "ncclChar" = 0 ncclUint8 "ncclUint8" = 1 ncclInt32 "ncclInt32" = 2 ncclInt "ncclInt" = 2 ncclUint32 "ncclUint32" = 3 ncclInt64 "ncclInt64" = 4 ncclUint64 "ncclUint64" = 5 ncclFloat16 "ncclFloat16" = 6 ncclHalf "ncclHalf" = 6 ncclFloat32 "ncclFloat32" = 7 ncclFloat "ncclFloat" = 7 ncclFloat64 "ncclFloat64" = 8 ncclDouble "ncclDouble" = 8 ncclBfloat16 "ncclBfloat16" = 9 ncclFloat8e4m3 "ncclFloat8e4m3" = 10 ncclFloat8e5m2 "ncclFloat8e5m2" = 11 ncclNumTypes "ncclNumTypes" = 12 ctypedef enum ncclScalarResidence_t "ncclScalarResidence_t": ncclScalarDevice "ncclScalarDevice" = 0 ncclScalarHostImmediate "ncclScalarHostImmediate" = 1 # types cdef extern from *: """ #include #include #include """ ctypedef void* cudaStream_t 'cudaStream_t' ctypedef void* ncclComm_t 'ncclComm_t' ctypedef void* ncclWindow_t 'ncclWindow_t' ctypedef struct ncclUniqueId 'ncclUniqueId': char internal[128] ctypedef struct ncclConfig_t 'ncclConfig_t': size_t size unsigned int magic unsigned int version int blocking int cgaClusterSize int minCTAs int maxCTAs char* netName int splitShare int trafficClass char* commName int collnetEnable int CTAPolicy int shrinkShare int nvlsCTAs int nChannelsPerNetPeer int nvlinkCentricSched int graphUsageMode int numRmaCtx ctypedef struct ncclSimInfo_t 'ncclSimInfo_t': size_t size unsigned int magic unsigned int version float estimatedTime ctypedef struct ncclWaitSignalDesc_t 'ncclWaitSignalDesc_t': int opCnt int peer int sigIdx int ctx ############################################################################### # Functions ############################################################################### cdef ncclResult_t ncclMemAlloc(void** ptr, size_t size) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclMemFree(void* ptr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGetVersion(int* version) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommInitRankConfig(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommInitRank(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommInitAll(ncclComm_t* comm, int ndev, const int* devlist) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommFinalize(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommDestroy(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommAbort(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommSplit(ncclComm_t comm, int color, int key, ncclComm_t* newcomm, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommShrink(ncclComm_t comm, int* excludeRanksList, int excludeRanksCount, ncclComm_t* newcomm, ncclConfig_t* config, int shrinkFlags) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommInitRankScalable(ncclComm_t* newcomm, int nranks, int myrank, int nId, ncclUniqueId* commIds, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef const char* ncclGetErrorString(ncclResult_t result) except?NULL nogil cdef const char* ncclGetLastError(ncclComm_t comm) except?NULL nogil cdef ncclResult_t ncclCommGetAsyncError(ncclComm_t comm, ncclResult_t* asyncError) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommCount(const ncclComm_t comm, int* count) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommCuDevice(const ncclComm_t comm, int* device) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommUserRank(const ncclComm_t comm, int* rank) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommRegister(const ncclComm_t comm, void* buff, size_t size, void** handle) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommDeregister(const ncclComm_t comm, void* handle) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommWindowRegister(ncclComm_t comm, void* buff, size_t size, ncclWindow_t* win, int winFlags) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclCommWindowDeregister(ncclComm_t comm, ncclWindow_t win) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclRedOpCreatePreMulSum(ncclRedOp_t* op, void* scalar, ncclDataType_t datatype, ncclScalarResidence_t residence, ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclRedOpDestroy(ncclRedOp_t op, ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclReduceScatter(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclAlltoAll(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGather(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclScatter(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclRecv(void* recvbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclSignal(int peer, int sigIdx, int ctx, unsigned int flags, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclWaitSignal(int nDesc, ncclWaitSignalDesc_t* signalDescs, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGroupStart() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGroupEnd() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil cdef ncclResult_t ncclGroupSimulateEnd(ncclSimInfo_t* simInfo) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/cynccl.pyx000066400000000000000000000217331515037102200225740ustar00rootroot00000000000000# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. # # SPDX-License-Identifier: Apache-2.0 # # This code was automatically generated with version 2.28.0. Do not modify it directly. from ._internal cimport nccl as _nccl ############################################################################### # Wrapper functions ############################################################################### cdef ncclResult_t ncclMemAlloc(void** ptr, size_t size) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclMemAlloc(ptr, size) cdef ncclResult_t ncclMemFree(void* ptr) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclMemFree(ptr) cdef ncclResult_t ncclGetVersion(int* version) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclGetVersion(version) cdef ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclGetUniqueId(uniqueId) cdef ncclResult_t ncclCommInitRankConfig(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommInitRankConfig(comm, nranks, commId, rank, config) cdef ncclResult_t ncclCommInitRank(ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommInitRank(comm, nranks, commId, rank) cdef ncclResult_t ncclCommInitAll(ncclComm_t* comm, int ndev, const int* devlist) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommInitAll(comm, ndev, devlist) cdef ncclResult_t ncclCommFinalize(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommFinalize(comm) cdef ncclResult_t ncclCommDestroy(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommDestroy(comm) cdef ncclResult_t ncclCommAbort(ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommAbort(comm) cdef ncclResult_t ncclCommSplit(ncclComm_t comm, int color, int key, ncclComm_t* newcomm, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommSplit(comm, color, key, newcomm, config) cdef ncclResult_t ncclCommShrink(ncclComm_t comm, int* excludeRanksList, int excludeRanksCount, ncclComm_t* newcomm, ncclConfig_t* config, int shrinkFlags) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommShrink(comm, excludeRanksList, excludeRanksCount, newcomm, config, shrinkFlags) cdef ncclResult_t ncclCommInitRankScalable(ncclComm_t* newcomm, int nranks, int myrank, int nId, ncclUniqueId* commIds, ncclConfig_t* config) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommInitRankScalable(newcomm, nranks, myrank, nId, commIds, config) cdef const char* ncclGetErrorString(ncclResult_t result) except?NULL nogil: return _nccl._ncclGetErrorString(result) cdef const char* ncclGetLastError(ncclComm_t comm) except?NULL nogil: return _nccl._ncclGetLastError(comm) cdef ncclResult_t ncclCommGetAsyncError(ncclComm_t comm, ncclResult_t* asyncError) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommGetAsyncError(comm, asyncError) cdef ncclResult_t ncclCommCount(const ncclComm_t comm, int* count) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommCount(comm, count) cdef ncclResult_t ncclCommCuDevice(const ncclComm_t comm, int* device) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommCuDevice(comm, device) cdef ncclResult_t ncclCommUserRank(const ncclComm_t comm, int* rank) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommUserRank(comm, rank) cdef ncclResult_t ncclCommRegister(const ncclComm_t comm, void* buff, size_t size, void** handle) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommRegister(comm, buff, size, handle) cdef ncclResult_t ncclCommDeregister(const ncclComm_t comm, void* handle) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommDeregister(comm, handle) cdef ncclResult_t ncclCommWindowRegister(ncclComm_t comm, void* buff, size_t size, ncclWindow_t* win, int winFlags) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommWindowRegister(comm, buff, size, win, winFlags) cdef ncclResult_t ncclCommWindowDeregister(ncclComm_t comm, ncclWindow_t win) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclCommWindowDeregister(comm, win) cdef ncclResult_t ncclRedOpCreatePreMulSum(ncclRedOp_t* op, void* scalar, ncclDataType_t datatype, ncclScalarResidence_t residence, ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclRedOpCreatePreMulSum(op, scalar, datatype, residence, comm) cdef ncclResult_t ncclRedOpDestroy(ncclRedOp_t op, ncclComm_t comm) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclRedOpDestroy(op, comm) cdef ncclResult_t ncclReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclReduce(sendbuff, recvbuff, count, datatype, op, root, comm, stream) cdef ncclResult_t ncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclBcast(buff, count, datatype, root, comm, stream) cdef ncclResult_t ncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclBroadcast(sendbuff, recvbuff, count, datatype, root, comm, stream) cdef ncclResult_t ncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclAllReduce(sendbuff, recvbuff, count, datatype, op, comm, stream) cdef ncclResult_t ncclReduceScatter(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclReduceScatter(sendbuff, recvbuff, recvcount, datatype, op, comm, stream) cdef ncclResult_t ncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclAllGather(sendbuff, recvbuff, sendcount, datatype, comm, stream) cdef ncclResult_t ncclAlltoAll(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclAlltoAll(sendbuff, recvbuff, count, datatype, comm, stream) cdef ncclResult_t ncclGather(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclGather(sendbuff, recvbuff, count, datatype, root, comm, stream) cdef ncclResult_t ncclScatter(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclScatter(sendbuff, recvbuff, count, datatype, root, comm, stream) cdef ncclResult_t ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclSend(sendbuff, count, datatype, peer, comm, stream) cdef ncclResult_t ncclRecv(void* recvbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclRecv(recvbuff, count, datatype, peer, comm, stream) cdef ncclResult_t ncclSignal(int peer, int sigIdx, int ctx, unsigned int flags, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclSignal(peer, sigIdx, ctx, flags, comm, stream) cdef ncclResult_t ncclWaitSignal(int nDesc, ncclWaitSignalDesc_t* signalDescs, ncclComm_t comm, cudaStream_t stream) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclWaitSignal(nDesc, signalDescs, comm, stream) cdef ncclResult_t ncclGroupStart() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclGroupStart() cdef ncclResult_t ncclGroupEnd() except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclGroupEnd() cdef ncclResult_t ncclGroupSimulateEnd(ncclSimInfo_t* simInfo) except?_NCCLRESULT_T_INTERNAL_LOADING_ERROR nogil: return _nccl._ncclGroupSimulateEnd(simInfo) nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/nccl.pxd000066400000000000000000000101141515037102200222020ustar00rootroot00000000000000# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. # # SPDX-License-Identifier: Apache-2.0 # # This code was automatically generated with version 2.28.0. Do not modify it directly. from libc.stdint cimport intptr_t from .cynccl cimport * ############################################################################### # Types ############################################################################### ctypedef ncclComm_t Comm ctypedef ncclWindow_t Window ctypedef cudaStream_t Stream ############################################################################### # Enum ############################################################################### ctypedef ncclResult_t _Result ctypedef ncclCommMemStat_t _CommMemStat ctypedef ncclRedOp_dummy_t _RedOp_dummy ctypedef ncclRedOp_t _RedOp ctypedef ncclDataType_t _DataType ctypedef ncclScalarResidence_t _ScalarResidence ############################################################################### # Functions ############################################################################### cpdef intptr_t mem_alloc(size_t size) except? 0 cpdef mem_free(intptr_t ptr) cpdef int get_version() except? -1 cpdef get_unique_id(intptr_t unique_id) cpdef intptr_t comm_init_rank_config(int nranks, comm_id, int rank, intptr_t config) except? 0 cpdef intptr_t comm_init_rank(int nranks, comm_id, int rank) except? 0 cpdef comm_init_all(intptr_t comm, int ndev, devlist) cpdef comm_finalize(intptr_t comm) cpdef comm_destroy(intptr_t comm) cpdef comm_abort(intptr_t comm) cpdef intptr_t comm_split(intptr_t comm, int color, int key, intptr_t config) except? 0 cpdef intptr_t comm_shrink(intptr_t comm, exclude_ranks_list, int exclude_ranks_count, intptr_t config, int shrink_flags) except? 0 cpdef intptr_t comm_init_rank_scalable(int nranks, int myrank, int n_id, comm_ids, intptr_t config) except? 0 cpdef str get_error_string(int result) cpdef str get_last_error(intptr_t comm) cpdef int comm_get_async_error(intptr_t comm) except? -1 cpdef int comm_count(intptr_t comm) except? -1 cpdef int comm_cu_device(intptr_t comm) except? -1 cpdef int comm_user_rank(intptr_t comm) except? -1 cpdef intptr_t comm_register(intptr_t comm, intptr_t buff, size_t size) except? 0 cpdef comm_deregister(intptr_t comm, intptr_t handle) cpdef intptr_t comm_window_register(intptr_t comm, intptr_t buff, size_t size, int win_flags) except? 0 cpdef comm_window_deregister(intptr_t comm, intptr_t win) cpdef int red_op_create_pre_mul_sum(intptr_t scalar, int datatype, int residence, intptr_t comm) except? -1 cpdef red_op_destroy(int op, intptr_t comm) cpdef reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, int root, intptr_t comm, intptr_t stream) cpdef bcast(intptr_t buff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream) cpdef broadcast(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream) cpdef all_reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, intptr_t comm, intptr_t stream) cpdef reduce_scatter(intptr_t sendbuff, intptr_t recvbuff, size_t recvcount, int datatype, int op, intptr_t comm, intptr_t stream) cpdef all_gather(intptr_t sendbuff, intptr_t recvbuff, size_t sendcount, int datatype, intptr_t comm, intptr_t stream) cpdef allto_all(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, intptr_t comm, intptr_t stream) cpdef gather(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream) cpdef scatter(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream) cpdef send(intptr_t sendbuff, size_t count, int datatype, int peer, intptr_t comm, intptr_t stream) cpdef recv(intptr_t recvbuff, size_t count, int datatype, int peer, intptr_t comm, intptr_t stream) cpdef signal(int peer, int sig_idx, int ctx, unsigned int flags, intptr_t comm, intptr_t stream) cpdef wait_signal(int n_desc, intptr_t signal_descs, intptr_t comm, intptr_t stream) cpdef group_start() cpdef group_end() cpdef group_simulate_end(intptr_t sim_info) nccl-2.29.7-1/bindings/nccl4py/nccl/bindings/nccl.pyx000066400000000000000000001136231515037102200222400ustar00rootroot00000000000000# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED. # # SPDX-License-Identifier: Apache-2.0 # # This code was automatically generated with version 2.28.0. Do not modify it directly. cimport cython # NOQA from libcpp.vector cimport vector from ._internal.utils cimport (nested_resource, nullable_unique_ptr, get_buffer_pointer, get_resource_ptr, get_nested_resource_ptr) from enum import IntEnum as _IntEnum from libc.stdlib cimport calloc, free, malloc from cython cimport view cimport cpython.buffer cimport cpython.memoryview cimport cpython from libc.string cimport memcmp, memcpy import numpy as _numpy cdef __from_data(data, dtype_name, expected_dtype, lowpp_type): # _numpy.recarray is a subclass of _numpy.ndarray, so implicitly handled here. if isinstance(data, lowpp_type): return data if not isinstance(data, _numpy.ndarray): raise TypeError("data argument must be a NumPy ndarray") if data.size != 1: raise ValueError("data array must have a size of 1") if data.dtype != expected_dtype: raise ValueError(f"data array must be of dtype {dtype_name}") return lowpp_type.from_ptr(data.ctypes.data, not data.flags.writeable, data) ############################################################################### # POD ############################################################################### cdef _get_unique_id_dtype_offsets(): cdef ncclUniqueId pod = ncclUniqueId() return _numpy.dtype({ 'names': ['internal'], 'formats': [(_numpy.int8, 128)], 'offsets': [ (&(pod.internal)) - (&pod), ], 'itemsize': sizeof(ncclUniqueId), }) unique_id_dtype = _get_unique_id_dtype_offsets() cdef class UniqueId: """Empty-initialize an instance of `ncclUniqueId`. .. seealso:: `ncclUniqueId` """ cdef: ncclUniqueId *_ptr object _owner bint _owned bint _readonly def __init__(self): self._ptr = calloc(1, sizeof(ncclUniqueId)) if self._ptr == NULL: raise MemoryError("Error allocating UniqueId") self._owner = None self._owned = True self._readonly = False def __dealloc__(self): cdef ncclUniqueId *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL free(ptr) def __repr__(self): return f"<{__name__}.UniqueId object at {hex(id(self))}>" @property def ptr(self): """Get the pointer address to the data as Python :class:`int`.""" return (self._ptr) cdef intptr_t _get_ptr(self): return (self._ptr) def __int__(self): return (self._ptr) def __eq__(self, other): cdef UniqueId other_ if not isinstance(other, UniqueId): return False other_ = other return (memcmp((self._ptr), (other_._ptr), sizeof(ncclUniqueId)) == 0) def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): self._ptr = malloc(sizeof(ncclUniqueId)) if self._ptr == NULL: raise MemoryError("Error allocating UniqueId") memcpy(self._ptr, val.ctypes.data, sizeof(ncclUniqueId)) self._owner = None self._owned = True self._readonly = not val.flags.writeable else: setattr(self, key, val) @staticmethod def from_data(data): """Create an UniqueId instance wrapping the given NumPy array. Args: data (_numpy.ndarray): a single-element array of dtype `unique_id_dtype` holding the data. """ return __from_data(data, "unique_id_dtype", unique_id_dtype, UniqueId) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): """Create an UniqueId instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. owner (object): The Python object that owns the pointer. If not provided, data will be copied. readonly (bool): whether the data is read-only (to the user). default is `False`. """ if ptr == 0: raise ValueError("ptr must not be null (0)") cdef UniqueId obj = UniqueId.__new__(UniqueId) if owner is None: obj._ptr = malloc(sizeof(ncclUniqueId)) if obj._ptr == NULL: raise MemoryError("Error allocating UniqueId") memcpy((obj._ptr), ptr, sizeof(ncclUniqueId)) obj._owner = None obj._owned = True else: obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly return obj cdef _get_config_dtype_offsets(): cdef ncclConfig_t pod = ncclConfig_t() return _numpy.dtype({ 'names': ['size_', 'magic', 'version', 'blocking', 'cga_cluster_size', 'min_ctas', 'max_ctas', 'net_name', 'split_share', 'traffic_class', 'comm_name', 'collnet_enable', 'cta_policy', 'shrink_share', 'nvls_ctas', 'n_channels_per_net_peer', 'nvlink_centric_sched', 'graph_usage_mode', 'num_rma_ctx'], 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.intp, _numpy.int32, _numpy.int32, _numpy.intp, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32], 'offsets': [ (&(pod.size)) - (&pod), (&(pod.magic)) - (&pod), (&(pod.version)) - (&pod), (&(pod.blocking)) - (&pod), (&(pod.cgaClusterSize)) - (&pod), (&(pod.minCTAs)) - (&pod), (&(pod.maxCTAs)) - (&pod), (&(pod.netName)) - (&pod), (&(pod.splitShare)) - (&pod), (&(pod.trafficClass)) - (&pod), (&(pod.commName)) - (&pod), (&(pod.collnetEnable)) - (&pod), (&(pod.CTAPolicy)) - (&pod), (&(pod.shrinkShare)) - (&pod), (&(pod.nvlsCTAs)) - (&pod), (&(pod.nChannelsPerNetPeer)) - (&pod), (&(pod.nvlinkCentricSched)) - (&pod), (&(pod.graphUsageMode)) - (&pod), (&(pod.numRmaCtx)) - (&pod), ], 'itemsize': sizeof(ncclConfig_t), }) config_dtype = _get_config_dtype_offsets() cdef class Config: """Empty-initialize an instance of `ncclConfig_t`. .. seealso:: `ncclConfig_t` """ cdef: ncclConfig_t *_ptr object _owner bint _owned bint _readonly dict _refs def __init__(self): self._ptr = calloc(1, sizeof(ncclConfig_t)) if self._ptr == NULL: raise MemoryError("Error allocating Config") self._owner = None self._owned = True self._readonly = False self._refs = {} def __dealloc__(self): cdef ncclConfig_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL free(ptr) def __repr__(self): return f"<{__name__}.Config object at {hex(id(self))}>" @property def ptr(self): """Get the pointer address to the data as Python :class:`int`.""" return (self._ptr) cdef intptr_t _get_ptr(self): return (self._ptr) def __int__(self): return (self._ptr) def __eq__(self, other): cdef Config other_ if not isinstance(other, Config): return False other_ = other return (memcmp((self._ptr), (other_._ptr), sizeof(ncclConfig_t)) == 0) def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): self._ptr = malloc(sizeof(ncclConfig_t)) if self._ptr == NULL: raise MemoryError("Error allocating Config") memcpy(self._ptr, val.ctypes.data, sizeof(ncclConfig_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable else: setattr(self, key, val) @property def size_(self): """int: """ return self._ptr[0].size @size_.setter def size_(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].size = val @property def magic(self): """int: """ return self._ptr[0].magic @magic.setter def magic(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].magic = val @property def version(self): """int: """ return self._ptr[0].version @version.setter def version(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].version = val @property def blocking(self): """int: """ return self._ptr[0].blocking @blocking.setter def blocking(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].blocking = val @property def cga_cluster_size(self): """int: """ return self._ptr[0].cgaClusterSize @cga_cluster_size.setter def cga_cluster_size(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].cgaClusterSize = val @property def min_ctas(self): """int: """ return self._ptr[0].minCTAs @min_ctas.setter def min_ctas(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].minCTAs = val @property def max_ctas(self): """int: """ return self._ptr[0].maxCTAs @max_ctas.setter def max_ctas(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].maxCTAs = val @property def net_name(self): """str: """ cdef char* ptr = self._ptr[0].netName if ptr: return cpython.PyUnicode_FromString(ptr) return "" @net_name.setter def net_name(self, val): if self._readonly: raise ValueError("This Config instance is read-only") cdef bytes buf = val.encode() cdef char *ptr = buf self._refs["net_name"] = buf self._ptr.netName = ptr @property def split_share(self): """int: """ return self._ptr[0].splitShare @split_share.setter def split_share(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].splitShare = val @property def traffic_class(self): """int: """ return self._ptr[0].trafficClass @traffic_class.setter def traffic_class(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].trafficClass = val @property def comm_name(self): """str: """ cdef char* ptr = self._ptr[0].commName if ptr: return cpython.PyUnicode_FromString(ptr) return "" @comm_name.setter def comm_name(self, val): if self._readonly: raise ValueError("This Config instance is read-only") cdef bytes buf = val.encode() cdef char *ptr = buf self._refs["comm_name"] = buf self._ptr.commName = ptr @property def collnet_enable(self): """int: """ return self._ptr[0].collnetEnable @collnet_enable.setter def collnet_enable(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].collnetEnable = val @property def cta_policy(self): """int: """ return self._ptr[0].CTAPolicy @cta_policy.setter def cta_policy(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].CTAPolicy = val @property def shrink_share(self): """int: """ return self._ptr[0].shrinkShare @shrink_share.setter def shrink_share(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].shrinkShare = val @property def nvls_ctas(self): """int: """ return self._ptr[0].nvlsCTAs @nvls_ctas.setter def nvls_ctas(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].nvlsCTAs = val @property def n_channels_per_net_peer(self): """int: """ return self._ptr[0].nChannelsPerNetPeer @n_channels_per_net_peer.setter def n_channels_per_net_peer(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].nChannelsPerNetPeer = val @property def nvlink_centric_sched(self): """int: """ return self._ptr[0].nvlinkCentricSched @nvlink_centric_sched.setter def nvlink_centric_sched(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].nvlinkCentricSched = val @property def graph_usage_mode(self): """int: """ return self._ptr[0].graphUsageMode @graph_usage_mode.setter def graph_usage_mode(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].graphUsageMode = val @property def num_rma_ctx(self): """int: """ return self._ptr[0].numRmaCtx @num_rma_ctx.setter def num_rma_ctx(self, val): if self._readonly: raise ValueError("This Config instance is read-only") self._ptr[0].numRmaCtx = val @staticmethod def from_data(data): """Create an Config instance wrapping the given NumPy array. Args: data (_numpy.ndarray): a single-element array of dtype `config_dtype` holding the data. """ return __from_data(data, "config_dtype", config_dtype, Config) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): """Create an Config instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. owner (object): The Python object that owns the pointer. If not provided, data will be copied. readonly (bool): whether the data is read-only (to the user). default is `False`. """ if ptr == 0: raise ValueError("ptr must not be null (0)") cdef Config obj = Config.__new__(Config) if owner is None: obj._ptr = malloc(sizeof(ncclConfig_t)) if obj._ptr == NULL: raise MemoryError("Error allocating Config") memcpy((obj._ptr), ptr, sizeof(ncclConfig_t)) obj._owner = None obj._owned = True else: obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly obj._refs = {} return obj cdef _get_sim_info_dtype_offsets(): cdef ncclSimInfo_t pod = ncclSimInfo_t() return _numpy.dtype({ 'names': ['size_', 'magic', 'version', 'estimated_time'], 'formats': [_numpy.uint64, _numpy.uint32, _numpy.uint32, _numpy.float32], 'offsets': [ (&(pod.size)) - (&pod), (&(pod.magic)) - (&pod), (&(pod.version)) - (&pod), (&(pod.estimatedTime)) - (&pod), ], 'itemsize': sizeof(ncclSimInfo_t), }) sim_info_dtype = _get_sim_info_dtype_offsets() cdef class SimInfo: """Empty-initialize an instance of `ncclSimInfo_t`. .. seealso:: `ncclSimInfo_t` """ cdef: ncclSimInfo_t *_ptr object _owner bint _owned bint _readonly def __init__(self): self._ptr = calloc(1, sizeof(ncclSimInfo_t)) if self._ptr == NULL: raise MemoryError("Error allocating SimInfo") self._owner = None self._owned = True self._readonly = False def __dealloc__(self): cdef ncclSimInfo_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL free(ptr) def __repr__(self): return f"<{__name__}.SimInfo object at {hex(id(self))}>" @property def ptr(self): """Get the pointer address to the data as Python :class:`int`.""" return (self._ptr) cdef intptr_t _get_ptr(self): return (self._ptr) def __int__(self): return (self._ptr) def __eq__(self, other): cdef SimInfo other_ if not isinstance(other, SimInfo): return False other_ = other return (memcmp((self._ptr), (other_._ptr), sizeof(ncclSimInfo_t)) == 0) def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): self._ptr = malloc(sizeof(ncclSimInfo_t)) if self._ptr == NULL: raise MemoryError("Error allocating SimInfo") memcpy(self._ptr, val.ctypes.data, sizeof(ncclSimInfo_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable else: setattr(self, key, val) @property def size_(self): """int: """ return self._ptr[0].size @size_.setter def size_(self, val): if self._readonly: raise ValueError("This SimInfo instance is read-only") self._ptr[0].size = val @property def magic(self): """int: """ return self._ptr[0].magic @magic.setter def magic(self, val): if self._readonly: raise ValueError("This SimInfo instance is read-only") self._ptr[0].magic = val @property def version(self): """int: """ return self._ptr[0].version @version.setter def version(self, val): if self._readonly: raise ValueError("This SimInfo instance is read-only") self._ptr[0].version = val @property def estimated_time(self): """float: """ return self._ptr[0].estimatedTime @estimated_time.setter def estimated_time(self, val): if self._readonly: raise ValueError("This SimInfo instance is read-only") self._ptr[0].estimatedTime = val @staticmethod def from_data(data): """Create an SimInfo instance wrapping the given NumPy array. Args: data (_numpy.ndarray): a single-element array of dtype `sim_info_dtype` holding the data. """ return __from_data(data, "sim_info_dtype", sim_info_dtype, SimInfo) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): """Create an SimInfo instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. owner (object): The Python object that owns the pointer. If not provided, data will be copied. readonly (bool): whether the data is read-only (to the user). default is `False`. """ if ptr == 0: raise ValueError("ptr must not be null (0)") cdef SimInfo obj = SimInfo.__new__(SimInfo) if owner is None: obj._ptr = malloc(sizeof(ncclSimInfo_t)) if obj._ptr == NULL: raise MemoryError("Error allocating SimInfo") memcpy((obj._ptr), ptr, sizeof(ncclSimInfo_t)) obj._owner = None obj._owned = True else: obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly return obj cdef _get_wait_signal_desc_dtype_offsets(): cdef ncclWaitSignalDesc_t pod = ncclWaitSignalDesc_t() return _numpy.dtype({ 'names': ['op_cnt', 'peer', 'sig_idx', 'ctx'], 'formats': [_numpy.int32, _numpy.int32, _numpy.int32, _numpy.int32], 'offsets': [ (&(pod.opCnt)) - (&pod), (&(pod.peer)) - (&pod), (&(pod.sigIdx)) - (&pod), (&(pod.ctx)) - (&pod), ], 'itemsize': sizeof(ncclWaitSignalDesc_t), }) wait_signal_desc_dtype = _get_wait_signal_desc_dtype_offsets() cdef class WaitSignalDesc: """Empty-initialize an instance of `ncclWaitSignalDesc_t`. .. seealso:: `ncclWaitSignalDesc_t` """ cdef: ncclWaitSignalDesc_t *_ptr object _owner bint _owned bint _readonly def __init__(self): self._ptr = calloc(1, sizeof(ncclWaitSignalDesc_t)) if self._ptr == NULL: raise MemoryError("Error allocating WaitSignalDesc") self._owner = None self._owned = True self._readonly = False def __dealloc__(self): cdef ncclWaitSignalDesc_t *ptr if self._owned and self._ptr != NULL: ptr = self._ptr self._ptr = NULL free(ptr) def __repr__(self): return f"<{__name__}.WaitSignalDesc object at {hex(id(self))}>" @property def ptr(self): """Get the pointer address to the data as Python :class:`int`.""" return (self._ptr) cdef intptr_t _get_ptr(self): return (self._ptr) def __int__(self): return (self._ptr) def __eq__(self, other): cdef WaitSignalDesc other_ if not isinstance(other, WaitSignalDesc): return False other_ = other return (memcmp((self._ptr), (other_._ptr), sizeof(ncclWaitSignalDesc_t)) == 0) def __setitem__(self, key, val): if key == 0 and isinstance(val, _numpy.ndarray): self._ptr = malloc(sizeof(ncclWaitSignalDesc_t)) if self._ptr == NULL: raise MemoryError("Error allocating WaitSignalDesc") memcpy(self._ptr, val.ctypes.data, sizeof(ncclWaitSignalDesc_t)) self._owner = None self._owned = True self._readonly = not val.flags.writeable else: setattr(self, key, val) @property def op_cnt(self): """int: """ return self._ptr[0].opCnt @op_cnt.setter def op_cnt(self, val): if self._readonly: raise ValueError("This WaitSignalDesc instance is read-only") self._ptr[0].opCnt = val @property def peer(self): """int: """ return self._ptr[0].peer @peer.setter def peer(self, val): if self._readonly: raise ValueError("This WaitSignalDesc instance is read-only") self._ptr[0].peer = val @property def sig_idx(self): """int: """ return self._ptr[0].sigIdx @sig_idx.setter def sig_idx(self, val): if self._readonly: raise ValueError("This WaitSignalDesc instance is read-only") self._ptr[0].sigIdx = val @property def ctx(self): """int: """ return self._ptr[0].ctx @ctx.setter def ctx(self, val): if self._readonly: raise ValueError("This WaitSignalDesc instance is read-only") self._ptr[0].ctx = val @staticmethod def from_data(data): """Create an WaitSignalDesc instance wrapping the given NumPy array. Args: data (_numpy.ndarray): a single-element array of dtype `wait_signal_desc_dtype` holding the data. """ return __from_data(data, "wait_signal_desc_dtype", wait_signal_desc_dtype, WaitSignalDesc) @staticmethod def from_ptr(intptr_t ptr, bint readonly=False, object owner=None): """Create an WaitSignalDesc instance wrapping the given pointer. Args: ptr (intptr_t): pointer address as Python :class:`int` to the data. owner (object): The Python object that owns the pointer. If not provided, data will be copied. readonly (bool): whether the data is read-only (to the user). default is `False`. """ if ptr == 0: raise ValueError("ptr must not be null (0)") cdef WaitSignalDesc obj = WaitSignalDesc.__new__(WaitSignalDesc) if owner is None: obj._ptr = malloc(sizeof(ncclWaitSignalDesc_t)) if obj._ptr == NULL: raise MemoryError("Error allocating WaitSignalDesc") memcpy((obj._ptr), ptr, sizeof(ncclWaitSignalDesc_t)) obj._owner = None obj._owned = True else: obj._ptr = ptr obj._owner = owner obj._owned = False obj._readonly = readonly return obj ############################################################################### # Enum ############################################################################### class Result(_IntEnum): """See `ncclResult_t`.""" Success = ncclSuccess UnhandledCudaError = ncclUnhandledCudaError SystemError = ncclSystemError InternalError = ncclInternalError InvalidArgument = ncclInvalidArgument InvalidUsage = ncclInvalidUsage RemoteError = ncclRemoteError InProgress = ncclInProgress NumResults = ncclNumResults class CommMemStat(_IntEnum): """See `ncclCommMemStat_t`.""" StatGpuMemSuspend = ncclStatGpuMemSuspend StatGpuMemSuspended = ncclStatGpuMemSuspended StatGpuMemPersist = ncclStatGpuMemPersist StatGpuMemTotal = ncclStatGpuMemTotal class RedOp_dummy(_IntEnum): """See `ncclRedOp_dummy_t`.""" NumOps_dummy = ncclNumOps_dummy class RedOp(_IntEnum): """See `ncclRedOp_t`.""" Sum = ncclSum Prod = ncclProd Max = ncclMax Min = ncclMin Avg = ncclAvg NumOps = ncclNumOps MaxRedOp = ncclMaxRedOp class DataType(_IntEnum): """See `ncclDataType_t`.""" Int8 = ncclInt8 Char = ncclChar Uint8 = ncclUint8 Int32 = ncclInt32 Int = ncclInt Uint32 = ncclUint32 Int64 = ncclInt64 Uint64 = ncclUint64 Float16 = ncclFloat16 Half = ncclHalf Float32 = ncclFloat32 Float = ncclFloat Float64 = ncclFloat64 Double = ncclDouble Bfloat16 = ncclBfloat16 Float8e4m3 = ncclFloat8e4m3 Float8e5m2 = ncclFloat8e5m2 NumTypes = ncclNumTypes class ScalarResidence(_IntEnum): """See `ncclScalarResidence_t`.""" Device = ncclScalarDevice HostImmediate = ncclScalarHostImmediate ############################################################################### # Error handling ############################################################################### class NCCLError(Exception): def __init__(self, status): self.status = status s = Result(status) cdef str err = f"{s.name} ({s.value}): {get_error_string(status)}" super(NCCLError, self).__init__(err) def __reduce__(self): return (type(self), (self.status,)) @cython.profile(False) cpdef inline check_status(int status): if status != Result.Success and status != Result.InProgress: raise NCCLError(status) ############################################################################### # Wrapper functions ############################################################################### cpdef intptr_t mem_alloc(size_t size) except? 0: cdef void* ptr with nogil: __status__ = ncclMemAlloc(&ptr, size) check_status(__status__) return ptr cpdef mem_free(intptr_t ptr): with nogil: __status__ = ncclMemFree(ptr) check_status(__status__) cpdef int get_version() except? -1: cdef int version with nogil: __status__ = ncclGetVersion(&version) check_status(__status__) return version cpdef get_unique_id(intptr_t unique_id): with nogil: __status__ = ncclGetUniqueId(unique_id) check_status(__status__) cpdef intptr_t comm_init_rank_config(int nranks, comm_id, int rank, intptr_t config) except? 0: cdef void* _comm_id_ = get_buffer_pointer(comm_id, -1, readonly=False) cdef Comm comm with nogil: __status__ = ncclCommInitRankConfig(&comm, nranks, ((_comm_id_))[0], rank, config) check_status(__status__) return comm cpdef intptr_t comm_init_rank(int nranks, comm_id, int rank) except? 0: cdef void* _comm_id_ = get_buffer_pointer(comm_id, -1, readonly=False) cdef Comm comm with nogil: __status__ = ncclCommInitRank(&comm, nranks, ((_comm_id_))[0], rank) check_status(__status__) return comm cpdef comm_init_all(intptr_t comm, int ndev, devlist): cdef nullable_unique_ptr[ vector[int] ] _devlist_ get_resource_ptr[int](_devlist_, devlist, NULL) with nogil: __status__ = ncclCommInitAll(comm, ndev, (_devlist_.data())) check_status(__status__) cpdef comm_finalize(intptr_t comm): with nogil: __status__ = ncclCommFinalize(comm) check_status(__status__) cpdef comm_destroy(intptr_t comm): with nogil: __status__ = ncclCommDestroy(comm) check_status(__status__) cpdef comm_abort(intptr_t comm): with nogil: __status__ = ncclCommAbort(comm) check_status(__status__) cpdef intptr_t comm_split(intptr_t comm, int color, int key, intptr_t config) except? 0: cdef Comm newcomm with nogil: __status__ = ncclCommSplit(comm, color, key, &newcomm, config) check_status(__status__) return newcomm cpdef intptr_t comm_shrink(intptr_t comm, exclude_ranks_list, int exclude_ranks_count, intptr_t config, int shrink_flags) except? 0: cdef nullable_unique_ptr[ vector[int] ] _exclude_ranks_list_ get_resource_ptr[int](_exclude_ranks_list_, exclude_ranks_list, NULL) cdef Comm newcomm with nogil: __status__ = ncclCommShrink(comm, (_exclude_ranks_list_.data()), exclude_ranks_count, &newcomm, config, shrink_flags) check_status(__status__) return newcomm cpdef intptr_t comm_init_rank_scalable(int nranks, int myrank, int n_id, comm_ids, intptr_t config) except? 0: cdef void* _comm_ids_ = get_buffer_pointer(comm_ids, -1, readonly=False) cdef Comm newcomm with nogil: __status__ = ncclCommInitRankScalable(&newcomm, nranks, myrank, n_id, _comm_ids_, config) check_status(__status__) return newcomm cpdef str get_error_string(int result): cdef bytes _output_ _output_ = ncclGetErrorString(<_Result>result) return _output_.decode() cpdef str get_last_error(intptr_t comm): cdef bytes _output_ _output_ = ncclGetLastError(comm) return _output_.decode() cpdef int comm_get_async_error(intptr_t comm) except? -1: cdef _Result async_error with nogil: __status__ = ncclCommGetAsyncError(comm, &async_error) check_status(__status__) return async_error cpdef int comm_count(intptr_t comm) except? -1: cdef int count with nogil: __status__ = ncclCommCount(comm, &count) check_status(__status__) return count cpdef int comm_cu_device(intptr_t comm) except? -1: cdef int device with nogil: __status__ = ncclCommCuDevice(comm, &device) check_status(__status__) return device cpdef int comm_user_rank(intptr_t comm) except? -1: cdef int rank with nogil: __status__ = ncclCommUserRank(comm, &rank) check_status(__status__) return rank cpdef intptr_t comm_register(intptr_t comm, intptr_t buff, size_t size) except? 0: cdef void* handle with nogil: __status__ = ncclCommRegister(comm, buff, size, &handle) check_status(__status__) return handle cpdef comm_deregister(intptr_t comm, intptr_t handle): with nogil: __status__ = ncclCommDeregister(comm, handle) check_status(__status__) cpdef intptr_t comm_window_register(intptr_t comm, intptr_t buff, size_t size, int win_flags) except? 0: cdef Window win with nogil: __status__ = ncclCommWindowRegister(comm, buff, size, &win, win_flags) check_status(__status__) return win cpdef comm_window_deregister(intptr_t comm, intptr_t win): with nogil: __status__ = ncclCommWindowDeregister(comm, win) check_status(__status__) cpdef int red_op_create_pre_mul_sum(intptr_t scalar, int datatype, int residence, intptr_t comm) except? -1: cdef _RedOp op with nogil: __status__ = ncclRedOpCreatePreMulSum(&op, scalar, <_DataType>datatype, <_ScalarResidence>residence, comm) check_status(__status__) return op cpdef red_op_destroy(int op, intptr_t comm): with nogil: __status__ = ncclRedOpDestroy(<_RedOp>op, comm) check_status(__status__) cpdef reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, int root, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclReduce(sendbuff, recvbuff, count, <_DataType>datatype, <_RedOp>op, root, comm, stream) check_status(__status__) cpdef bcast(intptr_t buff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclBcast(buff, count, <_DataType>datatype, root, comm, stream) check_status(__status__) cpdef broadcast(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclBroadcast(sendbuff, recvbuff, count, <_DataType>datatype, root, comm, stream) check_status(__status__) cpdef all_reduce(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int op, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclAllReduce(sendbuff, recvbuff, count, <_DataType>datatype, <_RedOp>op, comm, stream) check_status(__status__) cpdef reduce_scatter(intptr_t sendbuff, intptr_t recvbuff, size_t recvcount, int datatype, int op, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclReduceScatter(sendbuff, recvbuff, recvcount, <_DataType>datatype, <_RedOp>op, comm, stream) check_status(__status__) cpdef all_gather(intptr_t sendbuff, intptr_t recvbuff, size_t sendcount, int datatype, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclAllGather(sendbuff, recvbuff, sendcount, <_DataType>datatype, comm, stream) check_status(__status__) cpdef allto_all(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclAlltoAll(sendbuff, recvbuff, count, <_DataType>datatype, comm, stream) check_status(__status__) cpdef gather(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclGather(sendbuff, recvbuff, count, <_DataType>datatype, root, comm, stream) check_status(__status__) cpdef scatter(intptr_t sendbuff, intptr_t recvbuff, size_t count, int datatype, int root, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclScatter(sendbuff, recvbuff, count, <_DataType>datatype, root, comm, stream) check_status(__status__) cpdef send(intptr_t sendbuff, size_t count, int datatype, int peer, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclSend(sendbuff, count, <_DataType>datatype, peer, comm, stream) check_status(__status__) cpdef recv(intptr_t recvbuff, size_t count, int datatype, int peer, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclRecv(recvbuff, count, <_DataType>datatype, peer, comm, stream) check_status(__status__) cpdef signal(int peer, int sig_idx, int ctx, unsigned int flags, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclSignal(peer, sig_idx, ctx, flags, comm, stream) check_status(__status__) cpdef wait_signal(int n_desc, intptr_t signal_descs, intptr_t comm, intptr_t stream): with nogil: __status__ = ncclWaitSignal(n_desc, signal_descs, comm, stream) check_status(__status__) cpdef group_start(): with nogil: __status__ = ncclGroupStart() check_status(__status__) cpdef group_end(): with nogil: __status__ = ncclGroupEnd() check_status(__status__) cpdef group_simulate_end(intptr_t sim_info): with nogil: __status__ = ncclGroupSimulateEnd(sim_info) check_status(__status__) nccl-2.29.7-1/bindings/nccl4py/nccl/core/000077500000000000000000000000001515037102200177045ustar00rootroot00000000000000nccl-2.29.7-1/bindings/nccl4py/nccl/core/__init__.py000066400000000000000000000047731515037102200220300ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ NCCL4Py Core API: Pythonic access to NCCL for multi-GPU communication. This module provides the main public API for NCCL operations. """ # Core types and enums from nccl.core.typing import * # Constants from nccl.core.constants import * # Communicator and configuration from nccl.core.communicator import * # Resource management from nccl.core.resources import * # Group operations from nccl.core.group import * # Utilities from nccl.core.utils import * # Memory management from nccl.core.buffer import * # The following __all__ exports define the stable, public API surface of NCCL4Py. # Semantic versioning guarantees apply only to the symbols explicitly listed below. # All other modules, functions, and symbols are internal implementation details and are subject to change without notice. __all__ = [ # Types and specs "NcclDataType", "NcclRedOp", "NcclBufferSpec", "NcclScalarSpec", "NcclDeviceSpec", "NcclStreamSpec", # Exceptions "NcclInvalid", # Data type constants "INT8", "CHAR", "UINT8", "INT32", "INT", "UINT32", "INT64", "UINT64", "FLOAT16", "HALF", "FLOAT32", "FLOAT", "FLOAT64", "DOUBLE", "BFLOAT16", "FLOAT8E4M3", "FLOAT8E5M2", # Reduction op constants "SUM", "PROD", "MAX", "MIN", "AVG", # Constants and enums "NCCL_SPLIT_NOCOLOR", "CTAPolicy", "CommShrinkFlag", "WindowFlag", # Communicator "Communicator", "NCCLConfig", "WaitSignalDesc", # Resources "RegisteredBufferHandle", "RegisteredWindowHandle", "CustomRedOp", # Group "group", "group_start", "group_end", "group_simulate_end", "GroupSimInfo", # Utilities "Version", "get_version", "UniqueId", "get_unique_id", "get_error_string", # Memory "mem_alloc", "mem_free", # Interop modules (lazy-loaded) "cupy", "torch", ] def __getattr__(name): """Lazy-load interop submodules on first access to avoid importing cupy/torch unless needed.""" if name == "cupy": import nccl.core.interop.cupy return nccl.core.interop.cupy elif name == "torch": import nccl.core.interop.torch return nccl.core.interop.torch raise AttributeError(f"module {__name__!r} has no attribute {name!r}") nccl-2.29.7-1/bindings/nccl4py/nccl/core/buffer.py000066400000000000000000000141401515037102200215270ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ Buffer specification and memory allocation for NCCL operations. This module provides utilities for allocating NCCL-optimized device memory and resolving various buffer specifications (arrays, tensors, tuples) into the raw pointers, counts, and dtypes required by NCCL operations. """ from __future__ import annotations import math from cuda.core import Buffer from cuda.core.utils import StridedMemoryView, args_viewable_as_strided_memory from nccl.core.cuda import get_device_id, get_stream_ptr from nccl.core.memory import get_memory_resource from nccl.core.typing import ( NcclBufferSpec, NcclDataType, NcclDeviceSpec, NcclInvalid, NcclStreamSpec, ) __all__ = ["mem_alloc", "mem_free", "NcclBuffer"] def mem_alloc(size: int, device: NcclDeviceSpec | None = None) -> Buffer: """ Allocates GPU buffer memory using NCCL's memory allocator. The actual allocated size may be larger than requested due to buffer granularity requirements from NCCL optimizations. Args: - size (int): Number of bytes to allocate. - device (NcclDeviceSpec, optional): Target CUDA device. Defaults to current device. Returns: ``Buffer``: A CUDA buffer object backed by NCCL-managed memory. Notes: - The returned buffer size may exceed the requested size due to alignment requirements. - Buffer is allocated on the specified device, if provided; current device is restored after allocation. Memory Lifecycle: Memory can be explicitly freed using ``mem_free(buf)`` or automatically freed when the Buffer object is garbage collected. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclmemalloc """ device_id = get_device_id(device) mr = get_memory_resource(device_id) return mr.allocate(size) def mem_free(buf: Buffer) -> None: """ Frees memory allocated by ``mem_alloc()``. Args: - buf (Buffer): The buffer to free. Notes: Explicit deallocation is optional. Memory is automatically freed when the Buffer object is garbage collected. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclmemfree """ buf.close() @args_viewable_as_strided_memory((0,)) def _resolve_buffer(buffer, stream_ptr: int | None) -> StridedMemoryView: return buffer.view(stream_ptr) class NcclBuffer: """ Resolves user-provided buffer specifications to raw pointer, count, dtype, and device. This class handles various buffer input formats and extracts the necessary metadata for NCCL operations. Buffer specifications can be: - An object supporting DLPack or CUDA Array Interface - A ``Buffer`` object - A tuple: ``(buffer, dtype)`` Stream Handling: When stream is None, the special sentinel value -1 is used internally. This sentinel value (-1) is required by cuda.core's StridedMemoryView.view() to indicate that no stream synchronization should be performed between the producer stream (where the buffer was created) and the consumer stream (where NCCL operations will use it). This avoids implicit synchronization overhead when the user is managing stream dependencies manually. When a stream is provided, its handle (typically 0 for default stream or a valid stream pointer) is used for proper synchronization. Notes: - Element count is automatically derived from buffer size and dtype. Use slicing to control element count. Attributes: ptr (int): Raw device pointer address. count (int): Number of elements in the buffer. dtype (NcclDataType): Data type of buffer elements. device_id (int): CUDA device ID where buffer resides. """ def __init__(self, buffer: NcclBufferSpec, stream: NcclStreamSpec | None = None) -> None: """ Initializes an NcclBuffer from a buffer specification. Args: - buffer (NcclBufferSpec): Buffer specification to resolve. - stream (NcclStreamSpec, optional): CUDA stream for synchronization. Defaults to None. Raises: - ``NcclInvalid``: If the buffer specification is invalid or malformed. """ self.stream_ptr = -1 if stream is None else get_stream_ptr(stream) resolved = None try: from nccl.core.interop.torch import resolve_tensor resolved = resolve_tensor(buffer) except (ImportError, ModuleNotFoundError, NcclInvalid): pass try: from nccl.core.interop.cupy import resolve_array resolved = resolve_array(buffer) except (ImportError, ModuleNotFoundError, NcclInvalid): pass if resolved is not None: self._ptr = resolved[0] self._count = resolved[1] self._dtype = resolved[2] self._device_id = resolved[3] else: view = _resolve_buffer(buffer, self.stream_ptr) self._ptr = view.ptr self._count = math.prod(view.shape) self._dtype = NcclDataType(view.dtype) self._device_id = view.device_id @property def ptr(self) -> int: """ Raw device pointer address. Returns: ``int``: Device memory pointer. """ return self._ptr @property def count(self) -> int: """ Number of elements in the buffer. Returns: ``int``: Element count. """ return self._count @property def dtype(self) -> NcclDataType: """ Data type of buffer elements. Returns: ``NcclDataType``: NCCL data type. """ return self._dtype @property def device_id(self) -> int: """ CUDA device ID where buffer resides. Returns: ``int``: Device ID. """ return self._device_id nccl-2.29.7-1/bindings/nccl4py/nccl/core/communicator.py000066400000000000000000002152711515037102200227660ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ NCCL communicator creation, management, and operations. This module provides the core Communicator class and NCCLConfig for NCCL operations. Communicators manage groups of ranks for collective and point-to-point communication, with support for buffer registration, custom reduction operators, and resource management. """ from __future__ import annotations from typing import Sequence, Any import numpy as _np from cuda.core import Device from nccl import bindings as _nccl_bindings from nccl.core.buffer import NcclBuffer from nccl.core.constants import ( NCCL_SPLIT_NOCOLOR, NCCL_UNDEF_INT, CTAPolicy, CommShrinkFlag, WindowFlag, ) from nccl.core.cuda import get_stream_ptr, get_cuda_device from nccl.core.resources import ( CommResource, RegisteredBufferHandle, RegisteredWindowHandle, CustomRedOp, ) from nccl.core.typing import ( NcclDataType, NcclBufferSpec, NcclRedOp, NcclStreamSpec, NcclScalarSpec, NcclInvalid, ) from nccl.core.utils import UniqueId __all__ = [ "NCCLConfig", "WaitSignalDesc", "Communicator", ] class NCCLConfig: """ NCCL configuration for communicator initialization. This class provides configuration options for NCCL communicators, allowing fine-tuning of performance and behavior characteristics. Based on the official NCCL API documentation. """ def __init__( self, *, blocking: bool | None = None, cga_cluster_size: int | None = None, min_ctas: int | None = None, max_ctas: int | None = None, net_name: str | None = None, split_share: bool | None = None, traffic_class: int | None = None, comm_name: str | None = None, collnet_enable: bool | None = None, cta_policy: CTAPolicy | None = None, shrink_share: bool | None = None, nvls_ctas: int | None = None, n_channels_per_net_peer: int | None = None, nvlink_centric_sched: bool | None = None, graph_usage_mode: int | None = None, num_rma_ctx: int | None = None, ) -> None: """ Initializes NCCL configuration with custom parameters. All parameters are optional and default to NCCL's internal defaults (undefined). Args: - blocking (bool, optional): Blocking (True) or non-blocking (False) communicator behavior. Defaults to True. - cga_cluster_size (int, optional): Cooperative Group Array (CGA) size for kernels (0-8). Defaults to 4 for sm90+, 0 otherwise. - min_ctas (int, optional): Minimal number of CTAs per kernel. Positive integer up to 32. Defaults to 1. - max_ctas (int, optional): Maximal number of CTAs per kernel. Positive integer up to 32. Defaults to 32. - net_name (str, optional): Network module name (e.g., "IB", "Socket"). Case-insensitive. Defaults to NCCL auto-selection. - split_share (bool, optional): Share resources with child communicator during split. Defaults to False. - traffic_class (int, optional): Traffic class (TC) for network operations (>= 0). Network-specific meaning. - comm_name (str, optional): User-defined communicator name for logging and profiling. - collnet_enable (bool, optional): Enable (True) or disable (False) IB SHARP. Defaults to False. - cta_policy (CTAPolicy, optional): CTA scheduling policy. See CTAPolicy enum (Default, Efficiency, Zero). Defaults to CTAPolicy.Default. - shrink_share (bool, optional): Share resources with child communicator during shrink. Defaults to False. - nvls_ctas (int, optional): Total number of CTAs for NVLS kernels. Positive integer. Defaults to NCCL auto-determined value. - n_channels_per_net_peer (int, optional): Number of network channels for pairwise communication. Positive integer, rounded up to power of 2. Defaults to AlltoAll-optimized value. - nvlink_centric_sched (bool, optional): Enable (True) NVLink-centric scheduling. Defaults to False. - graph_usage_mode (int, optional): Graph usage mode (NCCL 2.29+). Supported values: 0 (no graphs), 1 (one graph), 2 (multiple graphs or mix of graph and non-graph). Defaults to 2. - num_rma_ctx (int, optional): Number of RMA contexts (NCCL 2.29+). Defaults to 1. Notes: Aborting any communicator may affect others in the same family when split_share or shrink_share is enabled. """ self._cfg: _nccl_bindings.Config = _nccl_bindings.Config() # Apply NCCL_CONFIG_INITIALIZER defaults self._cfg.size_ = int(_nccl_bindings.config_dtype.itemsize) self._cfg.magic = 0xCAFEBEEF # NCCL protocol magic number for ncclConfig_t validation self._cfg.version = _nccl_bindings.get_version() # Initialize all fields to undef self._cfg.blocking = NCCL_UNDEF_INT self._cfg.cga_cluster_size = NCCL_UNDEF_INT self._cfg.min_ctas = NCCL_UNDEF_INT self._cfg.max_ctas = NCCL_UNDEF_INT self._cfg.split_share = NCCL_UNDEF_INT self._cfg.traffic_class = NCCL_UNDEF_INT self._cfg.collnet_enable = NCCL_UNDEF_INT self._cfg.cta_policy = NCCL_UNDEF_INT self._cfg.shrink_share = NCCL_UNDEF_INT self._cfg.nvls_ctas = NCCL_UNDEF_INT self._cfg.n_channels_per_net_peer = NCCL_UNDEF_INT self._cfg.nvlink_centric_sched = NCCL_UNDEF_INT # NCCL 2.29 self._cfg.graph_usage_mode = NCCL_UNDEF_INT self._cfg.num_rma_ctx = NCCL_UNDEF_INT # Use setters for validation - they handle type checking and range validation if blocking is not None: self.blocking = blocking if cga_cluster_size is not None: self.cga_cluster_size = cga_cluster_size if min_ctas is not None: self.min_ctas = min_ctas if max_ctas is not None: self.max_ctas = max_ctas if net_name is not None: self.net_name = net_name if split_share is not None: self.split_share = split_share if traffic_class is not None: self.traffic_class = traffic_class if comm_name is not None: self.comm_name = comm_name if collnet_enable is not None: self.collnet_enable = collnet_enable if cta_policy is not None: self.cta_policy = cta_policy if shrink_share is not None: self.shrink_share = shrink_share if nvls_ctas is not None: self.nvls_ctas = nvls_ctas if n_channels_per_net_peer is not None: self.n_channels_per_net_peer = n_channels_per_net_peer if nvlink_centric_sched is not None: self.nvlink_centric_sched = nvlink_centric_sched if graph_usage_mode is not None: self.graph_usage_mode = graph_usage_mode if num_rma_ctx is not None: self.num_rma_ctx = num_rma_ctx def __repr__(self) -> str: """ Returns string representation showing non-default values. Returns: ``str``: String showing configured (non-default) values. """ parts = [] # Check each field and include if not undefined if self._cfg.blocking != NCCL_UNDEF_INT: parts.append(f"blocking={bool(self._cfg.blocking)}") if self._cfg.cga_cluster_size != NCCL_UNDEF_INT: parts.append(f"cga_cluster_size={self._cfg.cga_cluster_size}") if self._cfg.min_ctas != NCCL_UNDEF_INT: parts.append(f"min_ctas={self._cfg.min_ctas}") if self._cfg.max_ctas != NCCL_UNDEF_INT: parts.append(f"max_ctas={self._cfg.max_ctas}") if hasattr(self._cfg, "net_name") and self._cfg.net_name: parts.append(f"net_name='{self._cfg.net_name}'") if self._cfg.split_share != NCCL_UNDEF_INT: parts.append(f"split_share={bool(self._cfg.split_share)}") if self._cfg.traffic_class != NCCL_UNDEF_INT: parts.append(f"traffic_class={self._cfg.traffic_class}") if hasattr(self._cfg, "comm_name") and self._cfg.comm_name: parts.append(f"comm_name='{self._cfg.comm_name}'") if self._cfg.collnet_enable != NCCL_UNDEF_INT: parts.append(f"collnet_enable={bool(self._cfg.collnet_enable)}") if self._cfg.cta_policy != NCCL_UNDEF_INT: parts.append(f"cta_policy={CTAPolicy(self._cfg.cta_policy).name}") if self._cfg.shrink_share != NCCL_UNDEF_INT: parts.append(f"shrink_share={bool(self._cfg.shrink_share)}") if self._cfg.nvls_ctas != NCCL_UNDEF_INT: parts.append(f"nvls_ctas={self._cfg.nvls_ctas}") if self._cfg.n_channels_per_net_peer != NCCL_UNDEF_INT: parts.append(f"n_channels_per_net_peer={self._cfg.n_channels_per_net_peer}") if self._cfg.nvlink_centric_sched != NCCL_UNDEF_INT: parts.append(f"nvlink_centric_sched={bool(self._cfg.nvlink_centric_sched)}") if self._cfg.graph_usage_mode != NCCL_UNDEF_INT: parts.append(f"graph_usage_mode={self._cfg.graph_usage_mode}") if self._cfg.num_rma_ctx != NCCL_UNDEF_INT: parts.append(f"num_rma_ctx={self._cfg.num_rma_ctx}") if parts: return f"" else: return "" @property def ptr(self) -> int: """ Raw NCCL config pointer. Returns: ``int``: The configuration pointer. """ return int(self._cfg.ptr) # Field proxies @property def blocking(self) -> bool: """ Blocking or non-blocking communicator behavior. Returns: ``bool``: True for blocking, False for non-blocking. Default: True. """ return bool(self._cfg.blocking) @blocking.setter def blocking(self, val: bool) -> None: if not isinstance(val, bool): raise NcclInvalid(f"blocking must be bool, got {type(val).__name__}") self._cfg.blocking = int(val) @property def cga_cluster_size(self) -> int: """ Cooperative Group Array size (0-8). Returns: ``int``: CGA cluster size. Default: 4 for sm90+, 0 for older architectures. """ return self._cfg.cga_cluster_size @cga_cluster_size.setter def cga_cluster_size(self, val: int) -> None: if not isinstance(val, int): raise NcclInvalid(f"cga_cluster_size must be int, got {type(val).__name__}") if not (0 <= val <= 8): raise NcclInvalid(f"cga_cluster_size must be between 0 and 8, got {val}") self._cfg.cga_cluster_size = val @property def min_ctas(self) -> int: """ Minimum number of CTAs. Returns: ``int``: Positive integer up to 32. Default: 1. """ return self._cfg.min_ctas @min_ctas.setter def min_ctas(self, val: int) -> None: if not isinstance(val, int): raise NcclInvalid(f"min_ctas must be int, got {type(val).__name__}") if not (0 < val <= 32): raise NcclInvalid(f"min_ctas must be a positive integer up to 32, got {val}") self._cfg.min_ctas = val @property def max_ctas(self) -> int: """ Maximum number of CTAs. Returns: ``int``: Positive integer up to 32. Default: 32. """ return self._cfg.max_ctas @max_ctas.setter def max_ctas(self, val: int) -> None: if not isinstance(val, int): raise NcclInvalid(f"max_ctas must be int, got {type(val).__name__}") if not (0 < val <= 32): raise NcclInvalid(f"max_ctas must be a positive integer up to 32, got {val}") self._cfg.max_ctas = val @property def net_name(self) -> str: """ Network module name. Returns: ``str``: Network module (e.g., 'IB', 'Socket'). Case-insensitive. Default: auto-selected. """ return self._cfg.net_name @net_name.setter def net_name(self, val: str) -> None: if not isinstance(val, str): raise NcclInvalid(f"net_name must be str, got {type(val).__name__}") self._cfg.net_name = val @property def split_share(self) -> bool: """ Share resources with child communicator during split. Returns: ``bool``: True to share resources. Default: False. """ return bool(self._cfg.split_share) @split_share.setter def split_share(self, val: bool) -> None: if not isinstance(val, bool): raise NcclInvalid(f"split_share must be bool, got {type(val).__name__}") self._cfg.split_share = int(val) @property def traffic_class(self) -> int: """ Traffic class for network operations. Returns: ``int``: Traffic class (>= 0). Meaning is network-specific. """ return self._cfg.traffic_class @traffic_class.setter def traffic_class(self, val: int) -> None: if not isinstance(val, int): raise NcclInvalid(f"traffic_class must be int, got {type(val).__name__}") if val < 0: raise NcclInvalid(f"traffic_class must be >= 0, got {val}") self._cfg.traffic_class = val @property def comm_name(self) -> str: """ User-defined communicator name for logging and profiling. Returns: ``str``: Communicator name. """ return self._cfg.comm_name @comm_name.setter def comm_name(self, val: str) -> None: if not isinstance(val, str): raise NcclInvalid(f"comm_name must be str, got {type(val).__name__}") self._cfg.comm_name = val @property def collnet_enable(self) -> bool: """ Enable IB SHARP. Returns: ``bool``: True to enable. Default: False. """ return bool(self._cfg.collnet_enable) @collnet_enable.setter def collnet_enable(self, val: bool) -> None: if not isinstance(val, bool): raise NcclInvalid(f"collnet_enable must be bool, got {type(val).__name__}") self._cfg.collnet_enable = int(val) @property def cta_policy(self) -> CTAPolicy: """ CTA scheduling policy. Returns: ``CTAPolicy``: CTA policy enum. Default: CTAPolicy.Default. """ return CTAPolicy(self._cfg.cta_policy) @cta_policy.setter def cta_policy(self, val: int | CTAPolicy) -> None: if not isinstance(val, (int, CTAPolicy)): raise NcclInvalid(f"cta_policy must be int or CTAPolicy, got {type(val).__name__}") self._cfg.cta_policy = int(val) @property def shrink_share(self) -> bool: """ Share resources with child communicator during shrink. Returns: ``bool``: True to share resources. Default: False. """ return bool(self._cfg.shrink_share) @shrink_share.setter def shrink_share(self, val: bool) -> None: if not isinstance(val, bool): raise NcclInvalid(f"shrink_share must be bool, got {type(val).__name__}") self._cfg.shrink_share = int(val) @property def nvls_ctas(self) -> int: """ Total number of CTAs for NVLS kernels. Returns: ``int``: Positive integer. Auto-determined by default. """ return self._cfg.nvls_ctas @nvls_ctas.setter def nvls_ctas(self, val: int) -> None: if not isinstance(val, int): raise NcclInvalid(f"nvls_ctas must be int, got {type(val).__name__}") if val <= 0: raise NcclInvalid(f"nvls_ctas must be a positive integer, got {val}") self._cfg.nvls_ctas = val @property def n_channels_per_net_peer(self) -> int: """ Number of network channels for pairwise communication. Returns: ``int``: Positive integer, rounded up to power of 2. """ return self._cfg.n_channels_per_net_peer @n_channels_per_net_peer.setter def n_channels_per_net_peer(self, val: int) -> None: if not isinstance(val, int): raise NcclInvalid(f"n_channels_per_net_peer must be int, got {type(val).__name__}") if val <= 0: raise NcclInvalid(f"n_channels_per_net_peer must be a positive integer, got {val}") self._cfg.n_channels_per_net_peer = val @property def nvlink_centric_sched(self) -> bool: """ Enable NVLink-centric scheduling. Returns: ``bool``: True to enable. Default: False. """ return bool(self._cfg.nvlink_centric_sched) @nvlink_centric_sched.setter def nvlink_centric_sched(self, val: bool) -> None: if not isinstance(val, bool): raise NcclInvalid(f"nvlink_centric_sched must be bool, got {type(val).__name__}") self._cfg.nvlink_centric_sched = int(val) @property def graph_usage_mode(self) -> int: """ Graph usage mode. Returns: ``int``: Graph usage mode value. """ return int(self._cfg.graph_usage_mode) @graph_usage_mode.setter def graph_usage_mode(self, val: int) -> None: if not isinstance(val, int): raise NcclInvalid(f"graph_usage_mode must be int, got {type(val).__name__}") if val not in (0, 1, 2): raise NcclInvalid(f"graph_usage_mode must be one of 0, 1, 2; got {val}") self._cfg.graph_usage_mode = int(val) @property def num_rma_ctx(self) -> int: """ Number of RMA contexts. Returns: ``int``: Number of RMA contexts. """ return int(self._cfg.num_rma_ctx) @num_rma_ctx.setter def num_rma_ctx(self, val: int) -> None: if not isinstance(val, int): raise NcclInvalid(f"num_rma_ctx must be int, got {type(val).__name__}") if val <= 0: raise NcclInvalid(f"num_rma_ctx must be > 0, got {val}") self._cfg.num_rma_ctx = int(val) class WaitSignalDesc: """ Descriptor for wait signal operations in NCCL. This class describes a signal wait operation for use with :meth:`Communicator.wait_signal`. Each descriptor specifies which peer to wait for, how many signal operations to wait for, and additional context for the wait operation. Attributes: op_cnt (int): Number of signal operations to wait for from the peer. peer (int): Target peer rank to wait for signals from. sig_idx (int): Signal index identifier. Currently must be 0. ctx (int): Context identifier. Currently must be 0. Example: >>> desc = WaitSignalDesc(op_cnt=1, peer=0, sig_idx=0, ctx=0) >>> comm.wait_signal([desc], stream=stream) See Also: :meth:`Communicator.wait_signal`: The method that uses these descriptors. """ def __init__(self, op_cnt: int, peer: int, sig_idx: int, ctx: int) -> None: """ Initializes a wait signal descriptor. Args: op_cnt (int): Number of signal operations to wait for. Must be positive. peer (int): Target peer rank to wait for signals from. sig_idx (int): Signal index identifier. Currently must be 0. ctx (int): Context identifier. Currently must be 0. """ self.op_cnt = int(op_cnt) self.peer = int(peer) self.sig_idx = int(sig_idx) self.ctx = int(ctx) class Communicator: """ NCCL Communicator for collective and point-to-point operations. A Communicator represents a group of ranks that can perform collective operations (like reduce, broadcast) and point-to-point operations (send/recv). Each rank in the communicator has a unique ID (0 to nranks-1). Attributes: ptr (int): Raw NCCL communicator pointer (0 if destroyed) nranks (int): Total number of ranks in this communicator device (Device): CUDA device object associated with this communicator rank (int): This rank's ID within the communicator """ def __init__(self, ptr: int) -> None: """ Initializes communicator with a raw NCCL pointer. Args: - ptr (int): Integer representing NCCL communicator pointer (0 for sentinel/invalid). Raises: - ``NcclInvalid``: If ptr is not an integer. Notes: Unlike the class method ``init()``, this constructor allows ptr=0 for creating sentinel communicators (e.g., when ``split()`` excludes a rank). """ self._comm: int = int(ptr) self._resources: list[CommResource] = [] self._nranks = int(_nccl_bindings.comm_count(self._comm)) if ptr != 0 else None self._device = Device(int(_nccl_bindings.comm_cu_device(self._comm))) if ptr != 0 else None self._rank = int(_nccl_bindings.comm_user_rank(self._comm)) if ptr != 0 else None def _check_valid(self, operation: str) -> None: """ Checks if communicator is valid for the given operation. Args: - operation (str): Name of the operation being performed. Raises: - ``NcclInvalid``: If communicator is not initialized (ptr == 0). """ if self._comm == 0: raise NcclInvalid(f"Cannot {operation}: Communicator not initialized") def _validate_buffer_device(self, buffer: NcclBuffer, buffer_name: str = "buffer") -> None: """ Validates that buffer is on the same device as the communicator. Args: - buffer (NcclBuffer): Resolved buffer to validate. - buffer_name (str): Name of the buffer for error messages. Raises: - ``NcclInvalid``: If buffer device does not match communicator device. """ if buffer.device_id != self.device.device_id: raise NcclInvalid( f"{buffer_name} is on device {buffer.device_id}, but communicator " f"is on device {self.device.device_id}. Buffers must be on the same " f"device as the communicator." ) def __repr__(self) -> str: """ Returns string representation of the communicator. Returns: ``str``: String showing rank/count/device info if valid, or invalid status. """ if self._comm == 0: return "" try: return f"" except RuntimeError: # If we can't get properties, just show the pointer return f"" @classmethod def init( cls, nranks: int, rank: int, unique_id: UniqueId | Sequence[UniqueId], config: NCCLConfig | None = None, ) -> Communicator: """ Initializes a new NCCL communicator. Creates a communicator that connects multiple ranks. All ranks must call this method with the same nranks and unique_id, but with different rank values. Args: - nranks (int): Total number of ranks in the communicator. - rank (int): This rank (must be between 0 and nranks-1). - unique_id (UniqueId | Sequence[UniqueId]): Unique identifier(s) shared by all ranks. - config (NCCLConfig, optional): NCCL configuration options. Defaults to None. Returns: ``Communicator``: A new communicator instance. Raises: - ``NcclInvalid``: If unique_id has an invalid type. Notes: - This is a collective operation. All ranks must call this method. - See [ncclCommInitRankScalable](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcomminitrankscalable) for when multiple unique_ids are used. """ cfg_ptr = 0 if config is None else config.ptr if isinstance(unique_id, UniqueId): comm_ptr = _nccl_bindings.comm_init_rank_scalable( int(nranks), int(rank), 1, unique_id.ptr, cfg_ptr ) elif isinstance(unique_id, Sequence) and all( isinstance(uid, UniqueId) for uid in unique_id ): arr = _np.empty(len(unique_id), dtype=_nccl_bindings.unique_id_dtype) for i, uid in enumerate(unique_id): arr[i] = uid.as_ndarray[0].copy() comm_ptr = _nccl_bindings.comm_init_rank_scalable( int(nranks), int(rank), int(len(unique_id)), arr, cfg_ptr ) else: raise NcclInvalid("unique_id must be a UniqueId or a sequence of UniqueIds") comm = cls(comm_ptr) # reassign the values in case init() is called inside a group comm._nranks = int(nranks) comm._device = get_cuda_device() comm._rank = int(rank) return comm # --- Communicator APIs --- def split(self, color: int, key: int, config: NCCLConfig | None = None) -> Communicator: """ Splits this communicator into sub-communicators based on color values. Ranks which pass the same color value will be part of the same group. If color is NCCL_SPLIT_NOCOLOR, the rank will not be part of any group and receives a communicator with ptr=0. The key value determines rank ordering; smaller key means smaller rank in the new communicator. If keys are equal between ranks, the rank in the original communicator determines ordering. Args: - color (int): Non-negative color value for grouping ranks (use NCCL_SPLIT_NOCOLOR to exclude this rank). - key (int): Rank ordering key within each color group (smaller key = smaller rank). - config (NCCLConfig, optional): Configuration for the new communicator. If None, inherits parent's configuration. Defaults to None. Returns: ``Communicator``: New sub-communicator, or sentinel communicator (ptr=0) if color is NCCL_SPLIT_NOCOLOR. Raises: - ``NcclInvalid``: If communicator is not initialized or has outstanding operations. Notes: - This is a collective operation. All ranks in the communicator must call this method. - There must not be any outstanding NCCL operations on the communicator to avoid deadlock. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommsplit """ self._check_valid("split") if color == NCCL_SPLIT_NOCOLOR: # Return a sentinel communicator instead of None for consistent API return Communicator(0) cfg_ptr = 0 if config is None else config.ptr comm_ptr = _nccl_bindings.comm_split(self._comm, int(color), int(key), cfg_ptr) return Communicator(comm_ptr) def shrink( self, exclude_ranks: Sequence[int] | None = None, config: NCCLConfig | None = None, flag: CommShrinkFlag = CommShrinkFlag.Default, ) -> Communicator: """ Creates a new communicator by removing specified ranks from the existing communicator. Ranks listed in exclude_ranks will be excluded from the new communicator, and ranks within the new communicator will be updated to maintain a contiguous set of IDs. Args: - exclude_ranks (Sequence[int], optional): Ranks to exclude from the new communicator. Defaults to None (no exclusions). - config (NCCLConfig, optional): Configuration for the new communicator. If None, inherits parent's configuration. Defaults to None. - flag (CommShrinkFlag, optional): Shrink behavior flag. Use CommShrinkFlag.Default for normal operation or CommShrinkFlag.Abort after errors. Defaults to CommShrinkFlag.Default. Returns: ``Communicator``: New communicator without the excluded ranks. Raises: - ``NcclInvalid``: If communicator is not initialized. Notes: - This is a collective operation. All non-excluded ranks must call this method. - Excluded ranks must NOT call this function. - With CommShrinkFlag.Default: There must not be outstanding NCCL operations to avoid deadlock. - With CommShrinkFlag.Default + config.shrink_share=True: Parent communicator resources are reused. - With CommShrinkFlag.Abort: Automatically aborts outstanding operations; no resources shared with parent. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommshrink """ self._check_valid("shrink") ranks_to_exclude = list(exclude_ranks) if exclude_ranks is not None else [] cfg_ptr = 0 if config is None else config.ptr comm_ptr = _nccl_bindings.comm_shrink( self._comm, ranks_to_exclude, len(ranks_to_exclude), cfg_ptr, int(flag) ) return Communicator(comm_ptr) def destroy(self) -> None: """ Destroys the communicator and frees local resources allocated to it. This function only frees local resources if ``finalize()`` was previously called; otherwise, ``destroy()`` will call ``finalize()`` internally. If ``finalize()`` is called explicitly, users must ensure the communicator state becomes ncclSuccess before calling ``destroy()``. The communicator should not be accessed after ``destroy()`` returns. Args: None Raises: None (errors during cleanup are suppressed for safety) Notes: - All resources (registered buffers, windows, custom operators) owned by this communicator are automatically closed before destruction. - This is an intra-node collective call - all ranks on the same node must call it to avoid hanging. - Recommended pattern: Call ``finalize()`` then ``destroy()``. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommdestroy """ # Close all resources first (best-effort, ignore errors) self.close_all_resources() if self._comm == 0: return _nccl_bindings.comm_destroy(self._comm) self._comm = 0 def abort(self) -> None: """ Aborts the communicator and frees resources, terminating all uncompleted operations. This should be called when an unrecoverable error occurs. All active ranks are required to call this function in order to abort the NCCL communicator successfully. Args: None Raises: None (errors during cleanup are suppressed for safety) Notes: - All resources (registered buffers, windows, custom operators) owned by this communicator are automatically closed before aborting. - Unlike ``destroy()``, this immediately aborts uncompleted operations. - All active ranks must call this function to abort successfully. - For more details, see the Fault Tolerance section in NCCL documentation. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommabort """ # Close all resources first (best-effort, ignore errors) self.close_all_resources() if self._comm == 0: return _nccl_bindings.comm_abort(self._comm) self._comm = 0 def finalize(self) -> None: """ Finalizes the communicator, flushing all uncompleted operations and network resources. When the communicator is nonblocking, this is a nonblocking function. Successful return sets the communicator state to ncclInProgress, indicating finalization is in progress. Once all NCCL operations complete, the communicator transitions to ncclSuccess state. Users can query the state with ``get_async_error()``. Args: None Returns: None Notes: - This is typically called before ``destroy()`` to ensure all operations complete. - For nonblocking communicators, check completion status with ``get_async_error()``. - This is a collective operation that must be called by all ranks. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommfinalize """ if self._comm == 0: return _nccl_bindings.comm_finalize(self._comm) # --- Properties --- @property def ptr(self) -> int: """ Raw NCCL communicator pointer (0 if destroyed/invalid). Returns: ``int``: The communicator raw pointer. """ return self._comm @property def is_valid(self) -> bool: """ Checks if the communicator is valid. Returns: ``bool``: True if valid, False if destroyed or invalid. """ return self._comm != 0 @property def nranks(self) -> int: """ Total number of ranks in the communicator. Returns: ``int``: Number of ranks. Raises: - ``NcclInvalid``: If communicator is not initialized. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommcount """ self._check_valid("get nranks") if self._nranks is None: self._nranks = int(_nccl_bindings.comm_count(self._comm)) return self._nranks @property def device(self) -> Device: """ CUDA device object associated with this communicator. Returns: ``Device``: CUDA device object from cuda.core Raises: - ``NcclInvalid``: If communicator is not initialized. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommcudevice """ self._check_valid("get device") if self._device is None: self._device = Device(int(_nccl_bindings.comm_cu_device(self._comm))) return self._device @property def rank(self) -> int: """ This caller's rank within the communicator. Returns: ``int``: This rank's ID (0 to nranks-1). Raises: - ``NcclInvalid``: If communicator is not initialized. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommuserrank """ self._check_valid("get rank") if self._rank is None: self._rank = int(_nccl_bindings.comm_user_rank(self._comm)) return self._rank # --- Point-to-Point Communication --- def send( self, sendbuf: NcclBufferSpec, peer: int, *, stream: NcclStreamSpec | None = None ) -> None: """ Sends a buffer to a peer rank using this communicator. Args: - sendbuf (NcclBufferSpec): Source buffer to send. - peer (int): Destination rank ID. - stream (NcclStreamSpec, optional): CUDA stream for the operation. Defaults to None (uses default stream). Raises: - ``NcclInvalid``: If the buffer specification is invalid, buffer is on wrong device, or communicator is not initialized. """ self._check_valid("send") s = NcclBuffer(sendbuf) self._validate_buffer_device(s, "sendbuf") _nccl_bindings.send( s.ptr, s.count, int(s.dtype), int(peer), int(self._comm), get_stream_ptr(stream) ) def recv( self, recvbuf: NcclBufferSpec, peer: int, *, stream: NcclStreamSpec | None = None ) -> None: """ Receives data into a buffer from a peer rank using this communicator. Args: - recvbuf (NcclBufferSpec): Destination buffer to receive into. - peer (int): Source rank ID. - stream (NcclStreamSpec, optional): CUDA stream for the operation. Defaults to None (uses default stream). Raises: - ``NcclInvalid``: If the buffer specification is invalid, buffer is on wrong device, or communicator is not initialized. """ self._check_valid("recv") r = NcclBuffer(recvbuf) self._validate_buffer_device(r, "recvbuf") _nccl_bindings.recv( r.ptr, r.count, int(r.dtype), int(peer), int(self._comm), get_stream_ptr(stream) ) def wait_signal( self, signal_descs: Sequence[WaitSignalDesc], *, stream: NcclStreamSpec | None = None ) -> None: """ Waits for signals as described in the signal descriptor array. This function enqueues a wait operation on the specified CUDA stream that blocks until the required signals from peer ranks are received. Each descriptor specifies a peer rank and the number of signal operations to wait for from that peer. Args: signal_descs (list[WaitSignalDesc]): List of signal descriptors specifying which peers to wait for and how many signals to expect from each. stream (NcclStreamSpec | None): CUDA stream to enqueue the wait operation on. Raises: NcclInvalid: If communicator is not initialized or if any descriptor in the list is not a valid WaitSignalDesc instance. Example: >>> # Wait for 1 signal from peer rank 0 >>> desc = WaitSignalDesc(op_cnt=1, peer=0, sig_idx=0, ctx=0) >>> comm.wait_signal([desc], stream=stream) """ self._check_valid("wait_signal") nr_descs = int(len(signal_descs)) arr = _np.empty(nr_descs, dtype=_nccl_bindings.wait_signal_desc_dtype) for idx, desc in enumerate(signal_descs): if not isinstance(desc, WaitSignalDesc): raise NcclInvalid(f"Descriptor at index {idx} is not a valid WaitSignalDesc") arr[idx]["op_cnt"] = desc.op_cnt arr[idx]["peer"] = desc.peer arr[idx]["sig_idx"] = desc.sig_idx arr[idx]["ctx"] = desc.ctx ptr = 0 if nr_descs == 0 else int(arr.ctypes.data) _nccl_bindings.wait_signal(nr_descs, ptr, int(self._comm), get_stream_ptr(stream)) def signal(self, peer: int, sig_idx: int, ctx: int, flags: int, *, stream: NcclStreamSpec | None = None) -> None: """ Sends a signal to a peer rank. This function enqueues a signal operation on the specified CUDA stream that notifies the target peer rank. The peer can wait for this signal using :meth:`wait_signal`. Args: peer (int): Target rank to send the signal to. sig_idx (int): Signal index identifier for the operation. Currently must be 0. ctx (int): Context identifier for the operation. Currently must be 0. flags (int): Reserved for future use. Should be set to 0. stream (NcclStreamSpec | None): CUDA stream to enqueue the signal operation on. Raises: NcclInvalid: If communicator is not initialized. Example: >>> # Send a signal to peer rank 1 >>> comm.signal(peer=1, sig_idx=0, ctx=0, flags=0, stream=stream) See Also: :meth:`wait_signal`: The method used by peers to wait for signals. """ self._check_valid("signal") _nccl_bindings.signal(peer, sig_idx, ctx, flags, self._comm, get_stream_ptr(stream)) # --- Collective Communication Operations --- def allreduce( self, sendbuf: NcclBufferSpec, recvbuf: NcclBufferSpec, op: NcclRedOp | CustomRedOp, stream: NcclStreamSpec | None = None, ) -> None: """ Reduces data arrays of length count in sendbuf using the specified operation and leaves identical copies of the result in each recvbuf. All ranks receive the same reduced result in their receive buffers after this collective operation completes. This is a shortcut for ``reduce(sendbuf, recvbuf, op, root=None, stream=stream)``. Args: - sendbuf (NcclBufferSpec): Source buffer specification containing data to be reduced. - recvbuf (NcclBufferSpec): Destination buffer specification that will receive the reduced result. - op (NcclRedOp | CustomRedOp): Reduction operator to apply (e.g., SUM, MAX, MIN, AVG, PROD, or custom operator). - stream (NcclStreamSpec, optional): CUDA stream for the operation. Defaults to None (uses default stream). Raises: - ``NcclInvalid``: If send and receive buffers have mismatched dtypes, mismatched counts, buffers on wrong device, invalid buffer specifications, or communicator is not initialized. Notes: - Both send and receive buffers must have matching data types. - Element count is inferred from the sendbuf specification: count = sendcount. - Requires recvcount >= sendcount. - In-place operation occurs when sendbuf and recvbuf resolve to the same device memory address. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/colls.html#ncclallreduce """ self._check_valid("allreduce") self.reduce(sendbuf, recvbuf, op, stream=stream) def broadcast( self, sendbuf: NcclBufferSpec | Any, recvbuf: NcclBufferSpec, root: int, *, stream: NcclStreamSpec | None = None, ) -> None: """ Copies count elements from sendbuf on the root rank to all ranks' recvbuf. The sendbuf is only used on the root rank and is ignored for other ranks. Args: - sendbuf (NcclBufferSpec | Any): Source buffer specification (only used on root rank). - recvbuf (NcclBufferSpec): Destination buffer specification that will receive the broadcast data. - root (int): Root rank that broadcasts the data (must be between 0 and nranks-1). - stream (NcclStreamSpec, optional): CUDA stream for the operation. Defaults to None (uses default stream). Raises: - ``NcclInvalid``: If send and receive buffers have mismatched dtypes, mismatched counts, buffers on wrong device, invalid buffer specifications, or communicator is not initialized. Notes: - On root rank, both send and receive buffers must have matching data types. - Element count is inferred from the recvbuf specification: count = recvcount. - On root rank, requires sendcount == recvcount. - In-place operation occurs when sendbuf and recvbuf resolve to the same device memory address. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/colls.html#ncclbroadcast """ self._check_valid("broadcast") s, r = None, None if root == self.rank: s = NcclBuffer(sendbuf) self._validate_buffer_device(s, "sendbuf") r = NcclBuffer(recvbuf) self._validate_buffer_device(r, "recvbuf") if s is not None: if s.dtype != r.dtype: raise NcclInvalid( f"Dtype mismatch: sendbuf has dtype {s.dtype}, recvbuf has dtype {r.dtype}" ) if r.count != s.count: raise NcclInvalid( f"Buffer count mismatch: recvbuf must have exactly {s.count} elements, got {r.count}" ) s_ptr = s.ptr if s is not None else 0 r_ptr = r.ptr count = r.count dtype = r.dtype _nccl_bindings.broadcast( s_ptr, r_ptr, count, int(dtype), int(root), int(self._comm), get_stream_ptr(stream) ) def reduce( self, sendbuf: NcclBufferSpec, recvbuf: NcclBufferSpec | Any, op: NcclRedOp | CustomRedOp, root: int | None = None, *, stream: NcclStreamSpec | None = None, ) -> None: """ Reduces data arrays of length count in sendbuf using the specified operation. This method supports two modes of operation: 1. **AllReduce Mode** (root=None): Reduces data and leaves identical copies of the result in each rank's recvbuf. All ranks receive the same reduced result after the collective operation completes. 2. **Reduce Mode** (root specified): Reduces data and places the result only in recvbuf on the specified root rank. The recvbuf is only used on the root rank and is ignored for other ranks. Args: - sendbuf (NcclBufferSpec): Source buffer specification containing data to be reduced. - recvbuf (NcclBufferSpec | Any): Destination buffer specification that will receive the reduced result. In Reduce Mode (root specified), only used on root rank. - op (NcclRedOp | CustomRedOp): Reduction operator to apply (e.g., SUM, MAX, MIN, AVG, PROD, or custom operator). - root (int | None, optional): Root rank that receives the reduced result (must be between 0 and nranks-1). If None, performs an all-reduce where all ranks receive the result. Defaults to None. - stream (NcclStreamSpec, optional): CUDA stream for the operation. Defaults to None (uses default stream). Raises: - ``NcclInvalid``: If send and receive buffers have mismatched dtypes, mismatched counts, buffers on wrong device, invalid buffer specifications, or communicator is not initialized. Notes: - Both send and receive buffers must have matching data types if receive buffer is used. - Element count is inferred from the sendbuf specification: count = sendcount. - In All-Reduce Mode: All ranks must have recvcount >= sendcount. - In Reduce Mode: Only root rank requires recvcount >= sendcount. - In-place operation occurs when sendbuf and recvbuf resolve to the same device memory address. See Also: - All-Reduce Mode: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/colls.html#ncclallreduce - Reduce Mode: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/colls.html#ncclreduce """ self._check_valid("reduce") s, r = NcclBuffer(sendbuf), None self._validate_buffer_device(s, "sendbuf") if root is None or root == self.rank: r = NcclBuffer(recvbuf) self._validate_buffer_device(r, "recvbuf") if r is not None: if s.dtype != r.dtype: raise NcclInvalid( f"Dtype mismatch: sendbuf has dtype {s.dtype}, recvbuf has dtype {r.dtype}" ) if r.count < s.count: raise NcclInvalid( f"Buffer count mismatch: recvbuf must have at least {s.count} elements, got {r.count}" ) s_ptr = s.ptr r_ptr = r.ptr if r is not None else 0 count = s.count dtype = s.dtype if root is None: _nccl_bindings.all_reduce( s_ptr, r_ptr, count, int(dtype), int(op), int(self._comm), get_stream_ptr(stream) ) else: _nccl_bindings.reduce( s_ptr, r_ptr, count, int(dtype), int(op), int(root), int(self._comm), get_stream_ptr(stream), ) def allgather( self, sendbuf: NcclBufferSpec, recvbuf: NcclBufferSpec, stream: NcclStreamSpec | None = None, ) -> None: """ Gathers sendcount values from all ranks and leaves identical copies of the result in each recvbuf, receiving data from rank i at offset i*sendcount. All ranks receive the same concatenated result containing data from all ranks. This is a shortcut for ``gather(sendbuf, recvbuf, root=None, stream=stream)``. Args: - sendbuf (NcclBufferSpec): Source buffer specification containing sendcount elements. - recvbuf (NcclBufferSpec): Destination buffer specification (must have size at least nranks*sendcount elements). - stream (NcclStreamSpec, optional): CUDA stream for the operation. Defaults to None (uses default stream). Raises: - ``NcclInvalid``: If send and receive buffers have mismatched dtypes, recvbuf is too small, buffers on wrong device, invalid buffer specifications, or communicator is not initialized. Notes: - Both send and receive buffers must have matching data types. - Element count is inferred from the sendbuf specification: count = sendcount. - Requires recvcount >= nranks * sendcount. - Data from rank i is placed at recvbuf + i*sendcount. - In-place operation occurs when sendbuf resolves to device memory address: recvbuf_address + rank*sendcount. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/colls.html#ncclallgather """ self._check_valid("allgather") self.gather(sendbuf, recvbuf, stream=stream) def reduce_scatter( self, sendbuf: NcclBufferSpec, recvbuf: NcclBufferSpec, op: NcclRedOp | CustomRedOp, *, stream: NcclStreamSpec | None = None, ) -> None: """ Reduces data in sendbuf from all ranks using the specified operation and leaves the reduced result scattered over the devices so that recvbuf on rank i contains the i-th block of the result. Each rank receives a different portion of the reduced result. Args: - sendbuf (NcclBufferSpec): Source buffer specification (must have size at least nranks*recvcount elements). - recvbuf (NcclBufferSpec): Destination buffer specification containing recvcount elements. - op (NcclRedOp | CustomRedOp): Reduction operator to apply (e.g., SUM, MAX, MIN, AVG, PROD, or custom operator). - stream (NcclStreamSpec, optional): CUDA stream for the operation. Defaults to None (uses default stream). Raises: - ``NcclInvalid``: If send and receive buffers have mismatched dtypes, sendbuf is too small, buffers on wrong device, invalid buffer specifications, or communicator is not initialized. Notes: - Both send and receive buffers must have matching data types. - Element count is inferred from the sendbuf specification: count = sendcount / nranks. - Requires sendcount >= nranks and recvcount >= count. - Rank i receives the i-th block of the reduced result in its recvbuf. - In-place operation occurs when recvbuf resolves to device memory address: sendbuf_address + rank*count. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/colls.html#ncclreducescatter """ self._check_valid("reduce_scatter") s, r = NcclBuffer(sendbuf), NcclBuffer(recvbuf) self._validate_buffer_device(s, "sendbuf") self._validate_buffer_device(r, "recvbuf") if s.dtype != r.dtype: raise NcclInvalid( f"Dtype mismatch: sendbuf has dtype {s.dtype}, recvbuf has dtype {r.dtype}" ) per_rank_count = s.count // self.nranks if per_rank_count < 1: raise NcclInvalid( f"Buffer count mismatch: sendbuf must have at least {self.nranks} elements (nranks), got {s.count}" ) if r.count < per_rank_count: raise NcclInvalid( f"Buffer count mismatch: recvbuf must have at least {per_rank_count} elements (sendcount / nranks), got {r.count}" ) s_ptr = s.ptr r_ptr = r.ptr count = per_rank_count dtype = s.dtype _nccl_bindings.reduce_scatter( s_ptr, r_ptr, count, int(dtype), int(op), int(self._comm), get_stream_ptr(stream) ) def alltoall( self, sendbuf: NcclBufferSpec, recvbuf: NcclBufferSpec, *, stream: NcclStreamSpec | None = None, ) -> None: """ Each rank sends count values to all other ranks and receives count values from all other ranks. Data to send to destination rank j is taken from sendbuf+j*count and data received from source rank i is placed at recvbuf+i*count. Args: - sendbuf (NcclBufferSpec): Source buffer specification (must have size at least nranks*count elements). - recvbuf (NcclBufferSpec): Destination buffer specification (must have size at least nranks*count elements). - stream (NcclStreamSpec, optional): CUDA stream for the operation. Defaults to None (uses default stream). Raises: - ``NcclInvalid``: If send and receive buffers have mismatched dtypes, buffer sizes incompatible with nranks, buffers on wrong device, invalid buffer specifications, or communicator is not initialized. Notes: - Both send and receive buffers must have matching data types. - Element count is inferred from the sendbuf specification: count = sendcount / nranks. - Requires sendcount >= nranks and recvcount >= sendcount. - Data sent to rank j is at sendbuf + j*count and data received from rank i is at recvbuf + i*count. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/colls.html#ncclalltoall """ self._check_valid("alltoall") s, r = NcclBuffer(sendbuf), NcclBuffer(recvbuf) self._validate_buffer_device(s, "sendbuf") self._validate_buffer_device(r, "recvbuf") if s.dtype != r.dtype: raise NcclInvalid( f"Dtype mismatch: sendbuf has dtype {s.dtype}, recvbuf has dtype {r.dtype}" ) per_rank_count = s.count // self.nranks if per_rank_count < 1: raise NcclInvalid( f"Buffer count mismatch: sendbuf must have at least {self.nranks} elements (nranks), got {s.count}" ) if r.count < s.count: raise NcclInvalid( f"Buffer count mismatch: recvbuf must have at least {s.count} elements (nranks * count), got {r.count}" ) s_ptr = s.ptr r_ptr = r.ptr count = per_rank_count dtype = s.dtype _nccl_bindings.allto_all( s_ptr, r_ptr, count, int(dtype), int(self._comm), get_stream_ptr(stream) ) def gather( self, sendbuf: NcclBufferSpec, recvbuf: NcclBufferSpec | Any, root: int | None = None, *, stream: NcclStreamSpec | None = None, ) -> None: """ Gathers sendcount values from all ranks. This method supports two modes of operation: 1. **AllGather Mode** (root=None): Gathers values from all ranks and leaves identical copies of the result in each recvbuf. All ranks receive the same concatenated result containing data from all ranks. 2. **Gather Mode** (root specified): Gathers values from all ranks to the specified root rank. The recvbuf is only used on the root rank and is ignored for other ranks. Args: - sendbuf (NcclBufferSpec): Source buffer specification containing sendcount elements. - recvbuf (NcclBufferSpec | Any): Destination buffer specification (must have size at least nranks*sendcount elements). In Gather Mode (root specified), only used on root rank. - root (int | None, optional): Root rank that receives the gathered data (must be between 0 and nranks-1). If None, performs an all-gather where all ranks receive the result. Defaults to None. - stream (NcclStreamSpec, optional): CUDA stream for the operation. Defaults to None (uses default stream). Raises: - ``NcclInvalid``: If send and receive buffers have mismatched dtypes, recvbuf is too small, buffers on wrong device, invalid buffer specifications, or communicator is not initialized. Notes: - Both send and receive buffers must have matching data types if receive buffer is used. - Element count is inferred from the sendbuf specification: count = sendcount. - In AllGather Mode: All ranks must have recvcount >= nranks * sendcount. - In Gather Mode: Only root rank requires recvcount >= nranks * sendcount. - Data from rank i is placed at recvbuf + i*sendcount. - In AllGather Mode, in-place operation occurs when sendbuf resolves to device memory address: recvbuf_address + rank*sendcount. - In Gather Mode, in-place operation occurs when sendbuf resolves to device memory address: recvbuf_address + root*sendcount. See Also: - AllGather Mode: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/colls.html#ncclallgather - Gather Mode: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/colls.html#ncclgather """ self._check_valid("gather") s, r = NcclBuffer(sendbuf), None self._validate_buffer_device(s, "sendbuf") if root is None or root == self.rank: r = NcclBuffer(recvbuf) self._validate_buffer_device(r, "recvbuf") if r is not None: if r.dtype != s.dtype: raise NcclInvalid( f"Dtype mismatch: sendbuf has dtype {s.dtype}, recvbuf has dtype {r.dtype}" ) expected_recv_count = self.nranks * s.count if r.count < expected_recv_count: raise NcclInvalid( f"Buffer count mismatch: recvbuf must have at least {expected_recv_count} elements (nranks * sendcount), got {r.count}" ) s_ptr = s.ptr r_ptr = r.ptr if r is not None else 0 count = s.count dtype = s.dtype if root is None: _nccl_bindings.all_gather( s_ptr, r_ptr, count, int(dtype), int(self._comm), get_stream_ptr(stream) ) else: _nccl_bindings.gather( s_ptr, r_ptr, count, int(dtype), int(root), int(self._comm), get_stream_ptr(stream) ) def scatter( self, sendbuf: NcclBufferSpec | Any, recvbuf: NcclBufferSpec, root: int, *, stream: NcclStreamSpec | None = None, ) -> None: """ Each rank receives count elements from the root rank. On the root rank, count elements from sendbuf + i*count are sent to rank i. On non-root ranks, sendbuf is not used. Args: - sendbuf (NcclBufferSpec | Any): Source buffer specification (only used on root rank, must have size at least nranks*count elements). - recvbuf (NcclBufferSpec): Destination buffer specification containing count elements. - root (int): Root rank that scatters the data (must be between 0 and nranks-1). - stream (NcclStreamSpec, optional): CUDA stream for the operation. Defaults to None (uses default stream). Raises: - ``NcclInvalid``: If send and receive buffers have mismatched dtypes, sendbuf is too small on root rank, buffers on wrong device, invalid buffer specifications, or communicator is not initialized. Notes: - On root rank, both send and receive buffers must have matching data types. - Element count is inferred from the recvbuf specification: count = recvcount. - On root rank, requires sendcount >= nranks and sendcount / nranks == recvcount. - On root rank, data at sendbuf + i*count is sent to rank i. - In-place operation occurs when recvbuf resolves to device memory address: sendbuf_address + root*count. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/colls.html#ncclscatter """ self._check_valid("scatter") s, r = None, None if root == self.rank: s = NcclBuffer(sendbuf) self._validate_buffer_device(s, "sendbuf") r = NcclBuffer(recvbuf) self._validate_buffer_device(r, "recvbuf") if s is not None: if s.dtype != r.dtype: raise NcclInvalid( f"Dtype mismatch: sendbuf has dtype {s.dtype}, recvbuf has dtype {r.dtype}" ) per_rank_count = s.count // self.nranks if per_rank_count < 1: raise NcclInvalid( f"Buffer count mismatch: sendbuf must have at least {self.nranks} elements (nranks), got {s.count}" ) if r.count != per_rank_count: raise NcclInvalid( f"Buffer count mismatch: recvbuf must have exactly {per_rank_count} elements (sendcount / nranks), got {r.count}" ) s_ptr = s.ptr if s is not None else 0 r_ptr = r.ptr count = r.count dtype = r.dtype _nccl_bindings.scatter( s_ptr, r_ptr, count, int(dtype), int(root), int(self._comm), get_stream_ptr(stream) ) # --- Registration --- def register_buffer(self, buffer: NcclBufferSpec) -> RegisteredBufferHandle: """ Registers a buffer with this communicator for zero-copy communication. Registered buffers can enable performance optimizations in NCCL operations. The returned RegisteredBufferHandle must be explicitly closed when no longer needed. Args: - buffer (NcclBufferSpec): Buffer to register (array, Buffer, or buffer-like object). Returns: ``RegisteredBufferHandle``: Resource handle that can be closed manually or automatically when the communicator is destroyed / aborted. Raises: - ``NcclInvalid``: If buffer is on wrong device or communicator is not initialized. Notes: - Buffer size is automatically derived from buffer count and dtype. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommregister """ self._check_valid("register_buffer") nccl_buf = NcclBuffer(buffer) self._validate_buffer_device(nccl_buf, "buffer") buffer_ptr = nccl_buf.ptr size = nccl_buf.count * nccl_buf.dtype.itemsize resource = RegisteredBufferHandle(self._comm, buffer_ptr, size) self._resources.append(resource) return resource def register_window( self, buffer: NcclBufferSpec, flags: WindowFlag | None = None ) -> RegisteredWindowHandle | None: """ Collectively registers a local buffer into an NCCL window for optimized communication. Since this is a collective call, every rank in the communicator must participate, and buffer size must be equal among ranks by default. Windows enable optimized communication patterns in NCCL. Args: - buffer (NcclBufferSpec): Local buffer to register as window. - flags (WindowFlag, optional): Window registration flags to control behavior. Defaults to None. Returns: ``RegisteredWindowHandle``: Resource handle that can be closed manually or automatically when the communicator is destroyed / aborted, or ``None`` if NCCL returns a NULL handle (e.g., window unsupported on a platform). Raises: - ``NcclInvalid``: If buffer is on wrong device or communicator is not initialized. Notes: - This is a collective operation. All ranks in the communicator must call this method. - Buffer sizes must be equal among ranks by default. - Buffer size is automatically derived from buffer count and dtype. - If called within a group, the handle value may not be filled until ncclGroupEnd() completes. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommwindowregister """ self._check_valid("register_window") nccl_buf = NcclBuffer(buffer) self._validate_buffer_device(nccl_buf, "buffer") buffer_ptr = nccl_buf.ptr size = nccl_buf.count * nccl_buf.dtype.itemsize resource = RegisteredWindowHandle(self._comm, buffer_ptr, size, flags) if resource.handle == 0: return None self._resources.append(resource) return resource # --- Custom Reduction APIs --- def create_pre_mul_sum( self, scalar: NcclScalarSpec, datatype: NcclDataType | None = None, ) -> CustomRedOp: """ Creates a PreMulSum custom reduction operator. The PreMulSum operator performs: output = scalar * sum(inputs). This is useful for averaging (scalar = 1/N) or weighted reductions. The returned CustomRedOp must be explicitly closed when no longer needed. Args: scalar: Scalar multiplier value. Can be: - Python int/float: Converted to NumPy array, uses host memory - NumPy array: Must contain exactly 1 element, uses host memory - NcclSupportedBuffer: Device buffer datatype: NCCL data type of the scalar and reduction. If None, inferred from scalar. - For Python int/float: inferred from NumPy's natural dtype (int64/float64) - For NumPy array: inferred from array dtype - For device buffer: inferred from buffer dtype Returns: CustomRedOp resource that can be closed manually or automatically when the communicator is destroyed / aborted. Raises: - ``NcclInvalid``: If communicator is not initialized, scalar type is not supported, NumPy array or buffer doesn't contain exactly 1 element, or datatype cannot be inferred or is incompatible. """ self._check_valid("create_pre_mul_sum") # Determine residence and prepare scalar pointer scalar_array = None # Will hold host scalar to prevent GC residence: _nccl_bindings.ScalarResidence if isinstance(scalar, (int, float)): # Python scalar: Convert to NumPy array in host memory residence = _nccl_bindings.ScalarResidence.HostImmediate if datatype is None: scalar_array = _np.array([scalar]) datatype = NcclDataType(scalar_array.dtype) else: scalar_array = _np.array([scalar], dtype=datatype.numpy_dtype) scalar_ptr = scalar_array.ctypes.data elif isinstance(scalar, _np.ndarray): # NumPy array: Host memory residence = _nccl_bindings.ScalarResidence.HostImmediate # Validate array has exactly 1 element if scalar.size != 1: raise NcclInvalid( f"NumPy array must contain exactly 1 element for scalar, got {scalar.size} elements" ) # Ensure contiguous array scalar_array = _np.ascontiguousarray(scalar.ravel()) scalar_ptr = scalar_array.ctypes.data if datatype is None: datatype = NcclDataType(scalar_array.dtype) else: # Assume it's NcclSupportedBuffer (device buffer) # Use NcclBuffer to handle Buffer, DLPack, CAI, etc. residence = _nccl_bindings.ScalarResidence.Device try: buf = NcclBuffer(scalar) except (TypeError, ValueError) as e: raise NcclInvalid( f"scalar must be int, float, numpy.ndarray, or NcclSupportedBuffer, " f"got {type(scalar).__name__}: {e}" ) from e # Validate buffer contains exactly 1 element if buf.count != 1: raise NcclInvalid( f"Device buffer must contain exactly 1 element for scalar, got {buf.count} elements" ) # Infer datatype from buffer if not provided if datatype is None: datatype = buf.dtype else: # Validate buffer datatype matches requested datatype if buf.dtype != datatype: raise NcclInvalid( f"Device buffer datatype {buf.dtype} doesn't match requested datatype {datatype}" ) scalar_ptr = buf.ptr # Create the custom reduction operator resource = CustomRedOp(self._comm, scalar_ptr, datatype, residence) # Keep scalar_array alive by storing in resource to prevent premature GC # (only needed for host scalars; device buffers are managed by user) if scalar_array is not None: resource._scalar_array = scalar_array self._resources.append(resource) return resource def close_all_resources(self) -> None: """ Closes all resources owned by this communicator. This method is called automatically during ``destroy()`` and ``abort()`` but can be called manually if needed. It performs best-effort cleanup, ignoring any errors that occur during resource deallocation. Notes: This method is idempotent - calling it multiple times is safe. """ for resource in self._resources: try: resource.close() except Exception: # Best-effort cleanup - ignore errors to avoid masking # user exceptions or preventing communicator destruction pass self._resources.clear() # --- Miscellaneous --- def get_last_error(self) -> str: """ Gets the last error string for this communicator. Returns: ``str``: Error message string. Raises: - ``NcclInvalid``: If communicator is not initialized. """ self._check_valid("get last error") return _nccl_bindings.get_last_error(self._comm) def get_async_error(self) -> _nccl_bindings.Result: """ Queries the progress and potential errors of asynchronous NCCL operations. Operations without a stream argument (e.g., finalize) are complete when they return ncclSuccess. Operations with a stream argument (e.g., reduce) return ncclSuccess when posted but may report errors through this method until completed. If any NCCL function returns ncclInProgress, users must query communicator state until it becomes ncclSuccess before calling another NCCL function. Returns: ``Result``: Current state of the communicator (ncclSuccess, ncclInProgress, or error code). Raises: - ``NcclInvalid``: If communicator is not initialized. Notes: - Before state becomes ncclSuccess, do not issue CUDA kernels on streams used by NCCL. - If an error occurs, destroy the communicator with ``abort()``. - Nothing can be assumed about completion or correctness of enqueued operations after an error. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommgetasyncerror """ self._check_valid("get async error") return _nccl_bindings.comm_get_async_error(self._comm) nccl-2.29.7-1/bindings/nccl4py/nccl/core/constants.py000066400000000000000000000032371515037102200222770ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ NCCL constants and enums. This module centralizes all NCCL constants for easy access and organization. """ from enum import IntEnum __all__ = [ "NCCL_UNDEF_INT", "NCCL_UNDEF_FLOAT", "NCCL_SPLIT_NOCOLOR", "CTAPolicy", "CommShrinkFlag", "WindowFlag", ] # NCCL sentinel values for undefined config fields NCCL_UNDEF_INT: int = -2147483648 # INT_MIN """NCCL sentinel value for undefined integer configuration fields.""" NCCL_UNDEF_FLOAT: float = -1.0 """NCCL sentinel value for undefined float fields.""" # Communicator split constants NCCL_SPLIT_NOCOLOR: int = -1 """Color value for ncclCommSplit to indicate rank will not be part of any group.""" # CTA (Cooperative Thread Array) Policy flags class CTAPolicy(IntEnum): """ NCCL performance policy for CTA scheduling. """ Default = 0x00 """Default CTA policy.""" Efficiency = 0x01 """Optimize for efficiency.""" Zero = 0x02 """Zero-CTA optimization.""" # Communicator shrink flags class CommShrinkFlag(IntEnum): """ Flags for ncclCommShrink behavior. """ Default = 0x00 """Shrink the parent communicator.""" Abort = 0x01 """First terminate ongoing parent operations, then shrink the parent communicator.""" # Window registration flags class WindowFlag(IntEnum): """ Flags for window registration. """ Default = 0x00 """Default window registration.""" CollSymmetric = 0x01 """Collective symmetric window registration.""" nccl-2.29.7-1/bindings/nccl4py/nccl/core/cuda.py000066400000000000000000000042411515037102200211730ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ CUDA device and stream utilities for NCCL operations. This module provides context managers and helper functions for working with CUDA devices and streams in NCCL operations, including device context management and stream resolution. """ from __future__ import annotations from cuda.core import Device, Stream from nccl.core.typing import NcclDeviceSpec, NcclStreamSpec class CudaDeviceContext: def __init__(self, device: Device) -> None: self._device = device self._old_device = Device() def __enter__(self): if self._device.device_id != self._old_device.device_id: self._device.set_current() return self def __exit__(self, exc_type, exc_val, exc_tb): if self._device.device_id != self._old_device.device_id: self._old_device.set_current() return False # Re-raise any exception def get_cuda_device(device: NcclDeviceSpec | None = None) -> Device: if device is None: return Device() elif isinstance(device, Device): return device else: return Device(device) def get_device_id(device: NcclDeviceSpec | None = None) -> int: if isinstance(device, int): return device else: return get_cuda_device(device).device_id def get_cuda_stream( stream: NcclStreamSpec | None = None, device: NcclDeviceSpec | None = None ) -> Stream: if stream is None: device = get_cuda_device(device) return device.default_stream elif isinstance(stream, Stream): return stream elif isinstance(stream, int): return Stream.from_handle(handle=stream) else: device = get_cuda_device(device) return device.create_stream(stream) def get_stream_ptr(stream: NcclStreamSpec | None = None) -> int: if stream is None: return 0 elif isinstance(stream, int): return stream elif isinstance(stream, Stream): return int(stream.handle) else: return int(stream.__cuda_stream__()[1]) nccl-2.29.7-1/bindings/nccl4py/nccl/core/group.py000066400000000000000000000077671515037102200214330ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ NCCL group operations for batching collective communications. This module provides APIs for grouping multiple NCCL operations together to enable efficient batching and overlapping of communication operations. Group calls allow multiple collective operations to be launched together with better performance and the ability to simulate operations for performance analysis. See more details at: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/groups.html """ from __future__ import annotations import contextlib from typing import Generator from nccl import bindings as _nccl_bindings from nccl.core.constants import NCCL_UNDEF_FLOAT __all__ = ["GroupSimInfo", "group_start", "group_end", "group_simulate_end", "group"] class GroupSimInfo: """ Information for NCCL group operation simulation. This class holds simulation information that can be used to estimate the performance of group operations without actually executing them. """ def __init__(self) -> None: """ Initializes group simulation info with default values. """ self._sim_info = _nccl_bindings.SimInfo() # Apply NCCL_SIM_INFO_INITIALIZER defaults self._sim_info.size_ = int(_nccl_bindings.sim_info_dtype.itemsize) self._sim_info.magic = 0x74685283 # NCCL protocol magic number for ncclSimInfo_t validation self._sim_info.version = _nccl_bindings.get_version() self._sim_info.estimated_time = NCCL_UNDEF_FLOAT @property def ptr(self) -> int: """ Raw NCCL simulation info pointer. Returns: ``int``: The simulation info pointer. """ return int(self._sim_info.ptr) # Field proxies @property def estimated_time(self) -> float: """ Estimated execution time for the group operations (in seconds). Returns: ``float``: Estimated time in seconds. """ return self._sim_info.estimated_time def group_start() -> None: """ Starts a group of NCCL operations. All NCCL operations called after this will be batched together and executed when group_end() is called. This can improve performance by allowing NCCL to optimize the operation sequence. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/group.html#ncclgroupstart """ return _nccl_bindings.group_start() def group_end() -> None: """ Ends a group of NCCL operations. Executes all operations that were queued since the last group_start(). This must be called to actually perform the batched operations. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/group.html#ncclgroupend """ return _nccl_bindings.group_end() def group_simulate_end(sim_info: GroupSimInfo | None) -> None: """ Simulates the end of a group of NCCL operations. This estimates the execution time of the queued operations without actually executing them. The estimated time is stored in sim_info. Args: - sim_info (GroupSimInfo, optional): Simulation info object to store estimated time, or None. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/group.html#ncclgroupsimulateend """ if sim_info is None: return _nccl_bindings.group_simulate_end(None) return _nccl_bindings.group_simulate_end(int(sim_info.ptr)) @contextlib.contextmanager def group() -> Generator[None, None, None]: """ Context manager for NCCL group operations. Automatically calls group_start() on entry and group_end() on exit. This ensures proper cleanup even if an exception occurs. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/groups.html """ group_start() try: yield finally: group_end() nccl-2.29.7-1/bindings/nccl4py/nccl/core/interop/000077500000000000000000000000001515037102200213645ustar00rootroot00000000000000nccl-2.29.7-1/bindings/nccl4py/nccl/core/interop/__init__.py000066400000000000000000000006311515037102200234750ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ NCCL4Py interop modules: PyTorch and CuPy integration. This module provides utilities for integrating NCCL4Py with PyTorch and CuPy: - Memory allocation backed by NCCL allocator - Buffer resolution utilities """ nccl-2.29.7-1/bindings/nccl4py/nccl/core/interop/cupy.py000066400000000000000000000167371515037102200227340ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ CuPy interoperability for NCCL4Py. This module provides utilities for creating CuPy arrays backed by NCCL-allocated memory, enabling zero-copy integration between NCCL operations and CuPy workflows. """ from __future__ import annotations import numpy as np from typing import Literal from nccl.core.buffer import mem_alloc from nccl.core.typing import ( NcclInvalid, NcclDataType, INT8, UINT8, INT32, UINT32, INT64, UINT64, FLOAT16, FLOAT32, FLOAT64, BFLOAT16, FLOAT8E4M3, FLOAT8E5M2, ) __all__ = ["empty", "resolve_array"] try: import cupy _cupy_enabled = True except ImportError: _cupy_enabled = False def _to_nccl_dtype(cupy_dtype) -> NcclDataType: """ Converts CuPy dtype to NcclDataType. Note: CuPy dtypes are numpy.dtype objects. This function maps them to global NCCL data type constants. ml-dtypes is needed for bfloat16, float8_e4m3fn, float8_e5m2. Args: - cupy_dtype (cupy.dtype): CuPy data type. Returns: ``NcclDataType``: Corresponding NCCL data type (global constant). Raises: - ``ModuleNotFoundError``: If CuPy is not installed. - ``NcclInvalid``: If cupy_dtype has no NCCL equivalent. """ if not _cupy_enabled: raise ModuleNotFoundError("CuPy is not installed") # CuPy dtypes are numpy dtypes - convert to numpy dtype first try: np_dtype = np.dtype(cupy_dtype) except (TypeError, ValueError) as e: raise NcclInvalid( f"Invalid data type: could not convert {cupy_dtype} (type: {type(cupy_dtype).__name__}) " f"to numpy dtype: {e}" ) # Map numpy dtype to global NcclDataType constants # First try name-based for ml-dtypes (higher priority) if np_dtype.name == "bfloat16": return BFLOAT16 elif np_dtype.name == "float8_e4m3fn": return FLOAT8E4M3 elif np_dtype.name == "float8_e5m2": return FLOAT8E5M2 # Explicitly unsupported types - detect and give clear error _unsupported_kinds = { "c": "complex", "V": "void/structured", "O": "object", "S": "byte string", "U": "unicode string", "M": "datetime", "m": "timedelta", } if np_dtype.kind in _unsupported_kinds: kind_name = _unsupported_kinds[np_dtype.kind] raise NcclInvalid( f"Unsupported data type: numpy dtype {np_dtype} ({kind_name} type) has no NCCL equivalent. " f"NCCL only supports numeric types." ) # Map standard numpy types by kind+size _numpy_to_nccl = { ("f", 2): FLOAT16, ("f", 4): FLOAT32, ("f", 8): FLOAT64, ("i", 1): INT8, ("i", 2): None, # int16 - explicitly not supported by NCCL ("i", 4): INT32, ("i", 8): INT64, ("u", 1): UINT8, ("u", 2): None, # uint16 - explicitly not supported by NCCL ("u", 4): UINT32, ("u", 8): UINT64, ("b", 1): UINT8, # bool -> uint8 } kind_size = (np_dtype.kind, np_dtype.itemsize) if kind_size in _numpy_to_nccl: result = _numpy_to_nccl[kind_size] if result is None: raise NcclInvalid( f"Unsupported data type: numpy dtype {np_dtype} (kind={np_dtype.kind}, " f"itemsize={np_dtype.itemsize}). NCCL does not support {np_dtype.kind}{np_dtype.itemsize * 8}-bit types." ) return result else: raise NcclInvalid( f"Unsupported data type: numpy dtype {np_dtype} (kind={np_dtype.kind}, " f"itemsize={np_dtype.itemsize}, name={np_dtype.name}) has no NCCL equivalent. " f"NCCL supports: int8/32/64, uint8/32/64, float16/32/64, bfloat16, float8_e4m3fn, float8_e5m2." ) def _allocate_nccl_array(shape: tuple[int, ...], dtype: np.dtype, order: str) -> cupy.ndarray: """ Allocates NCCL-backed CuPy array with specified parameters. Args: - shape (tuple[int, ...]): Shape of array. - dtype (np.dtype): Data type. - order (str): Memory order ("C" or "F"). Returns: ``cupy.ndarray``: Allocated array with NCCL-backed memory. Raises: - ``ModuleNotFoundError``: If CuPy is not installed. Notes: Uses current CuPy device for allocation. """ if not _cupy_enabled: raise ModuleNotFoundError("CuPy is not installed") size = int(np.prod(shape) * dtype.itemsize) # Allocate buffer on current device (CuPy manages device context) buf = mem_alloc(size) # Important! Disable copy to force allocation to stay in NCCL memory cupy_array = cupy.from_dlpack(buf, copy=False) return cupy_array.view(dtype).reshape(shape, order=order) def empty( shape: int | tuple[int, ...], dtype: str | np.dtype | cupy.dtype | type = float, order: Literal["C", "F"] = "C", ) -> cupy.ndarray: """ Creates an uninitialized CuPy array backed by NCCL-allocated memory. Returns an array filled with uninitialized data using NCCL's memory allocator. This provides a CuPy-compatible interface while using NCCL's memory allocator for efficient GPU memory management in distributed scenarios. Args: - shape (int | tuple[int, ...]): Dimensionalities of the array. - dtype (str | np.dtype | cupy.dtype | type, optional): Data type specifier. Defaults to float. - order (Literal['C', 'F'], optional): Row-major (C-style) or column-major (Fortran-style) order. Only 'C' and 'F' are supported. Defaults to 'C'. Returns: ``cupy.ndarray``: An uninitialized CuPy array backed by NCCL-allocated memory. Raises: - ``NcclInvalid``: If order is not 'C' or 'F'. - ``ModuleNotFoundError``: If CuPy is not installed. Notes: - Unlike ``cupy.empty()``, this allocates memory using NCCL's memory allocator. - For zero-copy optimization, register using ``Communicator.register_buffer()`` or ``Communicator.register_window()``. - Memory is automatically freed when the array is garbage collected. No explicit free call is required. """ if not _cupy_enabled: raise ModuleNotFoundError("CuPy is not installed") # Validate order parameter if order not in ("C", "F"): raise NcclInvalid( f"Invalid memory order: must be 'C' or 'F', got '{order}'. " f"NCCL arrays support row-major (C) and column-major (F) layouts only." ) # Parse shape if isinstance(shape, int): shape = (shape,) else: shape = tuple(shape) # Parse dtype np_dtype = np.dtype(dtype) # Allocate NCCL-backed array on current device view = _allocate_nccl_array(shape, np_dtype, order) return view def resolve_array(array: cupy.ndarray) -> tuple[int, int, NcclDataType, int]: """ Resolves a CuPy array to a tuple of (ptr, count, dtype, device_id). Args: - array (cupy.ndarray): CuPy array to resolve. Returns: ``tuple[int, int, NcclDataType, int]``: Tuple of (ptr, count, dtype, device_id). """ if not _cupy_enabled: raise ModuleNotFoundError("CuPy is not installed") if not isinstance(array, cupy.ndarray): raise NcclInvalid(f"array must be a CuPy array, got {type(array).__name__}") return (array.data.ptr, array.size, _to_nccl_dtype(array.dtype), array.device.id) nccl-2.29.7-1/bindings/nccl4py/nccl/core/interop/torch.py000066400000000000000000000224041515037102200230570ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ PyTorch interoperability for NCCL4Py. This module provides utilities for creating PyTorch tensors backed by NCCL-allocated memory, enabling zero-copy integration between NCCL operations and PyTorch workflows. """ from __future__ import annotations import numpy as np from typing import Literal from nccl.core.buffer import mem_alloc from nccl.core.typing import ( NcclInvalid, NcclDataType, FLOAT16, FLOAT32, FLOAT64, INT8, INT32, INT64, UINT8, BFLOAT16, FLOAT8E4M3, FLOAT8E5M2, ) __all__ = ["empty", "resolve_tensor"] try: import torch _torch_enabled = True except ImportError: _torch_enabled = False def _to_nccl_dtype(torch_dtype) -> NcclDataType: """ Converts PyTorch dtype to NcclDataType. Args: - torch_dtype (torch.dtype): PyTorch data type. Returns: ``NcclDataType``: Corresponding NCCL data type (global constant). Raises: - ``ModuleNotFoundError``: If PyTorch is not installed. - ``NcclInvalid``: If torch_dtype has no NCCL equivalent. """ if not _torch_enabled: raise ModuleNotFoundError("PyTorch is not installed") _torch_to_nccl = { # Float types (all NCCL-supported) torch.float16: FLOAT16, torch.half: FLOAT16, # Alias torch.float32: FLOAT32, torch.float: FLOAT32, # Alias torch.float64: FLOAT64, torch.double: FLOAT64, # Alias torch.bfloat16: BFLOAT16, # Signed integer types torch.int8: INT8, torch.int32: INT32, torch.int64: INT64, torch.long: INT64, # Alias # Unsigned types torch.uint8: UINT8, torch.bool: UINT8, # Map bool to uint8 for convenience } # Add float8 types if available (PyTorch 2.1+) if hasattr(torch, "float8_e4m3fn"): _torch_to_nccl[torch.float8_e4m3fn] = FLOAT8E4M3 if hasattr(torch, "float8_e5m2"): _torch_to_nccl[torch.float8_e5m2] = FLOAT8E5M2 # Explicitly unsupported PyTorch dtypes (no NCCL equivalent) _unsupported_dtypes = set() # Integer types not in NCCL if hasattr(torch, "int16"): _unsupported_dtypes.add(torch.int16) if hasattr(torch, "short"): _unsupported_dtypes.add(torch.short) # Complex types (NCCL doesn't support) if hasattr(torch, "complex64"): _unsupported_dtypes.add(torch.complex64) _unsupported_dtypes.add(torch.cfloat) if hasattr(torch, "complex128"): _unsupported_dtypes.add(torch.complex128) _unsupported_dtypes.add(torch.cdouble) # Quantized types (NCCL doesn't support) for qtype in ["qint8", "quint8", "qint32", "quint4x2", "quint2x4"]: if hasattr(torch, qtype): _unsupported_dtypes.add(getattr(torch, qtype)) # Check if dtype is explicitly unsupported if torch_dtype in _unsupported_dtypes: raise NcclInvalid( f"Unsupported data type: torch dtype {torch_dtype} has no NCCL equivalent. " f"NCCL does not support complex, int16, or quantized types." ) # Check if dtype is supported if torch_dtype in _torch_to_nccl: return _torch_to_nccl[torch_dtype] else: # Unknown dtype - list what we do support supported_list = [str(dt) for dt in sorted(_torch_to_nccl.keys(), key=str)] raise NcclInvalid( f"Unknown torch dtype {torch_dtype}. " f"Supported types: {', '.join(supported_list[:12])}..." ) def _parse_device(device: torch.device | int | str | None) -> int: """ Parses device specification into device index. Args: - device (torch.device | int | str, optional): Device specification. Returns: ``int``: CUDA device index. Raises: - ``ModuleNotFoundError``: If PyTorch is not installed. - ``NcclInvalid``: If device is not a CUDA device. """ if not _torch_enabled: raise ModuleNotFoundError("PyTorch is not installed") if device is None: # Use current CUDA device return torch.cuda.current_device() elif isinstance(device, torch.device): if device.type != "cuda": raise NcclInvalid(f"NCCL tensors must be on CUDA device, got {device}") return device.index if device.index is not None else torch.cuda.current_device() elif isinstance(device, str): device_obj = torch.device(device) if device_obj.type != "cuda": raise NcclInvalid(f"NCCL tensors must be on CUDA device, got {device}") return device_obj.index if device_obj.index is not None else torch.cuda.current_device() else: # device is int, use it directly return device def _allocate_nccl_tensor( shape: tuple[int, ...], dtype: torch.dtype, device: int, morder: str ) -> torch.Tensor: """ Allocates NCCL-backed tensor with specified parameters. Args: - shape (tuple[int, ...]): Shape of tensor. - dtype (torch.dtype): Data type. - device (int): CUDA device index. - morder (str): Memory order ("C" or "F"). Returns: ``torch.Tensor``: Allocated tensor with NCCL-backed memory. Raises: - ``ModuleNotFoundError``: If PyTorch is not installed. """ if not _torch_enabled: raise ModuleNotFoundError("PyTorch is not installed") # Get itemsize for the dtype itemsize = torch.tensor([], dtype=dtype).element_size() size_bytes = int(np.prod(shape) * itemsize) # Allocate buffer on the specified device buf = mem_alloc(size_bytes, device=device) # Create tensor via DLPack torch_tensor = torch.from_dlpack(buf) # type: ignore[attr-defined] view = torch_tensor.view(dtype).view(shape) # Handle Fortran order if requested if morder == "F": # Compute Fortran-style (column-major) strides if len(shape) > 0: strides = [1] for dim in shape[:-1]: strides.append(strides[-1] * dim) view = view.as_strided(size=shape, stride=strides) return view def empty( *size, dtype: torch.dtype | None = None, device: torch.device | int | str | None = None, morder: Literal["C", "F"] = "C", ) -> torch.Tensor: """ Creates an uninitialized PyTorch tensor backed by NCCL-allocated memory. Returns a tensor filled with uninitialized data using NCCL's memory allocator. This provides a PyTorch-compatible interface while using NCCL's memory allocator for efficient GPU memory management in distributed scenarios. Args: - *size (int...): A sequence of integers defining the shape of the output tensor. Can be a variable number of arguments or a collection like a list or tuple. - dtype (torch.dtype, optional): The desired data type of returned tensor. If None, uses global default (see ``torch.set_default_dtype()``). Defaults to None. - device (torch.device | int | str, optional): The device of the constructed tensor. If None, uses the current CUDA device. Defaults to None. - morder (Literal["C", "F"], optional): Memory layout - "C" for row-major (C-style), "F" for column-major (Fortran-style). Defaults to "C". Returns: ``torch.Tensor``: An uninitialized PyTorch tensor backed by NCCL-allocated memory. Raises: - ``NcclInvalid``: If morder is invalid (must be "C" or "F"), or device is not a CUDA device. - ``ModuleNotFoundError``: If PyTorch is not installed. Notes: - Unlike ``torch.empty()``, this allocates memory using NCCL's memory allocator. - For zero-copy optimization, register using ``Communicator.register_buffer()`` or ``Communicator.register_window()``. - Memory is automatically freed when the tensor is garbage collected. No explicit free call is required. """ if not _torch_enabled: raise ModuleNotFoundError("PyTorch is not installed") # Validate morder parameter if morder not in ("C", "F"): raise NcclInvalid(f"Invalid memory order: must be 'C' or 'F', got '{morder}'") # Parse size arguments (can be *args or a single tuple/list) if len(size) == 1 and isinstance(size[0], (tuple, list)): shape = tuple(size[0]) else: shape = size # Handle dtype - use PyTorch's default if not specified if dtype is None: dtype = torch.get_default_dtype() # Parse device specification device_idx = _parse_device(device) # Allocate NCCL-backed tensor view = _allocate_nccl_tensor(shape, dtype, device_idx, morder) return view def resolve_tensor(tensor: torch.Tensor) -> tuple[int, int, NcclDataType, int]: """ Resolves a PyTorch tensor to a tuple of (ptr, count, dtype, device_id). Args: - tensor (torch.Tensor): PyTorch tensor to resolve. Returns: ``tuple[int, int, NcclDataType, int]``: Tuple of (ptr, count, dtype, device_id). """ if not _torch_enabled: raise ModuleNotFoundError("PyTorch is not installed") if not isinstance(tensor, torch.Tensor): raise NcclInvalid(f"tensor must be a PyTorch tensor, got {type(tensor).__name__}") return (tensor.data_ptr(), tensor.numel(), _to_nccl_dtype(tensor.dtype), tensor.device.index) nccl-2.29.7-1/bindings/nccl4py/nccl/core/memory.py000066400000000000000000000103301515037102200215630ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ NCCL-backed memory resource management. This module provides NcclMemoryResource, a MemoryResource implementation that uses NCCL's memory allocation functions to allocate device memory optimized for NCCL operations. Memory resources are thread-local and device-specific. """ from __future__ import annotations import threading from cuda.core import Buffer, Device, MemoryResource, Stream from nccl import bindings as _nccl_bindings from nccl.core.cuda import CudaDeviceContext try: from cuda.core import DevicePointerT except ImportError: try: from cuda.core._memory._buffer import DevicePointerT except ImportError: DevicePointerT = int __all__ = ["NcclMemoryResource", "get_memory_resource"] _mr_instances = threading.local() class NcclMemoryResource(MemoryResource): """ NCCL-backed device memory resource for Buffer allocations. This memory resource uses NCCL's memory allocation functions to allocate device memory that is optimized for NCCL operations. The resource is tied to a specific CUDA device and ensures proper device context during allocation. """ def __init__(self, device_id: int) -> None: """ Initializes NCCL memory resource for a specific device. Args: - device_id (int): CUDA device ID to allocate memory on. """ self.device = Device(device_id) def allocate(self, size: int, stream: Stream | None = None) -> Buffer: """ Allocates device memory using NCCL. Args: - size (int): Number of bytes to allocate. - stream (Stream, optional): CUDA stream for allocation (currently unused). Defaults to None. Returns: ``Buffer``: Buffer object wrapping the allocated memory. Raises: - ``NCCLError``: If allocation fails. """ with CudaDeviceContext(self.device): ptr = _nccl_bindings.mem_alloc(size) # mem_alloc raises NCCLError if ptr is 0 buf = Buffer.from_handle(ptr=ptr, size=size, mr=self) return buf def deallocate(self, ptr: DevicePointerT, size: int, stream: Stream | None = None) -> None: """ Deallocates device memory using NCCL. Args: - ptr (DevicePointerT): Device pointer to deallocate. - size (int): Size of the allocation (for compatibility). - stream (Stream, optional): CUDA stream for deallocation (currently unused). Defaults to None. """ with CudaDeviceContext(self.device): _nccl_bindings.mem_free(int(ptr)) @property def device_id(self) -> int: """ CUDA device ID this resource is associated with. Returns: ``int``: Device ID. """ return self.device.device_id @property def is_device_accessible(self) -> bool: """ Whether this memory is accessible from device code. Returns: ``bool``: Always True for NCCL-allocated memory. """ return True @property def is_host_accessible(self) -> bool: """ Whether this memory is accessible from host code. Returns: ``bool``: Always False for device memory. """ return False def get_memory_resource(device_id: int) -> NcclMemoryResource: """ Gets a thread-local ``NcclMemoryResource`` for the given device. This function provides a cached, thread-local memory resource for each device. Multiple calls with the same device will return the same resource instance, ensuring efficient memory management. Args: - device_id (int): CUDA device ID to get memory resource for. Returns: ``NcclMemoryResource``: Thread-local NcclMemoryResource for the device. """ if not hasattr(_mr_instances, "memory_resources"): _mr_instances.memory_resources = {} if device_id not in _mr_instances.memory_resources: _mr_instances.memory_resources[device_id] = NcclMemoryResource(device_id) return _mr_instances.memory_resources[device_id] nccl-2.29.7-1/bindings/nccl4py/nccl/core/resources.py000066400000000000000000000304221515037102200222710ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ Communicator-owned resource management. This module provides resource classes for NCCL communicator-owned objects including registered buffers for zero-copy communication, registered windows for RMA operations, and custom reduction operators. All resources are automatically cleaned up when the owning communicator is destroyed or aborted. """ from __future__ import annotations from abc import ABC, abstractmethod from nccl import bindings as _nccl_bindings from nccl.core.constants import WindowFlag from nccl.core.typing import NcclDataType, NcclInvalid __all__ = [ "RegisteredBufferHandle", "RegisteredWindowHandle", "CustomRedOp", ] class CommResource(ABC): """ Abstract base class for NCCL communicator-owned resources. Resources are tied to a specific communicator and must be explicitly deallocated before the communicator is destroyed. """ def __init__(self, comm_ptr: int): """ Initializes resource with communicator pointer. Args: - comm_ptr (int): Raw NCCL communicator pointer. Raises: - ``NcclInvalid``: If comm_ptr is 0 (invalid communicator). """ if comm_ptr == 0: raise NcclInvalid( "Invalid communicator: cannot create resource with communicator ptr=0" ) self._comm_ptr = comm_ptr self._closed = False @abstractmethod def _allocate(self) -> None: """ Allocates the underlying NCCL resource. Notes: This method is called during initialization and must be implemented by subclasses. """ pass @abstractmethod def _deallocate(self) -> None: """ Deallocates the underlying NCCL resource. Notes: This method is called during close() and must be implemented by subclasses. """ pass def close(self) -> None: """ Explicitly deallocates the resource. Notes: This method is idempotent and safe to call multiple times. """ if self._closed: return self._closed = True self._deallocate() def _check_valid(self) -> None: """ Checks if resource is valid and raises if closed. Raises: - ``RuntimeError``: If resource has been closed. """ if self._closed: raise RuntimeError(f"{self.__class__.__name__} has been closed and is no longer valid") @property def is_valid(self) -> bool: """ Checks if resource is still valid (not closed). Returns: ``bool``: True if valid, False if closed. """ return not self._closed class RegisteredBufferHandle(CommResource): """ NCCL registered buffer handle for zero-copy optimized communication. Registers a user buffer with the communicator to enable performance optimizations in NCCL operations. The registration handle can be closed manually or is automatically closed when the communicator is destroyed or aborted. Attributes: handle (int): Registration handle for NCCL operations. size (int): Size of registered buffer in bytes. is_valid (bool): Whether the registration is still valid. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommregister """ def __init__(self, comm_ptr: int, buffer_ptr: int, size: int): """ Creates and registers a buffer with NCCL. Args: - comm_ptr (int): NCCL communicator raw pointer. - buffer_ptr (int): Device pointer to the buffer. - size (int): Size of buffer in bytes. Raises: - ``NcclInvalid``: If comm_ptr is 0 (invalid communicator). """ self._buffer_ptr = buffer_ptr self._size = size self._handle: int | None = None super().__init__(comm_ptr) self._allocate() def _allocate(self) -> None: """ Registers buffer with NCCL for zero-copy communication. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommregister """ self._handle = _nccl_bindings.comm_register(self._comm_ptr, self._buffer_ptr, self._size) def _deallocate(self) -> None: """ Deregisters buffer from NCCL. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommderegister """ if self._handle is not None: _nccl_bindings.comm_deregister(self._comm_ptr, self._handle) self._handle = None @property def handle(self) -> int: """ Registration handle for NCCL operations. Returns: ``int``: The registration handle. Raises: - ``RuntimeError``: If buffer has been deregistered or handle is invalid. """ self._check_valid() if self._handle is None: raise RuntimeError("Buffer registration handle is invalid") return self._handle @property def size(self) -> int: """ Size of the registered buffer in bytes. Returns: ``int``: Buffer size in bytes. """ return self._size def __repr__(self) -> str: if not self.is_valid: return "" return f"" class RegisteredWindowHandle(CommResource): """ NCCL registered window handle for Remote Memory Access (RMA) operations. Registers a memory window with the communicator for one-sided communication patterns. The window handle can be closed manually or is automatically closed when the communicator is destroyed or aborted. Attributes: handle (int): Window handle for NCCL operations. size (int): Size of registered window in bytes. is_valid (bool): Whether the window registration is still valid. Notes: This is a collective operation. All ranks must call register_window() with equal buffer sizes. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommwindowregister """ def __init__(self, comm_ptr: int, buffer_ptr: int, size: int, flags: WindowFlag | None = None): """ Creates and registers a memory window with NCCL. Args: - comm_ptr (int): NCCL communicator raw pointer. - buffer_ptr (int): Device pointer to the buffer. - size (int): Size of window in bytes. - flags (WindowFlag, optional): Window registration flags to control behavior. Defaults to None. Raises: - ``NcclInvalid``: If comm_ptr is 0 (invalid communicator). """ self._buffer_ptr = buffer_ptr self._size = size self._flags = flags if flags is not None else WindowFlag.Default self._handle: int | None = None super().__init__(comm_ptr) self._allocate() def _allocate(self) -> None: """ Collectively registers window with NCCL. Notes: This is a collective operation called during initialization. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommwindowregister """ self._handle = _nccl_bindings.comm_window_register( self._comm_ptr, self._buffer_ptr, self._size, self._flags.value ) def _deallocate(self) -> None: """ Deregisters window from NCCL. Notes: Deregistration is local to the rank. Caller must ensure the buffer is not being accessed by any NCCL operation. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/comms.html#ncclcommwindowderegister """ if self._handle is not None: _nccl_bindings.comm_window_deregister(self._comm_ptr, self._handle) self._handle = None @property def handle(self) -> int: """ Window handle for NCCL operations. Returns: ``int``: The window handle. Raises: - ``RuntimeError``: If window has been deregistered or handle is invalid. """ self._check_valid() if self._handle is None: raise RuntimeError("Window registration handle is invalid") return self._handle @property def size(self) -> int: """ Size of the registered window in bytes. Returns: ``int``: Window size in bytes. """ return self._size def __repr__(self) -> str: if not self.is_valid: return "" return f"" class CustomRedOp(CommResource): """ NCCL user-defined custom reduction operator. Creates a custom reduction operator (e.g., PreMulSum) for collective operations. The PreMulSum operator performs: output = scalar * sum(inputs), useful for averaging or weighted reductions. The operator can be closed manually or is automatically destroyed when the communicator is destroyed or aborted. Attributes: op (int): Operator handle for use in reduction operations. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/ops.html#ncclredopcreatepremulsum """ def __init__( self, comm_ptr: int, scalar_ptr: int, datatype: NcclDataType, residence: _nccl_bindings.ScalarResidence, ): """ Creates a custom reduction operator. Args: - comm_ptr (int): NCCL communicator raw pointer. - scalar_ptr (int): Pointer to scalar value (host or device memory). - datatype (NcclDataType): NCCL data type of the scalar and reduction. - residence (ScalarResidence): Enum indicating scalar memory location (HostImmediate or Device). Raises: - ``NcclInvalid``: If comm_ptr is 0 (invalid communicator). """ self._scalar_ptr = scalar_ptr self._datatype = datatype self._residence = residence self._op: int | None = None super().__init__(comm_ptr) self._allocate() def _allocate(self) -> None: """ Creates the custom reduction operator in NCCL. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/ops.html#ncclredopcreatepremulsum """ self._op = _nccl_bindings.red_op_create_pre_mul_sum( self._scalar_ptr, int(self._datatype), int(self._residence), self._comm_ptr ) def _deallocate(self) -> None: """ Destroys the custom reduction operator in NCCL. See Also: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/ops.html#ncclredopdestroy """ if self._op is not None: _nccl_bindings.red_op_destroy(self._op, self._comm_ptr) self._op = None @property def op(self) -> int: """ Operator handle for use in reduction operations. Returns: ``int``: The custom reduction operator handle. Raises: - ``RuntimeError``: If operator has been destroyed or is invalid. """ self._check_valid() if self._op is None: raise RuntimeError("RedOp is invalid") return self._op def __int__(self) -> int: """ Returns operator handle as int for direct use in collective operations. Returns: ``int``: The custom reduction operator handle. Raises: - ``RuntimeError``: If operator has been destroyed or is invalid. """ self._check_valid() if self._op is None: raise RuntimeError("RedOp is invalid") return self._op def __repr__(self) -> str: if not self.is_valid: return "" return f"" nccl-2.29.7-1/bindings/nccl4py/nccl/core/typing.py000066400000000000000000000257561515037102200216070ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ Type definitions, protocols, and type aliases for NCCL4Py. This module defines all type specifications used throughout NCCL4Py including data types, reduction operators, buffer specifications, and protocol classes for DLPack and CUDA Array Interface compatibility. It provides type-safe interfaces for NCCL operations with comprehensive type hints. """ from __future__ import annotations from typing import Any, Protocol, runtime_checkable, TypeAlias import numpy as _np from cuda.core import Buffer, Device, Stream try: from cuda.core import IsStreamT except ImportError: try: from cuda.core._stream import IsStreamT except ImportError: # ---- Fallback definition ---- @runtime_checkable class IsStreamT(Protocol): def __cuda_stream__(self) -> tuple[int, int]: """ Fallback Protocol for CUDA stream objects. Returns: (version: int, cudaStream_t address: int) """ ... from nccl import bindings as _nccl_bindings from nccl.bindings import DataType, RedOp __all__ = [ "NcclDataType", "NcclRedOp", "NcclBufferSpec", "NcclScalarSpec", "NcclDeviceSpec", "NcclStreamSpec", "NcclInvalid", # Data type constants "INT8", "CHAR", "UINT8", "INT32", "INT", "UINT32", "INT64", "UINT64", "FLOAT16", "HALF", "FLOAT32", "FLOAT", "FLOAT64", "DOUBLE", "BFLOAT16", "FLOAT8E4M3", "FLOAT8E5M2", # Reduction op constants "SUM", "PROD", "MAX", "MIN", "AVG", ] _PyCapsule: TypeAlias = object _DeviceType: TypeAlias = int _DeviceID: TypeAlias = int class NcclInvalid(Exception): def __init__(self, msg): self.msg = msg def __repr__(self): return f"" class NcclDataType: def __init__(self, datatype: int | _np.dtype): if isinstance(datatype, _np.dtype): # First try name-based mapping for ml-dtypes (has higher priority) _name_mapping = { "bfloat16": DataType.Bfloat16, "float8_e4m3fn": DataType.Float8e4m3, "float8_e5m2": DataType.Float8e5m2, } if datatype.name in _name_mapping: self._datatype = _name_mapping[datatype.name] else: # Fall back to kind+size mapping for standard NumPy types _mapping = { ("f", 2): DataType.Float16, ("f", 4): DataType.Float32, ("f", 8): DataType.Float64, ("i", 1): DataType.Int8, ("i", 4): DataType.Int32, ("i", 8): DataType.Int64, ("u", 1): DataType.Uint8, ("u", 4): DataType.Uint32, ("u", 8): DataType.Uint64, } kind_size = (datatype.kind, datatype.itemsize) if kind_size in _mapping: self._datatype = _mapping[kind_size] else: raise NcclInvalid( f"Unsupported data type: numpy dtype {datatype} (kind={datatype.kind}, " f"itemsize={datatype.itemsize}, name={datatype.name}) has no NCCL equivalent" ) else: try: self._datatype = DataType(int(datatype)) except Exception: raise NcclInvalid( f"Unsupported data type: NCCL datatype value {datatype} is invalid" ) def __int__(self) -> int: return int(self._datatype) def __str__(self) -> str: return self._datatype.name def __repr__(self) -> str: return f"NcclDataType({self._datatype.name})" def __eq__(self, other: object) -> bool: """ Compares two NcclDataType instances for equality. Returns: ``bool``: True if data types are equal. """ if not isinstance(other, NcclDataType): return NotImplemented return self._datatype == other._datatype def __hash__(self) -> int: """ Makes NcclDataType hashable. Returns: ``int``: Hash value. """ return hash(self._datatype) @property def value(self) -> int: """ Integer value of the NCCL data type. Returns: ``int``: Data type value. """ return int(self._datatype) @property def name(self) -> str: """ Name of the NCCL data type. Returns: ``str``: Data type name (e.g., "Float32", "Int64"). """ return self._datatype.name @property def itemsize(self) -> int: """ Size in bytes of this data type. Returns: ``int``: Byte size (1, 2, 4, or 8). Raises: - ``NcclInvalid``: If data type is unsupported. """ if self._datatype in [ DataType.Int8, DataType.Char, DataType.Uint8, DataType.Float8e4m3, DataType.Float8e5m2, ]: return 1 elif self._datatype in [DataType.Float16, DataType.Half, DataType.Bfloat16]: return 2 elif self._datatype in [ DataType.Int32, DataType.Int, DataType.Uint32, DataType.Float32, DataType.Float, ]: return 4 elif self._datatype in [DataType.Int64, DataType.Uint64, DataType.Float64, DataType.Double]: return 8 else: raise NcclInvalid( f"Unsupported data type: NCCL datatype {self._datatype} has no byte size mapping" ) @property def numpy_dtype(self) -> _np.dtype: """ NumPy dtype corresponding to this NCCL data type. Returns: ``np.dtype``: Equivalent NumPy data type. Raises: - ``NcclInvalid``: If data type is unsupported. """ # Mapping from NCCL DataType to numpy dtype string _dtype_to_numpy = { DataType.Int8: "int8", DataType.Char: "int8", DataType.Uint8: "uint8", DataType.Int32: "int32", DataType.Int: "int32", DataType.Uint32: "uint32", DataType.Int64: "int64", DataType.Uint64: "uint64", DataType.Float16: "float16", DataType.Half: "float16", DataType.Float32: "float32", DataType.Float: "float32", DataType.Float64: "float64", DataType.Double: "float64", # ml-dtypes DataType.Bfloat16: "bfloat16", DataType.Float8e4m3: "float8_e4m3fn", DataType.Float8e5m2: "float8_e5m2", } if self._datatype not in _dtype_to_numpy: raise NcclInvalid( f"Unsupported data type: NCCL datatype {self._datatype} has no numpy dtype mapping" ) dtype_str = _dtype_to_numpy[self._datatype] # For ml-dtypes, provide helpful error if package is missing (optional dependency) if dtype_str in ("bfloat16", "float8_e4m3fn", "float8_e5m2"): try: return _np.dtype(dtype_str) except TypeError as e: raise NcclInvalid( f"Cannot create numpy dtype '{dtype_str}': ml-dtypes package is not installed. " f"ml-dtypes is required for bfloat16 and float8 support. " f"Install with: pip install ml-dtypes" ) from e else: return _np.dtype(dtype_str) INT8 = NcclDataType(DataType.Int8) CHAR = NcclDataType(DataType.Char) UINT8 = NcclDataType(DataType.Uint8) INT32 = NcclDataType(DataType.Int32) INT = NcclDataType(DataType.Int) UINT32 = NcclDataType(DataType.Uint32) INT64 = NcclDataType(DataType.Int64) UINT64 = NcclDataType(DataType.Uint64) FLOAT16 = NcclDataType(DataType.Float16) HALF = NcclDataType(DataType.Half) FLOAT32 = NcclDataType(DataType.Float32) FLOAT = NcclDataType(DataType.Float) FLOAT64 = NcclDataType(DataType.Float64) DOUBLE = NcclDataType(DataType.Double) BFLOAT16 = NcclDataType(DataType.Bfloat16) FLOAT8E4M3 = NcclDataType(DataType.Float8e4m3) FLOAT8E5M2 = NcclDataType(DataType.Float8e5m2) class NcclRedOp: """ NCCL reduction operator wrapper with validation. Wraps NCCL reduction operator values and validates them against the built-in operators (Sum, Prod, Max, Min, Avg). """ def __init__(self, value: int): """ Initializes NcclRedOp with validation. Args: - value (int): Integer value of the reduction operator. Raises: - ``NcclInvalid``: If the reduction operator value is invalid. """ try: _ro = getattr(_nccl_bindings, "RedOp") # Access a few known attributes to ensure class exists _ = getattr(_ro, "Sum") except Exception: raise NcclInvalid("NCCL bindings error: NcclRedOp bindings not found") # Validate that the value corresponds to a valid reduction operator try: self._redop_value = _ro(value) self._redop_name = self._redop_value.name except Exception: raise NcclInvalid( f"Invalid reduction operator: value {value} is not a valid NCCL reduction operator" ) def __int__(self) -> int: return int(self._redop_value) def __str__(self) -> str: return self._redop_name def __repr__(self) -> str: return f"" @property def value(self) -> int: """ Integer value of the reduction operator. Returns: ``int``: Operator value. """ return int(self._redop_value) @property def name(self) -> str: """ Gets the name of the reduction operator. Returns: ``str``: Operator name (e.g., "Sum", "Max"). """ return self._redop_name SUM = NcclRedOp(RedOp.Sum) PROD = NcclRedOp(RedOp.Prod) MAX = NcclRedOp(RedOp.Max) MIN = NcclRedOp(RedOp.Min) AVG = NcclRedOp(RedOp.Avg) class SupportsDLPack(Protocol): def __dlpack__(self, /, *, stream: Any | None = None) -> _PyCapsule: ... def __dlpack_device__(self) -> tuple[_DeviceType, _DeviceID]: ... class SupportsCAI(Protocol): @property def __cuda_array_interface__(self) -> dict[str, Any]: ... NcclSupportedBuffer: TypeAlias = Buffer | SupportsDLPack | SupportsCAI NcclBufferSpec: TypeAlias = NcclSupportedBuffer NcclScalarSpec: TypeAlias = ( int | float | _np.ndarray # NumPy array for host scalars | NcclSupportedBuffer # Device buffer (Buffer, DLPack, or CUDA Array Interface) ) NcclDeviceSpec: TypeAlias = Device | int NcclStreamSpec: TypeAlias = Stream | IsStreamT | int nccl-2.29.7-1/bindings/nccl4py/nccl/core/utils.py000066400000000000000000000147101515037102200214210ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ Utility functions and classes for NCCL operations. This module provides version information, unique identifiers for communicator initialization, and error string utilities for NCCL operations. """ from __future__ import annotations import ctypes as _ctypes import numpy as _np from packaging.version import Version as _Version from nccl._version import __version__ from nccl import bindings as _nccl_bindings __all__ = ["Version", "get_version", "UniqueId", "get_unique_id", "get_error_string"] _version_cache = None class Version: """ Version information for NCCL4Py and NCCL library. Attributes: nccl_version (Version): NCCL library version. nccl4py_version (Version): NCCL4Py package version. """ def __init__(self, nccl_version: int) -> None: """ Initializes Version object from NCCL version integer. Args: - nccl_version (int): NCCL version as an integer. """ v = nccl_version if v >= 10000: major = v // 10000 minor = (v % 10000) // 100 patch = v % 100 else: major = v // 1000 minor = (v % 1000) // 100 patch = v % 100 self.nccl_version = _Version(f"{major}.{minor}.{patch}") self.nccl4py_version = _Version(__version__) def __repr__(self) -> str: return f""" Versions: NCCL4Py version: {self.nccl4py_version} NCCL Library version: {self.nccl_version} """ def get_version() -> Version: """ Gets the version information for NCCL and NCCL4Py. Returns: ``Version``: Version object containing NCCL and NCCL4Py version information. """ global _version_cache if _version_cache is None: _version_cache = Version(int(_nccl_bindings.get_version())) return _version_cache class UniqueId: """ NCCL unique identifier for communicator initialization. A UniqueId is used to coordinate communicator initialization across multiple ranks. All ranks must use the same UniqueId to form a communicator. Attributes: ptr (int): Pointer to the internal NCCL unique ID structure. as_ndarray (np.ndarray): NumPy array representation of the unique ID. as_bytes (bytes): Bytes representation of the unique ID. """ def __init__(self) -> None: """ Initializes an empty UniqueId. Notes: Use ``get_unique_id()`` to generate a valid unique ID for communicator initialization. """ self._internal: _nccl_bindings.UniqueId = _nccl_bindings.UniqueId() def __repr__(self) -> str: """ Returns truncated bytes representation of UniqueId. Returns: ``str``: Hex representation showing first and last 8 bytes. """ # Show first 8 and last 8 bytes in hex bytes_data = self.as_bytes if len(bytes_data) <= 32: hex_str = bytes_data.hex() else: hex_str = bytes_data[:8].hex() + "..." + bytes_data[-8:].hex() return f"" @staticmethod def from_bytes(b: bytes | bytearray | memoryview) -> UniqueId: """ Creates a UniqueId from bytes. Args: - b (bytes | bytearray | memoryview): Bytes representation of a UniqueId. Returns: ``UniqueId``: Reconstructed UniqueId object. Raises: - ``TypeError``: If b is not a bytes-like object. - ``ValueError``: If b has incorrect length. """ try: buf = b if isinstance(b, (bytes, bytearray)) else b.tobytes() except Exception as e: raise TypeError("'b' must be a bytes-like object") from e expected = int(_nccl_bindings.unique_id_dtype.itemsize) if len(buf) != expected: raise ValueError(f"unique id must be {expected} bytes, got {len(buf)}") arr = _np.frombuffer(buf, dtype=_nccl_bindings.unique_id_dtype, count=1) uid = UniqueId.__new__(UniqueId) uid._internal = _nccl_bindings.UniqueId.from_data(arr) return uid @property def ptr(self) -> int: """ Pointer to the internal NCCL unique ID structure. Returns: ``int``: Internal structure pointer. """ return self._internal.ptr @property def as_ndarray(self) -> _np.ndarray: """ NumPy array representation of the unique ID. Returns: ``np.ndarray``: Array containing the unique ID data. """ size = int(_nccl_bindings.unique_id_dtype.itemsize) # Create a live, zero-copy view into the backing UniqueId storage. # Keep the Cython object alive via an attribute on the ctypes buffer. buf = (_ctypes.c_char * size).from_address(int(self.ptr)) # Use setattr to avoid relying on ctypes Array stubs having this attribute. setattr(buf, "_keepalive", self._internal) arr = _np.ndarray((1,), dtype=_nccl_bindings.unique_id_dtype, buffer=buf) return arr.view(_np.recarray) @property def as_bytes(self) -> bytes: """ Bytes representation of the unique ID. Returns: ``bytes``: Unique ID as bytes (for serialization/broadcast). """ size = int(_nccl_bindings.unique_id_dtype.itemsize) return _ctypes.string_at(int(self.ptr), size) def get_unique_id(empty: bool = False) -> UniqueId: """ Generates a new NCCL unique identifier for communicator initialization. Args: - empty (bool, optional): If True, return an empty UniqueId without calling NCCL. Defaults to False. Returns: ``UniqueId``: A new unique identifier to be shared across ranks. Notes: This should be called by one rank (typically rank 0) and the resulting UniqueId should be broadcast to all other ranks. """ uid = UniqueId() if empty: return uid _nccl_bindings.get_unique_id(uid.ptr) return uid def get_error_string(nccl_result: _nccl_bindings.Result | int) -> str: """ Gets the error string for an NCCL result code. Args: - nccl_result (Result | int): NCCL result code. Returns: ``str``: Human-readable error message corresponding to the result code. """ return _nccl_bindings.get_error_string(int(nccl_result)) nccl-2.29.7-1/bindings/nccl4py/pyproject.toml000066400000000000000000000053541515037102200207600ustar00rootroot00000000000000[project] name = "nccl4py" dynamic = ["version"] requires-python = ">=3.10" description = "NCCL4Py: Python bindings for NCCL" readme = {file = "DESCRIPTION.rst", content-type = "text/x-rst"} license = "Apache-2.0" license-files = ["LICENSE.txt"] authors = [ { name = "NVIDIA Corporation" } ] classifiers = [ "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Intended Audience :: End Users/Desktop", "Natural Language :: English", "Operating System :: POSIX :: Linux", "Topic :: Education", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries", "Programming Language :: Cython", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Environment :: GPU :: NVIDIA CUDA", "Environment :: GPU :: NVIDIA CUDA :: 12", "Environment :: GPU :: NVIDIA CUDA :: 13", ] dependencies = [ "packaging", "numpy", "cuda.core>=0.5.0,<1.0", "cuda-pathfinder~=1.1", ] [project.optional-dependencies] cu12 = [ "nvidia-nccl-cu12", "cuda-bindings~=12.0", ] cu13 = [ "nvidia-nccl-cu13", "cuda-bindings~=13.0", ] [dependency-groups] test = [ "pytest", "pytest-cov", "pytest-mpi", "mpi4py", "ml-dtypes", ] [project.urls] homepage = "https://developer.nvidia.com/nccl" documentation = "https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/index.html" repository = "https://github.com/NVIDIA/nccl" issues = "https://github.com/NVIDIA/nccl/issues" [build-system] requires = [ "setuptools>=77.0.3", "Cython>=0.29.24" ] build-backend = "setuptools.build_meta" [tool.setuptools] include-package-data = false [tool.setuptools.packages.find] include = ["nccl*"] [tool.setuptools.package-data] "nccl.bindings" = ["cynccl.pxd"] [tool.setuptools.dynamic] version = { attr = "nccl._version.__version__" } [tool.ruff] line-length = 100 [tool.pyright] include = ["nccl", "tests"] exclude = ["build", "dist", "*.egg-info"] pythonVersion = "3.10" typeCheckingMode = "basic" [tool.pytest.ini_options] minversion = "6.0" testpaths = ["tests"] python_files = ["test_*.py"] [tool.cibuildwheel] build = "cp310-* cp311-* cp312-* cp313-* cp314-*" skip = "*-musllinux_*" build-verbosity = 1 [tool.cibuildwheel.linux] archs = ["native"] # Fast local builds; override in CI for multi-arch manylinux-x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" [tool.uv] conflicts = [ [ { extra = "cu12" }, { extra = "cu13" }, ] ] nccl-2.29.7-1/bindings/nccl4py/setup.py000066400000000000000000000057341515037102200175600ustar00rootroot00000000000000import os import re import subprocess from pathlib import Path from Cython.Build import cythonize from setuptools import setup, Extension # Check CUDA_HOME is set and is a valid directory CUDA_HOME = os.environ.get("CUDA_HOME") if not CUDA_HOME: raise SystemExit("Error: CUDA_HOME is not set") cuda_path = Path(CUDA_HOME) if not cuda_path.exists() or not cuda_path.is_dir(): raise SystemExit(f"Error: CUDA_HOME does not exist or is not a directory: {CUDA_HOME}") CUDA_INC = str(cuda_path / "include") ext_modules = [ "nccl.bindings.nccl" ] def calculate_modules(module: str): module_parts = module.split(".") # nccl.bindings.nccl -> nccl/bindings/nccl.pyx lowpp_mod = module_parts.copy() lowpp_pyx = os.path.join(*lowpp_mod[:-1], f"{lowpp_mod[-1]}.pyx") lowpp_mod = ".".join(lowpp_mod) lowpp_ext = Extension( lowpp_mod, sources=[lowpp_pyx], include_dirs=[CUDA_INC], language="c++", extra_compile_args=["-std=c++14"], libraries=["dl"], ) # cy variant: nccl.bindings.nccl -> nccl/bindings/cynccl.pyx cy_mod = module_parts.copy() cy_mod[-1] = f"cy{cy_mod[-1]}" cy_mod_pyx = os.path.join(*cy_mod[:-1], f"{cy_mod[-1]}.pyx") cy_mod = ".".join(cy_mod) cy_ext = Extension( cy_mod, sources=[cy_mod_pyx], include_dirs=[CUDA_INC], language="c++", extra_compile_args=["-std=c++14"], libraries=["dl"], ) # internal variant: source is nccl_linux.pyx, but published module name is nccl.bindings._internal.nccl inter_mod = module_parts.copy() inter_mod.insert(-1, "_internal") inter_mod_pyx = os.path.join(*inter_mod[:-1], f"{inter_mod[-1]}_linux.pyx") inter_mod = ".".join(inter_mod) inter_ext = Extension( inter_mod, sources=[inter_mod_pyx], include_dirs=[CUDA_INC], language="c++", extra_compile_args=["-std=c++14"], libraries=["dl"], ) # internal variant: insert _internal and use utils.pyx inter_utils_mod = module_parts.copy() inter_utils_mod.insert(-1, "_internal") inter_utils_mod[-1] = "utils" inter_utils_mod_pyx = os.path.join(*inter_utils_mod[:-1], f"{inter_utils_mod[-1]}.pyx") inter_utils_mod = ".".join(inter_utils_mod) inter_utils_ext = Extension( inter_utils_mod, sources=[inter_utils_mod_pyx], include_dirs=[CUDA_INC], language="c++", extra_compile_args=["-std=c++14"], libraries=["dl"], ) return lowpp_ext, cy_ext, inter_ext, inter_utils_ext # Note: the extension attributes are overwritten in build_extension() ext_modules = [e for ext in ext_modules for e in calculate_modules(ext)] compiler_directives = {"embedsignature": True, "show_performance_hints": True} setup( ext_modules=cythonize( ext_modules, verbose=True, language_level=3, compiler_directives=compiler_directives, ), zip_safe=False, options={"build_ext": {"inplace": False}}, ) nccl-2.29.7-1/bindings/nccl4py/uv.lock000066400000000000000000004431111515037102200173450ustar00rootroot00000000000000version = 1 revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.13'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", "python_full_version < '3.11'", ] conflicts = [[ { package = "nccl4py", extra = "cu12" }, { package = "nccl4py", extra = "cu13" }, ]] [[package]] name = "colorama" version = "0.4.6" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] [[package]] name = "coverage" version = "7.13.4" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/24/56/95b7e30fa389756cb56630faa728da46a27b8c6eb46f9d557c68fff12b65/coverage-7.13.4.tar.gz", hash = "sha256:e5c8f6ed1e61a8b2dcdf31eb0b9bbf0130750ca79c1c49eb898e2ad86f5ccc91", size = 827239, upload-time = "2026-02-09T12:59:03.86Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/44/d4/7827d9ffa34d5d4d752eec907022aa417120936282fc488306f5da08c292/coverage-7.13.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0fc31c787a84f8cd6027eba44010517020e0d18487064cd3d8968941856d1415", size = 219152, upload-time = "2026-02-09T12:56:11.974Z" }, { url = "https://files.pythonhosted.org/packages/35/b0/d69df26607c64043292644dbb9dc54b0856fabaa2cbb1eeee3331cc9e280/coverage-7.13.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a32ebc02a1805adf637fc8dec324b5cdacd2e493515424f70ee33799573d661b", size = 219667, upload-time = "2026-02-09T12:56:13.33Z" }, { url = "https://files.pythonhosted.org/packages/82/a4/c1523f7c9e47b2271dbf8c2a097e7a1f89ef0d66f5840bb59b7e8814157b/coverage-7.13.4-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e24f9156097ff9dc286f2f913df3a7f63c0e333dcafa3c196f2c18b4175ca09a", size = 246425, upload-time = "2026-02-09T12:56:14.552Z" }, { url = "https://files.pythonhosted.org/packages/f8/02/aa7ec01d1a5023c4b680ab7257f9bfde9defe8fdddfe40be096ac19e8177/coverage-7.13.4-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8041b6c5bfdc03257666e9881d33b1abc88daccaf73f7b6340fb7946655cd10f", size = 248229, upload-time = "2026-02-09T12:56:16.31Z" }, { url = "https://files.pythonhosted.org/packages/35/98/85aba0aed5126d896162087ef3f0e789a225697245256fc6181b95f47207/coverage-7.13.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a09cfa6a5862bc2fc6ca7c3def5b2926194a56b8ab78ffcf617d28911123012", size = 250106, upload-time = "2026-02-09T12:56:18.024Z" }, { url = "https://files.pythonhosted.org/packages/96/72/1db59bd67494bc162e3e4cd5fbc7edba2c7026b22f7c8ef1496d58c2b94c/coverage-7.13.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:296f8b0af861d3970c2a4d8c91d48eb4dd4771bcef9baedec6a9b515d7de3def", size = 252021, upload-time = "2026-02-09T12:56:19.272Z" }, { url = "https://files.pythonhosted.org/packages/9d/97/72899c59c7066961de6e3daa142d459d47d104956db43e057e034f015c8a/coverage-7.13.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e101609bcbbfb04605ea1027b10dc3735c094d12d40826a60f897b98b1c30256", size = 247114, upload-time = "2026-02-09T12:56:21.051Z" }, { url = "https://files.pythonhosted.org/packages/39/1f/f1885573b5970235e908da4389176936c8933e86cb316b9620aab1585fa2/coverage-7.13.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:aa3feb8db2e87ff5e6d00d7e1480ae241876286691265657b500886c98f38bda", size = 248143, upload-time = "2026-02-09T12:56:22.585Z" }, { url = "https://files.pythonhosted.org/packages/a8/cf/e80390c5b7480b722fa3e994f8202807799b85bc562aa4f1dde209fbb7be/coverage-7.13.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:4fc7fa81bbaf5a02801b65346c8b3e657f1d93763e58c0abdf7c992addd81a92", size = 246152, upload-time = "2026-02-09T12:56:23.748Z" }, { url = "https://files.pythonhosted.org/packages/44/bf/f89a8350d85572f95412debb0fb9bb4795b1d5b5232bd652923c759e787b/coverage-7.13.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:33901f604424145c6e9c2398684b92e176c0b12df77d52db81c20abd48c3794c", size = 249959, upload-time = "2026-02-09T12:56:25.209Z" }, { url = "https://files.pythonhosted.org/packages/f7/6e/612a02aece8178c818df273e8d1642190c4875402ca2ba74514394b27aba/coverage-7.13.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:bb28c0f2cf2782508a40cec377935829d5fcc3ad9a3681375af4e84eb34b6b58", size = 246416, upload-time = "2026-02-09T12:56:26.475Z" }, { url = "https://files.pythonhosted.org/packages/cb/98/b5afc39af67c2fa6786b03c3a7091fc300947387ce8914b096db8a73d67a/coverage-7.13.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9d107aff57a83222ddbd8d9ee705ede2af2cc926608b57abed8ef96b50b7e8f9", size = 247025, upload-time = "2026-02-09T12:56:27.727Z" }, { url = "https://files.pythonhosted.org/packages/51/30/2bba8ef0682d5bd210c38fe497e12a06c9f8d663f7025e9f5c2c31ce847d/coverage-7.13.4-cp310-cp310-win32.whl", hash = "sha256:a6f94a7d00eb18f1b6d403c91a88fd58cfc92d4b16080dfdb774afc8294469bf", size = 221758, upload-time = "2026-02-09T12:56:29.051Z" }, { url = "https://files.pythonhosted.org/packages/78/13/331f94934cf6c092b8ea59ff868eb587bc8fe0893f02c55bc6c0183a192e/coverage-7.13.4-cp310-cp310-win_amd64.whl", hash = "sha256:2cb0f1e000ebc419632bbe04366a8990b6e32c4e0b51543a6484ffe15eaeda95", size = 222693, upload-time = "2026-02-09T12:56:30.366Z" }, { url = "https://files.pythonhosted.org/packages/b4/ad/b59e5b451cf7172b8d1043dc0fa718f23aab379bc1521ee13d4bd9bfa960/coverage-7.13.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d490ba50c3f35dd7c17953c68f3270e7ccd1c6642e2d2afe2d8e720b98f5a053", size = 219278, upload-time = "2026-02-09T12:56:31.673Z" }, { url = "https://files.pythonhosted.org/packages/f1/17/0cb7ca3de72e5f4ef2ec2fa0089beafbcaaaead1844e8b8a63d35173d77d/coverage-7.13.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:19bc3c88078789f8ef36acb014d7241961dbf883fd2533d18cb1e7a5b4e28b11", size = 219783, upload-time = "2026-02-09T12:56:33.104Z" }, { url = "https://files.pythonhosted.org/packages/ab/63/325d8e5b11e0eaf6d0f6a44fad444ae58820929a9b0de943fa377fe73e85/coverage-7.13.4-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3998e5a32e62fdf410c0dbd3115df86297995d6e3429af80b8798aad894ca7aa", size = 250200, upload-time = "2026-02-09T12:56:34.474Z" }, { url = "https://files.pythonhosted.org/packages/76/53/c16972708cbb79f2942922571a687c52bd109a7bd51175aeb7558dff2236/coverage-7.13.4-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e264226ec98e01a8e1054314af91ee6cde0eacac4f465cc93b03dbe0bce2fd7", size = 252114, upload-time = "2026-02-09T12:56:35.749Z" }, { url = "https://files.pythonhosted.org/packages/eb/c2/7ab36d8b8cc412bec9ea2d07c83c48930eb4ba649634ba00cb7e4e0f9017/coverage-7.13.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3aa4e7b9e416774b21797365b358a6e827ffadaaca81b69ee02946852449f00", size = 254220, upload-time = "2026-02-09T12:56:37.796Z" }, { url = "https://files.pythonhosted.org/packages/d6/4d/cf52c9a3322c89a0e6febdfbc83bb45c0ed3c64ad14081b9503adee702e7/coverage-7.13.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:71ca20079dd8f27fcf808817e281e90220475cd75115162218d0e27549f95fef", size = 256164, upload-time = "2026-02-09T12:56:39.016Z" }, { url = "https://files.pythonhosted.org/packages/78/e9/eb1dd17bd6de8289df3580e967e78294f352a5df8a57ff4671ee5fc3dcd0/coverage-7.13.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e2f25215f1a359ab17320b47bcdaca3e6e6356652e8256f2441e4ef972052903", size = 250325, upload-time = "2026-02-09T12:56:40.668Z" }, { url = "https://files.pythonhosted.org/packages/71/07/8c1542aa873728f72267c07278c5cc0ec91356daf974df21335ccdb46368/coverage-7.13.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d65b2d373032411e86960604dc4edac91fdfb5dca539461cf2cbe78327d1e64f", size = 251913, upload-time = "2026-02-09T12:56:41.97Z" }, { url = "https://files.pythonhosted.org/packages/74/d7/c62e2c5e4483a748e27868e4c32ad3daa9bdddbba58e1bc7a15e252baa74/coverage-7.13.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94eb63f9b363180aff17de3e7c8760c3ba94664ea2695c52f10111244d16a299", size = 249974, upload-time = "2026-02-09T12:56:43.323Z" }, { url = "https://files.pythonhosted.org/packages/98/9f/4c5c015a6e98ced54efd0f5cf8d31b88e5504ecb6857585fc0161bb1e600/coverage-7.13.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e856bf6616714c3a9fbc270ab54103f4e685ba236fa98c054e8f87f266c93505", size = 253741, upload-time = "2026-02-09T12:56:45.155Z" }, { url = "https://files.pythonhosted.org/packages/bd/59/0f4eef89b9f0fcd9633b5d350016f54126ab49426a70ff4c4e87446cabdc/coverage-7.13.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:65dfcbe305c3dfe658492df2d85259e0d79ead4177f9ae724b6fb245198f55d6", size = 249695, upload-time = "2026-02-09T12:56:46.636Z" }, { url = "https://files.pythonhosted.org/packages/b5/2c/b7476f938deb07166f3eb281a385c262675d688ff4659ad56c6c6b8e2e70/coverage-7.13.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b507778ae8a4c915436ed5c2e05b4a6cecfa70f734e19c22a005152a11c7b6a9", size = 250599, upload-time = "2026-02-09T12:56:48.13Z" }, { url = "https://files.pythonhosted.org/packages/b8/34/c3420709d9846ee3785b9f2831b4d94f276f38884032dca1457fa83f7476/coverage-7.13.4-cp311-cp311-win32.whl", hash = "sha256:784fc3cf8be001197b652d51d3fd259b1e2262888693a4636e18879f613a62a9", size = 221780, upload-time = "2026-02-09T12:56:50.479Z" }, { url = "https://files.pythonhosted.org/packages/61/08/3d9c8613079d2b11c185b865de9a4c1a68850cfda2b357fae365cf609f29/coverage-7.13.4-cp311-cp311-win_amd64.whl", hash = "sha256:2421d591f8ca05b308cf0092807308b2facbefe54af7c02ac22548b88b95c98f", size = 222715, upload-time = "2026-02-09T12:56:51.815Z" }, { url = "https://files.pythonhosted.org/packages/18/1a/54c3c80b2f056164cc0a6cdcb040733760c7c4be9d780fe655f356f433e4/coverage-7.13.4-cp311-cp311-win_arm64.whl", hash = "sha256:79e73a76b854d9c6088fe5d8b2ebe745f8681c55f7397c3c0a016192d681045f", size = 221385, upload-time = "2026-02-09T12:56:53.194Z" }, { url = "https://files.pythonhosted.org/packages/d1/81/4ce2fdd909c5a0ed1f6dedb88aa57ab79b6d1fbd9b588c1ac7ef45659566/coverage-7.13.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:02231499b08dabbe2b96612993e5fc34217cdae907a51b906ac7fca8027a4459", size = 219449, upload-time = "2026-02-09T12:56:54.889Z" }, { url = "https://files.pythonhosted.org/packages/5d/96/5238b1efc5922ddbdc9b0db9243152c09777804fb7c02ad1741eb18a11c0/coverage-7.13.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40aa8808140e55dc022b15d8aa7f651b6b3d68b365ea0398f1441e0b04d859c3", size = 219810, upload-time = "2026-02-09T12:56:56.33Z" }, { url = "https://files.pythonhosted.org/packages/78/72/2f372b726d433c9c35e56377cf1d513b4c16fe51841060d826b95caacec1/coverage-7.13.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5b856a8ccf749480024ff3bd7310adaef57bf31fd17e1bfc404b7940b6986634", size = 251308, upload-time = "2026-02-09T12:56:57.858Z" }, { url = "https://files.pythonhosted.org/packages/5d/a0/2ea570925524ef4e00bb6c82649f5682a77fac5ab910a65c9284de422600/coverage-7.13.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c048ea43875fbf8b45d476ad79f179809c590ec7b79e2035c662e7afa3192e3", size = 254052, upload-time = "2026-02-09T12:56:59.754Z" }, { url = "https://files.pythonhosted.org/packages/e8/ac/45dc2e19a1939098d783c846e130b8f862fbb50d09e0af663988f2f21973/coverage-7.13.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7b38448866e83176e28086674fe7368ab8590e4610fb662b44e345b86d63ffa", size = 255165, upload-time = "2026-02-09T12:57:01.287Z" }, { url = "https://files.pythonhosted.org/packages/2d/4d/26d236ff35abc3b5e63540d3386e4c3b192168c1d96da5cb2f43c640970f/coverage-7.13.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:de6defc1c9badbf8b9e67ae90fd00519186d6ab64e5cc5f3d21359c2a9b2c1d3", size = 257432, upload-time = "2026-02-09T12:57:02.637Z" }, { url = "https://files.pythonhosted.org/packages/ec/55/14a966c757d1348b2e19caf699415a2a4c4f7feaa4bbc6326a51f5c7dd1b/coverage-7.13.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7eda778067ad7ffccd23ecffce537dface96212576a07924cbf0d8799d2ded5a", size = 251716, upload-time = "2026-02-09T12:57:04.056Z" }, { url = "https://files.pythonhosted.org/packages/77/33/50116647905837c66d28b2af1321b845d5f5d19be9655cb84d4a0ea806b4/coverage-7.13.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e87f6c587c3f34356c3759f0420693e35e7eb0e2e41e4c011cb6ec6ecbbf1db7", size = 253089, upload-time = "2026-02-09T12:57:05.503Z" }, { url = "https://files.pythonhosted.org/packages/c2/b4/8efb11a46e3665d92635a56e4f2d4529de6d33f2cb38afd47d779d15fc99/coverage-7.13.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8248977c2e33aecb2ced42fef99f2d319e9904a36e55a8a68b69207fb7e43edc", size = 251232, upload-time = "2026-02-09T12:57:06.879Z" }, { url = "https://files.pythonhosted.org/packages/51/24/8cd73dd399b812cc76bb0ac260e671c4163093441847ffe058ac9fda1e32/coverage-7.13.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:25381386e80ae727608e662474db537d4df1ecd42379b5ba33c84633a2b36d47", size = 255299, upload-time = "2026-02-09T12:57:08.245Z" }, { url = "https://files.pythonhosted.org/packages/03/94/0a4b12f1d0e029ce1ccc1c800944a9984cbe7d678e470bb6d3c6bc38a0da/coverage-7.13.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ee756f00726693e5ba94d6df2bdfd64d4852d23b09bb0bc700e3b30e6f333985", size = 250796, upload-time = "2026-02-09T12:57:10.142Z" }, { url = "https://files.pythonhosted.org/packages/73/44/6002fbf88f6698ca034360ce474c406be6d5a985b3fdb3401128031eef6b/coverage-7.13.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fdfc1e28e7c7cdce44985b3043bc13bbd9c747520f94a4d7164af8260b3d91f0", size = 252673, upload-time = "2026-02-09T12:57:12.197Z" }, { url = "https://files.pythonhosted.org/packages/de/c6/a0279f7c00e786be75a749a5674e6fa267bcbd8209cd10c9a450c655dfa7/coverage-7.13.4-cp312-cp312-win32.whl", hash = "sha256:01d4cbc3c283a17fc1e42d614a119f7f438eabb593391283adca8dc86eff1246", size = 221990, upload-time = "2026-02-09T12:57:14.085Z" }, { url = "https://files.pythonhosted.org/packages/77/4e/c0a25a425fcf5557d9abd18419c95b63922e897bc86c1f327f155ef234a9/coverage-7.13.4-cp312-cp312-win_amd64.whl", hash = "sha256:9401ebc7ef522f01d01d45532c68c5ac40fb27113019b6b7d8b208f6e9baa126", size = 222800, upload-time = "2026-02-09T12:57:15.944Z" }, { url = "https://files.pythonhosted.org/packages/47/ac/92da44ad9a6f4e3a7debd178949d6f3769bedca33830ce9b1dcdab589a37/coverage-7.13.4-cp312-cp312-win_arm64.whl", hash = "sha256:b1ec7b6b6e93255f952e27ab58fbc68dcc468844b16ecbee881aeb29b6ab4d8d", size = 221415, upload-time = "2026-02-09T12:57:17.497Z" }, { url = "https://files.pythonhosted.org/packages/db/23/aad45061a31677d68e47499197a131eea55da4875d16c1f42021ab963503/coverage-7.13.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b66a2da594b6068b48b2692f043f35d4d3693fb639d5ea8b39533c2ad9ac3ab9", size = 219474, upload-time = "2026-02-09T12:57:19.332Z" }, { url = "https://files.pythonhosted.org/packages/a5/70/9b8b67a0945f3dfec1fd896c5cefb7c19d5a3a6d74630b99a895170999ae/coverage-7.13.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3599eb3992d814d23b35c536c28df1a882caa950f8f507cef23d1cbf334995ac", size = 219844, upload-time = "2026-02-09T12:57:20.66Z" }, { url = "https://files.pythonhosted.org/packages/97/fd/7e859f8fab324cef6c4ad7cff156ca7c489fef9179d5749b0c8d321281c2/coverage-7.13.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:93550784d9281e374fb5a12bf1324cc8a963fd63b2d2f223503ef0fd4aa339ea", size = 250832, upload-time = "2026-02-09T12:57:22.007Z" }, { url = "https://files.pythonhosted.org/packages/e4/dc/b2442d10020c2f52617828862d8b6ee337859cd8f3a1f13d607dddda9cf7/coverage-7.13.4-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b720ce6a88a2755f7c697c23268ddc47a571b88052e6b155224347389fdf6a3b", size = 253434, upload-time = "2026-02-09T12:57:23.339Z" }, { url = "https://files.pythonhosted.org/packages/5a/88/6728a7ad17428b18d836540630487231f5470fb82454871149502f5e5aa2/coverage-7.13.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7b322db1284a2ed3aa28ffd8ebe3db91c929b7a333c0820abec3d838ef5b3525", size = 254676, upload-time = "2026-02-09T12:57:24.774Z" }, { url = "https://files.pythonhosted.org/packages/7c/bc/21244b1b8cedf0dff0a2b53b208015fe798d5f2a8d5348dbfece04224fff/coverage-7.13.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f4594c67d8a7c89cf922d9df0438c7c7bb022ad506eddb0fdb2863359ff78242", size = 256807, upload-time = "2026-02-09T12:57:26.125Z" }, { url = "https://files.pythonhosted.org/packages/97/a0/ddba7ed3251cff51006737a727d84e05b61517d1784a9988a846ba508877/coverage-7.13.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:53d133df809c743eb8bce33b24bcababb371f4441340578cd406e084d94a6148", size = 251058, upload-time = "2026-02-09T12:57:27.614Z" }, { url = "https://files.pythonhosted.org/packages/9b/55/e289addf7ff54d3a540526f33751951bf0878f3809b47f6dfb3def69c6f7/coverage-7.13.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76451d1978b95ba6507a039090ba076105c87cc76fc3efd5d35d72093964d49a", size = 252805, upload-time = "2026-02-09T12:57:29.066Z" }, { url = "https://files.pythonhosted.org/packages/13/4e/cc276b1fa4a59be56d96f1dabddbdc30f4ba22e3b1cd42504c37b3313255/coverage-7.13.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7f57b33491e281e962021de110b451ab8a24182589be17e12a22c79047935e23", size = 250766, upload-time = "2026-02-09T12:57:30.522Z" }, { url = "https://files.pythonhosted.org/packages/94/44/1093b8f93018f8b41a8cf29636c9292502f05e4a113d4d107d14a3acd044/coverage-7.13.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1731dc33dc276dafc410a885cbf5992f1ff171393e48a21453b78727d090de80", size = 254923, upload-time = "2026-02-09T12:57:31.946Z" }, { url = "https://files.pythonhosted.org/packages/8b/55/ea2796da2d42257f37dbea1aab239ba9263b31bd91d5527cdd6db5efe174/coverage-7.13.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:bd60d4fe2f6fa7dff9223ca1bbc9f05d2b6697bc5961072e5d3b952d46e1b1ea", size = 250591, upload-time = "2026-02-09T12:57:33.842Z" }, { url = "https://files.pythonhosted.org/packages/d4/fa/7c4bb72aacf8af5020675aa633e59c1fbe296d22aed191b6a5b711eb2bc7/coverage-7.13.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9181a3ccead280b828fae232df12b16652702b49d41e99d657f46cc7b1f6ec7a", size = 252364, upload-time = "2026-02-09T12:57:35.743Z" }, { url = "https://files.pythonhosted.org/packages/5c/38/a8d2ec0146479c20bbaa7181b5b455a0c41101eed57f10dd19a78ab44c80/coverage-7.13.4-cp313-cp313-win32.whl", hash = "sha256:f53d492307962561ac7de4cd1de3e363589b000ab69617c6156a16ba7237998d", size = 222010, upload-time = "2026-02-09T12:57:37.25Z" }, { url = "https://files.pythonhosted.org/packages/e2/0c/dbfafbe90a185943dcfbc766fe0e1909f658811492d79b741523a414a6cc/coverage-7.13.4-cp313-cp313-win_amd64.whl", hash = "sha256:e6f70dec1cc557e52df5306d051ef56003f74d56e9c4dd7ddb07e07ef32a84dd", size = 222818, upload-time = "2026-02-09T12:57:38.734Z" }, { url = "https://files.pythonhosted.org/packages/04/d1/934918a138c932c90d78301f45f677fb05c39a3112b96fd2c8e60503cdc7/coverage-7.13.4-cp313-cp313-win_arm64.whl", hash = "sha256:fb07dc5da7e849e2ad31a5d74e9bece81f30ecf5a42909d0a695f8bd1874d6af", size = 221438, upload-time = "2026-02-09T12:57:40.223Z" }, { url = "https://files.pythonhosted.org/packages/52/57/ee93ced533bcb3e6df961c0c6e42da2fc6addae53fb95b94a89b1e33ebd7/coverage-7.13.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:40d74da8e6c4b9ac18b15331c4b5ebc35a17069410cad462ad4f40dcd2d50c0d", size = 220165, upload-time = "2026-02-09T12:57:41.639Z" }, { url = "https://files.pythonhosted.org/packages/c5/e0/969fc285a6fbdda49d91af278488d904dcd7651b2693872f0ff94e40e84a/coverage-7.13.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4223b4230a376138939a9173f1bdd6521994f2aff8047fae100d6d94d50c5a12", size = 220516, upload-time = "2026-02-09T12:57:44.215Z" }, { url = "https://files.pythonhosted.org/packages/b1/b8/9531944e16267e2735a30a9641ff49671f07e8138ecf1ca13db9fd2560c7/coverage-7.13.4-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1d4be36a5114c499f9f1f9195e95ebf979460dbe2d88e6816ea202010ba1c34b", size = 261804, upload-time = "2026-02-09T12:57:45.989Z" }, { url = "https://files.pythonhosted.org/packages/8a/f3/e63df6d500314a2a60390d1989240d5f27318a7a68fa30ad3806e2a9323e/coverage-7.13.4-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:200dea7d1e8095cc6e98cdabe3fd1d21ab17d3cee6dab00cadbb2fe35d9c15b9", size = 263885, upload-time = "2026-02-09T12:57:47.42Z" }, { url = "https://files.pythonhosted.org/packages/f3/67/7654810de580e14b37670b60a09c599fa348e48312db5b216d730857ffe6/coverage-7.13.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8eb931ee8e6d8243e253e5ed7336deea6904369d2fd8ae6e43f68abbf167092", size = 266308, upload-time = "2026-02-09T12:57:49.345Z" }, { url = "https://files.pythonhosted.org/packages/37/6f/39d41eca0eab3cc82115953ad41c4e77935286c930e8fad15eaed1389d83/coverage-7.13.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:75eab1ebe4f2f64d9509b984f9314d4aa788540368218b858dad56dc8f3e5eb9", size = 267452, upload-time = "2026-02-09T12:57:50.811Z" }, { url = "https://files.pythonhosted.org/packages/50/6d/39c0fbb8fc5cd4d2090811e553c2108cf5112e882f82505ee7495349a6bf/coverage-7.13.4-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c35eb28c1d085eb7d8c9b3296567a1bebe03ce72962e932431b9a61f28facf26", size = 261057, upload-time = "2026-02-09T12:57:52.447Z" }, { url = "https://files.pythonhosted.org/packages/a4/a2/60010c669df5fa603bb5a97fb75407e191a846510da70ac657eb696b7fce/coverage-7.13.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:eb88b316ec33760714a4720feb2816a3a59180fd58c1985012054fa7aebee4c2", size = 263875, upload-time = "2026-02-09T12:57:53.938Z" }, { url = "https://files.pythonhosted.org/packages/3e/d9/63b22a6bdbd17f1f96e9ed58604c2a6b0e72a9133e37d663bef185877cf6/coverage-7.13.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7d41eead3cc673cbd38a4417deb7fd0b4ca26954ff7dc6078e33f6ff97bed940", size = 261500, upload-time = "2026-02-09T12:57:56.012Z" }, { url = "https://files.pythonhosted.org/packages/70/bf/69f86ba1ad85bc3ad240e4c0e57a2e620fbc0e1645a47b5c62f0e941ad7f/coverage-7.13.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:fb26a934946a6afe0e326aebe0730cdff393a8bc0bbb65a2f41e30feddca399c", size = 265212, upload-time = "2026-02-09T12:57:57.5Z" }, { url = "https://files.pythonhosted.org/packages/ae/f2/5f65a278a8c2148731831574c73e42f57204243d33bedaaf18fa79c5958f/coverage-7.13.4-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:dae88bc0fc77edaa65c14be099bd57ee140cf507e6bfdeea7938457ab387efb0", size = 260398, upload-time = "2026-02-09T12:57:59.027Z" }, { url = "https://files.pythonhosted.org/packages/ef/80/6e8280a350ee9fea92f14b8357448a242dcaa243cb2c72ab0ca591f66c8c/coverage-7.13.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:845f352911777a8e722bfce168958214951e07e47e5d5d9744109fa5fe77f79b", size = 262584, upload-time = "2026-02-09T12:58:01.129Z" }, { url = "https://files.pythonhosted.org/packages/22/63/01ff182fc95f260b539590fb12c11ad3e21332c15f9799cb5e2386f71d9f/coverage-7.13.4-cp313-cp313t-win32.whl", hash = "sha256:2fa8d5f8de70688a28240de9e139fa16b153cc3cbb01c5f16d88d6505ebdadf9", size = 222688, upload-time = "2026-02-09T12:58:02.736Z" }, { url = "https://files.pythonhosted.org/packages/a9/43/89de4ef5d3cd53b886afa114065f7e9d3707bdb3e5efae13535b46ae483d/coverage-7.13.4-cp313-cp313t-win_amd64.whl", hash = "sha256:9351229c8c8407645840edcc277f4a2d44814d1bc34a2128c11c2a031d45a5dd", size = 223746, upload-time = "2026-02-09T12:58:05.362Z" }, { url = "https://files.pythonhosted.org/packages/35/39/7cf0aa9a10d470a5309b38b289b9bb07ddeac5d61af9b664fe9775a4cb3e/coverage-7.13.4-cp313-cp313t-win_arm64.whl", hash = "sha256:30b8d0512f2dc8c8747557e8fb459d6176a2c9e5731e2b74d311c03b78451997", size = 222003, upload-time = "2026-02-09T12:58:06.952Z" }, { url = "https://files.pythonhosted.org/packages/92/11/a9cf762bb83386467737d32187756a42094927150c3e107df4cb078e8590/coverage-7.13.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:300deaee342f90696ed186e3a00c71b5b3d27bffe9e827677954f4ee56969601", size = 219522, upload-time = "2026-02-09T12:58:08.623Z" }, { url = "https://files.pythonhosted.org/packages/d3/28/56e6d892b7b052236d67c95f1936b6a7cf7c3e2634bf27610b8cbd7f9c60/coverage-7.13.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29e3220258d682b6226a9b0925bc563ed9a1ebcff3cad30f043eceea7eaf2689", size = 219855, upload-time = "2026-02-09T12:58:10.176Z" }, { url = "https://files.pythonhosted.org/packages/e5/69/233459ee9eb0c0d10fcc2fe425a029b3fa5ce0f040c966ebce851d030c70/coverage-7.13.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:391ee8f19bef69210978363ca930f7328081c6a0152f1166c91f0b5fdd2a773c", size = 250887, upload-time = "2026-02-09T12:58:12.503Z" }, { url = "https://files.pythonhosted.org/packages/06/90/2cdab0974b9b5bbc1623f7876b73603aecac11b8d95b85b5b86b32de5eab/coverage-7.13.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0dd7ab8278f0d58a0128ba2fca25824321f05d059c1441800e934ff2efa52129", size = 253396, upload-time = "2026-02-09T12:58:14.615Z" }, { url = "https://files.pythonhosted.org/packages/ac/15/ea4da0f85bf7d7b27635039e649e99deb8173fe551096ea15017f7053537/coverage-7.13.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78cdf0d578b15148b009ccf18c686aa4f719d887e76e6b40c38ffb61d264a552", size = 254745, upload-time = "2026-02-09T12:58:16.162Z" }, { url = "https://files.pythonhosted.org/packages/99/11/bb356e86920c655ca4d61daee4e2bbc7258f0a37de0be32d233b561134ff/coverage-7.13.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:48685fee12c2eb3b27c62f2658e7ea21e9c3239cba5a8a242801a0a3f6a8c62a", size = 257055, upload-time = "2026-02-09T12:58:17.892Z" }, { url = "https://files.pythonhosted.org/packages/c9/0f/9ae1f8cb17029e09da06ca4e28c9e1d5c1c0a511c7074592e37e0836c915/coverage-7.13.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4e83efc079eb39480e6346a15a1bcb3e9b04759c5202d157e1dd4303cd619356", size = 250911, upload-time = "2026-02-09T12:58:19.495Z" }, { url = "https://files.pythonhosted.org/packages/89/3a/adfb68558fa815cbc29747b553bc833d2150228f251b127f1ce97e48547c/coverage-7.13.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecae9737b72408d6a950f7e525f30aca12d4bd8dd95e37342e5beb3a2a8c4f71", size = 252754, upload-time = "2026-02-09T12:58:21.064Z" }, { url = "https://files.pythonhosted.org/packages/32/b1/540d0c27c4e748bd3cd0bd001076ee416eda993c2bae47a73b7cc9357931/coverage-7.13.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ae4578f8528569d3cf303fef2ea569c7f4c4059a38c8667ccef15c6e1f118aa5", size = 250720, upload-time = "2026-02-09T12:58:22.622Z" }, { url = "https://files.pythonhosted.org/packages/c7/95/383609462b3ffb1fe133014a7c84fc0dd01ed55ac6140fa1093b5af7ebb1/coverage-7.13.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:6fdef321fdfbb30a197efa02d48fcd9981f0d8ad2ae8903ac318adc653f5df98", size = 254994, upload-time = "2026-02-09T12:58:24.548Z" }, { url = "https://files.pythonhosted.org/packages/f7/ba/1761138e86c81680bfc3c49579d66312865457f9fe405b033184e5793cb3/coverage-7.13.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b0f6ccf3dbe577170bebfce1318707d0e8c3650003cb4b3a9dd744575daa8b5", size = 250531, upload-time = "2026-02-09T12:58:26.271Z" }, { url = "https://files.pythonhosted.org/packages/f8/8e/05900df797a9c11837ab59c4d6fe94094e029582aab75c3309a93e6fb4e3/coverage-7.13.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:75fcd519f2a5765db3f0e391eb3b7d150cce1a771bf4c9f861aeab86c767a3c0", size = 252189, upload-time = "2026-02-09T12:58:27.807Z" }, { url = "https://files.pythonhosted.org/packages/00/bd/29c9f2db9ea4ed2738b8a9508c35626eb205d51af4ab7bf56a21a2e49926/coverage-7.13.4-cp314-cp314-win32.whl", hash = "sha256:8e798c266c378da2bd819b0677df41ab46d78065fb2a399558f3f6cae78b2fbb", size = 222258, upload-time = "2026-02-09T12:58:29.441Z" }, { url = "https://files.pythonhosted.org/packages/a7/4d/1f8e723f6829977410efeb88f73673d794075091c8c7c18848d273dc9d73/coverage-7.13.4-cp314-cp314-win_amd64.whl", hash = "sha256:245e37f664d89861cf2329c9afa2c1fe9e6d4e1a09d872c947e70718aeeac505", size = 223073, upload-time = "2026-02-09T12:58:31.026Z" }, { url = "https://files.pythonhosted.org/packages/51/5b/84100025be913b44e082ea32abcf1afbf4e872f5120b7a1cab1d331b1e13/coverage-7.13.4-cp314-cp314-win_arm64.whl", hash = "sha256:ad27098a189e5838900ce4c2a99f2fe42a0bf0c2093c17c69b45a71579e8d4a2", size = 221638, upload-time = "2026-02-09T12:58:32.599Z" }, { url = "https://files.pythonhosted.org/packages/a7/e4/c884a405d6ead1370433dad1e3720216b4f9fd8ef5b64bfd984a2a60a11a/coverage-7.13.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:85480adfb35ffc32d40918aad81b89c69c9cc5661a9b8a81476d3e645321a056", size = 220246, upload-time = "2026-02-09T12:58:34.181Z" }, { url = "https://files.pythonhosted.org/packages/81/5c/4d7ed8b23b233b0fffbc9dfec53c232be2e695468523242ea9fd30f97ad2/coverage-7.13.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:79be69cf7f3bf9b0deeeb062eab7ac7f36cd4cc4c4dd694bd28921ba4d8596cc", size = 220514, upload-time = "2026-02-09T12:58:35.704Z" }, { url = "https://files.pythonhosted.org/packages/2f/6f/3284d4203fd2f28edd73034968398cd2d4cb04ab192abc8cff007ea35679/coverage-7.13.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:caa421e2684e382c5d8973ac55e4f36bed6821a9bad5c953494de960c74595c9", size = 261877, upload-time = "2026-02-09T12:58:37.864Z" }, { url = "https://files.pythonhosted.org/packages/09/aa/b672a647bbe1556a85337dc95bfd40d146e9965ead9cc2fe81bde1e5cbce/coverage-7.13.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14375934243ee05f56c45393fe2ce81fe5cc503c07cee2bdf1725fb8bef3ffaf", size = 264004, upload-time = "2026-02-09T12:58:39.492Z" }, { url = "https://files.pythonhosted.org/packages/79/a1/aa384dbe9181f98bba87dd23dda436f0c6cf2e148aecbb4e50fc51c1a656/coverage-7.13.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25a41c3104d08edb094d9db0d905ca54d0cd41c928bb6be3c4c799a54753af55", size = 266408, upload-time = "2026-02-09T12:58:41.852Z" }, { url = "https://files.pythonhosted.org/packages/53/5e/5150bf17b4019bc600799f376bb9606941e55bd5a775dc1e096b6ffea952/coverage-7.13.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f01afcff62bf9a08fb32b2c1d6e924236c0383c02c790732b6537269e466a72", size = 267544, upload-time = "2026-02-09T12:58:44.093Z" }, { url = "https://files.pythonhosted.org/packages/e0/ed/f1de5c675987a4a7a672250d2c5c9d73d289dbf13410f00ed7181d8017dd/coverage-7.13.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eb9078108fbf0bcdde37c3f4779303673c2fa1fe8f7956e68d447d0dd426d38a", size = 260980, upload-time = "2026-02-09T12:58:45.721Z" }, { url = "https://files.pythonhosted.org/packages/b3/e3/fe758d01850aa172419a6743fe76ba8b92c29d181d4f676ffe2dae2ba631/coverage-7.13.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0e086334e8537ddd17e5f16a344777c1ab8194986ec533711cbe6c41cde841b6", size = 263871, upload-time = "2026-02-09T12:58:47.334Z" }, { url = "https://files.pythonhosted.org/packages/b6/76/b829869d464115e22499541def9796b25312b8cf235d3bb00b39f1675395/coverage-7.13.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:725d985c5ab621268b2edb8e50dfe57633dc69bda071abc470fed55a14935fd3", size = 261472, upload-time = "2026-02-09T12:58:48.995Z" }, { url = "https://files.pythonhosted.org/packages/14/9e/caedb1679e73e2f6ad240173f55218488bfe043e38da577c4ec977489915/coverage-7.13.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:3c06f0f1337c667b971ca2f975523347e63ec5e500b9aa5882d91931cd3ef750", size = 265210, upload-time = "2026-02-09T12:58:51.178Z" }, { url = "https://files.pythonhosted.org/packages/3a/10/0dd02cb009b16ede425b49ec344aba13a6ae1dc39600840ea6abcb085ac4/coverage-7.13.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:590c0ed4bf8e85f745e6b805b2e1c457b2e33d5255dd9729743165253bc9ad39", size = 260319, upload-time = "2026-02-09T12:58:53.081Z" }, { url = "https://files.pythonhosted.org/packages/92/8e/234d2c927af27c6d7a5ffad5bd2cf31634c46a477b4c7adfbfa66baf7ebb/coverage-7.13.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:eb30bf180de3f632cd043322dad5751390e5385108b2807368997d1a92a509d0", size = 262638, upload-time = "2026-02-09T12:58:55.258Z" }, { url = "https://files.pythonhosted.org/packages/2f/64/e5547c8ff6964e5965c35a480855911b61509cce544f4d442caa759a0702/coverage-7.13.4-cp314-cp314t-win32.whl", hash = "sha256:c4240e7eded42d131a2d2c4dec70374b781b043ddc79a9de4d55ca71f8e98aea", size = 223040, upload-time = "2026-02-09T12:58:56.936Z" }, { url = "https://files.pythonhosted.org/packages/c7/96/38086d58a181aac86d503dfa9c47eb20715a79c3e3acbdf786e92e5c09a8/coverage-7.13.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4c7d3cc01e7350f2f0f6f7036caaf5673fb56b6998889ccfe9e1c1fe75a9c932", size = 224148, upload-time = "2026-02-09T12:58:58.645Z" }, { url = "https://files.pythonhosted.org/packages/ce/72/8d10abd3740a0beb98c305e0c3faf454366221c0f37a8bcf8f60020bb65a/coverage-7.13.4-cp314-cp314t-win_arm64.whl", hash = "sha256:23e3f687cf945070d1c90f85db66d11e3025665d8dafa831301a0e0038f3db9b", size = 222172, upload-time = "2026-02-09T12:59:00.396Z" }, { url = "https://files.pythonhosted.org/packages/0d/4a/331fe2caf6799d591109bb9c08083080f6de90a823695d412a935622abb2/coverage-7.13.4-py3-none-any.whl", hash = "sha256:1af1641e57cf7ba1bd67d677c9abdbcd6cc2ab7da3bca7fa1e2b7e50e65f2ad0", size = 211242, upload-time = "2026-02-09T12:59:02.032Z" }, ] [package.optional-dependencies] toml = [ { name = "tomli", marker = "python_full_version <= '3.11' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, ] [[package]] name = "cuda-bindings" version = "12.9.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.13'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", "python_full_version < '3.11'", ] dependencies = [ { name = "cuda-pathfinder" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/77/3a/971ec4b608b89ce5817b8105fc71d7eef7bada022fe72f98c5e59849094b/cuda_bindings-12.9.5-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecea500213a917685fcf64cee8b96f02d35258920fe5f4deec6cbfaf94ff2198", size = 15621419, upload-time = "2025-12-18T16:30:49.81Z" }, { url = "https://files.pythonhosted.org/packages/ee/a1/86f1709105b0efa981619cd9f26890ed034ee9320e6527434afb10249a5d/cuda_bindings-12.9.5-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b2eb7041101f6386ac49f2bc0b4b194ac705d02c1b75a42a72e77f1be8d0bb6a", size = 16124212, upload-time = "2025-12-18T16:30:52.252Z" }, { url = "https://files.pythonhosted.org/packages/32/98/0da06b8d8e84953e411c56c683ea7d6c78ddbd3c877103058c0c6c0e3d7b/cuda_bindings-12.9.5-cp310-cp310-win_amd64.whl", hash = "sha256:42bb96bcf5bcf235341246f88c7e854623aceb68ba35eef6fcb1f665d54bfe81", size = 15350217, upload-time = "2025-12-18T16:30:54.299Z" }, { url = "https://files.pythonhosted.org/packages/91/6c/a90efb9e83c3830f10236c51b56b436e0b78efba79364659037dc93cdbbc/cuda_bindings-12.9.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:14dcc3a4b7bc50a7bec0e98a815b7cc36a0cfeafa20152e484c3a05068209da5", size = 15619944, upload-time = "2025-12-18T16:30:56.452Z" }, { url = "https://files.pythonhosted.org/packages/34/4a/f323d52849e4a85ea7ebbff0625c9c6297e35f0e91ce5e2d1cff4731bd1b/cuda_bindings-12.9.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:819f38fe6f3f8ea28f772e61c2583dd10152ee81051a970a291b7f916973729e", size = 16149366, upload-time = "2025-12-18T16:30:58.942Z" }, { url = "https://files.pythonhosted.org/packages/c6/07/41122c3073267645b5481f03b98d3de47848798967f64b51f25428e9d564/cuda_bindings-12.9.5-cp311-cp311-win_amd64.whl", hash = "sha256:6427fd9fef4c8cc171fbf9eb74763faeddc1fa3899882c269ff7f9b12e03b965", size = 15376641, upload-time = "2025-12-18T16:31:00.983Z" }, { url = "https://files.pythonhosted.org/packages/07/83/1720946a56090a004f375617fb2df61185a413daa779e8bb4bda313be9e3/cuda_bindings-12.9.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92f21accf6fcdea7998b2ee1bbed71e2a156737624c46dc9bc3b51a1c55bda88", size = 15400554, upload-time = "2025-12-18T16:31:03.599Z" }, { url = "https://files.pythonhosted.org/packages/10/5c/dd3cac662aad06e5b8661749b403cb6dc8359506a79e387ffc497cd25902/cuda_bindings-12.9.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5218388042d3415fba74030600fa25d59e3ec4183c344f227d43df52ca53f408", size = 15918529, upload-time = "2025-12-18T16:31:06.25Z" }, { url = "https://files.pythonhosted.org/packages/33/88/b9fb610955af5a79108744dd75cd921484b43da6b151988d889180a5ad16/cuda_bindings-12.9.5-cp312-cp312-win_amd64.whl", hash = "sha256:ee42798f639920d30292a1649b24388526067463f73a8469feeba1570121ce2d", size = 15399440, upload-time = "2025-12-18T16:31:08.894Z" }, { url = "https://files.pythonhosted.org/packages/dc/7a/fb65f880eb30ebf1bc648933ba4d4d37adbaf502bbede724d442a9512aca/cuda_bindings-12.9.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e06d272b25149e1a83b446ff3dbefcf6f82f66865c49e8d9db75683bbcff0b5f", size = 15269227, upload-time = "2025-12-18T16:31:10.952Z" }, { url = "https://files.pythonhosted.org/packages/68/b7/19ff68738d7397bdd58b5feb8385d5fbe14903ce9b60b3e772a5f25ea0ec/cuda_bindings-12.9.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da37458f8c5074d59040deddbd005460c0cf70d1ef90dd7d2c816e4686038e87", size = 15752484, upload-time = "2025-12-18T16:31:13.251Z" }, { url = "https://files.pythonhosted.org/packages/c7/7c/ee0e24167501e37fddb09a386f7ba7a7f377bf5adb3399277ff20185f26c/cuda_bindings-12.9.5-cp313-cp313-win_amd64.whl", hash = "sha256:5b9401e3bcd5d3a3c90cbb759e2cf6b8a446789b8dd6e452de9530e50b93683f", size = 15363723, upload-time = "2025-12-18T16:31:15.305Z" }, { url = "https://files.pythonhosted.org/packages/e2/ca/c4551ae51a133ca1e44fe3fedc6b2607d50fa8690f096cfd6c254bcf40c5/cuda_bindings-12.9.5-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3fb96c3d99f58ee5b3072bf1b40c215f4207481cc8b5f15b1a6c628eeb40cae", size = 15228703, upload-time = "2025-12-18T16:31:17.492Z" }, { url = "https://files.pythonhosted.org/packages/4f/17/5c5063639de93a27ba67fd9c12f7663014010b1fb50990399c923cafa8d3/cuda_bindings-12.9.5-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f5ee48922d723c626549b5644f69adcbf7537499a42963cc39d328319ff44ed", size = 15709656, upload-time = "2025-12-18T16:31:20.056Z" }, { url = "https://files.pythonhosted.org/packages/91/3d/72ae9c90aac0071e92bcbcb976b8fc3531dd74c4799dbbdc739a8b671b74/cuda_bindings-12.9.5-cp313-cp313t-win_amd64.whl", hash = "sha256:dbe330129380cc01a1a8f23894ae39c52e9fedc4b7db8ed69ac5f8ac48d55084", size = 15759852, upload-time = "2025-12-18T16:31:22.454Z" }, { url = "https://files.pythonhosted.org/packages/27/cb/eed7d79086adbb5f1ea96445170544f80ab1bd0d5b5e8f62823cfa054c90/cuda_bindings-12.9.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b3a89fccd6cc1ec535a940b43243761537ae573d977c3cd18e38f841e1f8a86", size = 15352988, upload-time = "2025-12-18T16:31:24.924Z" }, { url = "https://files.pythonhosted.org/packages/94/4c/6d2e59c2321efe9f219ba5eb5063a61facfe527655fe3ad7bdaafda1da1a/cuda_bindings-12.9.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40dc79c8cbf663e24d2607b2a979787e0d6840b624a4da48cd5303acb8482ad2", size = 15789364, upload-time = "2025-12-18T16:31:27.1Z" }, { url = "https://files.pythonhosted.org/packages/25/b0/21d9247995c9d34d7cb2c23bc50aa3ee6f4fbf92eb3a0d09be9223e221a0/cuda_bindings-12.9.5-cp314-cp314-win_amd64.whl", hash = "sha256:5faab6be8fc097076eabe591e084c7c743d6d4222ef15f42008122cdd5ca31e1", size = 15534026, upload-time = "2025-12-18T16:31:29.176Z" }, { url = "https://files.pythonhosted.org/packages/13/10/0d4980c6b9b7caffaa0e553378ce363f03e8e4b933c505c5117b26f8c067/cuda_bindings-12.9.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9a82dd5c30d9ef0956ddc9ae84e27ea027576633161de8646b00e7189d4f0a39", size = 15262883, upload-time = "2025-12-18T16:31:31.58Z" }, { url = "https://files.pythonhosted.org/packages/ed/9f/582ee268a5ebb96a643c29c69354b0c2d9634cd98deda1c5bd4abe4f61c8/cuda_bindings-12.9.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:77964179bc62504dc6bdba745bd594652233ec7796af8adce10edc0dfaf1ed95", size = 15740980, upload-time = "2025-12-18T16:31:33.972Z" }, { url = "https://files.pythonhosted.org/packages/a3/44/aab452c0a531a2c26e546745e0bba3047a7a30185620beeb68ed89a78f6d/cuda_bindings-12.9.5-cp314-cp314t-win_amd64.whl", hash = "sha256:ec376bfd8b07931f72bebed5e558c9bdae3a85473a6a1d5783ef3483a22fe86c", size = 16195220, upload-time = "2025-12-18T16:31:36.087Z" }, ] [[package]] name = "cuda-bindings" version = "13.1.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.13'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", "python_full_version < '3.11'", ] dependencies = [ { name = "cuda-pathfinder" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/60/63/579402b642f5b9b8ceb79e456b39b5771f27e132a8af3b140e54d69790fc/cuda_bindings-13.1.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4400370a83f1538e25ed4c18c34a0e9d5fad39741e282e69ce24d1479a11017d", size = 15777291, upload-time = "2025-12-09T22:05:41.109Z" }, { url = "https://files.pythonhosted.org/packages/df/6a/3a293cfb01cd4964444a0f75917b6edb1c31ea69d0230e329975da6991ba/cuda_bindings-13.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:81f92500e2f6aec2dac00a5a1ce77d5aa77ea77b606dc484d951f1f2cc3eaa13", size = 16311623, upload-time = "2025-12-09T22:05:43.897Z" }, { url = "https://files.pythonhosted.org/packages/72/b8/a5860b9e70faa53658236dc61efc3ecc51846beff4a0b73de9151130ff98/cuda_bindings-13.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:3f5bb8190267216f96597235252087accac4cbccefd1b60756cced114b2d6754", size = 15185932, upload-time = "2025-12-09T22:05:46.089Z" }, { url = "https://files.pythonhosted.org/packages/b0/58/b8d4c7c5fb29ba46088a7e78d1065484219f8fe41a08adc4a85b1ee56149/cuda_bindings-13.1.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5f5a6ade0ad45096568bc4dd1eb3377b65884d29124338fe9a4353130ef6631", size = 15771605, upload-time = "2025-12-09T22:05:48.266Z" }, { url = "https://files.pythonhosted.org/packages/17/af/710403f76f2d608d483d87089465e1f666351641dbd73d19bd025e652bad/cuda_bindings-13.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9348f69b03b257f07159dd4c869615e139722c2bd81e96c66f6b8f77615efd82", size = 16338970, upload-time = "2025-12-09T22:05:50.598Z" }, { url = "https://files.pythonhosted.org/packages/64/1c/e7ea27d4cb7d07331c88e3bbed3cacc947d2237471801086c7447b3e195d/cuda_bindings-13.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:ec33b84f4bd65a86a734427f2b9cb8f221bedab2c4cfb681488cabc82f1d64ab", size = 15210672, upload-time = "2025-12-09T22:05:53.369Z" }, { url = "https://files.pythonhosted.org/packages/53/3d/c8ed9d169843091f3f0d6b8218e826fd59520a37e0434c204feada597988/cuda_bindings-13.1.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e75ad0cb863330df784236d289612d71ca855c013d19ae00e5693574abd6915", size = 15530160, upload-time = "2025-12-09T22:05:55.386Z" }, { url = "https://files.pythonhosted.org/packages/4a/8e/368295623ee43fba622909d780fbb6863efc1638dff55f67a0f04eac6470/cuda_bindings-13.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25785d1a3cdcd98f151240fd5efd025609319a6720a217dee2a929241749d488", size = 16110386, upload-time = "2025-12-09T22:05:57.71Z" }, { url = "https://files.pythonhosted.org/packages/60/1f/ecc4701ade3e85f091c625a920574527b9daf7fb354189fbfbc5516af6cd/cuda_bindings-13.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:ccde9c95c0e953b31fe7731bb08da9d0a34b1770498df9a3c156fdfdbe3951ad", size = 15250028, upload-time = "2025-12-09T22:06:00.346Z" }, { url = "https://files.pythonhosted.org/packages/fe/c1/0ee8fd94bab7e23116e0e3da8c0902e299f3d9edc95f1d7d8ef894c897ed/cuda_bindings-13.1.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c9822a57c8f952dc367aacd7c32fe4cb17371104383606f455ea74635bff4c7", size = 15421116, upload-time = "2025-12-09T22:06:02.994Z" }, { url = "https://files.pythonhosted.org/packages/f3/c2/f272fad414b96299e010dcbe510cf17fc25deaf3443e0fdb55020a8298a3/cuda_bindings-13.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5837f5ea422c5653626dcfe22e9ab68142cd19af9e67a226100f224cc25a1b99", size = 15940152, upload-time = "2025-12-09T22:06:05.079Z" }, { url = "https://files.pythonhosted.org/packages/2a/56/433093bec0121f031edb582ea3a72f71031e8fbebecaaf329809344da4c7/cuda_bindings-13.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:9e4f348cd7a779657d51e6f71aac3965fb1738f40ff3bbe75265a3242fd6f29f", size = 15216463, upload-time = "2025-12-09T22:06:07.296Z" }, { url = "https://files.pythonhosted.org/packages/de/38/40416d037ed25db68f1dbd50e0232775a62d90c9f25af22b196c0a13b88c/cuda_bindings-13.1.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86258fe1b0d3998bea7f57dc891569e4996705b8dd00366e44c722d0a29b2090", size = 15498927, upload-time = "2025-12-09T22:06:09.476Z" }, { url = "https://files.pythonhosted.org/packages/ac/3f/f1f88b6cdb7d41ba076f8ff10edf6d3bd17e740da9a163544b43d6349653/cuda_bindings-13.1.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:daf8468fd603b2724c2d16cbd499348c64916ed72b1d04643f1660ce13cd12ae", size = 15984539, upload-time = "2025-12-09T22:06:11.882Z" }, { url = "https://files.pythonhosted.org/packages/f6/33/7739cc5e9a3373df8e7dea9060528bee5f70cf6e28b9c14f765502816c71/cuda_bindings-13.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:f2e079182014dbc162562b46467815272c14c7afe5b988978fa968728b0ac726", size = 15373212, upload-time = "2025-12-09T22:06:13.989Z" }, { url = "https://files.pythonhosted.org/packages/9e/0a/5c6d514e566ff86c4054bbbb6554bf49b9c55fefbc934eb456faecab53c9/cuda_bindings-13.1.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d0cd96a6ec00a78235947bff9462b2139bc5b83ce8e297d865802f0b52d1e23d", size = 15403944, upload-time = "2025-12-09T22:06:16.315Z" }, { url = "https://files.pythonhosted.org/packages/0b/5b/319cfa491a685d4d4757aa24223b6dbc0976954afac42f49fc47290ba6a3/cuda_bindings-13.1.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ff465829c6c394c2b4047250324a19925cf8c44633345b2746a4741e07bf827", size = 15911462, upload-time = "2025-12-09T22:06:18.403Z" }, { url = "https://files.pythonhosted.org/packages/e3/5c/38b92080c5b6c4ddb09f0be2536123f81c7e9e1a89e4573f20cb00347ee3/cuda_bindings-13.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:8205eee6b8b458a2110c0384923ace206855d0f1b436fc1b145fcbaa1653b501", size = 16044390, upload-time = "2025-12-09T22:06:20.945Z" }, ] [[package]] name = "cuda-core" version = "0.5.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/9b/2b/e6fa6721b0cac201d3dc30f2b6282f6d3d90f1223a1232453f410a2735b4/cuda_core-0.5.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6fb9cb9aba53f2e924dac34843165cbb64b9d6367c6113ec4d8845320f459bf", size = 17071396, upload-time = "2026-01-15T15:40:19.785Z" }, { url = "https://files.pythonhosted.org/packages/4b/19/2a9025982017edbb8beeda8475bfc038b521b031f2a54599bc3a07979181/cuda_core-0.5.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d32e780f6963253dc3d54646ad9aea733934c9df0899d7a0b01c3d564f6d55fd", size = 17137965, upload-time = "2026-01-15T15:40:24.316Z" }, { url = "https://files.pythonhosted.org/packages/52/37/0fb971ccefe74216e3de8e8b1c4a97a5d51666f366a325ad37c3e46985e9/cuda_core-0.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:0c25ece5b96252f2c56b17b2e49a28c35022681b64f32c3e92f02362ee481096", size = 4459925, upload-time = "2026-01-15T15:40:26.783Z" }, { url = "https://files.pythonhosted.org/packages/9a/7c/0344fab768d32b45241cc2655572f6f67a6ffa711085d2da9d61de8cf1fd/cuda_core-0.5.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0a998f241dde08d3e64b05d1f0d6bec004a5ed24fef42f9bebb9f17c274f31f", size = 17733195, upload-time = "2026-01-15T15:40:29.565Z" }, { url = "https://files.pythonhosted.org/packages/2d/1b/dc7a1117ed5bdc29284a05ef044f4f472bd035fd3765025feba67eca45a8/cuda_core-0.5.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:019894ed5799b687df8e603214cd241dcf24b94e4be9c3d4e957981e5e2eb0bd", size = 17809276, upload-time = "2026-01-15T15:40:33.784Z" }, { url = "https://files.pythonhosted.org/packages/c1/d3/af9cacea94546ab23c538a958da2f142b60354441dd4b49de37f536a4f61/cuda_core-0.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:8e2aa198e4baf4a5273cb3c27e22447d1e870189200b8fb889e956b26bcaf6be", size = 4465202, upload-time = "2026-01-15T15:40:36.088Z" }, { url = "https://files.pythonhosted.org/packages/4a/aa/073f74544f7e21d689986ed6367b94ea94f0145995db0d248f5d6936486e/cuda_core-0.5.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78876c513d484ec94ac08402d9b9059d1d8665ffd8cdc33a23b93424747c3c47", size = 17671566, upload-time = "2026-01-15T15:40:37.898Z" }, { url = "https://files.pythonhosted.org/packages/9d/01/12392ff1555e2ce77f4d7857c0027aca3c912e936828642206ba51d7149d/cuda_core-0.5.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82e687b0a5d4fde99d7bb266c60996fc266a55dc00c888eb0d7dba70bb127a09", size = 17887562, upload-time = "2026-01-15T15:40:40.658Z" }, { url = "https://files.pythonhosted.org/packages/28/97/2d189863d9d4d1f1a00aa1cc325584dc14b0b909132fc672f82c23faafff/cuda_core-0.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f192a0fd39836343158aa4e5b0e8e24280b26debb414ebad2af3794585731988", size = 4441446, upload-time = "2026-01-15T15:40:42.91Z" }, { url = "https://files.pythonhosted.org/packages/7f/41/2cd8225b2d95191b62b0da6ad4248ad5023bba9d23c355e0b3b151c1f21f/cuda_core-0.5.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c89270e8a332f8c9e18e423d7e1d08d6a82115419ec813f53784d48116fc6fc6", size = 17461993, upload-time = "2026-01-15T15:40:44.796Z" }, { url = "https://files.pythonhosted.org/packages/24/66/52f200a80f33c4a6a3c5da282fb2167192e20c8bc1fa4dbf33602d63be8f/cuda_core-0.5.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbfbdd753d58609c6b882bce5cbb6e341a338a150603b3849039a96641474b1d", size = 17689097, upload-time = "2026-01-15T15:40:47.486Z" }, { url = "https://files.pythonhosted.org/packages/41/a8/1bfeafa05974532e50086f9d56ffeb7742b0be8a827a63fb94c309b40b9f/cuda_core-0.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:1826a17a06dbc46e0d9ddfb306bd0baac5cf9218b700fe517d380f8113b676e6", size = 4427105, upload-time = "2026-01-15T15:40:49.885Z" }, { url = "https://files.pythonhosted.org/packages/d9/89/e74cecca255ccc38e717d882824a81fc24981c5f2d7eef72c5fcda4a3bf7/cuda_core-0.5.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d5ae3c8ecfa8f56afa6ca1b67c2516b1c32576427b6f57af7ebb6233373b46b0", size = 17461330, upload-time = "2026-01-15T15:40:51.964Z" }, { url = "https://files.pythonhosted.org/packages/ea/6d/dde1f35d2feaca86d900dbaebe8750fdd7692d651a185907a70683f3ae58/cuda_core-0.5.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0197d7e105266055166ce1bdfcf6a752b4fc54c5e1599fc76dfe258f9d1ccca4", size = 17580877, upload-time = "2026-01-15T15:40:54.42Z" }, { url = "https://files.pythonhosted.org/packages/e4/ea/91a5907da57b5c952e4aaa9dbee39cce29c0e160626900d04b561590db4c/cuda_core-0.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:45ec19853081bcf12de3173baf24aa2aedb63c62376dfa3504c86b9e9258f4c4", size = 4514450, upload-time = "2026-01-15T15:40:56.697Z" }, { url = "https://files.pythonhosted.org/packages/dc/45/37c5f493945cb53bcb7e20c2de9024f31c537713530d5b75cb3d8f76b278/cuda_core-0.5.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d3899efd6326df0097baeaadcf55ecbd5dd259a2cf97f27629f716d69ef1285", size = 18424313, upload-time = "2026-01-15T15:40:58.399Z" }, { url = "https://files.pythonhosted.org/packages/f1/ca/26a02fd2b7fd76f3d7384aecc7a9a6100b70923ee98a073025ebc9f7c51a/cuda_core-0.5.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66ea03faa965d594993fdc1cb00351debc1fff57479bfbe6c8715240f6dab9a5", size = 18216692, upload-time = "2026-01-15T15:41:00.933Z" }, { url = "https://files.pythonhosted.org/packages/02/02/eee51d861d8d4cb0a8e90301fe5b52d47ea4cf0129d8a51e00cdc0a8a366/cuda_core-0.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ce5d6745926fb42f23a7640fe66cb6cd0fda302a6a6b74ed2267625a9971fbea", size = 4838977, upload-time = "2026-01-15T15:41:03.407Z" }, ] [[package]] name = "cuda-pathfinder" version = "1.3.4" source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/b8/5e/db279a3bfbd18d59d0598922a3b3c1454908d0969e8372260afec9736376/cuda_pathfinder-1.3.4-py3-none-any.whl", hash = "sha256:fb983f6e0d43af27ef486e14d5989b5f904ef45cedf40538bfdcbffa6bb01fb2", size = 30878, upload-time = "2026-02-11T18:50:31.008Z" }, ] [[package]] name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, ] [[package]] name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] [[package]] name = "ml-dtypes" version = "0.5.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/fe/3a/c5b855752a70267ff729c349e650263adb3c206c29d28cc8ea7ace30a1d5/ml_dtypes-0.5.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b95e97e470fe60ed493fd9ae3911d8da4ebac16bd21f87ffa2b7c588bf22ea2c", size = 679735, upload-time = "2025-11-17T22:31:31.367Z" }, { url = "https://files.pythonhosted.org/packages/41/79/7433f30ee04bd4faa303844048f55e1eb939131c8e5195a00a96a0939b64/ml_dtypes-0.5.4-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4b801ebe0b477be666696bda493a9be8356f1f0057a57f1e35cd26928823e5a", size = 5051883, upload-time = "2025-11-17T22:31:33.658Z" }, { url = "https://files.pythonhosted.org/packages/10/b1/8938e8830b0ee2e167fc75a094dea766a1152bde46752cd9bfc57ee78a82/ml_dtypes-0.5.4-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:388d399a2152dd79a3f0456a952284a99ee5c93d3e2f8dfe25977511e0515270", size = 5030369, upload-time = "2025-11-17T22:31:35.595Z" }, { url = "https://files.pythonhosted.org/packages/c7/a3/51886727bd16e2f47587997b802dd56398692ce8c6c03c2e5bb32ecafe26/ml_dtypes-0.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:4ff7f3e7ca2972e7de850e7b8fcbb355304271e2933dd90814c1cb847414d6e2", size = 210738, upload-time = "2025-11-17T22:31:37.43Z" }, { url = "https://files.pythonhosted.org/packages/c6/5e/712092cfe7e5eb667b8ad9ca7c54442f21ed7ca8979745f1000e24cf8737/ml_dtypes-0.5.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6c7ecb74c4bd71db68a6bea1edf8da8c34f3d9fe218f038814fd1d310ac76c90", size = 679734, upload-time = "2025-11-17T22:31:39.223Z" }, { url = "https://files.pythonhosted.org/packages/4f/cf/912146dfd4b5c0eea956836c01dcd2fce6c9c844b2691f5152aca196ce4f/ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc11d7e8c44a65115d05e2ab9989d1e045125d7be8e05a071a48bc76eb6d6040", size = 5056165, upload-time = "2025-11-17T22:31:41.071Z" }, { url = "https://files.pythonhosted.org/packages/a9/80/19189ea605017473660e43762dc853d2797984b3c7bf30ce656099add30c/ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b9a53598f21e453ea2fbda8aa783c20faff8e1eeb0d7ab899309a0053f1483", size = 5034975, upload-time = "2025-11-17T22:31:42.758Z" }, { url = "https://files.pythonhosted.org/packages/b4/24/70bd59276883fdd91600ca20040b41efd4902a923283c4d6edcb1de128d2/ml_dtypes-0.5.4-cp311-cp311-win_amd64.whl", hash = "sha256:7c23c54a00ae43edf48d44066a7ec31e05fdc2eee0be2b8b50dd1903a1db94bb", size = 210742, upload-time = "2025-11-17T22:31:44.068Z" }, { url = "https://files.pythonhosted.org/packages/a0/c9/64230ef14e40aa3f1cb254ef623bf812735e6bec7772848d19131111ac0d/ml_dtypes-0.5.4-cp311-cp311-win_arm64.whl", hash = "sha256:557a31a390b7e9439056644cb80ed0735a6e3e3bb09d67fd5687e4b04238d1de", size = 160709, upload-time = "2025-11-17T22:31:46.557Z" }, { url = "https://files.pythonhosted.org/packages/a8/b8/3c70881695e056f8a32f8b941126cf78775d9a4d7feba8abcb52cb7b04f2/ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac", size = 676927, upload-time = "2025-11-17T22:31:48.182Z" }, { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, { url = "https://files.pythonhosted.org/packages/f5/f0/0cfadd537c5470378b1b32bd859cf2824972174b51b873c9d95cfd7475a5/ml_dtypes-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7", size = 212222, upload-time = "2025-11-17T22:31:53.742Z" }, { url = "https://files.pythonhosted.org/packages/16/2e/9acc86985bfad8f2c2d30291b27cd2bb4c74cea08695bd540906ed744249/ml_dtypes-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460", size = 160793, upload-time = "2025-11-17T22:31:55.358Z" }, { url = "https://files.pythonhosted.org/packages/d9/a1/4008f14bbc616cfb1ac5b39ea485f9c63031c4634ab3f4cf72e7541f816a/ml_dtypes-0.5.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48", size = 676888, upload-time = "2025-11-17T22:31:56.907Z" }, { url = "https://files.pythonhosted.org/packages/d3/b7/dff378afc2b0d5a7d6cd9d3209b60474d9819d1189d347521e1688a60a53/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b", size = 5036993, upload-time = "2025-11-17T22:31:58.497Z" }, { url = "https://files.pythonhosted.org/packages/eb/33/40cd74219417e78b97c47802037cf2d87b91973e18bb968a7da48a96ea44/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d", size = 5010956, upload-time = "2025-11-17T22:31:59.931Z" }, { url = "https://files.pythonhosted.org/packages/e1/8b/200088c6859d8221454825959df35b5244fa9bdf263fd0249ac5fb75e281/ml_dtypes-0.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328", size = 212224, upload-time = "2025-11-17T22:32:01.349Z" }, { url = "https://files.pythonhosted.org/packages/8f/75/dfc3775cb36367816e678f69a7843f6f03bd4e2bcd79941e01ea960a068e/ml_dtypes-0.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175", size = 160798, upload-time = "2025-11-17T22:32:02.864Z" }, { url = "https://files.pythonhosted.org/packages/4f/74/e9ddb35fd1dd43b1106c20ced3f53c2e8e7fc7598c15638e9f80677f81d4/ml_dtypes-0.5.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6", size = 702083, upload-time = "2025-11-17T22:32:04.08Z" }, { url = "https://files.pythonhosted.org/packages/74/f5/667060b0aed1aa63166b22897fdf16dca9eb704e6b4bbf86848d5a181aa7/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d", size = 5354111, upload-time = "2025-11-17T22:32:05.546Z" }, { url = "https://files.pythonhosted.org/packages/40/49/0f8c498a28c0efa5f5c95a9e374c83ec1385ca41d0e85e7cf40e5d519a21/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298", size = 5366453, upload-time = "2025-11-17T22:32:07.115Z" }, { url = "https://files.pythonhosted.org/packages/8c/27/12607423d0a9c6bbbcc780ad19f1f6baa2b68b18ce4bddcdc122c4c68dc9/ml_dtypes-0.5.4-cp313-cp313t-win_amd64.whl", hash = "sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6", size = 225612, upload-time = "2025-11-17T22:32:08.615Z" }, { url = "https://files.pythonhosted.org/packages/e5/80/5a5929e92c72936d5b19872c5fb8fc09327c1da67b3b68c6a13139e77e20/ml_dtypes-0.5.4-cp313-cp313t-win_arm64.whl", hash = "sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1", size = 164145, upload-time = "2025-11-17T22:32:09.782Z" }, { url = "https://files.pythonhosted.org/packages/72/4e/1339dc6e2557a344f5ba5590872e80346f76f6cb2ac3dd16e4666e88818c/ml_dtypes-0.5.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2b857d3af6ac0d39db1de7c706e69c7f9791627209c3d6dedbfca8c7e5faec22", size = 673781, upload-time = "2025-11-17T22:32:11.364Z" }, { url = "https://files.pythonhosted.org/packages/04/f9/067b84365c7e83bda15bba2b06c6ca250ce27b20630b1128c435fb7a09aa/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465", size = 5036145, upload-time = "2025-11-17T22:32:12.783Z" }, { url = "https://files.pythonhosted.org/packages/c6/bb/82c7dcf38070b46172a517e2334e665c5bf374a262f99a283ea454bece7c/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f", size = 5010230, upload-time = "2025-11-17T22:32:14.38Z" }, { url = "https://files.pythonhosted.org/packages/e9/93/2bfed22d2498c468f6bcd0d9f56b033eaa19f33320389314c19ef6766413/ml_dtypes-0.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:8c6a2dcebd6f3903e05d51960a8058d6e131fe69f952a5397e5dbabc841b6d56", size = 221032, upload-time = "2025-11-17T22:32:15.763Z" }, { url = "https://files.pythonhosted.org/packages/76/a3/9c912fe6ea747bb10fe2f8f54d027eb265db05dfb0c6335e3e063e74e6e8/ml_dtypes-0.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:5a0f68ca8fd8d16583dfa7793973feb86f2fbb56ce3966daf9c9f748f52a2049", size = 163353, upload-time = "2025-11-17T22:32:16.932Z" }, { url = "https://files.pythonhosted.org/packages/cd/02/48aa7d84cc30ab4ee37624a2fd98c56c02326785750cd212bc0826c2f15b/ml_dtypes-0.5.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:bfc534409c5d4b0bf945af29e5d0ab075eae9eecbb549ff8a29280db822f34f9", size = 702085, upload-time = "2025-11-17T22:32:18.175Z" }, { url = "https://files.pythonhosted.org/packages/5a/e7/85cb99fe80a7a5513253ec7faa88a65306be071163485e9a626fce1b6e84/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7", size = 5355358, upload-time = "2025-11-17T22:32:19.7Z" }, { url = "https://files.pythonhosted.org/packages/79/2b/a826ba18d2179a56e144aef69e57fb2ab7c464ef0b2111940ee8a3a223a2/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf", size = 5366332, upload-time = "2025-11-17T22:32:21.193Z" }, { url = "https://files.pythonhosted.org/packages/84/44/f4d18446eacb20ea11e82f133ea8f86e2bf2891785b67d9da8d0ab0ef525/ml_dtypes-0.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4381fe2f2452a2d7589689693d3162e876b3ddb0a832cde7a414f8e1adf7eab1", size = 236612, upload-time = "2025-11-17T22:32:22.579Z" }, { url = "https://files.pythonhosted.org/packages/ad/3f/3d42e9a78fe5edf792a83c074b13b9b770092a4fbf3462872f4303135f09/ml_dtypes-0.5.4-cp314-cp314t-win_arm64.whl", hash = "sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d", size = 168825, upload-time = "2025-11-17T22:32:23.766Z" }, ] [[package]] name = "mpi4py" version = "4.1.1" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/62/74/28ea85b0b949cad827ea50720e00e814e88c8fd536c27c3c491e4f025724/mpi4py-4.1.1.tar.gz", hash = "sha256:eb2c8489bdbc47fdc6b26ca7576e927a11b070b6de196a443132766b3d0a2a22", size = 500518, upload-time = "2025-10-10T13:55:20.402Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/36/b3/2e7df40608f2188dca16e38f8030add1071f06b1cd94dd8a4e16b9acbd84/mpi4py-4.1.1-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:1586f5d1557abed9cba7e984d18f32e787b353be0986e599974db177ae36329a", size = 1422849, upload-time = "2025-10-10T13:53:40.082Z" }, { url = "https://files.pythonhosted.org/packages/6d/ed/970bd3edc0e614eccc726fa406255b88f728a8bc059e81f96f28d6ede0af/mpi4py-4.1.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ba85e4778d63c750226de95115c92b709f38d7e661be660a275da4f0992ee197", size = 1326982, upload-time = "2025-10-10T13:53:42.32Z" }, { url = "https://files.pythonhosted.org/packages/5d/c3/f9a5d1f9ba52ac6386bf3d3550027f42a6b102b0432113cc43294420feb2/mpi4py-4.1.1-cp310-abi3-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0a8332884626994d9ef48da233dc7a0355f4868dd7ff59f078d5813a2935b930", size = 1373127, upload-time = "2025-10-10T13:53:43.957Z" }, { url = "https://files.pythonhosted.org/packages/84/d1/1fe75025df801d817ed49371c719559f742f3f263323442d34dbe3366af3/mpi4py-4.1.1-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6e0352860f0b3e18bc0dcb47e42e583ccb9472f89752d711a6fca46a38670554", size = 1225134, upload-time = "2025-10-10T13:53:45.583Z" }, { url = "https://files.pythonhosted.org/packages/40/44/d653fec0e4ca8181645da4bfb2763017625e5b3f151b208fadd932cb1766/mpi4py-4.1.1-cp310-abi3-win_amd64.whl", hash = "sha256:0f46dfe666a599e4bd2641116b2b4852a3ed9d37915edf98fae471d666663128", size = 1478863, upload-time = "2025-10-10T13:53:47.178Z" }, { url = "https://files.pythonhosted.org/packages/58/f7/793c9a532e5367cffb2b97ca6a879285ca73a14f79e6ff208bb390651a43/mpi4py-4.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9082e04c8afcffa7d650a262d800af1a617c555d610810deeab265a4a5f7d42e", size = 1585904, upload-time = "2025-10-10T13:53:49.129Z" }, { url = "https://files.pythonhosted.org/packages/b7/fe/cdead6721426b25d817a1bf45d5adc6dc90fd8bb0831f5ca06a4edd2015c/mpi4py-4.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1d618e6a5a8f6f86c33a954356d8ed398bec31f34b63321570661ac157063bb6", size = 1438343, upload-time = "2025-10-10T13:53:51.098Z" }, { url = "https://files.pythonhosted.org/packages/c0/c4/4a73c80cf483df603770278f0fdc57da5394edee376790c62f1eba04bb3b/mpi4py-4.1.1-cp310-cp310-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d4c460609bd6decc22ad89cbfe48e4c5a2461ff52ada9345a4c19edee39f93da", size = 1432321, upload-time = "2025-10-10T13:53:53.235Z" }, { url = "https://files.pythonhosted.org/packages/49/56/7b32631f3cc5cf741610a108a7f40a3714c9862c1f637b5ded525af32be9/mpi4py-4.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c04a388c7a945e751c82742c6bb277434d26a67768a01952f7494d1c25dff94b", size = 1299883, upload-time = "2025-10-10T13:53:55.22Z" }, { url = "https://files.pythonhosted.org/packages/14/76/53caf807ec74c042fbecf76162e071c09c53fb0ed66b1edf31dabd64c588/mpi4py-4.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:1ad4b225a5a1a02a2b89979ed8f328c6a2bc3bd6ad4a57e453727f90373fa5f8", size = 1622884, upload-time = "2025-10-10T13:53:56.882Z" }, { url = "https://files.pythonhosted.org/packages/20/8f/5d28174048ef02fb91dd0759a32c07b272c9f1df265e19145712aa7bd712/mpi4py-4.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a428ba96b992a8911cf932fa71dd8c0260d47ab7e5dee2b09239ad91fc540b79", size = 1596913, upload-time = "2025-10-10T13:53:58.466Z" }, { url = "https://files.pythonhosted.org/packages/ab/81/dce928b11816fac9713e93e609476ddac520fc50368aa7591728c329ff19/mpi4py-4.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fc0cf81445fac2ae2e5716c365fd72e1bb545df065f5a3f6731f64b3beed886e", size = 1433274, upload-time = "2025-10-10T13:54:00.508Z" }, { url = "https://files.pythonhosted.org/packages/5d/15/1a869a35d3e3438866dc8d8c9cb04dc6aa484171343627a8baf82c3c1ca9/mpi4py-4.1.1-cp311-cp311-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a753d5d61b46f90260247f344a6c57c527a6a4e7bea126830120ab41c3d057e5", size = 1423333, upload-time = "2025-10-10T13:54:03.679Z" }, { url = "https://files.pythonhosted.org/packages/25/33/072781fb85f5bc50b93ee7e8d3b3afb849d50570431b6cb2aa957db79b59/mpi4py-4.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4a36ef9d7b2b6b62026dbf9b59b44efb5430f7b9ca5fb855bfbf8d403218e37c", size = 1299183, upload-time = "2025-10-10T13:54:05.3Z" }, { url = "https://files.pythonhosted.org/packages/f9/a7/152af3c6412702a4e0fcfd0fe572307ed52821de13db9c96535f31a39aa7/mpi4py-4.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:20bf4c0c65fd67287664f8b1b6dc7c7b341838f10bba34a2e452d47530ce8a5f", size = 1632284, upload-time = "2025-10-10T13:54:06.786Z" }, { url = "https://files.pythonhosted.org/packages/ff/2c/e201cd4828555f10306a5439875cbd0ecfba766ace01ff5c6df43f795650/mpi4py-4.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4403a7cec985be9963efc626193e6df3f63f5ada0c26373c28e640e623e56c3", size = 1669517, upload-time = "2025-10-10T13:54:08.404Z" }, { url = "https://files.pythonhosted.org/packages/7b/53/18d978c3a19deecf38217ce54319e6c9162fec3569c4256c039b66eac2f4/mpi4py-4.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a2ffccc9f3a8c7c957403faad594d650c60234ac08cbedf45beaa96602debe9", size = 1454721, upload-time = "2025-10-10T13:54:09.977Z" }, { url = "https://files.pythonhosted.org/packages/ee/15/b908d1d23a4bd2bd7b2e98de5df23b26e43145119fe294728bf89211b935/mpi4py-4.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ed3d9b619bf197a290f7fd67eb61b1c2a5c204afd9621651a50dc0b1c1280d45", size = 1448977, upload-time = "2025-10-10T13:54:11.65Z" }, { url = "https://files.pythonhosted.org/packages/5d/19/088a2d37e80e0feb7851853b2a71cbe6f9b18bdf0eab680977864ea83aab/mpi4py-4.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0699c194db5d95fc2085711e4e0013083bd7ae9a88438e1fd64ddb67e9b0cf9e", size = 1318737, upload-time = "2025-10-10T13:54:13.075Z" }, { url = "https://files.pythonhosted.org/packages/97/3a/526261f39bf096e5ff396d18b76740a58d872425612ff84113dd85c2c08e/mpi4py-4.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:0abf5490c3d49c30542b461bfc5ad88dd7d147a4bdb456b7163640577fdfef88", size = 1725676, upload-time = "2025-10-10T13:54:14.681Z" }, { url = "https://files.pythonhosted.org/packages/30/75/2ffccd69360680a0216e71f90fd50dc8ff49711be54502d522a068196c68/mpi4py-4.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3dd973c509f2dbb6904c035a4a071509cde98decf0528fa21e2e7d5db5cc988", size = 1710002, upload-time = "2025-10-10T13:54:17.042Z" }, { url = "https://files.pythonhosted.org/packages/3c/13/22fa9dcbc5e4ae6fd10cba6d49b7c879c30c5bea88f450f79b373d200f40/mpi4py-4.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c8c83a359e62dd7fdd030360f430e0e8986df029c0953ab216ff97a110038dc4", size = 1484623, upload-time = "2025-10-10T13:54:19.097Z" }, { url = "https://files.pythonhosted.org/packages/47/01/476f0f9dc96261d02214009f42e10338fc56f260f1f10b23ee89c515c8b7/mpi4py-4.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:323ba354ba951c7736c033c5f2ad07bb1276f9696f0312ea6ff0a28cd0ab3e3d", size = 1448403, upload-time = "2025-10-10T13:54:21.211Z" }, { url = "https://files.pythonhosted.org/packages/a2/20/dc990edb7b075ecdba4e02bcd03d1583faeb84f664d1585c4c00a0f9851a/mpi4py-4.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c4ef9fe5fb211b1c5b6afe521397e3feb01e104024d6bc37aa4289c370605e2", size = 1318018, upload-time = "2025-10-10T13:54:23.23Z" }, { url = "https://files.pythonhosted.org/packages/4e/bf/b0ab43a99ac2a1d6d5765cb7d2a4f093656090ce07528043057ecc3e87cb/mpi4py-4.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:e13a1ba26604514a12c95b7d76058ce800d5740d5f5f3b50c4b782cfa0dfaa1f", size = 1722939, upload-time = "2025-10-10T13:54:24.862Z" }, { url = "https://files.pythonhosted.org/packages/84/26/3e00dc536311e758096414b4f33beb4c7f04dff875e87a6e88fbbe4fc2d8/mpi4py-4.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:28ce1f7412f5e99a6b9fe2547203633431d0ee45670413a475a07e6c785e63b1", size = 1798116, upload-time = "2025-10-10T13:54:26.378Z" }, { url = "https://files.pythonhosted.org/packages/15/51/d06d2b126be5660aca8c00fe0d940a8658085038f61a9cfc834d3d5ffa80/mpi4py-4.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd1e49b84a0651018517e87daf68085719eca25e5c9a7cd05d98a73418c88836", size = 1586285, upload-time = "2025-10-10T13:54:27.838Z" }, { url = "https://files.pythonhosted.org/packages/51/63/eeb936e0e8cfd8160b6b297645c730b22d242595861cf6a2fa627a358175/mpi4py-4.1.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:dd869ea7758b591ffbb1483588a6fbf84952a5090e80a45ea89674d55cf25f3b", size = 1514102, upload-time = "2025-10-10T13:54:29.297Z" }, { url = "https://files.pythonhosted.org/packages/1a/c1/06967d4c107ea7169d2120c4fb86c404707e6de82e277dc9f0fa5a9c1bf1/mpi4py-4.1.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:475da0797442cba723c0ad37da6a1c51d9624e697dd8bf89f23d0fad81e73eda", size = 1395247, upload-time = "2025-10-10T13:54:30.881Z" }, { url = "https://files.pythonhosted.org/packages/9e/7c/5f0f32b39185f0a7074c165dc37cdd235bfd737928a2fe223e41b308fb4c/mpi4py-4.1.1-cp313-cp313t-win_amd64.whl", hash = "sha256:8d3bfa074776d9507ee957f5230d11ecd03da23f601a85349a1a333eaf55e5fa", size = 1771515, upload-time = "2025-10-10T13:54:32.395Z" }, { url = "https://files.pythonhosted.org/packages/6a/e8/93ddde2b6ee7631b46bb79b851630b3527d9060b9b999844bcd882977539/mpi4py-4.1.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:1deb6f9df28ec6972305287cb2035c20d3f5af59f687f962080756374c16e48f", size = 1713353, upload-time = "2025-10-10T13:54:33.934Z" }, { url = "https://files.pythonhosted.org/packages/b2/23/449562bd23fcfbd7d01006b39429972bfed5dfb8541355d06d2e17c16c27/mpi4py-4.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1bb1e3ad0b9047b0dbc7b4014160a7ab2a84f1627be665527c7445fc312f189b", size = 1496415, upload-time = "2025-10-10T13:54:35.927Z" }, { url = "https://files.pythonhosted.org/packages/51/33/9a5b9ae66cbb095b711f4ddae6d2d4b0f55202ac9e503fd588b101f04a22/mpi4py-4.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5f757e3089abf2c9db69fac1665fa99c52ed392fdf799159f25cba9ee3b64f5a", size = 1450750, upload-time = "2025-10-10T13:54:37.608Z" }, { url = "https://files.pythonhosted.org/packages/d2/88/6acf948f19cb59c0e8843fed4ab4c471b7644e8a16c2d5d9c7ab6d73d573/mpi4py-4.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:807c6f1ed3adbc12952db52127e34cfbd6c48a05c3b3dd59deee2d2f09d78888", size = 1325773, upload-time = "2025-10-10T13:54:39.136Z" }, { url = "https://files.pythonhosted.org/packages/6a/b4/3021e073772cd9e1062a810b7298e68ea40933fb91b1c1c0d07c968dce5c/mpi4py-4.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:2c85983d38d77e6302a242e32afd2a9a9b3adedd770e199a38e5b8957150e7ac", size = 1721603, upload-time = "2025-10-10T13:54:41.396Z" }, { url = "https://files.pythonhosted.org/packages/ed/02/b6700c24fe28588a4e40adb23d02fe2aea82b33495fd6290235da5199383/mpi4py-4.1.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:729c4f625ad60e5cfb6c260608d249dc35a33cc16605faff01c6adbbd7e8ce0f", size = 1799551, upload-time = "2025-10-10T13:54:43.084Z" }, { url = "https://files.pythonhosted.org/packages/5a/93/9c9870174183869bd5a50bbfe7bda91a52bf7ca2d0851de4009590e735a2/mpi4py-4.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3cca235d46009f54cb319c779c6ac53d41ce1eee3cf07f157995bc7739329b97", size = 1587583, upload-time = "2025-10-10T13:54:45.989Z" }, { url = "https://files.pythonhosted.org/packages/29/12/c46bec2311fc937ed3767312f9feb5f11bc70058c20bc53ae7369d759424/mpi4py-4.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2580fab891db492f32a6e02717e824f6fd5588be6560b08627c1e9322f7ccbfb", size = 1513437, upload-time = "2025-10-10T13:54:48.145Z" }, { url = "https://files.pythonhosted.org/packages/09/3e/e46629867204b22ce6804096e0b7d35bb5b473df1d12272021843af726c3/mpi4py-4.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6beec4841f9436d49ec9cabfd76a19df61c10b21ca14eddafa58fe7977802ee7", size = 1395082, upload-time = "2025-10-10T13:54:49.744Z" }, { url = "https://files.pythonhosted.org/packages/1a/ca/7e27edf78cd8ba68aacafc836004cd092a978f0d5ffc8a3eac9e904a3e0e/mpi4py-4.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:b4b3813da9a7a1fc37ffb8dad314cb396313a40cd3fe150854ab29e999a9eb8c", size = 1771707, upload-time = "2025-10-10T13:54:51.756Z" }, { url = "https://files.pythonhosted.org/packages/e9/63/b6a2863fb7dd5a9eccfdb055bf1124b999ff755d0187223b307161479b76/mpi4py-4.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:95bb98d946eb88c9ae4dc6c42d11b3af8ce6b91e644c288cc3f85ec7596ffcd3", size = 1480110, upload-time = "2025-10-10T13:55:11.381Z" }, { url = "https://files.pythonhosted.org/packages/de/18/358f0eb58fb3b79f65861ed682af9e735d86669663dfbce396e8673ed518/mpi4py-4.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:84e9eb2e609b0b94cd0e9a3e3b57d897f748fb0207c4f72e81e5a95aba033767", size = 1340704, upload-time = "2025-10-10T13:55:12.973Z" }, { url = "https://files.pythonhosted.org/packages/b9/66/b342e330ac543d0147ebfab754f69854c4777ac9785cb5b7610e3cd0c29a/mpi4py-4.1.1-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:027b1a1ff9d57afed10af6b79041b95f85fd11b2af74e4c34ef4866ce81ecc24", size = 1380452, upload-time = "2025-10-10T13:55:14.582Z" }, { url = "https://files.pythonhosted.org/packages/dd/61/bbf87de6f3a8a9c54e7a4b72878c9069646ca9cafac8217fa5493a54b068/mpi4py-4.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c1191856906967a48fdcc484b326c179747e68c186261d76480a75156bcc73bf", size = 1255980, upload-time = "2025-10-10T13:55:17.075Z" }, { url = "https://files.pythonhosted.org/packages/8d/4b/227091dec11518e5545bd1ec91f52e06f64bdae697adc5fb33f9f20c04dc/mpi4py-4.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:189d49b0ae963f8f6f5dd8ed0f5f37923285c97bc725476990ec0556972bb4b2", size = 1452641, upload-time = "2025-10-10T13:55:18.562Z" }, ] [[package]] name = "nccl4py" source = { editable = "." } dependencies = [ { name = "cuda-core" }, { name = "cuda-pathfinder" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, { name = "packaging" }, ] [package.optional-dependencies] cu12 = [ { name = "cuda-bindings", version = "12.9.5", source = { registry = "https://pypi.org/simple" } }, { name = "nvidia-nccl-cu12" }, ] cu13 = [ { name = "cuda-bindings", version = "13.1.1", source = { registry = "https://pypi.org/simple" } }, { name = "nvidia-nccl-cu13" }, ] [package.dev-dependencies] test = [ { name = "ml-dtypes" }, { name = "mpi4py" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "pytest-mpi" }, ] [package.metadata] requires-dist = [ { name = "cuda-bindings", marker = "extra == 'cu12'", specifier = "~=12.0" }, { name = "cuda-bindings", marker = "extra == 'cu13'", specifier = "~=13.0" }, { name = "cuda-core", specifier = ">=0.5.0,<1.0" }, { name = "cuda-pathfinder", specifier = "~=1.1" }, { name = "numpy" }, { name = "nvidia-nccl-cu12", marker = "extra == 'cu12'" }, { name = "nvidia-nccl-cu13", marker = "extra == 'cu13'" }, { name = "packaging" }, ] provides-extras = ["cu12", "cu13"] [package.metadata.requires-dev] test = [ { name = "ml-dtypes" }, { name = "mpi4py" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "pytest-mpi" }, ] [[package]] name = "numpy" version = "2.2.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11'", ] sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245, upload-time = "2025-05-17T21:27:58.555Z" }, { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048, upload-time = "2025-05-17T21:28:21.406Z" }, { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542, upload-time = "2025-05-17T21:28:30.931Z" }, { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301, upload-time = "2025-05-17T21:28:41.613Z" }, { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320, upload-time = "2025-05-17T21:29:02.78Z" }, { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050, upload-time = "2025-05-17T21:29:27.675Z" }, { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034, upload-time = "2025-05-17T21:29:51.102Z" }, { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185, upload-time = "2025-05-17T21:30:18.703Z" }, { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149, upload-time = "2025-05-17T21:30:29.788Z" }, { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620, upload-time = "2025-05-17T21:30:48.994Z" }, { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" }, { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" }, { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" }, { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" }, { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" }, { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" }, { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" }, { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" }, { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" }, { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" }, { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828, upload-time = "2025-05-17T21:37:56.699Z" }, { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006, upload-time = "2025-05-17T21:38:18.291Z" }, { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765, upload-time = "2025-05-17T21:38:27.319Z" }, { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736, upload-time = "2025-05-17T21:38:38.141Z" }, { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719, upload-time = "2025-05-17T21:38:58.433Z" }, { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072, upload-time = "2025-05-17T21:39:22.638Z" }, { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213, upload-time = "2025-05-17T21:39:45.865Z" }, { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632, upload-time = "2025-05-17T21:40:13.331Z" }, { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532, upload-time = "2025-05-17T21:43:46.099Z" }, { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885, upload-time = "2025-05-17T21:44:05.145Z" }, { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467, upload-time = "2025-05-17T21:40:44Z" }, { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144, upload-time = "2025-05-17T21:41:05.695Z" }, { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217, upload-time = "2025-05-17T21:41:15.903Z" }, { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014, upload-time = "2025-05-17T21:41:27.321Z" }, { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935, upload-time = "2025-05-17T21:41:49.738Z" }, { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122, upload-time = "2025-05-17T21:42:14.046Z" }, { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143, upload-time = "2025-05-17T21:42:37.464Z" }, { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260, upload-time = "2025-05-17T21:43:05.189Z" }, { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225, upload-time = "2025-05-17T21:43:16.254Z" }, { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374, upload-time = "2025-05-17T21:43:35.479Z" }, { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391, upload-time = "2025-05-17T21:44:35.948Z" }, { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754, upload-time = "2025-05-17T21:44:47.446Z" }, { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476, upload-time = "2025-05-17T21:45:11.871Z" }, { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666, upload-time = "2025-05-17T21:45:31.426Z" }, ] [[package]] name = "numpy" version = "2.4.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.13'", "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/57/fd/0005efbd0af48e55eb3c7208af93f2862d4b1a56cd78e84309a2d959208d/numpy-2.4.2.tar.gz", hash = "sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae", size = 20723651, upload-time = "2026-01-31T23:13:10.135Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d3/44/71852273146957899753e69986246d6a176061ea183407e95418c2aa4d9a/numpy-2.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e7e88598032542bd49af7c4747541422884219056c268823ef6e5e89851c8825", size = 16955478, upload-time = "2026-01-31T23:10:25.623Z" }, { url = "https://files.pythonhosted.org/packages/74/41/5d17d4058bd0cd96bcbd4d9ff0fb2e21f52702aab9a72e4a594efa18692f/numpy-2.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7edc794af8b36ca37ef5fcb5e0d128c7e0595c7b96a2318d1badb6fcd8ee86b1", size = 14965467, upload-time = "2026-01-31T23:10:28.186Z" }, { url = "https://files.pythonhosted.org/packages/49/48/fb1ce8136c19452ed15f033f8aee91d5defe515094e330ce368a0647846f/numpy-2.4.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:6e9f61981ace1360e42737e2bae58b27bf28a1b27e781721047d84bd754d32e7", size = 5475172, upload-time = "2026-01-31T23:10:30.848Z" }, { url = "https://files.pythonhosted.org/packages/40/a9/3feb49f17bbd1300dd2570432961f5c8a4ffeff1db6f02c7273bd020a4c9/numpy-2.4.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:cb7bbb88aa74908950d979eeaa24dbdf1a865e3c7e45ff0121d8f70387b55f73", size = 6805145, upload-time = "2026-01-31T23:10:32.352Z" }, { url = "https://files.pythonhosted.org/packages/3f/39/fdf35cbd6d6e2fcad42fcf85ac04a85a0d0fbfbf34b30721c98d602fd70a/numpy-2.4.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f069069931240b3fc703f1e23df63443dbd6390614c8c44a87d96cd0ec81eb1", size = 15966084, upload-time = "2026-01-31T23:10:34.502Z" }, { url = "https://files.pythonhosted.org/packages/1b/46/6fa4ea94f1ddf969b2ee941290cca6f1bfac92b53c76ae5f44afe17ceb69/numpy-2.4.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c02ef4401a506fb60b411467ad501e1429a3487abca4664871d9ae0b46c8ba32", size = 16899477, upload-time = "2026-01-31T23:10:37.075Z" }, { url = "https://files.pythonhosted.org/packages/09/a1/2a424e162b1a14a5bd860a464ab4e07513916a64ab1683fae262f735ccd2/numpy-2.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2653de5c24910e49c2b106499803124dde62a5a1fe0eedeaecf4309a5f639390", size = 17323429, upload-time = "2026-01-31T23:10:39.704Z" }, { url = "https://files.pythonhosted.org/packages/ce/a2/73014149ff250628df72c58204822ac01d768697913881aacf839ff78680/numpy-2.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1ae241bbfc6ae276f94a170b14785e561cb5e7f626b6688cf076af4110887413", size = 18635109, upload-time = "2026-01-31T23:10:41.924Z" }, { url = "https://files.pythonhosted.org/packages/6c/0c/73e8be2f1accd56df74abc1c5e18527822067dced5ec0861b5bb882c2ce0/numpy-2.4.2-cp311-cp311-win32.whl", hash = "sha256:df1b10187212b198dd45fa943d8985a3c8cf854aed4923796e0e019e113a1bda", size = 6237915, upload-time = "2026-01-31T23:10:45.26Z" }, { url = "https://files.pythonhosted.org/packages/76/ae/e0265e0163cf127c24c3969d29f1c4c64551a1e375d95a13d32eab25d364/numpy-2.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:b9c618d56a29c9cb1c4da979e9899be7578d2e0b3c24d52079c166324c9e8695", size = 12607972, upload-time = "2026-01-31T23:10:47.021Z" }, { url = "https://files.pythonhosted.org/packages/29/a5/c43029af9b8014d6ea157f192652c50042e8911f4300f8f6ed3336bf437f/numpy-2.4.2-cp311-cp311-win_arm64.whl", hash = "sha256:47c5a6ed21d9452b10227e5e8a0e1c22979811cad7dcc19d8e3e2fb8fa03f1a3", size = 10485763, upload-time = "2026-01-31T23:10:50.087Z" }, { url = "https://files.pythonhosted.org/packages/51/6e/6f394c9c77668153e14d4da83bcc247beb5952f6ead7699a1a2992613bea/numpy-2.4.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:21982668592194c609de53ba4933a7471880ccbaadcc52352694a59ecc860b3a", size = 16667963, upload-time = "2026-01-31T23:10:52.147Z" }, { url = "https://files.pythonhosted.org/packages/1f/f8/55483431f2b2fd015ae6ed4fe62288823ce908437ed49db5a03d15151678/numpy-2.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40397bda92382fcec844066efb11f13e1c9a3e2a8e8f318fb72ed8b6db9f60f1", size = 14693571, upload-time = "2026-01-31T23:10:54.789Z" }, { url = "https://files.pythonhosted.org/packages/2f/20/18026832b1845cdc82248208dd929ca14c9d8f2bac391f67440707fff27c/numpy-2.4.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b3a24467af63c67829bfaa61eecf18d5432d4f11992688537be59ecd6ad32f5e", size = 5203469, upload-time = "2026-01-31T23:10:57.343Z" }, { url = "https://files.pythonhosted.org/packages/7d/33/2eb97c8a77daaba34eaa3fa7241a14ac5f51c46a6bd5911361b644c4a1e2/numpy-2.4.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:805cc8de9fd6e7a22da5aed858e0ab16be5a4db6c873dde1d7451c541553aa27", size = 6550820, upload-time = "2026-01-31T23:10:59.429Z" }, { url = "https://files.pythonhosted.org/packages/b1/91/b97fdfd12dc75b02c44e26c6638241cc004d4079a0321a69c62f51470c4c/numpy-2.4.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d82351358ffbcdcd7b686b90742a9b86632d6c1c051016484fa0b326a0a1548", size = 15663067, upload-time = "2026-01-31T23:11:01.291Z" }, { url = "https://files.pythonhosted.org/packages/f5/c6/a18e59f3f0b8071cc85cbc8d80cd02d68aa9710170b2553a117203d46936/numpy-2.4.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e35d3e0144137d9fdae62912e869136164534d64a169f86438bc9561b6ad49f", size = 16619782, upload-time = "2026-01-31T23:11:03.669Z" }, { url = "https://files.pythonhosted.org/packages/b7/83/9751502164601a79e18847309f5ceec0b1446d7b6aa12305759b72cf98b2/numpy-2.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adb6ed2ad29b9e15321d167d152ee909ec73395901b70936f029c3bc6d7f4460", size = 17013128, upload-time = "2026-01-31T23:11:05.913Z" }, { url = "https://files.pythonhosted.org/packages/61/c4/c4066322256ec740acc1c8923a10047818691d2f8aec254798f3dd90f5f2/numpy-2.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8906e71fd8afcb76580404e2a950caef2685df3d2a57fe82a86ac8d33cc007ba", size = 18345324, upload-time = "2026-01-31T23:11:08.248Z" }, { url = "https://files.pythonhosted.org/packages/ab/af/6157aa6da728fa4525a755bfad486ae7e3f76d4c1864138003eb84328497/numpy-2.4.2-cp312-cp312-win32.whl", hash = "sha256:ec055f6dae239a6299cace477b479cca2fc125c5675482daf1dd886933a1076f", size = 5960282, upload-time = "2026-01-31T23:11:10.497Z" }, { url = "https://files.pythonhosted.org/packages/92/0f/7ceaaeaacb40567071e94dbf2c9480c0ae453d5bb4f52bea3892c39dc83c/numpy-2.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:209fae046e62d0ce6435fcfe3b1a10537e858249b3d9b05829e2a05218296a85", size = 12314210, upload-time = "2026-01-31T23:11:12.176Z" }, { url = "https://files.pythonhosted.org/packages/2f/a3/56c5c604fae6dd40fa2ed3040d005fca97e91bd320d232ac9931d77ba13c/numpy-2.4.2-cp312-cp312-win_arm64.whl", hash = "sha256:fbde1b0c6e81d56f5dccd95dd4a711d9b95df1ae4009a60887e56b27e8d903fa", size = 10220171, upload-time = "2026-01-31T23:11:14.684Z" }, { url = "https://files.pythonhosted.org/packages/a1/22/815b9fe25d1d7ae7d492152adbc7226d3eff731dffc38fe970589fcaaa38/numpy-2.4.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:25f2059807faea4b077a2b6837391b5d830864b3543627f381821c646f31a63c", size = 16663696, upload-time = "2026-01-31T23:11:17.516Z" }, { url = "https://files.pythonhosted.org/packages/09/f0/817d03a03f93ba9c6c8993de509277d84e69f9453601915e4a69554102a1/numpy-2.4.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bd3a7a9f5847d2fb8c2c6d1c862fa109c31a9abeca1a3c2bd5a64572955b2979", size = 14688322, upload-time = "2026-01-31T23:11:19.883Z" }, { url = "https://files.pythonhosted.org/packages/da/b4/f805ab79293c728b9a99438775ce51885fd4f31b76178767cfc718701a39/numpy-2.4.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8e4549f8a3c6d13d55041925e912bfd834285ef1dd64d6bc7d542583355e2e98", size = 5198157, upload-time = "2026-01-31T23:11:22.375Z" }, { url = "https://files.pythonhosted.org/packages/74/09/826e4289844eccdcd64aac27d13b0fd3f32039915dd5b9ba01baae1f436c/numpy-2.4.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:aea4f66ff44dfddf8c2cffd66ba6538c5ec67d389285292fe428cb2c738c8aef", size = 6546330, upload-time = "2026-01-31T23:11:23.958Z" }, { url = "https://files.pythonhosted.org/packages/19/fb/cbfdbfa3057a10aea5422c558ac57538e6acc87ec1669e666d32ac198da7/numpy-2.4.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3cd545784805de05aafe1dde61752ea49a359ccba9760c1e5d1c88a93bbf2b7", size = 15660968, upload-time = "2026-01-31T23:11:25.713Z" }, { url = "https://files.pythonhosted.org/packages/04/dc/46066ce18d01645541f0186877377b9371b8fa8017fa8262002b4ef22612/numpy-2.4.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0d9b7c93578baafcbc5f0b83eaf17b79d345c6f36917ba0c67f45226911d499", size = 16607311, upload-time = "2026-01-31T23:11:28.117Z" }, { url = "https://files.pythonhosted.org/packages/14/d9/4b5adfc39a43fa6bf918c6d544bc60c05236cc2f6339847fc5b35e6cb5b0/numpy-2.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f74f0f7779cc7ae07d1810aab8ac6b1464c3eafb9e283a40da7309d5e6e48fbb", size = 17012850, upload-time = "2026-01-31T23:11:30.888Z" }, { url = "https://files.pythonhosted.org/packages/b7/20/adb6e6adde6d0130046e6fdfb7675cc62bc2f6b7b02239a09eb58435753d/numpy-2.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c7ac672d699bf36275c035e16b65539931347d68b70667d28984c9fb34e07fa7", size = 18334210, upload-time = "2026-01-31T23:11:33.214Z" }, { url = "https://files.pythonhosted.org/packages/78/0e/0a73b3dff26803a8c02baa76398015ea2a5434d9b8265a7898a6028c1591/numpy-2.4.2-cp313-cp313-win32.whl", hash = "sha256:8e9afaeb0beff068b4d9cd20d322ba0ee1cecfb0b08db145e4ab4dd44a6b5110", size = 5958199, upload-time = "2026-01-31T23:11:35.385Z" }, { url = "https://files.pythonhosted.org/packages/43/bc/6352f343522fcb2c04dbaf94cb30cca6fd32c1a750c06ad6231b4293708c/numpy-2.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:7df2de1e4fba69a51c06c28f5a3de36731eb9639feb8e1cf7e4a7b0daf4cf622", size = 12310848, upload-time = "2026-01-31T23:11:38.001Z" }, { url = "https://files.pythonhosted.org/packages/6e/8d/6da186483e308da5da1cc6918ce913dcfe14ffde98e710bfeff2a6158d4e/numpy-2.4.2-cp313-cp313-win_arm64.whl", hash = "sha256:0fece1d1f0a89c16b03442eae5c56dc0be0c7883b5d388e0c03f53019a4bfd71", size = 10221082, upload-time = "2026-01-31T23:11:40.392Z" }, { url = "https://files.pythonhosted.org/packages/25/a1/9510aa43555b44781968935c7548a8926274f815de42ad3997e9e83680dd/numpy-2.4.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5633c0da313330fd20c484c78cdd3f9b175b55e1a766c4a174230c6b70ad8262", size = 14815866, upload-time = "2026-01-31T23:11:42.495Z" }, { url = "https://files.pythonhosted.org/packages/36/30/6bbb5e76631a5ae46e7923dd16ca9d3f1c93cfa8d4ed79a129814a9d8db3/numpy-2.4.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d9f64d786b3b1dd742c946c42d15b07497ed14af1a1f3ce840cce27daa0ce913", size = 5325631, upload-time = "2026-01-31T23:11:44.7Z" }, { url = "https://files.pythonhosted.org/packages/46/00/3a490938800c1923b567b3a15cd17896e68052e2145d8662aaf3e1ffc58f/numpy-2.4.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:b21041e8cb6a1eb5312dd1d2f80a94d91efffb7a06b70597d44f1bd2dfc315ab", size = 6646254, upload-time = "2026-01-31T23:11:46.341Z" }, { url = "https://files.pythonhosted.org/packages/d3/e9/fac0890149898a9b609caa5af7455a948b544746e4b8fe7c212c8edd71f8/numpy-2.4.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:00ab83c56211a1d7c07c25e3217ea6695e50a3e2f255053686b081dc0b091a82", size = 15720138, upload-time = "2026-01-31T23:11:48.082Z" }, { url = "https://files.pythonhosted.org/packages/ea/5c/08887c54e68e1e28df53709f1893ce92932cc6f01f7c3d4dc952f61ffd4e/numpy-2.4.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fb882da679409066b4603579619341c6d6898fc83a8995199d5249f986e8e8f", size = 16655398, upload-time = "2026-01-31T23:11:50.293Z" }, { url = "https://files.pythonhosted.org/packages/4d/89/253db0fa0e66e9129c745e4ef25631dc37d5f1314dad2b53e907b8538e6d/numpy-2.4.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:66cb9422236317f9d44b67b4d18f44efe6e9c7f8794ac0462978513359461554", size = 17079064, upload-time = "2026-01-31T23:11:52.927Z" }, { url = "https://files.pythonhosted.org/packages/2a/d5/cbade46ce97c59c6c3da525e8d95b7abe8a42974a1dc5c1d489c10433e88/numpy-2.4.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0f01dcf33e73d80bd8dc0f20a71303abbafa26a19e23f6b68d1aa9990af90257", size = 18379680, upload-time = "2026-01-31T23:11:55.22Z" }, { url = "https://files.pythonhosted.org/packages/40/62/48f99ae172a4b63d981babe683685030e8a3df4f246c893ea5c6ef99f018/numpy-2.4.2-cp313-cp313t-win32.whl", hash = "sha256:52b913ec40ff7ae845687b0b34d8d93b60cb66dcee06996dd5c99f2fc9328657", size = 6082433, upload-time = "2026-01-31T23:11:58.096Z" }, { url = "https://files.pythonhosted.org/packages/07/38/e054a61cfe48ad9f1ed0d188e78b7e26859d0b60ef21cd9de4897cdb5326/numpy-2.4.2-cp313-cp313t-win_amd64.whl", hash = "sha256:5eea80d908b2c1f91486eb95b3fb6fab187e569ec9752ab7d9333d2e66bf2d6b", size = 12451181, upload-time = "2026-01-31T23:11:59.782Z" }, { url = "https://files.pythonhosted.org/packages/6e/a4/a05c3a6418575e185dd84d0b9680b6bb2e2dc3e4202f036b7b4e22d6e9dc/numpy-2.4.2-cp313-cp313t-win_arm64.whl", hash = "sha256:fd49860271d52127d61197bb50b64f58454e9f578cb4b2c001a6de8b1f50b0b1", size = 10290756, upload-time = "2026-01-31T23:12:02.438Z" }, { url = "https://files.pythonhosted.org/packages/18/88/b7df6050bf18fdcfb7046286c6535cabbdd2064a3440fca3f069d319c16e/numpy-2.4.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:444be170853f1f9d528428eceb55f12918e4fda5d8805480f36a002f1415e09b", size = 16663092, upload-time = "2026-01-31T23:12:04.521Z" }, { url = "https://files.pythonhosted.org/packages/25/7a/1fee4329abc705a469a4afe6e69b1ef7e915117747886327104a8493a955/numpy-2.4.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d1240d50adff70c2a88217698ca844723068533f3f5c5fa6ee2e3220e3bdb000", size = 14698770, upload-time = "2026-01-31T23:12:06.96Z" }, { url = "https://files.pythonhosted.org/packages/fb/0b/f9e49ba6c923678ad5bc38181c08ac5e53b7a5754dbca8e581aa1a56b1ff/numpy-2.4.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:7cdde6de52fb6664b00b056341265441192d1291c130e99183ec0d4b110ff8b1", size = 5208562, upload-time = "2026-01-31T23:12:09.632Z" }, { url = "https://files.pythonhosted.org/packages/7d/12/d7de8f6f53f9bb76997e5e4c069eda2051e3fe134e9181671c4391677bb2/numpy-2.4.2-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:cda077c2e5b780200b6b3e09d0b42205a3d1c68f30c6dceb90401c13bff8fe74", size = 6543710, upload-time = "2026-01-31T23:12:11.969Z" }, { url = "https://files.pythonhosted.org/packages/09/63/c66418c2e0268a31a4cf8a8b512685748200f8e8e8ec6c507ce14e773529/numpy-2.4.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d30291931c915b2ab5717c2974bb95ee891a1cf22ebc16a8006bd59cd210d40a", size = 15677205, upload-time = "2026-01-31T23:12:14.33Z" }, { url = "https://files.pythonhosted.org/packages/5d/6c/7f237821c9642fb2a04d2f1e88b4295677144ca93285fd76eff3bcba858d/numpy-2.4.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bba37bc29d4d85761deed3954a1bc62be7cf462b9510b51d367b769a8c8df325", size = 16611738, upload-time = "2026-01-31T23:12:16.525Z" }, { url = "https://files.pythonhosted.org/packages/c2/a7/39c4cdda9f019b609b5c473899d87abff092fc908cfe4d1ecb2fcff453b0/numpy-2.4.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b2f0073ed0868db1dcd86e052d37279eef185b9c8db5bf61f30f46adac63c909", size = 17028888, upload-time = "2026-01-31T23:12:19.306Z" }, { url = "https://files.pythonhosted.org/packages/da/b3/e84bb64bdfea967cc10950d71090ec2d84b49bc691df0025dddb7c26e8e3/numpy-2.4.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7f54844851cdb630ceb623dcec4db3240d1ac13d4990532446761baede94996a", size = 18339556, upload-time = "2026-01-31T23:12:21.816Z" }, { url = "https://files.pythonhosted.org/packages/88/f5/954a291bc1192a27081706862ac62bb5920fbecfbaa302f64682aa90beed/numpy-2.4.2-cp314-cp314-win32.whl", hash = "sha256:12e26134a0331d8dbd9351620f037ec470b7c75929cb8a1537f6bfe411152a1a", size = 6006899, upload-time = "2026-01-31T23:12:24.14Z" }, { url = "https://files.pythonhosted.org/packages/05/cb/eff72a91b2efdd1bc98b3b8759f6a1654aa87612fc86e3d87d6fe4f948c4/numpy-2.4.2-cp314-cp314-win_amd64.whl", hash = "sha256:068cdb2d0d644cdb45670810894f6a0600797a69c05f1ac478e8d31670b8ee75", size = 12443072, upload-time = "2026-01-31T23:12:26.33Z" }, { url = "https://files.pythonhosted.org/packages/37/75/62726948db36a56428fce4ba80a115716dc4fad6a3a4352487f8bb950966/numpy-2.4.2-cp314-cp314-win_arm64.whl", hash = "sha256:6ed0be1ee58eef41231a5c943d7d1375f093142702d5723ca2eb07db9b934b05", size = 10494886, upload-time = "2026-01-31T23:12:28.488Z" }, { url = "https://files.pythonhosted.org/packages/36/2f/ee93744f1e0661dc267e4b21940870cabfae187c092e1433b77b09b50ac4/numpy-2.4.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:98f16a80e917003a12c0580f97b5f875853ebc33e2eaa4bccfc8201ac6869308", size = 14818567, upload-time = "2026-01-31T23:12:30.709Z" }, { url = "https://files.pythonhosted.org/packages/a7/24/6535212add7d76ff938d8bdc654f53f88d35cddedf807a599e180dcb8e66/numpy-2.4.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:20abd069b9cda45874498b245c8015b18ace6de8546bf50dfa8cea1696ed06ef", size = 5328372, upload-time = "2026-01-31T23:12:32.962Z" }, { url = "https://files.pythonhosted.org/packages/5e/9d/c48f0a035725f925634bf6b8994253b43f2047f6778a54147d7e213bc5a7/numpy-2.4.2-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:e98c97502435b53741540a5717a6749ac2ada901056c7db951d33e11c885cc7d", size = 6649306, upload-time = "2026-01-31T23:12:34.797Z" }, { url = "https://files.pythonhosted.org/packages/81/05/7c73a9574cd4a53a25907bad38b59ac83919c0ddc8234ec157f344d57d9a/numpy-2.4.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da6cad4e82cb893db4b69105c604d805e0c3ce11501a55b5e9f9083b47d2ffe8", size = 15722394, upload-time = "2026-01-31T23:12:36.565Z" }, { url = "https://files.pythonhosted.org/packages/35/fa/4de10089f21fc7d18442c4a767ab156b25c2a6eaf187c0db6d9ecdaeb43f/numpy-2.4.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e4424677ce4b47fe73c8b5556d876571f7c6945d264201180db2dc34f676ab5", size = 16653343, upload-time = "2026-01-31T23:12:39.188Z" }, { url = "https://files.pythonhosted.org/packages/b8/f9/d33e4ffc857f3763a57aa85650f2e82486832d7492280ac21ba9efda80da/numpy-2.4.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2b8f157c8a6f20eb657e240f8985cc135598b2b46985c5bccbde7616dc9c6b1e", size = 17078045, upload-time = "2026-01-31T23:12:42.041Z" }, { url = "https://files.pythonhosted.org/packages/c8/b8/54bdb43b6225badbea6389fa038c4ef868c44f5890f95dd530a218706da3/numpy-2.4.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5daf6f3914a733336dab21a05cdec343144600e964d2fcdabaac0c0269874b2a", size = 18380024, upload-time = "2026-01-31T23:12:44.331Z" }, { url = "https://files.pythonhosted.org/packages/a5/55/6e1a61ded7af8df04016d81b5b02daa59f2ea9252ee0397cb9f631efe9e5/numpy-2.4.2-cp314-cp314t-win32.whl", hash = "sha256:8c50dd1fc8826f5b26a5ee4d77ca55d88a895f4e4819c7ecc2a9f5905047a443", size = 6153937, upload-time = "2026-01-31T23:12:47.229Z" }, { url = "https://files.pythonhosted.org/packages/45/aa/fa6118d1ed6d776b0983f3ceac9b1a5558e80df9365b1c3aa6d42bf9eee4/numpy-2.4.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fcf92bee92742edd401ba41135185866f7026c502617f422eb432cfeca4fe236", size = 12631844, upload-time = "2026-01-31T23:12:48.997Z" }, { url = "https://files.pythonhosted.org/packages/32/0a/2ec5deea6dcd158f254a7b372fb09cfba5719419c8d66343bab35237b3fb/numpy-2.4.2-cp314-cp314t-win_arm64.whl", hash = "sha256:1f92f53998a17265194018d1cc321b2e96e900ca52d54c7c77837b71b9465181", size = 10565379, upload-time = "2026-01-31T23:12:51.345Z" }, { url = "https://files.pythonhosted.org/packages/f4/f8/50e14d36d915ef64d8f8bc4a087fc8264d82c785eda6711f80ab7e620335/numpy-2.4.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:89f7268c009bc492f506abd6f5265defa7cb3f7487dc21d357c3d290add45082", size = 16833179, upload-time = "2026-01-31T23:12:53.5Z" }, { url = "https://files.pythonhosted.org/packages/17/17/809b5cad63812058a8189e91a1e2d55a5a18fd04611dbad244e8aeae465c/numpy-2.4.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:e6dee3bb76aa4009d5a912180bf5b2de012532998d094acee25d9cb8dee3e44a", size = 14889755, upload-time = "2026-01-31T23:12:55.933Z" }, { url = "https://files.pythonhosted.org/packages/3e/ea/181b9bcf7627fc8371720316c24db888dcb9829b1c0270abf3d288b2e29b/numpy-2.4.2-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:cd2bd2bbed13e213d6b55dc1d035a4f91748a7d3edc9480c13898b0353708920", size = 5399500, upload-time = "2026-01-31T23:12:58.671Z" }, { url = "https://files.pythonhosted.org/packages/33/9f/413adf3fc955541ff5536b78fcf0754680b3c6d95103230252a2c9408d23/numpy-2.4.2-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:cf28c0c1d4c4bf00f509fa7eb02c58d7caf221b50b467bcb0d9bbf1584d5c821", size = 6714252, upload-time = "2026-01-31T23:13:00.518Z" }, { url = "https://files.pythonhosted.org/packages/91/da/643aad274e29ccbdf42ecd94dafe524b81c87bcb56b83872d54827f10543/numpy-2.4.2-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e04ae107ac591763a47398bb45b568fc38f02dbc4aa44c063f67a131f99346cb", size = 15797142, upload-time = "2026-01-31T23:13:02.219Z" }, { url = "https://files.pythonhosted.org/packages/66/27/965b8525e9cb5dc16481b30a1b3c21e50c7ebf6e9dbd48d0c4d0d5089c7e/numpy-2.4.2-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:602f65afdef699cda27ec0b9224ae5dc43e328f4c24c689deaf77133dbee74d0", size = 16727979, upload-time = "2026-01-31T23:13:04.62Z" }, { url = "https://files.pythonhosted.org/packages/de/e5/b7d20451657664b07986c2f6e3be564433f5dcaf3482d68eaecd79afaf03/numpy-2.4.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be71bf1edb48ebbbf7f6337b5bfd2f895d1902f6335a5830b20141fc126ffba0", size = 12502577, upload-time = "2026-01-31T23:13:07.08Z" }, ] [[package]] name = "nvidia-nccl-cu12" version = "2.29.3" source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/28/cf/bcf8bb0c0030b1b9a345331f6281c37d2a8669758521eb93c382f6f87c8f/nvidia_nccl_cu12-2.29.3-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:6351b79dc7d2cc3d654ea1523616b9eeded71fe9c8da66b71eef9a5d1b2adad4", size = 289708535, upload-time = "2026-02-03T21:10:58.804Z" }, { url = "https://files.pythonhosted.org/packages/31/5a/cac7d231f322b66caa16fd4b136ebc8e4b18b2805811c2d58dc47210cdea/nvidia_nccl_cu12-2.29.3-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:35ad42e7d5d722a83c36a3a478e281c20a5646383deaf1b9ed1a9ab7d61bed53", size = 289760316, upload-time = "2026-02-03T21:11:37.899Z" }, ] [[package]] name = "nvidia-nccl-cu13" version = "2.29.3" source = { registry = "https://pypi.org/simple" } wheels = [ { url = "https://files.pythonhosted.org/packages/27/59/ff243ebe6fa1767a9135719829347f609a90607cfbba9637ba3e9b3e36ce/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:eab9f5c565ab3326906f1d1b5be5773a174c2a1b47002faed76f9e957392f713", size = 201042594, upload-time = "2026-02-03T21:10:54.736Z" }, { url = "https://files.pythonhosted.org/packages/7b/70/aae7806eeaed043b3e212da435880ad067b5f14052986a6b4c0a4c62f68a/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:2a321629f49490e4e0122ecb578a4b4a6f89e72740dd988e04dfa4758fab7fc3", size = 201104023, upload-time = "2026-02-03T21:11:24.071Z" }, ] [[package]] name = "packaging" version = "26.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, ] [[package]] name = "pluggy" version = "1.6.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] [[package]] name = "pygments" version = "2.19.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] [[package]] name = "pytest" version = "9.0.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, { name = "exceptiongroup", marker = "python_full_version < '3.11' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, { name = "pygments" }, { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-7-nccl4py-cu12' and extra == 'extra-7-nccl4py-cu13')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, ] [[package]] name = "pytest-cov" version = "7.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "coverage", extra = ["toml"] }, { name = "pluggy" }, { name = "pytest" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, ] [[package]] name = "pytest-mpi" version = "0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6c/db/b9d4b23750eb91865787656dce02a2864397aa1ee130df00ec73d3954e7e/pytest-mpi-0.6.tar.gz", hash = "sha256:09b3cd3511f8f3cd4d205f54d4a7223724fed0ab68b872ed1123d312152325a9", size = 35329, upload-time = "2022-01-08T02:19:26.461Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a6/2b/0ed49de84e96ebf771c86a16d88b48c08d291627cfcdce30973f8538c99e/pytest_mpi-0.6-py2.py3-none-any.whl", hash = "sha256:1b7e193fb3be31d08c8e4dd7435e8e13e14b17ead6a6fc6aa07a6d3c7145590b", size = 5907, upload-time = "2022-01-08T02:19:24.8Z" }, ] [[package]] name = "tomli" version = "2.4.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/82/30/31573e9457673ab10aa432461bee537ce6cef177667deca369efb79df071/tomli-2.4.0.tar.gz", hash = "sha256:aa89c3f6c277dd275d8e243ad24f3b5e701491a860d5121f2cdd399fbb31fc9c", size = 17477, upload-time = "2026-01-11T11:22:38.165Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/3c/d9/3dc2289e1f3b32eb19b9785b6a006b28ee99acb37d1d47f78d4c10e28bf8/tomli-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b5ef256a3fd497d4973c11bf142e9ed78b150d36f5773f1ca6088c230ffc5867", size = 153663, upload-time = "2026-01-11T11:21:45.27Z" }, { url = "https://files.pythonhosted.org/packages/51/32/ef9f6845e6b9ca392cd3f64f9ec185cc6f09f0a2df3db08cbe8809d1d435/tomli-2.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5572e41282d5268eb09a697c89a7bee84fae66511f87533a6f88bd2f7b652da9", size = 148469, upload-time = "2026-01-11T11:21:46.873Z" }, { url = "https://files.pythonhosted.org/packages/d6/c2/506e44cce89a8b1b1e047d64bd495c22c9f71f21e05f380f1a950dd9c217/tomli-2.4.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:551e321c6ba03b55676970b47cb1b73f14a0a4dce6a3e1a9458fd6d921d72e95", size = 236039, upload-time = "2026-01-11T11:21:48.503Z" }, { url = "https://files.pythonhosted.org/packages/b3/40/e1b65986dbc861b7e986e8ec394598187fa8aee85b1650b01dd925ca0be8/tomli-2.4.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e3f639a7a8f10069d0e15408c0b96a2a828cfdec6fca05296ebcdcc28ca7c76", size = 243007, upload-time = "2026-01-11T11:21:49.456Z" }, { url = "https://files.pythonhosted.org/packages/9c/6f/6e39ce66b58a5b7ae572a0f4352ff40c71e8573633deda43f6a379d56b3e/tomli-2.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1b168f2731796b045128c45982d3a4874057626da0e2ef1fdd722848b741361d", size = 240875, upload-time = "2026-01-11T11:21:50.755Z" }, { url = "https://files.pythonhosted.org/packages/aa/ad/cb089cb190487caa80204d503c7fd0f4d443f90b95cf4ef5cf5aa0f439b0/tomli-2.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:133e93646ec4300d651839d382d63edff11d8978be23da4cc106f5a18b7d0576", size = 246271, upload-time = "2026-01-11T11:21:51.81Z" }, { url = "https://files.pythonhosted.org/packages/0b/63/69125220e47fd7a3a27fd0de0c6398c89432fec41bc739823bcc66506af6/tomli-2.4.0-cp311-cp311-win32.whl", hash = "sha256:b6c78bdf37764092d369722d9946cb65b8767bfa4110f902a1b2542d8d173c8a", size = 96770, upload-time = "2026-01-11T11:21:52.647Z" }, { url = "https://files.pythonhosted.org/packages/1e/0d/a22bb6c83f83386b0008425a6cd1fa1c14b5f3dd4bad05e98cf3dbbf4a64/tomli-2.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:d3d1654e11d724760cdb37a3d7691f0be9db5fbdaef59c9f532aabf87006dbaa", size = 107626, upload-time = "2026-01-11T11:21:53.459Z" }, { url = "https://files.pythonhosted.org/packages/2f/6d/77be674a3485e75cacbf2ddba2b146911477bd887dda9d8c9dfb2f15e871/tomli-2.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:cae9c19ed12d4e8f3ebf46d1a75090e4c0dc16271c5bce1c833ac168f08fb614", size = 94842, upload-time = "2026-01-11T11:21:54.831Z" }, { url = "https://files.pythonhosted.org/packages/3c/43/7389a1869f2f26dba52404e1ef13b4784b6b37dac93bac53457e3ff24ca3/tomli-2.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:920b1de295e72887bafa3ad9f7a792f811847d57ea6b1215154030cf131f16b1", size = 154894, upload-time = "2026-01-11T11:21:56.07Z" }, { url = "https://files.pythonhosted.org/packages/e9/05/2f9bf110b5294132b2edf13fe6ca6ae456204f3d749f623307cbb7a946f2/tomli-2.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d6d9a4aee98fac3eab4952ad1d73aee87359452d1c086b5ceb43ed02ddb16b8", size = 149053, upload-time = "2026-01-11T11:21:57.467Z" }, { url = "https://files.pythonhosted.org/packages/e8/41/1eda3ca1abc6f6154a8db4d714a4d35c4ad90adc0bcf700657291593fbf3/tomli-2.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:36b9d05b51e65b254ea6c2585b59d2c4cb91c8a3d91d0ed0f17591a29aaea54a", size = 243481, upload-time = "2026-01-11T11:21:58.661Z" }, { url = "https://files.pythonhosted.org/packages/d2/6d/02ff5ab6c8868b41e7d4b987ce2b5f6a51d3335a70aa144edd999e055a01/tomli-2.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c8a885b370751837c029ef9bc014f27d80840e48bac415f3412e6593bbc18c1", size = 251720, upload-time = "2026-01-11T11:22:00.178Z" }, { url = "https://files.pythonhosted.org/packages/7b/57/0405c59a909c45d5b6f146107c6d997825aa87568b042042f7a9c0afed34/tomli-2.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8768715ffc41f0008abe25d808c20c3d990f42b6e2e58305d5da280ae7d1fa3b", size = 247014, upload-time = "2026-01-11T11:22:01.238Z" }, { url = "https://files.pythonhosted.org/packages/2c/0e/2e37568edd944b4165735687cbaf2fe3648129e440c26d02223672ee0630/tomli-2.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b438885858efd5be02a9a133caf5812b8776ee0c969fea02c45e8e3f296ba51", size = 251820, upload-time = "2026-01-11T11:22:02.727Z" }, { url = "https://files.pythonhosted.org/packages/5a/1c/ee3b707fdac82aeeb92d1a113f803cf6d0f37bdca0849cb489553e1f417a/tomli-2.4.0-cp312-cp312-win32.whl", hash = "sha256:0408e3de5ec77cc7f81960c362543cbbd91ef883e3138e81b729fc3eea5b9729", size = 97712, upload-time = "2026-01-11T11:22:03.777Z" }, { url = "https://files.pythonhosted.org/packages/69/13/c07a9177d0b3bab7913299b9278845fc6eaaca14a02667c6be0b0a2270c8/tomli-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:685306e2cc7da35be4ee914fd34ab801a6acacb061b6a7abca922aaf9ad368da", size = 108296, upload-time = "2026-01-11T11:22:04.86Z" }, { url = "https://files.pythonhosted.org/packages/18/27/e267a60bbeeee343bcc279bb9e8fbed0cbe224bc7b2a3dc2975f22809a09/tomli-2.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:5aa48d7c2356055feef06a43611fc401a07337d5b006be13a30f6c58f869e3c3", size = 94553, upload-time = "2026-01-11T11:22:05.854Z" }, { url = "https://files.pythonhosted.org/packages/34/91/7f65f9809f2936e1f4ce6268ae1903074563603b2a2bd969ebbda802744f/tomli-2.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84d081fbc252d1b6a982e1870660e7330fb8f90f676f6e78b052ad4e64714bf0", size = 154915, upload-time = "2026-01-11T11:22:06.703Z" }, { url = "https://files.pythonhosted.org/packages/20/aa/64dd73a5a849c2e8f216b755599c511badde80e91e9bc2271baa7b2cdbb1/tomli-2.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9a08144fa4cba33db5255f9b74f0b89888622109bd2776148f2597447f92a94e", size = 149038, upload-time = "2026-01-11T11:22:07.56Z" }, { url = "https://files.pythonhosted.org/packages/9e/8a/6d38870bd3d52c8d1505ce054469a73f73a0fe62c0eaf5dddf61447e32fa/tomli-2.4.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c73add4bb52a206fd0c0723432db123c0c75c280cbd67174dd9d2db228ebb1b4", size = 242245, upload-time = "2026-01-11T11:22:08.344Z" }, { url = "https://files.pythonhosted.org/packages/59/bb/8002fadefb64ab2669e5b977df3f5e444febea60e717e755b38bb7c41029/tomli-2.4.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fb2945cbe303b1419e2706e711b7113da57b7db31ee378d08712d678a34e51e", size = 250335, upload-time = "2026-01-11T11:22:09.951Z" }, { url = "https://files.pythonhosted.org/packages/a5/3d/4cdb6f791682b2ea916af2de96121b3cb1284d7c203d97d92d6003e91c8d/tomli-2.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bbb1b10aa643d973366dc2cb1ad94f99c1726a02343d43cbc011edbfac579e7c", size = 245962, upload-time = "2026-01-11T11:22:11.27Z" }, { url = "https://files.pythonhosted.org/packages/f2/4a/5f25789f9a460bd858ba9756ff52d0830d825b458e13f754952dd15fb7bb/tomli-2.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4cbcb367d44a1f0c2be408758b43e1ffb5308abe0ea222897d6bfc8e8281ef2f", size = 250396, upload-time = "2026-01-11T11:22:12.325Z" }, { url = "https://files.pythonhosted.org/packages/aa/2f/b73a36fea58dfa08e8b3a268750e6853a6aac2a349241a905ebd86f3047a/tomli-2.4.0-cp313-cp313-win32.whl", hash = "sha256:7d49c66a7d5e56ac959cb6fc583aff0651094ec071ba9ad43df785abc2320d86", size = 97530, upload-time = "2026-01-11T11:22:13.865Z" }, { url = "https://files.pythonhosted.org/packages/3b/af/ca18c134b5d75de7e8dc551c5234eaba2e8e951f6b30139599b53de9c187/tomli-2.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:3cf226acb51d8f1c394c1b310e0e0e61fecdd7adcb78d01e294ac297dd2e7f87", size = 108227, upload-time = "2026-01-11T11:22:15.224Z" }, { url = "https://files.pythonhosted.org/packages/22/c3/b386b832f209fee8073c8138ec50f27b4460db2fdae9ffe022df89a57f9b/tomli-2.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:d20b797a5c1ad80c516e41bc1fb0443ddb5006e9aaa7bda2d71978346aeb9132", size = 94748, upload-time = "2026-01-11T11:22:16.009Z" }, { url = "https://files.pythonhosted.org/packages/f3/c4/84047a97eb1004418bc10bdbcfebda209fca6338002eba2dc27cc6d13563/tomli-2.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:26ab906a1eb794cd4e103691daa23d95c6919cc2fa9160000ac02370cc9dd3f6", size = 154725, upload-time = "2026-01-11T11:22:17.269Z" }, { url = "https://files.pythonhosted.org/packages/a8/5d/d39038e646060b9d76274078cddf146ced86dc2b9e8bbf737ad5983609a0/tomli-2.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:20cedb4ee43278bc4f2fee6cb50daec836959aadaf948db5172e776dd3d993fc", size = 148901, upload-time = "2026-01-11T11:22:18.287Z" }, { url = "https://files.pythonhosted.org/packages/73/e5/383be1724cb30f4ce44983d249645684a48c435e1cd4f8b5cded8a816d3c/tomli-2.4.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:39b0b5d1b6dd03684b3fb276407ebed7090bbec989fa55838c98560c01113b66", size = 243375, upload-time = "2026-01-11T11:22:19.154Z" }, { url = "https://files.pythonhosted.org/packages/31/f0/bea80c17971c8d16d3cc109dc3585b0f2ce1036b5f4a8a183789023574f2/tomli-2.4.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a26d7ff68dfdb9f87a016ecfd1e1c2bacbe3108f4e0f8bcd2228ef9a766c787d", size = 250639, upload-time = "2026-01-11T11:22:20.168Z" }, { url = "https://files.pythonhosted.org/packages/2c/8f/2853c36abbb7608e3f945d8a74e32ed3a74ee3a1f468f1ffc7d1cb3abba6/tomli-2.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:20ffd184fb1df76a66e34bd1b36b4a4641bd2b82954befa32fe8163e79f1a702", size = 246897, upload-time = "2026-01-11T11:22:21.544Z" }, { url = "https://files.pythonhosted.org/packages/49/f0/6c05e3196ed5337b9fe7ea003e95fd3819a840b7a0f2bf5a408ef1dad8ed/tomli-2.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:75c2f8bbddf170e8effc98f5e9084a8751f8174ea6ccf4fca5398436e0320bc8", size = 254697, upload-time = "2026-01-11T11:22:23.058Z" }, { url = "https://files.pythonhosted.org/packages/f3/f5/2922ef29c9f2951883525def7429967fc4d8208494e5ab524234f06b688b/tomli-2.4.0-cp314-cp314-win32.whl", hash = "sha256:31d556d079d72db7c584c0627ff3a24c5d3fb4f730221d3444f3efb1b2514776", size = 98567, upload-time = "2026-01-11T11:22:24.033Z" }, { url = "https://files.pythonhosted.org/packages/7b/31/22b52e2e06dd2a5fdbc3ee73226d763b184ff21fc24e20316a44ccc4d96b/tomli-2.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:43e685b9b2341681907759cf3a04e14d7104b3580f808cfde1dfdb60ada85475", size = 108556, upload-time = "2026-01-11T11:22:25.378Z" }, { url = "https://files.pythonhosted.org/packages/48/3d/5058dff3255a3d01b705413f64f4306a141a8fd7a251e5a495e3f192a998/tomli-2.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:3d895d56bd3f82ddd6faaff993c275efc2ff38e52322ea264122d72729dca2b2", size = 96014, upload-time = "2026-01-11T11:22:26.138Z" }, { url = "https://files.pythonhosted.org/packages/b8/4e/75dab8586e268424202d3a1997ef6014919c941b50642a1682df43204c22/tomli-2.4.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5b5807f3999fb66776dbce568cc9a828544244a8eb84b84b9bafc080c99597b9", size = 163339, upload-time = "2026-01-11T11:22:27.143Z" }, { url = "https://files.pythonhosted.org/packages/06/e3/b904d9ab1016829a776d97f163f183a48be6a4deb87304d1e0116a349519/tomli-2.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c084ad935abe686bd9c898e62a02a19abfc9760b5a79bc29644463eaf2840cb0", size = 159490, upload-time = "2026-01-11T11:22:28.399Z" }, { url = "https://files.pythonhosted.org/packages/e3/5a/fc3622c8b1ad823e8ea98a35e3c632ee316d48f66f80f9708ceb4f2a0322/tomli-2.4.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f2e3955efea4d1cfbcb87bc321e00dc08d2bcb737fd1d5e398af111d86db5df", size = 269398, upload-time = "2026-01-11T11:22:29.345Z" }, { url = "https://files.pythonhosted.org/packages/fd/33/62bd6152c8bdd4c305ad9faca48f51d3acb2df1f8791b1477d46ff86e7f8/tomli-2.4.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e0fe8a0b8312acf3a88077a0802565cb09ee34107813bba1c7cd591fa6cfc8d", size = 276515, upload-time = "2026-01-11T11:22:30.327Z" }, { url = "https://files.pythonhosted.org/packages/4b/ff/ae53619499f5235ee4211e62a8d7982ba9e439a0fb4f2f351a93d67c1dd2/tomli-2.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:413540dce94673591859c4c6f794dfeaa845e98bf35d72ed59636f869ef9f86f", size = 273806, upload-time = "2026-01-11T11:22:32.56Z" }, { url = "https://files.pythonhosted.org/packages/47/71/cbca7787fa68d4d0a9f7072821980b39fbb1b6faeb5f5cf02f4a5559fa28/tomli-2.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0dc56fef0e2c1c470aeac5b6ca8cc7b640bb93e92d9803ddaf9ea03e198f5b0b", size = 281340, upload-time = "2026-01-11T11:22:33.505Z" }, { url = "https://files.pythonhosted.org/packages/f5/00/d595c120963ad42474cf6ee7771ad0d0e8a49d0f01e29576ee9195d9ecdf/tomli-2.4.0-cp314-cp314t-win32.whl", hash = "sha256:d878f2a6707cc9d53a1be1414bbb419e629c3d6e67f69230217bb663e76b5087", size = 108106, upload-time = "2026-01-11T11:22:34.451Z" }, { url = "https://files.pythonhosted.org/packages/de/69/9aa0c6a505c2f80e519b43764f8b4ba93b5a0bbd2d9a9de6e2b24271b9a5/tomli-2.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2add28aacc7425117ff6364fe9e06a183bb0251b03f986df0e78e974047571fd", size = 120504, upload-time = "2026-01-11T11:22:35.764Z" }, { url = "https://files.pythonhosted.org/packages/b3/9f/f1668c281c58cfae01482f7114a4b88d345e4c140386241a1a24dcc9e7bc/tomli-2.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2b1e3b80e1d5e52e40e9b924ec43d81570f0e7d09d11081b797bc4692765a3d4", size = 99561, upload-time = "2026-01-11T11:22:36.624Z" }, { url = "https://files.pythonhosted.org/packages/23/d1/136eb2cb77520a31e1f64cbae9d33ec6df0d78bdf4160398e86eec8a8754/tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a", size = 14477, upload-time = "2026-01-11T11:22:37.446Z" }, ] [[package]] name = "typing-extensions" version = "4.15.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] nccl-2.29.7-1/cmake/000077500000000000000000000000001515037102200137445ustar00rootroot00000000000000nccl-2.29.7-1/cmake/NCCLConfig.cmake.in000066400000000000000000000012671515037102200172260ustar00rootroot00000000000000@PACKAGE_INIT@ include(CMakeFindDependencyMacro) find_dependency(CUDAToolkit) find_dependency(Threads) include("${CMAKE_CURRENT_LIST_DIR}/NCCLTargets.cmake") if(TARGET CUDA::cudart) get_target_property(_nccl_cuda_includes CUDA::cudart INTERFACE_INCLUDE_DIRECTORIES) if(_nccl_cuda_includes) foreach(_t NCCL::nccl NCCL::nccl_static) if(TARGET ${_t}) set_property(TARGET ${_t} APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${_nccl_cuda_includes}") endif() endforeach() endif() unset(_nccl_cuda_includes) endif() set(NCCL_VERSION "@PROJECT_VERSION@") set(NCCL_INCLUDE_DIRS "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") set(NCCL_LIBRARIES NCCL::nccl) nccl-2.29.7-1/docs/000077500000000000000000000000001515037102200136145ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/000077500000000000000000000000001515037102200154325ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/01_communicators/000077500000000000000000000000001515037102200206155ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/01_communicators/01_multiple_devices_single_process/000077500000000000000000000000001515037102200275515ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/01_communicators/01_multiple_devices_single_process/Makefile000066400000000000000000000027651515037102200312230ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Include common build rules include ../../../../makefiles/common.mk include ../../../../makefiles/examples.mk # Target executable TARGET = multiple_devices_single_process # Source files SOURCES = main.cc OBJECTS = $(SOURCES:.cc=.o) # Default target all: $(TARGET) # Build executable $(TARGET): $(OBJECTS) $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ @echo "Built target $@" # Compile source files %.o: %.cc $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ # Test target test: $(TARGET) @echo "Testing $(TARGET)..." @echo "Running with all available GPUs" ./$(TARGET) # Clean build artifacts clean: rm -f $(OBJECTS) $(TARGET) # Install target install: $(TARGET) @mkdir -p $(PREFIX)/bin cp $(TARGET) $(PREFIX)/bin/ # Help help: @echo "NCCL Example: Multiple Devices Single Process" @echo "==============================================" @echo "" @echo "This example shows how to use ncclCommInitAll to create" @echo "communicators for multiple GPUs in a single process." @echo "" @echo "Targets:" @echo " all - Build the example (default)" @echo " test - Build and run test with all GPUs" @echo " clean - Remove build artifacts" @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" @echo " help - Show this help" .PHONY: all test clean install help nccl-2.29.7-1/docs/examples/01_communicators/01_multiple_devices_single_process/README.md000066400000000000000000000117131515037102200310330ustar00rootroot00000000000000 # NCCL Example: Multiple Devices Single Process This example demonstrates how to use `ncclCommInitAll` to create NCCL communicators for multiple GPUs within a single process, without requiring MPI or threading. ## Overview The `ncclCommInitAll` function provides a simplified way to initialize NCCL communicators when: - All GPUs are managed by a single process - Running on a single node - No multi-process coordination is needed This approach is ideal for single-node multi-GPU applications where simplicity is preferred over the flexibility of multi-process setups. ## What This Example Does 1. **Device Detection**: - Queries available CUDA devices - Lists device properties for each GPU 2. **Communicator Creation**: - Uses `ncclCommInitAll` to create all communicators in one call - Automatically assigns NCCL ranks 0 through n-1 - No NCCL unique ID distribution needed 3. **Verification**: - Displays communicator information for each GPU - Shows rank assignments and device mappings - Confirms successful initialization 4. **Cleanup**: - Properly destroys communicators and streams - Demonstrates correct resource management ## Building and Running ### Build ```shell make [NCCL_HOME=] [CUDA_HOME=] ``` ### Run with all available GPUs ```shell ./multiple_devices_single_process ``` ### Run with specific GPUs ```shell # Use only GPUs 0 and 1 CUDA_VISIBLE_DEVICES=0,1 ./multiple_devices_single_process ``` ### Run with NCCL debug output ```shell NCCL_DEBUG=INFO ./multiple_devices_single_process ``` ## Code Walk-through ### Key Function: ncclCommInitAll For single-node collective examples we use `ncclCommInitAll` as it creates a clique of communicators in one call. ```c int num_gpus; // num_gpus is set by querying the CUDA devices ncclComm_t* comms; int* devices; // devices needs to be populated with CUDA devices used // Create communicators for all devices in one call NCCLCHECK(ncclCommInitAll(comms, num_gpus, devices)); ``` This single function call: - Creates `num_gpus` communicators - Assigns ranks 0 to (num_gpus-1) - Sets up internal communication paths - No unique ID needed ### Comparison with ncclCommInitRank `ncclCommInitAll` is a convenience function and has the same functionality as: ```c ncclUniqueId id; ncclGetUniqueId(&id); ncclGroupStart(); for(int i = 0; i < num_gpus; i++) { cudaSetDevice(i); ncclCommInitRank(comms[i], num_gpus, id, devices[i]); } ncclGroupEnd(); ``` ## Expected Output ``` Found 4 CUDA device(s) available Available GPU devices: GPU 0: NVIDIA A100-SXM4-40GB (CUDA Device 0) Compute Capability: 8.0 Memory: 40.0 GB GPU 1: NVIDIA A100-SXM4-40GB (CUDA Device 1) Compute Capability: 8.0 Memory: 40.0 GB GPU 2: NVIDIA A100-SXM4-40GB (CUDA Device 2) Compute Capability: 8.0 Memory: 40.0 GB GPU 3: NVIDIA A100-SXM4-40GB (CUDA Device 3) Compute Capability: 8.0 Memory: 40.0 GB Using ncclCommInitAll() to create all communicators simultaneously All 4 NCCL communicators initialized successfully Communicator Details: Communicator 0: Rank 0/4 on CUDA device 0 Communicator 1: Rank 1/4 on CUDA device 1 Communicator 2: Rank 2/4 on CUDA device 2 Communicator 3: Rank 3/4 on CUDA device 3 All communicators have the expected size of 4 Synchronizing all CUDA streams... All streams synchronized Destroying NCCL communicators... All NCCL communicators destroyed Destroying CUDA streams... All CUDA streams destroyed ============================================================= SUCCESS: Multiple devices single process example completed! ============================================================= ``` ## When to Use ncclCommInitAll ### Ideal Use Cases - **Single-node workloads**: All GPUs on one machine - **Simple applications**: No multi-process complexity needed - **Testing/Development**: Quick setup for experiments ### When NOT to Use - **Multi-node clusters**: Need MPI for cross-node communication - **Process isolation**: When GPUs should be in separate processes ## Performance Considerations - **Advantages**: - Lower overhead (no inter process communication) - Simpler memory management - Direct access to all GPUs - **Disadvantages**: - Limited by single process resources - Cannot scale beyond one node ## Common Issues and Solutions 1. **Not all GPUs visible**: - Check `CUDA_VISIBLE_DEVICES` - Ensure user has permissions for all GPUs - Verify no other process is using GPUs exclusively 2. **Out of memory**: - Single process must handle memory for all GPUs - Consider using multiple processes if memory limited ## Next Steps After understanding this example: 1. Try the collective operation examples using `ncclCommInitAll` 2. Compare performance with MPI-based multi-process approach 3. Experiment with different GPU combinations nccl-2.29.7-1/docs/examples/01_communicators/01_multiple_devices_single_process/main.cc000066400000000000000000000244051515037102200310110ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "cuda_runtime.h" #include "nccl.h" #include #include #include /* * NCCL Example: Multiple Devices Single Process * ============================================= * * PURPOSE: * This example demonstrates how to initialize NCCL communicators for multiple * GPUs within a single process. This is the simplest NCCL setup and is ideal * for learning NCCL basics or for applications that want to use multiple GPUs * without the complexity of multi-process coordination. * * LEARNING OBJECTIVES: * - Learn how to use ncclCommInitAll() for simple multi-GPU setups * - See proper NCCL communicator lifecycle management * - Understand GPU device management in NCCL applications * - Learn proper resource cleanup patterns * * HOW IT WORKS: * 1. Detect all available CUDA devices * 2. Create communicators for all devices using ncclCommInitAll() * 3. Verify communicator properties (rank, size, device assignment) * 4. Clean up all resources properly * * KEY CONCEPTS: * - ncclCommInitAll(): Creates multiple communicators in a single call * - Single-process topology: All GPUs managed by one process * - Device management: Setting active CUDA device for operations * - Stream management: Each GPU gets its own CUDA stream * * WHEN TO USE THIS PATTERN: * - Learning NCCL fundamentals * - Single-node, multi-GPU applications * - Applications that don't need multi-node scaling * - Prototyping and testing NCCL functionality * * USAGE EXAMPLES: * ./multiple_devices_single_process # Use all available GPUs * * EXPECTED OUTPUT: * - Detection of all available GPUs * - Successful communicator initialization * - Display of rank/size information for each GPU * - Clean resource cleanup confirmation */ // Enhanced error checking macro for NCCL operations // Provides detailed error information including the failed operation #define NCCLCHECK(cmd) \ do { \ ncclResult_t res = cmd; \ if (res != ncclSuccess) { \ fprintf(stderr, "Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \ ncclGetErrorString(res)); \ fprintf(stderr, "Failed NCCL operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) #define CUDACHECK(cmd) \ do { \ cudaError_t err = cmd; \ if (err != cudaSuccess) { \ fprintf(stderr, "Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \ cudaGetErrorString(err)); \ fprintf(stderr, "Failed CUDA operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) // ============================================================================= // MAIN FUNCTION - NCCL Communicator Lifecycle Example // ============================================================================= int main(int argc, char *argv[]) { // Variables for managing multiple GPU communicators int num_gpus; // Number of available CUDA devices ncclComm_t *comms = NULL; // Array of NCCL communicators (one per GPU) cudaStream_t *streams = NULL; // Array of CUDA streams (one per GPU) int *devices = NULL; // Array of device IDs to use // Discover how many CUDA devices are available // This determines how many NCCL communicators we'll create CUDACHECK(cudaGetDeviceCount(&num_gpus)); if (num_gpus == 0) { fprintf(stderr, "ERROR: No CUDA devices found on this system\n"); fprintf( stderr, "Please ensure CUDA is properly installed and GPUs are available\n"); return 1; } printf("Found %d CUDA device(s) available\n\n", num_gpus); // ========================================================================= // STEP 1: Prepare Device Information and Memory Allocation // ========================================================================= // Allocate arrays to hold our per-device resources // We need one communicator, stream, and device ID per GPU devices = (int *)malloc(num_gpus * sizeof(int)); comms = (ncclComm_t *)malloc(num_gpus * sizeof(ncclComm_t)); streams = (cudaStream_t *)malloc(num_gpus * sizeof(cudaStream_t)); if (!devices || !comms || !streams) { fprintf(stderr, "ERROR: Failed to allocate memory for device arrays\n"); return 1; } // Create device list and display device information // By default, we use all available devices (0, 1, 2, ...) printf("Available GPU devices:\n"); for (int i = 0; i < num_gpus; i++) { devices[i] = i; // Use device i for communicator i // Query device properties for informational display cudaDeviceProp prop; CUDACHECK(cudaGetDeviceProperties(&prop, devices[i])); printf(" GPU %d: %s (CUDA Device %d)\n", i, prop.name, devices[i]); printf(" Compute Capability: %d.%d\n", prop.major, prop.minor); printf(" Memory: %.1f GB\n", prop.totalGlobalMem / (1024.0 * 1024.0 * 1024.0)); } // Create a CUDA stream for each GPU // Each GPU needs its own stream for optimal performance for (int i = 0; i < num_gpus; i++) { // Set the active CUDA device before creating resources on it // This ensures the stream is created on the correct GPU CUDACHECK(cudaSetDevice(devices[i])); CUDACHECK(cudaStreamCreate(&streams[i])); } // ========================================================================= // STEP 2 : Initialize NCCL Communicators // ========================================================================= printf("Using ncclCommInitAll() to create all communicators " "simultaneously\n"); // ncclCommInitAll() creates all communicators at once and handles the // coordination internally // // Parameters: // - comms: Array to store the created communicators // - num_gpus: Number of communicators to create // - devices: Array of CUDA device IDs to use // // After this call: // - comms[0] will be the communicator for devices[0] with rank 0 // - comms[1] will be the communicator for devices[1] with rank 1 // - ... and so on // // All communicators will have the same 'size' (total number of // participants) NCCLCHECK(ncclCommInitAll(comms, num_gpus, devices)); printf("All %d NCCL communicators initialized successfully\n\n", num_gpus); // ========================================================================= // STEP 3: Create CUDA Streams and Verify Communicator Properties // ========================================================================= printf("Communicator Details:\n"); bool sizes_match = true; for (int i = 0; i < num_gpus; i++) { // Query the communicator to verify it was set up correctly // These calls validate that NCCL properly assigned ranks and devices int rank, size, device; // Get this communicator's rank NCCLCHECK(ncclCommUserRank(comms[i], &rank)); // Get total number of participants NCCLCHECK(ncclCommCount(comms[i], &size)); // Get assigned CUDA device NCCLCHECK(ncclCommCuDevice(comms[i], &device)); printf(" Communicator %d: Rank %d/%d on CUDA device %d", i, rank, size, device); // Verify the assignment is correct if (rank != i) { printf(" [WARNING: Expected rank %d]", i); } if (device != devices[i]) { printf(" [WARNING: Expected device %d]", devices[i]); } printf("\n"); // Verify that all communicators have the expected size if (size != num_gpus) { printf("WARNING: Communicator %d has size %d, expected %d\n", i, size, num_gpus); sizes_match = false; } } if (sizes_match) printf("All communicators have the expected size of %d\n", num_gpus); printf("\n"); // ========================================================================= // STEP 4: Cleanup and Resource Management // ========================================================================= // IMPORTANT: Proper cleanup is critical for NCCL applications // Resources must be cleaned up in the correct order to avoid issues // First, synchronize all streams to ensure no operations are in flight // This prevents destroying resources while they're still being used printf("Synchronizing all CUDA streams...\n"); for (int i = 0; i < num_gpus; i++) { CUDACHECK(cudaSetDevice(devices[i])); CUDACHECK(cudaStreamSynchronize(streams[i])); } printf("All streams synchronized\n"); // Next, destroy NCCL communicators first // This must be done before destroying CUDA resources they depend on printf("Destroying NCCL communicators...\n"); for (int i = 0; i < num_gpus; i++) { NCCLCHECK(ncclCommFinalize(comms[i])); NCCLCHECK(ncclCommDestroy(comms[i])); } printf("All NCCL communicators destroyed\n"); // Finally, destroy CUDA streams // This is safe now that the communicators are gone printf("Destroying CUDA streams...\n"); for (int i = 0; i < num_gpus; i++) { CUDACHECK(cudaSetDevice(devices[i])); CUDACHECK(cudaStreamDestroy(streams[i])); } printf("All CUDA streams destroyed\n"); // Free host memory allocations free(devices); free(comms); free(streams); printf("\n=============================================================\n"); printf("SUCCESS: Multiple devices single process example completed!\n"); printf("=============================================================\n\n"); return 0; } nccl-2.29.7-1/docs/examples/01_communicators/02_one_device_per_pthread/000077500000000000000000000000001515037102200255735ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/01_communicators/02_one_device_per_pthread/Makefile000066400000000000000000000032241515037102200272340ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Include common build rules include ../../../../makefiles/common.mk include ../../../../makefiles/examples.mk # Target executable TARGET = one_device_per_pthread # Add pthread support LDFLAGS += -lpthread # Source files SOURCES = main.cc OBJECTS = $(SOURCES:.cc=.o) # Default target all: $(TARGET) # Build executable $(TARGET): $(OBJECTS) $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ @echo "Built target $@" # Compile source files %.o: %.cc $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ # Test target test: $(TARGET) @echo "Testing $(TARGET)..." @echo "Running with default thread count (number of GPUs)" ./$(TARGET) @echo "" @if [ "$$(nvidia-smi -L | wc -l)" -ge 2 ]; then \ echo "Running with 2 threads"; \ NTHREADS=2 ./$(TARGET); \ fi # Clean build artifacts clean: rm -f $(OBJECTS) $(TARGET) # Install target install: $(TARGET) @mkdir -p $(PREFIX)/bin cp $(TARGET) $(PREFIX)/bin/ # Help help: @echo "NCCL Example: One Device per Thread (pthread)" @echo "============================================" @echo "" @echo "This example shows how to use ncclCommInitRank to create" @echo "communicators for multiple GPUs using pthreads." @echo "" @echo "Targets:" @echo " all - Build the example (default)" @echo " test - Build and run basic tests" @echo " clean - Remove build artifacts" @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" @echo " help - Show this help" .PHONY: all test clean install help nccl-2.29.7-1/docs/examples/01_communicators/02_one_device_per_pthread/README.md000066400000000000000000000115361515037102200270600ustar00rootroot00000000000000 # NCCL Example: One Device per Thread (pthread) This example demonstrates NCCL communicator lifecycle management using pthreads, with one GPU per thread. ## Overview This example shows how to use NCCL in a multi-threaded environment where each pthread manages one GPU device. It demonstrates the proper initialization and cleanup sequence for NCCL communicators within threads. ## What This Example Does 1. **Thread Creation**: - Creates one pthread per available GPU or `NTHREADS` if set - Each thread manages its own CUDA device context 2. **Communicator Creation**: - Uses `ncclCommInitRank` with unique ID across threads - Each thread initializes its own communicator - Demonstrates thread-safe NCCL initialization 3. **Verification**: - Queries communicator properties (rank, size, device) - Confirms successful initialization across all threads 4. **Cleanup**: - Proper resource cleanup order within each thread - Demonstrates correct NCCL and CUDA resource management ## Building and Running ### Build ```shell make [NCCL_HOME=] [CUDA_HOME=] ``` ### Run with specific thread count (number of GPUs) ```shell [NTHREADS=n] ./one_device_per_pthread ``` ### Run with NCCL debug output ```shell NCCL_DEBUG=INFO ./one_device_per_pthread ``` ## Code Walk-through ### Key Function: ncclCommInitRank in threads ```c // Each thread creates it's own copy of struct `threadData_t`. typedef struct { int thread_id; // thread_id is set when thread is created int num_gpus; // num_gpus is set by querying the CUDA devices ncclUniqueId commId; // commId is set by ncclGetUniqueId ncclComm_t* comms; } threadData_t; threadData_t* data; // Each thread initializes its own communicator NCCLCHECK(ncclCommInitRank(&data->comms[thread_id], data->num_gpus, data->commId, data->thread_id)); ``` In this approach: - Each thread gets its own NCCL rank (0, 1, 2...) - Does not need explicit distribution of `uniqueId` since it uses a global variable. ## Expected Output ``` Using 4 devices with pthreads Creating 4 threads for NCCL communicators Thread 0: Set device 0 and created stream Thread 1: Set device 1 and created stream Thread 2: Set device 2 and created stream Thread 3: Set device 3 and created stream Thread 0: NCCL communicator initialized Thread 1: NCCL communicator initialized Thread 2: NCCL communicator initialized Thread 3: NCCL communicator initialized All threads synchronized - communicators ready Thread 0: Communicator rank 0 of 4 Thread 1: Communicator rank 1 of 4 Thread 2: Communicator rank 2 of 4 Thread 3: Communicator rank 3 of 4 Thread 0: Destroyed NCCL communicator Thread 1: Destroyed NCCL communicator Thread 2: Destroyed NCCL communicator Thread 3: Destroyed NCCL communicator Thread 0: Resources cleaned up Thread 1: Resources cleaned up Thread 2: Resources cleaned up Thread 3: Resources cleaned up All threads completed Success ``` ## When to Use pthread Approach ### Ideal Use Cases - **Thread-based applications**: When your application is already threaded - **Single-node workloads**: All GPUs on one machine - **Shared memory**: Need to share data structures between GPU contexts ### When NOT to Use - **Multi-node clusters**: Cannot scale beyond one node - **Process isolation**: When GPU contexts should be isolated - **Complex applications**: Multi-process approach may be cleaner ## Performance Considerations - **Advantages**: - Shared address space between threads - Easier data sharing between GPU contexts - No MPI overhead - **Disadvantages**: - Thread synchronization complexity - Limited to single node ## Common Issues and Solutions 1. **Thread synchronization errors**: - Ensure all threads use the same NCCL unique ID - Proper pthread synchronization (barriers, joins) 2. **CUDA context conflicts**: - Each thread must call `cudaSetDevice()` before CUDA operations - Don't share CUDA streams between threads 3. **Resource cleanup order**: - Always destroy NCCL communicators before CUDA resources - Synchronize streams before destroying communicators ## Error Handling The example uses simplified error handling with CHECK macros: - **CUDACHECK**: Exits immediately on CUDA errors - **NCCLCHECK**: Exits immediately on NCCL errors - **No async error checking**: Simplified for clarity - **Thread safety**: Each thread handles its own errors ## Highlighted Environment Variables - `NTHREADS`: Number of threads to create (defaults to number of GPUs) See examples/README.md for the full list. ## Next Steps After understanding this example: 1. Try using the collective examples and add the pthread approach 2. Compare with MPI-based multi-process approach nccl-2.29.7-1/docs/examples/01_communicators/02_one_device_per_pthread/main.cc000066400000000000000000000167451515037102200270430ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "cuda_runtime.h" #include "nccl.h" #include #include #include /** * NCCL Pthread Example - One Device Per Thread (Simple Version) * * This example demonstrates the basic lifecycle of NCCL communicators in a * multi-threaded environment. Each pthread manages one GPU device and shows * how to properly create and destroy NCCL communicators. * * Key Learning Points: * - NCCL communicator creation and destruction within threads * - CUDA stream management per thread * - Proper resource cleanup order * * This is a minimal example focusing purely on communicator lifecycle * management without performing actual collective operations. */ // Enhanced error checking macro for NCCL operations // Provides detailed error information including the failed operation #define NCCLCHECK(cmd) \ do { \ ncclResult_t res = cmd; \ if (res != ncclSuccess) { \ fprintf(stderr, "Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \ ncclGetErrorString(res)); \ fprintf(stderr, "Failed NCCL operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) #define CUDACHECK(cmd) \ do { \ cudaError_t err = cmd; \ if (err != cudaSuccess) { \ fprintf(stderr, "Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \ cudaGetErrorString(err)); \ fprintf(stderr, "Failed CUDA operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) // Thread data structure to pass parameters typedef struct { int thread_id; int num_gpus; ncclUniqueId commId; ncclComm_t *comms; } threadData_t; void *thread_worker(void *arg) { threadData_t *data = (threadData_t *)arg; int thread_id = data->thread_id; cudaStream_t stream; // ========================================================================= // Set Device Context and Create Stream // ========================================================================= // Each thread must set its device context before any CUDA operations CUDACHECK(cudaSetDevice(thread_id)); CUDACHECK(cudaStreamCreate(&stream)); printf(" Thread %d: Set device %d and created stream\n", thread_id, thread_id); // ========================================================================= // Initialize NCCL Communicator // ========================================================================= // Each thread creates its own communicator using the shared unique ID NCCLCHECK(ncclCommInitRank(&data->comms[thread_id], data->num_gpus, data->commId, thread_id)); printf(" Thread %d: NCCL communicator initialized\n", thread_id); if (thread_id == 0) { printf("All threads initialized - communicators ready\n"); } // ========================================================================= // Query Communicator Properties // ========================================================================= // Verify the communicator was created correctly int comm_thread_id, comm_size; NCCLCHECK(ncclCommUserRank(data->comms[thread_id], &comm_thread_id)); NCCLCHECK(ncclCommCount(data->comms[thread_id], &comm_size)); printf(" Thread %d: Communicator thread_id %d of %d\n", thread_id, comm_thread_id, comm_size); // Synchronize CUDA stream to ensure all GPU work is complete if (stream != NULL) { CUDACHECK(cudaStreamSynchronize(stream)); } // ========================================================================= // Cleanup Resources (Proper Order) // ========================================================================= // Destroy NCCL communicator FIRST (before CUDA resources) // This is important - NCCL cleanup should happen before CUDA cleanup if (data->comms[thread_id] != NULL) { NCCLCHECK(ncclCommFinalize(data->comms[thread_id])); NCCLCHECK(ncclCommDestroy(data->comms[thread_id])); printf(" Thread %d: Destroyed NCCL communicator\n", comm_thread_id); } // Now destroy CUDA stream if (stream != NULL) { CUDACHECK(cudaStreamDestroy(stream)); } printf(" Thread %d: Resources cleaned up\n", thread_id); return NULL; } int main(int argc, char *argv[]) { int num_gpus; pthread_t *threads; threadData_t *threadData; ncclComm_t *comms; ncclUniqueId commId; // ========================================================================= // STEP 1: Initialize Variables and Check GPU Availability // ========================================================================= CUDACHECK(cudaGetDeviceCount(&num_gpus)); const char *nThreadsEnv = getenv("NTHREADS"); if (nThreadsEnv) { num_gpus = atoi(nThreadsEnv); } if (num_gpus < 1) { printf("No CUDA devices found\n"); return EXIT_FAILURE; } printf("Using %d devices with pthreads\n", num_gpus); // ========================================================================= // STEP 2: Allocate Memory and Prepare Data Structures // ========================================================================= threads = (pthread_t *)malloc(num_gpus * sizeof(pthread_t)); threadData = (threadData_t *)malloc(num_gpus * sizeof(threadData_t)); comms = (ncclComm_t *)malloc(num_gpus * sizeof(ncclComm_t)); // Generate unique ID for NCCL communicator initialization NCCLCHECK(ncclGetUniqueId(&commId)); // ========================================================================= // STEP 3: Create and Launch Pthread Threads // ========================================================================= printf("Creating %d threads for NCCL communicators\n", num_gpus); for (int i = 0; i < num_gpus; i++) { threadData[i].thread_id = i; threadData[i].num_gpus = num_gpus; threadData[i].commId = commId; threadData[i].comms = comms; pthread_create(&threads[i], NULL, thread_worker, &threadData[i]); } // ========================================================================= // STEP 4: Wait for Thread Completion // ========================================================================= for (int i = 0; i < num_gpus; i++) { pthread_join(threads[i], NULL); } printf("All threads completed\n"); // ========================================================================= // STEP 5: Cleanup Resources // ========================================================================= free(threads); free(threadData); free(comms); printf("Success\n"); return 0; } nccl-2.29.7-1/docs/examples/01_communicators/03_one_device_per_process_mpi/000077500000000000000000000000001515037102200264705ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/01_communicators/03_one_device_per_process_mpi/Makefile000066400000000000000000000032341515037102200301320ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # This examples needs to be built with MPI support MPI = 1 # Include common build rules include ../../../../makefiles/common.mk include ../../../../makefiles/examples.mk # Target executable TARGET = one_device_per_process_mpi # Source files SOURCES = main.cc OBJECTS = $(SOURCES:.cc=.o) # Default target all: $(TARGET) # Build executable $(TARGET): $(OBJECTS) $(MPICXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ @echo "Built target $@" # Compile source files %.o: %.cc $(MPICXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ # Test target test: $(TARGET) @echo "Testing $(TARGET)..." @echo "Running with 1 process" $(MPIRUN) -np 1 ./$(TARGET) @echo "" @echo "Running with 2 processes" $(MPIRUN) -np 2 ./$(TARGET) # Clean build artifacts clean: rm -f $(OBJECTS) $(TARGET) # Install target install: $(TARGET) @mkdir -p $(PREFIX)/bin cp $(TARGET) $(PREFIX)/bin/ # Help help: @echo "NCCL Example: One Device per Process (MPI)" @echo "==========================================" @echo "" @echo "This example shows how to use ncclCommInitRank to create" @echo "communicators for multiple GPUs using pthreads." @echo "" @echo "Targets:" @echo " all - Build the example (default)" @echo " test - Build and run tests with different process counts" @echo " clean - Remove build artifacts" @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" @echo " help - Show this help" .PHONY: all test clean install help nccl-2.29.7-1/docs/examples/01_communicators/03_one_device_per_process_mpi/README.md000066400000000000000000000150551515037102200277550ustar00rootroot00000000000000 # NCCL Example: One Device per Process (MPI) This example demonstrates NCCL communicator lifecycle management using MPI, with one GPU per MPI process. ## Overview This example shows one of the most common NCCL deployment pattern: one GPU device per process. This approach is ideal for distributed training across multiple nodes and provides the foundation for scalable multi-GPU applications. MPI is used as it provides a parallel launcher and broadcast functions. It is, however, not a requirement for multi-node NCCL applications. Other approaches use server-client models or spawn parallel processes using sockets. NCCL only requires that the unique ID is distributed among each thread/process taking part in collective communication and all threads/processes call some NCCL initialization function. ## What This Example Does 1. **Multi-node Support**: - Determines local rank on each node automatically - Maps MPI processes to GPUs on each node - Uses `MPI_Comm_split_type` with `MPI_COMM_TYPE_SHARED` to assign each local rank a GPU. 2. **Communicator Creation**: - Uses `ncclCommInitRank` with MPI-coordinated unique ID - Rank `0` generates and broadcasts NCCL unique ID - Each process joins the distributed communicator 3. **Verification**: - Displays MPI rank → NCCL rank → GPU device mapping - Confirms successful initialization across all processes 4. **Cleanup**: - Proper resource cleanup order - MPI synchronization for clean shutdown ## Building and Running ### Build ```shell make MPI=1 [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] ``` ### Run example ```shell mpirun -np ./one_device_per_process_mpi ``` ### Run with NCCL debug output ```shell NCCL_DEBUG=INFO mpirun -np ./one_device_per_process_mpi ``` ## Code Walk-through This approach: - Automatically handles multi-node GPU assignment - Uses MPI for coordination and NCCL for GPU communication - Supports both single-node and multi-node deployments ### Unique ID Distribution The NCCL unique ID must be shared with all process which call `ncclCommInitRank`. We use MPI for that: ```c // Rank 0 generates unique ID if (mpi_rank == 0) { NCCLCHECK(ncclGetUniqueId(&nccl_id)); } // Broadcast to all processes MPI_Bcast(&nccl_id, sizeof(ncclUniqueId), MPI_BYTE, 0, MPI_COMM_WORLD); ``` ### Key Function: Multi-node GPU assignment ```c // Separate function to determine the node local rank via `MPI_Comm_split_type` int local_rank = getLocalRank(MPI_COMM_WORLD); // Use the local rank as the GPU device number. This assumes you only start as many processes as available GPUs CUDACHECK(cudaSetDevice(local_rank)); ncclComm_t comm; int mpi_rank, mpi_size; // mpi_rank & mpi_size are set during MPI initialization ncclUniqueId nccl_id; // nccl_id is generated and broadcasted as above // Initialize NCCL communicator across all processes NCCLCHECK(ncclCommInitRank(&comm, mpi_size, nccl_id, mpi_rank)); ``` ## Expected Output ### Single Node (4 processes) ``` Starting NCCL communicator lifecycle example with 4 processes MPI initialized - Process 0 of 4 total processes Found 4 CUDA devices on this node MPI rank 0 assigned to CUDA device 0 Rank 0 generated NCCL unique ID for all processes Rank 0 received NCCL unique ID Rank 0 created NCCL communicator MPI rank 0 → NCCL rank 0/4 on GPU device 0 [Similar output for ranks 1-3] All communicators initialized successfully! Beginning cleanup... Rank 0 destroyed NCCL communicator All NCCL communicators created and cleaned up properly! This example demonstrated the complete NCCL communicator lifecycle. Next steps: Try running NCCL collective operations (AllReduce, etc.) ``` ### Multi-node (8 processes, 2 nodes) ``` Starting NCCL communicator lifecycle example with 8 processes MPI initialized - Process 0 of 8 total processes MPI initialized - Process 1 of 8 total processes MPI initialized - Process 2 of 8 total processes MPI initialized - Process 3 of 8 total processes MPI initialized - Process 4 of 8 total processes MPI initialized - Process 5 of 8 total processes MPI initialized - Process 6 of 8 total processes MPI initialized - Process 7 of 8 total processes ... MPI rank 0 → NCCL rank 0/8 on GPU device 0 MPI rank 1 → NCCL rank 1/8 on GPU device 1 MPI rank 2 → NCCL rank 2/8 on GPU device 2 MPI rank 3 → NCCL rank 3/8 on GPU device 3 MPI rank 4 → NCCL rank 4/8 on GPU device 0 MPI rank 5 → NCCL rank 5/8 on GPU device 1 MPI rank 6 → NCCL rank 6/8 on GPU device 2 MPI rank 7 → NCCL rank 7/8 on GPU device 3 All NCCL communicators created and cleaned up properly! ``` ## When to Use MPI Approach ### Ideal Use Cases - **Multi-node clusters**: Scales across multiple machines - **Production deployments**: Industry standard for distributed training, inference, and most HPC codes - **Process isolation**: Each GPU in separate process for robustness - **Large scale**: Supports thousands of GPUs ### When NOT to Use - **Single-node testing**: Simpler approaches available - **No MPI available**: Some environments don't support MPI - **Shared memory needs**: Single-process approaches may be simpler ## Performance Considerations - **Advantages**: - MPI has been optimized for large parallel startup. - Industry standard deployment pattern - Optimal for large-scale training - **Disadvantages**: - MPI setup complexity - Inter-process communication overhead - Requires MPI runtime environment ## Common Issues and Solutions 1. **More MPI processes than GPUs on a node**: - The example reports an error if local rank exceeds available devices - Use fewer processes per node or more GPUs 2. **MPI broadcast hangs**: - Ensure all ranks participate in collective operations - Check MPI installation and network connectivity 3. **Multi-node communication fails**: - Check firewall settings and network configuration - Set `NCCL_SOCKET_IFNAME` to specify network interface ## Error Handling The example uses simplified error handling with CHECK macros: - **CUDACHECK**: Exits immediately on CUDA errors - **NCCLCHECK**: Exits immediately on NCCL errors - **No async error checking**: Simplified for clarity - **No global error coordination**: Each process exits on its own errors ## Next Steps After understanding this example: 1. Try running collective operations (AllReduce, AllGather, etc.) 2. Experiment with multi-node deployments nccl-2.29.7-1/docs/examples/01_communicators/03_one_device_per_process_mpi/main.cc000066400000000000000000000230321515037102200277230ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "cuda_runtime.h" #include "mpi.h" #include "nccl.h" #include #include #include #include /** * NCCL Example: One Device per Process with MPI * ============================================= * * LEARNING OBJECTIVE: * This example teaches the fundamental NCCL pattern: one GPU device per MPI * process. This is the most common deployment pattern for multi-GPU distributed * training. * * WHAT THIS CODE DEMONSTRATES: * - How to initialize NCCL communicators across multiple processes * - Proper GPU assignment in both single-node and multi-node environments * - Complete NCCL communicator lifecycle management * - Error handling best practices for production code * * STEP-BY-STEP PROCESS: * 1. MPI Setup: Initialize MPI and determine process layout * 2. GPU Assignment: Map each process to a local GPU device * 3. NCCL ID Sharing: Rank 0 creates unique ID, broadcasts to all processes * 4. Communicator Creation: Each process joins the NCCL communicator * 5. Verification: Query and verify communicator properties * 6. Clean Shutdown: Properly destroy all resources in correct order * * MULTI-NODE INTELLIGENCE: * - Automatically detects which processes are on the same physical node * - Assigns local GPU indices (0, 1, 2, 3...) to processes on each node * - Uses MPI_Comm_split_type with MPI_COMM_TYPE_SHARED for robust node * identification * - Leverages MPI's native shared memory detection for optimal performance * * USAGE EXAMPLES: * Single node (4 GPUs): mpirun -np 4 ./one_device_per_process_mpi * * EXPECTED OUTPUT: * Each process will report: MPI rank → NCCL rank → GPU device assignment * Success message confirms all communicators were created properly */ // Enhanced error checking macro for NCCL operations // Provides detailed error information including the failed operation #define NCCLCHECK(cmd) \ do { \ ncclResult_t res = cmd; \ if (res != ncclSuccess) { \ fprintf(stderr, "Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \ ncclGetErrorString(res)); \ fprintf(stderr, "Failed NCCL operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) #define CUDACHECK(cmd) \ do { \ cudaError_t err = cmd; \ if (err != cudaSuccess) { \ fprintf(stderr, "Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \ cudaGetErrorString(err)); \ fprintf(stderr, "Failed CUDA operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) // ============================================================================= // LOCAL RANK UTILITY FUNCTION - For Multi-Node GPU Assignment // ============================================================================= /** * Determine the local rank of this process on its physical node * * Algorithm: * 1. Split the communicator based on shared memory (i.e., nodes) * 2. Get the rank within the node communicator * 3. This rank becomes the local rank for GPU assignment * * @param comm The MPI communicator to use for determining local rank * @return Local rank (0, 1, 2...) for GPU assignment, or -1 on error */ int getLocalRank(MPI_Comm comm) { int world_size; MPI_Comm_size(comm, &world_size); int world_rank; MPI_Comm_rank(comm, &world_rank); // Split the communicator based on shared memory (i.e., nodes) MPI_Comm node_comm; MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, world_rank, MPI_INFO_NULL, &node_comm); // Get the rank and size within the node communicator int node_rank, node_size; MPI_Comm_rank(node_comm, &node_rank); MPI_Comm_size(node_comm, &node_size); // Clean up the node communicator MPI_Comm_free(&node_comm); return node_rank; } // ============================================================================= // MAIN FUNCTION - NCCL Communicator Lifecycle Example // ============================================================================= int main(int argc, char *argv[]) { // Variables for MPI, CUDA, and NCCL components int mpi_rank, mpi_size, local_rank; int num_gpus = 0; ncclComm_t comm = NULL; cudaStream_t stream = NULL; ncclUniqueId nccl_id; // ========================================================================= // STEP 1: Initialize MPI and determine process layout // ========================================================================= MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); if (mpi_rank == 0) { printf("Starting NCCL communicator lifecycle example with %d processes\n", mpi_size); } // Determine which local GPU this process should use local_rank = getLocalRank(MPI_COMM_WORLD); printf(" MPI initialized - Process %d of %d total processes\n", mpi_rank, mpi_size); // ========================================================================= // STEP 2: Setup CUDA device for this process // ========================================================================= // Check how many CUDA devices are available on this node CUDACHECK(cudaGetDeviceCount(&num_gpus)); printf(" Found %d CUDA devices on this node\n", num_gpus); if (num_gpus == 0) { fprintf(stderr, "ERROR: No CUDA devices found on this node!\n"); exit(EXIT_FAILURE); } if (local_rank >= num_gpus) { fprintf(stderr, "ERROR: Process %d needs GPU %d but only %d devices available\n", mpi_rank, local_rank, num_gpus); exit(EXIT_FAILURE); } // Assign this process to its designated GPU device CUDACHECK(cudaSetDevice(local_rank)); // Create CUDA stream for GPU operations CUDACHECK(cudaStreamCreate(&stream)); printf(" MPI rank %d assigned to CUDA device %d\n", mpi_rank, local_rank); // ========================================================================= // STEP 3: Initialize NCCL communicator // ========================================================================= // Generate NCCL unique ID (only rank 0 needs to do this) if (mpi_rank == 0) { NCCLCHECK(ncclGetUniqueId(&nccl_id)); printf("Rank 0 generated NCCL unique ID for all processes\n"); } // Share the unique ID with all processes using MPI broadcast MPI_Bcast(&nccl_id, NCCL_UNIQUE_ID_BYTES, MPI_CHAR, 0, MPI_COMM_WORLD); printf("INFO: Rank %d received NCCL unique ID\n", mpi_rank); // Create NCCL communicator for this process // This is where each process joins the distributed NCCL communicator NCCLCHECK(ncclCommInitRank(&comm, mpi_size, nccl_id, mpi_rank)); printf(" Rank %d created NCCL communicator\n", mpi_rank); // ========================================================================= // STEP 4: Verify communicator setup // ========================================================================= // Query communicator properties to verify everything is set up correctly int comm_rank, comm_size, comm_device; NCCLCHECK(ncclCommUserRank(comm, &comm_rank)); NCCLCHECK(ncclCommCount(comm, &comm_size)); NCCLCHECK(ncclCommCuDevice(comm, &comm_device)); printf(" MPI rank %d → NCCL rank %d/%d on GPU device %d\n", mpi_rank, comm_rank, comm_size, comm_device); // Give all processes a chance to finish their printf MPI_Barrier(MPI_COMM_WORLD); // ========================================================================= // STEP 5: Clean shutdown and resource cleanup // ========================================================================= if (mpi_rank == 0) { printf( "\nAll communicators initialized successfully! Beginning cleanup...\n"); } // Synchronize CUDA stream to ensure all GPU work is complete if (stream != NULL) { CUDACHECK(cudaStreamSynchronize(stream)); } // Destroy NCCL communicator FIRST (before CUDA resources) // This is important - NCCL cleanup should happen before CUDA cleanup if (comm != NULL) { NCCLCHECK(ncclCommFinalize(comm)); NCCLCHECK(ncclCommDestroy(comm)); printf(" Rank %d destroyed NCCL communicator\n", mpi_rank); } // Now destroy CUDA stream if (stream != NULL) { CUDACHECK(cudaStreamDestroy(stream)); } if (mpi_rank == 0) { printf( "\nAll NCCL communicators created and cleaned up properly!\n"); printf("This example demonstrated the complete NCCL communicator " "lifecycle.\n"); printf("Next steps: Try running NCCL collective operations (AllReduce, " "etc.)\n"); } MPI_Finalize(); return 0; } nccl-2.29.7-1/docs/examples/01_communicators/Makefile000066400000000000000000000030741515037102200222610ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Include common build rules include ../../../makefiles/common.mk # NCCL Fundamental Examples EXAMPLES = 01_multiple_devices_single_process 02_one_device_per_pthread ifeq ($(MPI), 1) EXAMPLES += 03_one_device_per_process_mpi endif # Default target all: $(EXAMPLES) # Build individual examples $(EXAMPLES): $(MAKE) -C $@ # Clean all build artifacts clean: for example in $(EXAMPLES); do \ $(MAKE) -C $$example clean; \ done ifneq ($(MPI),1) $(MAKE) -C 03_one_device_per_process_mpi clean endif # Test all examples test: all for example in $(EXAMPLES); do \ echo "Testing $$example..."; \ $(MAKE) -C $$example test || exit 1; \ done # Help help: @echo "NCCL Communicator Init Examples" @echo "===============================" @echo "" @echo "Targets:" @echo " all - Build all examples" @echo " clean - Clean all build artifacts" @echo " test - Test all examples" @echo " help - Show this help" @echo "" @echo "Examples:" @echo " 01_multiple_devices_single_process - Create communicators using multiple GPUs in a single thread" @echo " 02_one_device_per_pthread - Create communicators using one GPU per thread" @echo " 03_one_device_per_process_mpi - Create communicators using one GPU per MPI process" @echo "" @echo "To build/run individual examples:" @echo " make -C 01_multiple_devices_single_process" .PHONY: all clean test help $(EXAMPLES) nccl-2.29.7-1/docs/examples/01_communicators/README.md000066400000000000000000000070431515037102200221000ustar00rootroot00000000000000 # NCCL Communicator Examples ## Overview This directory contains minimal examples that demonstrate NCCL communicator lifecycle management (creation, query, and destruction) using different initialization patterns. ## Examples ### [01_multiple_devices_single_process](01_multiple_devices_single_process/) **Multiple Devices Single Process** - **Pattern**: Single process manages all GPUs - **API**: `ncclCommInitAll` (no external coordination) - **Use case**: Simple single-node applications - **Key features**: - Simplest initialization method - No MPI or threading required - Automatic rank assignment (0 to n-1) - Cannot span multiple nodes **Run command:** ```shell ./01_multiple_devices_single_process/multiple_devices_single_process ``` ### [02_one_device_per_pthread](02_one_device_per_pthread/) **One Device per Thread with pthreads** - **Pattern**: One thread per GPU within single process - **API**: `ncclCommInitRank` with pthread coordination - **Use case**: Single-node multi-GPU, thread-based parallelism - **Key features**: - pthread barriers for synchronization - Shared memory for unique ID - Lower overhead than multi-process - Cannot span multiple nodes **Run command:** ```shell [NTHREADS=n] ./02_one_device_per_pthread/one_device_per_pthread ``` ### [03_one_device_per_process_mpi](03_one_device_per_process_mpi/) **One Device per Process with MPI** - **Pattern**: One MPI process per GPU - **API**: `ncclCommInitRank` with MPI coordination - **Use case**: Multi-node clusters, distributed training - **Key features**: - MPI broadcast for unique ID distribution - Process-to-GPU mapping by local MPI ranks - Scalable to multiple nodes **Run command:** ```shell mpirun -np ./03_one_device_per_process_mpi/one_device_per_process_mpi ``` ## Choosing the Right Approach | Feature | ncclCommInitAll | pthread | MPI | |------------------------|-----------------|------------------|----------| | **Multi-node support** | ✗ | ✗ | ✓ | | **Process isolation** | ✗ | ✗ | ✓ | | **Setup complexity** | Low | Medium | High | | **Memory overhead** | Low | Medium | High | | **Best for** | Simple test | Single-node apps | Clusters | ### When to use each: - **ncclCommInitAll**: Development, testing, simple single-node apps - **pthread**: Single-node with thread-based parallelism needs - **MPI**: Production distributed training, multi-node setups ## Building ### **Quick Start** ```shell # Build all examples [or single directory] make [directory] # Test all examples make test ``` ### **Individual Examples** ```shell # Build specific example make 01_multiple_devices_single_process make 02_one_device_per_pthread make 03_one_device_per_process_mpi # Test individual example cd 01_multiple_devices_single_process && make test cd 02_one_device_per_pthread && make test cd 03_one_device_per_process_mpi && make test ``` ## References - [NVIDIA NCCL User Guide Examples](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/examples.html) - [NCCL API Reference](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api.html) - [CUDA Programming Guide](https://docs.nvidia.com/cuda/cuda-c-programming-guide/) - [MPI Standard](https://www.mpi-forum.org/docs/) nccl-2.29.7-1/docs/examples/02_point_to_point/000077500000000000000000000000001515037102200207775ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/02_point_to_point/01_ring_pattern/000077500000000000000000000000001515037102200237735ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/02_point_to_point/01_ring_pattern/Makefile000066400000000000000000000026761515037102200254460ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Include common build rules include ../../../../makefiles/common.mk include ../../../../makefiles/examples.mk # Target executable TARGET = ring_pattern # Common utilities (for utils.h, nccl_utils.h, etc.) COMMON_INC = ../../common/include INCLUDES += -I$(COMMON_INC) # Source files SOURCES = main.cc OBJECTS = $(SOURCES:.cc=.o) # Default target all: $(TARGET) # Build executable $(TARGET): $(OBJECTS) $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ @echo "Built target $@" # Compile source files %.o: %.cc $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ # Test target test: $(TARGET) @echo "Testing $(TARGET)..." @echo "Running with all available GPUs" ./$(TARGET) # Clean build artifacts clean: rm -f $(OBJECTS) $(TARGET) # Install target install: $(TARGET) @mkdir -p $(PREFIX)/bin cp $(TARGET) $(PREFIX)/bin/ # Help help: @echo "NCCL Example: P2P Ring Pattern" @echo "==============================================" @echo "" @echo "Targets:" @echo " all - Build the example (default)" @echo " test - Build and run test with all GPUs" @echo " clean - Remove build artifacts" @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" @echo " help - Show this help" .PHONY: all test clean install help nccl-2.29.7-1/docs/examples/02_point_to_point/01_ring_pattern/README.md000066400000000000000000000117771515037102200252670ustar00rootroot00000000000000 # NCCL Ring Communication Pattern Example This example demonstrates a ring communication pattern using NCCL P2P operations. It runs on a single node where a single process manages all GPUs and data flows in a circular pattern. ## Overview The ring communication pattern creates a circular data flow where each GPU sends data to its "next" neighbor and receives from its "previous" neighbor in the ring. This example uses `ncclCommInitAll` for simplified single-threaded, single-process multi-GPU setup. ## What This Example Does 1. **Detects and initializes all available GPUs** using `ncclCommInitAll` for simplified single-process setup 2. **Creates ring topology** where each GPU calculates its next and previous neighbors using modulo 3. **Executes simultaneous point-to-point communication** with each GPU sending to next and receiving from previous 4. **Verifies data correctness** by checking that each GPU received the expected data from its predecessor ## Building and Running ### Build the Example ```bash cd examples/02_point_to_point/01_ring_pattern make [NCCL_HOME=] [CUDA_HOME=] ``` ### Run with All Available GPUs ```bash ./ring_pattern ``` ## Code Walk-through ### Ring Topology Setup The example calculates ring neighbors using modulo arithmetic: ```cpp for (int i = 0; i < num_gpus; i++) { int next = (i + 1) % num_gpus; // Next neighbor in ring int prev = (i - 1 + num_gpus) % num_gpus; // Previous neighbor in ring } ``` ### Simultaneous Communication Uses `ncclGroupStart/End` to prevent deadlocks when scheduling all send and receive operations: ```cpp float **d_sendbuff; // device side send and receive buffer are allocated through cudaMalloc float **d_recvbuff; size_t count; // count is set to the number of floats to be sent (usually the size of the buffers) ncclComm_t *comms; // comms are set during ncclCommInitAll cudaStream_t *streams; // streams are set in cudaStreamCreate // Each GPU simultaneously sends to next and receives from previous NCCLCHECK(ncclGroupStart()); for (int i = 0; i < num_gpus; i++) { int next = (i + 1) % num_gpus; int prev = (i - 1 + num_gpus) % num_gpus; NCCLCHECK(ncclSend(d_sendbuff[i], count, ncclFloat, next, comms[i], streams[i])); NCCLCHECK(ncclRecv(d_recvbuff[i], count, ncclFloat, prev, comms[i], streams[i])); } NCCLCHECK(ncclGroupEnd()); ``` ## Expected Output ``` Starting NCCL ring communication example Using 4 GPUs for ring communication Preparing data structures Initializing NCCL communicators All communicators initialized successfully Creating CUDA streams and verifying setup GPU 0 -> NCCL rank 0/4 on CUDA device 0 GPU 1 -> NCCL rank 1/4 on CUDA device 1 GPU 2 -> NCCL rank 2/4 on CUDA device 2 GPU 3 -> NCCL rank 3/4 on CUDA device 3 Setting up ring topology Data flow -> GPU 0 -> GPU 1 -> ... -> GPU 3 -> GPU 0 Ring transfer with 268435456 elements (1.00 GB per GPU) Allocating and initializing buffers Executing ring communication GPU 0 sends to GPU 1, receives from GPU 3 GPU 1 sends to GPU 2, receives from GPU 0 GPU 2 sends to GPU 3, receives from GPU 1 GPU 3 sends to GPU 0, receives from GPU 2 Ring communication completed successfully Verifying data correctness GPU 0 received data from GPU 3: CORRECT GPU 1 received data from GPU 0: CORRECT GPU 2 received data from GPU 1: CORRECT GPU 3 received data from GPU 2: CORRECT SUCCESS - All GPUs received correct data Cleaning up resources Example completed successfully! ``` ## When to Use - **Learning NCCL fundamentals**: Understanding point-to-point communication patterns - **Algorithm development**: Building custom collective operations based on point to point communications - **Single-node applications**: Pipeline parallelism or custom data distribution patterns ## Key Insights - `ncclCommInitAll` simplifies single-node multi-GPU setup - No MPI or pthreads needed for single-node patterns - Ring pattern enables circular data flow among all GPUs - `ncclGroupStart/End` prevents deadlock in simultaneous operations - Each GPU both sends and receives in parallel ## Common Issues and Solutions ### Issue: Deadlock without group operations **Solution:** Always use `ncclGroupStart()` and `ncclGroupEnd()` when performing simultaneous send/recv operations. ### Issue: Verification failures **Solution:** Check ring topology calculations and data initialization patterns. Ensure correct neighbor calculations. ## Error Handling This example uses comprehensive error checking with `NCCLCHECK` and `CUDACHECK` macros that immediately exit on any failure. In production code, consider more graceful error handling and recovery mechanisms. ## Next Steps After this example, try: - **Collective operations**: Examples in `03_collectives/` - **Multi-node approach**: Use the MPI implementation from `01_communicators` to send data across nodes. nccl-2.29.7-1/docs/examples/02_point_to_point/01_ring_pattern/main.cc000066400000000000000000000244351515037102200252360ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "cuda_runtime.h" #include "nccl.h" #include #include #include #include /* * NCCL Ring Pattern Example - Educational Version * * This example demonstrates the fundamental ring communication pattern using * NCCL's point-to-point operations. Understanding ring patterns is essential * for NCCL programming as they form the basis of many collective algorithms. * * Learning Objectives: * - Understand ring topology and neighbor communication * - Learn NCCL point-to-point send/recv operations * - See how data flows in a ring pattern * - Practice deadlock avoidance with ncclGroup operations * - Understand single-process multi-GPU patterns * */ // Enhanced error checking macro for NCCL operations // Provides detailed error information including the failed operation #define NCCLCHECK(cmd) \ do { \ ncclResult_t res = cmd; \ if (res != ncclSuccess) { \ fprintf(stderr, "Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \ ncclGetErrorString(res)); \ fprintf(stderr, "Failed NCCL operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) #define CUDACHECK(cmd) \ do { \ cudaError_t err = cmd; \ if (err != cudaSuccess) { \ fprintf(stderr, "Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \ cudaGetErrorString(err)); \ fprintf(stderr, "Failed CUDA operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) int main(int argc, char *argv[]) { // ======================================================================== // STEP 1: Initialize Environment and Detect GPUs // ======================================================================== int num_gpus = 0; ncclComm_t *comms = NULL; cudaStream_t *streams = NULL; float **h_sendbuff = NULL; float **h_recvbuff = NULL; float **d_sendbuff = NULL; float **d_recvbuff = NULL; printf("Starting NCCL ring communication example\n"); // Get number of available CUDA devices CUDACHECK(cudaGetDeviceCount(&num_gpus)); if (num_gpus == 0) { fprintf(stderr, "No CUDA devices found\n"); return 1; } if (num_gpus < 2) { printf("At least 2 GPU are necessary to create inter-GPU traffic\n"); printf("Found only %d GPU(s) - pattern will be limited\n", num_gpus); } printf("Using %d GPUs for ring communication\n", num_gpus); // ======================================================================== // STEP 2: Prepare Data Structures and Device List // ======================================================================== printf("Preparing data structures\n"); // Create device list (use all available devices) int *devices = (int *)malloc(num_gpus * sizeof(int)); for (int i = 0; i < num_gpus; i++) { devices[i] = i; } // Allocate communicators, streams, and buffer pointers comms = (ncclComm_t *)malloc(num_gpus * sizeof(ncclComm_t)); streams = (cudaStream_t *)malloc(num_gpus * sizeof(cudaStream_t)); h_sendbuff = (float **)malloc(num_gpus * sizeof(float *)); h_recvbuff = (float **)malloc(num_gpus * sizeof(float *)); d_sendbuff = (float **)malloc(num_gpus * sizeof(float *)); d_recvbuff = (float **)malloc(num_gpus * sizeof(float *)); // ======================================================================== // STEP 3: Initialize NCCL Communicators // ======================================================================== /* * ncclCommInitAll is the simplest way to initialize NCCL communicators * for single-process, multi-GPU scenarios. It automatically: * - Creates one communicator per GPU * - Assigns ranks sequentially (GPU 0 = rank 0, GPU 1 = rank 1, etc.) */ printf("Initializing NCCL communicators\n"); NCCLCHECK(ncclCommInitAll(comms, num_gpus, devices)); printf("All communicators initialized successfully\n"); // ======================================================================== // STEP 4: Create Streams and Verify Communicator Setup // ======================================================================== printf("Creating CUDA streams and verifying setup\n"); // Create streams and verify communicator info for (int i = 0; i < num_gpus; i++) { CUDACHECK(cudaSetDevice(devices[i])); CUDACHECK(cudaStreamCreate(&streams[i])); // Query communicator information for verification int rank, size, device; NCCLCHECK(ncclCommUserRank(comms[i], &rank)); NCCLCHECK(ncclCommCount(comms[i], &size)); NCCLCHECK(ncclCommCuDevice(comms[i], &device)); printf(" GPU %d -> NCCL rank %d/%d on CUDA device %d\n", i, rank, size, device); } // ======================================================================== // STEP 5: Set Up Ring Topology and Allocate Buffers // ======================================================================== printf("Setting up ring topology\n"); printf("Data flow -> GPU 0 -> ... -> GPU %d -> GPU 0\n", num_gpus - 1); // Test with 1GB of data const size_t count = 256 * 1024 * 1024; // 256M floats = 1GB const size_t size_bytes = count * sizeof(float); printf("Ring transfer with %zu elements (%.2f GB per GPU)\n", count, size_bytes / (1024.0 * 1024.0 * 1024.0)); // Allocate buffers for each GPU printf("Allocating and initializing buffers\n"); for (int i = 0; i < num_gpus; i++) { CUDACHECK(cudaSetDevice(devices[i])); h_sendbuff[i] = (float *)malloc(size_bytes); h_recvbuff[i] = (float *)malloc(size_bytes); CUDACHECK(cudaMalloc((void **)&d_sendbuff[i], size_bytes)); CUDACHECK(cudaMalloc((void **)&d_recvbuff[i], size_bytes)); // Initialize data with GPU-specific pattern for verification for (size_t j = 0; j < count; j++) { h_sendbuff[i][j] = (float)(i * 1000 + j % 1000); } CUDACHECK(cudaMemcpy(d_sendbuff[i], h_sendbuff[i], size_bytes, cudaMemcpyHostToDevice)); } // ======================================================================== // STEP 6: Execute Ring Communication Pattern // ======================================================================== /* * The ring communication uses ncclGroup operations to avoid deadlock. * Without grouping, if all GPUs tried to send first, they would deadlock * waiting for receivers. Grouping allows NCCL to execute operations * in the optimal order. */ printf("Executing ring communication\n"); // NOTE: ncclGroupStart and ncclGroupEnd are essential to avoid deadlock // when using ncclCommInitAll! NCCLCHECK(ncclGroupStart()); for (int i = 0; i < num_gpus; i++) { int next = (i + 1) % num_gpus; int prev = (i - 1 + num_gpus) % num_gpus; printf(" GPU %d sends to GPU %d, receives from GPU %d\n", i, next, prev); // Each GPU simultaneously sends to next and receives from previous NCCLCHECK( ncclSend(d_sendbuff[i], count, ncclFloat, next, comms[i], streams[i])); NCCLCHECK( ncclRecv(d_recvbuff[i], count, ncclFloat, prev, comms[i], streams[i])); } NCCLCHECK(ncclGroupEnd()); // Synchronize all streams to ensure communication completes for (int i = 0; i < num_gpus; i++) { CUDACHECK(cudaSetDevice(devices[i])); CUDACHECK(cudaStreamSynchronize(streams[i])); } printf("Ring communication completed successfully\n"); // ======================================================================== // STEP 7: Verify Data Correctness and Report Results // ======================================================================== printf("Verifying data correctness\n"); bool all_correct = true; for (int i = 0; i < num_gpus; i++) { CUDACHECK(cudaSetDevice(devices[i])); CUDACHECK(cudaMemcpy(h_recvbuff[i], d_recvbuff[i], size_bytes, cudaMemcpyDeviceToHost)); int prev = (i - 1 + num_gpus) % num_gpus; // Verify that GPU i received data from GPU prev float expected = (float)(prev * 1000); bool correct = (h_recvbuff[i][0] == expected); printf(" GPU %d received data from GPU %d: %s\n", i, prev, correct ? "CORRECT" : "ERROR"); if (!correct) { all_correct = false; printf(" Expected %.0f, got %.0f\n", expected, h_recvbuff[i][0]); } } if (all_correct) { printf("SUCCESS - All GPUs received correct data\n"); } else { printf("FAILURE - Data verification failed\n"); } // ======================================================================== // STEP 8: Cleanup Resources // ======================================================================== printf("Cleaning up resources\n"); // Free buffers for (int i = 0; i < num_gpus; i++) { CUDACHECK(cudaSetDevice(devices[i])); free(h_sendbuff[i]); free(h_recvbuff[i]); CUDACHECK(cudaFree(d_sendbuff[i])); CUDACHECK(cudaFree(d_recvbuff[i])); } // Destroy communicators and streams for (int i = 0; i < num_gpus; i++) { NCCLCHECK(ncclCommFinalize(comms[i])); NCCLCHECK(ncclCommDestroy(comms[i])); CUDACHECK(cudaSetDevice(devices[i])); CUDACHECK(cudaStreamDestroy(streams[i])); } // Free allocated memory free(devices); free(comms); free(streams); free(h_sendbuff); free(h_recvbuff); free(d_sendbuff); free(d_recvbuff); printf("Example completed successfully!\n"); return 0; } nccl-2.29.7-1/docs/examples/02_point_to_point/Makefile000066400000000000000000000021311515037102200224340ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # NCCL Fundamental Examples EXAMPLES = 01_ring_pattern # Default target all: $(EXAMPLES) # Build individual examples $(EXAMPLES): $(MAKE) -C $@ # Clean all build artifacts clean: for example in $(EXAMPLES); do \ $(MAKE) -C $$example clean; \ done # Test all examples test: all for example in $(EXAMPLES); do \ echo "Testing $$example..."; \ $(MAKE) -C $$example test || exit 1; \ done # Help help: @echo "NCCL Point to Point Examples" @echo "============================" @echo "" @echo "Targets:" @echo " all - Build all examples" @echo " clean - Clean all build artifacts" @echo " test - Test all examples" @echo " help - Show this help" @echo "" @echo "Examples:" @echo " 01_ring_pattern - Use send and receive operations to form a ring pattern" @echo "" @echo "To build/run individual examples:" @echo " make -C 01_ring_pattern" .PHONY: all clean test $(EXAMPLES) nccl-2.29.7-1/docs/examples/02_point_to_point/README.md000066400000000000000000000037521515037102200222650ustar00rootroot00000000000000 # NCCL Point-to-Point Communication Examples ## Overview This directory contains minimal examples that demonstrate NCCL point-to-point (P2P) communication patterns on a single node. These examples focus on clarity and correct communicator usage, resource management, and verification. ## Examples ### [01_ring_pattern](01_ring_pattern/) **Ring Communication Pattern** - **Pattern**: Circular data flow among all GPUs - **API**: `ncclCommInitAll` with P2P operations (`ncclSend`/`ncclRecv`) - **Use case**: Learning P2P communication; pipeline/data movement patterns on a single node - **Key features**: - Initializes all GPUs in a single process - Computes ring neighbors with modulo arithmetic - Uses `ncclGroupStart/End` to prevent deadlocks - Verifies data correctness after transfers ## Choosing the Right Pattern *Scenario* : Pipeline parallel training needs to send data from one GPU to another *Addresses* : Individual transfers between two ranks *Dependencies* : A functional NCCL library and its dependencies ### Why `ncclCommInitAll` here? For single-node collective examples we use `ncclCommInitAll` as it creates a clique of communicators in one call. ```c // Initialize all GPUs in one call ncclComm_t* comms; int num_gpus; NCCLCHECK(ncclCommInitAll(comms, num_gpus, NULL)); ``` ## Building ### **Quick Start** ```shell # Build example by directory name make 01_ring_pattern ``` ### **Individual Examples** ```shell # Build and run the ring pattern cd 01_ring_pattern && make ./ring_pattern ``` ## References - [NCCL User Guide: Examples](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/examples.html) - [NCCL API Reference](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api.html) - [CUDA Programming Guide](https://docs.nvidia.com/cuda/cuda-c-programming-guide/) nccl-2.29.7-1/docs/examples/03_collectives/000077500000000000000000000000001515037102200202505ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/03_collectives/01_allreduce/000077500000000000000000000000001515037102200225105ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/03_collectives/01_allreduce/Makefile000066400000000000000000000025001515037102200241450ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Include common build rules include ../../../../makefiles/common.mk include ../../../../makefiles/examples.mk # Target executable TARGET = allreduce # Source files SOURCES = main.cc OBJECTS = $(SOURCES:.cc=.o) # Default target all: $(TARGET) # Build executable $(TARGET): $(OBJECTS) $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ @echo "Built target $@" # Compile source files %.o: %.cc $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ # Test target test: $(TARGET) @echo "Testing $(TARGET)..." @echo "Running with all available GPUs" ./$(TARGET) # Clean build artifacts clean: rm -f $(OBJECTS) $(TARGET) # Install target install: $(TARGET) @mkdir -p $(PREFIX)/bin cp $(TARGET) $(PREFIX)/bin/ # Help help: @echo "NCCL Example: Allreduce" @echo "==============================================" @echo "" @echo "Targets:" @echo " all - Build the example (default)" @echo " test - Build and run test with all GPUs" @echo " clean - Remove build artifacts" @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" @echo " help - Show this help" .PHONY: all test clean install help nccl-2.29.7-1/docs/examples/03_collectives/01_allreduce/README.md000066400000000000000000000113031515037102200237650ustar00rootroot00000000000000 # NCCL AllReduce Collective Operation Example This example demonstrates the fundamental AllReduce collective operation using NCCL's single-process, multi-GPU approach in which a single process manages all GPUs to perform a sum reduction. ## Overview AllReduce combines data from all participants using a reduction operation (sum, max, min, etc.) and distributes the result to all participants. This example shows how each GPU contributes its rank value and all GPUs receive the combined sum using `ncclCommInitAll` for simplified setup. ## What This Example Does 1. **Detects available GPUs** and initializes NCCL communicators for all devices using `ncclCommInitAll` 2. **Initializes data** with each GPU contributing its rank value (GPU 0→0, GPU 1→1, etc.) 3. **Performs AllReduce sum operation** where all GPU values are summed and distributed to all participants 4. **Verifies correctness** by checking that all GPUs received the expected sum: 0+1+2+...+(n-1) ## Building and Running ### Build the Example ```bash cd examples/03_collectives/01_allreduce make [NCCL_HOME=] [CUDA_HOME=] ``` ### Run with All Available GPUs ```bash ./allreduce ``` ### Run with Specific GPUs ```bash CUDA_VISIBLE_DEVICES=0,1,2,3 ./allreduce ``` ## Code Walk-through ### Data Initialization Each GPU sets a send buffer allocated on the GPU to its rank value: ```cpp float** sendbuff; float rank_value = (float)i; size_t size; // size is the number of float to be sent // Allocate device memory for send buffers CUDACHECK(cudaMalloc((void **)&sendbuff[i], size * sizeof(float))); // Each GPU contributes its rank (GPU i contributes value i) // Zero the entire buffer, then set first element to rank CUDACHECK(cudaMemset(sendbuff[i], 0, size * sizeof(float))); CUDACHECK(cudaMemcpy(sendbuff[i], &rank_value, sizeof(float), cudaMemcpyHostToDevice)); ``` ### AllReduce Operation All GPUs participate in the sum reduction. The operations are evaluated in parallel within a NCCL group to avoid any deadlocks. ```cpp float** recvbuff; ncclComm_t *comms; // comms are set during ncclCommInitAll cudaStream_t *streams; // streams are set in cudaStreamCreate // Allocate device memory for receive buffers CUDACHECK(cudaMalloc((void **)&recvbuff[i], size * sizeof(float))); NCCLCHECK(ncclGroupStart()); for (int i = 0; i < num_gpus; i++) { NCCLCHECK(ncclAllReduce(sendbuff[i], recvbuff[i], size, ncclFloat, ncclSum, comms[i], streams[i])); } NCCLCHECK(ncclGroupEnd()); ``` ## Expected Output ``` Using 4 devices for collective communication Memory allocated for 4 communicators and streams NCCL communicators initialized for all devices Device 0 initialized with data value 0 Device 1 initialized with data value 1 Device 2 initialized with data value 2 Device 3 initialized with data value 3 Starting collective sum operation across all devices Collective operation completed Verifying results (expected sum: 6) Device 0 correctly received sum: 6 Device 1 correctly received sum: 6 Device 2 correctly received sum: 6 Device 3 correctly received sum: 6 Example completed successfully! ``` ## When to Use - **Deep learning**: Gradient averaging in data-parallel training - **Scientific computing**: Global reductions in parallel algorithms - **Statistics**: Computing global sums, averages, or other reductions - **Distributed algorithms**: Any scenario requiring collective reduction operations ## Key Insights - `ncclCommInitAll` simplifies single-node multi-GPU setup - No MPI or pthreads needed for single-node patterns - Allocate device buffer via ``cudaMalloc` and initialize via `cudaMemset`. - Best practices to wrap all collective calls in ncclGroupStart/End - All communication happens in parallel ## Common Issues and Solutions ### Issue: Verification failures **Solution:** Ensure each GPU initializes its buffer correctly with its rank value. ### Issue: Out of memory errors **Solution:** Reduce the buffer size in the code or use fewer GPUs. ## Error Handling This example uses comprehensive error checking with `NCCLCHECK` and `CUDACHECK` macros that immediately exit on any failure. In production code, consider more graceful error handling and recovery mechanisms. ## Next Steps After understanding AllReduce, explore: - **Point-to-point communication**: Examples in `02_point_to_point/` - **Other collectives**: Implement Broadcast, Reduce, AllGather operations using this example - **Multi-node approach**: Use the MPI implementation from `01_communicators` to send data across nodes. nccl-2.29.7-1/docs/examples/03_collectives/01_allreduce/main.cc000066400000000000000000000200711515037102200237430ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "cuda_runtime.h" #include "nccl.h" #include #include #include #include /* * NCCL AllReduce Example - Collective Communication * * This example demonstrates the fundamental AllReduce collective operation * using NCCL's single-process, multi-GPU approach. AllReduce is one of the most * important collective operations in distributed and parallel computing. * * Learning Objectives: * - Understand AllReduce collective communication pattern * - Learn NCCL single-process multi-GPU programming model * - See how data reduction works across multiple devices * - Practice verification and validation of collective results * */ // Enhanced error checking macro for NCCL operations // Provides detailed error information including the failed operation #define NCCLCHECK(cmd) \ do { \ ncclResult_t res = cmd; \ if (res != ncclSuccess) { \ fprintf(stderr, "Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \ ncclGetErrorString(res)); \ fprintf(stderr, "Failed NCCL operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) #define CUDACHECK(cmd) \ do { \ cudaError_t err = cmd; \ if (err != cudaSuccess) { \ fprintf(stderr, "Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \ cudaGetErrorString(err)); \ fprintf(stderr, "Failed CUDA operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) int main(int argc, char *argv[]) { // ======================================================================== // STEP 1: Initialize Variables and Detect Available GPUs // ======================================================================== int num_gpus = 0; ncclComm_t *comms; cudaStream_t *streams; float **sendbuff; float **recvbuff; // Get number of CUDA devices CUDACHECK(cudaGetDeviceCount(&num_gpus)); if (num_gpus < 1) { printf("No CUDA devices found\n"); return EXIT_FAILURE; } printf("Using %d devices for collective communication\n", num_gpus); // ======================================================================== // STEP 2: Allocate Memory for Communicators, Streams, and Data Buffers // ======================================================================== // Allocate arrays for per-device resources, and array of pointers for buffers comms = (ncclComm_t *)malloc(num_gpus * sizeof(ncclComm_t)); streams = (cudaStream_t *)malloc(num_gpus * sizeof(cudaStream_t)); sendbuff = (float **)malloc(num_gpus * sizeof(float *)); recvbuff = (float **)malloc(num_gpus * sizeof(float *)); printf("Memory allocated for %d communicators and streams\n", num_gpus); // ======================================================================== // STEP 3: Initialize NCCL Communicators for All Devices // ======================================================================== // ncclCommInitAll creates communicators for all devices in one call // This is the simplest way to set up NCCL for single-process applications NCCLCHECK(ncclCommInitAll(comms, num_gpus, NULL)); printf("NCCL communicators initialized for all devices\n"); // ======================================================================== // STEP 4: Create CUDA Streams and Allocate Device Memory // ======================================================================== const size_t size = 32 * 1024 * 1024; // 32M floats for demonstration for (int i = 0; i < num_gpus; i++) { // Set device context for each GPU CUDACHECK(cudaSetDevice(i)); // Create stream for asynchronous operations CUDACHECK(cudaStreamCreate(&streams[i])); // Allocate device memory for send and receive buffers CUDACHECK(cudaMalloc((void **)&sendbuff[i], size * sizeof(float))); CUDACHECK(cudaMalloc((void **)&recvbuff[i], size * sizeof(float))); // Initialize send buffer: zero the entire buffer, then set first element to // rank CUDACHECK(cudaMemset(sendbuff[i], 0, size * sizeof(float))); float rank_value = (float)i; CUDACHECK(cudaMemcpy(sendbuff[i], &rank_value, sizeof(float), cudaMemcpyHostToDevice)); printf(" Device %d initialized with data value %d\n", i, i); } // ======================================================================== // STEP 5: Perform AllReduce Sum Operation // ======================================================================== printf("Starting collective sum operation across all devices\n"); // NOTE: ncclGroupStart and ncclGroupEnd are essential to avoid // deadlock when using ncclCommInitAll and multiple communication calls. NCCLCHECK(ncclGroupStart()); for (int i = 0; i < num_gpus; i++) { // Each device performs combines all contributions and distributes result NCCLCHECK(ncclAllReduce(sendbuff[i], recvbuff[i], size, ncclFloat, ncclSum, comms[i], streams[i])); } NCCLCHECK(ncclGroupEnd()); // Synchronize all streams to ensure completion for (int i = 0; i < num_gpus; i++) { CUDACHECK(cudaSetDevice(i)); CUDACHECK(cudaStreamSynchronize(streams[i])); } printf("Collective operation completed\n"); // ======================================================================== // STEP 6: Verify Results and Validate Correctness // ======================================================================== // Expected result: sum of all ranks = 0 + 1 + 2 + ... + (num_gpus-1) // Note: We only check the first element since that's all we initialized float expected = (float)(num_gpus * (num_gpus - 1) / 2); printf("Verifying results (expected sum: %.0f)\n", expected); bool success = true; for (int i = 0; i < num_gpus; i++) { float result; CUDACHECK(cudaSetDevice(i)); CUDACHECK(cudaMemcpy(&result, recvbuff[i], sizeof(float), cudaMemcpyDeviceToHost)); if (result != expected) { printf(" Device %d received incorrect result: %.0f (expected %.0f)\n", i, result, expected); success = false; } else { printf(" Device %d correctly received sum: %.0f\n", i, result); } } // ======================================================================== // STEP 7: Cleanup Resources and Report Results // ======================================================================== // Destroy NCCL communicators for (int i = 0; i < num_gpus; i++) { NCCLCHECK(ncclCommFinalize(comms[i])); NCCLCHECK(ncclCommDestroy(comms[i])); } // Free device memory and destroy streams for (int i = 0; i < num_gpus; i++) { CUDACHECK(cudaSetDevice(i)); CUDACHECK(cudaFree(sendbuff[i])); CUDACHECK(cudaFree(recvbuff[i])); CUDACHECK(cudaStreamDestroy(streams[i])); } // Free host memory free(comms); free(streams); free(sendbuff); free(recvbuff); if (success) { printf("Example completed successfully!\n"); } else { printf("Example failed - incorrect results detected\n"); return EXIT_FAILURE; } return 0; } nccl-2.29.7-1/docs/examples/03_collectives/Makefile000066400000000000000000000021051515037102200217060ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # NCCL Collective Examples EXAMPLES = 01_allreduce # Default target all: $(EXAMPLES) # Build individual examples $(EXAMPLES): $(MAKE) -C $@ # Clean all build artifacts clean: for example in $(EXAMPLES); do \ $(MAKE) -C $$example clean; \ done # Test all examples test: all for example in $(EXAMPLES); do \ echo "Testing $$example..."; \ $(MAKE) -C $$example test; \ done # Help help: @echo "NCCL Collective Communication Examples" @echo "=====================================" @echo "" @echo "Targets:" @echo " all - Build all examples" @echo " clean - Clean all build artifacts" @echo " test - Test all examples" @echo " help - Show this help" @echo "" @echo "Examples:" @echo " 01_allreduce - AllReduce collective operation" @echo "" @echo "To build/run individual examples:" @echo " make -C 01_allreduce" .PHONY: all clean test help $(EXAMPLES) nccl-2.29.7-1/docs/examples/03_collectives/README.md000066400000000000000000000041251515037102200215310ustar00rootroot00000000000000 # NCCL Collective Communication Examples ## Overview This directory contains minimal examples that demonstrate NCCL collective communication operations on a single node using a single process managing all GPUs. The focus is clarity, correct resource management, and result verification. ## Examples ### [01_allreduce](01_allreduce/) **AllReduce Collective Operation** - **Pattern**: All participants reduce and distribute the result - **API**: `ncclCommInitAll`, `ncclAllReduce` - **Use case**: Global reductions in ML and HPC (e.g., gradient averaging) - **Key features**: - Initializes all GPUs in a single process - Each GPU contributes its rank value - Executes AllReduce sum across all GPUs - Verifies the expected global sum ## Choosing the Right Pattern *Scenario* : Parallel training needs efficient global communication *Addresses* : Most commonly used collective algorithms *Dependencies* : A functional NCCL library and its dependencies ### Why `ncclCommInitAll` here? For single-node collective examples we use `ncclCommInitAll` as it creates a clique of communicators in one call. ```c // Initialize all GPUs in one call ncclComm_t* comms; int num_gpus; NCCLCHECK(ncclCommInitAll(comms, num_gpus, NULL)); ``` A more advanced setup using MPI to initialize communicators across multiple nodes is shown in [01_communicators/03_one_device_per_process_mpi](../01_communicators/03_one_device_per_process_mpi) ## Building ### **Quick Start** ```shell # Build example by directory name make 01_allreduce ``` ### **Individual Examples** ```shell # Build and run AllReduce cd 01_allreduce && make ./allreduce ``` ## References - [NCCL User Guide: Examples](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/examples.html) - [NCCL API Reference](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api.html) - [CUDA Programming Guide](https://docs.nvidia.com/cuda/cuda-c-programming-guide/) nccl-2.29.7-1/docs/examples/04_user_buffer_registration/000077500000000000000000000000001515037102200230365ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/04_user_buffer_registration/01_allreduce/000077500000000000000000000000001515037102200252765ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/04_user_buffer_registration/01_allreduce/Makefile000066400000000000000000000034031515037102200267360ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Include common build rules include ../../../../makefiles/common.mk include ../../../../makefiles/examples.mk # Target executable TARGET = allreduce_ub # Common utilities COMMON_INC = ../../common/include COMMON_SRC = ../../common/src # Build configuration INCLUDES += -I$(COMMON_INC) # Source files SOURCES = main.cc $(COMMON_SRC)/utils.cc OBJECTS = $(SOURCES:.cc=.o) # Default target all: $(TARGET) # Build executable $(TARGET): $(OBJECTS) ifeq ($(MPI),1) $(MPICXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ else $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -lpthread -o $@ endif @echo "Built target $@" # Compile source files %.o: %.cc ifeq ($(MPI),1) $(MPICXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ else $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ endif # Test target test: $(TARGET) @echo "Testing $(TARGET)..." ifeq ($(MPI),1) @echo "Running with 2 processes" $(MPIRUN) -np 2 ./$(TARGET) else @echo "Running with all available GPUs" ./$(TARGET) endif # Clean build artifacts clean: rm -f $(OBJECTS) $(TARGET) # Install target install: $(TARGET) @mkdir -p $(PREFIX)/bin cp $(TARGET) $(PREFIX)/bin/ # Help help: @echo "NCCL Example: User Buffer Registration Allreduce" @echo "==============================================" @echo "" @echo "Targets:" @echo " all - Build the example (default)" @echo " test - Build and run test with all GPUs" @echo " clean - Remove build artifacts" @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" @echo " help - Show this help" .PHONY: all test clean install help nccl-2.29.7-1/docs/examples/04_user_buffer_registration/01_allreduce/README.md000066400000000000000000000133631515037102200265630ustar00rootroot00000000000000 # NCCL User Buffer Registration AllReduce Example This example demonstrates how to use NCCL's user buffer registration feature to optimize performance for repeated collective operations on the same buffers. User Buffer Registration is a feature that allows NCCL to directly send/receive/operate data through the user buffer without extra internal copy (zero-copy). ## Overview User buffer registration allows NCCL to pre-register memory buffers with communicators, eliminating registration overhead on each operation. This is particularly beneficial for applications that repeatedly perform collective operations on the same memory regions, such as iterative training loops. ## What This Example Does 1. **Allocates memory using NCCL allocator** (`ncclMemAlloc`) which is provided by NCCL as convenience function 2. **Registers buffers with communicator** using `ncclCommRegister` for optimized performance 3. **Performs AllReduce sum operation** using the registered buffers for efficient communication ## Building and Running The advanced examples can be built using either pthread or MPI for parallelization. pthread is the default choice. To use MPI the user needs to provide a valid MPI installation under `MPI_HOME`. ### Build ```shell make [MPI=1] [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] ``` ### Run when compiled for pthreads (default) ```shell [NTHREADS=N] ./allreduce_ub ``` ### Run when compiled for MPI ```shell mpirun -np ./allreduce_ub ``` ## Code Structure ### Key Components 1. **Buffer Allocation and Registration**: ```c size_t size_bytes; // Is set to the size of the send/receive buffers void *d_sendbuff; void *d_recvbuff; // Allocate buffers using ncclMemAlloc (or another qualified allocator) on the device NCCLCHECK(ncclMemAlloc(&d_sendbuff, size_bytes)); NCCLCHECK(ncclMemAlloc(&d_recvbuff, size_bytes)); ncclComm_t comm; // comms is set during ncclCommInitRank void *send_handle; void *recv_handle; // Register buffers with NCCL, handle is returned for De-registration NCCLCHECK(ncclCommRegister(comm, d_sendbuff, size_bytes, &send_handle)); NCCLCHECK(ncclCommRegister(comm, d_recvbuff, size_bytes, &recv_handle)); ``` 2. **AllReduce with Group Operations**: ```c size_t count; // set to number of floats to exchange cudaStream_t stream; // stream is set in cudaStreamCreate NCCLCHECK(ncclAllReduce(d_sendbuff, d_recvbuff, count, ncclFloat, ncclSum, comm, stream)); ``` 3. **Buffer Deregistration and Cleanup**: ```c // Deregister buffers using handle from ncclCommRegister NCCLCHECK(ncclCommDeregister(comm, send_handle)); NCCLCHECK(ncclCommDeregister(comm, recv_handle)); // Free buffers allocated with ncclMemAlloc NCCLCHECK(ncclMemFree(d_sendbuff)); NCCLCHECK(ncclMemFree(d_recvbuff)); ``` ## Expected Output ### With 4 GPUs (using pthreads/MPI) ``` Starting AllReduce example with 4 ranks Rank 0 communicator initialized using device 0 Rank 1 communicator initialized using device 1 Rank 2 communicator initialized using device 2 Rank 3 communicator initialized using device 3 User Buffer allocation: Rank 0 allocating 4.00 MB per buffer Rank 1 allocating 4.00 MB per buffer Rank 2 allocating 4.00 MB per buffer Rank 3 allocating 4.00 MB per buffer Rank 0 data initialized (value: 0) Rank 1 data initialized (value: 1) Rank 2 data initialized (value: 2) Rank 3 data initialized (value: 3) Starting AllReduce with 1048576 elements (4 MB) AllReduce completed successfully Verification - Expected: 6.0, Got: 6.0 Results verified correctly Rank 0 buffers deregistered Rank 1 buffers deregistered Rank 2 buffers deregistered Rank 3 buffers deregistered All resources cleaned up successfully ``` ## Performance Benefits of User Buffer Registration User buffer registration provides several performance advantages: 1. **Reduced Overhead**: Pre-registration eliminates the need to register/deregister buffers for each operation 2. **Better Memory Pinning**: Registered buffers are pinned in memory, preventing page faults 3. **Lower Latency**: Especially beneficial for repeated operations on the same buffers **Important**: Buffers must be allocated with `ncclMemAlloc` or a compatible allocator for registration to work. See The [General Buffer Registration ](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/bufferreg.html#general-buffer-registration) section of the user guide. **Important**: If any rank in a communicator passes registered buffers to a NCCL communication function, all other ranks in the same communicator must pass their registered buffers; otherwise, mixing registered and non-registered buffers can result in undefined behavior. ## Key Insights - **User Buffer Registration** is most beneficial for: - Large data transfers - Repeated operations on the same buffers - Performance-critical applications - **Memory management** is critical - always deregister buffers before freeing ## Common Issues and Solutions 1. **Registration Failure**: Buffers MUST be allocated with `ncclMemAlloc` or another qualified allocator (not `cudaMalloc`) for registration. See [Buffer Registration](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/bufferreg.html) section for details. 2. **Allocation Error**: If `ncclMemAlloc` fails, check NCCL version (requires 2.19.x+) and available memory 3. **Deregistration Order**: Always deregister before freeing memory 4. **Handle Management**: Keep track of registration handles for proper cleanup 5. **Memory Leaks**: Always use `ncclMemFree` for buffers allocated with `ncclMemAlloc` nccl-2.29.7-1/docs/examples/04_user_buffer_registration/01_allreduce/main.cc000066400000000000000000000170231515037102200265340ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "cuda_runtime.h" #include "nccl.h" #include "utils.h" #include #include #include #include #include #include /* * NCCL User Buffer Registration AllReduce Example * * This example demonstrates how to use NCCL's user buffer registration feature * to optimize performance for repeated collective operations on the same * buffers. * * Learning Objectives: * - Learn how to register and deregister buffers with NCCL communicators * - See the proper lifecycle management of registered buffers * */ /* * This function can be called inside an MPI rank or pthread thread. The * initialization and broadcast are implemented in common/src/utils.cc for * easier readability. For fully integrated examples using pthreads or MPI see * examples in 01_communicators. */ void *allReduce(int my_rank, int total_ranks, int local_device, int devices_per_rank) { // ======================================================================== // STEP 1: Initialize NCCL Communicator and Setup // ======================================================================== ncclUniqueId nccl_unique_id; if (my_rank == 0) { printf("Starting AllReduce example with %d ranks\n", total_ranks); NCCLCHECK(ncclGetUniqueId(&nccl_unique_id)); } // Distribute unique ID. // This step ensures all ranks have the same unique ID for communicator // creation util_broadcast(0, my_rank, &nccl_unique_id); // Set device context for this rank // Each rank manages its assigned GPU device CUDACHECK(cudaSetDevice(local_device)); // Initialize NCCL communicator // This creates the communication context for collective operations ncclComm_t comm; NCCLCHECK(ncclCommInitRank(&comm, total_ranks, nccl_unique_id, my_rank)); printf(" Rank %d communicator initialized using device %d\n", my_rank, local_device); // ======================================================================== // STEP 2: Allocate Memory Using NCCL Allocator // ======================================================================== if (my_rank == 0) { printf("User Buffer allocation:\n"); } // Allocate memory - using larger buffers to demonstrate registration // benefits size_t count = 1024 * 1024; // 1M elements size_t size_bytes = count * sizeof(float); printf(" Rank %d allocating %.2f MB per buffer\n", my_rank, (float)size_bytes / (1024 * 1024)); // Allocate buffers using NCCL allocator // NCCL's allocator can provide optimized memory for communication void *d_sendbuff; void *d_recvbuff; NCCLCHECK(ncclMemAlloc(&d_sendbuff, size_bytes)); NCCLCHECK(ncclMemAlloc(&d_recvbuff, size_bytes)); // ======================================================================== // STEP 3: Register Buffers with NCCL Communicator // ======================================================================== // Register the buffers with NCCL // This is the key optimization - buffers are pre-registered for efficiency // The handles returned can be used to identify registered buffers void *send_handle; void *recv_handle; NCCLCHECK(ncclCommRegister(comm, d_sendbuff, size_bytes, &send_handle)); NCCLCHECK(ncclCommRegister(comm, d_recvbuff, size_bytes, &recv_handle)); // ======================================================================== // STEP 4: Initialize Data and Prepare for Communication // ======================================================================== // Initialize data - each rank contributes its rank value // This creates a simple test pattern for verification float *h_data = (float *)malloc(size_bytes); for (size_t i = 0; i < count; i++) { h_data[i] = (float)my_rank; } CUDACHECK(cudaMemcpy(d_sendbuff, h_data, size_bytes, cudaMemcpyHostToDevice)); printf(" Rank %d data initialized (value: %d)\n", my_rank, my_rank); // Create stream for asynchronous operations // Streams allow overlapping computation and communication cudaStream_t stream; CUDACHECK(cudaStreamCreate(&stream)); // ======================================================================== // STEP 5: Perform AllReduce Operation // ======================================================================== if (my_rank == 0) { printf("Starting AllReduce with %zu elements (%zu MB)\n", count, size_bytes / (1024 * 1024)); } // Perform AllReduce operation // Since buffers are registered, this should have optimized performance NCCLCHECK(ncclAllReduce(d_sendbuff, d_recvbuff, count, ncclFloat, ncclSum, comm, stream)); if (my_rank == 0) { printf("AllReduce completed successfully\n"); } // ======================================================================== // STEP 6: Verify Results and Validate Correctness // ======================================================================== // Synchronize to ensure completion CUDACHECK(cudaStreamSynchronize(stream)); // Verify results (optional - copy back and check a few elements) float *h_result = (float *)malloc(sizeof(float) * count); CUDACHECK(cudaMemcpy(h_result, d_recvbuff, sizeof(float) * count, cudaMemcpyDeviceToHost)); // Each element should be the sum of all ranks float expected_sum = (float)(total_ranks * (total_ranks - 1)) / 2; bool all_ok = true; if (my_rank == 0) { printf("Verification - Expected: %.1f, Got: %.1f\n", expected_sum, h_result[0]); for (size_t i = 1; i < count; i++) { if (fabsf(h_result[i] - expected_sum) > 0.001) { printf(" Results verification failed at index %zu: Expected %.1f, Got " "%.1f\n", i, expected_sum, h_result[i]); all_ok = false; break; } } if (all_ok) { printf("Results verified correctly\n"); } else { printf("Results verification failed\n"); } } // ======================================================================== // STEP 7: Cleanup and Resource Management // ======================================================================== // Important: Cleanup must happen in the correct order // 1. Free host memory // 2. Deregister buffers from communicator // 3. Free device memory // 4. Destroy CUDA resources // 5. Finalize and destroy NCCL communicator free(h_data); free(h_result); // Deregister buffers from communicator // This must happen before freeing the buffers or destroying the // communicator NCCLCHECK(ncclCommDeregister(comm, send_handle)); NCCLCHECK(ncclCommDeregister(comm, recv_handle)); printf(" Rank %d buffers deregistered\n", my_rank); // Free device memory allocated by NCCL NCCLCHECK(ncclMemFree(d_sendbuff)); NCCLCHECK(ncclMemFree(d_recvbuff)); // Finalize and destroy NCCL communicator NCCLCHECK(ncclCommFinalize(comm)); NCCLCHECK(ncclCommDestroy(comm)); // Destroy CUDA stream CUDACHECK(cudaStreamDestroy(stream)); if (my_rank == 0) { printf("All resources cleaned up successfully\n"); } return NULL; } int main(int argc, char *argv[]) { // Run example using the standard test framework // This handles MPI/pthread initialization, device assignment, and cleanup return run_example(argc, argv, allReduce); } nccl-2.29.7-1/docs/examples/04_user_buffer_registration/Makefile000066400000000000000000000021071515037102200244760ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # NCCL User Buffer Examples EXAMPLES = 01_allreduce # Default target all: $(EXAMPLES) # Build individual examples $(EXAMPLES): $(MAKE) -C $@ # Clean all build artifacts clean: for example in $(EXAMPLES); do \ $(MAKE) -C $$example clean; \ done # Test all examples test: all for example in $(EXAMPLES); do \ echo "Testing $$example..."; \ $(MAKE) -C $$example test; \ done # Help help: @echo "NCCL User Buffer Registration Examples" @echo "======================================" @echo "" @echo "Targets:" @echo " all - Build all examples" @echo " clean - Clean all build artifacts" @echo " test - Test all examples" @echo " help - Show this help" @echo "" @echo "Examples:" @echo " 01_allreduce - AllReduce collective operation" @echo "" @echo "To build/run individual examples:" @echo " make -C 01_allreduce" .PHONY: all clean test help $(EXAMPLES) nccl-2.29.7-1/docs/examples/04_user_buffer_registration/README.md000066400000000000000000000046711515037102200243250ustar00rootroot00000000000000 # NCCL User Buffer Registration Examples ## Overview This directory contains minimal examples that demonstrate NCCL user buffer registration for improving performance by allowing NCCL to operate directly on user-allocated buffers. ## Examples ### [01_allreduce](01_allreduce/) **AllReduce with User Buffer Registration** - **Pattern**: Register communication buffers once and reuse across operations - **API**: `ncclCommRegister`, `ncclCommDeregister`, `ncclMemAlloc`, `ncclAllReduce` - **Use case**: Repeated collectives on the same buffers; performance-critical workloads - **Key features**: - Buffers allocated via `ncclMemAlloc` for registration compatibility - Registration handles managed explicitly (register → use → deregister) - Collective operations executed on registered buffers - Correct cleanup and verification ## Choosing the Right Pattern *Scenario* : Optimize performance for repeated collectives on same buffers *Addresses* : Throughput-sensitive training loops *Dependencies* : pthread or MPI ### Why Buffer Registration? Pre-registering buffers eliminates per-call registration overhead and enables direct access. It can accelerate collectives and greatly reduce the resource usage (e.g. #channel usage). Also, this is a prerequisite for advanced features such as symmetric memory or device API calls. ```c // Allocate using NCCL convenience function and register buffers NCCLCHECK(ncclMemAlloc((void**)&d_send, size_bytes)); NCCLCHECK(ncclCommRegister(comm, d_send, size_bytes, &send_handle)); // Use in collectives NCCLCHECK(ncclAllReduce(d_send, d_recv, count, ncclFloat, ncclSum, comm, stream)); // Deregister and free NCCLCHECK(ncclCommDeregister(comm, send_handle)); NCCLCHECK(ncclMemFree(d_send)); ``` ## Building ### **Quick Start** ```shell # Build example by directory name make 01_allreduce ``` ### **Individual Examples** ```shell # Build and run AllReduce with user buffer registration cd 01_allreduce && make ./allreduce_ub ``` ## References - [NCCL User Guide: Examples](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/examples.html) - [NCCL API Reference](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api.html) - [CUDA Programming Guide](https://docs.nvidia.com/cuda/cuda-c-programming-guide/) nccl-2.29.7-1/docs/examples/05_symmetric_memory/000077500000000000000000000000001515037102200213425ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/05_symmetric_memory/01_allreduce/000077500000000000000000000000001515037102200236025ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/05_symmetric_memory/01_allreduce/Makefile000066400000000000000000000033741515037102200252510ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Include common build rules include ../../../../makefiles/common.mk include ../../../../makefiles/examples.mk # Target executable TARGET = allreduce_sm # Common utilities COMMON_INC = ../../common/include COMMON_SRC = ../../common/src # Build configuration INCLUDES += -I$(COMMON_INC) # Source files SOURCES = main.cc $(COMMON_SRC)/utils.cc OBJECTS = $(SOURCES:.cc=.o) # Default target all: $(TARGET) # Build executable $(TARGET): $(OBJECTS) ifeq ($(MPI),1) $(MPICXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ else $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -lpthread -o $@ endif @echo "Built target $@" # Compile source files %.o: %.cc ifeq ($(MPI),1) $(MPICXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ else $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ endif # Test target test: $(TARGET) @echo "Testing $(TARGET)..." ifeq ($(MPI),1) @echo "Running with 2 processes" $(MPIRUN) -np 2 ./$(TARGET) else @echo "Running with all available GPUs" ./$(TARGET) endif # Clean build artifacts clean: rm -f $(OBJECTS) $(TARGET) # Install target install: $(TARGET) @mkdir -p $(PREFIX)/bin cp $(TARGET) $(PREFIX)/bin/ # Help help: @echo "NCCL Example: Symmetric Memeory Allreduce" @echo "==============================================" @echo "" @echo "Targets:" @echo " all - Build the example (default)" @echo " test - Build and run test with all GPUs" @echo " clean - Remove build artifacts" @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" @echo " help - Show this help" .PHONY: all test clean install help nccl-2.29.7-1/docs/examples/05_symmetric_memory/01_allreduce/README.md000066400000000000000000000136711515037102200250710ustar00rootroot00000000000000 # NCCL Symmetric Memory AllReduce Example This example demonstrates how to use NCCL's symmetric memory feature for optimized collective operations. Symmetric memory provides optimized performance by leveraging consistent memory layouts across all participating ranks, enabling advanced communication algorithms. ## Overview Symmetric memory windows provide a way to register memory buffers that benefit from optimized collective operations. When using `NCCL_WIN_COLL_SYMMETRIC`, all ranks must provide symmetric buffers, enabling optimized communication patterns and better performance for large-scale multi-GPU operations. ## What This Example Does 1. **Allocates memory using NCCL allocator** (`ncclMemAlloc`) which provides memory compatible with symmetric windows 2. **Registers buffers as symmetric windows** using `ncclCommWindowRegister` with `NCCL_WIN_COLL_SYMMETRIC` flag 3. **Performs AllReduce sum operation** using the symmetric memory for optimized communication performance ## Building and Running The advanced examples can be built using either pthread or MPI for parallelization. pthread is the default choice. To use MPI the user needs to set `MPI=1` at build time and can optionally provide a valid MPI installation under `MPI_HOME`. ### Build ```shell make [MPI=1] [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] ``` ### Run when compiled for pthreads (default) ```shell [NTHREADS=N] ./allreduce_sm ``` ### Run when compiled for MPI ```shell mpirun -np ./allreduce_sm ``` ## Code Structure ### Key Components 1. **Buffer Allocation and Window Registration**: ```c size_t size_bytes; // Is set to the size of the send/receive buffers void *d_sendbuff; void *d_recvbuff; // Allocate buffers using ncclMemAlloc (compatible with symmetric memory) NCCLCHECK(ncclMemAlloc(&d_sendbuff, size_bytes)); NCCLCHECK(ncclMemAlloc(&d_recvbuff, size_bytes)); ncclComm_t comm; ncclWindow_t send_win; ncclWindow_t recv_win; // Register buffers as symmetric windows NCCLCHECK(ncclCommWindowRegister(comm, d_sendbuff, size_bytes, &send_win, NCCL_WIN_COLL_SYMMETRIC)); NCCLCHECK(ncclCommWindowRegister(comm, d_recvbuff, size_bytes, &recv_win, NCCL_WIN_COLL_SYMMETRIC)); ``` 2. **AllReduce Operation**: ```c size_t count; // set to number of floats to exchange cudaStream_t stream; // stream is set in cudaStreamCreate // Perform AllReduce with symmetric memory optimization NCCLCHECK(ncclAllReduce(d_sendbuff, d_recvbuff, count, ncclFloat, ncclSum, comm, stream)); ``` 3. **Window Deregistration and Cleanup**: ```c // Deregister symmetric memory windows NCCLCHECK(ncclCommWindowDeregister(comm, send_win)); NCCLCHECK(ncclCommWindowDeregister(comm, recv_win)); // Free buffers allocated with ncclMemAlloc NCCLCHECK(ncclMemFree(d_sendbuff)); NCCLCHECK(ncclMemFree(d_recvbuff)); ``` ## Expected Output ### With 4 GPUs (using pthreads/MPI) ``` Starting AllReduce example with 4 ranks Rank 0 communicator initialized using device 0 Rank 1 communicator initialized using device 1 Rank 2 communicator initialized using device 2 Rank 3 communicator initialized using device 3 Symmetric Memory allocation Rank 0 allocating 4.00 MB per buffer Rank 1 allocating 4.00 MB per buffer Rank 2 allocating 4.00 MB per buffer Rank 3 allocating 4.00 MB per buffer Rank 0 data initialized (value: 0) Rank 1 data initialized (value: 1) Rank 2 data initialized (value: 2) Rank 3 data initialized (value: 3) Starting AllReduce with 1048576 elements (4 MB) AllReduce completed successfully Verification - Expected: 6.0, Got: 6.0 Results verified correctly Rank 0 symmetric memory windows deregistered Rank 1 symmetric memory windows deregistered Rank 2 symmetric memory windows deregistered Rank 3 symmetric memory windows deregistered All resources cleaned up successfully Example completed - demonstrated symmetric memory lifecycle ``` ## Performance Benefits of Symmetric Memory Symmetric memory registration provides several performance advantages: - **Optimized Communication Algorithms**: NCCL can apply advanced optimizations when all ranks have symmetric layouts - **Better Memory Access Patterns**: Consistent layouts enable better caching and memory access optimization For more information on the performance benefits see the [Enabling Fast Inference and Resilient Training with NCCL 2.27]()https://developer.nvidia.com/blog/enabling-fast-inference-and-resilient-training-with-nccl-2-27/) blog. **Important**: Buffers must be allocated using the CUDA Virtual Memory Management (VMM) API. NCCL provides the `ncclMemAlloc` convenience function for symmetric memory registration. The `NCCL_WIN_COLL_SYMMETRIC` flag requires all ranks to provide symmetric buffers consistently. ## Key Insights - **Symmetric Memory Windows** are most beneficial for: - Large-scale collective operations with consistent memory patterns - Latency-sensitive kernels - Applications with predictable allocation patterns - **ncclCommInitRank** can be used for pthread or MPI parallel case - **Window registration** must happen on all ranks for collective operations - **Memory management** is critical - always deregister windows before freeing memory ## Common Issues and Solutions 1. **Window Registration Failure**: Buffers MUST be allocated with (VMM) API, e.g. `ncclMemAlloc` (not `cudaMalloc`) for symmetric memory. 2. **Allocation Error**: If `ncclMemAlloc` fails, check NCCL version (requires at least v2.27) and available memory 3. **Deregistration Order**: Always deregister windows before freeing memory or destroying communicators 4. **Symmetric Requirement**: All ranks must use `NCCL_WIN_COLL_SYMMETRIC` consistently in collective operations 5. **Memory Leaks**: Always use `ncclMemFree` for buffers allocated with `ncclMemAlloc` nccl-2.29.7-1/docs/examples/05_symmetric_memory/01_allreduce/main.cc000066400000000000000000000175321515037102200250450ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "cuda_runtime.h" #include "nccl.h" #include "utils.h" #include #include #include #include #include #include /* * NCCL Symmetric Memory AllReduce Example * * This example demonstrates how to use NCCL's symmetric memory feature * for collective operations. Symmetric memory provides optimized performance * by leveraging consistent memory layouts across all participating ranks. * * Learning Objectives: * - Learn how to register symmetric memory windows with NCCL communicators * - See the proper lifecycle management of symmetric memory * */ /* * This function can be called inside an MPI rank or pthread thread. The * initialization and broadcast are implemented in common/src/utils.cc for * easier readability. For fully integrated examples using pthreads or MPI see * examples in 01_communicators. */ void *allReduce(int my_rank, int total_ranks, int local_device, int devices_per_rank) { // ======================================================================== // STEP 1: Initialize NCCL Communicator and Setup // ======================================================================== ncclUniqueId nccl_unique_id; if (my_rank == 0) { printf("Starting AllReduce example with %d ranks\n", total_ranks); NCCLCHECK(ncclGetUniqueId(&nccl_unique_id)); } // Distribute unique ID. // This step ensures all ranks have the same unique ID for communicator // creation util_broadcast(0, my_rank, &nccl_unique_id); // Set device context for this rank // Each rank manages its assigned GPU device CUDACHECK(cudaSetDevice(local_device)); // Initialize NCCL communicator // This creates the communication context for collective operations ncclComm_t comm; NCCLCHECK(ncclCommInitRank(&comm, total_ranks, nccl_unique_id, my_rank)); printf(" Rank %d communicator initialized using device %d\n", my_rank, local_device); // ======================================================================== // STEP 2: Allocate Memory Using NCCL Allocator // ======================================================================== if (my_rank == 0) { printf("Symmetric Memory allocation\n"); } // Allocate memory - using larger buffers to demonstrate symmetric memory // benefits size_t count = 1024 * 1024; // 1M elements size_t size_bytes = count * sizeof(float); printf(" Rank %d allocating %.2f MB per buffer\n", my_rank, (float)size_bytes / (1024 * 1024)); float *h_data = (float *)malloc(size_bytes); // Allocate buffers using NCCL allocator // NCCL's allocator is compatible with symmetric memory layouts void *d_sendbuff; void *d_recvbuff; NCCLCHECK(ncclMemAlloc(&d_sendbuff, size_bytes)); NCCLCHECK(ncclMemAlloc(&d_recvbuff, size_bytes)); // ======================================================================== // STEP 3: Register Symmetric Memory Windows // ======================================================================== /* Passing NCCL_WIN_COLL_SYMMETRIC requires users to provide the symmetric * buffers among all ranks in collectives. * Every rank needs to call ncclCommWindowRegister to register its buffers. */ // Register symmetric memory windows with NCCL ncclWindow_t send_win; ncclWindow_t recv_win; NCCLCHECK(ncclCommWindowRegister(comm, d_sendbuff, size_bytes, &send_win, NCCL_WIN_COLL_SYMMETRIC)); NCCLCHECK(ncclCommWindowRegister(comm, d_recvbuff, size_bytes, &recv_win, NCCL_WIN_COLL_SYMMETRIC)); // ======================================================================== // STEP 4: Initialize Data and Prepare for Communication // ======================================================================== // Initialize data - each rank contributes its rank value // This creates a simple test pattern for verification for (size_t i = 0; i < count; i++) { h_data[i] = (float)my_rank; } CUDACHECK(cudaMemcpy(d_sendbuff, h_data, size_bytes, cudaMemcpyHostToDevice)); printf(" Rank %d data initialized (value: %d)\n", my_rank, my_rank); // Create stream for asynchronous operations // Streams allow overlapping computation and communication cudaStream_t stream; CUDACHECK(cudaStreamCreate(&stream)); // ======================================================================== // STEP 5: Perform AllReduce Operation // ======================================================================== if (my_rank == 0) { printf("Starting AllReduce with %zu elements (%zu MB)\n", count, size_bytes / (1024 * 1024)); } // Perform AllReduce operation // Since symmetric memory is registered, NCCL can apply optimized algorithms NCCLCHECK(ncclAllReduce(d_sendbuff, d_recvbuff, count, ncclFloat, ncclSum, comm, stream)); if (my_rank == 0) { printf("AllReduce completed successfully\n"); } // ======================================================================== // STEP 6: Verify Results and Validate Correctness // ======================================================================== // Synchronize to ensure completion CUDACHECK(cudaStreamSynchronize(stream)); // Verify results (optional - copy back and check) float *h_result = (float *)malloc(size_bytes); CUDACHECK(cudaMemcpy(h_result, d_recvbuff, size_bytes, cudaMemcpyDeviceToHost)); // Each element should be the sum of all ranks float expected_sum = (float)(total_ranks * (total_ranks - 1)) / 2; bool all_ok = true; if (my_rank == 0) { printf("Verification - Expected: %.1f, Got: %.1f\n", expected_sum, h_result[0]); for (size_t i = 1; i < count; i++) { if (fabsf(h_result[i] - expected_sum) > 0.001) { printf(" Results verification failed at index %zu: Expected %.1f, Got " "%.1f\n", i, expected_sum, h_result[i]); all_ok = false; break; } } if (all_ok) { printf("Results verified correctly\n"); } else { printf("Results verification failed\n"); } } // ======================================================================== // STEP 7: Cleanup and Resource Management // ======================================================================== // Important: Cleanup must happen in the correct order // 1. Free host memory // 2. Deregister symmetric memory windows // 3. Free device memory // 4. Destroy CUDA resources // 5. Finalize and destroy NCCL communicator free(h_data); free(h_result); // Deregister symmetric memory windows from communicator // This must happen before freeing the buffers or destroying the // communicator NCCLCHECK(ncclCommWindowDeregister(comm, send_win)); NCCLCHECK(ncclCommWindowDeregister(comm, recv_win)); printf(" Rank %d symmetric memory windows deregistered\n", my_rank); // Free device memory allocated by NCCL NCCLCHECK(ncclMemFree(d_sendbuff)); NCCLCHECK(ncclMemFree(d_recvbuff)); // Finalize and destroy NCCL communicator NCCLCHECK(ncclCommFinalize(comm)); NCCLCHECK(ncclCommDestroy(comm)); // Destroy CUDA stream CUDACHECK(cudaStreamDestroy(stream)); if (my_rank == 0) { printf("All resources cleaned up successfully\n"); printf("Example completed - demonstrated symmetric memory lifecycle\n"); } return NULL; } int main(int argc, char *argv[]) { // Run example using the standard test framework // This handles MPI/pthread initialization, device assignment, and cleanup return run_example(argc, argv, allReduce); } nccl-2.29.7-1/docs/examples/05_symmetric_memory/Makefile000066400000000000000000000020731515037102200230040ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # NCCL Shared Memory Examples EXAMPLES = 01_allreduce # Default target all: $(EXAMPLES) # Build individual examples $(EXAMPLES): $(MAKE) -C $@ # Clean all build artifacts clean: for example in $(EXAMPLES); do \ $(MAKE) -C $$example clean; \ done # Test all examples test: all for example in $(EXAMPLES); do \ echo "Testing $$example..."; \ $(MAKE) -C $$example test; \ done # Help help: @echo "NCCL Symmetric Memeory Examples" @echo "===============================" @echo "" @echo "Targets:" @echo " all - Build all examples" @echo " clean - Clean all build artifacts" @echo " test - Test all examples" @echo " help - Show this help" @echo "" @echo "Examples:" @echo " 01_allreduce - AllReduce collective operation" @echo "" @echo "To build/run individual examples:" @echo " make -C 01_allreduce" .PHONY: all clean test help $(EXAMPLES) nccl-2.29.7-1/docs/examples/05_symmetric_memory/README.md000066400000000000000000000046371515037102200226330ustar00rootroot00000000000000 # NCCL Symmetric Memory Examples ## Overview This directory contains minimal examples that demonstrate NCCL symmetric memory windows for improving performance of collective operations when all ranks use consistent memory layouts. ## Examples ### [01_allreduce](01_allreduce/) **AllReduce with Symmetric Memory Windows** - **Pattern**: Register symmetric windows per rank and use them for collectives - **API**: `ncclCommWindowRegister`, `ncclCommWindowDeregister`, `ncclMemAlloc`, `ncclAllReduce` - **Use case**: Large-scale collectives with consistent buffer layouts across ranks - **Key features**: - Buffers allocated via `ncclMemAlloc` for symmetric compatibility - Windows registered as `NCCL_WIN_COLL_SYMMETRIC` - Collective operations executed on symmetric windows - Correct deregistration and cleanup ## Choosing the Right Pattern *Scenario* : Large-scale training with consistent memory patterns *Addresses* : Low-latency, high-bandwidth collectives on supported systems *Dependencies* : pthread or MPI ### Why Symmetric Windows? Symmetric windows enable NCCL to apply optimized collective protocols when all ranks use consistent layouts. The memory needs to be allocated through the CUDA Virtual Memory Management (VMM) API and registered with NCCL. ```c // Allocate using NCCL provided convenience function and register symmetric windows NCCLCHECK(ncclMemAlloc(&buffer, size_bytes)); NCCLCHECK(ncclCommWindowRegister(comm, buffer, size_bytes, &win, NCCL_WIN_COLL_SYMMETRIC)); // Collective using symmetric windows NCCLCHECK(ncclAllReduce(buffer, buffer, count, ncclFloat, ncclSum, comm, stream)); // Deregister and free NCCLCHECK(ncclCommWindowDeregister(comm, win)); NCCLCHECK(ncclMemFree(buffer)); ``` ## Building ### **Quick Start** ```shell # Build example by directory name make 01_allreduce ``` ### **Individual Examples** ```shell # Build and run AllReduce with symmetric windows cd 01_allreduce && make ./allreduce_sm ``` ## References - [NCCL User Guide: Examples](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/examples.html) - [NCCL API Reference](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api.html) - [CUDA Programming Guide](https://docs.nvidia.com/cuda/cuda-c-programming-guide/) nccl-2.29.7-1/docs/examples/06_device_api/000077500000000000000000000000001515037102200200275ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/06_device_api/01_allreduce_lsa/000077500000000000000000000000001515037102200231265ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/06_device_api/01_allreduce_lsa/Makefile000066400000000000000000000035141515037102200245710ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Include common build rules include ../../../../makefiles/common.mk include ../../../../makefiles/examples.mk # Target executable TARGET = allreduce_lsa # Common utilities COMMON_INC = ../../common/include COMMON_SRC = ../../common/src # Build configuration INCLUDES += -I$(COMMON_INC) # Source files SOURCES = main.cu $(COMMON_SRC)/utils.cc OBJECTS = $(SOURCES:.cu=.o) OBJECTS := $(OBJECTS:.cc=.o) # Default target all: $(TARGET) # Build executable $(TARGET): $(OBJECTS) ifeq ($(MPI),1) $(MPICXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ else $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -lpthread -o $@ endif @echo "Built target $@" # Compile source files %.o: %.cu $(NVCC) $(NVCUFLAGS) $(INCLUDES) -c $< -o $@ %.o: %.cc ifeq ($(MPI),1) $(MPICXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ else $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ endif # Test target test: $(TARGET) @echo "Testing $(TARGET)..." ifeq ($(MPI),1) @echo "Running with 2 processes" $(MPIRUN) -np 2 ./$(TARGET) else @echo "Running with all available GPUs" ./$(TARGET) endif # Clean build artifacts clean: rm -f $(OBJECTS) $(TARGET) # Install target install: $(TARGET) @mkdir -p $(PREFIX)/bin cp $(TARGET) $(PREFIX)/bin/ # Help help: @echo "NCCL Example: Device API Allreduce" @echo "==============================================" @echo "" @echo "Targets:" @echo " all - Build the example (default)" @echo " test - Build and run test with all GPUs" @echo " clean - Remove build artifacts" @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" @echo " help - Show this help" .PHONY: all test clean install help nccl-2.29.7-1/docs/examples/06_device_api/01_allreduce_lsa/README.md000066400000000000000000000205041515037102200244060ustar00rootroot00000000000000 # NCCL Device API AllReduce Example This example shows how to implement AllReduce sum operation directly in a kernel using the NCCL device API. We first create a device communicator with `ncclDevCommCreate` to enable kernel-initiated communication. After that, device-side synchronization is performed with barriers and symmetric memory windows are used to enable Load Store Accessible (LSA) memory access of peers. ## Overview This example shows how to implement AllReduce sum operation using a GPU kernel that directly performs the collective operations. The device communicators are created with `ncclDevCommCreate` and device-side synchronization is ensured with Load Store Accessible (LSA) barriers. LSA windows are used for peer memory access. ## What This Example Does 1. **Creates device communicators** using `ncclDevCommCreate` for GPU kernel access to NCCL operations 2. **Registers symmetric memory windows** with `ncclCommWindowRegister` for direct peer-to-peer access 3. **Launches GPU kernel** that performs AllReduce sum operation entirely on device using LSA barriers ## Building and Running The advanced examples can be built using either pthread or MPI for parallelization. pthread is the default choice. To use MPI the user needs to set `MPI=1` at build time and can optionally provide a valid MPI installation under `MPI_HOME`. ### Build ```shell make [MPI=1] [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] ``` ### Run when compiled for pthreads (default) ```shell [NTHREADS=N] ./allreduce_lsa ``` ### Run when compiled for MPI ```shell mpirun -np ./allreduce_lsa ``` ## Code Walk-through ### Device Communicator Creation (Host-side) The `ncclDevComm` is the core component of the device API, enabling GPU kernels to perform inter-GPU communication and fuse computation with communication. The `ncclDevCommRequirements` specifies what resources the device communicator should allocate. In this example, we set `lsaBarrierCount` to match our thread block count, giving each block its own barrier for independent cross-GPU synchronization. ```cpp ncclDevComm devComm; ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; // Allocate one barrier per CTA we intend to launch reqs.lsaBarrierCount = NCCL_DEVICE_CTA_COUNT; // Create device communicator with LSA barrier support NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); ``` ### Memory Window Registration (Host-side) The device API requires symmetric memory windows registered using `NCCL_WIN_COLL_SYMMETRIC`. See the [symmetric memory example](../../05_symmetric_memory/) for allocation and requirements details. ```cpp ncclComm_t comm; void* d_sendbuff; void* d_recvbuff; ncclWindow_t send_win; ncclWindow_t recv_win; // Register symmetric windows for device-side peer access NCCLCHECK(ncclCommWindowRegister(comm, d_sendbuff, size_bytes, &send_win, NCCL_WIN_COLL_SYMMETRIC)); NCCLCHECK(ncclCommWindowRegister(comm, d_recvbuff, size_bytes, &recv_win, NCCL_WIN_COLL_SYMMETRIC)); ``` ### LSA Barriers (Device-side) LSA barriers enable cross-GPU synchronization from device code. Each thread block uses `blockIdx.x` to select its dedicated barrier, allowing blocks to progress independently while coordinating with corresponding blocks on other GPUs. ```cpp // LSA barriers enable coordination between GPU threads across different ranks // This ensures all ranks reach the same synchronization point before proceeding ncclLsaBarrierSession bar { ncclCoopCta(), // Barrier scope: entire CTA (thread block) devComm, ncclTeamLsa(devComm), devComm.lsaBarrier, blockIdx.x // Barrier index: matches our CTA index (0 to lsaBarrierCount-1) }; bar.sync(ncclCoopCta(), cuda::memory_order_relaxed); // ... // Release barrier ensures that we received data from everyone before we unblock the stream and allow the next kernel(s) to process the data. // Critical for correctness in device-side collective operations bar.sync(ncclCoopCta(), cuda::memory_order_release); ``` ### Memory Access (Device-side) `ncclGetLsaPointer` allows CUDA kernels to directly access other GPUs' memory within the LSA team. ```cpp // Access peer memory directly using LSA (Load/Store Accessible) pointers float* peerPtr = (float*)ncclGetLsaPointer(sendwin, sendoffset, peer); ``` ## Expected Output ``` Starting Device API AllReduce initialization Rank 0 using GPU device 0 Rank 1 using GPU device 1 Rank 2 using GPU device 2 Rank 3 using GPU device 3 Rank 0 initialized NCCL communicator for 4 total ranks Rank 1 initialized NCCL communicator for 4 total ranks Rank 2 initialized NCCL communicator for 4 total ranks Rank 3 initialized NCCL communicator for 4 total ranks Rank 0 initialized data with value 0 Rank 1 initialized data with value 1 Rank 2 initialized data with value 2 Rank 3 initialized data with value 3 Rank 0 created device communicator with 16 LSA barriers Rank 1 created device communicator with 16 LSA barriers Rank 2 created device communicator with 16 LSA barriers Rank 3 created device communicator with 16 LSA barriers Starting AllReduce with 1048576 elements (4 MB) using Device API Expected result: sum of ranks 0 to 3 = 6 per element Rank 0 completed AllReduce kernel execution Rank 1 completed AllReduce kernel execution Rank 2 completed AllReduce kernel execution Rank 3 completed AllReduce kernel execution AllReduce completed. Result verification: PASSED All elements correctly sum to 6 (ranks 0-3) ``` ## When to Use - **Kernel-level communication**: When compute kernels need immediate access to communication results - **Low-latency scenarios**: Reduced host-device synchronization overhead - **Custom collectives**: Implementing specialized reduction or communication patterns - **Iterative algorithms**: Repeated communication with minimal CPU involvement ## Performance Considerations **Advantages:** - Lower latency for small to medium message sizes - Eliminates host-device synchronization bottlenecks - Enables computation-communication fusion within kernels - Direct peer memory access without CPU copying **Disadvantages:** - More complex programming model requiring LSA barriers - Requires careful memory ordering and synchronization - Higher development complexity compared to host API - CUDA Compute Capability 7.0+ and GPUs with P2P support (e.g., NVLink or PCI) required. ## Common Issues and Solutions ### Issue: NCCL warning communicator does not support symmetric memory NCCL selects support for symmetric memory operations based on GPU connectivity. If the GPUs on a node are only connected through e.g. the inter-CPU link, symmetric memory will not be supported. **Solution:** Use `nvidia-smi` to identify and select a subset of GPUs (e.g. via `CUDA_VISIBLE_DEVICES`) connected through NVlink or PCIe. ### Issue: LSA barrier synchronization failures **Solution:** Ensure `lsaBarrierCount` matches the number of thread blocks in kernel launch configuration. ### Issue: Memory access violations in device kernel **Solution:** Verify memory windows are registered as `NCCL_WIN_COLL_SYMMETRIC` and all ranks use identical buffer sizes. ### Issue: Incomplete results or race conditions **Solution:** Use proper memory ordering in LSA barriers (`cuda::memory_order_relaxed` vs `cuda::memory_order_release`). ## Performance Notes - These are educational examples, not optimized for performance - Real implementations should use vectorization, loop unrolling, and memory coalescing - Consider NCCL's optimized device kernels for best practices related to performance - NCCL library implementation of device kernels for collective operations - NCCL perf tests implementations of optimized device kernels ## Error Handling The example uses comprehensive error checking for both CUDA and NCCL operations. Device kernels should implement proper error handling for LSA operations and memory access patterns. ## Next Steps After understanding this example, explore: - **Custom reduction operations**: Implement non-standard reduction patterns - **Mixed host-device patterns**: Combine host and device API for complex workflows - **Performance optimization**: Fine-tune LSA barrier usage and memory access patterns nccl-2.29.7-1/docs/examples/06_device_api/01_allreduce_lsa/main.cu000066400000000000000000000262541515037102200244140ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "cuda_runtime.h" #include "nccl.h" #include "nccl_device.h" #include "utils.h" #include #include #include #include #include /* * NCCL Device API AllReduce Example * * This example demonstrates NCCL's Device API, which enables GPU kernels to * directly interact with NCCL without CPU intervention. This is particularly * powerful for applications that need to perform communication * from within CUDA kernels. * * Learning Objectives: * - Understand NCCL Device API vs Host API differences * - Learn how to register memory windows for device-side access * - See how GPU kernels can perform collective operations directly * - Practice LSA (Load Store Access) barrier synchronization * * Key Device API Concepts: * - ncclDevComm: Device-side communicator for kernel use * - ncclWindow_t: Memory windows that enable direct peer access * - LSA barriers: Synchronization primitives for device-side coordination * - ncclGetLsaPointer: Direct access to peer memory from device code * * When to Use Device API: * - Compute kernels that need immediate communication results * - Fusion of computation and communication in a single kernel * - Reduced host-device synchronization overhead * - Custom collective operations not available in standard NCCL * * Performance Considerations: * - Lower latency than host API for small operations * - Enables computation-communication overlap within kernels * - Requires careful synchronization and memory ordering * - LSA barriers add coordination overhead but enable correctness */ // Device API kernel launch configuration // CTA count must match lsaBarrierCount for proper barrier synchronization #define NCCL_DEVICE_CTA_COUNT 16 #define NCCL_DEVICE_THREADS_PER_CTA 512 // ========================================================================== // Device Kernel Implementation // ========================================================================== // Device kernel that performs AllReduce sum operation // This kernel demonstrates direct NCCL communication from GPU threads __global__ void simpleAllReduceKernel(ncclWindow_t sendwin, size_t sendoffset, ncclWindow_t recvwin, size_t recvoffset, size_t count, int root, struct ncclDevComm devComm) { // LSA barriers enable coordination between GPU threads across different ranks // Barrier scope: CTA (all threads in this block participate) // Barrier index: blockIdx.x selects this CTA's dedicated barrier (one barrier per CTA) ncclLsaBarrierSession bar { ncclCoopCta(), devComm, ncclTeamLsa(devComm), devComm.lsaBarrier, blockIdx.x }; bar.sync(ncclCoopCta(), cuda::memory_order_relaxed); const int rank = devComm.rank, nRanks = devComm.nRanks; // We are going to spread the workload accross all GPU ranks. // So calculate the global thread ID accross all ranks. // This maps global threads to data elements in the data to be reduced const int globalTid = threadIdx.x + blockDim.x * (rank + blockIdx.x * nRanks); const int globalNthreads = blockDim.x * gridDim.x * nRanks; // Grid stride loop over all elements with the globalThreads for (size_t offset = globalTid; offset < count; offset += globalNthreads) { float v = 0; // Access remote (and local [peer==rank]) memory and reduce locally for (int peer=0; peer>>( send_win, 0, recv_win, 0, count, 0, devComm); // Wait for completion - kernel performs AllReduce. CUDACHECK(cudaStreamSynchronize(stream)); printf(" Rank %d completed AllReduce kernel execution\n", my_rank); // ========================================================================== // STEP 7: Verify Results and Cleanup Resources // ========================================================================== // Verify results by copying back and checking CUDACHECK(cudaMemcpy(h_data, d_recvbuff, size_bytes, cudaMemcpyDeviceToHost)); float expected = (float)((total_ranks * (total_ranks - 1)) / 2); bool success = true; for (int i = 0; i < count; i++) { if (h_data[i] != expected) { success = false; break; } } if (my_rank == 0) { printf("AllReduce completed. Result verification: %s\n", success ? "PASSED" : "FAILED"); if (success) { printf("All elements correctly sum to %.0f (ranks 0-%d)\n", expected, total_ranks - 1); } } // Cleanup resources in proper order free(h_data); // Device API specific cleanup NCCLCHECK(ncclDevCommDestroy(comm, &devComm)); NCCLCHECK(ncclCommWindowDeregister(comm, send_win)); NCCLCHECK(ncclCommWindowDeregister(comm, recv_win)); NCCLCHECK(ncclMemFree(d_sendbuff)); NCCLCHECK(ncclMemFree(d_recvbuff)); // Standard NCCL cleanup NCCLCHECK(ncclCommFinalize(comm)); NCCLCHECK(ncclCommDestroy(comm)); CUDACHECK(cudaStreamDestroy(stream)); return NULL; } int main(int argc, char* argv[]) { // Run example using the provided utility framework return run_example(argc, argv, allReduce); } nccl-2.29.7-1/docs/examples/06_device_api/02_alltoall_gin/000077500000000000000000000000001515037102200227715ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/06_device_api/02_alltoall_gin/Makefile000066400000000000000000000037401515037102200244350ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Include common build rules include ../../../../makefiles/common.mk include ../../../../makefiles/examples.mk # Target executable TARGET = alltoall_gin # Common utilities COMMON_INC = ../../common/include COMMON_SRC = ../../common/src # Build configuration INCLUDES += -I$(COMMON_INC) # Source files SOURCES = main.cu $(COMMON_SRC)/utils.cc OBJECTS = $(SOURCES:.cu=.o) OBJECTS := $(OBJECTS:.cc=.o) # Default target all: $(TARGET) # Build executable $(TARGET): $(OBJECTS) ifeq ($(MPI),1) $(MPICXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ else $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -lpthread -o $@ endif @echo "Built target $@" # Compile source files %.o: %.cu $(NVCC) $(NVCUFLAGS) $(INCLUDES) -c $< -o $@ %.o: %.cc ifeq ($(MPI),1) $(MPICXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ else $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ endif # Test target test: $(TARGET) @echo "Testing $(TARGET)..." ifeq ($(MPI),1) @echo "Running with 2 processes" $(MPIRUN) -np 2 ./$(TARGET) else @echo "Running with all available GPUs" ./$(TARGET) endif # Clean build artifacts clean: rm -f $(OBJECTS) $(TARGET) # Install target install: $(TARGET) @mkdir -p $(PREFIX)/bin cp $(TARGET) $(PREFIX)/bin/ # Help help: @echo "NCCL Example: Pure GIN AlltoAll Device API" @echo "==============================================" @echo "" @echo "This example demonstrates pure GPU-Initiated Networking (GIN)" @echo "for AlltoAll operations without LSA optimizations." @echo "" @echo "Targets:" @echo " all - Build the example (default)" @echo " test - Build and run test with all GPUs" @echo " clean - Remove build artifacts" @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" @echo " help - Show this help" .PHONY: all test clean install help nccl-2.29.7-1/docs/examples/06_device_api/02_alltoall_gin/README.md000066400000000000000000000176201515037102200242560ustar00rootroot00000000000000 # NCCL Device API Pure GIN AlltoAll Example This example demonstrates NCCL's GPU-Initiated Networking (GIN) capabilities for performing AlltoAll collective operations directly from GPU kernels using only network-based communication. ## Overview This example showcases **pure GIN communication** where all data exchange happens through the network, without any Load Store Access (LSA) optimizations. This is particularly useful for: - Multi-node environments where ranks cannot use LSA - Testing network performance without local optimizations - Understanding the baseline GIN communication patterns - Scenarios where all communication must go through the network ## What This Example Does 1. **Creates device communicators** using `ncclDevCommCreate` for GPU kernel access to NCCL operations 2. **Registers symmetric memory windows** with `ncclCommWindowRegister` for direct peer-to-peer access 3. **Launches GPU kernel** that performs AlltoAll operations using pure GIN for all peer communication ## Building and Running The advanced examples can be built using either pthread or MPI for parallelization. pthread is the default choice. To use MPI the user needs to set `MPI=1` at build time and can optionally provide a valid MPI installation under `MPI_HOME`. ### Build ```bash make [MPI=1] [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] ``` ### Run when compiled for pthreads (default) ```bash [NTHREADS=N] ./alltoall_gin ``` ### Run when compiled for MPI ```bash mpirun -np ./alltoall_gin ``` ## Code Walk-through ### Device Communicator Creation (Host-side) The `ncclDevComm` is the core component enabling GPU kernels to perform network communication directly. For pure GIN communication, we configure the device communicator with GIN-specific resources. The `ncclDevCommRequirements` specifies GIN barriers for network synchronization and signals for completion detection. Unlike LSA-based examples, we don't need LSA barriers since all communication goes through the network. ```cpp ncclDevComm devComm; ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; // GIN barriers enable cross-node synchronization over the network reqs.railGinBarrierCount = NCCL_DEVICE_CTA_COUNT; // GIN signals provide completion notifications for asynchronous operations reqs.ginSignalCount = 1; // Enable full GIN connectivity, i.e., connect each rank to all other ranks reqs.ginConnectionType = NCCL_GIN_CONNECTION_FULL; // Create device communicator with pure GIN support NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); ``` ### Memory Window Registration (Host-side) The device API requires symmetric memory windows registered using `NCCL_WIN_COLL_SYMMETRIC`. These windows enable GPU kernels to access remote memory through GIN operations. Unlike LSA which provides direct memory access, GIN windows are accessed through network put/get operations. ```cpp ncclWindow_t send_win; ncclWindow_t recv_win; // Register symmetric windows for GIN network access NCCLCHECK(ncclCommWindowRegister(comm, d_sendbuff, size_bytes, &send_win, NCCL_WIN_COLL_SYMMETRIC)); NCCLCHECK(ncclCommWindowRegister(comm, d_recvbuff, size_bytes, &recv_win, NCCL_WIN_COLL_SYMMETRIC)); ``` ### GIN Barriers (Device-side) GIN barriers enable cross-node synchronization from device code over the network. Each thread block uses `blockIdx.x` to select its dedicated barrier, allowing blocks to progress independently while coordinating with corresponding blocks on other nodes. This is crucial for ensuring all ranks are ready before starting the AlltoAll exchange. ```cpp // GIN barriers coordinate GPU threads across different nodes over network ncclGinBarrierSession bar { ncclCoopCta(), // Barrier scope: entire CTA (thread block) gin, // GIN context for network operations ncclTeamWorld(devComm), // Team spanning all ranks devComm.railGinBarrier, // GIN barrier handle blockIdx.x // Barrier index: matches our CTA index }; bar.sync(ncclCoopCta(), cuda::memory_order_relaxed, ncclGinFenceLevel::Relaxed); ``` ### GIN Put Operations (Device-side) GIN provides one-sided put operations for direct remote memory writes over the network. Each thread handles a subset of destination ranks, writing its rank's data to the appropriate location in each peer's receive buffer. The `ncclGin_SignalInc` parameter increments a signal counter, enabling asynchronous completion detection. ```cpp // Send data to all peers via GIN network operations const size_t size = count * sizeof(T); for (int r = tid; r < devComm.nRanks; r += nthreads) { gin.put(ncclTeamWorld(devComm), r, recvwin, recvoffset + devComm.rank * size, // Destination: peer r's buffer sendwin, sendoffset + r * size, // Source: data for peer r size, ncclGin_SignalInc{signalIndex}); // Signal increment for completion } ``` ### Signal-based Completion (Device-side) GIN uses signals for asynchronous completion detection of network operations. The kernel waits for the signal value to reach the expected count (initial value + number of ranks), indicating all put operations have completed. The `gin.flush()` ensures all pending operations are committed before proceeding. ```cpp // Wait for all remote puts to complete gin.waitSignal(ncclCoopCta(), signalIndex, signalValue + devComm.nRanks); gin.flush(ncclCoopCta()); // Ensure all operations are committed ``` ## Expected Output ``` Starting Pure GIN AlltoAll initialization Rank 0 using GPU device 0 Rank 1 using GPU device 1 Rank 0 initialized NCCL communicator for 2 total ranks Rank 1 initialized NCCL communicator for 2 total ranks Rank 0 initialized send data Rank 1 initialized send data Rank 0 created device communicator with GIN support Rank 1 created device communicator with GIN support Starting Pure GIN AlltoAll with 1024 elements per rank (2048 total elements, 0 MB) === Executing Pure GIN AlltoAll === Rank 0 completed pure GIN AlltoAll kernel Rank 1 completed pure GIN AlltoAll kernel Pure GIN AlltoAll result: PASSED ``` ## When to Use - **Multi-node environments**: When ranks cannot use LSA - **Testing network performance**: Without local optimizations ## Performance Considerations - **Network overhead**: All communication goes through the network stack - **Signal-based completion**: Enables asynchronous operation patterns - **Barrier synchronization**: Ensures proper ordering of network operations - **Multiple GIN contexts**: Can improve parallel communication performance ## Common Issues and Solutions ### Issue: Deadlock at util_broadcast **Solution:** Ensure you're running with multiple GPUs/processes ```bash NTHREADS=2 ./alltoall_gin # For 2 GPUs ``` ### Issue: CUDA out of memory **Solution:** Reduce the data size in the example ### Issue: Network errors **Solution:** Ensure proper network configuration for multi-node setups ## Performance Notes - These are educational examples, not optimized for performance - Real implementations should consider: - Optimal GIN context usage for parallel operations - Signal pool management for high-throughput scenarios - Memory coalescing patterns for network operations - Network topology-aware communication strategies ## Error Handling The example uses comprehensive error checking for CUDA, NCCL, and GIN operations. Device kernels should implement proper error handling for network operations and signal management. ## Next Steps After understanding this example, explore: - **Performance optimization**: Fine-tune GIN context usage and signal management - **Hybrid approaches**: Combine GIN with LSA for topology-aware optimizations - **Integration with compute**: Fuse network communication with computation kernels nccl-2.29.7-1/docs/examples/06_device_api/02_alltoall_gin/main.cu000066400000000000000000000252351515037102200242550ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "cuda_runtime.h" #include "nccl.h" #include "nccl_device.h" #include "utils.h" #include #include #include #include #include /* * NCCL Device API Pure GIN AlltoAll Example * * This example demonstrates NCCL's GPU-Initiated Networking (GIN) capabilities * for performing AlltoAll collective operations directly from GPU kernels using * only network-based communication. * GIN enables GPU kernels to initiate network communication without CPU * intervention, providing low-latency communication for distributed applications. * * Learning Objectives: * - Understand pure GIN (GPU-Initiated Networking) communication * - Learn how to use ncclGin for device-initiated network communication * - See pure GIN AlltoAll implementation for network-based communication * - Practice GIN barriers and signal-based synchronization * * Key GIN Concepts: * - ncclGin: Device-side networking object for kernel-initiated communication * - GIN contexts: Network communication channels for parallel operations * - GIN signals: Completion notifications for asynchronous operations * - GIN barriers: Network-based synchronization across ranks * - One-sided put operations: Direct remote memory writes over network * * When to Use Pure GIN: * - Communication between ranks that cannot use LSA (different nodes) * - Network-based collective operations in multi-node environments * - Scenarios where all communication must go through the network * - Testing network performance without local optimizations * * Performance Considerations: * - GIN provides network communication from GPU kernels * - All communication goes through the network (no local optimizations) * - Signal-based completion detection enables asynchronous operation * - Multiple GIN contexts can improve parallel communication performance */ // Device API kernel launch configuration // CTA count must match railGinBarrierCount for proper barrier synchronization #define NCCL_DEVICE_CTA_COUNT 1 #define NCCL_DEVICE_THREADS_PER_CTA 512 // ========================================================================== // Device Kernel Implementations // ========================================================================== // Pure GIN AlltoAll kernel - uses GIN for all peer communication // This kernel demonstrates network-based AlltoAll using GPU-initiated networking template __global__ void PureGinAlltoAllKernel(ncclWindow_t sendwin, size_t sendoffset, ncclWindow_t recvwin, size_t recvoffset, size_t count, int root, struct ncclDevComm devComm) { int ginContext = 0; unsigned int signalIndex = 0; ncclGin gin { devComm, ginContext }; uint64_t signalValue = gin.readSignal(signalIndex); // GIN barriers enable coordination between GPU threads across different ranks over network ncclGinBarrierSession bar { ncclCoopCta(), gin, ncclTeamWorld(devComm), devComm.railGinBarrier, blockIdx.x }; bar.sync(ncclCoopCta(), cuda::memory_order_relaxed, ncclGinFenceLevel::Relaxed); int tid = threadIdx.x + blockIdx.x * blockDim.x; int nthreads = blockDim.x * gridDim.x; // Send to all peers via GIN (GPU-initiated networking) const size_t size = count * sizeof(T); for (int r = tid; r < devComm.nRanks; r += nthreads) { gin.put(ncclTeamWorld(devComm), r, recvwin, recvoffset + devComm.rank * size, sendwin, sendoffset + r * size, size, ncclGin_SignalInc{signalIndex}); } // Wait for all remote puts to complete using signal-based synchronization gin.waitSignal(ncclCoopCta(), signalIndex, signalValue + devComm.nRanks); gin.flush(ncclCoopCta()); } // ========================================================================== // Host-Side Setup and Device API Initialization // ========================================================================== void* pureGinAlltoAll(int my_rank, int total_ranks, int local_device, int devices_per_rank) { ncclComm_t comm; ncclUniqueId nccl_unique_id; if (my_rank == 0) { printf("Starting Pure GIN AlltoAll initialization\n"); } // Standard NCCL communicator initialization if (my_rank == 0) { NCCLCHECK(ncclGetUniqueId(&nccl_unique_id)); } // Distribute unique ID util_broadcast(0, my_rank, &nccl_unique_id); // Set device context for this rank CUDACHECK(cudaSetDevice(local_device)); printf(" Rank %d using GPU device %d\n", my_rank, local_device); // ========================================================================== // STEP 2: Initialize NCCL Communicator and Allocate Memory // ========================================================================== // Initialize NCCL communicator NCCLCHECK(ncclCommInitRank(&comm, total_ranks, nccl_unique_id, my_rank)); printf(" Rank %d initialized NCCL communicator for %d total ranks\n", my_rank, total_ranks); // Check for Device API and GIN support ncclCommProperties_t props = NCCL_COMM_PROPERTIES_INITIALIZER; NCCLCHECK(ncclCommQueryProperties(comm, &props)); if (!props.deviceApiSupport) { printf("ERROR: rank %d communicator does not support Device API!\n", my_rank); NCCLCHECK(ncclCommFinalize(comm)); NCCLCHECK(ncclCommDestroy(comm)); return NULL; } if (props.ginType == NCCL_GIN_TYPE_NONE) { printf("ERROR: rank %d communicator does not support GIN!\n", my_rank); NCCLCHECK(ncclCommFinalize(comm)); NCCLCHECK(ncclCommDestroy(comm)); return NULL; } // Allocate memory for AlltoAll operation size_t count = 1024; // Elements per rank size_t total_elements = count * total_ranks; size_t size_bytes = total_elements * sizeof(float); float *h_sendbuff = (float*)malloc(size_bytes); float *h_recvbuff = (float*)malloc(size_bytes); void* d_sendbuff; void* d_recvbuff; ncclWindow_t send_win; ncclWindow_t recv_win; // Device API requires symmetric memory allocation NCCLCHECK(ncclMemAlloc(&d_sendbuff, size_bytes)); NCCLCHECK(ncclMemAlloc(&d_recvbuff, size_bytes)); // ========================================================================== // STEP 3: Register Memory Windows for Device-Side Access // ========================================================================== // Register symmetric windows for GIN access NCCLCHECK(ncclCommWindowRegister(comm, d_sendbuff, size_bytes, &send_win, NCCL_WIN_COLL_SYMMETRIC)); NCCLCHECK(ncclCommWindowRegister(comm, d_recvbuff, size_bytes, &recv_win, NCCL_WIN_COLL_SYMMETRIC)); // Initialize data: each rank sends unique values to each destination for (size_t i = 0; i < total_elements; i++) { int dest_rank = i / count; int element_idx = i % count; h_sendbuff[i] = (float)(my_rank * 1000 + dest_rank * 100 + element_idx); } CUDACHECK(cudaMemcpy(d_sendbuff, h_sendbuff, size_bytes, cudaMemcpyHostToDevice)); printf(" Rank %d initialized send data\n", my_rank); // ========================================================================== // STEP 4: Create Device Communicator with GIN Support // ========================================================================== // Create stream for kernel execution cudaStream_t stream; CUDACHECK(cudaStreamCreate(&stream)); // Create device communicator with GIN support ncclDevComm devComm; ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; reqs.railGinBarrierCount = NCCL_DEVICE_CTA_COUNT; // GIN barriers for network synchronization reqs.ginSignalCount = 1; // GIN signals for completion detection reqs.ginConnectionType = NCCL_GIN_CONNECTION_FULL; // Enable full GIN connectivity NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); printf(" Rank %d created device communicator with GIN support\n", my_rank); if (my_rank == 0) { printf("Starting Pure GIN AlltoAll with %zu elements per rank (%zu total elements, %zu MB)\n", count, total_elements, size_bytes / (1024 * 1024)); } // ========================================================================== // STEP 5: Execute Pure GIN AlltoAll Kernel // ========================================================================== if (my_rank == 0) { printf("\n=== Executing Pure GIN AlltoAll ===\n"); } // Clear receive buffer CUDACHECK(cudaMemset(d_recvbuff, 0, size_bytes)); // Launch pure GIN AlltoAll kernel PureGinAlltoAllKernel<<>>( send_win, 0, recv_win, 0, count, 0, devComm); // Wait for completion CUDACHECK(cudaStreamSynchronize(stream)); printf(" Rank %d completed pure GIN AlltoAll kernel\n", my_rank); // ========================================================================== // STEP 6: Verify Results // ========================================================================== // Verify pure GIN results CUDACHECK(cudaMemcpy(h_recvbuff, d_recvbuff, size_bytes, cudaMemcpyDeviceToHost)); bool gin_success = true; for (int src_rank = 0; src_rank < total_ranks; src_rank++) { for (size_t i = 0; i < count; i++) { size_t recv_idx = src_rank * count + i; float expected = (float)(src_rank * 1000 + my_rank * 100 + i); if (h_recvbuff[recv_idx] != expected) { gin_success = false; printf(" Rank %d: Pure GIN mismatch at [%d][%zu]: got %.0f, expected %.0f\n", my_rank, src_rank, i, h_recvbuff[recv_idx], expected); break; } } if (!gin_success) break; } if (my_rank == 0) { printf("Pure GIN AlltoAll result: %s\n", gin_success ? "PASSED" : "FAILED"); } // ========================================================================== // STEP 7: Cleanup Resources // ========================================================================== // Cleanup host memory free(h_sendbuff); free(h_recvbuff); // Device API specific cleanup NCCLCHECK(ncclDevCommDestroy(comm, &devComm)); NCCLCHECK(ncclCommWindowDeregister(comm, send_win)); NCCLCHECK(ncclCommWindowDeregister(comm, recv_win)); NCCLCHECK(ncclMemFree(d_sendbuff)); NCCLCHECK(ncclMemFree(d_recvbuff)); // Standard NCCL cleanup NCCLCHECK(ncclCommFinalize(comm)); NCCLCHECK(ncclCommDestroy(comm)); CUDACHECK(cudaStreamDestroy(stream)); return NULL; } int main(int argc, char* argv[]) { // Run example using the provided utility framework return run_example(argc, argv, pureGinAlltoAll); } nccl-2.29.7-1/docs/examples/06_device_api/03_alltoall_hybrid/000077500000000000000000000000001515037102200234765ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/06_device_api/03_alltoall_hybrid/Makefile000066400000000000000000000040131515037102200251340ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Include common build rules include ../../../../makefiles/common.mk include ../../../../makefiles/examples.mk # Target executable TARGET = alltoall_hybrid # Common utilities COMMON_INC = ../../common/include COMMON_SRC = ../../common/src # Build configuration INCLUDES += -I$(COMMON_INC) # Source files SOURCES = main.cu $(COMMON_SRC)/utils.cc OBJECTS = $(SOURCES:.cu=.o) OBJECTS := $(OBJECTS:.cc=.o) # Default target all: $(TARGET) # Build executable $(TARGET): $(OBJECTS) ifeq ($(MPI),1) $(MPICXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -o $@ else $(CXX) $(CXXFLAGS) $(OBJECTS) $(LIBRARIES) $(LDFLAGS) -lpthread -o $@ endif @echo "Built target $@" # Compile source files %.o: %.cu $(NVCC) $(NVCUFLAGS) $(INCLUDES) -c $< -o $@ %.o: %.cc ifeq ($(MPI),1) $(MPICXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ else $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ endif # Test target test: $(TARGET) @echo "Testing $(TARGET)..." ifeq ($(MPI),1) @echo "Running with 2 processes" $(MPIRUN) -np 2 ./$(TARGET) else @echo "Running with all available GPUs" ./$(TARGET) endif # Clean build artifacts clean: rm -f $(OBJECTS) $(TARGET) # Install target install: $(TARGET) @mkdir -p $(PREFIX)/bin cp $(TARGET) $(PREFIX)/bin/ # Help help: @echo "NCCL Example: Hybrid AlltoAll Device API" @echo "=========================================" @echo "" @echo "This example demonstrates hybrid communication combining" @echo "GPU-Initiated Networking (GIN) for remote peers with" @echo "Load Store Access (LSA) for local peers." @echo "" @echo "Targets:" @echo " all - Build the example (default)" @echo " test - Build and run test with all GPUs" @echo " clean - Remove build artifacts" @echo " install - Install to PREFIX/bin (default: /usr/local/bin)" @echo " help - Show this help" .PHONY: all test clean install help nccl-2.29.7-1/docs/examples/06_device_api/03_alltoall_hybrid/README.md000066400000000000000000000231231515037102200247560ustar00rootroot00000000000000 # NCCL Device API Hybrid AlltoAll Example This example shows how to implement AlltoAll operations using a hybrid approach that combines Load Store Access (LSA) for local peers with GPU-Initiated Networking (GIN) for remote peers. We create a device communicator with `ncclDevCommCreate` supporting both LSA and GIN capabilities, enabling optimal communication performance across different peer types. ## Overview This example showcases **hybrid communication** that intelligently selects the optimal communication method for each peer: - **LSA (Load Store Access)** for local peers (same node/memory space) - **GIN (GPU-Initiated Networking)** for remote peers (different nodes) ## What This Example Does 1. **Creates hybrid device communicators** using `ncclDevCommCreate` with both LSA and GIN support for optimal peer communication 2. **Registers symmetric memory windows** with `ncclCommWindowRegister` for both LSA direct access and GIN network operations 3. **Launches GPU kernel** that performs AlltoAll operations using LSA for local peers and GIN for remote peers 4. **Demonstrates hybrid synchronization** coordinating both LSA barriers and GIN signals for correctness ## Building and Running The advanced examples can be built using either pthread or MPI for parallelization. pthread is the default choice. To use MPI the user needs to set `MPI=1` at build time and can optionally provide a valid MPI installation under `MPI_HOME`. ### Build ```bash make [MPI=1] [MPI_HOME=] [NCCL_HOME=] [CUDA_HOME=] ``` ### Run when compiled for pthreads (default) ```bash [NTHREADS=N] ./alltoall_hybrid ``` ### Run when compiled for MPI ```bash mpirun -np ./alltoall_hybrid ``` ## Code Walk-through ### Device Communicator Creation (Host-side) The `ncclDevComm` is the core component enabling GPU kernels to perform both local and remote communication. For hybrid communication, we configure the device communicator with both LSA and GIN resources. The `ncclDevCommRequirements` specifies LSA barriers for local synchronization, GIN barriers for network synchronization, and GIN signals for completion detection. This dual setup enables optimal communication for each peer type. ```cpp ncclDevComm devComm; ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; // LSA barriers enable direct memory access coordination for local peers reqs.lsaBarrierCount = NCCL_DEVICE_CTA_COUNT; // GIN barriers enable cross-node synchronization over the network reqs.railGinBarrierCount = NCCL_DEVICE_CTA_COUNT; // GIN signals provide completion notifications for asynchronous network operations reqs.ginSignalCount = 1; // Enable full GIN connectivity, i.e., connect each rank to all other ranks reqs.ginConnectionType = NCCL_GIN_CONNECTION_FULL; // Create device communicator with hybrid LSA+GIN support NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); ``` ### Memory Window Registration (Host-side) The device API requires symmetric memory windows registered using `NCCL_WIN_COLL_SYMMETRIC`. These windows enable both LSA direct access for local peers and GIN network operations for remote peers. The same memory windows support both communication methods, with the kernel automatically selecting the appropriate access pattern based on peer locality. ```cpp ncclWindow_t send_win; ncclWindow_t recv_win; // Register symmetric windows for both LSA and GIN access NCCLCHECK(ncclCommWindowRegister(comm, d_sendbuff, size_bytes, &send_win, NCCL_WIN_COLL_SYMMETRIC)); NCCLCHECK(ncclCommWindowRegister(comm, d_recvbuff, size_bytes, &recv_win, NCCL_WIN_COLL_SYMMETRIC)); ``` ### Hybrid Barriers (Device-side) Hybrid barriers coordinate both local LSA operations and remote GIN operations. The barrier session uses the world team and GIN context to ensure synchronization across all ranks, regardless of their communication method. This unified barrier approach ensures all peers reach the same synchronization point before proceeding with data exchange. ```cpp // Hybrid barriers coordinate both LSA and GIN operations across all ranks ncclBarrierSession bar { ncclCoopCta(), // Barrier scope: entire CTA (thread block) ncclTeamTagWorld(), // Team spanning all ranks (local + remote) gin, // GIN context for network coordination blockIdx.x // Barrier index: matches our CTA index }; bar.sync(ncclCoopCta(), cuda::memory_order_relaxed, ncclGinFenceLevel::Relaxed); ``` ### Peer Classification (Device-side) The hybrid kernel intelligently classifies peers into local (LSA-accessible) and remote (GIN-only) categories. This classification determines the optimal communication method for each peer. Local peers benefit from direct memory access, while remote peers use network communication. ```cpp // Classify peers into local (LSA) and remote (GIN) categories ncclTeam world = ncclTeamWorld(devComm); // All ranks ncclTeam lsa = ncclTeamLsa(devComm); // Local ranks only const int startLsa = world.rank - lsa.rank; // First local rank in world const int lsaSize = lsa.nRanks; // Number of local peers ``` ### Memory Access (Device-side) `ncclGetLsaPointer` allows CUDA kernels to directly access other GPUs' memory within the LSA team, while `gin.put` handles remote communication over the network. The hybrid approach uses the most efficient method for each peer type. ```cpp // Handle local peers using direct memory access (LSA) T* sendLocal = (T*)ncclGetLocalPointer(sendwin, sendoffset); T* recvPtr = (T*)ncclGetLsaPointer(recvwin, recvoffset, lp); // Handle remote peers using network operations (GIN) gin.put(world, r, recvwin, recvoffset + world.rank * size, sendwin, sendoffset + r * size, size, ncclGin_SignalInc{signalIndex}); ``` ## Building and Running ### Build ```bash make ``` ### Run with pthread mode (default) ```bash # Run with all available GPUs ./alltoall_hybrid # Run with specific number of GPUs NTHREADS=4 ./alltoall_hybrid ``` ### Run with MPI mode ```bash # Build with MPI support make MPI=1 # Run with MPI across multiple nodes mpirun -np 4 --hostfile hosts ./alltoall_hybrid ``` ### Test ```bash make test ``` ## Expected Output ``` Starting Hybrid AlltoAll initialization Rank 0 using GPU device 0 Rank 1 using GPU device 1 Rank 2 using GPU device 2 Rank 3 using GPU device 3 Rank 0 initialized NCCL communicator for 4 total ranks Rank 1 initialized NCCL communicator for 4 total ranks Rank 2 initialized NCCL communicator for 4 total ranks Rank 3 initialized NCCL communicator for 4 total ranks Rank 0 initialized send data Rank 1 initialized send data Rank 2 initialized send data Rank 3 initialized send data Rank 0 created device communicator with hybrid support Rank 1 created device communicator with hybrid support Rank 2 created device communicator with hybrid support Rank 3 created device communicator with hybrid support Starting Hybrid AlltoAll with 1024 elements per rank (4096 total elements, 0 MB) Using LSA for local peers and GIN for remote peers === Executing Hybrid AlltoAll === Rank 0 completed hybrid AlltoAll kernel Rank 1 completed hybrid AlltoAll kernel Rank 2 completed hybrid AlltoAll kernel Rank 3 completed hybrid AlltoAll kernel Hybrid AlltoAll result: PASSED ✓ All 4096 elements correctly exchanged using hybrid communication ``` ## When to Use - **Multi-node usage**: Mixed local/remote communication patterns - **Production workloads**: Where performance is critical - **Heterogeneous clusters**: Different node configurations ## Performance Considerations **Advantages:** - **Reduced Latency**: LSA provides low latency for local communication - **Optimal Bandwidth**: GIN efficiently handles remote communication - **Reduced Network Load**: Local traffic stays off the network - **Scalable Design**: Efficient across different node configurations **Disadvantages:** - More complex programming model requiring coordination of both LSA and GIN - Requires careful synchronization between different communication methods - Higher development complexity compared to pure approaches ## Common Issues and Solutions ### Issue: LSA barriers not supported **Cause:** GPUs not connected through NVLink or PCIe for direct memory access **Solution:** Verify GPU topology with `nvidia-smi topo -m` and ensure proper LSA-capable connections ### Issue: Hybrid synchronization failures **Solution:** Ensure both `lsaBarrierCount` and `railGinBarrierCount` match the number of thread blocks in kernel launch configuration ## Performance Notes - These are educational examples, not optimized for performance - Real implementations should consider: - Optimal balance between LSA and GIN operations based on topology - Memory coalescing patterns for both LSA and GIN operations - Barrier synchronization overhead minimization - Signal pool management for high-throughput GIN scenarios ## Error Handling The example uses comprehensive error checking for CUDA, NCCL, LSA, and GIN operations. Device kernels should implement proper error handling for both direct memory access patterns and network operations. ## Next Steps After understanding this example, explore: - **Topology-aware optimization**: Fine-tune LSA/GIN balance based on hardware topology - **Custom hybrid patterns**: Implement specialized communication strategies - **Performance profiling**: Analyze LSA vs GIN performance characteristics - **Advanced synchronization**: Optimize barrier usage for complex communication patterns nccl-2.29.7-1/docs/examples/06_device_api/03_alltoall_hybrid/main.cu000066400000000000000000000271511515037102200247610ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "cuda_runtime.h" #include "nccl.h" #include "nccl_device.h" #include "utils.h" #include #include #include #include #include /* * NCCL Device API Hybrid AlltoAll Example * * This example demonstrates NCCL's hybrid communication approach that combines * GPU-Initiated Networking (GIN) for remote peers with Load Store Access (LSA) * for local peers, optimizing AlltoAll collective operations. * * Learning Objectives: * - Understand hybrid communication optimization * - Learn when to use GIN vs LSA for different peer types * - Practice combining network and memory-based communication * - See performance optimization through intelligent peer selection * * Key Hybrid Concepts: * - **LSA (Load Store Access)**: Direct memory access for local peers * - **GIN (GPU-Initiated Networking)**: Network communication for remote peers * - **Peer classification**: Distinguishing between local and remote peers * - **Hybrid synchronization**: Combining LSA and GIN completion mechanisms * - **Performance optimization**: Using the fastest method for each peer type * * When to Use Hybrid: * - Multi-node environments with both local and remote peers * - Performance-critical applications requiring optimal communication * - Mixed communication patterns (intra-node + inter-node) * - Production workloads where efficiency matters * * Performance Benefits: * - LSA provides low-latency local communication * - GIN handles remote communication efficiently * - Reduced network traffic for local operations * - Optimal bandwidth utilization across communication types */ // Device API kernel launch configuration // CTA count must match railGinBarrierCount for proper barrier synchronization #define NCCL_DEVICE_CTA_COUNT 16 #define NCCL_DEVICE_THREADS_PER_CTA 512 // ========================================================================== // Device Kernel Implementation // ========================================================================== // Hybrid AlltoAll kernel - optimizes by using LSA for local peers, GIN for remote // This kernel demonstrates performance optimization using both communication methods template __global__ void HybridAlltoAllKernel(ncclWindow_t sendwin, size_t sendoffset, ncclWindow_t recvwin, size_t recvoffset, size_t count, int root, struct ncclDevComm devComm) { int ginContext = 0; unsigned int signalIndex = 0; ncclGin gin { devComm, ginContext }; uint64_t signalValue = gin.readSignal(signalIndex); // GIN barriers for cross-node synchronization ncclBarrierSession bar { ncclCoopCta(), ncclTeamTagWorld(), gin, blockIdx.x }; bar.sync(ncclCoopCta(), cuda::memory_order_relaxed, ncclGinFenceLevel::Relaxed); int tid = threadIdx.x + blockIdx.x * blockDim.x; int nthreads = blockDim.x * gridDim.x; ncclTeam world = ncclTeamWorld(devComm); ncclTeam lsa = ncclTeamLsa(devComm); const int startLsa = world.rank - lsa.rank; const int lsaSize = lsa.nRanks; // Handle remote peers (i.e., non-LSA) using GIN for network communication const size_t size = count * sizeof(T); for (int r = tid; r < startLsa; r += nthreads) { gin.put(world, r, recvwin, recvoffset + world.rank * size, sendwin, sendoffset + r * size, size, ncclGin_SignalInc{signalIndex}); } for (int r = startLsa + lsaSize + tid; r < world.nRanks; r += nthreads) { gin.put(world, r, recvwin, recvoffset + world.rank * size, sendwin, sendoffset + r * size, size, ncclGin_SignalInc{signalIndex}); } // Handle local peers with LSA (Load Store Access) for optimal performance T* sendLocal = (T*)ncclGetLocalPointer(sendwin, sendoffset); for (size_t offset = tid; offset < count; offset += nthreads) { for (int lp = 0; lp < lsa.nRanks; lp++) { int wr = startLsa + lp; T* recvPtr = (T*)ncclGetLsaPointer(recvwin, recvoffset, lp); recvPtr[world.rank * count + offset] = sendLocal[wr * count + offset]; } } // Wait for remote GIN operations to complete int numRemotePeers = world.nRanks - lsa.nRanks; gin.waitSignal(ncclCoopCta(), signalIndex, signalValue + numRemotePeers); gin.flush(ncclCoopCta()); // Final synchronization barrier bar.sync(ncclCoopCta(), cuda::memory_order_release, ncclGinFenceLevel::Relaxed); } // ========================================================================== // Host-Side Setup and Device API Initialization // ========================================================================== void* hybridAlltoAll(int my_rank, int total_ranks, int local_device, int devices_per_rank) { ncclComm_t comm; ncclUniqueId nccl_unique_id; if (my_rank == 0) { printf("Starting Hybrid AlltoAll initialization\n"); } // Standard NCCL communicator initialization if (my_rank == 0) { NCCLCHECK(ncclGetUniqueId(&nccl_unique_id)); } // Distribute unique ID util_broadcast(0, my_rank, &nccl_unique_id); // Set device context for this rank CUDACHECK(cudaSetDevice(local_device)); printf(" Rank %d using GPU device %d\n", my_rank, local_device); // ========================================================================== // STEP 2: Initialize NCCL Communicator and Allocate Memory // ========================================================================== // Initialize NCCL communicator NCCLCHECK(ncclCommInitRank(&comm, total_ranks, nccl_unique_id, my_rank)); printf(" Rank %d initialized NCCL communicator for %d total ranks\n", my_rank, total_ranks); // Check for Device API and GIN support ncclCommProperties_t props = NCCL_COMM_PROPERTIES_INITIALIZER; NCCLCHECK(ncclCommQueryProperties(comm, &props)); if (!props.deviceApiSupport) { printf("ERROR: rank %d communicator does not support Device API!\n", my_rank); NCCLCHECK(ncclCommFinalize(comm)); NCCLCHECK(ncclCommDestroy(comm)); return NULL; } if (props.ginType == NCCL_GIN_TYPE_NONE) { printf("ERROR: rank %d communicator does not support GIN!\n", my_rank); NCCLCHECK(ncclCommFinalize(comm)); NCCLCHECK(ncclCommDestroy(comm)); return NULL; } // Allocate memory for AlltoAll operation size_t count = 1024; // Elements per rank size_t total_elements = count * total_ranks; size_t size_bytes = total_elements * sizeof(float); float *h_sendbuff = (float*)malloc(size_bytes); float *h_recvbuff = (float*)malloc(size_bytes); void* d_sendbuff; void* d_recvbuff; ncclWindow_t send_win; ncclWindow_t recv_win; // Device API requires symmetric memory allocation NCCLCHECK(ncclMemAlloc(&d_sendbuff, size_bytes)); NCCLCHECK(ncclMemAlloc(&d_recvbuff, size_bytes)); // ========================================================================== // STEP 3: Register Memory Windows for Device-Side Access // ========================================================================== // Register symmetric windows for both LSA and GIN access NCCLCHECK(ncclCommWindowRegister(comm, d_sendbuff, size_bytes, &send_win, NCCL_WIN_COLL_SYMMETRIC)); NCCLCHECK(ncclCommWindowRegister(comm, d_recvbuff, size_bytes, &recv_win, NCCL_WIN_COLL_SYMMETRIC)); // Initialize data: each rank sends unique values to each destination for (size_t i = 0; i < total_elements; i++) { int dest_rank = i / count; int element_idx = i % count; h_sendbuff[i] = (float)(my_rank * 1000 + dest_rank * 100 + element_idx); } CUDACHECK(cudaMemcpy(d_sendbuff, h_sendbuff, size_bytes, cudaMemcpyHostToDevice)); printf(" Rank %d initialized send data\n", my_rank); // ========================================================================== // STEP 4: Create Device Communicator with Hybrid Support // ========================================================================== // Create stream for kernel execution cudaStream_t stream; CUDACHECK(cudaStreamCreate(&stream)); // Create device communicator with both LSA and GIN support ncclDevComm devComm; ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; reqs.lsaBarrierCount = NCCL_DEVICE_CTA_COUNT; // LSA barriers for local synchronization reqs.railGinBarrierCount = NCCL_DEVICE_CTA_COUNT; // GIN barriers for network synchronization reqs.ginSignalCount = 1; // GIN signals for completion detection reqs.ginConnectionType = NCCL_GIN_CONNECTION_FULL; // Enable full GIN connectivity NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); printf(" Rank %d created device communicator with hybrid support\n", my_rank); if (my_rank == 0) { printf("Starting Hybrid AlltoAll with %zu elements per rank (%zu total elements, %zu MB)\n", count, total_elements, size_bytes / (1024 * 1024)); printf("Using LSA for local peers and GIN for remote peers\n"); } // ========================================================================== // STEP 5: Execute Hybrid AlltoAll Kernel // ========================================================================== if (my_rank == 0) { printf("\n=== Executing Hybrid AlltoAll ===\n"); } // Clear receive buffer CUDACHECK(cudaMemset(d_recvbuff, 0, size_bytes)); // Launch hybrid AlltoAll kernel HybridAlltoAllKernel<<>>( send_win, 0, recv_win, 0, count, 0, devComm); // Wait for completion CUDACHECK(cudaStreamSynchronize(stream)); printf(" Rank %d completed hybrid AlltoAll kernel\n", my_rank); // ========================================================================== // STEP 6: Verify Results // ========================================================================== // Verify hybrid results CUDACHECK(cudaMemcpy(h_recvbuff, d_recvbuff, size_bytes, cudaMemcpyDeviceToHost)); bool hybrid_success = true; for (int src_rank = 0; src_rank < total_ranks; src_rank++) { for (size_t i = 0; i < count; i++) { size_t recv_idx = src_rank * count + i; float expected = (float)(src_rank * 1000 + my_rank * 100 + i); if (h_recvbuff[recv_idx] != expected) { hybrid_success = false; printf(" Rank %d: Hybrid mismatch at [%d][%zu]: got %.0f, expected %.0f\n", my_rank, src_rank, i, h_recvbuff[recv_idx], expected); break; } } if (!hybrid_success) break; } if (my_rank == 0) { printf("Hybrid AlltoAll result: %s\n", hybrid_success ? "PASSED" : "FAILED"); if (hybrid_success) { printf("✓ All %zu elements correctly exchanged using hybrid communication\n", total_elements); } } // ========================================================================== // STEP 7: Cleanup Resources // ========================================================================== // Cleanup host memory free(h_sendbuff); free(h_recvbuff); // Device API specific cleanup NCCLCHECK(ncclDevCommDestroy(comm, &devComm)); NCCLCHECK(ncclCommWindowDeregister(comm, send_win)); NCCLCHECK(ncclCommWindowDeregister(comm, recv_win)); NCCLCHECK(ncclMemFree(d_sendbuff)); NCCLCHECK(ncclMemFree(d_recvbuff)); // Standard NCCL cleanup NCCLCHECK(ncclCommFinalize(comm)); NCCLCHECK(ncclCommDestroy(comm)); CUDACHECK(cudaStreamDestroy(stream)); return NULL; } int main(int argc, char* argv[]) { // Run example using the provided utility framework return run_example(argc, argv, hybridAlltoAll); } nccl-2.29.7-1/docs/examples/06_device_api/Makefile000066400000000000000000000024761515037102200215000ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # NCCL Device API Examples EXAMPLES = 01_allreduce_lsa 02_alltoall_gin 03_alltoall_hybrid # Default target all: $(EXAMPLES) # Build individual examples $(EXAMPLES): $(MAKE) -C $@ # Clean all build artifacts clean: for example in $(EXAMPLES); do \ $(MAKE) -C $$example clean; \ done # Test all examples test: all for example in $(EXAMPLES); do \ echo "Testing $$example..."; \ $(MAKE) -C $$example test; \ done # Help help: @echo "NCCL Device API Examples" @echo "========================" @echo "" @echo "Targets:" @echo " all - Build all examples" @echo " clean - Clean all build artifacts" @echo " test - Test all examples" @echo " help - Show this help" @echo "" @echo "Examples:" @echo " 01_allreduce_lsa - AllReduce collective operation" @echo " 02_alltoall_gin - Pure GIN AlltoAll (network-only)" @echo " 03_alltoall_hybrid - Hybrid GIN+LSA AlltoAll (optimized)" @echo "" @echo "To build/run individual examples:" @echo " make -C 01_allreduce_lsa" @echo " make -C 02_alltoall_gin" @echo " make -C 03_alltoall_hybrid" .PHONY: all clean test help $(EXAMPLES) nccl-2.29.7-1/docs/examples/06_device_api/README.md000066400000000000000000000110451515037102200213070ustar00rootroot00000000000000 # NCCL Device API Examples ## Overview This directory contains minimal examples that demonstrate NCCL's device API, enabling users to perform inter-GPU communication within their own kernels. ## Examples ### [01_allreduce_lsa](01_allreduce_lsa/) **AllReduce with Device Kernel Implementation** - **Pattern**: GPU kernel performs collectives using device communicators - **API**: `ncclDevCommCreate`, `ncclCommWindowRegister`, device-side LSA barriers, `ncclAllReduce` - **Use case**: Allreduce operations with custom operations, fusing allreduce operation with previous/next compute operation. - **Key features**: - Device communicator creation with LSA barrier support - Symmetric memory windows for peer memory access - Device kernels coordinating via LSA barriers - Host launches kernel; kernel performs AllReduce on-device ### [02_alltoall_gin](02_alltoall_gin/) **Pure GIN AlltoAll - Network-Only Communication** - **Pattern**: GPU kernel performs AlltoAll using only GIN for all peers - **API**: `ncclDevCommCreate` with GIN support, `ncclGin`, GIN barriers and signals - **Use case**: Multi-node AlltoAll with consistent network-based communication - **Key features**: - Pure GIN implementation (no LSA optimizations) - Network barriers for cross-node synchronization - Signal-based completion detection - Baseline network performance measurements ### [03_alltoall_hybrid](03_alltoall_hybrid/) **Hybrid AlltoAll - Optimized Communication** - **Pattern**: GPU kernel performs AlltoAll using LSA for local peers, GIN for remote - **API**: `ncclDevCommCreate` with both LSA and GIN support, peer classification - **Use case**: Multi-node AlltoAll with optimal performance across topologies - **Key features**: - Hybrid implementation for optimal performance - Intelligent peer classification (local vs remote) - Combined LSA and GIN synchronization - Production-ready optimized communication patterns ## Choosing the Right Pattern *Scenario* : Custom kernels fusing computation and communication. *Addresses* : Schedule communication from inside a CUDA kernel. *Dependencies* : pthread or MPI ### Why the Device API? The device API allows NCCL communication within CUDA kernels, fusing communication and computation steps: ```cpp // Host: // 1) Create device communicator + requirements // 2) Register symmetric memory window for peer access ncclDevComm devComm; ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; reqs.lsaBarrierCount = NCCL_DEVICE_CTA_COUNT; NCCLCHECK(ncclDevCommCreate(comm, &reqs, &devComm)); NCCLCHECK(ncclCommWindowRegister(comm, buffer, size, &win, NCCL_WIN_COLL_SYMMETRIC)); // Device: // - Use barriers for cross-GPU synchronization // - Access peers via symmetric window (LSA pointers) myAllReduceKernel<<>>(win, devComm); ``` ### Checking for Device API and GIN Support To check for the required Device API and GIN support, the communicator properties are queried via `ncclCommQueryProperties`: ```cpp ncclCommProperties_t props = NCCL_COMM_PROPERTIES_INITIALIZER; NCCLCHECK(ncclCommQueryProperties(comm, &props)); // Check for Device API support if (!props.deviceApiSupport) { printf("ERROR: communicator does not support Device API!\n"); // Exit gracefully... } // Check for GIN support (ginType should not be NCCL_GIN_TYPE_NONE) if (props.ginType == NCCL_GIN_TYPE_NONE) { printf("ERROR: communicator does not support GIN!\n"); // Exit gracefully... } // For pure LSA examples, ensure a single team where all ranks can access each other if (props.nLsaTeams != 1) { printf("ERROR: expected 1 LSA team for pure LSA example!\n"); // Exit gracefully... } ``` ## Building ### **Quick Start** ```shell # Build example by directory name make 01_allreduce_lsa make 02_alltoall_gin make 03_alltoall_hybrid ``` ### **Individual Examples** ```shell # Build and run the device API AllReduce cd 01_allreduce_lsa && make ./allreduce_lsa # Build and run the Pure GIN AlltoAll example cd 02_alltoall_gin && make ./allreduce_gin # Build and run the Hybrid AlltoAll example cd 03_alltoall_hybrid && make ./allreduce_hybrid ``` ## References - [NCCL User Guide: Examples](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/examples.html) - [NCCL API Reference](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api.html) - [CUDA Programming Guide](https://docs.nvidia.com/cuda/cuda-c-programming-guide/) nccl-2.29.7-1/docs/examples/Makefile000066400000000000000000000025161515037102200170760ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # NCCL Examples Main Makefile # Define all category directories CATEGORIES := 01_communicators 02_point_to_point 03_collectives 04_user_buffer_registration 05_symmetric_memory 06_device_api # Default target all: $(CATEGORIES) # Build all categories $(CATEGORIES): @echo "Building $@..." @$(MAKE) -C $@ # Clean all categories clean: @echo "Cleaning all examples..." @for category in $(CATEGORIES); do \ $(MAKE) -C $$category clean; \ done # Test all examples test: all @echo "Testing all examples..." @for category in $(CATEGORIES); do \ $(MAKE) -C $$category test; \ done # Install all examples install: all @echo "Installing all examples..." @for category in $(CATEGORIES); do \ $(MAKE) -C $$category install; \ done # Help target help: @echo "NCCL Examples Main Makefile" @echo "===========================" @echo "" @echo "Available targets:" @echo " all - Build all examples" @echo " clean - Clean all build artifacts" @echo " test - Build and test all examples" @echo " install - Install all examples" @echo " help - Show this help message" @echo "" .PHONY: all clean test install help $(CATEGORIES) nccl-2.29.7-1/docs/examples/README.md000066400000000000000000000132151515037102200167130ustar00rootroot00000000000000 # NCCL Library Examples Welcome to the NCCL examples directory. This collection of NCCL (NVIDIA Collective Communications Library) examples is designed to teach developers how to effectively use NCCL in their applications. The examples progress from basic concepts to advanced usage patterns, with each example featuring a detailed README file. The APIs and features covered here are far from the complete set of what NCCL provides. The [NCCL Documentation](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/index.html) includes a detailed description of NCCL features and APIs. These examples showcase individual features but are not intended to maximize the performance for an individual communication pattern. For a performance implementation please refer to the [nccl-tests](https://github.com/NVIDIA/nccl-tests/) GitHub repository. ## Basic Examples We start with the most basic NCCL operations. All examples in this section are self-contained, meaning you can copy-paste one file and compile it on its own. The only dependencies are the NCCL library itself and, in the MPI case, an MPI implementation. These templates tend to new users coming up to speed with NCCL for GPU communication. ### [Communicators](01_communicators/) This section teaches you how to create, test, and destroy a communicator. We have provided 3 examples using a single thread, multiple threads, and multiple processes. This section shows the different options of launching an NCCL application. ### [Point 2 Point](02_point_to_point/) This sample send/recv implementation uses point to point communication to communicate in a simple ring pattern. ### [Collectives](03_collectives/) This sample implementation shows the most basic NCCL collective communication call. ## Advanced Features These examples are intended for experienced users looking for best practices to use a specific feature. For complete end-to-end templates please use the basic examples. Since NCCL does not include its own launcher, we have provided two popular bootstrap mechanisms. By default these examples will be launched as separate threads, one thread per GPU. Users can set `MPI=1` to build an MPI parallel version which can run across multiple compute nodes. Users can optionally provide a valid MPI installation under `MPI_HOME`. Each example can be run individually. By default you can run each executable via ``` [NTHREADS=] ./ ``` If `NTHREADS` is unset, the examples will use the number of visible GPUs as number threads. If the applications are built with `MPI` support, you can run each executable as ``` mpirun -np ./ ``` To ease the readability of these examples we have moved the bootstrap and broadcast part to the [common](common/) directory. Completely self-contained examples are provided in the sections above. ### [User Buffer Registration](04_user_buffer_registration/) User Buffer Registration eliminates the overhead of copying data between user buffers and internal NCCL buffers. This folder provides sample implementation using User Buffer Registration with common collectives. ### [Symmetric Memory Registration](05_symmetric_memory/) Since 2.27, NCCL supports window registration, which allows users to register local buffers into NCCL window and enables extreme low latency and high bandwidth communication in NCCL. This folder provides sample implementation using Symmetric Memory Registration with common collectives. ### [Device APIs](06_device_api/) Device API enables GPU kernels to directly perform inter-GPU communication. This enables applications to perform communication from within CUDA kernels and fuse computation and communication, and fine-grained control over collective implementation. This folder demonstrates how to implement collectives using device-side kernels. ## Prerequisites - The same prerequisites as building NCCL from source. - Users can optionally add `MPI_HOME` for an MPI library in a non-standard location. ## Build Steps The examples can be built while building the NCCL library from source. Users can choose to build the examples with MPI support (`MPI=1`). ``` git clone https://github.com/NVIDIA/nccl.git cd nccl make -j examples [MPI=1] ``` or, if NCCL has already been built, the user can optionally add a non-standard NCCL installation location: ``` cd docs/examples make NCCL_HOME= [MPI=1] ``` ## Environment Variables ### Build Stage Users can use these optional variables to choose which libraries are used to build these examples: - `NCCL_HOME=`: Local base directory of a NCCL installation. - `MPI` : [0,1] Build the examples with MPI support. - `MPI_HOME=` : Local base directory of a MPI installation. - `CUDA_HOME=` : Local base directory of a CUDA installation. ### Run Stage - `NTHREADS=`: Number of threads to create for the threaded examples. Defaults to number of visible GPUs. - `CUDA_VISIBLE_DEVICES`: Comma delimited list of GPUs visible to the application. - All other NCCL [environment variables](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html) apply. ## Supported OS Linux ## Troubleshooting Each example includes a /Common Issues and Solutions/ section for the individual tests. For general runtime issues use the debug output enabled by setting `NCCL_DEBUG=INFO` for detailed logging. The [Troubleshooting](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/troubleshooting.html) section of the NCCL documentation also includes many helpful tips. nccl-2.29.7-1/docs/examples/common/000077500000000000000000000000001515037102200167225ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/common/README.md000066400000000000000000000017741515037102200202120ustar00rootroot00000000000000# NCCL Common Utilities ## Description This directory contains shared utilities and helper functions used across all NCCL examples. These utilities provide common functionality for error handling, device management, and MPI integration. ## Components ### Headers (`include/`) - **utils.h**: General utility functions - **nccl_utils.h**: NCCL error checking macros - **mpi_utils.h**: MPI error checking macros ### Source Files (`src/`) - **utils.cc**: General utility functions ## Key Features ### Error Checking Macros ```c #define NCCLCHECK(cmd) // NCCL error checking #define CUDACHECK(cmd) // CUDA error checking #define MPICHECK(cmd) // MPI error checking ``` ## Usage in Examples Include the headers in your example source files: ```c #include "utils.h" #include "mpi_utils.h" ``` ## Notes - All utilities include comprehensive error checking - Functions are designed to be thread-safe - Memory management functions handle null pointers safely - MPI utilities are only needed for multi-process examples nccl-2.29.7-1/docs/examples/common/include/000077500000000000000000000000001515037102200203455ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/common/include/mpi_utils.h000066400000000000000000000026751515037102200225350ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef MPI_UTILS_H_ #define MPI_UTILS_H_ #include "mpi.h" #include #include // MPI error checking macro #define MPICHECK(cmd) \ do { \ int err = cmd; \ if (err != MPI_SUCCESS) { \ char error_string[MPI_MAX_ERROR_STRING]; \ int length; \ MPI_Error_string(err, error_string, &length); \ fprintf(stderr, "MPI error at %s:%d - %s\n", __FILE__, __LINE__, \ error_string); \ fprintf(stderr, "Failed MPI operation: %s\n", #cmd); \ MPI_Abort(MPI_COMM_WORLD, err); \ } \ } while (0) #endif nccl-2.29.7-1/docs/examples/common/include/nccl_utils.h000066400000000000000000000037361515037102200226660ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_UTILS_H_ #define NCCL_UTILS_H_ #include #include #include #include #include "cuda_runtime.h" // Error checking #define NCCLCHECK(cmd) \ do { \ ncclResult_t res = cmd; \ if (res != ncclSuccess) { \ fprintf(stderr, "Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \ ncclGetErrorString(res)); \ fprintf(stderr, "Failed NCCL operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) #define CUDACHECK(cmd) \ do { \ cudaError_t err = cmd; \ if (err != cudaSuccess) { \ fprintf(stderr, "Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \ cudaGetErrorString(err)); \ fprintf(stderr, "Failed CUDA operation: %s\n", #cmd); \ exit(EXIT_FAILURE); \ } \ } while (0) #endif nccl-2.29.7-1/docs/examples/common/include/utils.h000066400000000000000000000031561515037102200216630ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef UTILS_H_ #define UTILS_H_ #include "cuda_runtime.h" #include "nccl.h" #include "nccl_utils.h" #include #include #ifdef MPI_SUPPORT #include "mpi.h" #include "mpi_utils.h" #else #include #include #endif /** * Broadcast NCCL unique ID * * Broadcasts the NCCL unique ID from the root rank to all other ranks. * Uses MPI_Bcast in MPI mode and pthread barrier in pthread mode. * * @param root Root rank that holds the NCCL unique ID * @param my_rank Current rank or thread id * @param arg Pointer to NCCL unique ID to broadcast * * @return 0 on success, non-zero on error */ int util_broadcast(int root, int my_rank, ncclUniqueId *arg); /** * Run the given NCCL example in parallel * * This function performs the complete NCCL example lifecycle: * 1. Initialize backend (MPI or pthread) * 2. Execute NCCL communicator setup function * 3. Cleanup of all resources * * @param argc Command line argument count * @param argv Command line arguments * @param ncclExample Function pointer to example-specific NCCL setup * * @return 0 on success, non-zero on error */ int run_example(int argc, char *argv[], void *(*ncclExample)(int, int, int, int)); #endif // UTILS_H_ nccl-2.29.7-1/docs/examples/common/src/000077500000000000000000000000001515037102200175115ustar00rootroot00000000000000nccl-2.29.7-1/docs/examples/common/src/utils.cc000066400000000000000000000232131515037102200211610ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "utils.h" #include #ifndef MPI_SUPPORT pthread_barrier_t barrier; ncclUniqueId nccl_unique_id; #endif /** * Common context structure for both MPI and pthread examples * * This structure provides a unified interface for NCCL examples that can * run in either MPI mode (one process per device) or pthread mode * (one thread per device). */ typedef struct { // Common variables int total_ranks; // Total number of MPI ranks or pthreads int devices_per_rank; // Number of devices per rank or thread int local_device; // Node local rank or thread id (0 to total_ranks-1) int my_rank; // Rank or thread id for NCCL ncclUniqueId nccl_id; // NCCL unique ID ncclComm_t comm; // NCCL communicator (single for all modes) ncclUniqueId *nccl_unique_id; // NCCL unique ID pointer void *func; // pthread-specific variables #ifndef MPI_SUPPORT pthread_t *threads; // Thread array int *thread_ranks; // Thread rank array #endif } context_t; /** * Initialize MPI or pthread backend * * Sets up the backend and populates common context variables. * * MPI Mode (compiled with MPI=1): * - Initializes MPI (rank, size) * - Calculates local rank based on splitting communicator by node multi-node * support * - Generates and broadcasts NCCL unique ID * - Sets device assignment based on local rank * * pthread Mode (default): * - Gets thread count from NTHREADS environment or GPU count * - Validates thread count against available GPUs * - Generates NCCL unique ID for sharing across threads * - Allocates thread management resources * * @param argc Command line argument count * @param argv Command line arguments * @param ctx Output: Populated example context * * @return 0 on success, non-zero on error */ int initialize(int argc, char *argv[], context_t *ctx); /** * Wrap function to call the example function in a thread * * Note: This function is needed since pthread only allows a single void* * argument. */ void *thread_wrapper(void *arg); /** * Run ncclExample in parallel using MPI or pthreads * * Starts the execution of the given NCCL example function in parallel * * MPI Mode: * - Starts the function on each rank * - Checks the output and calls MPI_Barrier to synchronize * * pthread Mode: * - Creates threads (one per device) * - Each thread runs the given example function * - Waits for all threads to complete * * @param ctx Context with backend setup completed * @param ncclExample Function pointer to example-specific NCCL setup * * @return 0 on success, non-zero on error * * Note: This function expects ncclExample() to be defined by the example. * The ncclExample function should have signature: * void* ncclExample(int, int, int, int) */ int run_parallel(context_t *ctx, void *(*ncclExample)(int, int, int, int)); /** * Clean up resources * * Properly cleans up all resources allocated during initialization. * Note: NCCL communicators are destroyed by ncclCommSetup function. * * MPI Mode: * - Finalizes MPI * * pthread Mode: * - Frees thread arrays * * @param ctx Context to clean up */ void cleanup(context_t *ctx); /** * Broadcast NCCL unique ID */ int util_broadcast(int root, int my_rank, ncclUniqueId *arg) { #ifdef MPI_SUPPORT MPICHECK( MPI_Bcast(arg, sizeof(ncclUniqueId), MPI_BYTE, root, MPI_COMM_WORLD)); #else if (my_rank == root) { nccl_unique_id = *arg; } int barrier_err = pthread_barrier_wait(&barrier); if (barrier_err != 0 && barrier_err != PTHREAD_BARRIER_SERIAL_THREAD) { fprintf(stderr, "pthread_barrier_wait failed at %s:%d with error code %d\n", __FILE__, __LINE__, barrier_err); abort(); } if (my_rank != root) { *arg = nccl_unique_id; } #endif return 0; } /** * Initialize MPI or pthread backend */ int initialize(int argc, char *argv[], context_t *ctx) { #ifdef MPI_SUPPORT // Initialize MPI MPICHECK(MPI_Init(&argc, &argv)); MPICHECK(MPI_Comm_rank(MPI_COMM_WORLD, &ctx->my_rank)); MPICHECK(MPI_Comm_size(MPI_COMM_WORLD, &ctx->total_ranks)); if (ctx->my_rank == 0) { printf("Number of processes: %d\n", ctx->total_ranks); } // Only for printing the output in order MPI_Barrier(MPI_COMM_WORLD); printf("MPI initialized: rank %d of %d\n", ctx->my_rank, ctx->total_ranks); // Split the communicator based on shared memory (i.e., nodes) MPI_Comm node_comm; MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, ctx->my_rank, MPI_INFO_NULL, &node_comm); // Get the rank within the node communicator MPI_Comm_rank(node_comm, &ctx->local_device); // Clean up the node communicator MPI_Comm_free(&node_comm); #else // Get number of devices (threads) from environment or default to available // GPUs int num_gpus = 0; CUDACHECK(cudaGetDeviceCount(&num_gpus)); ctx->total_ranks = num_gpus; // Default to all available GPUs const char *nThreadsEnv = getenv("NTHREADS"); if (nThreadsEnv) { ctx->total_ranks = atoi(nThreadsEnv); } printf("Creating %d threads for %d devices\n", ctx->total_ranks, num_gpus); if (ctx->total_ranks < 1) { printf("Invalid number of threads: %d\n", ctx->total_ranks); return 1; } // Check if we have enough GPUs if (ctx->total_ranks > num_gpus) { printf("Error: Requested %d threads but only %d GPUs available\n", ctx->total_ranks, num_gpus); printf("Please reduce NTHREADS to %d or fewer\n", num_gpus); return 1; } // Thread synchronization needed for unique ID sharing later on pthread_barrier_init(&barrier, NULL, ctx->total_ranks); // Generate NCCL unique ID (shared across all threads) NCCLCHECK(ncclGetUniqueId(&ctx->nccl_id)); // Allocate thread resources ctx->threads = (pthread_t *)malloc(ctx->total_ranks * sizeof(pthread_t)); ctx->thread_ranks = (int *)malloc(ctx->total_ranks * sizeof(int)); if (ctx->threads == NULL || ctx->thread_ranks == NULL) { printf("Failed to allocate memory for threads\n"); return 1; } #endif return 0; } /** * Wrap function to call the example function in a thread * * Note: This function is needed since pthread only allows a single void* * argument. */ void *thread_wrapper(void *arg) { context_t *ctx = (context_t *)arg; void *(*example_func)(int, int, int, int) = (void *(*)(int, int, int, int))ctx->func; return example_func(ctx->my_rank, ctx->total_ranks, ctx->local_device, ctx->devices_per_rank); } /** * Run ncclExample in parallel using MPI or pthreads */ int run_parallel(context_t *ctx, void *(*ncclExample)(int, int, int, int)) { #ifdef MPI_SUPPORT if (ctx->my_rank == 0) { printf("NCCL Example: One Device per Process\n"); printf("====================================\n"); } if (ncclExample(ctx->my_rank, ctx->total_ranks, ctx->local_device, ctx->devices_per_rank) != NULL) return 1; // Synchronize to ensure ordered output MPICHECK(MPI_Barrier(MPI_COMM_WORLD)); #else printf("NCCL Example: One Device per Thread\n"); printf("===================================\n"); // Create separate context for each thread context_t *thread_contexts = (context_t *)malloc(ctx->total_ranks * sizeof(context_t)); if (thread_contexts == NULL) { printf("Failed to allocate thread contexts\n"); return 1; } ncclUniqueId *nccl_unique_id = (ncclUniqueId *)calloc(1, sizeof(ncclUniqueId)); for (int i = 0; i < ctx->total_ranks; i++) { // Copy main context to thread context memcpy(&thread_contexts[i], ctx, sizeof(context_t)); thread_contexts[i].threads = NULL; thread_contexts[i].thread_ranks = NULL; thread_contexts[i].my_rank = i; // Set NCCL rank to thread id thread_contexts[i].local_device = i; thread_contexts[i].total_ranks = ctx->total_ranks; thread_contexts[i].devices_per_rank = 1; thread_contexts[i].func = (void *)ncclExample; thread_contexts[i].nccl_unique_id = nccl_unique_id; ctx->thread_ranks[i] = i; pthread_create(&ctx->threads[i], NULL, thread_wrapper, &thread_contexts[i]); } // Wait for all threads to complete for (int i = 0; i < ctx->total_ranks; i++) { pthread_join(ctx->threads[i], NULL); } free(thread_contexts); #endif return 0; } /** * Run the given NCCL example in parallel */ int run_example(int argc, char *argv[], void *(*ncclExample)(int, int, int, int)) { // 1. Allocate context context_t *ctx = (context_t *)calloc(1, sizeof(context_t)); if (ctx == NULL) { printf("Failed to allocate memory for context\n"); return 1; } // 2. Initialize backend (MPI or pthread) if (initialize(argc, argv, ctx) != 0) { printf("Failed to initialize backend\n"); return 1; } // 3. Start the given example code in parallel if (run_parallel(ctx, ncclExample) != 0) { printf("Failed to execute NCCL operations\n"); cleanup(ctx); // Cleanup on failure return 1; } // 3. Cleanup cleanup(ctx); // 4. Print common success message #ifdef MPI_SUPPORT if (ctx->my_rank == 0) { #endif printf("\nAll NCCL communicators finalized successfully!\n"); #ifdef MPI_SUPPORT } #endif return 0; } /** * Clean up resources */ void cleanup(context_t *ctx) { #ifdef MPI_SUPPORT // Free MPI resources MPICHECK(MPI_Finalize()); #else free(ctx->threads); free(ctx->thread_ranks); pthread_barrier_destroy(&barrier); #endif } nccl-2.29.7-1/makefiles/000077500000000000000000000000001515037102200146245ustar00rootroot00000000000000nccl-2.29.7-1/makefiles/common.mk000066400000000000000000000143701515037102200164520ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # CUDA_HOME ?= /usr/local/cuda PREFIX ?= /usr/local VERBOSE ?= 0 KEEP ?= 0 DEBUG ?= 0 ASAN ?= 0 UBSAN ?= 0 TRACE ?= 0 WERROR ?= 0 PROFAPI ?= 1 NVTX ?= 1 RDMA_CORE ?= 0 NET_PROFILER ?= 0 MLX5DV ?= 0 MAX_EXT_NET_PLUGINS ?= 0 EMIT_LLVM_IR ?= 0 NVCC ?= $(CUDA_HOME)/bin/nvcc CUDA_LIB ?= $(CUDA_HOME)/lib64 CUDA_INC ?= $(CUDA_HOME)/include CUDA_VERSION = $(strip $(shell which $(NVCC) >/dev/null && $(NVCC) --version | grep release | sed 's/.*release //' | sed 's/\,.*//')) #CUDA_VERSION ?= $(shell ls $(CUDA_LIB)/libcudart.so.* | head -1 | rev | cut -d "." -f -2 | rev) CUDA_MAJOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 1) CUDA_MINOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 2) #$(info CUDA_VERSION ${CUDA_MAJOR}.${CUDA_MINOR}) # You should define NVCC_GENCODE in your environment to the minimal set # of archs to reduce compile time. CUDA8_GENCODE = -gencode=arch=compute_60,code=sm_60 \ -gencode=arch=compute_61,code=sm_61 CUDA9_GENCODE = -gencode=arch=compute_70,code=sm_70 CUDA10_GENCODE = -gencode=arch=compute_75,code=sm_75 CUDA11_GENCODE = -gencode=arch=compute_80,code=sm_80 CUDA12_GENCODE = -gencode=arch=compute_90,code=sm_90 CUDA12_8_GENCODE = -gencode=arch=compute_100,code=sm_100 \ -gencode=arch=compute_120,code=sm_120 CUDA13_GENCODE = -gencode=arch=compute_110,code=sm_110 CUDA8_PTX = -gencode=arch=compute_61,code=compute_61 CUDA9_PTX = -gencode=arch=compute_70,code=compute_70 CUDA11_PTX = -gencode=arch=compute_80,code=compute_80 CUDA12_PTX = -gencode=arch=compute_90,code=compute_90 CUDA13_PTX = -gencode=arch=compute_120,code=compute_120 ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 13; echo $$?),0) # Prior to SM75 is deprecated from CUDA13.0 onwards NVCC_GENCODE ?= $(CUDA10_GENCODE) $(CUDA11_GENCODE) $(CUDA12_GENCODE) $(CUDA12_8_GENCODE) $(CUDA13_GENCODE) $(CUDA13_PTX) else ifeq ($(shell test "0$(CUDA_MAJOR)" -eq 12 -a "0$(CUDA_MINOR)" -ge 8; echo $$?),0) # Include Blackwell support if we're using CUDA12.8 or above NVCC_GENCODE ?= $(CUDA8_GENCODE) $(CUDA9_GENCODE) $(CUDA11_GENCODE) $(CUDA12_GENCODE) $(CUDA12_8_GENCODE) $(CUDA13_PTX) else ifeq ($(shell test "0$(CUDA_MAJOR)" -eq 11 -a "0$(CUDA_MINOR)" -ge 8 -o "0$(CUDA_MAJOR)" -gt 11; echo $$?),0) # Include Hopper support if we're using CUDA11.8 or above NVCC_GENCODE ?= $(CUDA8_GENCODE) $(CUDA9_GENCODE) $(CUDA11_GENCODE) $(CUDA12_GENCODE) $(CUDA12_PTX) else ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 11; echo $$?),0) NVCC_GENCODE ?= $(CUDA8_GENCODE) $(CUDA9_GENCODE) $(CUDA11_GENCODE) $(CUDA11_PTX) # Include Volta support if we're using CUDA9 or above else ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 9; echo $$?),0) NVCC_GENCODE ?= $(CUDA8_GENCODE) $(CUDA9_GENCODE) $(CUDA9_PTX) else NVCC_GENCODE ?= $(CUDA8_GENCODE) $(CUDA8_PTX) endif $(info NVCC_GENCODE is ${NVCC_GENCODE}) # CUDA 13.0 requires c++17 ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 13; echo $$?),0) CXXSTD ?= -std=c++17 else CXXSTD ?= -std=c++14 endif CXXFLAGS := -DCUDA_MAJOR=$(CUDA_MAJOR) -DCUDA_MINOR=$(CUDA_MINOR) -fPIC -fvisibility=hidden \ -Wall -Wno-unused-function -Wno-sign-compare $(CXXSTD) -Wvla \ -I $(CUDA_INC) -I $(CUDA_INC)/cccl \ $(CXXFLAGS) # Maxrregcount needs to be set accordingly to NCCL_MAX_NTHREADS (otherwise it will cause kernel launch errors) # 512 : 120, 640 : 96, 768 : 80, 1024 : 60 # We would not have to set this if we used __launch_bounds__, but this only works on kernels, not on functions. NVCUFLAGS := -ccbin $(CXX) $(NVCC_GENCODE) $(CXXSTD) --expt-extended-lambda -Xptxas -maxrregcount=96 -Xfatbin -compress-all # Use addprefix so that we can specify more than one path NVLDFLAGS := -L${CUDA_LIB} -lcudart -lrt NVCUFLAGS_SYM := ########## GCOV ########## GCOV ?= 0 # disable by default. GCOV_FLAGS := $(if $(filter 0,${GCOV} ${DEBUG}),,--coverage) # only gcov=1 and debug =1 CXXFLAGS += ${GCOV_FLAGS} NVCUFLAGS += ${GCOV_FLAGS:%=-Xcompiler %} LDFLAGS += ${GCOV_FLAGS} NVLDFLAGS += ${GCOV_FLAGS:%=-Xcompiler %} # $(warning GCOV_FLAGS=${GCOV_FLAGS}) ########## GCOV ########## ifeq ($(DEBUG), 0) NVCUFLAGS += -O3 CXXFLAGS += -O3 -g else NVCUFLAGS += -O0 -G -g CXXFLAGS += -O0 -g -ggdb3 endif # Make sure to run with ASAN_OPTIONS=protect_shadow_gap=0 otherwise CUDA will fail with OOM ifneq ($(ASAN), 0) CXXFLAGS += -fsanitize=address LDFLAGS += -fsanitize=address -static-libasan NVLDFLAGS += -Xcompiler -fsanitize=address,-static-libasan endif ifneq ($(UBSAN), 0) CXXFLAGS += -fsanitize=undefined LDFLAGS += -fsanitize=undefined -static-libubsan NVLDFLAGS += -Xcompiler -fsanitize=undefined,-static-libubsan endif ifneq ($(VERBOSE), 0) NVCUFLAGS += -Xptxas -v -Xcompiler -Wall,-Wextra,-Wno-unused-parameter CXXFLAGS += -Wall -Wextra else .SILENT: endif ifneq ($(TRACE), 0) CXXFLAGS += -DENABLE_TRACE endif ifeq ($(NVTX), 0) CXXFLAGS += -DNVTX_DISABLE endif ifneq ($(WERROR), 0) CXXFLAGS += -Werror endif ifneq ($(KEEP), 0) NVCUFLAGS += -keep endif ifneq ($(PROFAPI), 0) CXXFLAGS += -DPROFAPI endif ifneq ($(RDMA_CORE), 0) CXXFLAGS += -DNCCL_BUILD_RDMA_CORE=1 -libverbs endif ifneq ($(MLX5DV), 0) CXXFLAGS += -DNCCL_BUILD_MLX5DV=1 -lmlx5 endif ifneq ($(NET_PROFILER), 0) CXXFLAGS += -DNCCL_ENABLE_NET_PROFILING=1 endif ifneq ($(MAX_EXT_NET_PLUGINS), 0) CXXFLAGS += -DNCCL_NET_MAX_PLUGINS=$(MAX_EXT_NET_PLUGINS) endif CXXFLAGS += -DDOCA_VERBS_USE_CUDA_WRAPPER -DDOCA_VERBS_USE_NET_WRAPPER NVCUFLAGS += -DDOCA_VERBS_USE_CUDA_WRAPPER -DDOCA_VERBS_USE_NET_WRAPPER CXXFLAGS += -DNCCL_GIN_PROXY_ENABLE=1 # Detect OS Linux or Windows ifeq ($(shell uname -s), Linux) NCCL_OS_LINUX := 1 CXXFLAGS += -DNCCL_OS_LINUX NVCUFLAGS += -DNCCL_OS_LINUX else ifeq ($(shell uname -s), Windows) NCCL_OS_WINDOWS := 1 CXXFLAGS += -DNCCL_OS_WINDOWS NVCUFLAGS += -DNCCL_OS_WINDOWS endif # Check and enable LLVM IR generation ifneq ($(EMIT_LLVM_IR), 0) CXXFLAGS += -DEMIT_LLVM_IR=1 endif # Git version overrides (set via command line: make NCCL_GIT_BRANCH=xxx NCCL_GIT_COMMIT_HASH=yyy) ifneq ($(NCCL_GIT_BRANCH),) CXXFLAGS += -DNCCL_GIT_BRANCH='"$(NCCL_GIT_BRANCH)"' endif ifneq ($(NCCL_GIT_COMMIT_HASH),) CXXFLAGS += -DNCCL_GIT_COMMIT_HASH='"$(NCCL_GIT_COMMIT_HASH)"' endif nccl-2.29.7-1/makefiles/examples.mk000066400000000000000000000013451515037102200167760ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Make sure NCCL headers are found and libraries are linked ifneq ($(NCCL_HOME), "") NVCUFLAGS += -I$(NCCL_HOME)/include/ NVLDFLAGS += -L$(NCCL_HOME)/lib endif # Build configuration INCLUDES = -I$(CUDA_HOME)/include -I$(NCCL_HOME)/include LIBRARIES = -L$(CUDA_HOME)/lib64 -L$(NCCL_HOME)/lib LDFLAGS = -lcudart -lnccl -Wl,-rpath,$(NCCL_HOME)/lib # MPI configuration ifeq ($(MPI), 1) ifdef MPI_HOME MPICXX ?= $(MPI_HOME)/bin/mpicxx MPIRUN ?= $(MPI_HOME)/bin/mpirun else MPICXX ?= mpicxx MPIRUN ?= mpirun endif CXXFLAGS += -DMPI_SUPPORT endif nccl-2.29.7-1/makefiles/formatting.mk000066400000000000000000000023621515037102200173320ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # Prerequisite: $(FILESTOFORMAT) contains the list of files of interest for formatting # As this file defines a new target (format), it should be included at least after the definition of the # default target. ASTYLE_FORMAT_OPTS=-Qv --style=java --indent-after-parens --indent-modifiers --indent-switches --indent-continuation=2 --keep-one-line-blocks --keep-one-line-statements --indent=spaces=2 --lineend=linux --suffix=none ASTYLEDIR := $(BUILDDIR)/contrib ASTYLETAR := $(ASTYLEDIR)/astyle.tar.gz ASTYLEBIN := $(ASTYLEDIR)/astyle/build/gcc/bin/astyle ASTYLEBLD := $(ASTYLEDIR)/astyle/build/gcc/ ASTYLEVER := 3.1 ASTYLEURL := "https://versaweb.dl.sourceforge.net/project/astyle/astyle/astyle%20$(ASTYLEVER)/astyle_$(ASTYLEVER)_linux.tar.gz" $(ASTYLEDIR) : @mkdir -p $(ASTYLEDIR) $(ASTYLETAR) : $(ASTYLEDIR) @wget -q -O $(ASTYLETAR) $(ASTYLEURL) $(ASTYLEBLD) : $(ASTYLETAR) @cd $(ASTYLEDIR) && tar xzf $(ASTYLETAR) $(ASTYLEBIN) : $(ASTYLEBLD) ${MAKE} -C $(ASTYLEBLD) .PHONY : format format : $(ASTYLEBIN) @$(ASTYLEBIN) $(ASTYLE_FORMAT_OPTS) $(FILESTOFORMAT) nccl-2.29.7-1/makefiles/version.mk000066400000000000000000000004531515037102200166440ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # ##### version NCCL_MAJOR := 2 NCCL_MINOR := 29 NCCL_PATCH := 7 NCCL_SUFFIX := PKG_REVISION := 1 nccl-2.29.7-1/pkg/000077500000000000000000000000001515037102200134455ustar00rootroot00000000000000nccl-2.29.7-1/pkg/Makefile000066400000000000000000000011351515037102200151050ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # .PHONY : all clean default : build build : debian.build txz.build BUILDDIR ?= $(abspath ../build) ABSBUILDDIR := $(abspath $(BUILDDIR)) TARGETS := debian txz all: ${TARGETS:%=%.build} prep: ${TARGETS:%=%.prep} build: ${TARGETS:%=%.build} clean: ${TARGETS:%=%.clean} %.prep: ${MAKE} -C $* prep BUILDDIR=${ABSBUILDDIR} %.build: ${MAKE} -C $* build BUILDDIR=${ABSBUILDDIR} %.clean: ${MAKE} -C $* clean nccl-2.29.7-1/pkg/debian/000077500000000000000000000000001515037102200146675ustar00rootroot00000000000000nccl-2.29.7-1/pkg/debian/.gitignore000066400000000000000000000001211515037102200166510ustar00rootroot00000000000000/*.debhelper.log /*.debhelper /*.substvars /tmp/ /files /libnccl1/ /libnccl-dev/ nccl-2.29.7-1/pkg/debian/CMakeLists.txt000066400000000000000000000056541515037102200174410ustar00rootroot00000000000000# Debian package building for NCCL find_program(DEBUILD_EXECUTABLE debuild) find_program(SED_EXECUTABLE sed) # Set package directories set(DEBPREPDIR "${CMAKE_BINARY_DIR}/debian") set(PKGDIR "${CMAKE_BINARY_DIR}/pkg/deb") set(DEBGENFILES libnccl-dev.install libnccl2.install control changelog) set(DEBFILES compat copyright rules $(DEBGENFILES)) # Get current timestamp in RFC 2822 format execute_process(COMMAND date -R OUTPUT_VARIABLE PKG_TIMESTAMP OUTPUT_STRIP_TRAILING_WHITESPACE) # Get Debian multiarch tuple execute_process(COMMAND dpkg-architecture -qDEB_HOST_MULTIARCH OUTPUT_VARIABLE PKG_MULTIARCH OUTPUT_STRIP_TRAILING_WHITESPACE) # Get Debian architecture execute_process(COMMAND dpkg-architecture -qDEB_HOST_ARCH OUTPUT_VARIABLE PKG_ARCH OUTPUT_STRIP_TRAILING_WHITESPACE) # Custom target for debian package preparation add_custom_target(debian_prep ALL # Create debian prep directory COMMAND ${CMAKE_COMMAND} -E make_directory ${DEBPREPDIR} COMMENT "Preparing Debian package files" ) # for loop to generate the files foreach(DEBGENFILE ${DEBGENFILES}) add_custom_command( TARGET debian_prep COMMAND ${SED_EXECUTABLE} -e 's/\\$$\{nccl:Major\}/${NCCL_MAJOR}/g' -e 's/\\$$\{nccl:Minor\}/${NCCL_MINOR}/g' -e 's/\\$$\{nccl:Patch\}/${NCCL_PATCH}/g' -e 's/\\$$\{nccl:Suffix\}/${NCCL_SUFFIX}/g' -e 's/\\$$\{pkg:Revision\}/${PKG_REVISION}/g' -e 's/\\$$\{cuda:Major\}/${CUDA_MAJOR}/g' -e 's/\\$$\{cuda:Minor\}/${CUDA_MINOR}/g' -e 's/\\$$\{pkg:Timestamp\}/${PKG_TIMESTAMP}/g' -e 's/\\$$\{pkg:MultiArch\}/${PKG_MULTIARCH}/g' -e 's/\\$$\{pkg:Arch\}/${PKG_ARCH}/g' ${CMAKE_CURRENT_SOURCE_DIR}/${DEBGENFILE}.in > ${DEBPREPDIR}/${DEBGENFILE} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${DEBGENFILE}.in ) endforeach() foreach(DEBFILE ${DEBFILES}) add_custom_command(TARGET debian_prep COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/${DEBFILE} ${DEBPREPDIR}/${DEBFILE} DEPENDS ${DEBFILE}) endforeach() # Build the package add_custom_target(debian_build COMMAND cd ${CMAKE_BINARY_DIR} && ${DEBUILD_EXECUTABLE} -eLD_LIBRARY_PATH -uc -us -d -b -Zxz COMMAND ${CMAKE_COMMAND} -E make_directory ${PKGDIR} COMMAND mv ${CMAKE_SOURCE_DIR}/libnccl*.deb ${PKGDIR} WORKING_DIRECTORY ${DEBPREPDIR} DEPENDS debian_prep COMMENT "Building Debian package with debuild" ) # Custom target for cleaning debian package add_custom_target(debian_clean COMMAND ${CMAKE_COMMAND} -E remove_directory ${DEBPREPDIR} COMMAND ${CMAKE_COMMAND} -E remove_directory ${PKGDIR} COMMENT "Cleaning Debian package build files" ) nccl-2.29.7-1/pkg/debian/Makefile000066400000000000000000000032741515037102200163350ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # include ../../makefiles/common.mk include ../../makefiles/version.mk BUILDDIR ?= $(abspath ../../build) DEBPREPDIR := $(BUILDDIR)/debian PKGDIR := $(BUILDDIR)/pkg/deb/ DEBGEN_IN := $(wildcard *.in) DEBGEN := $(DEBGEN_IN:.in=) DEBFILES := compat copyright libnccl-dev.install rules $(DEBGEN) DEBTARGETS := $(patsubst %, $(DEBPREPDIR)/%, $(DEBFILES)) PKG_TIMESTAMP := $(shell date -R) PKG_ARCH ?= $(shell dpkg-architecture -qDEB_HOST_ARCH) PKG_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) prep : $(DEBTARGETS) $(MAKE) -C ../.. lic BUILDDIR=$(BUILDDIR) build : prep $(MAKE) -C ../.. src.build BUILDDIR=$(BUILDDIR) @printf "Building Debian package\n" (cd $(BUILDDIR); debuild -eLD_LIBRARY_PATH -uc -us -d -b -Zxz) mkdir -p $(PKGDIR) mv $(BUILDDIR)/../libnccl*.deb $(PKGDIR)/ clean: rm -Rf $(DEBPREPDIR) $(PKGDIR) $(DEBPREPDIR)/% : %.in @printf "Generating %-35s > %s\n" $< $@ mkdir -p $(DEBPREPDIR) sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \ -e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \ -e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \ -e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \ -e "s/\$${cuda:Major}/$(CUDA_MAJOR)/g" \ -e "s/\$${cuda:Minor}/$(CUDA_MINOR)/g" \ -e "s/\$${pkg:Revision}/$(PKG_REVISION)/g" \ -e "s/\$${pkg:Timestamp}/$(PKG_TIMESTAMP)/g" \ -e "s/\$${pkg:Arch}/$(PKG_ARCH)/g" \ -e "s/\$${pkg:MultiArch}/$(PKG_MULTIARCH)/g" \ $< > $@ $(DEBPREPDIR)/% : % @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(DEBPREPDIR) cp -f $< $@ nccl-2.29.7-1/pkg/debian/changelog.in000066400000000000000000000003471515037102200171520ustar00rootroot00000000000000nccl (${nccl:Major}.${nccl:Minor}.${nccl:Patch}${nccl:Suffix}-${pkg:Revision}+cuda${cuda:Major}.${cuda:Minor}) trusty; urgency=medium * Automatic Debian package from build -- cudatools ${pkg:Timestamp} nccl-2.29.7-1/pkg/debian/compat000066400000000000000000000000021515037102200160650ustar00rootroot000000000000009 nccl-2.29.7-1/pkg/debian/control.in000066400000000000000000000023511515037102200167000ustar00rootroot00000000000000Source: nccl Section: libs Maintainer: cudatools Priority: optional Build-depends: debhelper(>=9) Standards-Version: 3.9.5 Package: libnccl${nccl:Major} Section: libs Architecture: ${pkg:Arch} Depends: ${misc:Depends}, ${shlibs:Depends} Description: NVIDIA Collective Communication Library (NCCL) Runtime NCCL (pronounced "Nickel") is a stand-alone library of standard collective communication routines for GPUs, implementing all-reduce, all-gather, reduce, broadcast, and reduce-scatter. It has been optimized to achieve high bandwidth on any platform using PCIe, NVLink, NVswitch, as well as networking using InfiniBand Verbs or TCP/IP sockets. Package: libnccl-dev Section: libdevel Architecture: ${pkg:Arch} Depends: ${misc:Depends}, ${shlibs:Depends}, libnccl${nccl:Major} (= ${binary:Version}) Description: NVIDIA Collective Communication Library (NCCL) Development Files NCCL (pronounced "Nickel") is a stand-alone library of standard collective communication routines for GPUs, implementing all-reduce, all-gather, reduce, broadcast, and reduce-scatter. It has been optimized to achieve high bandwidth on any platform using PCIe, NVLink, NVswitch, as well as networking using InfiniBand Verbs or TCP/IP sockets. nccl-2.29.7-1/pkg/debian/copyright000077700000000000000000000000001515037102200210662../../LICENSE.txtustar00rootroot00000000000000nccl-2.29.7-1/pkg/debian/gbp.conf000066400000000000000000000001641515037102200163070ustar00rootroot00000000000000[DEFAULT] debian-branch = master upstream-branch = master ignore-new = True [git-buildpackage] no-purge = True nccl-2.29.7-1/pkg/debian/libnccl-dev.install.in000066400000000000000000000002041515037102200210420ustar00rootroot00000000000000bin/ncclras /usr/bin include/* /usr/include lib/libnccl.so /usr/lib/${pkg:MultiArch} lib/libnccl_static.a /usr/lib/${pkg:MultiArch} nccl-2.29.7-1/pkg/debian/libnccl2.install.in000066400000000000000000000002121515037102200203470ustar00rootroot00000000000000lib/libnccl.so.${nccl:Major} /usr/lib/${pkg:MultiArch} lib/libnccl.so.${nccl:Major}.${nccl:Minor}.${nccl:Patch} /usr/lib/${pkg:MultiArch} nccl-2.29.7-1/pkg/debian/rules000077500000000000000000000003551515037102200157520ustar00rootroot00000000000000#!/usr/bin/make -f %: dh $@ --parallel override_dh_auto_install: PREFIX=debian/tmp dh_auto_install override_dh_auto_test: # Do not make test override_dh_auto_clean: # Do not make clean override_dh_builddeb: dh_builddeb -- -Zxz nccl-2.29.7-1/pkg/debian/source/000077500000000000000000000000001515037102200161675ustar00rootroot00000000000000nccl-2.29.7-1/pkg/debian/source/format000066400000000000000000000000151515037102200173760ustar00rootroot000000000000003.0 (native) nccl-2.29.7-1/pkg/redhat/000077500000000000000000000000001515037102200147145ustar00rootroot00000000000000nccl-2.29.7-1/pkg/redhat/CMakeLists.txt000066400000000000000000000060331515037102200174560ustar00rootroot00000000000000# Redhat package building for NCCL find_program(RPMBUILD_EXECUTABLE rpmbuild) find_program(SED_EXECUTABLE sed) # Set package directories set(RPMPREPDIR "${CMAKE_BINARY_DIR}/redhat") set(PKGDIR "${CMAKE_BINARY_DIR}/pkg/rpm") set(RPMGEN_IN nccl.spec.in) set(RPMGEN nccl.spec) set(RPMFILES ${RPMGEN}) set(RPMTARGETS ${RPMPREPDIR}/${RPMGEN}) # Get current timestamp in RFC 2822 format execute_process(COMMAND date -R OUTPUT_VARIABLE PKG_TIMESTAMP OUTPUT_STRIP_TRAILING_WHITESPACE) # Get architecture set(ARCH ${CMAKE_HOST_SYSTEM_PROCESSOR}) set(PKG_ARCH ${CMAKE_HOST_SYSTEM_PROCESSOR}) # Get multiarch tuple execute_process(COMMAND ${CMAKE_CXX_COMPILER} -print-multiarch OUTPUT_VARIABLE PKG_MULTIARCH OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) if(NOT PKG_MULTIARCH) # Hardwire the PKG_MULTIARCH directory as the RHEL6 distribution agnostic compiler (gcc 4.8.3) doesn't set it set(PKG_MULTIARCH "${ARCH}-linux-gnu") endif() # Custom target for redhat package preparation add_custom_target(redhat_prep ALL # Create redhat prep directory COMMAND ${CMAKE_COMMAND} -E make_directory ${RPMPREPDIR} COMMENT "Preparing Redhat package files" ) # Generate spec file from template add_custom_command( TARGET redhat_prep COMMAND ${SED_EXECUTABLE} -e 's/\\$$\{nccl:Major\}/${NCCL_MAJOR}/g' -e 's/\\$$\{nccl:Minor\}/${NCCL_MINOR}/g' -e 's/\\$$\{nccl:Patch\}/${NCCL_PATCH}/g' -e 's/\\$$\{nccl:Suffix\}/${NCCL_SUFFIX}/g' -e 's/\\$$\{cuda:Major\}/${CUDA_MAJOR}/g' -e 's/\\$$\{cuda:Minor\}/${CUDA_MINOR}/g' -e 's/\\$$\{pkg:Revision\}/${PKG_REVISION}/g' -e 's/\\$$\{pkg:Timestamp\}/${PKG_TIMESTAMP}/g' -e 's/\\$$\{pkg:Arch\}/${PKG_ARCH}/g' -e 's/\\$$\{pkg:MultiArch\}/${PKG_MULTIARCH}/g' ${CMAKE_CURRENT_SOURCE_DIR}/${RPMGEN_IN} > ${RPMPREPDIR}/${RPMGEN} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${RPMGEN_IN} ) # Build the package add_custom_target(redhat_build COMMAND ${CMAKE_COMMAND} -E make_directory ${PKGDIR} COMMAND ${RPMBUILD_EXECUTABLE} --define "_sourcedir ${CMAKE_BINARY_DIR}/pkg/txz" --define "_rpmdir ${PKGDIR}" --define "_builddir ${PKGDIR}/build/" --define "_buildrootdir ${PKGDIR}/buildroot/" -bb ${CMAKE_BINARY_DIR}/redhat/nccl.spec WORKING_DIRECTORY ${RPMPREPDIR} DEPENDS redhat_prep nccl_static txz_build COMMENT "Building Redhat package with rpmbuild" ) # Custom target for cleaning redhat package add_custom_target(redhat_clean COMMAND ${CMAKE_COMMAND} -E remove_directory ${RPMPREPDIR} COMMAND ${CMAKE_COMMAND} -E remove_directory ${PKGDIR} COMMENT "Cleaning Redhat package build files" ) # Add redhat targets to the main project add_dependencies(redhat_build nccl_static txz_build) nccl-2.29.7-1/pkg/redhat/Makefile000066400000000000000000000040231515037102200163530ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # include ../../makefiles/common.mk include ../../makefiles/version.mk BUILDDIR ?= $(abspath ../../build) RPMPREPDIR := $(BUILDDIR)/redhat PKGDIR := $(BUILDDIR)/pkg/rpm/ RPMGEN_IN := $(wildcard *.in) RPMGEN := $(RPMGEN_IN:.in=) RPMFILES := $(RPMGEN) RPMTARGETS := $(patsubst %, $(RPMPREPDIR)/%, $(RPMFILES)) PKG_TIMESTAMP := $(shell date -R) ARCH := $(shell uname -m) PKG_ARCH ?= $(shell uname -m) PKG_MULTIARCH ?= $(shell $(CXX) -print-multiarch) ifeq ($(PKG_MULTIARCH),) # Hardwire the PKG_MULTIARCH directory as the RHEL6 distribution agnostic compiler (gcc 4.8.3) doesn't set it PKG_MULTIARCH := $(ARCH)-linux-gnu endif prep : $(RPMTARGETS) $(MAKE) -C ../.. lic BUILDDIR=$(BUILDDIR) build : prep $(MAKE) -C ../.. src.build BUILDDIR=$(BUILDDIR) $(MAKE) -C ../txz build BUILDDIR=$(BUILDDIR) @printf "Building Redhat package\n" mkdir -p $(PKGDIR) rpmbuild --define "_sourcedir $(BUILDDIR)/pkg/txz" \ --define "_rpmdir $(PKGDIR)" \ --define "_builddir $(PKGDIR)/build/" \ --define "_buildrootdir $(PKGDIR)/buildroot/" \ -bb $(BUILDDIR)/redhat/nccl.spec clean: rm -Rf $(RPMPREPDIR) $(PKGDIR) $(RPMPREPDIR)/% : %.in @printf "Generating %-35s > %s\n" $< $@ mkdir -p $(RPMPREPDIR) sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \ -e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \ -e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \ -e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \ -e "s/\$${cuda:Major}/$(CUDA_MAJOR)/g" \ -e "s/\$${cuda:Minor}/$(CUDA_MINOR)/g" \ -e "s/\$${pkg:Revision}/$(PKG_REVISION)/g" \ -e "s/\$${pkg:Timestamp}/$(PKG_TIMESTAMP)/g" \ -e "s/\$${pkg:Arch}/$(PKG_ARCH)/g" \ -e "s/\$${pkg:MultiArch}/$(PKG_MULTIARCH)/g" \ $< > $@ $(RPMPREPDIR)/% : % @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(RPMPREPDIR) cp -f $< $@ nccl-2.29.7-1/pkg/redhat/nccl.spec.in000066400000000000000000000051101515037102200171110ustar00rootroot00000000000000Name: libnccl Version: ${nccl:Major}.${nccl:Minor}.${nccl:Patch}${nccl:Suffix} Release: ${pkg:Revision}+cuda${cuda:Major}.${cuda:Minor} Summary: NVIDIA Collective Communication Library (NCCL) Runtime Group: Development/Libraries License: Apache-2.0 URL: http://developer.nvidia.com/nccl Source0: nccl_${nccl:Major}.${nccl:Minor}.${nccl:Patch}${nccl:Suffix}-${pkg:Revision}+cuda${cuda:Major}.${cuda:Minor}_${pkg:Arch}.txz Requires(pre,preun): /sbin/ldconfig %description NCCL (pronounced "Nickel") is a stand-alone library of standard collective communication routines for GPUs, implementing all-reduce, all-gather, reduce, broadcast, and reduce-scatter. It has been optimized to achieve high bandwidth on any platform using PCIe, NVLink, NVswitch, as well as networking using InfiniBand Verbs or TCP/IP sockets. %package devel Summary: NVIDIA Collective Communication Library (NCCL) Runtime Group: Development/Libraries Requires: libnccl >= ${nccl:Major}.${nccl:Minor}.${nccl:Patch} %description devel NCCL development files %package static Summary: NVIDIA Collective Communication Library (NCCL) Runtime Group: Development/Libraries %description static NCCL static library %define debug_package %{nil} %prep %setup -n nccl_${nccl:Major}.${nccl:Minor}.${nccl:Patch}${nccl:Suffix}-${pkg:Revision}+cuda${cuda:Major}.${cuda:Minor}_${pkg:Arch} -q %build %install rm -rf $RPM_BUILD_ROOT install -m 755 -d $RPM_BUILD_ROOT install -m 755 -d $RPM_BUILD_ROOT/%{_libdir} install -m 755 lib/libnccl.so.${nccl:Major}.${nccl:Minor}.${nccl:Patch} $RPM_BUILD_ROOT/%{_libdir} ln -s libnccl.so.${nccl:Major}.${nccl:Minor}.${nccl:Patch} $RPM_BUILD_ROOT/%{_libdir}/libnccl.so.${nccl:Major} # devel install -m 755 -d $RPM_BUILD_ROOT/%{_bindir} install -m 755 -d $RPM_BUILD_ROOT/%{_includedir} cp -a include/* $RPM_BUILD_ROOT/%{_includedir}/ install -m 755 bin/ncclras $RPM_BUILD_ROOT/%{_bindir} ln -s libnccl.so.${nccl:Major} $RPM_BUILD_ROOT/%{_libdir}/libnccl.so # static install -m 644 lib/libnccl_static.a $RPM_BUILD_ROOT/%{_libdir} %post -p /sbin/ldconfig %postun -p /sbin/ldconfig %post devel -p /sbin/ldconfig %postun devel -p /sbin/ldconfig %clean rm -rf $RPM_BUILD_ROOT %files devel %doc LICENSE.txt %defattr(-,root,root,-) %{_bindir}/ncclras %{_includedir}/* %{_libdir}/libnccl.so %files static %doc LICENSE.txt %defattr(-,root,root,-) %{_libdir}/libnccl_static.a %files %doc LICENSE.txt %defattr(-,root,root,-) %{_libdir}/libnccl.so.${nccl:Major} %{_libdir}/libnccl.so.${nccl:Major}.${nccl:Minor}.${nccl:Patch} %changelog nccl-2.29.7-1/pkg/srctxz/000077500000000000000000000000001515037102200150025ustar00rootroot00000000000000nccl-2.29.7-1/pkg/srctxz/CMakeLists.txt000066400000000000000000000035731515037102200175520ustar00rootroot00000000000000# Source TXZ package building for NCCL find_program(SED_EXECUTABLE sed) # Set package directories set(TXZPREPDIR "${CMAKE_BINARY_DIR}/srctxz") set(PKGDIR "${CMAKE_BINARY_DIR}/pkg/srctxz") set(TXZGEN_IN create_srctxz.sh.in) set(TXZGEN create_srctxz.sh) set(TXZTARGETS ${TXZPREPDIR}/${TXZGEN}) # Set package revision set(PKG_REVISION 3) set(PKG_ARCH ${CMAKE_HOST_SYSTEM_PROCESSOR}) # Custom target for srctxz package preparation add_custom_target(srctxz_prep ALL # Create srctxz prep directory COMMAND ${CMAKE_COMMAND} -E make_directory ${TXZPREPDIR} COMMENT "Preparing Source TXZ package files" ) # Generate create_srctxz.sh from template add_custom_command( TARGET srctxz_prep COMMAND ${SED_EXECUTABLE} -e 's/\\$$\{nccl:Major\}/${NCCL_MAJOR}/g' -e 's/\\$$\{nccl:Minor\}/${NCCL_MINOR}/g' -e 's/\\$$\{nccl:Patch\}/${NCCL_PATCH}/g' -e 's/\\$$\{nccl:Suffix\}/${NCCL_SUFFIX}/g' -e 's/\\$$\{pkg:Revision\}/${PKG_REVISION}/g' ${CMAKE_CURRENT_SOURCE_DIR}/${TXZGEN_IN} > ${TXZPREPDIR}/${TXZGEN} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${TXZGEN_IN} ) # Build the package add_custom_target(srctxz_build COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_SOURCE_DIR}/src ${CMAKE_COMMAND} --build . --target clean COMMAND cd ${CMAKE_BINARY_DIR} && SRCTXZ_APITESTS=${SRCTXZ_APITESTS} bash srctxz/create_srctxz.sh COMMAND ${CMAKE_COMMAND} -E make_directory ${PKGDIR} COMMAND mv ${CMAKE_SOURCE_DIR}/../nccl-src*.txz ${PKGDIR} WORKING_DIRECTORY ${TXZPREPDIR} DEPENDS srctxz_prep COMMENT "Building Source TXZ package with create_srctxz.sh" ) # Custom target for cleaning srctxz package add_custom_target(srctxz_clean COMMAND ${CMAKE_COMMAND} -E remove_directory ${TXZPREPDIR} COMMAND ${CMAKE_COMMAND} -E remove_directory ${PKGDIR} COMMENT "Cleaning Source TXZ package build files" ) nccl-2.29.7-1/pkg/srctxz/Makefile000066400000000000000000000022331515037102200164420ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # include ../../makefiles/common.mk include ../../makefiles/version.mk BUILDDIR ?= $(abspath ../../build) TXZPREPDIR := $(BUILDDIR)/srctxz PKGDIR := $(BUILDDIR)/pkg/srctxz/ TXZGEN_IN := $(wildcard *.in) TXZGEN := $(TXZGEN_IN:.in=) TXZTARGETS := $(patsubst %, $(TXZPREPDIR)/%, $(TXZGEN)) PKG_REVISION ?= 3 PKG_ARCH := $(shell uname -m) prep: $(TXZTARGETS) build: prep $(MAKE) -C ../../src clean @printf "Building source tar.xz package\n" (cd $(BUILDDIR); SRCTXZ_APITESTS=$(SRCTXZ_APITESTS) bash srctxz/create_srctxz.sh) mkdir -p $(PKGDIR) mv $(BUILDDIR)/../../nccl-src*.txz $(PKGDIR) clean: rm -Rf $(TXZPREPDIR) $(PKGDIR) $(TXZPREPDIR)/% : %.in @printf "Generating %-35s > %s\n" $< $@ mkdir -p $(TXZPREPDIR) sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \ -e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \ -e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \ -e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \ -e "s/\$${pkg:Revision}/$(PKG_REVISION)/g" \ $< > $@ nccl-2.29.7-1/pkg/srctxz/create_srctxz.sh.in000066400000000000000000000030701515037102200206230ustar00rootroot00000000000000#!/bin/bash # # SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # To run from $BUILDDIR/ cd .. NCCLDIR=`basename $PWD` echo "Checking for unclean directory ..." git clean -x -i echo "Clean done." echo "Checking for uncommited files ..." if [ "`git status -s | wc -l`" != "0" ]; then git status -s echo "Some changes are not committed yet. Continue ? (Ctrl-C to abort)" read fi cd .. NCCL_MAJOR=${nccl:Major} NCCL_MINOR=${nccl:Minor} NCCL_PATCH=${nccl:Patch} NCCL_SUFFIX=${nccl:Suffix} NCCL_BUILD=${pkg:Revision} NCCLNAME="nccl-src_${NCCL_MAJOR}.${NCCL_MINOR}.${NCCL_PATCH}${NCCL_SUFFIX}-${NCCL_BUILD}" if [ "${SRCTXZ_APITESTS}" = "1" ]; then NCCLNAME+="-apitest" fi INCLUDE_TEST_ENTRIES=("apitest" "googletest" "gtest.mk") if [ "${SRCTXZ_APITESTS}" = "1" ]; then # Exclude all entries inside test folder except those in INCLUDE_TEST_ENTRIES for entry in $(ls $NCCLDIR/test); do if [[ ! " ${INCLUDE_TEST_ENTRIES[@]} " =~ " $entry " ]]; then EXCLUDE_TEST+=" --exclude $NCCLDIR/test/$entry" fi done else # Exclude the entire test directory EXCLUDE_TEST+=" --exclude test" fi tar --exclude fortran \ --exclude doc \ --exclude plc \ --exclude build \ --exclude ".git*" \ --exclude share \ --exclude ompi \ --exclude ext-net \ --exclude pkg/srctxz \ --exclude docker \ $EXCLUDE_TEST \ --transform "s/^$NCCLDIR/$NCCLNAME/" -Jcf $NCCLNAME.txz --owner=0 --group=0 $NCCLDIR nccl-2.29.7-1/pkg/txz/000077500000000000000000000000001515037102200142725ustar00rootroot00000000000000nccl-2.29.7-1/pkg/txz/CMakeLists.txt000066400000000000000000000036701515037102200170400ustar00rootroot00000000000000# TXZ package building for NCCL find_program(SED_EXECUTABLE sed) # Set package directories set(TXZPREPDIR "${CMAKE_BINARY_DIR}/txz") set(PKGDIR "${CMAKE_BINARY_DIR}/pkg/txz") set(TXZGEN_IN create_txz.sh.in) set(TXZGEN create_txz.sh) set(TXZTARGETS ${TXZPREPDIR}/${TXZGEN}) # Get package architecture set(PKG_ARCH ${CMAKE_HOST_SYSTEM_PROCESSOR}) # Custom target for txz package preparation add_custom_target(txz_prep ALL # Create txz prep directory COMMAND ${CMAKE_COMMAND} -E make_directory ${TXZPREPDIR} COMMENT "Preparing TXZ package files" ) # Generate create_txz.sh from template add_custom_command( TARGET txz_prep COMMAND ${SED_EXECUTABLE} -e 's/\\$$\{nccl:Major\}/${NCCL_MAJOR}/g' -e 's/\\$$\{nccl:Minor\}/${NCCL_MINOR}/g' -e 's/\\$$\{nccl:Patch\}/${NCCL_PATCH}/g' -e 's/\\$$\{nccl:Suffix\}/${NCCL_SUFFIX}/g' -e 's/\\$$\{cuda:Major\}/${CUDA_MAJOR}/g' -e 's/\\$$\{cuda:Minor\}/${CUDA_MINOR}/g' -e 's/\\$$\{pkg:Revision\}/${PKG_REVISION}/g' -e 's/\\$$\{pkg:Arch\}/${PKG_ARCH}/g' ${CMAKE_CURRENT_SOURCE_DIR}/${TXZGEN_IN} > ${TXZPREPDIR}/${TXZGEN} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${TXZGEN_IN} ) # Build the package add_custom_target(txz_build COMMAND cd ${CMAKE_BINARY_DIR} && bash txz/create_txz.sh COMMAND ${CMAKE_COMMAND} -E make_directory ${PKGDIR} COMMAND mv ${CMAKE_SOURCE_DIR}/nccl*.txz ${PKGDIR} WORKING_DIRECTORY ${TXZPREPDIR} DEPENDS txz_prep nccl_static COMMENT "Building TXZ package with create_txz.sh" ) # Custom target for cleaning txz package add_custom_target(txz_clean COMMAND ${CMAKE_COMMAND} -E remove_directory ${TXZPREPDIR} COMMAND ${CMAKE_COMMAND} -E remove_directory ${PKGDIR} COMMENT "Cleaning TXZ package build files" ) # Add txz targets to the main project add_dependencies(txz_build nccl_static) nccl-2.29.7-1/pkg/txz/Makefile000066400000000000000000000024741515037102200157410ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # include ../../makefiles/common.mk include ../../makefiles/version.mk BUILDDIR ?= $(abspath ../../build) TXZPREPDIR := $(BUILDDIR)/txz PKGDIR := $(BUILDDIR)/pkg/txz/ TXZGEN_IN := $(wildcard *.in) TXZGEN := $(TXZGEN_IN:.in=) TXZTARGETS := $(patsubst %, $(TXZPREPDIR)/%, $(TXZGEN)) PKG_ARCH := $(shell uname -m) prep: $(TXZTARGETS) $(MAKE) -C ../.. lic BUILDDIR=$(BUILDDIR) build: prep $(MAKE) -C ../.. src.build BUILDDIR=$(BUILDDIR) $(MAKE) -C ../.. ir.build BUILDDIR=$(BUILDDIR) @printf "Building tar.xz package\n" (cd $(BUILDDIR); bash txz/create_txz.sh) mkdir -p $(PKGDIR) mv $(BUILDDIR)/../nccl*.txz $(PKGDIR) clean: rm -Rf $(TXZPREPDIR) $(PKGDIR) $(TXZPREPDIR)/% : %.in @printf "Generating %-35s > %s\n" $< $@ mkdir -p $(TXZPREPDIR) sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \ -e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \ -e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \ -e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \ -e "s/\$${cuda:Major}/$(CUDA_MAJOR)/g" \ -e "s/\$${cuda:Minor}/$(CUDA_MINOR)/g" \ -e "s/\$${pkg:Revision}/$(PKG_REVISION)/g" \ -e "s/\$${pkg:Arch}/$(PKG_ARCH)/g" \ $< > $@ nccl-2.29.7-1/pkg/txz/create_txz.sh.in000066400000000000000000000013511515037102200174030ustar00rootroot00000000000000#!/bin/bash # # SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # To run from $BUILDDIR/ BUILDDIR=`basename $PWD` cd .. NCCL_MAJOR=${nccl:Major} NCCL_MINOR=${nccl:Minor} NCCL_PATCH=${nccl:Patch} NCCL_SUFFIX=${nccl:Suffix} CUDA_MAJOR=${cuda:Major} CUDA_MINOR=${cuda:Minor} PKG_REVISION=${pkg:Revision} PKG_ARCH=${pkg:Arch} NCCLNAME="nccl_${NCCL_MAJOR}.${NCCL_MINOR}.${NCCL_PATCH}${NCCL_SUFFIX}-${PKG_REVISION}+cuda${CUDA_MAJOR}.${CUDA_MINOR}_${PKG_ARCH}" tar --transform "s/^$BUILDDIR/$NCCLNAME/" -Jcf $NCCLNAME.txz --owner=0 --group=0 $BUILDDIR/bin $BUILDDIR/include $BUILDDIR/lib $BUILDDIR/*.txt nccl-2.29.7-1/plugins/000077500000000000000000000000001515037102200143455ustar00rootroot00000000000000nccl-2.29.7-1/plugins/env/000077500000000000000000000000001515037102200151355ustar00rootroot00000000000000nccl-2.29.7-1/plugins/env/README.md000066400000000000000000000242431515037102200164210ustar00rootroot00000000000000# NCCL Environment Plugin Documentation This page describes the NCCL Environment plugin API and how to implement an environment plugin for NCCL. # Overview To allow NCCL to customize environment variable handling and provide enhanced configuration management, NCCL provides an environment plugin interface. Environment plugins allow users to implement custom environment variable resolution, validation, and transformation logic without modifying NCCL core code. Environment plugins come as a shared library called `libnccl-env.so`. That shared library contains one or more implementations of the NCCL ENV API, in the form of versioned structs, filled with pointers to all required functions. # Plugin architecture ## Plugin name and supporting multiple environment plugins When NCCL is initialized, it will look for a `libnccl-env.so` library and dynamically load it, then look for symbols inside the library. The `NCCL_ENV_PLUGIN` environment variable allows multiple plugins to coexist. If set, NCCL will look for a library with a name of `libnccl-env-${NCCL_ENV_PLUGIN}.so`. It is therefore advised to name the library following that pattern, with a symlink pointing `libnccl-env.so` to `libnccl-env-${NCCL_ENV_PLUGIN}.so`. That way, if there are multiple plugins in the path, setting `NCCL_ENV_PLUGIN` will allow users to select the right plugin. ## Struct versioning Once a library is found, NCCL will look for a symbol named `ncclEnvPlugin_vX`, with `X` increasing over time. The versioning ensures that the plugin and the NCCL core are compatible. Plugins are encouraged to provide multiple of those symbols, implementing multiple versions of the NCCL ENV API, so that the same plugin can be compiled and support a wide range of NCCL versions. Conversely, and to ease transition, NCCL can choose to support different plugin versions, looking for the latest ncclEnv struct version, but also looking for older ones so that older plugins would still work. ## Headers management To help users build plugins effortlessly, plugins should copy the `ncclEnv_vX` definitions they support to their internal includes. An example is shown in `plugins/env/example/` where we keep all headers in the `nccl/` directory and provide thin layers to implement old versions on top of newer ones. The `nccl/` directory is populated with `env_vX.h` files extracting all relevant definitions from old API versions. It also provides error codes in `err.h`. # API (v1) Below is the main `ncclEnv_v1` struct. Each function is explained in later sections. ```c typedef struct { const char* name; // Initialize the environment plugin // Input // - ncclMajor: NCCL major version number // - ncclMinor: NCCL minor version number // - ncclPatch: NCCL patch version number // - suffix: NCCL version suffix string ncclResult_t (*init)(uint8_t ncclMajor, uint8_t ncclMinor, uint8_t ncclPatch, const char* suffix); // Finalize the environment plugin ncclResult_t (*finalize)(void); // Get environment variable value // Input // - name: environment variable name // Output // - returns: pointer to environment variable value string, or NULL if not found. The plugin is responsible for keeping the // returned value (address) valid until it is no longer needed by NCCL. This happens when NCCL calls ``finalize`` // or ``getEnv`` again on the same variable name. In any other case, modifying the variable (e.g., through // ``setenv``) is considered undefined behavior since NCCL might access the returned address after the plugin has // reset the variable. const char* (*getEnv)(const char* name); } ncclEnv_v1_t; ``` ## Error codes All plugin functions use NCCL error codes as return value. `ncclSuccess` should be returned upon success. Otherwise, plugins can return one of the following: - `ncclSystemError` is returned when a system call fails, such as memory allocation or file I/O errors. - `ncclInternalError` is returned when the plugin encounters an internal error during initialization or finalization. - `ncclInvalidUsage` should be returned when the error is most likely a user error, such as invalid configuration. - `ncclInvalidArgument` should be returned when invalid arguments are passed to plugin functions. ## Operation overview NCCL will call the `init` function first during initialization, passing the NCCL version information. This allows the plugin to initialize its internal state and validate compatibility with the NCCL version. The `getEnv` function is called whenever NCCL needs to retrieve an environment variable value. This provides the plugin with the opportunity to implement custom environment variable resolution, validation, or transformation logic. The plugin should keep every variable accessible thoroughout the plugin lifetime (i.e., until NCCL calls finalize). When NCCL is finalized, the `finalize` function is called to allow the plugin to clean up any resources and perform any necessary cleanup operations. ## API Functions ### Initialization #### name The `name` field should point to a character string with the name of the environment plugin. This will be used for all logging, especially when `NCCL_DEBUG=INFO` is set. #### init As soon as NCCL finds the plugin and the correct ncclEnv symbol, it calls its `init` function. This allows the plugin to initialize its internal context and validate compatibility with the NCCL version. The function receives: - `ncclMajor`, `ncclMinor`, `ncclPatch`: NCCL version numbers for compatibility checking - `suffix`: NCCL version suffix string (e.g., "+cuda12.0") If the `init` function does not return `ncclSuccess`, NCCL will fall back to the internal environment plugin. #### finalize When the environment plugin is no longer needed, a call to `finalize` allows the plugin to clean up resources and perform any necessary cleanup operations. ### Environment variable handling #### getEnv The `getEnv` function is called whenever NCCL needs to retrieve an environment variable value. This function provides the plugin with the opportunity to implement custom environment variable resolution logic. The function receives: - `name`: The name of the environment variable to retrieve The function should return: - A pointer to the environment variable value string if found - `NULL` if the environment variable is not set or not found This allows plugins to implement various features such as: - Environment variable validation and sanitization - Dynamic environment variable resolution - Configuration file integration - Environment variable transformation or substitution - Hierarchical configuration management The returned memory address for the variable should not be modified by the plugin until it is safe to do so. That is, when NCCL calls the plugin ``finalize`` or ``getEnv`` function for the same variable again. In any other case, modifying the variable is considered undefined behavior. # Plugin implementation examples ## Basic environment plugin A basic environment plugin that simply delegates to the system `getenv` function: ```c #include "nccl_env.h" static ncclResult_t ncclEnvInit(uint8_t ncclMajor, uint8_t ncclMinor, uint8_t ncclPatch, const char* suffix) { return ncclSuccess; } static ncclResult_t ncclEnvFinalize(void) { return ncclSuccess; } static const char* ncclEnvGetEnv(const char* name) { return getenv(name); } const ncclEnv_v1_t ncclEnvPlugin_v1 = { .name = "ncclEnvBasic", .init = ncclEnvInit, .finalize = ncclEnvFinalize, .getEnv = ncclEnvGetEnv, }; ``` ## Loading the plugin Set the `LD_LIBRARY_PATH` to include your plugin directory: ```bash export LD_LIBRARY_PATH=/path/to/your/plugin:$LD_LIBRARY_PATH ``` Set `NCCL_ENV_PLUGIN` to either the plugin name or the absolute path to the plugin file: ```bash export NCCL_ENV_PLUGIN=myenv export NCCL_ENV_PLUGIN=libnccl-env-myenv.so export NCCL_ENV_PLUGIN=/path/to/your/plugin/libnccl-env-myenv.so ``` NCCL will automatically discover and load the plugin based on the exported symbol names. # Advanced topics ## Plugin versioning NCCL supports multiple plugin interface versions. Make sure your plugin exports the correct version: ```c const ncclEnv_v1_t ncclEnvPlugin_v1 = { .name = "YourPluginName", .init = yourInitFunction, .finalize = yourFinalizeFunction, .getEnv = yourGetEnvFunction, }; ``` ## Environment variable caching For performance reasons, plugins may want to implement caching of environment variable values. However, care should be taken to ensure that cached values remain consistent with the actual environment state. ## Integration with configuration management systems Environment plugins can integrate with external configuration management systems by: - Reading configuration from files or databases - Implementing hierarchical configuration resolution - Supporting configuration hot-reloading - Providing configuration validation and schema enforcement # Best practices 1. **Test thoroughly**: Verify your plugin works with various environment variable configurations 2. **Handle edge cases**: Ensure your plugin behaves correctly with unusual or malformed input 3. **Document your approach**: Clearly document your environment variable handling strategy 4. **Version your plugin**: Use meaningful version numbers and maintain backward compatibility 5. **Performance optimization**: Keep plugin logic lightweight to avoid impacting NCCL performance 6. **Error handling**: Implement robust error handling and graceful degradation 7. **Security**: Validate and sanitize environment variable values appropriately # Known limitations - Environment plugins are called synchronously during NCCL initialization and environment variable access - Plugins should avoid blocking operations in the `getEnv` function - The plugin interface does not support asynchronous environment variable updates # Contributing When developing new environment plugins: - Follow the existing code style and structure - Include comprehensive documentation - Add example configurations and test cases - Consider contributing useful plugins back to the community # Resources - [NCCL Documentation](https://docs.nvidia.com/deeplearning/nccl/) - Example plugin implementations in this directory For questions and support, refer to the NCCL community resources and documentation. nccl-2.29.7-1/plugins/env/example/000077500000000000000000000000001515037102200165705ustar00rootroot00000000000000nccl-2.29.7-1/plugins/env/example/CMakeLists.txt000066400000000000000000000012571515037102200213350ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/plugin.c ) # Create shared library add_library(nccl-env-example SHARED ${SRC_FILES}) # Set include directories target_include_directories(nccl-env-example PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/nccl ) # Set output name to match Makefile set_target_properties(nccl-env-example PROPERTIES OUTPUT_NAME "nccl-env-example" PREFIX "lib" POSITION_INDEPENDENT_CODE ON LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test/unit/plugins ) nccl-2.29.7-1/plugins/env/example/Makefile000066400000000000000000000010651515037102200202320ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # .DEFAULT_GOAL: build include ../../../makefiles/common.mk BUILDDIR ?= . NCCLDIR := $(BUILDDIR) SRC_FILES := $(wildcard *.c) build: ${BUILDDIR}/libnccl-env-example.so ${BUILDDIR}/libnccl-env-example.so: ${SRC_FILES} @printf "Compiling %-35s > %s\n" $< $@ @mkdir -p ${BUILDDIR} $(CC) -Inccl -fPIC -shared -o $@ $^ clean: rm -f ${BUILDDIR}/libnccl-env-example.so nccl-2.29.7-1/plugins/env/example/nccl/000077500000000000000000000000001515037102200175075ustar00rootroot00000000000000nccl-2.29.7-1/plugins/env/example/nccl/env.h000066400000000000000000000003501515037102200204460ustar00rootroot00000000000000/* * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. */ #ifndef NCCL_ENV_H_ #define NCCL_ENV_H_ #include #include "env_v1.h" // Plugin symbol name #define NCCL_ENV_PLUGIN_SYMBOL ncclEnvPlugin_v1 #endif nccl-2.29.7-1/plugins/env/example/nccl/env_v1.h000066400000000000000000000023671515037102200210660ustar00rootroot00000000000000/* * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. */ #ifndef ENV_V1_H_ #define ENV_V1_H_ #include "err.h" typedef struct { const char* name; // Initialize the environment plugin // Input // - ncclMajor: NCCL major version number // - ncclMinor: NCCL minor version number // - ncclPatch: NCCL patch version number // - suffix: NCCL version suffix string ncclResult_t (*init)(uint8_t ncclMajor, uint8_t ncclMinor, uint8_t ncclPatch, const char* suffix); // Finalize the environment plugin ncclResult_t (*finalize)(void); // Get environment variable value // Input // - name: environment variable name // Output // - returns: pointer to environment variable value string, or NULL if not found. The plugin is responsible for keeping the // returned value (address) valid until it is no longer needed by NCCL. This happens when NCCL calls ``finalize`` // or ``getEnv`` again on the same variable name. In any other case, modifying the variable (e.g., through // ``setenv``) is considered undefined behavior since NCCL might access the returned address after the plugin has // reset the variable. const char* (*getEnv)(const char* name); } ncclEnv_v1_t; #endif nccl-2.29.7-1/plugins/env/example/nccl/err.h000066400000000000000000000005041515037102200204470ustar00rootroot00000000000000/* * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. */ #ifndef ERR_H_ #define ERR_H_ // NCCL error codes #define ncclSuccess 0 #define ncclSystemError 1 #define ncclInternalError 2 #define ncclInvalidUsage 3 #define ncclInvalidArgument 4 #define ncclUnhandledCudaError 5 typedef int ncclResult_t; #endif nccl-2.29.7-1/plugins/env/example/plugin.c000066400000000000000000000033521515037102200202350ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include #include #include #include "nccl/env.h" /** * Initialize the environment plugin * * This function is called by NCCL during initialization to set up the plugin. * It receives NCCL version information and a logging function for debug output. */ static ncclResult_t ncclEnvInit(uint8_t ncclMajor, uint8_t ncclMinor, uint8_t ncclPatch, const char* suffix) { return ncclSuccess; } /** * Finalize the environment plugin * * This function is called by NCCL during finalization to clean up plugin resources. */ static ncclResult_t ncclEnvFinalize(void) { return ncclSuccess; } /** * Get environment variable value * * This function is called by NCCL whenever it needs to retrieve an environment variable. * It delegates to the system getenv function and provides optional logging. * * @param name The name of the environment variable to retrieve * @return Pointer to the environment variable value string, or NULL if not found */ static const char* ncclEnvGetEnv(const char* name) { return getenv(name); } /** * Export the plugin structure * * This structure must be exported with the correct symbol name for NCCL to find it. * The symbol name should match NCCL_ENV_PLUGIN_SYMBOL defined in nccl_env.h. */ const ncclEnv_v1_t ncclEnvPlugin_v1 = { .name = "ncclEnvExample", .init = ncclEnvInit, .finalize = ncclEnvFinalize, .getEnv = ncclEnvGetEnv, }; nccl-2.29.7-1/plugins/mixed/000077500000000000000000000000001515037102200154535ustar00rootroot00000000000000nccl-2.29.7-1/plugins/mixed/README.md000066400000000000000000000002711515037102200167320ustar00rootroot00000000000000# NCCL Mixed Plugin Documentation The NCCL mixed plugin API is a combination of Net and Tuner plugin APIs. It demonstrates how different plugins can be combined into a single library. nccl-2.29.7-1/plugins/mixed/example/000077500000000000000000000000001515037102200171065ustar00rootroot00000000000000nccl-2.29.7-1/plugins/mixed/example/CMakeLists.txt000066400000000000000000000013661515037102200216540ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/plugin.c ) # Create shared library add_library(nccl-mixed-example SHARED ${SRC_FILES}) # Set include directories target_include_directories(nccl-mixed-example PRIVATE ${CMAKE_SOURCE_DIR}/plugins/net/example/nccl ${CMAKE_SOURCE_DIR}/plugins/tuner/example/nccl ) # Set output name to match Makefile set_target_properties(nccl-mixed-example PROPERTIES OUTPUT_NAME "nccl-mixed-example" PREFIX "lib" POSITION_INDEPENDENT_CODE ON LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test/unit/plugins ) nccl-2.29.7-1/plugins/mixed/example/Makefile000066400000000000000000000011351515037102200205460ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # .PHONY=test NCCL_PLUGINS:=../.. NCCL_HOME:=../../../build CUDA_HOME:=/usr/local/cuda INC:= -I$(NCCL_HOME)/include -I$(CUDA_HOME)/include -I$(NCCL_PLUGINS)/net/example/nccl -I$(NCCL_PLUGINS)/tuner/example/nccl PLUGIN_SO:=libnccl-mixed.so default: $(PLUGIN_SO) $(PLUGIN_SO): plugin.c $(CC) $(INC) -fPIC -shared -o $@ -Wl,-soname,$(PLUGIN_SO) $^ test: $(PLUGIN_SO) @bash test.sh clean: rm -f $(PLUGIN_SO) nccl-2.29.7-1/plugins/mixed/example/plugin.c000066400000000000000000000144501515037102200205540ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "net.h" #define PLUGIN_NAME "Plugin" #define __hidden __attribute__ ((visibility("hidden"))) #define NCCL_PLUGIN_MAX_RECVS 1 int max_requests = NCCL_NET_MAX_REQUESTS; __hidden ncclResult_t pluginInit(void** ctx, uint64_t commId, ncclNetCommConfig_t* config, ncclDebugLogger_t logFunction, ncclProfilerCallback_t profFunction) { return ncclSuccess; } __hidden ncclResult_t pluginDevices(int* ndev) { *ndev = 0; return ncclSuccess; } __hidden ncclResult_t pluginPciPath(int dev, char** path) { return ncclInternalError; } __hidden ncclResult_t pluginPtrSupport(int dev, int* supportedTypes) { return ncclInternalError; } __hidden ncclResult_t pluginGetProperties(int dev, ncclNetProperties_t* props) { // Below are default values, if unsure don't change. props->name = "Example"; // Fill for proper topology detection, e.g. /sys/devices/pci0000:00/0000:00:10.0/0000:0b:00.0 props->pciPath = NULL; // Only used to detect NICs with multiple PCI attachments. props->guid = 0; // Add NCCL_PTR_CUDA if GPU Direct RDMA is supported and regMr can take CUDA pointers. props->ptrSupport = NCCL_PTR_HOST; // If you regMr has a fast registration cache, set to 1. If set to 0, user buffer registration may be disabled. props->regIsGlobal = 0; // Force flush after receive. Needed if the control path and data path use a different path to the GPU props->forceFlush = 0; // Speed in *Mbps*. 100000 means 100G props->speed = 100000; // Port number, used in conjunction with guid props->port = 0; // Custom latency (used to help tuning if latency is high. If set to 0, use default NCCL values. props->latency = 0; // Maximum number of comm objects we can create. props->maxComms = 1024*1024; // Maximum number of receive operations taken by irecv(). props->maxRecvs = NCCL_PLUGIN_MAX_RECVS; // Coupling with NCCL network device-side code. props->netDeviceType = NCCL_NET_DEVICE_HOST; props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION; // Used to tell NCCL core whether this is a virtual device fusing multiple physical devices. props->vProps.ndevs = 1; props->vProps.devs[0] = dev; // maximum transfer sizes the plugin can handle props->maxP2pBytes = NCCL_MAX_NET_SIZE_BYTES; props->maxCollBytes = NCCL_MAX_NET_SIZE_BYTES; return ncclSuccess; } __hidden ncclResult_t pluginListen(void* ctx, int dev, void* handle, void** listenComm) { return ncclInternalError; } __hidden ncclResult_t pluginConnect(void* ctx, int dev, void* handle, void** sendComm, ncclNetDeviceHandle_t** sendDevComm) { return ncclInternalError; } __hidden ncclResult_t pluginAccept(void* listenComm, void** recvComm, ncclNetDeviceHandle_t** recvDevComm) { return ncclInternalError; } __hidden ncclResult_t pluginRegMr(void* collComm, void* data, size_t size, int type, void** mhandle) { return ncclInternalError; } __hidden ncclResult_t pluginRegMrDmaBuf(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle) { return ncclInternalError; } __hidden ncclResult_t pluginDeregMr(void* collComm, void* mhandle) { return ncclInternalError;} __hidden ncclResult_t pluginIsend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* phandle, void** request) { return ncclInternalError; } __hidden ncclResult_t pluginIrecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** phandles, void** request) { return ncclInternalError; } __hidden ncclResult_t pluginIflush(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request) { return ncclInternalError; } __hidden ncclResult_t pluginTest(void* request, int* done, int* size) { return ncclInternalError; } __hidden ncclResult_t pluginCloseSend(void* sendComm) { return ncclInternalError; } __hidden ncclResult_t pluginCloseRecv(void* recvComm) { return ncclInternalError; } __hidden ncclResult_t pluginCloseListen(void* listenComm) { return ncclInternalError; } __hidden ncclResult_t pluginIrecvConsumed(void* recvComm, int n, void* request) { return ncclInternalError; } __hidden ncclResult_t pluginGetDeviceMr(void* comm, void* mhandle, void** dptr_mhandle) { return ncclInternalError; } __hidden ncclResult_t pluginMakeVDevice(int* d, ncclNetVDeviceProps_t* props) { return ncclInternalError; } __hidden ncclResult_t pluginFinalize(void* ctx) { return ncclSuccess; } const ncclNet_v11_t ncclNetPlugin_v11 = { .name = PLUGIN_NAME, .init = pluginInit, .devices = pluginDevices, .getProperties = pluginGetProperties, .listen = pluginListen, .connect = pluginConnect, .accept = pluginAccept, .regMr = pluginRegMr, .regMrDmaBuf = pluginRegMrDmaBuf, .deregMr = pluginDeregMr, .isend = pluginIsend, .irecv = pluginIrecv, .iflush = pluginIflush, .test = pluginTest, .closeSend = pluginCloseSend, .closeRecv = pluginCloseRecv, .closeListen = pluginCloseListen, .getDeviceMr = pluginGetDeviceMr, .irecvConsumed = pluginIrecvConsumed, .makeVDevice = pluginMakeVDevice, .finalize = pluginFinalize }; #include "tuner.h" __hidden ncclResult_t tunerPluginInit(void** context, uint64_t commId, size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, ncclNvlDomainInfo_v5_t* nvlDomainInfo, ncclTunerConstants_v5_t* constants) { return ncclSuccess; } __hidden ncclResult_t tunerPluginGetCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff, int* nChannels) { // Update NCCL core generated cost table. Updated table will be evaluated by NCCL to pick the best algo/proto combo if (collCostTable[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE) { collCostTable[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] = 0.0; } *nChannels = 1; return ncclSuccess; } __hidden ncclResult_t tunerPluginFinalize(void* context) { return ncclSuccess; } const ncclTuner_v5_t ncclTunerPlugin_v5 = { .name = PLUGIN_NAME, .init = tunerPluginInit, .getCollInfo = tunerPluginGetCollInfo, .finalize = tunerPluginFinalize }; nccl-2.29.7-1/plugins/net/000077500000000000000000000000001515037102200151335ustar00rootroot00000000000000nccl-2.29.7-1/plugins/net/CMakeLists.txt000066400000000000000000000002431515037102200176720ustar00rootroot00000000000000# Since all the plugins generate binary with the same name, build only one of them add_subdirectory(example) # add_subdirectory(ib_sharp) # add_subdirectory(mock) nccl-2.29.7-1/plugins/net/README.md000066400000000000000000000551161515037102200164220ustar00rootroot00000000000000# NCCL Net Plugin Documentation This page describes the NCCL Net plugin API and how to implement a network plugin for NCCL. # Overview To allow NCCL to work on any network type, NCCL provides a way to use external plugins. Plugins implement the NCCL network API, and decouple NCCL binary builds which are built against a particular version of the GPU stack (i.e. CUDA) from the network code which is built against a particular version of the networking stack. That way, we can easily integrate any CUDA version with any network stack version. NCCL network plugins come as a shared library called `libnccl-net.so`. That shared library contains one or more implementations of the NCCL NET API, in the form of versioned structs, filled with pointers to all required functions. # Plugin architecture ## Plugin name and supporting multiple network plugins When NCCL is initialized, it will look for a `libnccl-net.so` library and dynamically load it, then look for symbols inside the library. The `NCCL_NET_PLUGIN` environment variable allows multiple plugins to coexist. If set, NCCL will look for a library with a name of `libnccl-net-${NCCL_NET_PLUGIN}.so`. It is therefore advised to name the library following that pattern, with a symlink pointing `libnccl-net.so` to `libnccl-net-${NCCL_NET_PLUGIN}.so`. That way, if there are multiple plugins in the path, setting `NCCL_NET_PLUGIN` will allow users to select the right plugin. ## Struct versioning Once a library is found, NCCL will look for a symbol named `ncclNet_vX`, with `X` increasing over time. The versioning ensures that the plugin and the NCCL core are compatible. Plugins are encouraged to provide multiple of those symbols, implementing multiple versions of the NCCL NET API, so that the same plugin can be compiled and support a wide range of NCCL versions. Conversely, and to ease transition, NCCL can choose to support different plugin versions, looking for the latest ncclNet struct version, but also looking for older ones so that older plugins would still work. ## In-network collective operations, a.k.a. collNet Additionally to the ncclNet structure, network plugins can provide a collNet structure which implements in-network collective operations, if supported. That can be used by the NCCL collNet algorithm to accelerate inter-node reductions in allReduce. The collNet struct is a different, optional struct provided by the network plugin, but its versioning is tied to the ncclNet struct and many functions are common between the two to ease the implementation. ## Headers management To help users build plugins effortlessly, plugins should copy the `ncclNet_vX` definitions they support to their internal includes. An example is shown in `plugins/net/example/` where we keep all headers in the `nccl/` directory and provide thin layers to implement old versions on top of newer ones. The `nccl/` directory is populated with `net_vX.h` files extracting all relevant definitions from old API versions. It also provides error codes in `err.h`. # API (v11) Below is the main `ncclNet_v11` struct. Each function is explained in later sections. ``` typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(void** ctx, uint64_t commId, ncclNetCommConfig_v11_t* config, ncclDebugLogger_t logFunction, ncclProfilerCallback_t profFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Get various device properties. ncclResult_t (*getProperties)(int dev, ncclNetProperties_v11_t* props); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(void* ctx, int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. // This call must not block for the connection to be established, and instead // should return successfully with sendComm == NULL with the expectation that // it will be called again until sendComm != NULL. // If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection ncclResult_t (*connect)(void* ctx, int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v11_t** sendDevComm); // Finalize connection establishment after remote peer has called connect. // This call must not block for the connection to be established, and instead // should return successfully with recvComm == NULL with the expectation that // it will be called again until recvComm != NULL. // If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v11_t** recvDevComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle); /* DMA-BUF support */ ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* pHandle, void** request); // Asynchronous recv from a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** pHandles, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* sizes); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); // Copy the given mhandle to a dptr in a format usable by this plugin's device code ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle); // Notify the plugin that a recv has completed by the device ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request); // Virtual NIC APIs. makeVDevice will create a virtual NIC given the specified properties, and tell the caller // what index this new vNIC exists at ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_t* props); } ncclNet_t; ``` ## Error codes All plugins functions use NCCL error codes as return value. `ncclSuccess` should be returned upon success. Otherwise, plugins can return one of the following: - `ncclSystemError` is the most common error for network plugins, when a call to the linux kernel or a system library fails. This typically includes all network/hardware errors. - `ncclInternalError` is returned when the NCCL core code is using the network plugin in an incorrect way, for example allocating more requests than it should, or passing an invalid argument to calls. - `ncclInvalidUsage` should be returned when the error is most likely a user error. This can include misconfiguration, but also sizes mismatch. - `ncclInvalidArgument` should usually not be used by plugins since arguments should be checked by the NCCL core layer. - `ncclUnhandledCudaError` is returned when an error comes from CUDA. Since network plugins should not need to rely on CUDA, this should not be common. ## Operation overview NCCL will call the `init` function first, then query the number of network devices with the `devices` function, getting each network device properties with `getProperties`. If NCCL wishes to initialize virtual devices, used in NIC fusion currently, it can call `makeVDevice` specifying a list of physical devices (the original devices listed from `devices`) it wishes to merge together. If the plugin does not support NIC fusion, it can set `makeVDevice` to null. To establish a connection between two network devices, NCCL will first call `listen` on the receiving side, pass the returned handle to the sender side of the connection, and call `connect` with that handle. Finally, `accept` will be called on the receiving side to finalize the connection establishment. `connect` and `accept` can receive an optional `netDevComm` pointer from the caller, if the caller wishes to make use of device networking. This parameter may be ignored by the plugin if it does not support device-side networking. Once the connection is established, communication will be done using the functions `isend`, `irecv` and `test`. Prior to calling `isend` or `irecv`, NCCL will call the `regMr` function on all buffers to allow RDMA NICs to prepare buffers. `deregMr` will be used to unregister buffers. In certain conditions, `iflush` will be called after a receive calls completes to allow the network plugin to flush data and ensure the GPU will observe the newly written data. To close the connections NCCL will call `closeListen` to close the object returned by `listen`, `closeSend` to close the object returned by `connect` and `closeRecv` to close the object returned by `accept`. ## API Functions ### Initialization `name` The `name` field should point to a character string with the name of the network plugin. This will be used for all logging, especially when `NCCL_DEBUG=INFO` is set. Note: setting `NCCL_NET=` will ensure a specific network implementation is used, with a matching `name`. This is not to be confused with `NCCL_NET_PLUGIN` which defines a suffix to the `libnccl-net.so`library name to load. `init` As soon as NCCL finds the plugin and the correct ncclNet symbol, it will call the `init` function. This will allow the plugin to discover network devices and make sure they are usable. If the `init` function does not return `ncclSuccess`, then NCCL will not use the plugin and fall back on internal ones. Every call to `init` returns an opaque context that the plugin uses internally to allocate resources and manage state. Such context is passed to other net plugin calls that create further resources, such as `listen` and `connect`. Every context is uniquely associated to a communicator using the commId. The network can also be initialized with a per communicator configuration using the `config` argument. To allow the plugin logs to integrate into the NCCL logs seemlessly, NCCL provides a logging function to `init`. This function is typically used to allow for `INFO` and `WARN` macros within the plugin code adding the following definitions: ``` #define WARN(...) logFunction(NCCL_LOG_WARN, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__) #define INFO(FLAGS, ...) logFunction(NCCL_LOG_INFO, (FLAGS), __func__, __LINE__, __VA_ARGS__) ``` The `ncclProfilerCallback_t` argument is a NCCL core callback that allows the plugin to define and record its own events with the NCCL profiler plugin. `devices` Once the plugin is initialized, NCCL will query the number of devices available. It should not be zero, otherwise NCCL initialization will fail. If no device is present or usable, the `init` function should not return `ncclSuccess`. `getProperties` Right after getting the number of devices, NCCL will query properties for each available network device. These properties are critical when multiple adapters are present to ensure NCCL uses each adapter in the most optimized way. The `name` is only used for logging. The `pciPath` is the base for all topology detection and should point to the PCI device directory in /sys. This is typically the directory pointed by `/sys/class/net/eth0/device` or `/sys/class/infiniband/mlx5_0/device`. If the network interface is virtual, then `pciPath` should be `NULL`. The `guid` field is used to determine when network adapters are connected to multiple PCI endpoints. For normal cases, it can be set to the device number. If multiple network devices have the same guid, then NCCL will consider the are sharing the same network port to the fabric, hence it will not use the port multiple times. The `ptrSupport` field indicates whether or not CUDA pointers are supported. If so, it should be set to `NCCL_PTR_HOST|NCCL_PTR_CUDA`, otherwise it should be set to `NCCL_PTR_HOST`. If the plugin supports `dmabuf`, it should set `ptrSupport` to `NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF` and provide a `regMrDmaBuf` function. The `regIsGlobal` field allows NCCL to register buffers in advance using e.g. a loopback connection and later on, expect that another registration on a buffer contained within a previous registration will be nearly immediate, as the buffer is already known by the network adapter. A typical implementation would maintain a registration cache; the call to ncclCommRegister will create the initial entry in the cache using regMr() on a loopback connection. Any later call to NCCL operations will call regMr() again on the real connection, with the real buffer (could be at a different offset within the original buffer, with a smaller size, etc), then deregMr() right after. The call to ncclCommDeregister should call the final deregMr() and effectively remove the mapping on the network adapter. The `forceFlush` field can request the NCCL core to call flush for all transfers. By default, flushes are only called when the GPU architecture or PCI topology would not not guarantee correct PCI ordering. Plugins can set it to one if the NIC operates in a mode where e.g. the data and the completion paths use different PCI links and therefore need a call to flush() to guarantee ordering. The `speed` field indicates the speed of the network port in Mbps (10^6 bits per second). This is important to ensure proper optimization of flows within the node. The `port` field indicates the port number. This is important again for topology detection and flow optimization within the node when a NIC with a single PCI connection is connected to the fabric with multiple ports. The `latency` field indicates the network latency in microseconds. This can be useful to improve the NCCL tuning and make sure NCCL switches from tree to ring at the right size. The `maxComms` field indicates the maximum number of connections we can create. The `maxRecvs` field indicates the maximum number for grouped receive operations (see grouped receive). The `netDeviceType` indicates which type of device networking this plugin supports. The current supported options are `NCCL_NET_DEVICE_HOST` and `NCCL_NET_DEVICE_UNPACK`. The `netDeviceVersion` indicates the version of device networking this plugin supports. Currently, this must match the associated netDeviceVersion of this netDeviceType compiled into NCCL core. Net device functionality is built as apart of NCCL core's device code. The `maxP2pBytes` and `maxCollBytes` fields indicate the maximum size the plugin can handle for point-to-point and collective calls. This will tell the NCCL core to cut large operations into multiple smaller chunks if needed. `vProps` is the list of devices that have been fused into the current device. Each entry is an index pointing to the child device. ### Connection establishment Connections are used in an unidirectional manner. There is therefore a sender side and a receiver side. `listen` To create a connection, NCCL will start by calling `listen` on the receiver side. This function takes the opaque plugin context returned by `init` and a device number as input argument, and should return a local `listenComm` object, and a `handle` to pass to the other side, so that the sender side can connect to the receiver. The `handle` is a buffer of size `NCCL_NET_HANDLE_MAXSIZE` and is provided by NCCL. This call should never block, but contrary to `connect` and `accept`, `listenComm` should never be `NULL` if the call succeeds. `connect` NCCL will use its bootstrap infrastructure to provide the `handle` to the sender side, then call `connect` on the sender side on a given device index `dev`, providing the `handle`. `connect` should not block either, and instead set `sendComm` to `NULL` and return `ncclSuccess`. In that case, NCCL will call `accept` again until it succeeds. `accept` To finalize the connection, the receiver side will call `accept` on the `listenComm` returned by the `listen` call previously. If the sender did not connect yet, `accept` should not block. It should return `ncclSuccess`, setting `recvComm` to `NULL`. NCCL will call `accept` again until it succeeds. The `connect` API takes the opaque plugin context returned by `init`. The plugin context can reference the `ncclNetCommConfig_t` passed to the `init` function and containing a trafficClass field. This field can be used by the network plugin to specify the QoS level of the connection. By default, `trafficClass` is set to -1 but can be configured by the application during communicator initialization to select a plugin-supported QoS level. `closeListen`/`closeSend`/`closeRecv` Once a `listenComm`/`sendComm`/`recvComm` is no longer needed, NCCL will call `closeListen`/`closeSend`/`closeRecv` to free the associated resources. ### Communication Communication is done using asynchronous send and receive operations: `isend`, `irecv` and `test`. To support RDMA capabilities, buffer registration and flush functions are provided. To keep track of asynchronous send, receive and flush operations, requests are returned to NCCL, then queried with `test`. Each `sendComm` or `recvComm` must be able to handle `NCCL_NET_MAX_REQUESTS` requests in parallel. Note: That value should be multiplied by the multi-receive capability of the plugin for the sender side, so that we can effectively have `NCCL_NET_MAX_REQUESTS` multi-receive operations happening in parallel. So, if we have a `maxRecvs`value of 8 and `NCCL_NET_MAX_REQUESTS` is 8, then each `sendComm` must be able to handle up to 8x8=64 concurrent `isend` operations. `regMr` Prior to sending or receiving data, NCCL will call `regMr` with any buffers later used for communication. It will provide a `sendComm` or `recvComm` as `comm` argument, then the buffer pointer `data`, `size`, and `type` being either `NCCL_PTR_HOST`, or `NCCL_PTR_CUDA` if the network supports CUDA pointers. The network plugin can use the output argument `mhandle` to keep any reference to that memory registration, as this `mhandle` will be passed back for all `isend`, `irecv`, `iflush` and `deregMr` calls. `regMrDmaBuf` If the plugin has set the `NCCL_PTR_DMABUF` property in `ptrSupport`, NCCL will use `regMrDmaBuf` instead of `regMr`. If the property was not set, `regMrDmaBuf` can be set to `NULL`. `deregMr` When buffers will no longer be used for communication, NCCL will call `deregMr` to let the plugin free resources. This function is used to deregister handles returned by both `regMr` and `regMrDmaBuf`. `isend` Data will be sent through the connection using `isend`, passing the `sendComm` previously created by `connect`, and the buffer described by `data`, `size`, and `mhandle`. A `tag` must be used if the network supports multi-receive operations (see `irecv`) to distinguish between different sends matching the same multi-receive. Otherwise it can be set to 0. The `isend` operation returns a handle in the `request` argument for further calls to `test`. If the `isend` operation cannot be initiated, `request` can be set to `NULL` and NCCL will call `isend` again later. The `pHandle` argument allows NCCL to pass an opaque handle that can be used by the network plugin to support network defined events. `irecv` To receive data, NCCL will call `irecv` with the `recvComm` returned by `accept`. The argument `n` will allow NCCL to perform a multi-receive, to allow grouping of multiple sends through a single network connection. Each buffer will be described by the `data`, `sizes`, and `mhandles` arrays. `tags` will specify a tag for each receive so that each of the `n` independent `isend` operations is received into the right buffer. If all receive operations can be initiated, `irecv` will return a handle in the `request` pointer, otherwise it will set it to `NULL`. In the case of multi-receive, all `n` receive operations are handled by a single request handle. The sizes provided to `irecv` can (and will) be larger than the size of the `isend` operation. The contrary (receive size being lower than the send size) is an error, however. NCCL sets request pointer in `irecv` to `NCCL_NET_OPTIONAL_RECV_COMPLETION` when it is using LL or LL128 protocols. In these cases, NCCL polls on flag embedded in data to detect completion of irecv and is resilient to redundant network writes. This allows the plugin to optimize request completions on such irecvs (for example, complete the request immediately). The plugin is still expected to set a valid request pointer on return which NCCL can poll to check for completion. The `pHandle` argument allows NCCL to pass an array of opaque handles that can be used by the network plugin to support network defined events. Note: for a given connection, send/receive operations should always match in the order they were posted. Tags provided for receive operations are only used to assign a given send operation to one of the buffers of the first (multi-)receive in the queue, not to allow for out-of-order tag matching on any receive operation posted. `test` After an `isend` or `irecv` operation is initiated, NCCL will call `test` on the request handles until they complete. When that happens, `done` will be set to 1 and `sizes` will be set to the real size sent or received, the latter being potentially lower than the size passed to `irecv`. In the case of a multi-receive, all receives will be considered as done as a single operation (the goal being to allow aggregation), hence they share a single request and a single `done` status. However, they can have different sizes, so when `done` is non-zero, the `sizes` array should contain the `n` sizes corresponding to the buffers passed to `irecv`. Once `test` returns 1 in `done`, the request handle can be freed, meaning that NCCL will never call `test` again on that request (until it is reallocated by another call to `isend` or `irecv`). `iflush` After a receive operation completes, if the operation was targeting GPU memory and received a non-zero number of bytes, NCCL will call `iflush` to let the network flush any buffer and ensure the GPU can read it right after without seeing stale data. This flush operation is decoupled from the `test` code to improve latency of `LL*` protocols, as those are capable of determining when data is valid or not. `iflush` returns a request which needs to be queried with `test` until it completes. nccl-2.29.7-1/plugins/net/example/000077500000000000000000000000001515037102200165665ustar00rootroot00000000000000nccl-2.29.7-1/plugins/net/example/CMakeLists.txt000066400000000000000000000007531515037102200213330ustar00rootroot00000000000000set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/plugin.c ) # Create shared library add_library(nccl-net-example SHARED ${SRC_FILES}) # Set include directories target_include_directories(nccl-net-example PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/nccl ) # Set output name to match Makefile set_target_properties(nccl-net-example PROPERTIES OUTPUT_NAME "nccl-net-example" PREFIX "lib" POSITION_INDEPENDENT_CODE ON LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test/unit/plugins ) nccl-2.29.7-1/plugins/net/example/Makefile000066400000000000000000000011251515037102200202250ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # .DEFAULT_GOAL: build include ../../../makefiles/common.mk SRCDIR ?= $(abspath ../../..) BUILDDIR ?= . NCCLDIR := $(BUILDDIR) SRC_FILES := $(wildcard *.c) build: ${BUILDDIR}/libnccl-net-example.so ${BUILDDIR}/libnccl-net-example.so: ${SRC_FILES} @printf "Compiling %-35s > %s\n" $< $@ @mkdir -p ${BUILDDIR} $(CC) -Inccl -fPIC -shared -o $@ $^ clean: rm -f ${BUILDDIR}/libnccl-net-example.so nccl-2.29.7-1/plugins/net/example/nccl/000077500000000000000000000000001515037102200175055ustar00rootroot00000000000000nccl-2.29.7-1/plugins/net/example/nccl/common.h000066400000000000000000000022511515037102200211460ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef COMMON_H_ #define COMMON_H_ #include typedef enum {NCCL_LOG_NONE=0, NCCL_LOG_VERSION=1, NCCL_LOG_WARN=2, NCCL_LOG_INFO=3, NCCL_LOG_ABORT=4, NCCL_LOG_TRACE=5} ncclDebugLogLevel; typedef enum {NCCL_INIT=1, NCCL_COLL=2, NCCL_P2P=4, NCCL_SHM=8, NCCL_NET=16, NCCL_GRAPH=32, NCCL_TUNING=64, NCCL_ENV=128, NCCL_ALLOC=256, NCCL_CALL=512, NCCL_PROXY=1024, NCCL_NVLS=2048, NCCL_BOOTSTRAP=4096, NCCL_REG=8192, NCCL_ALL=~0} ncclDebugLogSubSys; typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags, const char *file, int line, const char *fmt, ...); enum { ncclProfilerNetEventStart = 0, ncclProfilerNetEventStop, ncclProfilerNetEventUpdate, ncclProfilerNetEventUpdateAndStop }; typedef ncclResult_t (*ncclProfilerCallback_t)(void** eHandle, int type, void* phandle, int64_t pluginId, void* extData); #endif nccl-2.29.7-1/plugins/net/example/nccl/err.h000066400000000000000000000014171515037102200204510ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_ERR_H_ #define NCCL_ERR_H_ /* Error type for plugins */ typedef enum { ncclSuccess = 0, ncclUnhandledCudaError = 1, ncclSystemError = 2, ncclInternalError = 3, ncclInvalidArgument = 4, ncclInvalidUsage = 5, ncclRemoteError = 6 } ncclResult_t; #endif nccl-2.29.7-1/plugins/net/example/nccl/net.h000066400000000000000000000023441515037102200204470ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_H_ #define NET_H_ #include #include #include "err.h" #include "net_device.h" #include "common.h" #define NCCL_NET_HANDLE_MAXSIZE 128 #define NCCL_MAX_NET_SIZE_BYTES (1*1024*1024*1024*1024L) //1TB #define NCCL_NET_OPTIONAL_RECV_COMPLETION 0x1 #define NCCL_PTR_HOST 0x1 #define NCCL_PTR_CUDA 0x2 #define NCCL_PTR_DMABUF 0x4 // Maximum number of requests per comm object #define NCCL_NET_MAX_REQUESTS 32 #define NCCL_NET_MAX_DEVS_PER_NIC 4 #include "net_v11.h" #include "net_v10.h" #include "net_v9.h" #include "net_v8.h" #include "net_v7.h" #include "net_v6.h" #include "net_v5.h" #include "net_v4.h" #include "net_v3.h" #include "net_v2.h" typedef ncclNet_v11_t ncclNet_t; typedef ncclNetProperties_v11_t ncclNetProperties_t; typedef ncclNetVDeviceProps_v11_t ncclNetVDeviceProps_t; typedef ncclNetCommConfig_v11_t ncclNetCommConfig_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/net_device.h000066400000000000000000000025561515037102200217730ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_DEVICE_H_ #define NET_DEVICE_H_ #define NCCL_NET_DEVICE_INVALID_VERSION 0x0 #define NCCL_NET_MTU_SIZE 4096 // Arbitrary version number - A given NCCL build will only be compatible with a single device networking plugin // version. NCCL will check the supplied version number from net->getProperties() and compare to its internal version. #define NCCL_NET_DEVICE_UNPACK_VERSION 0x7 typedef enum {NCCL_NET_DEVICE_HOST=0, NCCL_NET_DEVICE_UNPACK=1} ncclNetDeviceType; typedef struct { ncclNetDeviceType netDeviceType; // Network offload type int netDeviceVersion; // Version number for network offload void* handle; size_t size; int needsProxyProgress; } ncclNetDeviceHandle_v7_t; typedef ncclNetDeviceHandle_v7_t ncclNetDeviceHandle_v8_t; typedef ncclNetDeviceHandle_v8_t ncclNetDeviceHandle_v9_t; typedef ncclNetDeviceHandle_v9_t ncclNetDeviceHandle_v10_t; typedef ncclNetDeviceHandle_v10_t ncclNetDeviceHandle_v11_t; typedef ncclNetDeviceHandle_v11_t ncclNetDeviceHandle_t; #endif nccl-2.29.7-1/plugins/net/example/nccl/net_v10.h000066400000000000000000000127161515037102200211410ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_V10_H_ #define NET_V10_H_ typedef struct { int ndevs; int devs[NCCL_NET_MAX_DEVS_PER_NIC]; } ncclNetVDeviceProps_v10_t; #define NCCL_NET_TRAFFIC_CLASS_UNDEF -1 typedef struct { // Plugin-specific TC value int trafficClass; } ncclNetCommConfig_v10_t; typedef struct { char* name; // Used mostly for logging. char* pciPath; // Path to the PCI device in /sys. uint64_t guid; // Unique identifier for the NIC chip. Important for // cards with multiple PCI functions (Physical or virtual). int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF] int regIsGlobal; // regMr is not tied to a particular comm int forceFlush; // Force a flush on receives int speed; // Port speed in Mbps. int port; // Port number. float latency; // Network latency int maxComms; // Maximum number of comms we can create int maxRecvs; // Maximum number of grouped receives. ncclNetDeviceType netDeviceType; // Network offload type int netDeviceVersion; // Version number for network offload ncclNetVDeviceProps_v10_t vProps; size_t maxP2pBytes; // Max transfer size for point-to-point operations size_t maxCollBytes; // Max transfer size for collective operations } ncclNetProperties_v10_t; typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(ncclDebugLogger_t logFunction, ncclProfilerCallback_t profFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Get various device properties. ncclResult_t (*getProperties)(int dev, ncclNetProperties_v10_t* props); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. // This call must not block for the connection to be established, and instead // should return successfully with sendComm == NULL with the expectation that // it will be called again until sendComm != NULL. // If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection ncclResult_t (*connect)(int dev, ncclNetCommConfig_v10_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_v10_t** sendDevComm); // Finalize connection establishment after remote peer has called connect. // This call must not block for the connection to be established, and instead // should return successfully with recvComm == NULL with the expectation that // it will be called again until recvComm != NULL. // If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v10_t** recvDevComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle); /* DMA-BUF support */ ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* phandle, void** request); // Asynchronous recv from a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** phandles, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* sizes); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); // Copy the given mhandle to a dptr in a format usable by this plugin's device code ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle); // Notify the plugin that a recv has completed by the device ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request); // Virtual NIC APIs. makeVDevice will create a virtual NIC given the specified properties, and tell the caller // what index this new vNIC exists at ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v10_t* props); } ncclNet_v10_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/net_v11.h000066400000000000000000000140611515037102200211350ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_V11_H_ #define NET_V11_H_ typedef struct { int ndevs; int devs[NCCL_NET_MAX_DEVS_PER_NIC]; } ncclNetVDeviceProps_v11_t; #define NCCL_NET_TRAFFIC_CLASS_UNDEF -1 typedef struct { // Plugin-specific TC value int trafficClass; } ncclNetCommConfig_v11_t; typedef struct { char* name; // Used mostly for logging. char* pciPath; // Path to the PCI device in /sys. uint64_t guid; // Unique identifier for the NIC chip. Important for // cards with multiple PCI functions (Physical or virtual). int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF] int regIsGlobal; // regMr is not tied to a particular comm int forceFlush; // Force a flush on receives int speed; // Port speed in Mbps. int port; // Port number. float latency; // Network latency int maxComms; // Maximum number of comms we can create int maxRecvs; // Maximum number of grouped receives. ncclNetDeviceType netDeviceType; // Network offload type int netDeviceVersion; // Version number for network offload ncclNetVDeviceProps_v11_t vProps; size_t maxP2pBytes; // Max transfer size for point-to-point operations size_t maxCollBytes; // Max transfer size for collective operations int maxMultiRequestSize; // Maximum number of requests supported in a single multi-request. } ncclNetProperties_v11_t; typedef struct { int32_t maxConcurrentPeers; int32_t minConcurrentPeers; int32_t maxFlowsPerPeer; int32_t minFlowsPerPeer; } ncclNetCommAttr_v11_t; typedef struct { ncclNetCommAttr_v11_t sendCommAttr; ncclNetCommAttr_v11_t recvCommAttr; uint32_t op; uint32_t algo; uint32_t proto; } ncclNetAttr_v11_t; typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(void** ctx, uint64_t commId, ncclNetCommConfig_v11_t* config, ncclDebugLogger_t logFunction, ncclProfilerCallback_t profFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Get various device properties. ncclResult_t (*getProperties)(int dev, ncclNetProperties_v11_t* props); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(void* ctx, int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. // This call must not block for the connection to be established, and instead // should return successfully with sendComm == NULL with the expectation that // it will be called again until sendComm != NULL. // If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection ncclResult_t (*connect)(void* ctx, int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v11_t** sendDevComm); // Finalize connection establishment after remote peer has called connect. // This call must not block for the connection to be established, and instead // should return successfully with recvComm == NULL with the expectation that // it will be called again until recvComm != NULL. // If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v11_t** recvDevComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle); /* DMA-BUF support */ ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* phandle, void** request); // Asynchronous recv from a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** phandles, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* sizes); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); // Copy the given mhandle to a dptr in a format usable by this plugin's device code ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle); // Notify the plugin that a recv has completed by the device ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request); // Virtual NIC APIs. makeVDevice will create a virtual NIC given the specified properties, and tell the caller // what index this new vNIC exists at ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v11_t* props); // Finalize the network. ncclResult_t (*finalize)(void* ctx); ncclResult_t (*setNetAttr)(void* ctx, ncclNetAttr_v11_t* netAttr); } ncclNet_v11_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/net_v2.h000066400000000000000000000054421515037102200210600ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_V2_H_ #define NET_V2_H_ typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(ncclDebugLogger_t logFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Return the device path in /sys. NCCL will call free on this path. ncclResult_t (*pciPath)(int dev, char** path); // Return whether this device supports host pointers and/or CUDA pointers // as data from the current GPU. Supported types should be composed with // NCCL_PTR_HOST and NCCL_PTR_CUDA. ncclResult_t (*ptrSupport)(int dev, int* supportedTypes); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. ncclResult_t (*connect)(int dev, void* handle, void** sendComm); // Finalize connection establishment after remote peer has called connectHandle ncclResult_t (*accept)(void* listenComm, void** recvComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, int size, void* mhandle, void** request); // Asynchronous recv from a peer. Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, void* data, int size, void* mhandle, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*flush)(void* recvComm, void* data, int size, void* mhandle); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* size); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); } ncclNet_v2_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/net_v3.h000066400000000000000000000051201515037102200210520ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_V3_H_ #define NET_V3_H_ #define NCCL_NET_MAX_REQUESTS_V3 16 typedef ncclNetProperties_v4_t ncclNetProperties_v3_t; typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(ncclDebugLogger_t logFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Get various device properties. ncclResult_t (*getProperties)(int dev, ncclNetProperties_v3_t* props); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. ncclResult_t (*connect)(int dev, void* handle, void** sendComm); // Finalize connection establishment after remote peer has called connectHandle ncclResult_t (*accept)(void* listenComm, void** recvComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, int size, void* mhandle, void** request); // Asynchronous recv from a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, void* data, int size, void* mhandle, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*flush)(void* recvComm, void* data, int size, void* mhandle); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* size); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); } ncclNet_v3_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/net_v4.h000066400000000000000000000061011515037102200210530ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_V4_H_ #define NET_V4_H_ #define NCCL_NET_HANDLE_MAXSIZE_V4 64 typedef struct { char* name; // Used mostly for logging. char* pciPath; // Path to the PCI device in /sys. uint64_t guid; // Unique identifier for the NIC chip. Important for // cards with multiple PCI functions (Physical or virtual). int ptrSupport; // NCCL_PTR_HOST or NCCL_PTR_HOST|NCCL_PTR_CUDA int speed; // Port speed in Mbps. int port; // Port number. int maxComms; // Maximum number of comms we can create } ncclNetProperties_v4_t; // v4 struct for backwards compatibility typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(ncclDebugLogger_t logFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Get various device properties. ncclResult_t (*getProperties)(int dev, ncclNetProperties_v4_t* props); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. ncclResult_t (*connect)(int dev, void* handle, void** sendComm); // Finalize connection establishment after remote peer has called connectHandle ncclResult_t (*accept)(void* listenComm, void** recvComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, int size, void* mhandle, void** request); // Asynchronous recv from a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, void* data, int size, void* mhandle, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*iflush)(void* recvComm, void* data, int size, void* mhandle, void** request); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* size); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); } ncclNet_v4_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/net_v5.h000066400000000000000000000060161515037102200210610ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_V5_H_ #define NET_V5_H_ typedef ncclNetProperties_v6_t ncclNetProperties_v5_t; typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(ncclDebugLogger_t logFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Get various device properties. ncclResult_t (*getProperties)(int dev, ncclNetProperties_v5_t* props); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. // This call must not block for the connection to be established, and instead // should return successfully with sendComm == NULL with the expectation that // it will be called again until sendComm != NULL. ncclResult_t (*connect)(int dev, void* handle, void** sendComm); // Finalize connection establishment after remote peer has called connect. // This call must not block for the connection to be established, and instead // should return successfully with recvComm == NULL with the expectation that // it will be called again until recvComm != NULL. ncclResult_t (*accept)(void* listenComm, void** recvComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request); // Asynchronous recv from a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* sizes); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); } ncclNet_v5_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/net_v6.h000066400000000000000000000072601515037102200210640ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_V6_H_ #define NET_V6_H_ typedef struct { char* name; // Used mostly for logging. char* pciPath; // Path to the PCI device in /sys. uint64_t guid; // Unique identifier for the NIC chip. Important for // cards with multiple PCI functions (Physical or virtual). int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF] int speed; // Port speed in Mbps. int port; // Port number. float latency; // Network latency int maxComms; // Maximum number of comms we can create int maxRecvs; // Maximum number of grouped receives. }ncclNetProperties_v6_t; typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(ncclDebugLogger_t logFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Get various device properties. ncclResult_t (*getProperties)(int dev, ncclNetProperties_v6_t* props); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. // This call must not block for the connection to be established, and instead // should return successfully with sendComm == NULL with the expectation that // it will be called again until sendComm != NULL. ncclResult_t (*connect)(int dev, void* handle, void** sendComm); // Finalize connection establishment after remote peer has called connect. // This call must not block for the connection to be established, and instead // should return successfully with recvComm == NULL with the expectation that // it will be called again until recvComm != NULL. ncclResult_t (*accept)(void* listenComm, void** recvComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle); /* DMA-BUF support */ ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request); // Asynchronous recv from a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* sizes); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); } ncclNet_v6_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/net_v7.h000066400000000000000000000105331515037102200210620ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_V7_H_ #define NET_V7_H_ typedef struct { char* name; // Used mostly for logging. char* pciPath; // Path to the PCI device in /sys. uint64_t guid; // Unique identifier for the NIC chip. Important for // cards with multiple PCI functions (Physical or virtual). int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF] int speed; // Port speed in Mbps. int port; // Port number. float latency; // Network latency int maxComms; // Maximum number of comms we can create int maxRecvs; // Maximum number of grouped receives. ncclNetDeviceType netDeviceType; // Network offload type int netDeviceVersion; // Version number for network offload } ncclNetProperties_v7_t; typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(ncclDebugLogger_t logFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Get various device properties. ncclResult_t (*getProperties)(int dev, ncclNetProperties_v7_t* props); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. // This call must not block for the connection to be established, and instead // should return successfully with sendComm == NULL with the expectation that // it will be called again until sendComm != NULL. ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v7_t** sendDevComm); // Finalize connection establishment after remote peer has called connect. // This call must not block for the connection to be established, and instead // should return successfully with recvComm == NULL with the expectation that // it will be called again until recvComm != NULL. ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v7_t** recvDevComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. ncclResult_t (*regMr)(void* comm, void* data, int size, int type, void** mhandle); /* DMA-BUF support */ ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request); // Asynchronous recv from a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* sizes); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); // Copy the given mhandle to a dptr in a format usable by this plugin's device code ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle); // Notify the plugin that a recv has completed by the device ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request); } ncclNet_v7_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/net_v8.h000066400000000000000000000112001515037102200210530ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_V8_H_ #define NET_V8_H_ typedef struct { char* name; // Used mostly for logging. char* pciPath; // Path to the PCI device in /sys. uint64_t guid; // Unique identifier for the NIC chip. Important for // cards with multiple PCI functions (Physical or virtual). int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF] int regIsGlobal; // regMr is not tied to a particular comm int speed; // Port speed in Mbps. int port; // Port number. float latency; // Network latency int maxComms; // Maximum number of comms we can create int maxRecvs; // Maximum number of grouped receives. ncclNetDeviceType netDeviceType; // Network offload type int netDeviceVersion; // Version number for network offload } ncclNetProperties_v8_t; typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(ncclDebugLogger_t logFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Get various device properties. ncclResult_t (*getProperties)(int dev, ncclNetProperties_v8_t* props); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. // This call must not block for the connection to be established, and instead // should return successfully with sendComm == NULL with the expectation that // it will be called again until sendComm != NULL. // If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v8_t** sendDevComm); // Finalize connection establishment after remote peer has called connect. // This call must not block for the connection to be established, and instead // should return successfully with recvComm == NULL with the expectation that // it will be called again until recvComm != NULL. // If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v8_t** recvDevComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle); /* DMA-BUF support */ ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, int size, int tag, void* mhandle, void** request); // Asynchronous recv from a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* sizes); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); // Copy the given mhandle to a dptr in a format usable by this plugin's device code ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle); // Notify the plugin that a recv has completed by the device ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request); } ncclNet_v8_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/net_v9.h000066400000000000000000000123251515037102200210650ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_V9_H_ #define NET_V9_H_ typedef struct { int ndevs; int devs[NCCL_NET_MAX_DEVS_PER_NIC]; } ncclNetVDeviceProps_v9_t; typedef struct { char* name; // Used mostly for logging. char* pciPath; // Path to the PCI device in /sys. uint64_t guid; // Unique identifier for the NIC chip. Important for // cards with multiple PCI functions (Physical or virtual). int ptrSupport; // [NCCL_PTR_HOST|NCCL_PTR_CUDA|NCCL_PTR_DMABUF] int regIsGlobal; // regMr is not tied to a particular comm int forceFlush; // Force a flush on receives int speed; // Port speed in Mbps. int port; // Port number. float latency; // Network latency int maxComms; // Maximum number of comms we can create int maxRecvs; // Maximum number of grouped receives. ncclNetDeviceType netDeviceType; // Network offload type int netDeviceVersion; // Version number for network offload ncclNetVDeviceProps_v9_t vProps; size_t maxP2pBytes; // Max transfer size for point-to-point operations size_t maxCollBytes; // Max transfer size for collective operations } ncclNetProperties_v9_t; typedef struct { // Name of the network (mainly for logs) const char* name; // Initialize the network. ncclResult_t (*init)(ncclDebugLogger_t logFunction); // Return the number of adapters. ncclResult_t (*devices)(int* ndev); // Get various device properties. ncclResult_t (*getProperties)(int dev, ncclNetProperties_v9_t* props); // Create a receiving object and provide a handle to connect to it. The // handle can be up to NCCL_NET_HANDLE_MAXSIZE bytes and will be exchanged // between ranks to create a connection. ncclResult_t (*listen)(int dev, void* handle, void** listenComm); // Connect to a handle and return a sending comm object for that peer. // This call must not block for the connection to be established, and instead // should return successfully with sendComm == NULL with the expectation that // it will be called again until sendComm != NULL. // If *sendDevComm points to a valid object, then NCCL is requesting device offload for this connection ncclResult_t (*connect)(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_v9_t** sendDevComm); // Finalize connection establishment after remote peer has called connect. // This call must not block for the connection to be established, and instead // should return successfully with recvComm == NULL with the expectation that // it will be called again until recvComm != NULL. // If *recvDevComm points to a valid object, then NCCL is requesting device offload for this connection ncclResult_t (*accept)(void* listenComm, void** recvComm, ncclNetDeviceHandle_v9_t** recvDevComm); // Register/Deregister memory. Comm can be either a sendComm or a recvComm. // Type is either NCCL_PTR_HOST or NCCL_PTR_CUDA. ncclResult_t (*regMr)(void* comm, void* data, size_t size, int type, void** mhandle); /* DMA-BUF support */ ncclResult_t (*regMrDmaBuf)(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle); ncclResult_t (*deregMr)(void* comm, void* mhandle); // Asynchronous send to a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*isend)(void* sendComm, void* data, size_t size, int tag, void* mhandle, void** request); // Asynchronous recv from a peer. // May return request == NULL if the call cannot be performed (or would block) ncclResult_t (*irecv)(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** request); // Perform a flush/fence to make sure all data received with NCCL_PTR_CUDA is // visible to the GPU ncclResult_t (*iflush)(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request); // Test whether a request is complete. If size is not NULL, it returns the // number of bytes sent/received. ncclResult_t (*test)(void* request, int* done, int* sizes); // Close and free send/recv comm objects ncclResult_t (*closeSend)(void* sendComm); ncclResult_t (*closeRecv)(void* recvComm); ncclResult_t (*closeListen)(void* listenComm); // Copy the given mhandle to a dptr in a format usable by this plugin's device code ncclResult_t (*getDeviceMr)(void* comm, void* mhandle, void** dptr_mhandle); // Notify the plugin that a recv has completed by the device ncclResult_t (*irecvConsumed)(void* recvComm, int n, void* request); // Virtual NIC APIs. makeVDevice will create a virtual NIC given the specified properties, and tell the caller // what index this new vNIC exists at ncclResult_t (*makeVDevice)(int* d, ncclNetVDeviceProps_v9_t* props); } ncclNet_v9_t; #endif // end include guard nccl-2.29.7-1/plugins/net/example/nccl/types.h000066400000000000000000000015651515037102200210310ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_TYPES_H_ #define NCCL_TYPES_H_ /* Data types */ typedef enum { ncclInt8 = 0, ncclChar = 0, ncclUint8 = 1, ncclInt32 = 2, ncclInt = 2, ncclUint32 = 3, ncclInt64 = 4, ncclUint64 = 5, ncclFloat16 = 6, ncclHalf = 6, ncclFloat32 = 7, ncclFloat = 7, ncclFloat64 = 8, ncclDouble = 8, ncclBfloat16 = 9, } ncclDataType_t; #endif nccl-2.29.7-1/plugins/net/example/plugin.c000066400000000000000000000454021515037102200202350ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "net.h" #define __hidden __attribute__ ((visibility("hidden"))) #define NCCL_PLUGIN_MAX_RECVS 1 int max_requests = NCCL_NET_MAX_REQUESTS; __hidden ncclResult_t pluginInit(void** ctx, uint64_t commId, ncclNetCommConfig_t* config, ncclDebugLogger_t logFunction, ncclProfilerCallback_t profFunction) { return ncclSuccess; } __hidden ncclResult_t pluginDevices(int* ndev) { *ndev = 0; return ncclSuccess; } __hidden ncclResult_t pluginPciPath(int dev, char** path) { return ncclInternalError; } __hidden ncclResult_t pluginPtrSupport(int dev, int* supportedTypes) { return ncclInternalError; } __hidden ncclResult_t pluginGetProperties(int dev, ncclNetProperties_t* props) { // Below are default values, if unsure don't change. props->name = "Example"; // Fill for proper topology detection, e.g. /sys/devices/pci0000:00/0000:00:10.0/0000:0b:00.0 props->pciPath = NULL; // Only used to detect NICs with multiple PCI attachments. props->guid = 0; // Add NCCL_PTR_CUDA if GPU Direct RDMA is supported and regMr can take CUDA pointers. props->ptrSupport = NCCL_PTR_HOST; // If you regMr has a fast registration cache, set to 1. If set to 0, user buffer registration may be disabled. props->regIsGlobal = 0; // Force flush after receive. Needed if the control path and data path use a different path to the GPU props->forceFlush = 0; // Speed in *Mbps*. 100000 means 100G props->speed = 100000; // Port number, used in conjunction with guid props->port = 0; // Custom latency (used to help tuning if latency is high. If set to 0, use default NCCL values. props->latency = 0; // Maximum number of comm objects we can create. props->maxComms = 1024*1024; // Maximum number of receive operations taken by irecv(). props->maxRecvs = NCCL_PLUGIN_MAX_RECVS; // Coupling with NCCL network device-side code. props->netDeviceType = NCCL_NET_DEVICE_HOST; props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION; // Used to tell NCCL core whether this is a virtual device fusing multiple physical devices. props->vProps.ndevs = 1; props->vProps.devs[0] = dev; // maximum transfer sizes the plugin can handle props->maxP2pBytes = NCCL_MAX_NET_SIZE_BYTES; props->maxCollBytes = NCCL_MAX_NET_SIZE_BYTES; return ncclSuccess; } __hidden ncclResult_t pluginListen(void* ctx, int dev, void* handle, void** listenComm) { return ncclInternalError; } __hidden ncclResult_t pluginConnect(void* ctx, int dev, void* handle, void** sendComm, ncclNetDeviceHandle_t** sendDevComm) { return ncclInternalError; } __hidden ncclResult_t pluginAccept(void* listenComm, void** recvComm, ncclNetDeviceHandle_t** recvDevComm) { return ncclInternalError; } __hidden ncclResult_t pluginRegMr(void* collComm, void* data, size_t size, int type, void** mhandle) { return ncclInternalError; } __hidden ncclResult_t pluginRegMrDmaBuf(void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle) { return ncclInternalError; } __hidden ncclResult_t pluginDeregMr(void* collComm, void* mhandle) { return ncclInternalError;} __hidden ncclResult_t pluginIsend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* phandle, void** request) { return ncclInternalError; } __hidden ncclResult_t pluginIrecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** phandles, void** request) { return ncclInternalError; } __hidden ncclResult_t pluginIflush(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request) { return ncclInternalError; } __hidden ncclResult_t pluginTest(void* request, int* done, int* size) { return ncclInternalError; } __hidden ncclResult_t pluginCloseSend(void* sendComm) { return ncclInternalError; } __hidden ncclResult_t pluginCloseRecv(void* recvComm) { return ncclInternalError; } __hidden ncclResult_t pluginCloseListen(void* listenComm) { return ncclInternalError; } __hidden ncclResult_t pluginIrecvConsumed(void* recvComm, int n, void* request) { return ncclInternalError; } __hidden ncclResult_t pluginGetDeviceMr(void* comm, void* mhandle, void** dptr_mhandle) { return ncclInternalError; } __hidden ncclResult_t pluginMakeVDevice(int* d, ncclNetVDeviceProps_t* props) { return ncclInternalError; } __hidden ncclResult_t pluginFinalize(void* ctx) { return ncclSuccess; } #define PLUGIN_NAME "Plugin" const ncclNet_v11_t ncclNetPlugin_v11 = { .name = PLUGIN_NAME, .init = pluginInit, .devices = pluginDevices, .getProperties = pluginGetProperties, .listen = pluginListen, .connect = pluginConnect, .accept = pluginAccept, .regMr = pluginRegMr, .regMrDmaBuf = pluginRegMrDmaBuf, .deregMr = pluginDeregMr, .isend = pluginIsend, .irecv = pluginIrecv, .iflush = pluginIflush, .test = pluginTest, .closeSend = pluginCloseSend, .closeRecv = pluginCloseRecv, .closeListen = pluginCloseListen, .getDeviceMr = pluginGetDeviceMr, .irecvConsumed = pluginIrecvConsumed, .makeVDevice = pluginMakeVDevice, .finalize = pluginFinalize, }; __hidden ncclResult_t pluginInit_v10(ncclDebugLogger_t logFunction, ncclProfilerCallback_t profFunction) { return ncclSuccess; } __hidden ncclResult_t pluginGetProperties_v10(int dev, ncclNetProperties_v10_t* props) { // Below are default values, if unsure don't change. props->name = "Example"; // Fill for proper topology detection, e.g. /sys/devices/pci0000:00/0000:00:10.0/0000:0b:00.0 props->pciPath = NULL; // Only used to detect NICs with multiple PCI attachments. props->guid = 0; // Add NCCL_PTR_CUDA if GPU Direct RDMA is supported and regMr can take CUDA pointers. props->ptrSupport = NCCL_PTR_HOST; // If you regMr has a fast registration cache, set to 1. If set to 0, user buffer registration may be disabled. props->regIsGlobal = 0; // Force flush after receive. Needed if the control path and data path use a different path to the GPU props->forceFlush = 0; // Speed in *Mbps*. 100000 means 100G props->speed = 100000; // Port number, used in conjunction with guid props->port = 0; // Custom latency (used to help tuning if latency is high. If set to 0, use default NCCL values. props->latency = 0; // Maximum number of comm objects we can create. props->maxComms = 1024*1024; // Maximum number of receive operations taken by irecv(). props->maxRecvs = NCCL_PLUGIN_MAX_RECVS; // Coupling with NCCL network device-side code. props->netDeviceType = NCCL_NET_DEVICE_HOST; props->netDeviceVersion = NCCL_NET_DEVICE_INVALID_VERSION; // Used to tell NCCL core whether this is a virtual device fusing multiple physical devices. props->vProps.ndevs = 1; props->vProps.devs[0] = dev; // maximum transfer sizes the plugin can handle props->maxP2pBytes = NCCL_MAX_NET_SIZE_BYTES; props->maxCollBytes = NCCL_MAX_NET_SIZE_BYTES; return ncclSuccess; } __hidden ncclResult_t pluginListen_v10(int d, void* handle, void** listenComm) { return ncclInternalError; } __hidden ncclResult_t pluginConnect_v10(int dev, ncclNetCommConfig_v10_t* config, void* handle, void** sendComm, ncclNetDeviceHandle_v10_t** sendDevComm) { return ncclInternalError; } __hidden ncclResult_t pluginMakeVDevice_v10(int* d, ncclNetVDeviceProps_v10_t* props) { return ncclInternalError; } const ncclNet_v10_t ncclNetPlugin_v10 = { .name = PLUGIN_NAME, .init = pluginInit_v10, .devices = pluginDevices, .getProperties = pluginGetProperties_v10, .listen = pluginListen_v10, .connect = pluginConnect_v10, .accept = pluginAccept, .regMr = pluginRegMr, .regMrDmaBuf = pluginRegMrDmaBuf, .deregMr = pluginDeregMr, .isend = pluginIsend, .irecv = pluginIrecv, .iflush = pluginIflush, .test = pluginTest, .closeSend = pluginCloseSend, .closeRecv = pluginCloseRecv, .closeListen = pluginCloseListen, .getDeviceMr = pluginGetDeviceMr, .irecvConsumed = pluginIrecvConsumed, .makeVDevice = pluginMakeVDevice_v10, }; __hidden ncclResult_t pluginInit_v9(ncclDebugLogger_t logFunction) { return pluginInit_v10(logFunction, NULL); } __hidden ncclResult_t pluginGetProperties_v9(int dev, ncclNetProperties_v9_t* props) { return pluginGetProperties_v10(dev, (ncclNetProperties_v10_t*)props); } __hidden ncclResult_t pluginConnect_v9(int dev, void* handle, void** sendComm, ncclNetDeviceHandle_t** sendDevComm){ return pluginConnect_v10(dev, NULL, handle, sendComm, sendDevComm); } __hidden ncclResult_t pluginIsend_v9(void* sendComm, void* data, size_t size, int tag, void* mhandle, void** request) { return pluginIsend(sendComm, data, size, tag, mhandle, NULL, request); } __hidden ncclResult_t pluginIrecv_v9(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** request) { return pluginIrecv(recvComm, n, data, sizes, tags, mhandles, NULL, request); } __hidden ncclResult_t pluginMakeVDevice_v9(int* d, ncclNetVDeviceProps_v9_t* props) { return ncclInternalError; } const ncclNet_v9_t ncclNetPlugin_v9 = { .name = PLUGIN_NAME, .init = pluginInit_v9, .devices = pluginDevices, .getProperties = pluginGetProperties_v9, .listen = pluginListen_v10, .connect = pluginConnect_v9, .accept = pluginAccept, .regMr = pluginRegMr, .regMrDmaBuf = pluginRegMrDmaBuf, .deregMr = pluginDeregMr, .isend = pluginIsend_v9, .irecv = pluginIrecv_v9, .iflush = pluginIflush, .test = pluginTest, .closeSend = pluginCloseSend, .closeRecv = pluginCloseRecv, .closeListen = pluginCloseListen, .getDeviceMr = pluginGetDeviceMr, .irecvConsumed = pluginIrecvConsumed, .makeVDevice = pluginMakeVDevice_v9, }; __hidden ncclResult_t pluginGetProperties_v8(int dev, ncclNetProperties_v8_t* props_v8) { ncclNetProperties_t props; ncclResult_t ret = pluginGetProperties(dev, &props); if (ret != ncclSuccess) return ret; props_v8->name = props.name; props_v8->pciPath = props.pciPath; props_v8->guid = props.guid; props_v8->ptrSupport = props.ptrSupport; props_v8->regIsGlobal = props.regIsGlobal; props_v8->speed = props.speed; props_v8->latency = props.latency; props_v8->port = props.port; props_v8->maxComms = props.maxComms; props_v8->maxRecvs = props.maxRecvs; props_v8->netDeviceType = props.netDeviceType; props_v8->netDeviceVersion = props.netDeviceVersion; return ncclSuccess; } __hidden ncclResult_t pluginIsend_v8(void* sendComm, void* data, int size, int tag, void* mhandle, void** request) { return pluginIsend(sendComm, data, (int)size, tag, mhandle, NULL, request); } __hidden ncclResult_t pluginIrecv_v8(void* recvComm, int n, void** data, int* sizes, int* tags, void** mhandles, void** request) { size_t sizesOut[NCCL_PLUGIN_MAX_RECVS]; for (int i=0; iname = props.name; props_v7->pciPath = props.pciPath; props_v7->guid = props.guid; props_v7->ptrSupport = props.ptrSupport; props_v7->speed = props.speed; props_v7->latency = props.latency; props_v7->port = props.port; props_v7->maxComms = props.maxComms; props_v7->maxRecvs = props.maxRecvs; props_v7->netDeviceType = props.netDeviceType; props_v7->netDeviceVersion = props.netDeviceVersion; return ncclSuccess; } __hidden ncclResult_t pluginRegMr_v7(void* collComm, void* data, int size, int type, void** mhandle) { return pluginRegMr(collComm, data, size, type, mhandle); } const ncclNet_v7_t ncclNetPlugin_v7 = { .name = PLUGIN_NAME, .init = pluginInit_v9, .devices = pluginDevices, .getProperties = pluginGetProperties_v7, .listen = pluginListen_v10, .connect = pluginConnect_v9, .accept = pluginAccept, .regMr = pluginRegMr_v7, .regMrDmaBuf = pluginRegMrDmaBuf, .deregMr = pluginDeregMr, .isend = pluginIsend_v8, .irecv = pluginIrecv_v8, .iflush = pluginIflush, .test = pluginTest, .closeSend = pluginCloseSend, .closeRecv = pluginCloseRecv, .closeListen = pluginCloseListen, .getDeviceMr = pluginGetDeviceMr, .irecvConsumed = pluginIrecvConsumed, }; __hidden ncclResult_t pluginGetProperties_v6(int dev, ncclNetProperties_v6_t* props_v6) { ncclNetProperties_t props; ncclResult_t ret = pluginGetProperties(dev, &props); if (ret != ncclSuccess) return ret; props_v6->name = props.name; props_v6->pciPath = props.pciPath; props_v6->guid = props.guid; props_v6->ptrSupport = props.ptrSupport; props_v6->speed = props.speed; props_v6->latency = props.latency; props_v6->port = props.port; props_v6->maxComms = props.maxComms; props_v6->maxRecvs = props.maxRecvs; return ncclSuccess; } __hidden ncclResult_t pluginConnect_v6(int dev, void* handle, void** sendComm) { return ncclInternalError; } __hidden ncclResult_t pluginAccept_v6(void* listenComm, void** recvComm) { return ncclInternalError; } const ncclNet_v6_t ncclNetPlugin_v6 = { .name = PLUGIN_NAME, .init = pluginInit_v9, .devices = pluginDevices, .getProperties = pluginGetProperties_v6, .listen = pluginListen_v10, .connect = pluginConnect_v6, .accept = pluginAccept_v6, .regMr = pluginRegMr_v7, .regMrDmaBuf = pluginRegMrDmaBuf, .deregMr = pluginDeregMr, .isend = pluginIsend_v8, .irecv = pluginIrecv_v8, .iflush = pluginIflush, .test = pluginTest, .closeSend = pluginCloseSend, .closeRecv = pluginCloseRecv, .closeListen = pluginCloseListen }; /* v5 Compat */ const ncclNet_v5_t ncclNetPlugin_v5 = { .name = PLUGIN_NAME, .init = pluginInit_v9, .devices = pluginDevices, .getProperties = pluginGetProperties_v6, .listen = pluginListen_v10, .connect = pluginConnect_v6, .accept = pluginAccept_v6, .regMr = pluginRegMr_v7, .deregMr = pluginDeregMr, .isend = pluginIsend_v8, .irecv = pluginIrecv_v8, .iflush = pluginIflush, .test = pluginTest, .closeSend = pluginCloseSend, .closeRecv = pluginCloseRecv, .closeListen = pluginCloseListen, }; /* v4 Compat */ static ncclResult_t pluginGetProperties_v4(int dev, ncclNetProperties_v4_t* props_v4) { ncclNetProperties_t props; ncclResult_t ret = pluginGetProperties(dev, &props); if (ret != ncclSuccess) return ret; props_v4->name = props.name; props_v4->pciPath = props.pciPath; props_v4->guid = props.guid; props_v4->ptrSupport = props.ptrSupport; props_v4->speed = props.speed; props_v4->port = props.port; props_v4->maxComms = props.maxComms; return ncclSuccess; } static ncclResult_t pluginIsend_v4(void *sendComm, void* data, int size, void *mhandle, void** request) { return pluginIsend_v8(sendComm, data, size, 0, mhandle, request); } static ncclResult_t pluginIrecv_v4(void* recvComm, void* data, int size, void* mhandle, void** request) { int tag = 0; return pluginIrecv_v8(recvComm, 1, &data, &size, &tag, &mhandle, request); } static ncclResult_t pluginIflush_v4(void* recvComm, void* data, int size, void* mhandle, void** request) { return pluginIflush(recvComm, 1, &data, &size, &mhandle, request); } static ncclResult_t pluginConnect_v4(int dev, void* handle, void** sendComm) { ncclResult_t ret; do { ncclNetDeviceHandle_v7_t* handle = NULL; ret = pluginConnect_v10(dev, NULL, handle, sendComm, &handle); } while (ret == ncclSuccess && *sendComm == NULL); return ret; } static ncclResult_t pluginAccept_v4(void* listenComm, void** recvComm) { ncclResult_t ret; do { ncclNetDeviceHandle_v7_t* handle = NULL; ret = pluginAccept(listenComm, recvComm, &handle); } while (ret == ncclSuccess && *recvComm == NULL); return ret; } const ncclNet_v4_t ncclNetPlugin_v4 = { .name = PLUGIN_NAME, .init = pluginInit_v9, .devices = pluginDevices, .getProperties = pluginGetProperties_v4, .listen = pluginListen_v10, .connect = pluginConnect_v4, .accept = pluginAccept_v4, .regMr = pluginRegMr_v7, .deregMr = pluginDeregMr, .isend = pluginIsend_v4, .irecv = pluginIrecv_v4, .iflush = pluginIflush_v4, .test = pluginTest, .closeSend = pluginCloseSend, .closeRecv = pluginCloseRecv, .closeListen = pluginCloseListen, }; /* v3 Compat */ static ncclResult_t pluginFlush(void* recvComm, void* data, int size, void* mhandle) { void* req; ncclResult_t ret = pluginIflush_v4(recvComm, data, size, mhandle, &req); int done = 0; while (ret == ncclSuccess && done == 0) { ret = pluginTest(req, &done, NULL); } return ret; } static ncclResult_t pluginInit_v3(ncclDebugLogger_t logFunction) { max_requests = NCCL_NET_MAX_REQUESTS_V3; return pluginInit_v10(logFunction, NULL); } #include static ncclResult_t pluginListen_v3(int dev, void* handle, void** listenComm) { char pluginHandle[NCCL_NET_HANDLE_MAXSIZE]; ncclResult_t ret = pluginListen_v10(dev, &pluginHandle, listenComm); memcpy(handle, &pluginHandle, NCCL_NET_HANDLE_MAXSIZE_V4); return ret; } static ncclResult_t pluginConnect_v3(int dev, void* handle, void** sendComm) { char pluginHandle[NCCL_NET_HANDLE_MAXSIZE]; memcpy(&pluginHandle, handle, NCCL_NET_HANDLE_MAXSIZE_V4); return pluginConnect_v4(dev, &pluginHandle, sendComm); } const ncclNet_v3_t ncclNetPlugin_v3 = { .name = PLUGIN_NAME, .init = pluginInit_v3, .devices = pluginDevices, .getProperties = pluginGetProperties_v4, .listen = pluginListen_v3, .connect = pluginConnect_v3, .accept = pluginAccept_v4, .regMr = pluginRegMr_v7, .deregMr = pluginDeregMr, .isend = pluginIsend_v4, .irecv = pluginIrecv_v4, .flush = pluginFlush, .test = pluginTest, .closeSend = pluginCloseSend, .closeRecv = pluginCloseRecv, .closeListen = pluginCloseListen, }; /* v2 Compat */ const ncclNet_v2_t ncclNetPlugin_v2 = { .name = PLUGIN_NAME, .init = pluginInit_v3, .devices = pluginDevices, .pciPath = pluginPciPath, .ptrSupport = pluginPtrSupport, .listen = pluginListen_v3, .connect = pluginConnect_v4, .accept = pluginAccept_v4, .regMr = pluginRegMr_v7, .deregMr = pluginDeregMr, .isend = pluginIsend_v4, .irecv = pluginIrecv_v4, .flush = pluginFlush, .test = pluginTest, .closeSend = pluginCloseSend, .closeRecv = pluginCloseRecv, .closeListen = pluginCloseListen, }; nccl-2.29.7-1/plugins/net/google-fastsocket/000077500000000000000000000000001515037102200205535ustar00rootroot00000000000000nccl-2.29.7-1/plugins/net/google-fastsocket/Makefile000066400000000000000000000007701515037102200222170ustar00rootroot00000000000000CUDA_HOME?=/usr/local/cuda INC:=-I$(CUDA_HOME)/include PLUGIN_SO:=libnccl-net.so default: $(PLUGIN_SO) $(PLUGIN_SO): nccl-fastsocket/*.cc $(CC) $(INC) -fPIC -shared -o $@ -Wl,-soname,$(PLUGIN_SO) $^ nccl-fastsocket/*.cc: git clone https://github.com/google/nccl-fastsocket.git install: $(BUILDDIR)/lib/$(PLUGIN_SO) $(BUILDDIR)/lib/$(PLUGIN_SO): $(PLUGIN_SO) @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(BUILDDIR)/lib install -m 644 $< $@ clean: rm -f $(PLUGIN_SO) rm -Rf nccl-fastsocket nccl-2.29.7-1/plugins/profiler/000077500000000000000000000000001515037102200161675ustar00rootroot00000000000000nccl-2.29.7-1/plugins/profiler/README.md000066400000000000000000000600001515037102200174420ustar00rootroot00000000000000# NCCL Profiler Plugin Documentation This page describes the NCCL Profiler plugin API and how to implement a profiler plugin for NCCL. # Overview To allow NCCL to better integrate with DL frameworks, NCCL v2.23 introduced a profiler plugin interface. Any NCCL user can write profiler plugins to extract performance data from NCCL and use it for debugging and analysis. Similarly to other plugins (e.g., network plugin), the profiler plugins come as a shared library called `libnccl-profiler.so`. That shared library contains one or more implementations of the NCCL PROFILER API, in the form of versioned structs, filled with pointers to all required functions. # Plugin architecture ## Plugin name and supporting multiple profiler plugins When NCCL is initialized, it will look for a `libnccl-profiler.so` library and dynamically load it, then look for symbols inside the library. The `NCCL_PROFILER_PLUGIN` environment variable allows multiple plugins to coexist. If set, NCCL will look for a library with a name of `libnccl-profiler-${NCCL_PROFILER_PLUGIN}.so`. It is therefore advised to name the library following that pattern, with a symlink pointing `libnccl-profiler.so` to `libnccl-profiler-${NCCL_PROFILER_PLUGIN}.so`. That way, if there are multiple plugins in the path, setting `NCCL_PROFILER_PLUGIN` will allow users to select the right plugin. Alternatively, the user can also set `NCCL_PROFILER_PLUGIN` to the pathname of the `libnccl-profiler.so` library. ## Struct versioning Once a library is found, NCCL will look for a symbol named `ncclProfiler_vX`, with `X` increasing over time. The versioning ensures that the plugin and the NCCL core are compatible. Plugins are encouraged to provide multiple of those symbols, implementing multiple versions of the NCCL PROFILER API, so that the same plugin can be compiled and support a wide range of NCCL versions. Conversely, and to ease transition, NCCL can choose to support different plugin versions, looking for the latest ncclProfiler struct version, but also looking for older ones so that older plugins would still work. ## Headers management To help users build plugins effortlessly, plugins should copy the `ncclProfiler_vX` definitions they support to their internal includes. An example is shown in `plugins/profiler/example` where we keep all headers in the `nccl/` directory and provide thin layers to implement old version on top of newer ones. The `nccl/` directory is populated with `profiler_vX.h` files extracting all relevant definitions from old API versions. It also provides error codes in `err.h`. # API (v5) Below is the main `ncclProfiler_v5` struct. Each function is explained in later sections. ``` typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // - commId : communicator id // - commName : user assigned communicator name // - nNodes : number of nodes in communicator // - nranks : number of ranks in communicator // - rank : rank identifier in communicator // - logfn : logger function // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, uint64_t commId, int* eActivationMask, const char* commName, int nNodes, int nranks, int rank, ncclDebugLogger_t logfn); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v5_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v5_t eState, ncclProfilerEventStateArgs_v5_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v5_t; ``` ## Error codes As rule of thumb, profiler generated errors should not be propagated to NCCL and alter its normal functioning. Nevertheless, the profiler interface returns NCCL error codes, in case any need for them arises in the future. For now, any profiler interface call should only return `ncclSuccess`. The only exception is `init` that can return an error so that NCCL can disable the plugin. ## Operation overview NCCL will call the `init` function first for every new communicator that is initialized. The profiler returns an opaque context handle that is used to isolate profiler instances across communicators. Similarly, NCCL will call `finalize` to destroy the profiler context, thus freeing resources. The NCCL core code is instrumented with calls to `startEvent`, `stopEvent` and `recordEventState`. These are used to start, stop and update events in the profiler, respectively. ## API Functions ### Initialization #### name The `name` field should point to a character string with the name of the profiler plugin. This will be used for all logging, especially when `NCCL_DEBUG=INFO` is set. #### init As soon as NCCL finds the plugin and the correct ncclProfiler symbol, it calls its `init` function. This allows the plugin to initialize its internal context, used during profiling of NCCL events. If the `init` function does not return `ncclSuccess`, NCCL disables the plugin. #### finalize When the profiler is no longer needed, a call to `finalize` destroys the profiler context and frees up resources. ### Profiling #### startEvent When NCCL needs to start profiling a new event it calls `startEvent`. `startEvent` takes the profiler context, previously created by `init`, an event descriptor of type `ncclProfilerEventDescr_t` and returns an opaque profiler event handle that can be passed to other profiler functions, as discussed later in the document. The event descriptor contains all the event metadata. Every event type has its own descriptor. Below is the `ncclProfilerEventDescr_t` struct. ``` typedef struct { uint64_t type; // event type descriptor: ncclProfileGroupApi, ncclProfileCollApi, ... void* parentObj; // pointer to parent event used to expose the event hierarchy to the profiler int rank; // rank that generated the event union { struct { // GroupAPI event metadata bool graphCaptured; // Set to true if the Group API event is emitted inside a CUDA graph capture int groupDepth; // Determines the depth of a ncclGroup. A depth of 1 implies that the Group API call is implicit (internal to NCCL) // and not called by the user. Any depth greater than 1 means that the user made the Group API call. } groupApi; struct { // Collective API call metadata const char* func; // string containing name of the collective operation during size_t count; // data count const char* datatype; // string containing the name of the datatype int root; // root rank void* stream; // Opaque handle that points to the CUDA stream that the operation is enqueued in bool graphCaptured; // Set to true if the Collective API event is emitted inside a CUDA graph capture } collApi; struct { // Point-to-point API call metadata const char* func; // string containing name of the p2p operation size_t count; // data count const char* datatype; // string containing the name of the datatype void* stream; // Opaque handle that points to a CUDA stream object bool graphCaptured; // Set to true if the Collective API event is emitted inside a CUDA graph capture } p2pApi; struct { // Kernel Launch event metadata void* stream; // Opaque handle that points to the CUDA stream that the operation is enqueued in } kernelLaunch; struct { // collective events metadata uint64_t seqNumber; // sequence number of this collective operation in the communicator const char* func; // string containing name of the collective void const* sendBuff; // address of send buffer void* recvBuff; // address of recv buffer size_t count; // data count int root; // root rank const char* datatype; // string containing the name of the datatype uint8_t nChannels; // number of channels for this collective uint8_t nWarps; // number of GPU warps for this collective const char* algo; // string containing name of the algorithm for this collective const char* proto; // string containing name of the protocol for this collective void* parentGroup; // for backward compatibility with v4 - this points to the legacy v4 group parent } coll; struct { // point-to-point events metadata const char* func; void* buff; const char* datatype; size_t count; int peer; // peer rank for this point-to-point uint8_t nChannels; // number of channels for this p2p void* parentGroup; // for backward compatibility with v4 - this points to the legacy v4 group parent } p2p; struct { // proxyOp events metadata pid_t pid; // process id that generated the associated `ncclProxyOp` object uint8_t channelId; // id of the channel used by the associated `ncclProxyOp` object int peer; // peer rank int nSteps; // number of network transfers/steps required by the `ncclProxyOp` int chunkSize; // chunk size for this `ncclProxyOp` int isSend; // type of network operation } proxyOp; struct { // proxyStep events metadata int step; // individual step in `ncclProxyOp` } proxyStep; struct { uint8_t channelId; // id of the channel used by the kernel uint64_t ptimer; // kernel supplied timestamp } kernelCh; struct { int64_t id; // net plugin id (used by net and profiler plugins to agree on event definitions) void* data; // pointer to network plugin defined event } netPlugin; }; } ncclProfilerEventDescr_v5_t; ``` NCCL defines the following events: `ncclProfileGroupApi`, `ncclProfileCollApi`, `ncclProfileP2pApi`, `ncclProfileKernelLaunch`, `ncclProfileGroup`, `ncclProfileColl`, `ncclProfileP2p`,`ncclProfileProxyOp`, `ncclProfileProxyStep`, `ncclProfileProxyCtrl`, `ncclProfileKernelCh` and `ncclProfileNetPlugin`. #### stopEvent `stopEvent` takes the event handle returned by `startEvent` to stop the event. After the event has been stopped the handle can no longer be used with other profiler calls. Using the event handle after `eventStop` is undefined behavior. #### recordEventState Some events can only be started and stopped. For example, `ncclProfileP2pApi`, `ncclProfileCollApi`, `ncclProfileGroup`, `ncclProfileColl`, `ncclProfileP2p` cannot be updated through calls to `recordEventState`. `ncclProfileGroupApi`, `ncclProfileProxyOp`, `ncclProfileProxyStep`, `ncclProfileNetPlugin`, `ncclProfileKernelCh`, and `ncclProfileProxyCtrl` can be updated through calls to `recordEventState`. The state of these events can be updated, along with event attributes, using `recordEventState`. These events can go through several states during their lifecycle. The list of supported states for the updatable events is reported below. ``` typedef enum { // ncclProfileProxyOp event states ncclProfilerProxyOpSendPosted = 0, // deprecated in v4 ncclProfilerProxyOpSendRemFifoWait = 1, // deprecated in v4 ncclProfilerProxyOpSendTransmitted = 2, // deprecated in v4 ncclProfilerProxyOpSendDone = 3, // deprecated in v4 ncclProfilerProxyOpRecvPosted = 4, // deprecated in v4 ncclProfilerProxyOpRecvReceived = 5, // deprecated in v4 ncclProfilerProxyOpRecvTransmitted = 6, // deprecated in v4 ncclProfilerProxyOpRecvDone = 7, // deprecated in v4 ncclProfilerProxyOpInProgress_v4 = 19,// state marks transition of proxy op to progress // ncclProfileProxyStep event states ncclProfilerProxyStepSendGPUWait = 8, // state marks the waiting of send data from GPU for given network transfer/step ncclProfilerProxyStepSendPeerWait_v4 = 20,// state marks the waiting of recv clear to send credits for given network transfer/step ncclProfilerProxyStepSendWait = 9, // state marks the waiting of send data from network for given network transfer/step ncclProfilerProxyStepRecvWait = 10,// state marks the waiting of recv data from network for given network transfer/step ncclProfilerProxyStepRecvFlushWait = 11,// state marks the waiting of recv data flush to GPU for given network transfer/step ncclProfilerProxyStepRecvGPUWait = 12,// state marks the waiting of recv data consumption from GPU for given network transfer/step // ncclProfileProxyCtrl event states ncclProfilerProxyCtrlIdle = 13,// state marks proxy progress thread idle ncclProfilerProxyCtrlActive = 14,// state marks proxy progress thread active ncclProfilerProxyCtrlSleep = 15,// state marks proxy progress thread sleeping ncclProfilerProxyCtrlWakeup = 16,// state marks proxy progress thread waking up ncclProfilerProxyCtrlAppend = 17,// state marks append of new network work item begin ncclProfilerProxyCtrlAppendEnd = 18,// state marks append of new network work item end // ncclProfileNetPlugin event states ncclProfilerNetPluginUpdate = 21,// state marks update of network defined event // ncclProfileKernelCh event states ncclProfilerKernelChStop = 22,// state marks stop of kernelCh event and timestamp update // Group API States ncclProfilerGroupStartApiStop = 23,// state marks the end of a ncclGroupStart() API call ncclProfilerEndGroupApiStart = 24 // state marks the start of a ncclGroupEnd() API call } ncclProfilerEventState_v5_t; ``` NCCL profile API events are generated when the API calls are made, right after NCCL checks for graph capture information. They parent collective, point-to-point and kernel launch events and persist across multiple operations in a group. `ncclProfileKernelLaunch` events are generated when the CUDA call to a kernel launch is made. In the case of graph capture, the event start indicates that the kernel launch operation has been recorded, not launched. `ncclProfileProxyOp` events are generated by the proxy progress thread while it is processing network requests for the GPU kernel. ProxyOp events are generated for every active channel and provide a summary of the activity of the proxy progress thread for that channel. Most of the states for this event were duplicated with `ncclProfileProxyStep` events. Therefore, starting with version 4 of the profiler interface these states have been deprecated. The same level of information can still be obtained through the `ncclProfileProxyStep` events. `ncclProfileProxyStep` events are generated by the proxy progress thread while it is processing network requests for the GPU kernel. ProxyStep events describe individual network transfer in the channel. Thus, they provide a more fine-grained view w.r.t. ProxyOp events. `ncclProfileProxyCtrl` events are generated by the proxy progress thread while it is not processing network requests for the GPU kernel. This includes everything else that the proxy thread might be doing, including appending new `ncclProxyOp` objects to the list of work elements to process. `ncclProfileKernelCh` events are generated by the profiler proxy progress function while the kernel processes work items for the enqueued NCCL operations. `ncclProfileNetPlugin` events are generated by the network plugin. Network plugins are free to define their own set of events and communicate them to the profiler plugin using `ncclProfileNetPlugin` and the `ncclProfilerCallback\_t` NCCL core callback. The network and profiler plugin can agree on the network defined event definition using the plugin id in the event descriptor. The plugin identifier is a 64-bit integer that has two parts: the 16 LSB are assigned to the plugin event version, the next 16 bits are assigned to the plugin type (NCCL\_PROFILER\_NET\_TYPE\_IB, ...). The rest of the bits are unused and available for future extensions. A network IB plugin can use this infrastructure to define a QP event as: ```C #define NCCL_PROFILER_NET_IB_VER 1 enum { ncclProfileQp = (1 << 0), }; // The data structure version is encoded in the plugin identifier bitmask and // passed to NCCL core through the profiler callback. NCCL copies the plugin // identifier in the event descriptor before calling the profiler startEvent // function. The profiler should inspect the plugin id to find out the source // plugin as well as the version of the event struct typedef struct { uint8_t type; // event type (plugin defined) union { struct { int device; // network device id uint64_t wr_id; // work request id int opcode; // ibv opcode int qpNum; // QP number size_t length; // work request data length } qp; }; } ncclProfilerNetIbDescr_v1_t; ``` The network event infrastructure is network agnostic. A different network socket plugin can use it to define a socket event as: ```C #define NCCL_PROFILER_NET_SOCKET_VER 1 enum { ncclProfileSocket = (1 << 0), }; // The data structure version is encoded in the plugin identifier bitmask and // passed to NCCL core through the profiler callback. NCCL copies the plugin // identifier in the event descriptor before calling the profiler startEvent // function. The profiler should inspect the plugin id to find out the source // plugin as well as the version of the event struct typedef struct { uint8_t type; // event type (plugin defined) union { struct { int fd; int op; size_t length; } sock; }; } ncclProfilerNetSockDescr_v1_t; ``` The network plugin creates an event (descriptor) and passes it to the profiler callback, along with the network type and version (plugin id). NCCL then creates a `ncclProfileNetPlugin` event descriptor, attaches the network plugin defined event as external data, and calls the profiler `startEvent` function. ```C ncclResult_t isend(..., void* phandle, ...) { ... int pluginId = NCCL_PROFILER_NET_TYPE_IB | NCCL_PROFILER_NET_IB_VER; ncclProfilerNetIbDescr_v1_t eDescr = { }; eDescr.type = ncclProfileQp; eDescr.qp = { ... }; ncclProfilerCallback(&eHandle, 0 /* start net event */, phandle, pluginId, &eDescr); ... } ``` State transitions for the events described can also come with event attribute updates. For this reason the profiler defines the `ncclProfilerEventStateArgs_t` struct, reported below. ``` typedef union { struct { // attributes for update for ncclProfileProxyStep events size_t transSize; // transfer size field for this proxy step } proxyStep; struct { // attributes to update for ncclProfileProxyCtrl events int appendedProxyOps; // number of appended proxy ops thus far } proxyCtrl; struct { // attributes to update for ncclProfileNetPlugin events void* data; // network plugin opaque update data field } netPlugin; struct { // attribute to update for ncclProfileKernelCh events uint64_t pTimer; // timestamp provided by the NCCL kernel } kernelCh; } ncclProfilerEventStateArgs_v5_t; ``` The example profiler in `plugins/profiler/example` contains details on how to capture and use the events above. ### Event hierarchy NCCL core events (reported above) are organized into a hierarchy as reported below: ``` Group API event | +- Collective API event | | | +- Collective event | | | +- ProxyOp event | | | | | +- ProxyStep event | | | | | +- NetPlugin event | | | +- KernelCh event | +- Point-to-point API event | | | +- Point-to-point event | | | +- ProxyOp event | | | | | +- ProxyStep event | | | | | +- NetPlugin event | | | +- KernelCh event | +- Kernel Launch event ProxyCtrl event ``` # Profiler instrumentation and logging ## Profiling of collective and p2p operations The NCCL code is instrumented with profiler callbacks at different levels to capture start/stop of groups, collective and point-to-point operations, as well as proxy, kernel and network activity. Due to the asynchronous nature of NCCL operations, events associated to collective and point-to-point operations are not easy to delimit precisely. For example, without both proxy and/or kernel activity it is impossible for the profiler to figure out when a collective operation completes. Therefore, `stopEvent` for collectives simply indicates to the profiler that the collective has been enqueued. The profiler can leverage proxy and/or kernel event information, if these are enabled, to estimate when the collective ends. For example, the profiler can look at the `stopEvent` call of the last `ncclProfileProxyOp` event to mark the completion of the associated collective event. This can be achieved by reference counting the collective event and letting calls to `startEvent` and `stopEvent` increment and decrement the reference counter, respectively. ## PXN PXN causes some proxy operations to be processed in a remote proxy thread that differs from the one that generated the operation. When this happens, the event hierarchy reported above breaks. Because the profiler can use the hierarchy information, provided by NCCL in the event descriptor, to dereference the parent event during `startEvent`, the remote proxy thread must be in the same address space of the proxy thread originating the operation. To avoid the profiler instance in the remote proxy address space to dereference a pointer from another address space the event descriptor includes the PID of the originator. The profiler plugin needs to check that the originator PID matches the local PID before dereferencing the parent event. # Known Limitations In intra-node communication, or whenever a rank does not have any network activity for which proxy events are unavailable, the profiler will only report the enqueue events (e.g., ncclAllReduce). The events from enqueue can be time stamped by the profiler (at start and stop) to reconstruct the execution time of the collective. However, this time only represents the launch time of the collective and not the actual execution time. To reconstruct the execution time more accurately proxy and kernel events are provided. With version 3 of the profiler interface network activity is no longer required to do intra-node profiling. Kernel events instrumentation leverages counters exposed by the kernel to the host and the proxy progress thread. Thus, the proxy progress thread infrastructure is shared between the network and the profiler. If the proxy is serving network requests the kernel profiling probing can be delayed, causing loss of accuracy. Similarly, if the CPU is under heavy load and the scheduling of the proxy progress thread is delayed, a similar loss of accuracy can be encountered. To mitigate this effect, with version 4 of the profiler NCCL uses a per-channel ring buffer of 64 elements. Every counter is complemented by two timestamps (ptimers) supplied by the NCCL kernel (one for start and one for stop of the operation in the kernel). NCCL propagates these timestamps to the profiler plugin that it can convert them to CPU time domain. nccl-2.29.7-1/plugins/profiler/example/000077500000000000000000000000001515037102200176225ustar00rootroot00000000000000nccl-2.29.7-1/plugins/profiler/example/CMakeLists.txt000066400000000000000000000024411515037102200223630ustar00rootroot00000000000000# Find all C source files in current directory set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/plugin.cc ${CMAKE_CURRENT_SOURCE_DIR}/print_event.cc ${CMAKE_CURRENT_SOURCE_DIR}/profiler_plugin_ce.cc ) # Create shared library add_library(nccl-profiler-example SHARED ${SRC_FILES}) # Set include directories target_include_directories(nccl-profiler-example PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/nccl ${CUDAToolkit_INCLUDE_DIRS} ) # Set output name to match Makefile set_target_properties(nccl-profiler-example PROPERTIES OUTPUT_NAME "nccl-profiler-example" PREFIX "lib" POSITION_INDEPENDENT_CODE ON LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib ) add_custom_command(TARGET nccl-profiler-example POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/test/unit/plugins COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/lib/libnccl-profiler-example.so ${CMAKE_BINARY_DIR}/test/unit/plugins ) # Add custom target for clean (equivalent to Makefile clean target) add_custom_target(clean-profiler-lib COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_BINARY_DIR}/lib/libnccl-profiler-example.so COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_BINARY_DIR}/test/unit/plugins/libnccl-profiler-example.so COMMENT "Cleaning libnccl-profiler-example.so" ) nccl-2.29.7-1/plugins/profiler/example/Makefile000066400000000000000000000012731515037102200212650ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # .DEFAULT_GOAL: build include ../../../makefiles/common.mk BUILDDIR ?= . NCCLDIR := $(BUILDDIR) SRC_FILES := $(wildcard *.cc) DST_DIR := $(BUILDDIR) OBJ_FILES := $(SRC_FILES:%.cc=${DST_DIR}/%.o) DEP_FILES := $(OBJ_FILES:%.o=%.dep) build: ${DST_DIR}/libnccl-profiler-example.so ${DST_DIR}/libnccl-profiler-example.so: ${SRC_FILES} @printf "Compiling %-35s > %s\n" $< $@ @mkdir -p ${DST_DIR} $(CXX) -Inccl -I${CUDA_INC} -fPIC -shared -o $@ $^ clean: rm -f ${DST_DIR}/libnccl-profiler-example.so nccl-2.29.7-1/plugins/profiler/example/README.md000066400000000000000000000305301515037102200211020ustar00rootroot00000000000000# NCCL Example Profiler Plugin Usage This page describes how to use the NCCL example profiler plugin # Overview The example profiler plugin implements the NCCL profiler plugin API introduced in NCCL v2.23. The API defines a set of events and data structures that NCCL uses to share event information with profiler plugins. The user can control what events are instrumented by NCCL and when traces collected by the profiler should be dumped through environment variables, as described in the rest of the document. The user can also control other profiler parameters that alter its behavior. For example, users can change the size of the event window the profiler keeps track of. ## Building the profiler plugin To build the example plugin shipped as part of NCCL, just type `make`. ## Using the profiler plugin 1. Add the directory of this profiler plugin to your `LD_LIBRARY_PATH` or set the `NCCL_PROFILER_PLUGIN`, as documented in `plugins/profiler/README.md`. 2. Set `NCCL_PROFILE_EVENT_MASK` bitmask to specify the NCCL events you want to instrument. By default, all collectives and send/recv operations will be traced. For more details about the event representation used by the profiler refer to `plugins/profiler/README.md`. As an example, setting: `NCCL_PROFILE_EVENT_MASK` to 256 (`ncclProfileGroupApi`) | 2 (`ncclProfileColl`) | 8 (`ncclProfileProxyOp`) enables the profiling of the group API, the collective and the proxy op events. The same events can be expressed more concisely by setting `NCCL_PROFILE_EVENT_MASK` to 8 (`ncclProfileProxyOp`). Indeed, in NCCL all the events above (in the event hierarchy) the one requested are also captured. The advantage is that the profiler can easily correlate events that belong to the same NCCL operation and present them accordingly. Setting `NCCL_PROFILE_EVENT_MASK` to 4095 enables all events supported by the v5 profiler. 3. Set `NCCL_PROFILE_DUMP_FILE` to the name of the dump file for the collected traces. A file named ${NCCL_PROFILE_DUMP_FILE}-hostname-tid.txt is created. Profiler traces are saved using the chrome event format (more precisely, using asynchronous events). 4. If you set the dump file variable, type chrome://tracing on your chromium browser search bar and open the created dump file to visualize the traces. # Changing the profiler memory pool sizes The example profiler uses separate memory pools for different types of events. The size of these memory pools (i.e., the # events) determines the number of events that the profiler can keep track of at the same time. When NCCL requests a new event (e.g., collective event) to profile a `ncclAllReduce` operation, by calling `startEvent`, the profiler searches in the collective pool for a free event. If it finds one, it marks it as in use and returns the handle to NCCL. If the pool is completely used the profiler returns `NULL` to NCCL and ignores all the following NCCL profiler calls for the `NULL` event handle. When the `ncclAllReduce` has been processed, NCCL calls `stopEvent` with the previosly returned event handle. The profiler has a total of 5 memory pools. The group, collective and p2p pools contain objects for the corresponding events. The `ProxyCtrl` pool contains objects for `ProxyCtrl` events and the `ProxyDetach` pool contains objects for `ProxyOp` events generated by remote proxies. A list of pools and their size is reported below: - `NCCL_PROFILE_GROUP_API_POOL_SIZE` (256) - `NCCL_PROFILE_COLL_API_POOL_SIZE` (256) - `NCCL_PROFILE_P2P_API_POOL_SIZE` (256) - `NCCL_PROFILE_KERNEL_LAUNCH_POOL_SIZE` (256) - `NCCL_PROFILE_COLL_POOL_SIZE` (256) - `NCCL_PROFILE_P2P_POOL_SIZE` (256) - `NCCL_PROFILE_PROXY_CTRL_POOL_SIZE` (16) - `NCCL_PROFILE_PROXY_DETACH_POOL_SIZE` (256) Remote proxy operations are generated when PXN is in use. Refer to this article for more information about PXN and how it works: https://developer.nvidia.com/blog/doubling-all2all-performance-with-nvidia-collective-communication-library-2-12/ # Reported events The example profiler generates traces using the json format. An example of trace is reported below: ``` [ {"name": "Group API", "cat": "GROUP_API", "ph": "b", "id": 0, "pid": 225798, "tid": 1, "ts": 3433.595001, "args": {"groupApiId": 0, "groupDepth":1}}, {"name": "KernelLaunch", "cat": "KERNEL_LAUNCH", "ph": "b", "id": 0, "pid": 225798, "tid": 1, "ts": 0.000000, "args": {"groupId": 0, "Stream": 0x5020000567d0}}, {"name": "KernelLaunch", "cat": "KERNEL_LAUNCH", "ph": "e", "id": 0, "pid": 225798, "tid": 1, "ts": 111991.558990}, {"name": "AllReduce", "cat": "COLL_API", "ph": "b", "id": 0, "pid": 225798, "tid": 1, "ts": 0.000000, "args": {"count": 262144, "datatype": ncclFloat32, "root": 0, "GraphCaptured":0, "Stream": 0x5020000567d0}}, {"name": "AllReduce", "cat": "COLL", "ph": "b", "id": 0, "pid": 225798, "tid": 1, "ts": 111994.477997, "args": {"SeqNum": 0, "CommHash": 1493613951195738943, "Rank": 0, "Count": 262144, "Datatype": "ncclFloat32", "Algorithm": "RING", "Protocol": "SIMPLE", "nChannels": 2}}, {"name": "KernelCh", "cat": "GPU", "ph": "b", "id": 0, "pid": 225798, "tid": 1, "ts": 119711.888000, "args": {"Channel": 0, "StartGpuClk": 1756135989724672000, "StopGpuClk": 1756135989732831232}}, {"name": "ScheduleRecv", "cat": "PROXY", "ph": "b", "id": 0, "pid": 225798, "tid": 1, "ts": 119652.709991, "args": {"Channel": 0, "Peer": 1, "Steps": 4, "ChunkSize": 4194304, "transSize": 524288}}, {"name": "ScheduleRecv", "cat": "PROXY", "ph": "e", "id": 0, "pid": 225798, "tid": 1, "ts": 119686.300995}, {"name": "ProgressRecv", "cat": "PROXY", "ph": "b", "id": 0, "pid": 225798, "tid": 1, "ts": 119686.300995, "args": {"Channel": 0, "Peer": 1, "Steps": 4, "ChunkSize": 4194304, "transSize": 524288}}, {“name": "RecvWait", "cat": "NET", "ph": "b", "id": 0, "pid": 225798, "tid": 1, "ts": 119707.677979, "args": {"Step": 0}}, {"name": "RecvWait", "cat": "NET", "ph": "e", "id": 0, "pid": 225798, "tid": 1, "ts": 119807.691986}, {"name": "RecvFlushWait", "cat": "NET", "ph": "b", "id": 0, "pid": 225798, "tid": 1, "ts": 119807.691986, "args": {"Step": 0}}, {"name": "RecvFlushWait", "cat": "NET", "ph": "e", "id": 0, "pid": 225798, "tid": 1, "ts": 119867.338989}, {"name": "RecvGpuWait", "cat": "NET", "ph": "b", "id": 0, "pid": 225798, "tid": 1, "ts": 119867.338989, "args": {"Step": 0}}, {"name": "RecvGpuWait", "cat": "NET", "ph": "e", "id": 0, "pid": 225798, "tid": 1, "ts": 120120.983002}, {"name": "RecvWait", "cat": "NET", "ph": "b", "id": 1, "pid": 225798, "tid": 1, "ts": 119733.647980, "args": {"Step": 1}}, {"name": "RecvWait", "cat": "NET", "ph": "e", "id": 1, "pid": 225798, "tid": 1, "ts": 119844.401001}, {"name": "RecvFlushWait", "cat": "NET", "ph": "b", "id": 1, "pid": 225798, "tid": 1, "ts": 119844.401001, "args": {"Step": 1}}, {"name": "RecvFlushWait", "cat": "NET", "ph": "e", "id": 1, "pid": 225798, "tid": 1, "ts": 119890.567993}, {"name": "RecvGpuWait", "cat": "NET", "ph": "b", "id": 1, "pid": 225798, "tid": 1, "ts": 119890.567993, "args": {"Step": 1}}, {"name": "RecvGpuWait", "cat": "NET", "ph": "e", "id": 1, "pid": 225798, "tid": 1, "ts": 120121.129974}, {"name": "RecvWait", "cat": "NET", "ph": "b", "id": 2, "pid": 225798, "tid": 1, "ts": 119753.023987, "args": {"Step": 2}}, {"name": "RecvWait", "cat": "NET", "ph": "e", "id": 2, "pid": 225798, "tid": 1, "ts": 120038.847992}, {"name": "RecvFlushWait", "cat": "NET", "ph": "b", "id": 2, "pid": 225798, "tid": 1, "ts": 120038.847992, "args": {"Step": 2}}, {"name": "RecvFlushWait", "cat": "NET", "ph": "e", "id": 2, "pid": 225798, "tid": 1, "ts": 120085.685974}, {"name": "RecvGpuWait", "cat": "NET", "ph": "b", "id": 2, "pid": 225798, "tid": 1, "ts": 120085.685974, "args": {"Step": 2}}, {"name": "RecvGpuWait", "cat": "NET", "ph": "e", "id": 2, "pid": 225798, "tid": 1, "ts": 120121.244995}, {"name": "RecvWait", "cat": "NET", "ph": "b", "id": 3, "pid": 225798, "tid": 1, "ts": 119772.510986, "args": {"Step": 3}}, {"name": "RecvWait", "cat": "NET", "ph": "e", "id": 3, "pid": 225798, "tid": 1, "ts": 120062.944977}, {"name": "RecvFlushWait", "cat": "NET", "ph": "b", "id": 3, "pid": 225798, "tid": 1, "ts": 120062.944977, "args": {"Step": 3}}, {"name": "RecvFlushWait", "cat": "NET", "ph": "e", "id": 3, "pid": 225798, "tid": 1, "ts": 120101.089996}, {"name": "RecvGpuWait", "cat": "NET", "ph": "b", "id": 3, "pid": 225798, "tid": 1, "ts": 120101.089996, "args": {"Step": 3}}, {"name": "RecvGpuWait", "cat": "NET", "ph": "e", "id": 3, "pid": 225798, "tid": 1, "ts": 120165.115997}, {"name": "ProgressRecv", "cat": "PROXY", "ph": "e", "id": 0, "pid": 225798, "tid": 1, "ts": 120165.356995}, {"name": "ScheduleSend", "cat": "PROXY", "ph": "b", "id": 1, "pid": 225798, "tid": 1, "ts": 119656.950989, "args": {"Channel": 0, "Peer": 1, "Steps": 4, "ChunkSize": 4194304, "transSize": 524288}}, {"name": "ScheduleSend", "cat": "PROXY", "ph": "e", "id": 1, "pid": 225798, "tid": 1, "ts": 119709.078979}, {"name": "ProgressSend", "cat": "PROXY", "ph": "b", "id": 1, "pid": 225798, "tid": 1, "ts": 119709.078979, "args": {"Channel": 0, "Peer": 1, "Steps": 4, "ChunkSize": 4194304, "transSize": 524288}}, {"name": "SendGpuWait", "cat": "NET", "ph": "b", "id": 4, "pid": 225798, "tid": 1, "ts": 119710.632996, "args": {"Step": 0}}, {"name": "SendGpuWait", "cat": "NET", "ph": "e", "id": 4, "pid": 225798, "tid": 1, "ts": 119808.636993}, {"name": "SendPeerWait", "cat": "NET", "ph": "b", "id": 4, "pid": 225798, "tid": 1, "ts": 119808.636993, "args": {"Step": 0}}, {"name": "SendPeerWait", "cat": "NET", "ph": "e", "id": 4, "pid": 225798, "tid": 1, "ts": 119818.972992}, ... [ trace truncated for brevity ] {"name": "AllReduce", "cat": "COLL", "ph": "e", "id": 17, "pid": 225798, "tid": 1, "ts": 170633.535980}, {"name": "AllReduce", "cat": "COLL_API", "ph": "e", "id": 17, "pid": 225798, "tid": 1, "ts": 170582.923981}, {"name": "Group API", "cat": "GROUP_API", "ph": "e", "id": 17, "pid": 225798, "tid": 1, "ts": 170637.582001}, {}] ``` Details about the fields used in the trace can be found at this link: https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview?tab=t.0#heading=h.yr4qxyxotyw The trace above is obtained by running a `ncclAllReduce` operation on 2 GPUs, communicating with each other through the network interface. The `Group` event encloses all traces that are related to the single `ncclAllReduce` call. (Note that for single collective invocations, where there are no explicit group calls, NCCL creates a group with only one collective and this is what is presented in the traces above). The `AllReduce` event encloses traces for the proxy operation associated to the `ncclAllReduce` operation. The `args` field in the traces contains NCCL specific information (aside from the chrome trace event format). ## AllReduce trace The `AllReduce` entry presents information about the `ncclAllReduce` operation. It contains the following info in the args field: - seqNum : sequential number of the collective in the communicator (every collective type has its own sequence number in the communicator) - commHash : communicator unique identifier - rank : NCCL rank for the ncclAllReduce - datatype : NCCL datatype - algorithm : algorithm used to process the ncclAllReduce - protocol : protocol used to process the ncclAllReduce - nChannels : Number of channels used to process the ncclAllReduce If the proxy events are not active (e.g., the `ncclAllReduce` is intranode) the end timestamp will match the time consumed by the CPU to launch the collective. For more details refer to `plugins/profiler/README.md`, section `Profiling of collective and p2p operations`. The Proxy send trace gives a summary of the proxy progress thread activity for the channel. If more details are needed, these can be obtained by enabling the proxy step event (`ncclProfileProxyStep`). In which case the trace entries below are also reported by the profiler. #### Proxy SendGpuWait Presents, for every network step, the time the CPU proxy spends waiting for the GPU to provide the data in the staging buffer. #### Proxy SendWait Presents, for every network step, the time the CPU proxy spends waiting for the `isend` to complete #### Proxy RecvWait Presents, for every network step, the time the CPU proxy spends waiting for a posted `irecv` to complete #### Proxy RecvFlushWait Presents, for every network step, the time the CPU proxy spends waitng for the recv data to be flushed to the GPU #### Proxy RecvGpuWait Presents, for every network step, the time the CPU proxy spends waiting for the GPU to consume the recv data nccl-2.29.7-1/plugins/profiler/example/event.h000066400000000000000000000302101515037102200211100ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef EVENT_H_ #define EVENT_H_ #include #include #include #include #include "err.h" #include "profiler.h" #include "queue.h" #include // CE timing modes typedef enum { CE_TIMING_CPU = 0, CE_TIMING_GPU = 1 } CeTimingMode_t; #define MAX_CHANNELS 32 #define MAX_STEPS 32 #define MAX_OPS 16 // Up to 64K ranks for PAT #define MAX_EVENTS_PER_REQ (8) struct proxyOp; struct proxyStep; struct netPlugin { uint64_t type; int pluginType; int pluginVer; uint8_t pluginEvent; union { struct { int device; int qpNum; int opcode; uint64_t wr_id; size_t length; } qp; struct { int fd; int op; size_t length; } sock; }; double startTs; double stopTs; struct proxyStep* parent; }; struct kernelCh { uint8_t type; uint8_t channelId; struct taskEventBase* parent; double startTs; double stopTs; uint64_t startGpuClk; uint64_t stopGpuClk; }; #define PROXY_STEP_SEND_GPU_WAIT 0 #define PROXY_STEP_SEND_PEER_WAIT 1 #define PROXY_STEP_SEND_WAIT 2 #define PROXY_STEP_RECV_WAIT 0 #define PROXY_STEP_RECV_FLUSH_WAIT 1 #define PROXY_STEP_RECV_GPU_WAIT 2 #define PROXY_STEP_MAX_STATES 3 struct proxyStep { uint64_t type; // type of event: network transfer int state; int step; // network transfer id in given channel int isSend; // send/recv channel operation double timestamp[PROXY_STEP_MAX_STATES]; double startTs; double stopTs; struct proxyOp* parent; struct netPlugin net[MAX_EVENTS_PER_REQ]; int nNetEvents; }; struct proxyOp { uint64_t type; // type of event: proxy operation uint8_t channelId; // channel id for this proxy operation pid_t pid; int rank; int peer; // peer rank for this proxy operation int nSteps; // total number of network transfers for this proxy operation int chunkSize; // chunk size for this proxy operation int isSend; // send/recv channel operation size_t transSize; // transfer data size for this proxy operation double startTs; double progrTs; // In progress state transition double stopTs; int stepCount; // last processed network operation for this proxy operation struct proxyStep step[MAX_STEPS]; // array of network transfer events struct taskEventBase* parent; // parent event p2p/collective }; struct group; struct context; struct proxyCtrl { uint64_t type; struct context* ctx; // profiler context double startTs; double stopTs; int state; int appended; // appended proxy operations }; // task level event base structure struct taskEventBase { uint64_t type; // event type: collective/p2p int rank; // rank of the operation in NCCL communicator const char* func; // ncclFunc* int refCount; // number of references for this operation void* parent; // parent API event struct taskEventBase* next; // next top level event double startTs; double stopTs; }; struct collective { struct taskEventBase base; // base structure for this event uint64_t seqNumber; // sequence number for this collective in communicator void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; uint8_t nChannels; const char* algo; const char* proto; int nWarps; struct proxyOp op[MAX_CHANNELS][2*MAX_OPS]; int nProxyOps[MAX_CHANNELS]; struct kernelCh kernel[MAX_CHANNELS]; }; struct p2p { struct taskEventBase base; // base structure for this event uint8_t func; void const* buff; size_t count; const char* datatype; int peer; uint8_t nChannels; struct proxyOp op[MAX_CHANNELS]; struct kernelCh kernel[MAX_CHANNELS]; }; struct group { uint64_t type; struct context* ctx; // profiler context int groupId; int refCount; struct taskEventBase* eventHead; // queue head for task events struct taskEventBase* eventTail; // queue tail for task events double startTs; double stopTs; struct group* next; // next group event in queue }; struct collApi { uint64_t type; struct groupApi* parent; struct context* ctx; // profiler context int collApiId; int refCount; cudaStream_t stream; const char* func; size_t count; const char* datatype; int root; bool graphCaptured; struct taskEventBase* eventHead; // queue head for task events struct taskEventBase* eventTail; // queue tail for task events double startTs; double stopTs; struct collApi* next; }; struct p2pApi { uint64_t type; struct groupApi* parent; struct context* ctx; // profiler context int p2pApiId; int refCount; const char* func; cudaStream_t stream; size_t count; const char* datatype; bool graphCaptured; struct taskEventBase* eventHead; // queue head for task events struct taskEventBase* eventTail; // queue tail for task events double startTs; double stopTs; struct p2pApi* next; }; struct kernelLaunch { uint64_t type; struct groupApi* parent; cudaStream_t stream; int kernelLaunchId; double startTs; double stopTs; struct kernelLaunch* next; }; // CE event structures struct ceColl { struct taskEventBase base; // Must be first for task event queue (uses base.next) struct collApi* parent; int ceCollId; uint64_t seqNumber; size_t count; const char* datatype; int root; const char* syncStrategy; cudaStream_t stream; uint64_t eventId; int timingMode; // Timing fields: // - cpuStartTime/cpuStopTime: Captured using CLOCK_MONOTONIC (via gettime()), units: microseconds (double) // - cpuDuration: Always CPU-measured time difference (cpuStopTime - cpuStartTime), units: microseconds (double) // - elapsedTime: Final reported timing, units: microseconds (uint64_t) // * If timingMode==CE_TIMING_GPU: GPU-measured time from cudaEventElapsedTime (converted from ms to us) // * If timingMode==CE_TIMING_CPU: Same as cpuDuration (CPU-measured) double cpuStartTime; double cpuStopTime; double cpuDuration; uint64_t elapsedTime; // Child events (CeSync and CeBatch) struct taskEventBase* eventHead; struct taskEventBase* eventTail; // Plugin-managed CUDA events for timing cudaEvent_t startEvent; cudaEvent_t stopEvent; bool startCompleted; bool stopCompleted; struct ceColl* pollerNext; // For poller tracking list (separate from base.next) }; struct ceSync { struct taskEventBase base; // For parent CeColl's event queue struct ceColl* parent; int ceSyncId; bool isComplete; uint32_t seqNumber; int nRanks; cudaStream_t stream; uint64_t eventId; int timingMode; // Timing fields: See ceColl struct for detailed clock/unit documentation // - cpuStartTime/cpuStopTime: CLOCK_MONOTONIC, microseconds (double) // - cpuDuration: CPU-measured (cpuStopTime - cpuStartTime), microseconds (double) // - elapsedTime: GPU or CPU-measured depending on timingMode, microseconds (uint64_t) double cpuStartTime; double cpuStopTime; double cpuDuration; uint64_t elapsedTime; // Plugin-managed CUDA events for timing cudaEvent_t startEvent; cudaEvent_t stopEvent; bool startCompleted; bool stopCompleted; struct ceSync* pollerNext; // For poller tracking list }; struct ceBatch { struct taskEventBase base; // For parent CeColl's event queue struct ceColl* parent; int ceBatchId; int numOps; size_t totalBytes; bool useIntraSync; cudaStream_t stream; uint64_t eventId; int timingMode; // Timing fields: See ceColl struct for detailed clock/unit documentation // - cpuStartTime/cpuStopTime: CLOCK_MONOTONIC, microseconds (double) // - cpuDuration: CPU-measured (cpuStopTime - cpuStartTime), microseconds (double) // - elapsedTime: GPU or CPU-measured depending on timingMode, microseconds (uint64_t) double cpuStartTime; double cpuStopTime; double cpuDuration; uint64_t elapsedTime; // Plugin-managed CUDA events for timing cudaEvent_t startEvent; cudaEvent_t stopEvent; bool startCompleted; bool stopCompleted; struct ceBatch* pollerNext; // For poller tracking list }; struct groupApi { uint64_t type; struct context* ctx; int groupApiId; int refCount; bool graphCaptured; int groupDepth; struct profilerQueue p2pApiEvents; struct profilerQueue collApiEvents; struct profilerQueue kernelLaunchEvents; double endOfncclGroupStartTs; double startOfncclGroupEndTs; double startTs; double stopTs; struct groupApi* next; }; // CE event poller tracking struct ceEventList { struct ceColl* ceCollHead; struct ceSync* ceSyncHead; struct ceBatch* ceBatchHead; pthread_mutex_t mutex; }; // arrays for different event objects struct context { const char* commName; uint64_t commHash; int nranks; int rank; // CE event tracking for poller struct ceEventList ceEvents; int groupApiPoolSize; int groupApiPoolBase; int groupApiPoolIndex; struct groupApi* groupApiPool; int collApiPoolSize; int collApiPoolBase; int collApiPoolIndex; struct collApi* collApiPool; int p2pApiPoolSize; int p2pApiPoolBase; int p2pApiPoolIndex; struct p2pApi* p2pApiPool; int kernelLaunchPoolSize; int kernelLaunchPoolBase; int kernelLaunchPoolIndex; struct kernelLaunch* kernelLaunchPool; int groupPoolSize; int groupPoolBase; int groupPoolIndex; struct group* groupPool; int collPoolSize; int collPoolBase; int collPoolIndex; struct collective* collPool; int p2pPoolSize; int p2pPoolBase; int p2pPoolIndex; struct p2p* p2pPool; int proxyCtrlPoolSize; int proxyCtrlPoolBase; int proxyCtrlPoolIndex; struct proxyCtrl* proxyCtrlPool; // CE event pools int ceCollPoolSize; int ceCollPoolBase; int ceCollPoolIndex; struct ceColl* ceCollPool; int ceSyncPoolSize; int ceSyncPoolBase; int ceSyncPoolIndex; struct ceSync* ceSyncPool; int ceBatchPoolSize; int ceBatchPoolBase; int ceBatchPoolIndex; struct ceBatch* ceBatchPool; }; template inline int taskEventQueueEmpty(T *obj) { return obj->eventHead == NULL; } template inline void taskEventQueueEnqueue(T* obj, struct taskEventBase* event) { event->next = NULL; if (obj->eventHead) obj->eventTail->next = event; else obj->eventHead = event; obj->eventTail = event; } template inline struct taskEventBase* taskEventQueueHead(T *obj) { return obj->eventHead; } template inline struct taskEventBase* taskEventQueueDequeue(T* obj) { struct taskEventBase* tmp = obj->eventHead; obj->eventHead = obj->eventHead->next; if (obj->eventHead == NULL) obj->eventTail = NULL; return tmp; } template inline void resetTaskEvents(T *obj, struct context* ctx) { while (!taskEventQueueEmpty(obj)) { struct taskEventBase* base = taskEventQueueDequeue(obj); if (base->type == ncclProfileColl) { struct collective* c = (struct collective *)base; // reset event proxyOps & proxySteps memset(c->nProxyOps, 0, sizeof(int)*MAX_CHANNELS); // release collective events in the group and return them to the collective pool __atomic_fetch_add(&ctx->collPoolBase, 1, __ATOMIC_RELAXED); } else if (base->type == ncclProfileP2p) { struct p2p* p = (struct p2p *)base; // reset event proxyOp and proxySteps memset(&p->op, 0, sizeof(struct proxyOp)*MAX_CHANNELS); // release p2p events in the group and return them to the p2p pool __atomic_fetch_add(&ctx->p2pPoolBase, 1, __ATOMIC_RELAXED); } } } #endif nccl-2.29.7-1/plugins/profiler/example/nccl/000077500000000000000000000000001515037102200205415ustar00rootroot00000000000000nccl-2.29.7-1/plugins/profiler/example/nccl/common.h000066400000000000000000000016271515037102200222100ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef COMMON_H_ #define COMMON_H_ typedef enum {NCCL_LOG_NONE=0, NCCL_LOG_VERSION=1, NCCL_LOG_WARN=2, NCCL_LOG_INFO=3, NCCL_LOG_ABORT=4, NCCL_LOG_TRACE=5} ncclDebugLogLevel; typedef enum {NCCL_INIT=1, NCCL_COLL=2, NCCL_P2P=4, NCCL_SHM=8, NCCL_NET=16, NCCL_GRAPH=32, NCCL_TUNING=64, NCCL_ENV=128, NCCL_ALLOC=256, NCCL_CALL=512, NCCL_PROXY=1024, NCCL_NVLS=2048, NCCL_BOOTSTRAP=4096, NCCL_REG=8192, NCCL_ALL=~0} ncclDebugLogSubSys; typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags, const char *file, int line, const char *fmt, ...); #endif nccl-2.29.7-1/plugins/profiler/example/nccl/err.h000066400000000000000000000014161515037102200215040ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_ERR_H_ #define NCCL_ERR_H_ /* Error type for plugins */ typedef enum { ncclSuccess = 0, ncclUnhandledCudaError = 1, ncclSystemError = 2, ncclInternalError = 3, ncclInvalidArgument = 4, ncclInvalidUsage = 5, ncclRemoteError = 6 } ncclResult_t; #endif nccl-2.29.7-1/plugins/profiler/example/nccl/net_ib_v1.h000066400000000000000000000022421515037102200225600ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_IB_V1_H_ #define NET_IB_V1_H_ #define NCCL_PROFILER_NET_IB_VER 1 enum { ncclProfileQp = (1 << 0), }; // The data structure version is encoded in the plugin identifier bitmask and // passed to NCCL core through the profiler callback. NCCL copies the plugin // identifier in the event descriptor before calling the profiler startEvent // function. The profiler should inspect the plugin id to find out the source // plugin as well as the version of the event struct typedef struct { uint8_t type; // event type (plugin defined) union { struct { int device; // network device id uint64_t wr_id; // work request id int opcode; // ibv opcode int qpNum; // QP number size_t length; // work request data length } qp; }; } ncclProfilerNetIbDescr_v1_t; #endif nccl-2.29.7-1/plugins/profiler/example/nccl/net_socket_v1.h000066400000000000000000000020251515037102200234550ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_SOCKET_V1_H_ #define NET_SOCKET_V1_H_ #define NCCL_PROFILER_NET_SOCKET_VER 1 enum { ncclProfileSocket = (1 << 0), }; // The data structure version is encoded in the plugin identifier bitmask and // passed to NCCL core through the profiler callback. NCCL copies the plugin // identifier in the event descriptor before calling the profiler startEvent // function. The profiler should inspect the plugin id to find out the source // plugin as well as the version of the event struct typedef struct { uint8_t type; // event type (plugin defined) union { struct { int fd; int op; size_t length; } sock; }; } ncclProfilerNetSockDescr_v1_t; #endif nccl-2.29.7-1/plugins/profiler/example/nccl/profiler.h000066400000000000000000000104351515037102200225370ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_H_ #define PROFILER_H_ #include #include #include "common.h" enum { ncclProfileGroup = (1 << 0), // group event type ncclProfileColl = (1 << 1), // host collective call event type ncclProfileP2p = (1 << 2), // host point-to-point call event type ncclProfileProxyOp = (1 << 3), // proxy operation event type ncclProfileProxyStep = (1 << 4), // proxy step event type ncclProfileProxyCtrl = (1 << 5), // proxy control event type ncclProfileKernelCh = (1 << 6), // kernel channel event type ncclProfileNetPlugin = (1 << 7), // network plugin-defined, events ncclProfileGroupApi = (1 << 8), // Group API events ncclProfileCollApi = (1 << 9), // Collective API events ncclProfileP2pApi = (1 << 10), // Point-to-Point API events ncclProfileKernelLaunch = (1 << 11), // Kernel launch events // CE events (v6) ncclProfileCeColl = (1 << 12), // CE collective operation ncclProfileCeSync = (1 << 13), // CE synchronization operation ncclProfileCeBatch = (1 << 14), // CE batch operation }; typedef enum { ncclProfilerProxyOpSendPosted = 0, // deprecated in v4 ncclProfilerProxyOpSendRemFifoWait = 1, // deprecated in v4 ncclProfilerProxyOpSendTransmitted = 2, // deprecated in v4 ncclProfilerProxyOpSendDone = 3, // deprecated in v4 ncclProfilerProxyOpRecvPosted = 4, // deprecated in v4 ncclProfilerProxyOpRecvReceived = 5, // deprecated in v4 ncclProfilerProxyOpRecvTransmitted = 6, // deprecated in v4 ncclProfilerProxyOpRecvDone = 7, // deprecated in v4 ncclProfilerProxyOpInProgress_v4 = 19, /* Legacy proxy profiler states */ ncclProfilerProxyStepSendGPUWait = 8, ncclProfilerProxyStepSendPeerWait_v4 = 20, ncclProfilerProxyStepSendWait = 9, ncclProfilerProxyStepRecvWait = 10, ncclProfilerProxyStepRecvFlushWait = 11, ncclProfilerProxyStepRecvGPUWait = 12, /* Legacy proxy control states */ ncclProfilerProxyCtrlIdle = 13, ncclProfilerProxyCtrlActive = 14, ncclProfilerProxyCtrlSleep = 15, ncclProfilerProxyCtrlWakeup = 16, ncclProfilerProxyCtrlAppend = 17, ncclProfilerProxyCtrlAppendEnd = 18, /* Network defined event states */ ncclProfilerNetPluginUpdate = 21, /* Kernel event states */ ncclProfilerKernelChStop = 22, /* Group API States */ ncclProfilerGroupStartApiStop = 23, ncclProfilerGroupEndApiStart = 24, /* CE-specific states (v6) */ ncclProfilerCeCollStart = 25, // CE collective operation begins ncclProfilerCeCollComplete = 26, // CE collective operation completes ncclProfilerCeSyncStart = 27, // CE synchronization begins ncclProfilerCeSyncComplete = 28, // CE synchronization completes ncclProfilerCeBatchStart = 29, // CE batch operation begins ncclProfilerCeBatchComplete = 30, // CE batch operation completes } ncclProfilerEventState_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v1_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v2_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v3_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v4_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v5_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v6_t; #include "profiler_v6.h" #include "profiler_v5.h" #include "profiler_v4.h" #include "profiler_v3.h" #include "profiler_v2.h" #include "profiler_v1.h" #include "profiler_net.h" // Use v6 as default to support CE events // v5 and earlier versions are still supported for backward compatibility typedef ncclProfiler_v6_t ncclProfiler_t; typedef ncclProfilerEventDescr_v6_t ncclProfilerEventDescr_t; typedef ncclProfilerEventStateArgs_v6_t ncclProfilerEventStateArgs_t; #endif // end include guard nccl-2.29.7-1/plugins/profiler/example/nccl/profiler_net.h000066400000000000000000000014571515037102200234110ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_NET_H_ #define PROFILER_NET_H_ #define NCCL_PROFILER_NET_VER_BITS (16) #define NCCL_PROFILER_NET_VER_MASK (~0U >> NCCL_PROFILER_NET_VER_BITS) #define NCCL_PROFILER_NET_TYPE_MASK (~0U << NCCL_PROFILER_NET_VER_BITS) typedef enum { NCCL_PROFILER_NET_TYPE_IB = (1U << NCCL_PROFILER_NET_VER_BITS), NCCL_PROFILER_NET_TYPE_SOCK = (2U << NCCL_PROFILER_NET_VER_BITS), } ncclProfilerNetType; #include "net_ib_v1.h" #include "net_socket_v1.h" #endif nccl-2.29.7-1/plugins/profiler/example/nccl/profiler_v1.h000066400000000000000000000065571515037102200231570ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V1_H_ #define PROFILER_V1_H_ #include typedef struct { uint8_t type; // event type descriptor: ncclProfileColl, ... void* parentObj; // pointer to the profiler parent object (for coll is the group) int rank; // originating rank union { struct { const char* name; uint64_t commHash; uint64_t seqNumber; uint8_t func; void const* sendBuff; void* recvBuff; size_t count; int root; uint8_t datatype; uint32_t op; size_t trafficBytes; uint8_t nMaxChannels; uint8_t nWarps; uint8_t algo; uint8_t proto; int isCollnet; int isNvls; } coll; struct { const char* name; uint64_t commHash; uint8_t func; void* buff; uint8_t datatype; size_t count; int peer; } p2p; struct { pid_t pid; // pid of the originating process uint8_t channelId; // channel id for this proxy operation int peer; // remote rank for send/recv int nSteps; // number of steps for this proxy operation int chunkSize; // amount of data transferred by this proxy operation int isSend; } proxyOp; struct { int step; } proxyStep; }; } ncclProfilerEventDescr_v1_t; typedef union { struct { size_t transSize; int steps; } proxyOp; struct { int appendedProxyOps; } proxyCtrl; } ncclProfilerEventStateArgs_v1_t; typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, int* eActivationMask); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v1_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v1_t eState, ncclProfilerEventStateArgs_v1_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v1_t; #endif nccl-2.29.7-1/plugins/profiler/example/nccl/profiler_v2.h000066400000000000000000000065151515037102200231520ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V2_H_ #define PROFILER_V2_H_ #include typedef struct { uint8_t type; // event type descriptor: ncclProfileColl, ... void* parentObj; // pointer to the profiler parent object (for coll is the group) int rank; // originating rank union { struct { const char* name; uint64_t commHash; uint64_t seqNumber; const char* func; void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; size_t trafficBytes; uint8_t nMaxChannels; uint8_t nWarps; const char* algo; const char* proto; } coll; struct { const char* name; uint64_t commHash; const char* func; void* buff; const char* datatype; size_t count; int peer; } p2p; struct { pid_t pid; // pid of the originating process uint8_t channelId; // channel id for this proxy operation int peer; // remote rank for send/recv int nSteps; // number of steps for this proxy operation int chunkSize; // amount of data transferred by this proxy operation int isSend; } proxyOp; struct { int step; } proxyStep; }; } ncclProfilerEventDescr_v2_t; typedef union { struct { size_t transSize; int steps; } proxyOp; struct { int appendedProxyOps; } proxyCtrl; } ncclProfilerEventStateArgs_v2_t; typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, int* eActivationMask); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v2_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v2_t eState, ncclProfilerEventStateArgs_v2_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v2_t; #endif nccl-2.29.7-1/plugins/profiler/example/nccl/profiler_v3.h000066400000000000000000000066541515037102200231570ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V3_H_ #define PROFILER_V3_H_ #include typedef struct { uint8_t type; // event type descriptor: ncclProfileColl, ... void* parentObj; // pointer to the profiler parent object (for coll is the group) int rank; // originating rank union { struct { const char* name; uint64_t commHash; uint64_t seqNumber; const char* func; void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; uint8_t nMaxChannels; uint8_t nWarps; const char* algo; const char* proto; } coll; struct { const char* name; uint64_t commHash; const char* func; void* buff; const char* datatype; size_t count; int peer; } p2p; struct { pid_t pid; // pid of the originating process uint8_t channelId; // channel id for this proxy operation int peer; // remote rank for send/recv int nSteps; // number of steps for this proxy operation int chunkSize; // amount of data transferred by this proxy operation int isSend; } proxyOp; struct { int step; } proxyStep; struct { uint8_t channelId; } kernelCh; struct { int64_t id; void* data; } netPlugin; }; } ncclProfilerEventDescr_v3_t; typedef union { struct { size_t transSize; int steps; } proxyOp; struct { int appendedProxyOps; } proxyCtrl; } ncclProfilerEventStateArgs_v3_t; typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, int* eActivationMask); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v3_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v3_t eState, ncclProfilerEventStateArgs_v3_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v3_t; #endif nccl-2.29.7-1/plugins/profiler/example/nccl/profiler_v4.h000066400000000000000000000075721515037102200231600ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V4_H_ #define PROFILER_V4_H_ typedef struct { uint8_t type; // event type descriptor: ncclProfileColl, ... void* parentObj; // pointer to the profiler parent object (for coll is the group) int rank; // originating rank union { struct { uint64_t seqNumber; const char* func; void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; uint8_t nChannels; uint8_t nWarps; const char* algo; const char* proto; } coll; struct { const char* func; void* buff; const char* datatype; size_t count; int peer; uint8_t nChannels; } p2p; struct { pid_t pid; // pid of the originating process uint8_t channelId; // channel id for this proxy operation int peer; // remote rank for send/recv int nSteps; // number of steps for this proxy operation int chunkSize; // amount of data transferred by this proxy operation int isSend; } proxyOp; struct { int step; } proxyStep; struct { uint8_t channelId; uint64_t pTimer; // start timestamp from GPU globaltimer } kernelCh; struct { int64_t id; void* data; } netPlugin; }; } ncclProfilerEventDescr_v4_t; typedef union { struct { size_t transSize; } proxyStep; struct { int appendedProxyOps; } proxyCtrl; struct { void* data; } netPlugin; struct { uint64_t pTimer; } kernelCh; } ncclProfilerEventStateArgs_v4_t; typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // - commName : user assigned communicator name // - commHash : communicator id // - nNodes : number of nodes in communicator // - nranks : number of ranks in communciator // - rank : rank identifier in communicator // - logfn : logger function // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, int* eActivationMask, const char* commName, uint64_t commHash, int nNodes, int nranks, int rank, ncclDebugLogger_t logfn); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v4_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v4_t eState, ncclProfilerEventStateArgs_v4_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v4_t; #endif nccl-2.29.7-1/plugins/profiler/example/nccl/profiler_v5.h000066400000000000000000000106601515037102200231510ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V5_H_ #define PROFILER_V5_H_ #include typedef struct { uint64_t type; // event type descriptor: ncclProfileGroupApi, ... void* parentObj; // pointer to the profiler parent object int rank; // originating rank union { struct { bool graphCaptured; int groupDepth; } groupApi; struct { const char* func; size_t count; const char* datatype; int root; void* stream; bool graphCaptured; } collApi; struct { const char* func; size_t count; const char* datatype; void* stream; bool graphCaptured; } p2pApi; struct { void* stream; } kernelLaunch; struct { uint64_t seqNumber; const char* func; void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; uint8_t nChannels; uint8_t nWarps; const char* algo; const char* proto; void* parentGroup; // for backward compatibility with v4 } coll; struct { const char* func; void* buff; const char* datatype; size_t count; int peer; uint8_t nChannels; void* parentGroup; // for backward compatibility with v4 } p2p; struct { pid_t pid; // pid of the originating process uint8_t channelId; // channel id for this proxy operation int peer; // remote rank for send/recv int nSteps; // number of steps for this proxy operation int chunkSize; // amount of data transferred by this proxy operation int isSend; } proxyOp; struct { int step; } proxyStep; struct { uint8_t channelId; uint64_t pTimer; // start timestamp from GPU globaltimer } kernelCh; struct { int64_t id; void* data; } netPlugin; }; } ncclProfilerEventDescr_v5_t; typedef union { struct { size_t transSize; } proxyStep; struct { int appendedProxyOps; } proxyCtrl; struct { void* data; } netPlugin; struct { uint64_t pTimer; } kernelCh; } ncclProfilerEventStateArgs_v5_t; typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // - commId : communicator id // - commName : user assigned communicator name // - nNodes : number of nodes in communicator // - nranks : number of ranks in communicator // - rank : rank identifier in communicator // - logfn : logger function // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, uint64_t commId, int* eActivationMask, const char* commName, int nNodes, int nranks, int rank, ncclDebugLogger_t logfn); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v5_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v5_t eState, ncclProfilerEventStateArgs_v5_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v5_t; #endif nccl-2.29.7-1/plugins/profiler/example/nccl/profiler_v6.h000066400000000000000000000071551515037102200231570ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V6_H_ #define PROFILER_V6_H_ #include "profiler_v5.h" // Extend v5 descriptor with CE-specific fields typedef struct { uint64_t type; // event type descriptor void* parentObj; // pointer to the profiler parent object int rank; // originating rank union { // All v5 descriptors (groupApi, collApi, p2pApi, kernelLaunch, coll, p2p, proxyOp, proxyStep, kernelCh, netPlugin) struct { bool graphCaptured; int groupDepth; } groupApi; struct { const char* func; size_t count; const char* datatype; int root; void* stream; bool graphCaptured; } collApi; struct { const char* func; size_t count; const char* datatype; void* stream; bool graphCaptured; } p2pApi; struct { void* stream; } kernelLaunch; struct { uint64_t seqNumber; const char* func; void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; uint8_t nChannels; uint8_t nWarps; const char* algo; const char* proto; void* parentGroup; } coll; struct { const char* func; void* buff; const char* datatype; size_t count; int peer; uint8_t nChannels; void* parentGroup; } p2p; struct { pid_t pid; uint8_t channelId; int peer; int nSteps; int chunkSize; int isSend; } proxyOp; struct { int step; } proxyStep; struct { uint8_t channelId; uint64_t pTimer; } kernelCh; struct { int64_t id; void* data; } netPlugin; // v6 CE-specific descriptors struct { uint64_t seqNumber; const char* func; void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; const char* syncStrategy; bool intraBatchSync; uint32_t batchSize; uint32_t numBatches; uint32_t ceSeqNum; void* stream; } ceColl; struct { bool isComplete; int nRanks; } ceCollSync; struct { int numOps; size_t totalBytes; bool useIntraSync; } ceCollBatch; }; } ncclProfilerEventDescr_v6_t; // v6 uses same state args as v5 (no CE-specific state args needed) // CE events don't use recordEventState - plugin manages all timing internally typedef ncclProfilerEventStateArgs_v5_t ncclProfilerEventStateArgs_v6_t; typedef struct { const char* name; // init - initialize the profiler plugin ncclResult_t (*init)(void** context, uint64_t commId, int* eActivationMask, const char* commName, int nNodes, int nranks, int rank, ncclDebugLogger_t logfn); // startEvent - initialize and start a new event ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v6_t* eDescr); // stopEvent - stop/finalize an event ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and updates ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v6_t eState, ncclProfilerEventStateArgs_v6_t* eStateArgs); // finalize - finalize the profiler plugin ncclResult_t (*finalize)(void* context); } ncclProfiler_v6_t; #endif // PROFILER_V6_H_ nccl-2.29.7-1/plugins/profiler/example/nccl/types.h000066400000000000000000000015651515037102200220650ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_TYPES_H_ #define NCCL_TYPES_H_ /* Data types */ typedef enum { ncclInt8 = 0, ncclChar = 0, ncclUint8 = 1, ncclInt32 = 2, ncclInt = 2, ncclUint32 = 3, ncclInt64 = 4, ncclUint64 = 5, ncclFloat16 = 6, ncclHalf = 6, ncclFloat32 = 7, ncclFloat = 7, ncclFloat64 = 8, ncclDouble = 8, ncclBfloat16 = 9, } ncclDataType_t; #endif nccl-2.29.7-1/plugins/profiler/example/plugin.cc000066400000000000000000001163211515037102200214330ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include #include #include #include #include #include #include #include #include "event.h" #include "print_event.h" #include "profiler_plugin_ce.h" #define __hidden __attribute__ ((visibility("hidden"))) static int initialized; // initialization counter for profiler static double startTime; // profiler start time static const int defaultEActivationMask = ncclProfileColl | ncclProfileP2p; static const int defaultGroupApiPoolSize = 8; static const int defaultCollApiPoolSize = 8; static const int defaultP2pApiPoolSize = 8; static const int defaultKernelLaunchPoolSize = 8; static const int defaultGroupPoolSize = 8; static const int defaultCeCollPoolSize = 8; static const int defaultCeSyncPoolSize = 8; static const int defaultCeBatchPoolSize = 8; static const int defaultCollPoolSize = 8; static const int defaultP2pPoolSize = 8; static const int defaultProxyCtrlPoolSize = 16; static const int defaultDetachPoolSize = 8; static int groupApiPoolSize; static int collApiPoolSize; static int p2pApiPoolSize; static int kernelLaunchPoolSize; static int groupPoolSize; static int collPoolSize; static int p2pPoolSize; static int proxyCtrlPoolSize; static int ceCollPoolSize; static int ceSyncPoolSize; static int ceBatchPoolSize; static int detachPoolSize; static int detachPoolBase; static int detachPoolIndex; static int detachPoolDone; static struct proxyOp* detachPool; ncclDebugLogger_t logFn; #define INFO(FLAGS, ...) logFn(NCCL_LOG_INFO, (FLAGS), __func__, __LINE__, __VA_ARGS__) __hidden double gettime(void) { using namespace std::chrono; auto now = steady_clock::now(); return duration_cast>(now.time_since_epoch()).count(); } // Export startTime for CE profiler double getProfilerStartTime(void) { return startTime; } static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; static pid_t pid; static int* eActivationMaskPtr; // Initialize pool sizes from environment variables static void initPoolSizes(void) { const char* str; str = getenv("NCCL_PROFILE_GROUP_API_POOL_SIZE"); groupApiPoolSize = str ? atoi(str) : defaultGroupApiPoolSize; str = getenv("NCCL_PROFILE_COLL_API_POOL_SIZE"); collApiPoolSize = str ? atoi(str) : defaultCollApiPoolSize; str = getenv("NCCL_PROFILE_P2P_API_POOL_SIZE"); p2pApiPoolSize = str ? atoi(str) : defaultP2pApiPoolSize; str = getenv("NCCL_PROFILE_KERNEL_LAUNCH_POOL_SIZE"); kernelLaunchPoolSize = str ? atoi(str) : defaultKernelLaunchPoolSize; str = getenv("NCCL_PROFILE_GROUP_POOL_SIZE"); groupPoolSize = str ? atoi(str) : defaultGroupPoolSize; str = getenv("NCCL_PROFILE_COLL_POOL_SIZE"); collPoolSize = str ? atoi(str) : defaultCollPoolSize; str = getenv("NCCL_PROFILE_P2P_POOL_SIZE"); p2pPoolSize = str ? atoi(str) : defaultP2pPoolSize; str = getenv("NCCL_PROFILE_PROXY_CTRL_POOL_SIZE"); proxyCtrlPoolSize = str ? atoi(str) : defaultProxyCtrlPoolSize; str = getenv("NCCL_PROFILE_CE_COLL_POOL_SIZE"); ceCollPoolSize = str ? atoi(str) : defaultCeCollPoolSize; str = getenv("NCCL_PROFILE_CE_SYNC_POOL_SIZE"); ceSyncPoolSize = str ? atoi(str) : defaultCeSyncPoolSize; str = getenv("NCCL_PROFILE_CE_BATCH_POOL_SIZE"); ceBatchPoolSize = str ? atoi(str) : defaultCeBatchPoolSize; str = getenv("NCCL_PROFILE_PROXY_DETACH_POOL_SIZE"); detachPoolSize = str ? atoi(str) : defaultDetachPoolSize; } // Allocate global shared pools static ncclResult_t allocateGlobalPools(void) { detachPool = (struct proxyOp *)calloc(detachPoolSize, sizeof(*detachPool)); if (detachPool == NULL) { return ncclSystemError; } return ncclSuccess; } // Allocate event pools for a context static ncclResult_t allocateContextPools(struct context* ctx) { ctx->groupApiPool = (struct groupApi *)calloc(groupApiPoolSize, sizeof(*ctx->groupApiPool)); if (!ctx->groupApiPool) goto fail; ctx->collApiPool = (struct collApi *)calloc(collApiPoolSize, sizeof(*ctx->collApiPool)); if (!ctx->collApiPool) goto fail; ctx->p2pApiPool = (struct p2pApi *)calloc(p2pApiPoolSize, sizeof(*ctx->p2pApiPool)); if (!ctx->p2pApiPool) goto fail; ctx->kernelLaunchPool = (struct kernelLaunch *)calloc(kernelLaunchPoolSize, sizeof(*ctx->kernelLaunchPool)); if (!ctx->kernelLaunchPool) goto fail; ctx->groupPool = (struct group *)calloc(groupPoolSize, sizeof(*ctx->groupPool)); if (!ctx->groupPool) goto fail; ctx->collPool = (struct collective *)calloc(collPoolSize, sizeof(*ctx->collPool)); if (!ctx->collPool) goto fail; ctx->p2pPool = (struct p2p *)calloc(p2pPoolSize, sizeof(*ctx->p2pPool)); if (!ctx->p2pPool) goto fail; ctx->proxyCtrlPool = (struct proxyCtrl *)calloc(proxyCtrlPoolSize, sizeof(*ctx->proxyCtrlPool)); if (!ctx->proxyCtrlPool) goto fail; ctx->ceCollPool = (struct ceColl *)calloc(ceCollPoolSize, sizeof(*ctx->ceCollPool)); if (!ctx->ceCollPool) goto fail; ctx->ceCollPoolSize = ceCollPoolSize; ctx->ceCollPoolBase = 0; ctx->ceCollPoolIndex = 0; ctx->ceSyncPool = (struct ceSync *)calloc(ceSyncPoolSize, sizeof(*ctx->ceSyncPool)); if (!ctx->ceSyncPool) goto fail; ctx->ceSyncPoolSize = ceSyncPoolSize; ctx->ceSyncPoolBase = 0; ctx->ceSyncPoolIndex = 0; ctx->ceBatchPool = (struct ceBatch *)calloc(ceBatchPoolSize, sizeof(*ctx->ceBatchPool)); if (!ctx->ceBatchPool) goto fail; ctx->ceBatchPoolSize = ceBatchPoolSize; ctx->ceBatchPoolBase = 0; ctx->ceBatchPoolIndex = 0; return ncclSuccess; fail: if (ctx->ceBatchPool) free(ctx->ceBatchPool); if (ctx->ceSyncPool) free(ctx->ceSyncPool); if (ctx->ceCollPool) free(ctx->ceCollPool); if (ctx->proxyCtrlPool) free(ctx->proxyCtrlPool); if (ctx->p2pPool) free(ctx->p2pPool); if (ctx->collPool) free(ctx->collPool); if (ctx->groupPool) free(ctx->groupPool); if (ctx->collApiPool) free(ctx->collApiPool); if (ctx->p2pApiPool) free(ctx->p2pApiPool); if (ctx->kernelLaunchPool) free(ctx->kernelLaunchPool); if (ctx->groupApiPool) free(ctx->groupApiPool); return ncclSystemError; } // One-time global profiler initialization static ncclResult_t initGlobalProfiler(int* eActivationMask) { const char* str = getenv("NCCL_PROFILE_EVENT_MASK"); __atomic_store_n(eActivationMask, str ? atoi(str) : 0, __ATOMIC_RELAXED); initPoolSizes(); ncclResult_t ret = allocateGlobalPools(); if (ret != ncclSuccess) { return ret; } pid = getpid(); startTime = gettime(); // Only start CE poller thread if CE events are enabled AND at least one CE pool is allocated if ((*eActivationMask & (ncclProfileCeColl | ncclProfileCeSync | ncclProfileCeBatch)) && (ceCollPoolSize > 0 || ceSyncPoolSize > 0 || ceBatchPoolSize > 0)) { ncclResult_t ret = ceProfilerInitGlobal(); if (ret != ncclSuccess) { return ret; } } return ncclSuccess; } __hidden ncclResult_t exampleProfilerInit(void** context, uint64_t commId, int* eActivationMask, const char* commName, int nNodes, int nranks, int rank, ncclDebugLogger_t logfn) { if (pthread_mutex_trylock(&lock) != 0) { *context = NULL; return ncclSuccess; } if (__atomic_fetch_add(&initialized, 1, __ATOMIC_RELAXED) == 0) { ncclResult_t ret = initGlobalProfiler(eActivationMask); if (ret != ncclSuccess) { pthread_mutex_unlock(&lock); return ret; } } pthread_mutex_unlock(&lock); eActivationMaskPtr = eActivationMask; struct context* ctx = (struct context *)calloc(1, sizeof(*ctx)); if (!ctx) return ncclSystemError; ctx->commName = commName; ctx->commHash = commId; ctx->nranks = nranks; ctx->rank = rank; logFn = logfn; INFO(NCCL_INIT, "PROFILER/Plugin: init commName: %s commHash: %lu nranks: %d rank: %d", commName ? commName : "", commId, nranks, rank); ncclResult_t ret = allocateContextPools(ctx); if (ret != ncclSuccess) { free(ctx); return ret; } ceProfilerRegisterContext(ctx); *context = ctx; return ncclSuccess; } static const char* profilerDumpFile; // Open trace file for writing static FILE* openTraceFile(struct context* ctx, char* filename, size_t filenameSize) { const char* dump = profilerDumpFile ? profilerDumpFile : getenv("NCCL_PROFILE_DUMP_FILE"); if (!dump) return NULL; snprintf(filename, filenameSize, "%s_%lu_%d.json", dump, ctx->commHash, ctx->rank); FILE* fh = fopen(filename, "w"); if (fh) { fprintf(fh, "[\n"); } return fh; } // Print all events to trace file static void printAllEvents(FILE* fh, struct context* ctx) { if (!fh) return; int start, end; start = (ctx->groupApiPoolIndex - groupApiPoolSize >= 0) ? ctx->groupApiPoolIndex - groupApiPoolSize : 0; end = ctx->groupApiPoolIndex; for (int i = start; i < end; i++) { printEvent(fh, &ctx->groupApiPool[i % groupApiPoolSize]); } start = (ctx->proxyCtrlPoolIndex - proxyCtrlPoolSize >= 0) ? ctx->proxyCtrlPoolIndex - proxyCtrlPoolSize : 0; end = ctx->proxyCtrlPoolIndex; for (int i = start; i < end; i++) { printEvent(fh, &ctx->proxyCtrlPool[i % proxyCtrlPoolSize]); } // Print orphan CeColl events (those without CollApi parent) start = (ctx->ceCollPoolIndex - ctx->ceCollPoolSize >= 0) ? ctx->ceCollPoolIndex - ctx->ceCollPoolSize : 0; end = ctx->ceCollPoolIndex; for (int i = start; i < end; i++) { struct ceColl* event = &ctx->ceCollPool[i % ctx->ceCollPoolSize]; // Only print if no parent (orphan) AND completed if (!event->parent && event->stopCompleted) { printEvent(fh, event); } } // CeSync and CeBatch are printed via their CeColl parent } // Free all context pools static void freeContextPools(struct context* ctx) { free(ctx->groupPool); free(ctx->collApiPool); free(ctx->p2pApiPool); free(ctx->kernelLaunchPool); free(ctx->groupApiPool); free(ctx->collPool); free(ctx->p2pPool); free(ctx->proxyCtrlPool); free(ctx->ceCollPool); free(ctx->ceSyncPool); free(ctx->ceBatchPool); } // Global cleanup on last thread static void finalizeGlobalProfiler(FILE* fh) { if (fh) { int start = (detachPoolIndex - detachPoolSize >= 0) ? detachPoolIndex - detachPoolSize : 0; int end = detachPoolIndex; for (int i = start; i < end; i++) { printEvent(fh, &detachPool[i % detachPoolSize]); } } free(detachPool); ceProfilerFinalizeGlobal(fh); } __hidden ncclResult_t exampleProfilerFinalize(void* context) { struct context* ctx = (struct context *)context; if (ctx == NULL) { return ncclSuccess; } char filename[PATH_MAX] = { 0 }; FILE* fh = openTraceFile(ctx, filename, sizeof(filename)); INFO(NCCL_INIT, "PROFILER/Plugin: finalize commName: %s commHash: %lu nranks: %d rank: %d traceFile: %s", ctx->commName ? ctx->commName : "", ctx->commHash, ctx->nranks, ctx->rank, filename[0] ? filename : "none"); // Wait for poller to complete pending events usleep(10000); // Check how many completed events we have int completedCount = 0; int totalCount = 0; for (int i = 0; i < ctx->ceCollPoolSize && i < ctx->ceCollPoolIndex; i++) { totalCount++; if (ctx->ceCollPool[i].stopCompleted) completedCount++; } INFO(NCCL_INIT, "PROFILER/Plugin: CeColl events - total=%d completed=%d", totalCount, completedCount); // Print events first (while pools are still valid) printAllEvents(fh, ctx); // Then cleanup and free resources ceProfilerCleanupPendingEvents(ctx); ceProfilerDeregisterContext(ctx); freeContextPools(ctx); free(ctx); if (__atomic_sub_fetch(&initialized, 1, __ATOMIC_RELAXED) == 0) { finalizeGlobalProfiler(fh); } if (fh) fprintf(fh, "{}]\n"); if (fh) fclose(fh); return ncclSuccess; } __hidden void updateEvent(void* handle); __hidden ncclResult_t exampleProfilerStartEvent(void* context, void** eHandle, ncclProfilerEventDescr_t* eDescr) { *eHandle = NULL; struct context* ctx = (struct context *)context; if (ctx == NULL) { return ncclSuccess; } if (eDescr->type == ncclProfileGroupApi) { struct groupApi* event; int groupApiId = __atomic_fetch_add(&ctx->groupApiPoolIndex, 1, __ATOMIC_RELAXED); if ((groupApiId - __atomic_load_n(&ctx->groupApiPoolBase, __ATOMIC_RELAXED)) < groupApiPoolSize) { // if there are available group API events grab one event = &ctx->groupApiPool[groupApiId%groupApiPoolSize]; // Make sure all child events of the picked group API event are cleared while (!profilerQueueEmpty(&event->collApiEvents)) { struct collApi *collApiEvent = profilerQueueDequeue(&event->collApiEvents); resetTaskEvents(collApiEvent, ctx); __atomic_fetch_add(&ctx->collApiPoolBase, 1, __ATOMIC_RELAXED); } while (!profilerQueueEmpty(&event->p2pApiEvents)) { struct p2pApi *p2pApiEvent = profilerQueueDequeue(&event->p2pApiEvents); resetTaskEvents(p2pApiEvent, ctx); __atomic_fetch_add(&ctx->p2pApiPoolBase, 1, __ATOMIC_RELAXED); } while (!profilerQueueEmpty(&event->kernelLaunchEvents)) { profilerQueueDequeue(&event->kernelLaunchEvents); __atomic_fetch_add(&ctx->kernelLaunchPoolBase, 1, __ATOMIC_RELAXED); } } else { // else drop this event __atomic_fetch_sub(&ctx->groupApiPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } event->type = ncclProfileGroupApi; event->ctx = ctx; event->groupApiId = groupApiId; event->graphCaptured = eDescr->groupApi.graphCaptured; event->groupDepth = eDescr->groupApi.groupDepth; event->startTs = gettime() - startTime; *eHandle = event; } else if (eDescr->type == ncclProfileCollApi) { if (eDescr->parentObj == NULL) return ncclSuccess; struct collApi* event; int collApiId = __atomic_fetch_add(&ctx->collApiPoolIndex, 1, __ATOMIC_RELAXED); if ((collApiId - __atomic_load_n(&ctx->collApiPoolBase, __ATOMIC_RELAXED)) < collApiPoolSize) { // if there are available Coll API events grab one event = &ctx->collApiPool[collApiId%collApiPoolSize]; resetTaskEvents(event, ctx); } else { // else drop this event __atomic_fetch_sub(&ctx->collApiPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } event->type = ncclProfileCollApi; event->collApiId = collApiId; event->ctx = ctx; event->func = eDescr->collApi.func; event->stream = (cudaStream_t) eDescr->collApi.stream; event->count = eDescr->collApi.count; event->datatype = eDescr->collApi.datatype; event->root = eDescr->collApi.root; event->graphCaptured = eDescr->collApi.graphCaptured; struct groupApi* parent = (struct groupApi *) eDescr->parentObj; event->parent = parent; profilerQueueEnqueue(&parent->collApiEvents, event); __atomic_fetch_add(&parent->refCount, 1, __ATOMIC_RELAXED); *eHandle = event; } else if (eDescr->type == ncclProfileP2pApi) { if (eDescr->parentObj == NULL) return ncclSuccess; struct p2pApi* event; int p2pApiId = __atomic_fetch_add(&ctx->p2pApiPoolIndex, 1, __ATOMIC_RELAXED); if ((p2pApiId - __atomic_load_n(&ctx->p2pApiPoolBase, __ATOMIC_RELAXED)) < p2pApiPoolSize) { // if there are available p2p API events grab one event = &ctx->p2pApiPool[p2pApiId%p2pApiPoolSize]; resetTaskEvents(event, ctx); } else { // else drop this event __atomic_fetch_sub(&ctx->p2pApiPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } event->type = ncclProfileP2pApi; event->p2pApiId = p2pApiId; event->ctx = ctx; event->func = eDescr->p2pApi.func; event->stream = (cudaStream_t) eDescr->p2pApi.stream; event->count = eDescr->p2pApi.count; event->datatype = eDescr->p2pApi.datatype; event->graphCaptured = eDescr->p2pApi.graphCaptured; struct groupApi* parent = (struct groupApi *) eDescr->parentObj; event->parent = parent; profilerQueueEnqueue(&parent->p2pApiEvents, event); __atomic_fetch_add(&parent->refCount, 1, __ATOMIC_RELAXED); *eHandle = event; } else if (eDescr->type == ncclProfileKernelLaunch) { if (eDescr->parentObj == NULL) return ncclSuccess; struct kernelLaunch* event; int kernelLaunchId = __atomic_fetch_add(&ctx->kernelLaunchPoolIndex, 1, __ATOMIC_RELAXED); if ((kernelLaunchId - __atomic_load_n(&ctx->kernelLaunchPoolBase, __ATOMIC_RELAXED)) < kernelLaunchPoolSize) { // if there are available kernel API events grab one event = &ctx->kernelLaunchPool[kernelLaunchId%kernelLaunchPoolSize]; } else { // else drop this event __atomic_fetch_sub(&ctx->kernelLaunchPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } event->type = ncclProfileKernelLaunch; event->stream = (cudaStream_t) eDescr->kernelLaunch.stream; struct groupApi* parent = (struct groupApi *) eDescr->parentObj; event->parent = parent; profilerQueueEnqueue(&parent->kernelLaunchEvents, event); __atomic_fetch_add(&parent->refCount, 1, __ATOMIC_RELAXED); *eHandle = event; } else if (eDescr->type == ncclProfileGroup) { if (eDescr->parentObj == NULL) return ncclSuccess; struct group* event; int groupId = __atomic_fetch_add(&ctx->groupPoolIndex, 1, __ATOMIC_RELAXED); if ((groupId - __atomic_load_n(&ctx->groupPoolBase, __ATOMIC_RELAXED)) < groupPoolSize) { // if there are available group events grab one event = &ctx->groupPool[groupId%groupPoolSize]; while (!taskEventQueueEmpty(event)) { struct taskEventBase* base = taskEventQueueDequeue(event); if (base->type == ncclProfileColl) { struct collective* c = (struct collective *)base; // reset event proxyOps & proxySteps memset(c->nProxyOps, 0, sizeof(int)*MAX_CHANNELS); // release collective events in the group and return them to the collective pool __atomic_fetch_add(&ctx->collPoolBase, 1, __ATOMIC_RELAXED); } else if (base->type == ncclProfileP2p) { struct p2p* p = (struct p2p *)base; // reset event proxyOp and proxySteps memset(&p->op, 0, sizeof(struct proxyOp)*MAX_CHANNELS); // release p2p events in the group and return them to the p2p pool __atomic_fetch_add(&ctx->p2pPoolBase, 1, __ATOMIC_RELAXED); } } } else { // else drop this event __atomic_fetch_sub(&ctx->groupPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } event->type = ncclProfileGroup; event->ctx = ctx; event->groupId = groupId; event->startTs = gettime() - startTime; *eHandle = event; debugEvent(event, "GroupStart"); } else if (eDescr->type == ncclProfileColl) { // the parent might be null if we run out of events struct collApi* parent = (struct collApi *)eDescr->parentObj; if (parent == NULL) return ncclSuccess; struct collective* event; int collId = __atomic_fetch_add(&ctx->collPoolIndex, 1, __ATOMIC_RELAXED); if ((collId - __atomic_load_n(&ctx->collPoolBase, __ATOMIC_RELAXED)) < collPoolSize) { // if there are available collective events grab one event = &ctx->collPool[collId%collPoolSize]; } else { // else drop this event __atomic_fetch_sub(&ctx->collPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } event->base.type = ncclProfileColl; event->base.rank = eDescr->rank; event->base.func = eDescr->coll.func; event->base.startTs = gettime() - startTime; event->base.parent = parent; event->seqNumber = eDescr->coll.seqNumber; event->sendBuff = eDescr->coll.sendBuff; event->recvBuff = eDescr->coll.recvBuff; event->count = eDescr->coll.count; event->root = eDescr->coll.root; event->datatype = eDescr->coll.datatype; event->nChannels = eDescr->coll.nChannels; event->nWarps = eDescr->coll.nWarps; event->algo = eDescr->coll.algo; event->proto = eDescr->coll.proto; *eHandle = event; taskEventQueueEnqueue(parent, (struct taskEventBase *)event); // increment the group ref counter so the event will stay open __atomic_fetch_add(&parent->refCount, 1, __ATOMIC_RELAXED); debugEvent(event, "CollStart"); } else if (eDescr->type == ncclProfileP2p) { // the parent might be null if we run out of events struct p2pApi* parent = (struct p2pApi*) eDescr->parentObj; if (parent == NULL) return ncclSuccess; struct p2p* event; int p2pId = __atomic_fetch_add(&ctx->p2pPoolIndex, 1, __ATOMIC_RELAXED); if ((p2pId - __atomic_load_n(&ctx->p2pPoolBase, __ATOMIC_RELAXED)) < p2pPoolSize) { // if there are available p2p events grab one event = &ctx->p2pPool[p2pId%p2pPoolSize]; } else { // else drop this event __atomic_fetch_sub(&ctx->p2pPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } event->base.type = ncclProfileP2p; event->base.rank = eDescr->rank; event->base.func = eDescr->p2p.func; event->base.next = parent->eventHead; event->base.startTs = gettime() - startTime; event->base.parent = parent; event->buff = eDescr->p2p.buff; event->count = eDescr->p2p.count; event->datatype = eDescr->p2p.datatype; event->peer = eDescr->p2p.peer; event->nChannels = eDescr->p2p.nChannels; *eHandle = event; // increment the group ref counter so the event will staty open taskEventQueueEnqueue(parent, (struct taskEventBase *)event); __atomic_fetch_add(&parent->refCount, 1, __ATOMIC_RELAXED); debugEvent(event, "P2pStart"); } else if (eDescr->type == ncclProfileProxyCtrl) { int proxyCtrlId = __atomic_fetch_add(&ctx->proxyCtrlPoolIndex, 1, __ATOMIC_RELAXED); struct proxyCtrl* event = &ctx->proxyCtrlPool[proxyCtrlId%proxyCtrlPoolSize]; event->type = ncclProfileProxyCtrl; event->ctx = ctx; event->startTs = gettime() - startTime; *eHandle = event; } else if (eDescr->type == ncclProfileProxyOp) { // the eventBase might be null if we run out of events struct taskEventBase* eventBase = (struct taskEventBase *)eDescr->parentObj; if (eventBase == NULL) return ncclSuccess; if (eDescr->proxyOp.pid != pid) { // PXN captured proxyOp events struct proxyOp* event; int detachId = __atomic_fetch_add(&detachPoolIndex, 1, __ATOMIC_RELAXED); if ((detachId - detachPoolBase) < detachPoolSize) { // if there are available detached proxyOp events grab one event = &detachPool[detachId%detachPoolSize]; } else { // else drop this event __atomic_fetch_sub(&detachPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } event->type = ncclProfileProxyOp; event->channelId = eDescr->proxyOp.channelId; event->pid = eDescr->proxyOp.pid; event->rank = eDescr->rank; event->peer = eDescr->proxyOp.peer; event->nSteps = eDescr->proxyOp.nSteps; event->chunkSize = eDescr->proxyOp.chunkSize; event->isSend = eDescr->proxyOp.isSend; event->startTs = gettime() - startTime; event->parent = NULL; event->stepCount = 0; *eHandle = event; debugEvent(event, "PxnProxyOpStart"); return ncclSuccess; } if (eventBase->type == ncclProfileColl) { struct collective* parent = (struct collective *)eDescr->parentObj; int channelId = eDescr->proxyOp.channelId; struct proxyOp* event = &parent->op[channelId][parent->nProxyOps[channelId]++]; event->type = ncclProfileProxyOp; event->channelId = channelId; event->pid = eDescr->proxyOp.pid; event->rank = eDescr->rank; event->peer = eDescr->proxyOp.peer; event->nSteps = eDescr->proxyOp.nSteps; event->chunkSize = eDescr->proxyOp.chunkSize; event->isSend = eDescr->proxyOp.isSend; event->parent = eventBase; event->startTs = gettime() - startTime; event->stepCount = 0; *eHandle = event; __atomic_fetch_add(&parent->base.refCount, 1, __ATOMIC_RELAXED); debugEvent(event, "ProxyOpStart"); } else { // ncclProfileP2p struct p2p* parent = (struct p2p *)eDescr->parentObj; int channelId = eDescr->proxyOp.channelId; struct proxyOp* event = &parent->op[channelId]; event->type = ncclProfileProxyOp; event->channelId = channelId; event->pid = eDescr->proxyOp.pid; event->rank = eDescr->rank; event->peer = eDescr->proxyOp.peer; event->nSteps = eDescr->proxyOp.nSteps; event->chunkSize = eDescr->proxyOp.chunkSize; event->isSend = eDescr->proxyOp.isSend; event->parent = eventBase; event->startTs = gettime() - startTime; event->stepCount = 0; *eHandle = event; __atomic_fetch_add(&parent->base.refCount, 1, __ATOMIC_RELAXED); debugEvent(event, "ProxyOpStart"); } } else if (eDescr->type == ncclProfileProxyStep) { // the parent might be null if we run out of events struct proxyOp* parent = (struct proxyOp *)eDescr->parentObj; if (parent == NULL) return ncclSuccess; int s = parent->stepCount++ % MAX_STEPS; struct proxyStep* event = &parent->step[s]; event->type = ncclProfileProxyStep; event->state = 0; event->step = eDescr->proxyStep.step; event->parent = parent; event->isSend = parent->isSend; event->startTs = gettime() - startTime; event->nNetEvents = 0; *eHandle = event; debugEvent(event, "ProxyStepStart"); } else if (eDescr->type == ncclProfileKernelCh) { struct taskEventBase* eventBase = (struct taskEventBase *)eDescr->parentObj; if (eventBase == NULL) return ncclSuccess; if (eventBase->type == ncclProfileColl) { struct collective* parent = (struct collective *)eDescr->parentObj; struct kernelCh* event = &parent->kernel[eDescr->kernelCh.channelId]; event->type = ncclProfileKernelCh; event->channelId = eDescr->kernelCh.channelId; event->startGpuClk = eDescr->kernelCh.pTimer; event->parent = eventBase; event->startTs = gettime() - startTime; *eHandle = event; __atomic_fetch_add(&parent->base.refCount, 1, __ATOMIC_RELAXED); debugEvent(event, "KernelChStart"); } else { // ncclProfileP2p struct p2p* parent = (struct p2p *)eDescr->parentObj; struct kernelCh* event = &parent->kernel[eDescr->kernelCh.channelId]; event->type = ncclProfileKernelCh; event->channelId = eDescr->kernelCh.channelId; event->startGpuClk = eDescr->kernelCh.pTimer; event->parent = eventBase; event->startTs = gettime() - startTime; *eHandle = event; __atomic_fetch_add(&parent->base.refCount, 1, __ATOMIC_RELAXED); debugEvent(event, "KernelChStart"); } } else if (eDescr->type == ncclProfileNetPlugin) { struct proxyStep* parent = (struct proxyStep *)eDescr->parentObj; if (parent == NULL) return ncclSuccess; int64_t pluginId = eDescr->netPlugin.id; int64_t type = pluginId & NCCL_PROFILER_NET_TYPE_MASK; int64_t ver = pluginId & NCCL_PROFILER_NET_VER_MASK; if (type == NCCL_PROFILER_NET_TYPE_IB) { if (ver == 1) { ncclProfilerNetIbDescr_v1_t* descr = (ncclProfilerNetIbDescr_v1_t *)eDescr->netPlugin.data; struct netPlugin* event = parent->net + __atomic_fetch_add(&parent->nNetEvents, 1, __ATOMIC_RELAXED); event->type = ncclProfileNetPlugin; event->pluginType = type; event->pluginVer = ver; if (descr->type == ncclProfileQp) { event->pluginEvent = ncclProfileQp; event->qp.device = descr->qp.device; event->qp.wr_id = descr->qp.wr_id; event->qp.opcode = descr->qp.opcode; event->qp.qpNum = descr->qp.qpNum; event->qp.length = descr->qp.length; } event->startTs = gettime() - startTime; *eHandle = event; debugEvent(event, "NetPluginStart"); } } else if (type == NCCL_PROFILER_NET_TYPE_SOCK) { if (ver == 1) { ncclProfilerNetSockDescr_v1_t* descr = (ncclProfilerNetSockDescr_v1_t *)eDescr->netPlugin.data; struct netPlugin* event = parent->net + __atomic_fetch_add(&parent->nNetEvents, 1, __ATOMIC_RELAXED); event->type = ncclProfileNetPlugin; event->pluginType = type; event->pluginVer = ver; if (descr->type == ncclProfileSocket) { event->pluginEvent = ncclProfileSocket; event->sock.fd = descr->sock.fd; event->sock.op = descr->sock.op; event->sock.length = descr->sock.length; } event->startTs = gettime() - startTime; *eHandle = event; debugEvent(event, "NetPluginStart"); } } } return ncclSuccess; } void updateEvent(void* handle) { uint64_t type = *(uint64_t *)handle; if (type == ncclProfileGroupApi) { struct groupApi* event = (struct groupApi*) handle; if (__atomic_sub_fetch(&event->refCount, 1, __ATOMIC_RELAXED) == 0) { event->stopTs = gettime() - startTime; __atomic_fetch_add(&event->ctx->groupApiPoolBase, 1, __ATOMIC_RELAXED); } } else if (type == ncclProfileCollApi) { struct collApi* event = (struct collApi*) handle; if (__atomic_sub_fetch(&event->refCount, 1, __ATOMIC_RELAXED) == 0) { event->stopTs = gettime() - startTime; __atomic_fetch_add(&event->ctx->collApiPoolBase, 1, __ATOMIC_RELAXED); } updateEvent(event->parent); return; } else if (type == ncclProfileP2pApi) { struct p2pApi* event = (struct p2pApi*) handle; if (__atomic_sub_fetch(&event->refCount, 1, __ATOMIC_RELAXED) == 0) { event->stopTs = gettime() - startTime; __atomic_fetch_add(&event->ctx->p2pApiPoolBase, 1, __ATOMIC_RELAXED); } updateEvent(event->parent); event->stopTs = gettime() - startTime; } else if (type == ncclProfileKernelLaunch) { struct kernelLaunch* event = (struct kernelLaunch*) handle; event->stopTs = gettime() - startTime; updateEvent(event->parent); } else if (type == ncclProfileGroup) { struct group* event = (struct group *)handle; if (__atomic_sub_fetch(&event->refCount, 1, __ATOMIC_RELAXED) == 0) { event->stopTs = gettime() - startTime; // return group event to the pool __atomic_fetch_add(&event->ctx->groupPoolBase, 1, __ATOMIC_RELAXED); } debugEvent(event, "GroupStop"); } else if (type == ncclProfileColl) { struct collective* event = (struct collective *)handle; if (__atomic_sub_fetch(&event->base.refCount, 1, __ATOMIC_RELAXED) == 0) { event->base.stopTs = gettime() - startTime; debugEvent(event, "CollStop"); updateEvent(event->base.parent); return; } debugEvent(event, "CollStop"); } else if (type == ncclProfileP2p) { struct p2p* event = (struct p2p *)handle; if (__atomic_sub_fetch(&event->base.refCount, 1, __ATOMIC_RELAXED) == 0) { event->base.stopTs = gettime() - startTime; debugEvent(event, "P2pStop"); updateEvent(event->base.parent); return; } debugEvent(event, "P2pStop"); } else if (type == ncclProfileProxyOp) { struct proxyOp* event = (struct proxyOp *)handle; event->stopTs = gettime() - startTime; if (event->pid != pid) { // only for proxyOps that don't have a parent collective/p2p (i.e., PXN) int done = __atomic_add_fetch(&detachPoolDone, 1, __ATOMIC_RELAXED); if (done == detachPoolSize) { // reset the event completed (done) counter __atomic_store_n(&detachPoolDone, 0, __ATOMIC_RELAXED); // update the base pointer to the top of the pool int index = __atomic_load_n(&detachPoolIndex, __ATOMIC_RELAXED); __atomic_store_n(&detachPoolBase, index, __ATOMIC_RELAXED); } debugEvent(event, "ProxyOpStop"); return; } updateEvent(event->parent); debugEvent(event, "ProxyOpStop"); } else if (type == ncclProfileProxyStep) { struct proxyStep* event = (struct proxyStep *)handle; event->stopTs = gettime() - startTime; debugEvent(event, "ProxyStepStop"); } else if (type == ncclProfileProxyCtrl) { struct proxyCtrl* event = (struct proxyCtrl *)handle; event->stopTs = gettime() - startTime; debugEvent(event, "ProxyCtrlStop"); } else if (type == ncclProfileKernelCh) { struct kernelCh* event = (struct kernelCh *)handle; event->stopTs = gettime() - startTime; updateEvent(event->parent); debugEvent(event, "KernelChStop"); } else if (type == ncclProfileNetPlugin) { struct netPlugin* event = (struct netPlugin *)handle; event->stopTs = gettime() - startTime; debugEvent(event, "NetPluginStop"); } } __hidden ncclResult_t exampleProfilerStopEvent(void* eHandle) { // the event handle might be null if we run out of events if (eHandle == NULL) return ncclSuccess; uint64_t type = *(uint64_t *)eHandle; // Stopping API events, Kernel Launch events, collective/p2p task events // in NCCL core do not mean that they are complete. It means that the // operation was enqueued so we need to keep the events open if (type == ncclProfileGroupApi) { struct groupApi* event = (struct groupApi*) eHandle; event->stopTs = gettime() - startTime; return ncclSuccess; } else if (type == ncclProfileCollApi) { struct collApi* event = (struct collApi*) eHandle; event->stopTs = gettime() - startTime; return ncclSuccess; } else if (type == ncclProfileP2pApi) { struct p2pApi* event = (struct p2pApi*) eHandle; event->stopTs = gettime() - startTime; return ncclSuccess; } else if (type == ncclProfileKernelLaunch) { struct kernelLaunch* event = (struct kernelLaunch*) eHandle; event->stopTs = gettime() - startTime; return ncclSuccess; } else if (type == ncclProfileGroup) { struct group* event = (struct group *)eHandle; event->stopTs = gettime() - startTime; return ncclSuccess; } else if (type == ncclProfileColl) { struct collective* event = (struct collective *)eHandle; event->base.stopTs = gettime() - startTime; return ncclSuccess; } else if (type == ncclProfileP2p) { struct p2p* event = (struct p2p *)eHandle; event->base.stopTs = gettime() - startTime; return ncclSuccess; } updateEvent(eHandle); return ncclSuccess; } __hidden ncclResult_t exampleProfilerRecordEventState(void* eHandle, ncclProfilerEventState_t eState, ncclProfilerEventStateArgs_t* eStateArgs) { // the event handle might be null if we run out of events if (eHandle == NULL) return ncclSuccess; uint64_t type = *(uint64_t *)eHandle; if (type == ncclProfileGroupApi) { struct groupApi* event = (struct groupApi*) eHandle; if (eState == ncclProfilerGroupEndApiStart) { event->endOfncclGroupStartTs = gettime() - startTime; } else if (eState == ncclProfilerGroupStartApiStop) { event->startOfncclGroupEndTs = gettime() - startTime; } } else if (type == ncclProfileProxyOp) { struct proxyOp* event = (struct proxyOp *)eHandle; if (eState == ncclProfilerProxyOpInProgress_v4) { event->progrTs = gettime() - startTime; } } else if (type == ncclProfileProxyStep) { struct proxyStep* event = (struct proxyStep *)eHandle; struct proxyOp* parent = event->parent; switch (eState) { case ncclProfilerProxyStepSendGPUWait: event->timestamp[PROXY_STEP_SEND_GPU_WAIT] = gettime() - startTime; break; case ncclProfilerProxyStepSendPeerWait_v4: // do not update step event if in SendPeerWait if (event->state == ncclProfilerProxyStepSendPeerWait_v4) break; event->timestamp[PROXY_STEP_SEND_PEER_WAIT] = gettime() - startTime; event->state = ncclProfilerProxyStepSendPeerWait_v4; break; case ncclProfilerProxyStepSendWait: event->timestamp[PROXY_STEP_SEND_WAIT] = gettime() - startTime; parent->transSize += eStateArgs->proxyStep.transSize; break; case ncclProfilerProxyStepRecvWait: event->timestamp[PROXY_STEP_RECV_WAIT] = gettime() - startTime; break; case ncclProfilerProxyStepRecvFlushWait: event->timestamp[PROXY_STEP_RECV_FLUSH_WAIT] = gettime() - startTime; parent->transSize += eStateArgs->proxyStep.transSize; break; case ncclProfilerProxyStepRecvGPUWait: event->timestamp[PROXY_STEP_RECV_GPU_WAIT] = gettime() - startTime; break; default: break; } } else if (type == ncclProfileProxyCtrl) { struct proxyCtrl* event = (struct proxyCtrl *)eHandle; if (eState == ncclProfilerProxyCtrlAppendEnd) { event->appended = eStateArgs->proxyCtrl.appendedProxyOps; } event->state = eState; } else if (type == ncclProfileKernelCh) { struct kernelCh* event = (struct kernelCh *)eHandle; if (eState == ncclProfilerKernelChStop) { event->stopGpuClk = eStateArgs->kernelCh.pTimer; } } debugEvent(eHandle, "RecordEventState"); return ncclSuccess; } ncclProfiler_t ncclProfiler_v5 = { "Example-profiler", exampleProfilerInit, exampleProfilerStartEvent, exampleProfilerStopEvent, exampleProfilerRecordEventState, exampleProfilerFinalize, }; __attribute__((visibility("default"))) int exampleProfilerStart(int eActivationMask, const char* name) { profilerDumpFile = name; if (__atomic_load_n(&initialized, __ATOMIC_RELAXED)) { __atomic_store_n(eActivationMaskPtr, eActivationMask, __ATOMIC_RELAXED); } return ncclSuccess; } __attribute__((visibility("default"))) int exampleProfilerStop(void) { if (__atomic_load_n(&initialized, __ATOMIC_RELAXED)) { __atomic_store_n(eActivationMaskPtr, 0, __ATOMIC_RELAXED); } return ncclSuccess; } // ============================================================================ // v6 implementation with CE events support // ============================================================================ #include "nccl/profiler_v6.h" __hidden ncclResult_t exampleProfilerStartEvent_v6(void* context, void** eHandle, ncclProfilerEventDescr_v6_t* eDescr) { struct context* ctx = (struct context*)context; if (ctx == NULL) { *eHandle = NULL; return ncclSuccess; } if (eDescr->type == ncclProfileCeColl) { return ceProfilerStartCeCollEvent(ctx, eHandle, eDescr, startTime); } if (eDescr->type == ncclProfileCeSync) { return ceProfilerStartCeSyncEvent(ctx, eHandle, eDescr, startTime); } if (eDescr->type == ncclProfileCeBatch) { return ceProfilerStartCeBatchEvent(ctx, eHandle, eDescr, startTime); } return exampleProfilerStartEvent(context, eHandle, (ncclProfilerEventDescr_t*)eDescr); } __hidden ncclResult_t exampleProfilerStopEvent_v6(void* eHandle) { if (!eHandle) return ncclSuccess; uint64_t type = *(uint64_t*)eHandle; // Handle CE events - record stop event to stream if (type == ncclProfileCeColl) { return ceProfilerStopCeCollEvent(eHandle); } if (type == ncclProfileCeSync) { return ceProfilerStopCeSyncEvent(eHandle); } if (type == ncclProfileCeBatch) { return ceProfilerStopCeBatchEvent(eHandle); } // Fall through to v5 handling for non-CE events return exampleProfilerStopEvent(eHandle); } __hidden ncclResult_t exampleProfilerRecordEventState_v6(void* eHandle, ncclProfilerEventState_v6_t eState, ncclProfilerEventStateArgs_v6_t* eStateArgs) { // CE events don't use recordEventState - poller handles all timing // Just fall through to v5 for non-CE events return exampleProfilerRecordEventState(eHandle, (ncclProfilerEventState_t)eState, (ncclProfilerEventStateArgs_t*)eStateArgs); } ncclProfiler_v6_t ncclProfiler_v6 = { "Example-profiler-v6", exampleProfilerInit, exampleProfilerStartEvent_v6, exampleProfilerStopEvent_v6, exampleProfilerRecordEventState_v6, exampleProfilerFinalize, }; nccl-2.29.7-1/plugins/profiler/example/plugin.h000066400000000000000000000010671515037102200212750ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PLUGIN_H_ #define PLUGIN_H_ __attribute__((visibility("default"))) int exampleProfilerStart(int eActivationMask, const char* name); __attribute__((visibility("default"))) int exampleProfilerStop(void); #endif nccl-2.29.7-1/plugins/profiler/example/print_event.cc000066400000000000000000000720341515037102200224740ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include #include "err.h" #include "profiler.h" #include "event.h" #include "print_event.h" #include #define __hidden __attribute__ ((visibility("hidden"))) // FIXME: chrome tracing asynchronous events (following used) allow event nesting for events that have same id and category // It appears that nesting more than three events causes issues. Therefore, every event is given an increasing id and a // category that matches the type of event (GROUP API, COLL API, P2P API, GROUP, COLL, P2P, PROXY, NET) static __thread int groupApiId; __hidden void printGroupApiEventHeader(FILE* fh, struct groupApi* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GROUP_API\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"groupApiId\": %d, \"groupDepth\": %d}},\n", "Group API", groupApiId, getpid(), 1, event->startTs, event->groupApiId, event->groupDepth); } __hidden void printGroupApiEventTrailer(FILE* fh, struct groupApi* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GROUP_API\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "Group API", groupApiId++, getpid(), 1, event->stopTs); } static __thread int p2pApiId; __hidden void printP2pApiEventHeader(FILE* fh, struct p2pApi* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"P2P_API\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"count\": %lu, \"datatype\": \"%s\", \"GraphCaptured\": %d, \"Stream\": \"%p\"}},\n", event->func, p2pApiId, getpid(), 1, event->startTs, event->count, event->datatype, event->graphCaptured, event->stream); } __hidden void printP2pApiEventTrailer(FILE* fh, struct p2pApi* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"P2P_API\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", event->func, p2pApiId++, getpid(), 1, event->stopTs); } static __thread int collApiId; __hidden void printCollApiEventHeader(FILE* fh, struct collApi* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"COLL_API\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"count\": %lu, \"datatype\": \"%s\", \"root\": %d, \"GraphCaptured\": %d, \"Stream\": \"%p\"}},\n", event->func, collApiId, getpid(), 1, event->startTs, event->count, event->datatype, event->root, event->graphCaptured, event->stream); } __hidden void printCollApiEventTrailer(FILE* fh, struct collApi* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"COLL_API\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", event->func, collApiId++, getpid(), 1, event->stopTs); } static __thread int kernelLaunchId; __hidden void printKernelLaunchEventHeader(FILE* fh, struct kernelLaunch* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"KERNEL_LAUNCH\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"groupId\": %d, \"Stream\": \"%p\"}},\n", "KernelLaunch", kernelLaunchId, getpid(), 1, event->startTs, event->kernelLaunchId, event->stream); } __hidden void printKernelLaunchEventTrailer(FILE* fh, struct kernelLaunch* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"KERNEL_LAUNCH\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "KernelLaunch", kernelLaunchId++, getpid(), 1, event->stopTs); } static __thread int groupId; __hidden void printGroupEventHeader(FILE* fh, struct group* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GROUP\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"groupId\": %d}},\n", "Group", groupId, getpid(), 1, event->startTs, event->groupId); } __hidden void printGroupEventTrailer(FILE* fh, struct group* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GROUP\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "Group", groupId++, getpid(), 1, event->stopTs); } static __thread int collId; __hidden void printCollEventHeader(FILE* fh, struct collective* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"COLL\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"SeqNum\": %lu, \"CommHash\": %lu, \"Rank\": %d, \"Count\": %lu, \"Datatype\": \"%s\", \"Algorithm\": \"%s\", \"Protocol\": \"%s\", \"nChannels\": %d}},\n", event->base.func, collId, getpid(), 1, event->base.startTs, event->seqNumber, ((struct collApi*)event->base.parent)->ctx->commHash, event->base.rank, event->count, event->datatype, event->algo, event->proto, event->nChannels); } __hidden void printCollEventTrailer(FILE* fh, struct collective* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"COLL\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", event->base.func, collId++, getpid(), 1, event->base.stopTs); } static __thread int p2pId; __hidden void printP2pEventHeader(FILE* fh, struct p2p* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"P2P\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"CommHash\": %lu, \"Rank\": %d, \"Peer\": %d, \"Count\": %lu, \"Datatype\": \"%s\", \"nChannels\": %d}},\n", event->base.func, p2pId, getpid(), 1, event->base.startTs, ((struct p2pApi*)event->base.parent)->ctx->commHash, event->base.rank, event->peer, event->count, event->datatype, event->nChannels); } __hidden void printP2pEventTrailer(FILE* fh, struct p2p* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"P2P\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", event->base.func, p2pId++, getpid(), 1, event->base.stopTs); } static __thread int proxyOpId; __hidden void printProxyOpEventHeader(FILE* fh, struct proxyOp* event) { if (event->isSend) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"Peer\": %d, \"Steps\": %d, \"ChunkSize\": %d, \"transSize\": %lu}},\n", "ScheduleSend", proxyOpId, getpid(), 1, event->startTs, event->channelId, event->peer, event->nSteps, event->chunkSize, event->transSize); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "ScheduleSend", proxyOpId, getpid(), 1, event->progrTs); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"Peer\": %d, \"Steps\": %d, \"ChunkSize\": %d, \"transSize\": %lu}},\n", "ProgressSend", proxyOpId, getpid(), 1, event->progrTs, event->channelId, event->peer, event->nSteps, event->chunkSize, event->transSize); } else { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"Peer\": %d, \"Steps\": %d, \"ChunkSize\": %d, \"transSize\": %lu}},\n", "ScheduleRecv", proxyOpId, getpid(), 1, event->startTs, event->channelId, event->peer, event->nSteps, event->chunkSize, event->transSize); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "ScheduleRecv", proxyOpId, getpid(), 1, event->progrTs); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"Peer\": %d, \"Steps\": %d, \"ChunkSize\": %d, \"transSize\": %lu}},\n", "ProgressRecv", proxyOpId, getpid(), 1, event->progrTs, event->channelId, event->peer, event->nSteps, event->chunkSize, event->transSize); } } __hidden void printProxyOpEventTrailer(FILE* fh, struct proxyOp* event) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", event->isSend ? "ProgressSend" : "ProgressRecv", proxyOpId++, getpid(), 1, event->stopTs); } static __thread int proxyStepId; __hidden void printProxyStepEventHeader(FILE* fh, struct proxyStep* event) { if (event->isSend) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n", "SendGpuWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_GPU_WAIT], event->step); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "SendGpuWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_PEER_WAIT]); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n", "SendPeerWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_PEER_WAIT], event->step); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "SendPeerWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_WAIT]); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n", "SendWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_SEND_WAIT], event->step); } else { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n", "RecvWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_WAIT], event->step); } } __hidden void printProxyStepEventTrailer(FILE* fh, struct proxyStep* event) { if (event->isSend) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "SendWait", proxyStepId++, getpid(), 1, event->stopTs); } else { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "RecvWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_FLUSH_WAIT]); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n", "RecvFlushWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_FLUSH_WAIT], event->step); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "RecvFlushWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_GPU_WAIT]); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Step\": %d}},\n", "RecvGpuWait", proxyStepId, getpid(), 1, event->timestamp[PROXY_STEP_RECV_GPU_WAIT], event->step); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "RecvGpuWait", proxyStepId++, getpid(), 1, event->stopTs); } } static __thread int kernelId; __hidden void printKernelChEventHeader(FILE* fh, struct kernelCh* event) { if (event->type != ncclProfileKernelCh) return; fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GPU\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"Channel\": %d, \"StartGpuClk\": %lu, \"StopGpuClk\": %lu}},\n", "KernelCh", kernelId, getpid(), 1, event->startTs, event->channelId, event->startGpuClk, event->stopGpuClk); } __hidden void printKernelChEventTrailer(FILE* fh, struct kernelCh* event) { if (event->type != ncclProfileKernelCh) return; fprintf(fh, "{\"name\": \"%s\", \"cat\": \"GPU\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "KernelCh", kernelId, getpid(), 1, event->stopTs); } static __thread int proxyCtrlId; __hidden void printProxyCtrlEvent(FILE* fh, struct proxyCtrl* event) { const char* str; if (event->state == ncclProfilerProxyCtrlIdle || event->state == ncclProfilerProxyCtrlActive) { str = "Idle"; } else if (event->state == ncclProfilerProxyCtrlSleep || event->state == ncclProfilerProxyCtrlWakeup) { str = "Sleep"; } else if (event->state == ncclProfilerProxyCtrlAppend || event->state == ncclProfilerProxyCtrlAppendEnd) { str = "Append"; } else { return; } if (event->state == ncclProfilerProxyCtrlAppendEnd) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"appended\": %d}},\n", str, proxyCtrlId, getpid(), 1, event->startTs, event->appended); } else { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", str, proxyCtrlId, getpid(), 1, event->startTs); } fprintf(fh, "{\"name\": \"%s\", \"cat\": \"PROXY\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", str, proxyCtrlId++, getpid(), 1, event->stopTs); } static __thread int ibQpId, sockId; __hidden void printNetPluginEvent(FILE* fh, struct netPlugin* event) { if (event->pluginType == NCCL_PROFILER_NET_TYPE_IB) { if (event->pluginVer == 1) { if (event->pluginEvent == ncclProfileQp) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET_IB\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"device\": %d, \"qp_num\": %d, \"opcode\": %d, \"wr_id\": %lu, \"size\": %lu}},\n", "Qp", ibQpId, getpid(), 1, event->startTs, event->qp.device, event->qp.qpNum, event->qp.opcode, event->qp.wr_id, event->qp.length); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET_IB\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "Qp", ibQpId++, getpid(), 1, event->stopTs); } } } else if (event->pluginType == NCCL_PROFILER_NET_TYPE_SOCK) { if (event->pluginVer == 1) { if (event->pluginEvent == ncclProfileSocket) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET_SOCK\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"sock\": %d, \"op\": %d, \"size\": %lu}},\n", "Sock", sockId, getpid(), 1, event->startTs, event->sock.fd, event->sock.op, event->sock.length); fprintf(fh, "{\"name\": \"%s\", \"cat\": \"NET_SOCK\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", "Sock", sockId++, getpid(), 1, event->stopTs); } } } } //#define DEBUG_EVENTS // Debug helper functions for CE events #ifdef DEBUG_EVENTS static void debugCeCollEvent(FILE* fh, struct ceColl* event, const char* tag) { fprintf(fh, "CeColl event %p tag = %s {\n", event, tag); fprintf(fh, " func = %s\n", event->base.func); fprintf(fh, " eventId = %lu\n", event->eventId); fprintf(fh, " timingMode = %s\n", event->timingMode == CE_TIMING_GPU ? "gpu" : "cpu"); fprintf(fh, " startCompleted = %d\n", event->startCompleted); fprintf(fh, " stopCompleted = %d\n", event->stopCompleted); fprintf(fh, " startTs = %f\n", event->base.startTs); fprintf(fh, " stopTs = %f\n", event->base.stopTs); fprintf(fh, " cpuDuration = %lu us\n", event->cpuDuration); fprintf(fh, " elapsedTime = %lu us\n", event->elapsedTime); fprintf(fh, "}\n"); } static void debugCeSyncEvent(FILE* fh, struct ceSync* event, const char* tag) { fprintf(fh, "CeSync event %p tag = %s {\n", event, tag); fprintf(fh, " syncType = %s\n", event->isComplete ? "Complete" : "Ready"); fprintf(fh, " eventId = %lu\n", event->eventId); fprintf(fh, " timingMode = %s\n", event->timingMode == CE_TIMING_GPU ? "gpu" : "cpu"); fprintf(fh, " startCompleted = %d\n", event->startCompleted); fprintf(fh, " stopCompleted = %d\n", event->stopCompleted); fprintf(fh, " startTs = %f\n", event->base.startTs); fprintf(fh, " stopTs = %f\n", event->base.stopTs); fprintf(fh, " cpuDuration = %lu us\n", event->cpuDuration); fprintf(fh, "}\n"); } static void debugCeBatchEvent(FILE* fh, struct ceBatch* event, const char* tag) { fprintf(fh, "CeBatch event %p tag = %s {\n", event, tag); fprintf(fh, " numOps = %d\n", event->numOps); fprintf(fh, " totalBytes = %lu\n", event->totalBytes); fprintf(fh, " eventId = %lu\n", event->eventId); fprintf(fh, " timingMode = %s\n", event->timingMode == CE_TIMING_GPU ? "gpu" : "cpu"); fprintf(fh, " startCompleted = %d\n", event->startCompleted); fprintf(fh, " stopCompleted = %d\n", event->stopCompleted); fprintf(fh, " startTs = %f\n", event->base.startTs); fprintf(fh, " stopTs = %f\n", event->base.stopTs); fprintf(fh, " cpuDuration = %lu us\n", event->cpuDuration); fprintf(fh, "}\n"); } #endif void debugEvent(void* eHandle, const char* tag) { #ifdef DEBUG_EVENTS char filename[64] = { 0 }; sprintf(filename, "EventDebug-%d", getpid()); FILE* fh = fopen(filename, "a+"); uint64_t type = *(uint64_t *)eHandle; if (type == ncclProfileGroup) { struct group* event = (struct group *)eHandle; fprintf(fh, "Group event %p tag = %s {\n", event, tag); fprintf(fh, " refCount = %d\n", __atomic_load_n(&event->refCount, __ATOMIC_RELAXED)); fprintf(fh, " startTs = %f\n", event->startTs); fprintf(fh, " stopTs = %f\n", event->stopTs); fprintf(fh, "}\n"); } else if (type == ncclProfileColl) { struct collective* event = (struct collective *)eHandle; fprintf(fh, "Collective event %p tag = %s {\n", event, tag); fprintf(fh, " refCount = %d\n", __atomic_load_n(&event->base.refCount, __ATOMIC_RELAXED)); fprintf(fh, " parent = %p\n", event->base.parent); for (int j = 0; j < 2*MAX_OPS; j++) { for (int i = 0; i < MAX_CHANNELS; i++) if (event->op[i][j].type == ncclProfileProxyOp) fprintf(fh, " op[%d] = %p\n", i, &event->op[i]); } fprintf(fh, " startTs = %f\n", event->base.startTs); fprintf(fh, " stopTs = %f\n", event->base.stopTs); fprintf(fh, "}\n"); } else if (type == ncclProfileP2p) { struct p2p* event = (struct p2p *)eHandle; fprintf(fh, "P2p event %p tag = %s {\n", event, tag); fprintf(fh, " refCount = %d\n", __atomic_load_n(&event->base.refCount, __ATOMIC_RELAXED)); fprintf(fh, " parent = %p\n", event->base.parent); fprintf(fh, " op = %p\n", &event->op); fprintf(fh, " startTs = %f\n", event->base.startTs); fprintf(fh, " stopTs = %f\n", event->base.stopTs); fprintf(fh, "}\n"); } else if (type == ncclProfileProxyOp) { struct proxyOp* event = (struct proxyOp *)eHandle; fprintf(fh, "ProxyOp event %p tag = %s {\n", event, tag); fprintf(fh, " type = %s\n", event->isSend < 0 ? "Unknown" : event->isSend ? "Send" : "Recv"); fprintf(fh, " channel = %d\n", event->channelId); fprintf(fh, " parent = %p\n", event->parent); fprintf(fh, " rank = %d\n", event->rank); fprintf(fh, " startTs = %f\n", event->startTs); fprintf(fh, " progrTs = %f\n", event->progrTs); fprintf(fh, " stopTs = %f\n", event->stopTs); fprintf(fh, "}\n"); } else if (type == ncclProfileProxyStep) { struct proxyStep* event = (struct proxyStep *)eHandle; fprintf(fh, "ProxyStep event %p tag = %s {\n", event, tag); fprintf(fh, " type = %s\n", event->isSend < 0 ? "Unknown" : event->isSend ? "Send" : "Recv"); fprintf(fh, " parent = %p\n", event->parent); fprintf(fh, " startTs = %f\n", event->startTs); fprintf(fh, " stopTs = %f\n", event->stopTs); fprintf(fh, "}\n"); } else if (type == ncclProfileKernelCh) { struct kernelCh* event = (struct kernelCh *)eHandle; fprintf(fh, "KernelCh event %p tag = %s {\n", event, tag); fprintf(fh, " parent = %p\n", event->parent); fprintf(fh, " channel = %d\n", event->channelId); } else if (type == ncclProfileNetPlugin) { struct netPlugin* event = (struct netPlugin *)eHandle; fprintf(fh, "NetPlugin event %p tag = %s {\n", event, tag); fprintf(fh, " pluginType = %d\n", event->pluginType); fprintf(fh, " pluginVer = %d\n", event->pluginVer); fprintf(fh, " pluginEvent = %d\n", event->pluginEvent); fprintf(fh, " startTs = %f\n", event->startTs); fprintf(fh, " stopTs = %f\n", event->stopTs); fprintf(fh, "}\n"); } else if (type == ncclProfileCeColl) { debugCeCollEvent(fh, (struct ceColl*)eHandle, tag); } else if (type == ncclProfileCeSync) { debugCeSyncEvent(fh, (struct ceSync*)eHandle, tag); } else if (type == ncclProfileCeBatch) { debugCeBatchEvent(fh, (struct ceBatch*)eHandle, tag); } fclose(fh); #endif } // CE event print functions static int ceCollId = 0; static void printCeCollEvent(FILE* fh, struct ceColl* event) { if (event->timingMode == CE_TIMING_GPU) { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"CE_COLL\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"eventId\": %lu, \"count\": %lu, \"datatype\": \"%s\", \"strategy\": \"%s\", \"start_ts_cpu\": %f, \"stop_ts_cpu\": %f, \"duration_cpu_us\": %f, \"duration_gpu_us\": %lu}},\n", event->base.func, ceCollId, getpid(), 1, event->base.startTs, event->eventId, event->count, event->datatype, event->syncStrategy, event->cpuStartTime, event->cpuStopTime, event->cpuDuration, event->elapsedTime); } else { fprintf(fh, "{\"name\": \"%s\", \"cat\": \"CE_COLL\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"eventId\": %lu, \"count\": %lu, \"datatype\": \"%s\", \"strategy\": \"%s\", \"start_ts_cpu\": %f, \"stop_ts_cpu\": %f, \"duration_cpu_us\": %f}},\n", event->base.func, ceCollId, getpid(), 1, event->base.startTs, event->eventId, event->count, event->datatype, event->syncStrategy, event->cpuStartTime, event->cpuStopTime, event->cpuDuration); } // Print child events (CeSync and CeBatch) struct taskEventBase* child = taskEventQueueHead(event); while (child) { struct taskEventBase* next = child->next; printEvent(fh, child); child = next; } fprintf(fh, "{\"name\": \"%s\", \"cat\": \"CE_COLL\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", event->base.func, ceCollId++, getpid(), 1, event->base.stopTs); } static int ceSyncId = 0; static void printCeSyncEvent(FILE* fh, struct ceSync* event) { const char* syncTypeStr = event->isComplete ? "Complete" : "Ready"; if (event->timingMode == CE_TIMING_GPU) { fprintf(fh, "{\"name\": \"CeSync\", \"cat\": \"CE_SYNC\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"eventId\": %lu, \"type\": \"%s\", \"strategy\": \"%s\", \"nRanks\": %d, \"start_ts_cpu\": %f, \"stop_ts_cpu\": %f, \"duration_cpu_us\": %f, \"duration_gpu_us\": %lu}},\n", ceSyncId, getpid(), 1, event->base.startTs, event->eventId, syncTypeStr, event->parent->syncStrategy, event->nRanks, event->cpuStartTime, event->cpuStopTime, event->cpuDuration, event->elapsedTime); } else { fprintf(fh, "{\"name\": \"CeSync\", \"cat\": \"CE_SYNC\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"eventId\": %lu, \"type\": \"%s\", \"strategy\": \"%s\", \"nRanks\": %d, \"start_ts_cpu\": %f, \"stop_ts_cpu\": %f, \"duration_cpu_us\": %f}},\n", ceSyncId, getpid(), 1, event->base.startTs, event->eventId, syncTypeStr, event->parent->syncStrategy, event->nRanks, event->cpuStartTime, event->cpuStopTime, event->cpuDuration); } fprintf(fh, "{\"name\": \"CeSync\", \"cat\": \"CE_SYNC\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", ceSyncId++, getpid(), 1, event->base.stopTs); } static int ceBatchId = 0; static void printCeBatchEvent(FILE* fh, struct ceBatch* event) { if (event->timingMode == CE_TIMING_GPU) { fprintf(fh, "{\"name\": \"CeBatch\", \"cat\": \"CE_BATCH\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"eventId\": %lu, \"numOps\": %d, \"totalBytes\": %lu, \"start_ts_cpu\": %f, \"stop_ts_cpu\": %f, \"duration_cpu_us\": %f, \"duration_gpu_us\": %lu}},\n", ceBatchId, getpid(), 1, event->base.startTs, event->eventId, event->numOps, event->totalBytes, event->cpuStartTime, event->cpuStopTime, event->cpuDuration, event->elapsedTime); } else { fprintf(fh, "{\"name\": \"CeBatch\", \"cat\": \"CE_BATCH\", \"ph\": \"b\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f, \"args\": {\"eventId\": %lu, \"numOps\": %d, \"totalBytes\": %lu, \"start_ts_cpu\": %f, \"stop_ts_cpu\": %f, \"duration_cpu_us\": %f}},\n", ceBatchId, getpid(), 1, event->base.startTs, event->eventId, event->numOps, event->totalBytes, event->cpuStartTime, event->cpuStopTime, event->cpuDuration); } fprintf(fh, "{\"name\": \"CeBatch\", \"cat\": \"CE_BATCH\", \"ph\": \"e\", \"id\": %d, \"pid\": %d, \"tid\": %d, \"ts\": %f},\n", ceBatchId++, getpid(), 1, event->base.stopTs); } void printEvent(FILE* fh, void* handle) { if (handle == NULL || fh == NULL) return; uint64_t type = *(uint64_t *)handle; if (type == ncclProfileGroupApi) { struct groupApi* g = (struct groupApi*) handle; printGroupApiEventHeader(fh, g); struct kernelLaunch* kernelLaunchHead = profilerQueueHead(&g->kernelLaunchEvents); while (kernelLaunchHead != NULL) { printEvent(fh, kernelLaunchHead); kernelLaunchHead = kernelLaunchHead->next; } struct collApi* collApiHead = profilerQueueHead(&g->collApiEvents); while (collApiHead != NULL) { printEvent(fh, collApiHead); collApiHead = collApiHead->next; } struct p2pApi* p2pApiHead = profilerQueueHead(&g->p2pApiEvents); while (p2pApiHead != NULL) { printEvent(fh, p2pApiHead); p2pApiHead = p2pApiHead->next; } printGroupApiEventTrailer(fh, g); } else if (type == ncclProfileCollApi) { struct collApi* collApiEvent = (struct collApi *) handle; printCollApiEventHeader(fh, collApiEvent); struct taskEventBase* base = taskEventQueueHead(collApiEvent); while (base) { struct taskEventBase* next = base->next; printEvent(fh, base); base = next; } printCollApiEventTrailer(fh, collApiEvent); } else if (type == ncclProfileP2pApi) { struct p2pApi* p2pApiEvent = (struct p2pApi *) handle; printP2pApiEventHeader(fh, p2pApiEvent); struct taskEventBase* base = taskEventQueueHead(p2pApiEvent); while (base) { struct taskEventBase* next = base->next; printEvent(fh, base); base = next; } printP2pApiEventTrailer(fh, p2pApiEvent); } else if (type == ncclProfileKernelLaunch) { struct kernelLaunch* kernelLaunchEvent = (struct kernelLaunch *) handle; printKernelLaunchEventHeader(fh, kernelLaunchEvent); printKernelLaunchEventTrailer(fh, kernelLaunchEvent); } else if (type == ncclProfileGroup) { struct group* g = (struct group *)handle; printGroupEventHeader(fh, g); struct taskEventBase* base = taskEventQueueHead(g); while (base) { struct taskEventBase* next = base->next; printEvent(fh, base); base = next; } printGroupEventTrailer(fh, g); } else if (type == ncclProfileColl) { struct collective* c = (struct collective *)handle; printCollEventHeader(fh, c); for (int i = 0; i < MAX_CHANNELS; i++) { printKernelChEventHeader(fh, &c->kernel[i]); for (int j = 0; j < c->nProxyOps[i]; j++) { printEvent(fh, &c->op[i][j]); } printKernelChEventTrailer(fh, &c->kernel[i]); } printCollEventTrailer(fh, c); } else if (type == ncclProfileP2p) { struct p2p* p = (struct p2p *)handle; printP2pEventHeader(fh, p); for (int i = 0; i < MAX_CHANNELS; i++) { printKernelChEventHeader(fh, &p->kernel[i]); printEvent(fh, &p->op[i]); printKernelChEventTrailer(fh, &p->kernel[i]); } printP2pEventTrailer(fh, p); } else if (type == ncclProfileProxyOp) { struct proxyOp* p = (struct proxyOp *)handle; printProxyOpEventHeader(fh, p); for (int i = 0; i < MAX_STEPS; i++) { printEvent(fh, &p->step[i]); } printProxyOpEventTrailer(fh, p); } else if (type == ncclProfileProxyStep) { struct proxyStep* p = (struct proxyStep *)handle; printProxyStepEventHeader(fh, p); for (int q = 0; q < p->nNetEvents; q++) { printNetPluginEvent(fh, &p->net[q]); } printProxyStepEventTrailer(fh, p); } else if (type == ncclProfileProxyCtrl) { struct proxyCtrl* p = (struct proxyCtrl *)handle; printProxyCtrlEvent(fh, p); } else if (type == ncclProfileCeColl) { struct ceColl* ce = (struct ceColl*)handle; printCeCollEvent(fh, ce); } else if (type == ncclProfileCeSync) { struct ceSync* ce = (struct ceSync*)handle; printCeSyncEvent(fh, ce); } else if (type == ncclProfileCeBatch) { struct ceBatch* ce = (struct ceBatch*)handle; printCeBatchEvent(fh, ce); } return; } nccl-2.29.7-1/plugins/profiler/example/print_event.h000066400000000000000000000010461515037102200223310ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PRINT_EVENT_H_ #define PRINT_EVENT_H_ #include "nccl/common.h" extern ncclDebugLogger_t logFn; void debugEvent(void* eHandle, const char* tag); void printEvent(FILE* fh, void* handle); #endif nccl-2.29.7-1/plugins/profiler/example/profiler_plugin_ce.cc000066400000000000000000000456421515037102200240130ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include #include #include #include #include #include "profiler_plugin_ce.h" #include "event.h" #include "print_event.h" #define __hidden __attribute__ ((visibility("hidden"))) // External reference to gettime() from plugin.cc extern double gettime(void); // External reference to startTime from plugin.cc extern double getProfilerStartTime(void); // CE profiler global state static struct { pthread_t pollerThread; bool pollerRunning; pthread_mutex_t mutex; struct context** contextRegistry; int contextCount; int contextCapacity; CeTimingMode_t timingMode; int pollerIntervalUs; } ceProfilerCtxt = { .pollerThread = 0, .pollerRunning = false, .mutex = PTHREAD_MUTEX_INITIALIZER, .contextRegistry = NULL, .contextCount = 0, .contextCapacity = 0, .timingMode = CE_TIMING_CPU, .pollerIntervalUs = 500 }; // Poll CE Coll events for a context static void pollCeCollEvents(struct context* ctx) { if (ctx->ceCollPoolSize == 0 || ctx->ceCollPool == NULL) return; double startTime = getProfilerStartTime(); struct ceColl** ceCollPtr = &ctx->ceEvents.ceCollHead; while (*ceCollPtr) { struct ceColl* event = *ceCollPtr; if (!event->startCompleted && cudaEventQuery(event->startEvent) == cudaSuccess) { event->startCompleted = true; event->cpuStartTime = gettime(); event->base.startTs = gettime() - startTime; } if (event->startCompleted && !event->stopCompleted && cudaEventQuery(event->stopEvent) == cudaSuccess) { event->stopCompleted = true; event->cpuStopTime = gettime(); event->base.stopTs = gettime() - startTime; event->cpuDuration = event->cpuStopTime - event->cpuStartTime; if (event->timingMode == CE_TIMING_GPU) { float elapsedMs; if (cudaEventElapsedTime(&elapsedMs, event->startEvent, event->stopEvent) == cudaSuccess) { event->elapsedTime = (uint64_t)(elapsedMs * 1000); } else { event->elapsedTime = (uint64_t)event->cpuDuration; } } else { event->elapsedTime = (uint64_t)event->cpuDuration; } // Decrement parent refCount when complete if (event->parent) { __atomic_fetch_sub(&event->parent->refCount, 1, __ATOMIC_RELAXED); } // Return event to pool for reuse (ring buffer behavior) __atomic_fetch_add(&ctx->ceCollPoolBase, 1, __ATOMIC_RELAXED); *ceCollPtr = event->pollerNext; continue; } ceCollPtr = &event->pollerNext; } } // Poll CE Sync events for a context static void pollCeSyncEvents(struct context* ctx) { if (ctx->ceSyncPoolSize == 0 || ctx->ceSyncPool == NULL) return; double startTime = getProfilerStartTime(); struct ceSync** ceSyncPtr = &ctx->ceEvents.ceSyncHead; while (*ceSyncPtr) { struct ceSync* event = *ceSyncPtr; if (!event->startCompleted && cudaEventQuery(event->startEvent) == cudaSuccess) { event->startCompleted = true; event->cpuStartTime = gettime(); event->base.startTs = gettime() - startTime; } if (event->startCompleted && !event->stopCompleted && cudaEventQuery(event->stopEvent) == cudaSuccess) { event->stopCompleted = true; event->cpuStopTime = gettime(); event->base.stopTs = gettime() - startTime; event->cpuDuration = event->cpuStopTime - event->cpuStartTime; if (event->timingMode == CE_TIMING_GPU) { float elapsedMs; if (cudaEventElapsedTime(&elapsedMs, event->startEvent, event->stopEvent) == cudaSuccess) { event->elapsedTime = (uint64_t)(elapsedMs * 1000); } else { event->elapsedTime = (uint64_t)event->cpuDuration; } } else { event->elapsedTime = (uint64_t)event->cpuDuration; } // Decrement parent refCount when complete if (event->parent) { __atomic_fetch_sub(&event->parent->base.refCount, 1, __ATOMIC_RELAXED); } // Return event to pool for reuse (ring buffer behavior) __atomic_fetch_add(&ctx->ceSyncPoolBase, 1, __ATOMIC_RELAXED); *ceSyncPtr = event->pollerNext; continue; } ceSyncPtr = &event->pollerNext; } } // Poll CE Batch events for a context static void pollCeBatchEvents(struct context* ctx) { if (ctx->ceBatchPoolSize == 0 || ctx->ceBatchPool == NULL) return; double startTime = getProfilerStartTime(); struct ceBatch** ceBatchPtr = &ctx->ceEvents.ceBatchHead; while (*ceBatchPtr) { struct ceBatch* event = *ceBatchPtr; if (!event->startCompleted && cudaEventQuery(event->startEvent) == cudaSuccess) { event->startCompleted = true; event->cpuStartTime = gettime(); event->base.startTs = gettime() - startTime; } if (event->startCompleted && !event->stopCompleted && cudaEventQuery(event->stopEvent) == cudaSuccess) { event->stopCompleted = true; event->cpuStopTime = gettime(); event->base.stopTs = gettime() - startTime; event->cpuDuration = event->cpuStopTime - event->cpuStartTime; if (event->timingMode == CE_TIMING_GPU) { float elapsedMs; if (cudaEventElapsedTime(&elapsedMs, event->startEvent, event->stopEvent) == cudaSuccess) { event->elapsedTime = (uint64_t)(elapsedMs * 1000); } else { event->elapsedTime = (uint64_t)event->cpuDuration; } } else { event->elapsedTime = (uint64_t)event->cpuDuration; } // Decrement parent refCount when complete if (event->parent) { __atomic_fetch_sub(&event->parent->base.refCount, 1, __ATOMIC_RELAXED); } // Return event to pool for reuse (ring buffer behavior) __atomic_fetch_add(&ctx->ceBatchPoolBase, 1, __ATOMIC_RELAXED); *ceBatchPtr = event->pollerNext; continue; } ceBatchPtr = &event->pollerNext; } } // CE poller thread main function static void* cePollerThreadMain(void* arg) { while (__atomic_load_n(&ceProfilerCtxt.pollerRunning, __ATOMIC_RELAXED)) { if (pthread_mutex_trylock(&ceProfilerCtxt.mutex) != 0) { usleep(ceProfilerCtxt.pollerIntervalUs); continue; } for (int i = 0; i < ceProfilerCtxt.contextCount; i++) { struct context* ctx = ceProfilerCtxt.contextRegistry[i]; if (!ctx) continue; if (pthread_mutex_trylock(&ctx->ceEvents.mutex) != 0) { continue; } pollCeCollEvents(ctx); pollCeSyncEvents(ctx); pollCeBatchEvents(ctx); pthread_mutex_unlock(&ctx->ceEvents.mutex); } pthread_mutex_unlock(&ceProfilerCtxt.mutex); usleep(ceProfilerCtxt.pollerIntervalUs); } return NULL; } // Initialize CE profiler global state ncclResult_t ceProfilerInitGlobal(void) { const char* ceTimingStr = getenv("NCCL_PROFILER_CE_TIMING"); if (ceTimingStr && strcasecmp(ceTimingStr, "gpu") == 0) { ceProfilerCtxt.timingMode = CE_TIMING_GPU; } else { ceProfilerCtxt.timingMode = CE_TIMING_CPU; } const char* intervalStr = getenv("NCCL_PROFILER_CE_POLLER_INTERVAL_MICROSECONDS"); if (intervalStr) { ceProfilerCtxt.pollerIntervalUs = atoi(intervalStr); } ceProfilerCtxt.contextCapacity = 16; ceProfilerCtxt.contextRegistry = (struct context**)calloc(ceProfilerCtxt.contextCapacity, sizeof(struct context*)); if (!ceProfilerCtxt.contextRegistry) { return ncclSystemError; } ceProfilerCtxt.pollerRunning = true; if (pthread_create(&ceProfilerCtxt.pollerThread, NULL, cePollerThreadMain, NULL) != 0) { free(ceProfilerCtxt.contextRegistry); return ncclSystemError; } return ncclSuccess; } // Finalize CE profiler global state ncclResult_t ceProfilerFinalizeGlobal(FILE* fh) { if (ceProfilerCtxt.contextRegistry) { __atomic_store_n(&ceProfilerCtxt.pollerRunning, false, __ATOMIC_RELAXED); pthread_join(ceProfilerCtxt.pollerThread, NULL); free(ceProfilerCtxt.contextRegistry); ceProfilerCtxt.contextRegistry = NULL; } return ncclSuccess; } // Register context with CE poller for tracking void ceProfilerRegisterContext(struct context* ctx) { if (pthread_mutex_init(&ctx->ceEvents.mutex, NULL) != 0) { return; } ctx->ceEvents.ceCollHead = NULL; ctx->ceEvents.ceSyncHead = NULL; ctx->ceEvents.ceBatchHead = NULL; if (pthread_mutex_trylock(&ceProfilerCtxt.mutex) != 0) { pthread_mutex_destroy(&ctx->ceEvents.mutex); return; } // Check if context with this commHash+rank already exists for (int i = 0; i < ceProfilerCtxt.contextCount; i++) { if (ceProfilerCtxt.contextRegistry[i] && ceProfilerCtxt.contextRegistry[i]->commHash == ctx->commHash && ceProfilerCtxt.contextRegistry[i]->rank == ctx->rank) { pthread_mutex_unlock(&ceProfilerCtxt.mutex); return; } } // Resize registry if needed if (ceProfilerCtxt.contextCount > ceProfilerCtxt.contextCapacity) { int newCapacity = ceProfilerCtxt.contextCapacity * 2; struct context** newRegistry = (struct context**)calloc(newCapacity, sizeof(struct context*)); if (newRegistry) { memcpy(newRegistry, ceProfilerCtxt.contextRegistry, ceProfilerCtxt.contextCount * sizeof(struct context*)); free(ceProfilerCtxt.contextRegistry); ceProfilerCtxt.contextRegistry = newRegistry; ceProfilerCtxt.contextCapacity = newCapacity; } } if (ceProfilerCtxt.contextCount < ceProfilerCtxt.contextCapacity) { ceProfilerCtxt.contextRegistry[ceProfilerCtxt.contextCount++] = ctx; } pthread_mutex_unlock(&ceProfilerCtxt.mutex); } // Deregister context from CE poller void ceProfilerDeregisterContext(struct context* ctx) { if (pthread_mutex_trylock(&ceProfilerCtxt.mutex) != 0) { return; } for (int i = 0; i < ceProfilerCtxt.contextCount; i++) { if (ceProfilerCtxt.contextRegistry[i] && ceProfilerCtxt.contextRegistry[i]->commHash == ctx->commHash && ceProfilerCtxt.contextRegistry[i]->rank == ctx->rank) { ceProfilerCtxt.contextRegistry[i] = ceProfilerCtxt.contextRegistry[ceProfilerCtxt.contextCount - 1]; ceProfilerCtxt.contextCount--; break; } } pthread_mutex_unlock(&ceProfilerCtxt.mutex); } void ceProfilerCleanupPendingEvents(struct context* ctx) { if (pthread_mutex_trylock(&ctx->ceEvents.mutex) != 0) { return; } struct ceColl* ceColl = ctx->ceEvents.ceCollHead; while (ceColl) { if (ceColl->startEvent) cudaEventDestroy(ceColl->startEvent); if (ceColl->stopEvent) cudaEventDestroy(ceColl->stopEvent); ceColl = ceColl->pollerNext; } struct ceSync* ceSync = ctx->ceEvents.ceSyncHead; while (ceSync) { if (ceSync->startEvent) cudaEventDestroy(ceSync->startEvent); if (ceSync->stopEvent) cudaEventDestroy(ceSync->stopEvent); ceSync = ceSync->pollerNext; } struct ceBatch* ceBatch = ctx->ceEvents.ceBatchHead; while (ceBatch) { if (ceBatch->startEvent) cudaEventDestroy(ceBatch->startEvent); if (ceBatch->stopEvent) cudaEventDestroy(ceBatch->stopEvent); ceBatch = ceBatch->pollerNext; } pthread_mutex_unlock(&ctx->ceEvents.mutex); } // Get CE timing mode CeTimingMode_t ceProfilerGetTimingMode(void) { return ceProfilerCtxt.timingMode; } // Start CE Coll event ncclResult_t ceProfilerStartCeCollEvent(struct context* ctx, void** eHandle, ncclProfilerEventDescr_v6_t* eDescr, double startTime) { struct ceColl* event; int ceCollId = __atomic_fetch_add(&ctx->ceCollPoolIndex, 1, __ATOMIC_RELAXED); if ((ceCollId - __atomic_load_n(&ctx->ceCollPoolBase, __ATOMIC_RELAXED)) < ctx->ceCollPoolSize) { event = &ctx->ceCollPool[ceCollId % ctx->ceCollPoolSize]; event->parent = (struct collApi*)eDescr->parentObj; event->ceCollId = ceCollId; event->seqNumber = eDescr->ceColl.seqNumber; event->count = eDescr->ceColl.count; event->datatype = eDescr->ceColl.datatype; event->root = eDescr->ceColl.root; event->syncStrategy = eDescr->ceColl.syncStrategy; event->stream = (cudaStream_t)eDescr->ceColl.stream; event->eventId = ceCollId; event->timingMode = ceProfilerCtxt.timingMode; event->startCompleted = false; event->stopCompleted = false; // Initialize taskEventBase fields event->base.type = eDescr->type; event->base.rank = eDescr->rank; event->base.func = eDescr->ceColl.func; event->base.refCount = 1; event->base.parent = event->parent; event->base.next = NULL; event->base.startTs = -1; event->base.stopTs = -1; // Initialize child event queue for CeSync/CeBatch event->eventHead = NULL; event->eventTail = NULL; // Add to parent CollApi's task event queue if (event->parent) { taskEventQueueEnqueue(event->parent, &event->base); __atomic_fetch_add(&event->parent->refCount, 1, __ATOMIC_RELAXED); } // Create CUDA events with appropriate flags if (ceProfilerCtxt.timingMode == CE_TIMING_GPU) { cudaEventCreate(&event->startEvent, 0); cudaEventCreate(&event->stopEvent, 0); } else { cudaEventCreateWithFlags(&event->startEvent, cudaEventDisableTiming); cudaEventCreateWithFlags(&event->stopEvent, cudaEventDisableTiming); } // Record start event to stream cudaEventRecord(event->startEvent, event->stream); if (pthread_mutex_trylock(&ctx->ceEvents.mutex) == 0) { event->pollerNext = ctx->ceEvents.ceCollHead; ctx->ceEvents.ceCollHead = event; pthread_mutex_unlock(&ctx->ceEvents.mutex); } *eHandle = event; debugEvent(*eHandle, "CeCollStartEvent"); return ncclSuccess; } else { __atomic_fetch_sub(&ctx->ceCollPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } } // Stop CE Coll event ncclResult_t ceProfilerStopCeCollEvent(void* eHandle) { struct ceColl* event = (struct ceColl*)eHandle; cudaEventRecord(event->stopEvent, event->stream); debugEvent(eHandle, "CeCollStopEvent"); return ncclSuccess; } // Start CE Sync event ncclResult_t ceProfilerStartCeSyncEvent(struct context* ctx, void** eHandle, ncclProfilerEventDescr_v6_t* eDescr, double startTime) { struct ceSync* event; int ceSyncId = __atomic_fetch_add(&ctx->ceSyncPoolIndex, 1, __ATOMIC_RELAXED); if ((ceSyncId - __atomic_load_n(&ctx->ceSyncPoolBase, __ATOMIC_RELAXED)) < ctx->ceSyncPoolSize) { event = &ctx->ceSyncPool[ceSyncId % ctx->ceSyncPoolSize]; event->parent = (struct ceColl*)eDescr->parentObj; event->ceSyncId = ceSyncId; event->isComplete = eDescr->ceCollSync.isComplete; event->nRanks = eDescr->ceCollSync.nRanks; // Get seqNumber and stream from parent CeColl event event->seqNumber = event->parent->seqNumber; event->stream = event->parent->stream; event->eventId = ceSyncId; event->timingMode = ceProfilerCtxt.timingMode; event->startCompleted = false; event->stopCompleted = false; // Initialize taskEventBase fields event->base.type = eDescr->type; event->base.rank = eDescr->rank; event->base.func = "CeSync"; event->base.refCount = 1; event->base.parent = event->parent; event->base.next = NULL; event->base.startTs = -1; event->base.stopTs = -1; // Add to parent CeColl's event queue if it exists if (event->parent) { taskEventQueueEnqueue(event->parent, &event->base); __atomic_fetch_add(&event->parent->base.refCount, 1, __ATOMIC_RELAXED); } // Create CUDA events with appropriate flags if (ceProfilerCtxt.timingMode == CE_TIMING_GPU) { cudaEventCreate(&event->startEvent, 0); cudaEventCreate(&event->stopEvent, 0); } else { cudaEventCreateWithFlags(&event->startEvent, cudaEventDisableTiming); cudaEventCreateWithFlags(&event->stopEvent, cudaEventDisableTiming); } // Record start event to stream cudaEventRecord(event->startEvent, event->stream); if (pthread_mutex_trylock(&ctx->ceEvents.mutex) == 0) { event->pollerNext = ctx->ceEvents.ceSyncHead; ctx->ceEvents.ceSyncHead = event; pthread_mutex_unlock(&ctx->ceEvents.mutex); } *eHandle = event; debugEvent(*eHandle, "CeSyncStartEvent"); return ncclSuccess; } else { __atomic_fetch_sub(&ctx->ceSyncPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } } // Stop CE Sync event ncclResult_t ceProfilerStopCeSyncEvent(void* eHandle) { struct ceSync* event = (struct ceSync*)eHandle; cudaEventRecord(event->stopEvent, event->stream); debugEvent(eHandle, "CeSyncStopEvent"); return ncclSuccess; } // Start CE Batch event ncclResult_t ceProfilerStartCeBatchEvent(struct context* ctx, void** eHandle, ncclProfilerEventDescr_v6_t* eDescr, double startTime) { struct ceBatch* event; int ceBatchId = __atomic_fetch_add(&ctx->ceBatchPoolIndex, 1, __ATOMIC_RELAXED); if ((ceBatchId - __atomic_load_n(&ctx->ceBatchPoolBase, __ATOMIC_RELAXED)) < ctx->ceBatchPoolSize) { event = &ctx->ceBatchPool[ceBatchId % ctx->ceBatchPoolSize]; event->parent = (struct ceColl*)eDescr->parentObj; event->ceBatchId = ceBatchId; event->numOps = eDescr->ceCollBatch.numOps; event->totalBytes = eDescr->ceCollBatch.totalBytes; event->useIntraSync = eDescr->ceCollBatch.useIntraSync; // Get stream from parent CeColl event event->stream = event->parent->stream; event->eventId = ceBatchId; event->timingMode = ceProfilerCtxt.timingMode; event->startCompleted = false; event->stopCompleted = false; // Initialize taskEventBase fields event->base.type = eDescr->type; event->base.rank = eDescr->rank; event->base.func = "CeBatch"; event->base.refCount = 1; event->base.parent = event->parent; event->base.next = NULL; event->base.startTs = -1; event->base.stopTs = -1; // Add to parent CeColl's event queue if it exists if (event->parent) { taskEventQueueEnqueue(event->parent, &event->base); __atomic_fetch_add(&event->parent->base.refCount, 1, __ATOMIC_RELAXED); } // Create CUDA events with appropriate flags if (ceProfilerCtxt.timingMode == CE_TIMING_GPU) { cudaEventCreate(&event->startEvent, 0); cudaEventCreate(&event->stopEvent, 0); } else { cudaEventCreateWithFlags(&event->startEvent, cudaEventDisableTiming); cudaEventCreateWithFlags(&event->stopEvent, cudaEventDisableTiming); } // Record start event to stream cudaEventRecord(event->startEvent, event->stream); if (pthread_mutex_trylock(&ctx->ceEvents.mutex) == 0) { event->pollerNext = ctx->ceEvents.ceBatchHead; ctx->ceEvents.ceBatchHead = event; pthread_mutex_unlock(&ctx->ceEvents.mutex); } *eHandle = event; debugEvent(*eHandle, "CeBatchStartEvent"); return ncclSuccess; } else { __atomic_fetch_sub(&ctx->ceBatchPoolIndex, 1, __ATOMIC_RELAXED); return ncclSuccess; } } // Stop CE Batch event ncclResult_t ceProfilerStopCeBatchEvent(void* eHandle) { struct ceBatch* event = (struct ceBatch*)eHandle; cudaEventRecord(event->stopEvent, event->stream); debugEvent(eHandle, "CeBatchStopEvent"); return ncclSuccess; } nccl-2.29.7-1/plugins/profiler/example/profiler_plugin_ce.h000066400000000000000000000027771515037102200236570ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_PLUGIN_CE_H_ #define PROFILER_PLUGIN_CE_H_ #include "event.h" #include "nccl/profiler_v6.h" // CE profiler initialization and cleanup ncclResult_t ceProfilerInitGlobal(void); ncclResult_t ceProfilerFinalizeGlobal(FILE* fh); // CE context management void ceProfilerRegisterContext(struct context* ctx); void ceProfilerDeregisterContext(struct context* ctx); // CE event cleanup void ceProfilerCleanupPendingEvents(struct context* ctx); // CE event start/stop functions ncclResult_t ceProfilerStartCeCollEvent(struct context* ctx, void** eHandle, ncclProfilerEventDescr_v6_t* eDescr, double startTime); ncclResult_t ceProfilerStopCeCollEvent(void* eHandle); ncclResult_t ceProfilerStartCeSyncEvent(struct context* ctx, void** eHandle, ncclProfilerEventDescr_v6_t* eDescr, double startTime); ncclResult_t ceProfilerStopCeSyncEvent(void* eHandle); ncclResult_t ceProfilerStartCeBatchEvent(struct context* ctx, void** eHandle, ncclProfilerEventDescr_v6_t* eDescr, double startTime); ncclResult_t ceProfilerStopCeBatchEvent(void* eHandle); // Get CE timing mode for context initialization CeTimingMode_t ceProfilerGetTimingMode(void); #endif // PROFILER_PLUGIN_CE_H_ nccl-2.29.7-1/plugins/profiler/example/queue.h000066400000000000000000000025221515037102200211200ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef QUEUE_H #define QUEUE_H template struct profilerQueue { T *head, *tail; }; template inline void profilerQueueConstruct(profilerQueue *me) { me->head = nullptr; me->tail = nullptr; } template inline bool profilerQueueEmpty(profilerQueue *me) { return me->head == nullptr; } template inline T* profilerQueueHead(profilerQueue *me) { return me->head; } template inline T* profilerQueueTail(profilerQueue *me) { return me->tail; } template inline void profilerQueueEnqueue(profilerQueue *me, T *x) { x->*next = nullptr; (me->head ? me->tail->*next : me->head) = x; me->tail = x; } template inline T* profilerQueueDequeue(profilerQueue *me) { T *ans = me->head; me->head = ans->*next; if (me->head == nullptr) me->tail = nullptr; return ans; } #endif nccl-2.29.7-1/plugins/profiler/google-CoMMA/000077500000000000000000000000001515037102200203355ustar00rootroot00000000000000nccl-2.29.7-1/plugins/profiler/google-CoMMA/Makefile000066400000000000000000000005661515037102200220040ustar00rootroot00000000000000.PHONY: build-CoMMA all: build-CoMMA build-CoMMA: clone-CoMMA cd CoMMA && cargo build clone-CoMMA: @if [ ! -d CoMMA ] ; then \ git clone https://github.com/google/CoMMA.git; \ ln -s $(PWD)/.. CoMMA/third_party/nccl/plugins/profiler; \ fi clean: @if [ -d CoMMA ] ; then \ cd CoMMA && cargo clean; \ fi delete: @if [ -d CoMMA ] ; then \ rm -rf CoMMA; \ fi nccl-2.29.7-1/plugins/profiler/inspector/000077500000000000000000000000001515037102200201755ustar00rootroot00000000000000nccl-2.29.7-1/plugins/profiler/inspector/Makefile000066400000000000000000000103471515037102200216420ustar00rootroot00000000000000# Standalone Makefile for NCCL Inspector (independent from main NCCL) # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # Compiler settings CXX ?= g++ NVCC ?= nvcc # Build options DEBUG ?= 0 ASAN ?= 0 UBSAN ?= 0 # CUDA paths - default to /usr/local/cuda or require explicit CUDA_HOME CUDA_HOME ?= $(shell \ if [ -d /usr/local/cuda ]; then echo /usr/local/cuda; \ else echo "ERROR: CUDA installation not found. Please set CUDA_HOME." >&2; exit 1; fi) CUDA_INC ?= $(CUDA_HOME)/include CUDA_LIB ?= $(CUDA_HOME)/lib64 # Detect CUDA version NVCC := $(CUDA_HOME)/bin/nvcc CUDA_VERSION = $(strip $(shell which $(NVCC) >/dev/null && $(NVCC) --version | grep release | sed 's/.*release //' | sed 's/\,.*//')) CUDA_MAJOR = $(shell echo $(CUDA_VERSION) | cut -d "." -f 1) # CUDA 13.0 requires c++17 ifeq ($(shell test "0$(CUDA_MAJOR)" -ge 13; echo $$?),0) CXXSTD ?= -std=c++17 else CXXSTD ?= -std=c++14 endif # Build directories OBJDIR := obj SRCDIR := . # Compiler flags CXXFLAGS := $(CXXSTD) -fPIC -Wall -Wextra -O2 INCLUDES := -I$(CUDA_INC) -I$(SRCDIR) -I$(SRCDIR)/nccl LDFLAGS := -shared -L$(CUDA_LIB) LIBS := -lcudart -ldl -lpthread # Debug flags ifneq ($(DEBUG), 0) CXXFLAGS += -g -DDEBUG endif # Sanitizer flags ifneq ($(ASAN), 0) CXXFLAGS += -fsanitize=address LDFLAGS += -fsanitize=address -static-libasan endif ifneq ($(UBSAN), 0) CXXFLAGS += -fsanitize=undefined LDFLAGS += -fsanitize=undefined -static-libubsan endif # Source files SRCS := inspector.cc \ inspector_prom.cc \ inspector_plugin.cc \ inspector_cudawrap.cc \ json.cc \ version.cc OBJS := $(SRCS:%.cc=$(OBJDIR)/%.o) # Target library TARGET := libnccl-profiler-inspector.so # Build rules .PHONY: all clean help check-cuda all: check-cuda $(TARGET) # Check that CUDA installation exists check-cuda: @if [ ! -d "$(CUDA_HOME)" ]; then \ echo "ERROR: CUDA installation not found at $(CUDA_HOME)"; \ echo "Please install CUDA or set CUDA_HOME to the correct path"; \ echo "Example: make CUDA_HOME=/path/to/cuda"; \ exit 1; \ fi @if [ ! -f "$(CUDA_INC)/cuda.h" ]; then \ echo "ERROR: cuda.h not found at $(CUDA_INC)/cuda.h"; \ echo "Please check your CUDA installation"; \ exit 1; \ fi @if [ ! -d "$(CUDA_LIB)" ]; then \ echo "ERROR: CUDA library directory not found at $(CUDA_LIB)"; \ echo "Please check your CUDA installation"; \ exit 1; \ fi @if ! ls $(CUDA_LIB)/libcudart.so* >/dev/null 2>&1; then \ echo "ERROR: libcudart.so not found in $(CUDA_LIB)"; \ echo "Please check your CUDA installation"; \ exit 1; \ fi @echo "Using CUDA installation at: $(CUDA_HOME)" $(TARGET): $(OBJDIR) $(OBJS) $(CXX) $(LDFLAGS) -o $@ $(OBJS) $(LIBS) @echo "Built standalone NCCL Inspector: $@" $(OBJDIR)/%.o: %.cc $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ $(OBJDIR)/version.o: version.cc $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ version.cc: @echo "// Auto-generated version info" > version.cc @echo "#ifdef __cplusplus" >> version.cc @echo "extern \"C\" {" >> version.cc @echo "#endif" >> version.cc @echo "const char* get_git_version_info() {" >> version.cc @if command -v git >/dev/null 2>&1; then \ echo " return \"$(shell git describe --always --dirty 2>/dev/null || echo 'unknown')\"; " >> version.cc; \ else \ echo " return \"standalone-build\"; " >> version.cc; \ fi @echo "}" >> version.cc @echo "#ifdef __cplusplus" >> version.cc @echo "}" >> version.cc @echo "#endif" >> version.cc $(OBJDIR): mkdir -p $(OBJDIR) clean: rm -rf $(OBJDIR) $(TARGET) version.cc # Help target help: @echo "Standalone NCCL Inspector Build System" @echo "" @echo "Targets:" @echo " all - Build the inspector library (default)" @echo " clean - Clean build artifacts" @echo " help - Show this help" @echo "" @echo "Options:" @echo " DEBUG=1 - Enable debug build" @echo " ASAN=1 - Enable address sanitizer" @echo " UBSAN=1 - Enable undefined behavior sanitizer" @echo "" @echo "Environment Variables:" @echo " CUDA_HOME - Path to CUDA installation (default: /usr/local/cuda)" @echo " CXX - C++ compiler (default: g++)" @echo "" @echo "Usage Examples:" @echo " make DEBUG=1" @echo " make CUDA_HOME=/path/to/cuda" nccl-2.29.7-1/plugins/profiler/inspector/README.md000066400000000000000000000224411515037102200214570ustar00rootroot00000000000000# NCCL Inspector Plugin The NCCL Inspector is a plugin for the NVIDIA Collective Communications Library (NCCL) that provides detailed, per-communicator, per-collective performance and metadata logging. It is designed to help users analyze and debug NCCL collective operations by generating structured JSON output for each operation. ## Related Documentation - **[Performance Exporter](exporter/example/README.md)** - Tool for analyzing and visualizing NCCL performance data from inspector logs ## Folder Location The Inspector plugin source is located in: ``` plugins/profiler/inspector/ ``` ## Building the Inspector Plugin To build the Inspector plugin, run: ```bash make ``` The build system will automatically detect CUDA and NCCL installations from your environment. If you need to specify custom paths, you can set `CUDA_HOME` and `NCCL_HOME` environment variables or pass them as make arguments. ### Build Options The Makefile supports several build options: - **DEBUG=1**: Enable debug build with additional debugging information - **ASAN=1**: Enable Address Sanitizer for memory error detection - **UBSAN=1**: Enable Undefined Behavior Sanitizer Example debug build: ```bash make DEBUG=1 ``` ### Build Output The build process creates: - `libnccl-profiler-inspector.so`: The main inspector plugin library - `version.cc`: Auto-generated version information from git ## Using NCCL Inspector ### Key Differences from Normal NCCL Usage The main difference between running NCCL with the Inspector plugin versus running NCCL normally is the addition of environment variables that enable detailed performance logging: **Normal NCCL Run:** ```bash # Standard NCCL execution ./your_nccl_application ``` **NCCL Inspector Run:** ```bash # NCCL Inspector enabled execution export NCCL_PROFILER_PLUGIN=/path/to/nccl/plugins/profiler/inspector/libnccl-profiler-inspector.so export NCCL_INSPECTOR_ENABLE=1 export NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS=500 ./your_nccl_application ``` ### Required Environment Variables - `NCCL_PROFILER_PLUGIN=/path/to/nccl/plugins/profiler/inspector/libnccl-profiler-inspector.so` Loads the Inspector plugin into NCCL. - `NCCL_INSPECTOR_ENABLE=1` Enables the Inspector plugin. - `NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS=` Sets the interval (in microseconds) for the internal dump thread to write output. Example: `500`. - `NCCL_INSPECTOR_DUMP_DIR=` (optional) Sets the output directory for logs. If not set, defaults to `nccl-inspector-unknown-jobid` or `nccl-inspector-` if running under SLURM. - `NCCL_INSPECTOR_DUMP_VERBOSE=<0|1>` (optional) Enables verbose output including event trace information. Set to `1` to enable, `0` to disable (default). - `NCCL_INSPECTOR_PROM_DUMP=<0|1>` (optional) Enables Prometheus format for textfile node exporter output instead of custom JSON. Set to `1` to enable, `0` to disable (default). ### Debugging To see detailed Inspector plugin messages, use NCCL's debug subsystem filtering. The Inspector uses the `PROFILE` subsystem: ```bash # Show only Inspector messages export NCCL_DEBUG=INFO export NCCL_DEBUG_SUBSYS=PROFILE # Show Inspector messages along with other subsystems export NCCL_DEBUG=INFO export NCCL_DEBUG_SUBSYS=INIT,PROFILE # Show all debug messages (including Inspector) export NCCL_DEBUG=INFO export NCCL_DEBUG_SUBSYS=ALL ``` Inspector messages will appear with your configured NCCL_DEBUG level and will show: - Plugin initialization and configuration - Dump thread status and intervals - File creation and locations (with device UUIDs for Prometheus mode) - Error conditions and warnings ### Example Usage **Single Node:** ```bash export NCCL_PROFILER_PLUGIN=/path/to/nccl/plugins/profiler/inspector/libnccl-profiler-inspector.so export NCCL_INSPECTOR_ENABLE=1 export NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS=500 ./build/test/perf/all_reduce_perf -b 8 -e 16G -f 2 -g 8 ``` **Multi-Node (SLURM):** ```bash # Add these environment variables to your SLURM script export NCCL_PROFILER_PLUGIN=/path/to/nccl/plugins/profiler/inspector/libnccl-profiler-inspector.so export NCCL_INSPECTOR_ENABLE=1 export NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS=500 export NCCL_INSPECTOR_DUMP_DIR=/path/to/logs/${SLURM_JOB_ID}/ # Then run your normal NCCL application srun your_nccl_application ``` **Prometheus Output Mode(For Node exporter)** **Example Prometheus Setup:** ```bash export NCCL_PROFILER_PLUGIN=/path/to/nccl/plugins/profiler/inspector/libnccl-profiler-inspector.so export NCCL_INSPECTOR_ENABLE=1 export NCCL_INSPECTOR_PROM_DUMP=1 export NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS=30000000 # 30 seconds export NCCL_INSPECTOR_DUMP_DIR=/var/lib/node_exporter/nccl_inspector/ ``` **Exported Metrics:** - `nccl_algorithm_bandwidth_gbs` - NCCL algorithm bandwidth in GB/s - `nccl_bus_bandwidth_gbs` - NCCL bus bandwidth in GB/s - `nccl_collective_exec_time_microseconds` - Execution time in microseconds All metrics include labels: `comm_id`, `collective`, `hostname`, `rank`, `slurm_job`, `slurm_job_id`, `pid`, `n_ranks`, `n_nodes`, `coll_sn`, `coll_timing_source`. e.g.: comm_id="0xd152c00f111816",collective="AllReduce",hostname="pool0-0002",rank="4",slurm_job="unknown",slurm_job_id="unknown",n_ranks="8",n_nodes="1",coll_sn="228924",timestamp="2025-10-17T03:31:47Z",gpu_device_id="GPU4",message_size="2.00GB" ## Output Example Each output file contains JSON objects with the following structure: ```json { "header": { "id": "0x7f8c496ae9f661", "rank": 2, "n_ranks": 8, "nnodes": 1 }, "metadata": { "inspector_output_format_version": "v4.0", "git_rev": "", "rec_mechanism": "profiler_plugin", "dump_timestamp_us": 1748030377748202, "hostname": "example-hostname", "pid": 1639453 }, "coll_perf": { "coll": "AllReduce", "coll_sn": 1407, "coll_msg_size_bytes": 17179869184, "coll_exec_time_us": 61974, "coll_algobw_gbs": 277.210914, "coll_busbw_gbs": 485.119099 } } ``` ## Output Example Verbose To enable verbose output with event trace information, set the `NCCL_INSPECTOR_DUMP_VERBOSE=1` environment variable: ```bash export NCCL_INSPECTOR_DUMP_VERBOSE=1 ``` This will include additional event trace information in the JSON output, showing the sequence of callbacks and timestamps for each individual event. ```json { "header": { "id": "0xe62dedaa97644a", "rank": 4, "n_ranks": 8, "nnodes": 1 }, "metadata": { "inspector_output_format_version": "v4.0", "git_rev": "9019a1912-dirty", "rec_mechanism": "nccl_profiler_interface", "dump_timestamp_us": 1752867229276385, "hostname": "example-hostname", "pid": 438776 }, "coll_perf": { "coll": "ReduceScatter", "coll_sn": 1231, "coll_msg_size_bytes": 2147483648, "coll_exec_time_us": 41057, "coll_timing_source": "kernel_gpu", "coll_algobw_gbs": 418.439467, "coll_busbw_gbs": 366.134533, "event_trace_sn": { "coll_start_sn": 1, "coll_stop_sn": 2, "kernel_events": [ { "channel_id": 0, "kernel_start_sn": 3, "kernel_stop_sn": 48, "kernel_record_sn": 47 } ] }, "event_trace_ts": { "coll_start_ts": 1752867229235059, "coll_stop_ts": 1752867229235064, "kernel_events": [ { "channel_id": 0, "kernel_start_ts": 1752867229235181, "kernel_stop_ts": 1752867229275811, "kernel_record_ts": 1752867229275811 } ] } } } ``` Multiple such JSON objects are written, one per collective operation per communicator. ## Output Directory - By default, output directory is auto-generated based on: - `nccl-inspector-` if `SLURM_JOBID` is set - `nccl-inspector-unknown-jobid` otherwise - You can override this with the `NCCL_INSPECTOR_DUMP_DIR` environment variable. - For Prometheus integration, set it to a directory where Prometheus exporter can scrape it from (e.g., `NCCL_INSPECTOR_DUMP_DIR=/var/lib/node_exporter/nccl_inspector`). ## Output File Size Estimates The size of output files depends on the output format and usage patterns: **JSON Mode** (`NCCL_INSPECTOR_PROM_DUMP=0`, default): - File size **grows continuously** throughout the application lifetime - Each collective operation adds a new JSON entry to the log file - File size is proportional to: - Total number of collective operations executed - Number of parallel/overlapping communicators the process (PID) participates in - Estimate: ~200-500 bytes per collective operation - Example: A workload with 1M collectives across 4 communicators ≈ 200-500 MB per process **Prometheus Mode** (`NCCL_INSPECTOR_PROM_DUMP=1`): - File size is **bounded** (does not grow indefinitely) - Files are rewritten periodically (default: every 30 seconds based on `NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS`) - File size is proportional to: - Number of parallel/overlapping communicators using the same GPU device - Each file contains only the most recent metrics snapshot - Estimate: ~500-1000 bytes per communicator per metric - Example: 8 communicators on one GPU with 3 metrics ≈ 12-24 KB per GPU (fixed size) ## Additional Notes - The plugin is compatible with standard NCCL workflows and can be used in both single-node and multi-node (SLURM) environments. - For more details, see the source code and comments in `plugins/profiler/inspector/`. nccl-2.29.7-1/plugins/profiler/inspector/exporter/000077500000000000000000000000001515037102200220455ustar00rootroot00000000000000nccl-2.29.7-1/plugins/profiler/inspector/exporter/example/000077500000000000000000000000001515037102200235005ustar00rootroot00000000000000nccl-2.29.7-1/plugins/profiler/inspector/exporter/example/README.md000066400000000000000000000120261515037102200247600ustar00rootroot00000000000000# NCCL Inspector Performance Summary Exporter This tool processes NCCL Inspector log files and generates comprehensive performance analysis reports including visualizations and statistical summaries. One can build similar exporters to integrate with various observability systems like Elastic, Prometheus or other Custom Metric systems. ## Features - **Performance Analysis**: Generates statistical summaries for collective operations - **Communication Type Classification**: Automatically categorizes communication patterns - **Visualizations**: Creates scatter plots, histograms, and box plots for performance metrics - **Data Export**: Converts logs to Parquet format for efficient processing - **Multi-format Log Support**: Processes `.log`, `.log.gz`, `.jsonl`, and `.jsonl.gz` files - **Parallel Processing**: Utilizes multi-core processing for faster analysis ## Requirements - Python 3.7+ - Access to NCCL Inspector log files ## Installation ### Clone the Repository ```bash git clone https://github.com/NVIDIA/nccl.git cd nccl/plugins/profiler/inspector/exporter/example ``` Install the required dependencies using the provided `requirements.txt` file: ```bash pip install -r requirements.txt ``` ## Usage The script processes NCCL Inspector log files from a specified directory. **Note:** To generate NCCL Inspector log files, you need to run your NCCL application with the inspector plugin enabled. The log files will be output to a directory specified by the `NCCL_INSPECTOR_DUMP_DIR` environment variable. For detailed setup instructions and environment variable configuration, see the [Inspector README](../../README.md). ### Basic Usage ```bash python perf_summary_exporter.py --input_dir /path/to/nccl/inspector/logs ``` This mode processes all log files in the specified directory and its subdirectories recursively. ### Command Line Arguments - `--input_dir `: **Required**. Directory containing NCCL Inspector log files (searches recursively in subdirectories) - `--output_dir `: **Optional**. Custom output directory name (default: `-analysis`) ## Output The tool generates: 1. **Parquet Files**: One per log file containing processed log data (stored in `parquet_files/` subdirectory) 2. **Summary Directory**: Contains comprehensive analysis results 3. **Visualizations**: Scatter plots, histograms, and box plots for each message size 4. **CSV Files**: Detailed summaries for each message size and collective type 5. **Log File**: Processing log with detailed information ## Example Output Structure ``` / ├── output.log ├── parquet_files/ │ ├── .parquet │ ├── .parquet │ └── ... └── summary/ ├── scatter_plot__.png ├── combined_scatter_plot__.png └── msg_size_/ ├── histograms/ │ └── histogram___.png ├── boxplots/ │ └── boxplot___.png └── summary___.csv ``` ## Supported Communicator Types - `single-rank` - `nvlink-only` - `hca-only` - `mixed` ## Supported Collective Types - `AllReduce` - `AllGather` - `ReduceScatter` - `Broadcast` ## Log File Formats ### Supported Formats - `.log` - Plain text JSON lines - `.log.gz` - Compressed JSON lines - `.jsonl` - JSON lines format - `.jsonl.gz` - Compressed JSON lines ### Expected JSON Structure ```json { "header": { "id": "0x9e7a479f95a66c", "rank": 31, "n_ranks": 32, "nnodes": 4 }, "metadata": { "inspector_output_format_version": "v4.0", "git_rev": "75e61acda-dirty", "rec_mechanism": "nccl_profiler_interface", "dump_timestamp_us": 1749490229087081, "hostname": "example-hostname", "pid": 468528 }, "coll_perf": { "coll": "ReduceScatter", "coll_sn": 129, "coll_msg_size_bytes": 65536, "coll_exec_time_us": 110, "coll_timing_source": "kernel_gpu", "coll_algobw_gbs": 19.065018, "coll_busbw_gbs": 18.469236 } } ``` ## Troubleshooting ### Common Issues 1. **No log files found**: Ensure the log directory path is correct and contains valid log files 2. **Missing dependencies**: Ensure all requirements are installed in your virtual environment 3. **Mixed file formats**: The tool will exit if it detects mixed `.log`, `.log.gz`, `.jsonl`, and `.jsonl.gz` files in the same directory. This is typically indicative of corrupt input directories caused by multiple overlapping NCCL Inspector runs with different output format options. Clean the directory and re-run with consistent settings. ### Log Files The tool creates detailed logs in the output directory. Check `output.log` for processing information and any error messages. ## Support Please refer to the github issues page at https://github.com/NVIDIA/nccl/issues. Your question may already have been asked by another user. If not, feel free to create a new issue and refer to the "inspector plugin" in the title. nccl-2.29.7-1/plugins/profiler/inspector/exporter/example/perf_summary_exporter.py000066400000000000000000000477671515037102200305400ustar00rootroot00000000000000# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information from pathlib import Path import argparse import glob import gzip import sys import pandas as pd from concurrent.futures import ProcessPoolExecutor import json from tqdm.auto import tqdm import duckdb import math import matplotlib.pyplot as plt import matplotlib.dates from matplotlib.gridspec import GridSpec import os import logging import contextlib from datetime import datetime import numpy as np def setup_logging(output_dir): log_file = output_dir / "output.log" logging.basicConfig( filename=log_file, level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", ) @contextlib.contextmanager def smart_open(filename, mode="r"): if filename.endswith(".gz"): opener = gzip.open else: opener = open with opener(filename, mode) as f: yield f def get_log_files_and_output_dir(): parser = argparse.ArgumentParser(description="Process log files in a directory.") parser.add_argument( "--input_dir", type=str, help="The directory containing NCCL Inspector log files to process.", ) parser.add_argument( "--output_dir", type=str, help="Custom output directory name (default: auto-generated from input directory)." ) args = parser.parse_args() if args.input_dir: # Use the provided input directory root_dir = Path(args.input_dir) if not root_dir.exists(): raise FileNotFoundError(f"Input directory not found: {root_dir}") logfiles = list(glob.iglob(str(Path(root_dir) / "**" / "*.log"), recursive=True)) gzlogfiles = list( glob.iglob(str(Path(root_dir) / "**" / "*.log.gz"), recursive=True) ) jsonlfiles = list( glob.iglob(str(Path(root_dir) / "**" / "*.jsonl"), recursive=True) ) gzjsonlfiles = list( glob.iglob(str(Path(root_dir) / "**" / "*.jsonl.gz"), recursive=True) ) if ( sum((1 for x in [logfiles, gzlogfiles, jsonlfiles, gzjsonlfiles] if len(x) > 0)) > 1 ): ### TODO: we could probably generate some logic to pick the "right" file to load, but for now, bail logging.critical("Appear to have mixed .log/.log.gz/.jsonl/.jsonl.gz; bailing!") sys.exit(1) files = logfiles + gzlogfiles + jsonlfiles + gzjsonlfiles if not files: print("No inspector logs found") sys.exit(1) # Generate output directory name from input directory if args.output_dir: output_dir_name = args.output_dir else: output_dir_name = f"{root_dir.name}-analysis" return files, output_dir_name def bytes_to_human_readable(size_bytes): """ Convert bytes to human-readable format using decimal (SI) units. Uses powers of 1000 (decimal/SI standard): - 1 KB = 1,000 bytes - 1 MB = 1,000,000 bytes - 1 GB = 1,000,000,000 bytes Not binary units (powers of 1024): - Does NOT use KiB, MiB, GiB (1024-based) Args: size_bytes: Number of bytes to convert Returns: Human-readable string (e.g., "1.50MB", "2.34GB") """ if size_bytes == 0: return "0B" size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") i = int(math.log10(int(size_bytes)) / 3) s = round(size_bytes * math.pow(10, -3 * i), 2) return f"{s:.2f}{size_name[i]}" def timestamp_to_datetime(timestamp_us): """Convert microsecond timestamp to datetime string""" return datetime.fromtimestamp(timestamp_us / 1000000).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] def microseconds_to_human_readable(microseconds): """Convert microseconds to human readable format""" if microseconds < 1000: return f"{microseconds:.1f}μs" elif microseconds < 1000000: return f"{microseconds/1000:.1f}ms" else: return f"{microseconds/1000000:.1f}s" def get_comm_type(row) -> str: if row["n_ranks"] == 1: return "single-rank" elif row["nnodes"] == 1: return "nvlink-only" elif row["n_ranks"] == row["nnodes"]: return "hca-only" else: return "mixed" def parse_file(filepath: Path, output_dir): filename = Path(filepath).stem parquet_file = output_dir / f"{filename}.parquet" # Check if parquet file exists and is newer than source file if parquet_file.exists(): source_mtime = Path(filepath).stat().st_mtime parquet_mtime = parquet_file.stat().st_mtime if parquet_mtime >= source_mtime: logging.info(f"Parquet file {parquet_file} is up to date. Skipping...") return else: logging.info(f"Source file {filepath} is newer than parquet. Regenerating...") # Check if file is empty or too small file_size = Path(filepath).stat().st_size if file_size == 0: logging.warning(f"Skipping empty file: {filepath}") return recs = [] try: with smart_open(filepath, "r") as infile: for lineno, line in enumerate(infile): try: json_recs = json.loads(line) except json.JSONDecodeError: logging.error(f"Failed to parse line {filepath}:{lineno}") continue # Validate that required fields exist if not all(key in json_recs for key in ["header", "metadata", "coll_perf"]): logging.error(f"Missing required fields in {filepath}:{lineno}") continue header = json_recs["header"] metadata = json_recs["metadata"] comm_type = get_comm_type(header) coll_perf = json_recs["coll_perf"] recs.append( dict( **header, comm_type=comm_type, **coll_perf, **metadata, ) ) except Exception as e: logging.error(f"Error reading file {filepath}: {e}") return # Skip files with no valid records if not recs: logging.warning(f"No valid records found in file: {filepath}. Skipping...") return df = pd.DataFrame(recs) df.to_parquet(parquet_file) logging.info(f"Created parquet file {parquet_file} with {len(recs)} records") def create_per_node_parquet_files(files, output_dir): output_dir = Path(output_dir) / "parquet_files" output_dir.mkdir(parents=True, exist_ok=True) max_workers = min(64, len(files), os.cpu_count() or 1) with ProcessPoolExecutor(max_workers=max_workers) as executor: list( tqdm( executor.map(parse_file, files, [output_dir] * len(files)), total=len(files), desc="Processing files", unit="file", ) ) return output_dir def generate_scatter_plot(df, comm_type, coll_type, output_file): plt.figure(figsize=(10, 6), dpi=100) distinct_msg_sizes = df["coll_msg_size_bytes"].unique() for msg_size in distinct_msg_sizes: df_msg_size = df[df["coll_msg_size_bytes"] == msg_size] mean_busbw = df_msg_size["mean_coll_busbw_gbs"].mean() plt.scatter( df_msg_size["coll_sn"], df_msg_size["mean_coll_busbw_gbs"], label=f"MsgSize: {bytes_to_human_readable(msg_size)} (Mean: {mean_busbw:.2f} GB/s)", alpha=0.5, ) plt.xlabel("Operation Sequence Number") plt.ylabel("Mean Collective Bus BW (GB/s)") plt.title(f"Comm Type: {comm_type}, Coll Type: {coll_type}") plt.legend(title="Message Size", loc="upper right") plt.tight_layout() plt.savefig(output_file) plt.close() logging.info(f"Scatter plot saved to {output_file}") def generate_combined_scatter_plot(df, comm_type, coll_type, output_file, max_cols=3): distinct_msg_sizes = df["coll_msg_size_bytes"].unique() num_plots = len(distinct_msg_sizes) # Compute number of rows and columns num_cols = min(max_cols, num_plots) # Limit max columns num_rows = (num_plots + num_cols - 1) // num_cols # Calculate rows dynamically # Create figure with GridSpec fig = plt.figure(figsize=(5 * num_cols, 5 * num_rows), dpi=100) gs = GridSpec(num_rows, num_cols, figure=fig) for i, msg_size in enumerate(distinct_msg_sizes): row, col = divmod(i, num_cols) # Determine row & column index ax = fig.add_subplot(gs[row, col]) # Create subplot at position df_msg_size = df[df["coll_msg_size_bytes"] == msg_size] mean_busbw = df_msg_size["mean_coll_busbw_gbs"].mean() ax.scatter( df_msg_size["coll_sn"], df_msg_size["mean_coll_busbw_gbs"], label=f"MsgSize: {bytes_to_human_readable(msg_size)} (Mean: {mean_busbw:.2f} GB/s)", alpha=0.5, ) ax.set_xlabel("Op Seq No") ax.set_ylabel("Mean Collective Bus BW (GB/s)") ax.set_title(f"Message Size: {bytes_to_human_readable(msg_size)}({msg_size})") ax.legend(loc="upper right") fig.suptitle(f"Comm Type: {comm_type}, Coll Type: {coll_type}", ha="center", y=0.98) plt.tight_layout() plt.savefig(output_file) plt.close() logging.info(f"Combined scatter plot saved to {output_file}") def generate_histogram(df, comm_type, coll_type, output_file, message_size): plt.figure(figsize=(10, 6), dpi=100) data_range = df["mean_coll_busbw_gbs"].max() - df["mean_coll_busbw_gbs"].min() num_bins = min(50, int(data_range) + 1) plt.hist( df["mean_coll_busbw_gbs"], bins=num_bins, alpha=0.7, color="b", edgecolor="black", linewidth=1.2, ) plt.xlabel("Mean Collective Bus BW (GB/s)") plt.ylabel("Frequency") plt.title( f"Comm Type: {comm_type}, Coll Type: {coll_type} Mean Collective Bus BW Histogram\nMsg Size: {message_size}" ) plt.gca().yaxis.set_major_formatter(plt.FuncFormatter(lambda y, _: f"{y:.0f}")) plt.gca().xaxis.set_major_formatter(plt.FuncFormatter(lambda x, _: f"{x:.2f} GB/s")) plt.gca().xaxis.get_offset_text().set_visible(False) plt.tight_layout() plt.savefig(output_file) plt.close() logging.info(f"Histogram saved to {output_file}") def generate_boxplot(df, comm_type, coll_type, output_file, message_size): plt.figure(figsize=(10, 6)) boxprops = dict(linestyle="-", linewidth=2, color="blue") flierprops = dict(marker="o", color="red", alpha=0.5) medianprops = dict(linestyle="-", linewidth=2.5, color="orange") whiskerprops = dict(linestyle="--", linewidth=2, color="green") capprops = dict(linestyle="-", linewidth=2, color="black") plt.boxplot( df["mean_coll_busbw_gbs"], vert=False, patch_artist=True, boxprops=boxprops, flierprops=flierprops, medianprops=medianprops, whiskerprops=whiskerprops, capprops=capprops, ) plt.xlabel("Mean Coll Bus BW (GB/s)") plt.title( f"Box Plot of Coll Bus BW (CommType: {comm_type} - Coll Type: {coll_type} - Msg Size: {message_size})" ) # Adding labels for min, max, and median stats = df["mean_coll_busbw_gbs"].describe(percentiles=[0.5]) plt.annotate( f"Min: {stats['min']:.2f}", xy=(stats["min"], 1), xytext=(stats["min"], 1.1), arrowprops=dict(facecolor="black", shrink=0.05), ) plt.annotate( f"Median: {stats['50%']:.2f}", xy=(stats["50%"], 1), xytext=(stats["50%"], 1.1), arrowprops=dict(facecolor="black", shrink=0.05), ) plt.annotate( f"Max: {stats['max']:.2f}", xy=(stats["max"], 1), xytext=(stats["max"], 1.1), arrowprops=dict(facecolor="black", shrink=0.05), ) plt.tight_layout() plt.savefig(output_file) plt.close() logging.info(f"Box plot saved to {output_file}") def summarize_data_per_comm_coll_type(output_root, comm_type, coll_type, output_dir_name): """Summarize parquet data per communication and collective type using DuckDB""" logging.info(f"Summarizing data per comm/coll type for {output_dir_name}, {comm_type} and {coll_type}") # Check if there are any parquet files parquet_dir = output_root / "parquet_files" parquet_files = list(parquet_dir.glob("*.parquet")) if not parquet_files: logging.warning(f"No parquet files found for {comm_type} and {coll_type}") return None # Clean up invalid/empty parquet files by moving them to a separate directory invalid_dir = parquet_dir / "invalid" invalid_dir.mkdir(exist_ok=True) invalid_count = 0 for pf in parquet_files: try: # Check file size first if pf.stat().st_size == 0: logging.warning(f"Moving zero-byte parquet file {pf} to invalid directory") pf.rename(invalid_dir / pf.name) invalid_count += 1 continue # Use pyarrow to check parquet metadata without reading data import pyarrow.parquet as pq parquet_file = pq.ParquetFile(pf) if parquet_file.metadata.num_rows == 0: logging.warning(f"Moving empty parquet file {pf} (0 rows) to invalid directory") pf.rename(invalid_dir / pf.name) invalid_count += 1 except Exception as e: logging.warning(f"Moving invalid parquet file {pf} to invalid directory: {e}") pf.rename(invalid_dir / pf.name) invalid_count += 1 # Check if any valid files remain remaining_files = list(parquet_dir.glob("*.parquet")) if not remaining_files: logging.warning(f"No valid parquet files found for {comm_type} and {coll_type} (moved {invalid_count} invalid files)") return None logging.info(f"Found {len(remaining_files)} valid parquet files (moved {invalid_count} invalid files)") try: duckdb.execute( f"CREATE OR REPLACE VIEW logs AS SELECT * FROM read_parquet('{parquet_dir}/*.parquet')" ) df = duckdb.execute(f""" SELECT id, coll_sn, coll_msg_size_bytes, AVG(coll_busbw_gbs) as mean_coll_busbw_gbs, COUNT(*) as log_count, ARRAY_DISTINCT(LIST(n_ranks)) as n_ranks, ARRAY_DISTINCT(LIST(nnodes)) as nnodes, MIN(dump_timestamp_us) as coll_start_timestamp_us, MAX(dump_timestamp_us) as coll_end_timestamp_us, (MAX(dump_timestamp_us) - MIN(dump_timestamp_us)) as coll_duration_us FROM logs WHERE coll = '{coll_type}' and comm_type = '{comm_type}' GROUP BY id, coll_sn, coll_msg_size_bytes ORDER BY coll_sn """).df() except Exception as e: logging.error(f"Error executing DuckDB query for {comm_type} and {coll_type}: {e}") return None if df.empty: logging.info(f"No data for {comm_type} and {coll_type}") return None # Add human-readable formatting df["human_readable_coll_msg_size_bytes"] = df["coll_msg_size_bytes"].apply( bytes_to_human_readable ) # Log example of time range data for first few rows if len(df) > 0: sample_row = df.iloc[0] start_time = timestamp_to_datetime(sample_row['coll_start_timestamp_us']) end_time = timestamp_to_datetime(sample_row['coll_end_timestamp_us']) duration = microseconds_to_human_readable(sample_row['coll_duration_us']) logging.info(f"Example time range - ID: {sample_row['id']}, Coll_SN: {sample_row['coll_sn']}, " f"Start: {start_time}, End: {end_time}, Duration: {duration}") return df def generate_visualizations(df, output_root, comm_type, coll_type): """Generate all visualizations and save CSV files for the processed data""" logging.info(f"Generating visualizations for {comm_type} and {coll_type}") summary_dir = output_root / "summary" summary_dir.mkdir(parents=True, exist_ok=True) # Scatter Plot for all message sizes output_file = summary_dir / f"scatter_plot_{comm_type}_{coll_type}.png" generate_scatter_plot(df, comm_type, coll_type, output_file) # Combined Scatter Plot for all message sizes output_file = summary_dir / f"combined_scatter_plot_{comm_type}_{coll_type}.png" generate_combined_scatter_plot(df, comm_type, coll_type, output_file) distinct_msg_sizes = df["coll_msg_size_bytes"].unique() for msg_size in distinct_msg_sizes: hr_msg_size = bytes_to_human_readable(msg_size) msg_size_dir = summary_dir / f"msg_size_{msg_size}_{hr_msg_size}" msg_size_hist_dir = msg_size_dir / "histograms" msg_size_boxplot_dir = msg_size_dir / "boxplots" msg_size_dir.mkdir(parents=True, exist_ok=True) msg_size_hist_dir.mkdir(parents=True, exist_ok=True) msg_size_boxplot_dir.mkdir(parents=True, exist_ok=True) df_msg_size = df[df["coll_msg_size_bytes"] == msg_size] # Add human-readable time formatting df_msg_size = df_msg_size.copy() df_msg_size["coll_start_datetime"] = df_msg_size["coll_start_timestamp_us"].apply(timestamp_to_datetime) df_msg_size["coll_end_datetime"] = df_msg_size["coll_end_timestamp_us"].apply(timestamp_to_datetime) df_msg_size["coll_duration_human"] = df_msg_size["coll_duration_us"].apply(microseconds_to_human_readable) # Histogram output_file = ( msg_size_hist_dir / f"histogram_{comm_type}_{coll_type}_{msg_size}.png" ) generate_histogram( df_msg_size, comm_type, coll_type, output_file, bytes_to_human_readable(msg_size), ) # Box Plot output_file = ( msg_size_boxplot_dir / f"boxplot_{comm_type}_{coll_type}_{msg_size}.png" ) generate_boxplot( df_msg_size, comm_type, coll_type, output_file, bytes_to_human_readable(msg_size), ) output_file = msg_size_dir / f"summary_{comm_type}_{coll_type}_{msg_size}.csv" df_msg_size.to_csv(output_file, index=False) logging.info( f"Summary for {comm_type}, {coll_type}, and msg_size {msg_size} written to {output_file}" ) def generate_summary(output_root, comm_type, coll_type, output_dir_name): """Generate summary by summarizing data per comm/coll type and creating visualizations""" logging.info(f"Generating summary for {output_dir_name}, {comm_type} and {coll_type}") # Step 1: Summarize data per communication and collective type df = summarize_data_per_comm_coll_type(output_root, comm_type, coll_type, output_dir_name) # Step 2: Generate visualizations if data exists if df is not None: generate_visualizations(df, output_root, comm_type, coll_type) else: logging.warning(f"No data found for {comm_type} and {coll_type} - skipping visualization generation") def generate_summary_wrapper(args): return generate_summary(*args) if __name__ == "__main__": files, output_dir_name = get_log_files_and_output_dir() print(f"Number of log files found: {len(files)}") print(f"Output directory: {output_dir_name}") output_dir = Path(output_dir_name) output_dir.mkdir(parents=True, exist_ok=True) setup_logging(output_dir) create_per_node_parquet_files(files, output_dir) comm_types = ["single-rank", "nvlink-only", "hca-only", "mixed"] coll_types = ["AllReduce", "AllGather", "ReduceScatter", "Broadcast"] summary_args = [ (output_dir, comm_type, coll_type, output_dir_name) for comm_type in comm_types for coll_type in coll_types ] max_workers = min(64, len(summary_args), os.cpu_count() or 1) with ProcessPoolExecutor(max_workers=max_workers) as executor: list( tqdm( executor.map(generate_summary_wrapper, summary_args), total=len(summary_args), desc="Generating summaries", ) ) print("Done!") nccl-2.29.7-1/plugins/profiler/inspector/exporter/example/requirements.txt000066400000000000000000000001301515037102200267560ustar00rootroot00000000000000pandas>=1.3.0 tqdm>=4.60.0 duckdb>=0.8.0 matplotlib>=3.3.0 pyarrow>=5.0.0 numpy>=1.21.0 nccl-2.29.7-1/plugins/profiler/inspector/inspector.cc000066400000000000000000001532521515037102200225220ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "inspector.h" #include "inspector_prom.h" #include "inspector_cudawrap.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common.h" #define JSON_CHK(expr) \ do { \ const jsonResult_t res = (expr); \ if (res != jsonSuccess) { \ INFO_INSPECTOR("jsonError: %s\n", jsonErrorString(res)); \ return inspectorJsonError; \ } \ } while (0) #define JSON_CHK_GOTO(expr, res, label) \ do { \ const jsonResult_t macro_res = (expr); \ if (macro_res != jsonSuccess) { \ INFO_INSPECTOR("jsonError: %s\n", jsonErrorString(macro_res)); \ res = inspectorJsonError; \ goto label; \ } \ } while (0) #define INS_CUDA_CHK(cmd) \ do { \ cudaError_t err = cmd; \ if (err != cudaSuccess) { \ INFO_INSPECTOR("Cuda failure '%s'", cudaGetErrorString(err)); \ return inspectorCudaError; \ } \ } while (false) // Global flag to control inspector use static bool enableNcclInspector = false; // Global flag to control starting internal dump thread static bool enableNcclInspectorDumpThread = false; // Global flag to control verbose dumping (event_trace) static bool enableNcclInspectorDumpVerbose = false; // Global flag to control prometheus format dumping static bool enableNcclInspectorPromDump = false; // Global dump interval in microseconds static uint64_t ncclInspectorDumpIntervalUsecs = 0; // Extra guard to prevent spurious messages for eager pollers that try to dump // out results before we have initialized static bool ncclInspectorInit = false; // Define the global logFn variable ncclDebugLogger_t logFn = nullptr; /* * Description: * * Returns the current time in microseconds since the epoch. * * Thread Safety: * * Thread-safe (uses gettimeofday). * * Input: * * None. * * Output: * * None. * * Return: * uint64_t - current time in microseconds. * * Error Handling: * This function uses gettimeofday() which rarely fails. In case of * failure, the function returns 0. Callers should check for 0 return * value if precise error handling is required. * */ uint64_t inspectorGetTime() { uint64_t ts = 0; timeval tv; gettimeofday(&tv, 0); ts = tv.tv_sec * 1000000 + tv.tv_usec; return ts; } /* * Description: * * Wrapper around inspectorGetTime() that returns formatted UTC datetime string. * * Thread Safety: * * Not thread-safe. Onus of thread safety is on the caller/owner of * the buffer. * * Input: * char* buffer - output buffer for datetime string. * size_t bufferSize - size of output buffer. * * Output: * buffer contains UTC datetime string in ISO 8601 format. * * Return: * inspectorResult_t - success or error code. */ inspectorResult_t inspectorGetTimeUTC(char* buffer, size_t bufferSize) { if (!buffer || bufferSize < 21) { // Need at least 20 chars for "YYYY-MM-DDTHH:MM:SSZ" return inspectorMemoryError; } uint64_t timestampUsec = inspectorGetTime(); time_t timestampSec = timestampUsec / 1000000; // Convert microseconds to seconds struct tm* utc_tm = gmtime(×tampSec); if (utc_tm) { // Format as ISO 8601 datetime: YYYY-MM-DDTHH:MM:SSZ if (strftime(buffer, bufferSize, "%Y-%m-%dT%H:%M:%SZ", utc_tm) == 0) { return inspectorMemoryError; // Buffer too small } } else { // Fallback if gmtime fails snprintf(buffer, bufferSize, "unknown"); } return inspectorSuccess; } /* * Description: * * Converts a string to the corresponding ncclDataType_t enum value. * * Thread Safety: * Thread-safe (read-only string input). * * Input: * * const char* str - string representation of the datatype. * * Output: * * None. * * Return: * * ncclDataType_t - corresponding enum value, or -1 if unknown. * */ ncclDataType_t inspectorStringToDatatype(const char* str) { if (strcmp(str, "ncclInt8") == 0) return ncclInt8; if (strcmp(str, "ncclInt32") == 0) return ncclInt32; if (strcmp(str, "ncclUint32") == 0) return ncclUint32; if (strcmp(str, "ncclInt64") == 0) return ncclInt64; if (strcmp(str, "ncclUint64") == 0) return ncclUint64; if (strcmp(str, "ncclFloat16") == 0) return ncclFloat16; if (strcmp(str, "ncclFloat32") == 0) return ncclFloat32; if (strcmp(str, "ncclFloat64") == 0) return ncclFloat64; if (strcmp(str, "ncclBfloat16") == 0) return ncclBfloat16; if (strcmp(str, "ncclFloat8e4m3") == 0) return ncclFloat8e4m3; if (strcmp(str, "ncclFloat8e5m2") == 0) return ncclFloat8e5m2; return (ncclDataType_t)-1; // Or handle error as appropriate } /* * Description: * * Converts a string to the corresponding ncclFunc_t enum value. * * Thread Safety: * Thread-safe (read-only string input). * * Input: * const char* str - string representation of the function (must not be NULL). * * Output: * None. * * Return: * ncclFunc_t - corresponding enum value, or ncclNumFuncs if unknown. * * Preconditions: * - str must not be NULL */ ncclFunc_t ncclStringToFunc(const char* str) { if (strcmp(str, "AllGather") == 0) return ncclFuncAllGather; if (strcmp(str, "AllReduce") == 0) return ncclFuncAllReduce; if (strcmp(str, "Broadcast") == 0) return ncclFuncBroadcast; if (strcmp(str, "Recv") == 0) return ncclFuncRecv; if (strcmp(str, "Reduce") == 0) return ncclFuncReduce; if (strcmp(str, "ReduceScatter") == 0) return ncclFuncReduceScatter; if (strcmp(str, "SendRecv") == 0) return ncclFuncSendRecv; if (strcmp(str, "Send") == 0) return ncclFuncSend; return ncclNumFuncs; // Invalid / unknown } const char* ncclFuncToString(ncclFunc_t fn) { switch (fn) { case ncclFuncAllGather: return "AllGather"; case ncclFuncAllReduce: return "AllReduce"; case ncclFuncBroadcast: return "Broadcast"; case ncclFuncRecv: return "Recv"; case ncclFuncReduce: return "Reduce"; case ncclFuncReduceScatter: return "ReduceScatter"; case ncclFuncSendRecv: return "SendRecv"; case ncclFuncSend: return "Send"; default: return "Invalid"; } } struct inspectorDumpThread; static inspectorDumpThread* dumper = nullptr; #define UNUSED(x) (void)(x) inspectorResult_t inspectorLockInit(pthread_rwlock_t* lockRef) { if (0 != pthread_rwlock_init(lockRef, nullptr)) { return inspectorLockError; } else { return inspectorSuccess; } } inspectorResult_t inspectorLockDestroy(pthread_rwlock_t* lockRef) { if (0 != pthread_rwlock_destroy(lockRef)) { return inspectorLockError; } else { return inspectorSuccess; } } inspectorResult_t inspectorLockRd(pthread_rwlock_t* lockRef) { if (0 != pthread_rwlock_rdlock(lockRef)) { return inspectorLockError; } else { return inspectorSuccess; } } inspectorResult_t inspectorLockWr(pthread_rwlock_t* lockRef) { if (0 != pthread_rwlock_wrlock(lockRef)) { return inspectorLockError; } else { return inspectorSuccess; } } inspectorResult_t inspectorUnlockRWLock(pthread_rwlock_t* lockRef) { if (0 != pthread_rwlock_unlock(lockRef)) { return inspectorLockError; } else { return inspectorSuccess; } } inspectorState g_state; static inspectorResult_t inspectorCommInfoListInit(struct inspectorCommInfoList* commList) { if (commList->comms) { return inspectorGlobalInitError; } commList->comms = nullptr; commList->ncomms = 0; INS_CHK(inspectorLockInit(&commList->guard)); return inspectorSuccess; } static inspectorResult_t inspectorGlobalStateInit() { memset(&g_state, 0, sizeof(struct inspectorState)); INS_CHK(inspectorCommInfoListInit(&g_state.liveComms)); INS_CHK(inspectorCommInfoListInit(&g_state.deletedComms)); return inspectorSuccess; } /* * Description: * * Converts inspectorTimingSource_t enum to a string representation. * * Thread Safety: * Thread-safe (read-only operation). * * Input: * inspectorTimingSource_t timingSource - timing source enum value. * * Output: * None. * * Return: * const char* - string representation of the timing source. */ const char* inspectorTimingSourceToString(inspectorTimingSource_t timingSource) { switch (timingSource) { case inspectorTimingSourceKernelGpu: return "kernel_gpu"; case inspectorTimingSourceKernelCpu: return "kernel_cpu"; case inspectorTimingSourceCollectiveCpu: return "collective_cpu"; default: return "unknown"; } } /* * Description: * * Writes the header information for a communicator to the JSON output. * * Thread Safety: * Not thread-safe (should be called with proper locking). * * Input: * jsonFileOutput* jfo - JSON output handle. * struct inspectorCommInfo* commInfo - communicator info. * * Output: * Header is written to JSON output. * * Return: * inspectorResult_t - success or error code. * */ static inspectorResult_t inspectorCommInfoHeader(jsonFileOutput* jfo, struct inspectorCommInfo* commInfo) { JSON_CHK(jsonStartObject(jfo)); JSON_CHK(jsonKey(jfo, "id")); JSON_CHK(jsonStr(jfo, commInfo->commHashStr)); JSON_CHK(jsonKey(jfo, "rank")); JSON_CHK(jsonInt(jfo, commInfo->rank)); JSON_CHK(jsonKey(jfo, "n_ranks")); JSON_CHK(jsonInt(jfo, commInfo->nranks)); JSON_CHK(jsonKey(jfo, "nnodes")); JSON_CHK(jsonUint64(jfo, commInfo->nnodes)); JSON_CHK(jsonFinishObject(jfo)); return inspectorSuccess; } /* * Description: * * Writes metadata header information to the JSON output. * * Thread Safety: * Not thread-safe (should be called with proper locking). * * Input: * jsonFileOutput* jfo - JSON output handle. * * Output: * Metadata header is written to JSON output. * * Return: * inspectorResult_t - success or error code. * */ static inspectorResult_t inspectorCommInfoMetaHeader(jsonFileOutput* jfo) { JSON_CHK(jsonStartObject(jfo)); { JSON_CHK(jsonKey(jfo, "inspector_output_format_version")); JSON_CHK(jsonStr(jfo, "v4.0")); JSON_CHK(jsonKey(jfo, "git_rev")); JSON_CHK(jsonStr(jfo, get_git_version_info())); JSON_CHK(jsonKey(jfo, "rec_mechanism")); JSON_CHK(jsonStr(jfo, "nccl_profiler_interface")); JSON_CHK(jsonKey(jfo, "dump_timestamp_us")); JSON_CHK(jsonUint64(jfo, inspectorGetTime())); char hostname[256]; gethostname(hostname, 255); JSON_CHK(jsonKey(jfo, "hostname")); JSON_CHK(jsonStr(jfo, hostname)); JSON_CHK(jsonKey(jfo, "pid")); JSON_CHK(jsonUint64(jfo, getpid())); } JSON_CHK(jsonFinishObject(jfo)); return inspectorSuccess; } /* * Description: * * Writes verbose information (event_trace) for a completed * collective operation to the JSON output. * * Thread Safety: * Not thread-safe (should be called with proper locking). * * Input: * jsonFileOutput* jfo - JSON output handle. * const struct inspectorCompletedCollInfo* collInfo - completed * collective info. * * Output: * Verbose collective info is written to JSON output. * * Return: * inspectorResult_t - success or error code. * */ static inline inspectorResult_t inspectorCompletedCollVerbose(jsonFileOutput* jfo, struct inspectorCompletedCollInfo* collInfo) { // Add event trace information JSON_CHK(jsonKey(jfo, "event_trace_sn")); JSON_CHK(jsonStartObject(jfo)); { // Collective events JSON_CHK(jsonKey(jfo, "coll_start_sn")); JSON_CHK(jsonUint64(jfo, collInfo->collEvtTrk.evntTrace[NCCL_INSP_EVT_TRK_COLL_START].sn)); JSON_CHK(jsonKey(jfo, "coll_stop_sn")); JSON_CHK(jsonUint64(jfo, collInfo->collEvtTrk.evntTrace[NCCL_INSP_EVT_TRK_COLL_STOP].sn)); // Kernel events JSON_CHK(jsonKey(jfo, "kernel_events")); JSON_CHK(jsonStartList(jfo)); for (uint32_t ch = 0; ch < collInfo->collEvtTrk.nChannels; ch++) { JSON_CHK(jsonStartObject(jfo)); JSON_CHK(jsonKey(jfo, "channel_id")); JSON_CHK(jsonInt(jfo, ch)); JSON_CHK(jsonKey(jfo, "kernel_start_sn")); JSON_CHK(jsonUint64(jfo, collInfo->collEvtTrk.kernelCh[ch].evntTrace[NCCL_INSP_EVT_TRK_KERNEL_START].sn)); JSON_CHK(jsonKey(jfo, "kernel_stop_sn")); JSON_CHK(jsonUint64(jfo, collInfo->collEvtTrk.kernelCh[ch].evntTrace[NCCL_INSP_EVT_TRK_KERNEL_STOP].sn)); JSON_CHK(jsonKey(jfo, "kernel_record_sn")); JSON_CHK(jsonUint64(jfo, collInfo->collEvtTrk.kernelCh[ch].evntTrace[NCCL_INSP_EVT_TRK_KERNEL_RECORD].sn)); JSON_CHK(jsonFinishObject(jfo)); } JSON_CHK(jsonFinishList(jfo)); } JSON_CHK(jsonFinishObject(jfo)); JSON_CHK(jsonKey(jfo, "event_trace_ts")); JSON_CHK(jsonStartObject(jfo)); { // Collective events JSON_CHK(jsonKey(jfo, "coll_start_ts")); JSON_CHK(jsonUint64(jfo, collInfo->collEvtTrk.evntTrace[NCCL_INSP_EVT_TRK_COLL_START].ts)); JSON_CHK(jsonKey(jfo, "coll_stop_ts")); JSON_CHK(jsonUint64(jfo, collInfo->collEvtTrk.evntTrace[NCCL_INSP_EVT_TRK_COLL_STOP].ts)); // Kernel events JSON_CHK(jsonKey(jfo, "kernel_events")); JSON_CHK(jsonStartList(jfo)); for (uint32_t ch = 0; ch < collInfo->collEvtTrk.nChannels; ch++) { JSON_CHK(jsonStartObject(jfo)); JSON_CHK(jsonKey(jfo, "channel_id")); JSON_CHK(jsonInt(jfo, ch)); JSON_CHK(jsonKey(jfo, "kernel_start_ts")); JSON_CHK(jsonUint64(jfo, collInfo->collEvtTrk.kernelCh[ch].evntTrace[NCCL_INSP_EVT_TRK_KERNEL_START].ts)); JSON_CHK(jsonKey(jfo, "kernel_stop_ts")); JSON_CHK(jsonUint64(jfo, collInfo->collEvtTrk.kernelCh[ch].evntTrace[NCCL_INSP_EVT_TRK_KERNEL_STOP].ts)); JSON_CHK(jsonKey(jfo, "kernel_record_ts")); JSON_CHK(jsonUint64(jfo, collInfo->collEvtTrk.kernelCh[ch].evntTrace[NCCL_INSP_EVT_TRK_KERNEL_RECORD].ts)); JSON_CHK(jsonFinishObject(jfo)); } JSON_CHK(jsonFinishList(jfo)); } JSON_CHK(jsonFinishObject(jfo)); return inspectorSuccess; } /* * Description: * * Writes completed collective operation information to the JSON * output. * * Thread Safety: * Not thread-safe (should be called with proper locking). * * Input: * jsonFileOutput* jfo - JSON output handle. * const struct inspectorCompletedCollInfo* collInfo - completed * collective info. * * Output: * Collective info is written to JSON output. * * Return: * inspectorResult_t - success or error code. * */ static inline inspectorResult_t inspectorCompletedColl(jsonFileOutput* jfo, struct inspectorCompletedCollInfo* collInfo) { JSON_CHK(jsonStartObject(jfo)); { JSON_CHK(jsonKey(jfo, "coll")); JSON_CHK(jsonStr(jfo, ncclFuncToString(collInfo->func))); JSON_CHK(jsonKey(jfo, "coll_sn")); JSON_CHK(jsonUint64(jfo, collInfo->sn)); JSON_CHK(jsonKey(jfo, "coll_msg_size_bytes")); JSON_CHK(jsonUint64(jfo, collInfo->msgSizeBytes)); JSON_CHK(jsonKey(jfo, "coll_exec_time_us")); JSON_CHK(jsonUint64(jfo, collInfo->execTimeUsecs)); JSON_CHK(jsonKey(jfo, "coll_timing_source")); JSON_CHK(jsonStr(jfo, inspectorTimingSourceToString(collInfo->timingSource))); JSON_CHK(jsonKey(jfo, "coll_algobw_gbs")); JSON_CHK(jsonDouble(jfo, collInfo->algoBwGbs)); JSON_CHK(jsonKey(jfo, "coll_busbw_gbs")); JSON_CHK(jsonDouble(jfo, collInfo->busBwGbs)); if (enableNcclInspectorDumpVerbose) { INS_CHK(inspectorCompletedCollVerbose(jfo, collInfo)); } } JSON_CHK(jsonFinishObject(jfo)); return inspectorSuccess; } /* * Description: * * Dumps the state of a communicator to the JSON output if needed. * * Thread Safety: * Not thread-safe (should be called with proper locking). * * Input: * jsonFileOutput* jfo - JSON output handle. * inspectorCommInfo* commInfo - communicator info. * bool* needs_writing - set to true if output was written. * * Output: * State is dumped to JSON output if needed. * * Return: * inspectorResult_t - success or error code. * */ static inspectorResult_t inspectorCommInfoDump(jsonFileOutput* jfo, inspectorCommInfo* commInfo, bool* needs_writing) { *needs_writing = false; if (commInfo == nullptr) return inspectorSuccess; struct inspectorCompletedCollInfo collInfo; memset(&collInfo, 0, sizeof(struct inspectorCompletedCollInfo)); inspectorLockWr(&commInfo->guard); if (commInfo->dump) { *needs_writing = true; memcpy(&collInfo, &commInfo->completedCollInfo, sizeof(struct inspectorCompletedCollInfo)); commInfo->dump = false; } inspectorUnlockRWLock(&commInfo->guard); if (*needs_writing) { JSON_CHK(jsonLockOutput(jfo)); JSON_CHK(jsonStartObject(jfo)); { JSON_CHK(jsonKey(jfo, "header")); inspectorCommInfoHeader(jfo, commInfo); JSON_CHK(jsonKey(jfo, "metadata")); inspectorCommInfoMetaHeader(jfo); JSON_CHK(jsonKey(jfo, "coll_perf")); INS_CHK(inspectorCompletedColl(jfo, &collInfo)); } JSON_CHK(jsonFinishObject(jfo)); JSON_CHK(jsonNewline(jfo)); JSON_CHK(jsonUnlockOutput(jfo)); } return inspectorSuccess; } /* * Description: * * Dumps the state of all communicators in a commList to the JSON * output. * * Thread Safety: * Thread-safe - assumes no locks are taken and acquires all necessary * locks to iterate through all communicator objects and dump their state. * * Input: * jsonFileOutput* jfo - JSON output handle (must not be NULL). * struct inspectorCommInfoList* commList - list of communicators (must not be NULL). * * Output: * State of all communicators is dumped to JSON output. * * Return: * inspectorResult_t - success or error code. * */ static inspectorResult_t inspectorCommInfoListDump(jsonFileOutput* jfo, struct inspectorCommInfoList* commList) { bool flush = false; INS_CHK(inspectorLockRd(&commList->guard)); inspectorResult_t res = inspectorSuccess; if (commList->ncomms > 0) { for (struct inspectorCommInfo* itr = commList->comms; itr != nullptr; itr = itr->next) { bool needs_writing; INS_CHK_GOTO(inspectorCommInfoDump(jfo, itr, &needs_writing), res, exit); if (needs_writing) { flush = true; } } if (flush) { JSON_CHK_GOTO(jsonLockOutput(jfo), res, exit); JSON_CHK_GOTO(jsonFlushOutput(jfo), res, exit); JSON_CHK_GOTO(jsonUnlockOutput(jfo), res, exit); } } exit: INS_CHK(inspectorUnlockRWLock(&commList->guard)); return res; } /* * Description: * Finalizes and cleans up a commList, freeing all communicators. * * Thread Safety: * Not thread-safe (should be called with proper locking). * * Input: * struct commList* commList - list of communicators. * * Output: * All communicators are freed. * * Return: * inspectorResult_t - success or error code. * */ inspectorResult_t inspectorCommInfoListFinalize(struct inspectorCommInfoList* commList) { struct inspectorCommInfo* nextComm = nullptr; INS_CHK(inspectorLockWr(&commList->guard)); while (commList->comms != nullptr && commList->ncomms != 0) { TRACE_INSPECTOR("NCCL Inspector: comm %lu still in tracker", commList->comms->commHash); nextComm = commList->comms->next; INS_CHK(inspectorLockDestroy(&commList->comms->guard)); free(commList->comms); commList->comms = nextComm; commList->ncomms--; } INS_CHK(inspectorUnlockRWLock(&commList->guard)); return inspectorSuccess; } /* * Description: * * Ensures the given directory exists and is writable, creating it * if necessary. * * Thread Safety: * Not thread-safe (should be called during initialization). * * Input: * char* workdir - directory path. * * Output: * Directory is created if needed. * * Return: * * bool - true if directory exists and is writable, false otherwise. * */ static bool ensureDir(char* workdir) { struct stat st; // Check if directory exists if (stat(workdir, &st) == 0) { if (S_ISDIR(st.st_mode)) { // Directory exists, check if it's writable if (access(workdir, W_OK) == 0) { return true; // Directory exists and is writable } else { INFO_INSPECTOR( "NCCL Inspectoer: dump directory %s exists, but is not " "writable", workdir); return false; } } else { INFO_INSPECTOR( "NCCL Inspector: dump location %s exists, but is not a " "directory", workdir); return false; } } else { // Directory doesn't exist, try to create it const mode_t mode = 0777; if (mkdir(workdir, mode) == 0) { return true; // Directory created successfully } else { INFO_INSPECTOR( "NCCL Inspector: failed to create dump directory %s: %s", workdir, strerror(errno)); return false; } } } /* * Description: * * Generates the output dump directory path based on environment * variables. * * Thread Safety: * Not thread-safe (should be called during initialization). * * Input: * char** workdir - pointer to output directory string. * * Output: * workdir is set to the generated directory path. * * Return: * None. */ static void genDumpDir(char** workdir) { const char* dumpdir = getenv("NCCL_INSPECTOR_DUMP_DIR"); if (dumpdir != NULL) { *workdir = strdup(dumpdir); // TODO check errors here return; } const char* jobid = getenv("SLURM_JOBID"); bool badJobId = true; if (jobid != NULL) { errno = 0; const int intid = strtol(jobid, NULL, 10); if (errno == 0) { char tmp[2048]; snprintf(tmp, 2048, "nccl-inspector-%d", intid); *workdir = strdup(tmp); badJobId = false; } } if (badJobId) { *workdir = strdup("nccl-inspector-unknown-jobid"); } } inspectorDumpThread::inspectorDumpThread(const char* _outputRoot, uint64_t _sampleIntervalUsecs) : jfo(nullptr), outputRoot(strdup(_outputRoot)), sampleIntervalUsecs(_sampleIntervalUsecs) { if (inspectorLockInit(&guard) != inspectorSuccess) { INFO_INSPECTOR("NCCL Inspector inspectorDumpThread: couldn't init lock"); } } inspectorDumpThread::~inspectorDumpThread() { // Close and cleanup Prometheus files, only in Prom mode if (enableNcclInspectorPromDump) { // Close any open Prometheus file handles for (size_t i = 0; i < deviceFlushEntries.size(); i++) { if (deviceFlushEntries[i].fileHandle) { fclose(deviceFlushEntries[i].fileHandle); deviceFlushEntries[i].fileHandle = NULL; } } // Cleanup (delete) prom files after closing them for (size_t i = 0; i < deviceFlushEntries.size(); i++) { if (deviceFlushEntries[i].filename[0] != '\0') { if (unlink(deviceFlushEntries[i].filename) == 0) { TRACE_INSPECTOR("NCCL Inspector: Cleaned up Prometheus file %s", deviceFlushEntries[i].filename); } else { INFO_INSPECTOR("NCCL Inspector: Failed to cleanup Prometheus file %s: %s", deviceFlushEntries[i].filename, strerror(errno)); } } } } if (jfo != nullptr) { jsonFinalizeFileOutput(jfo); jfo = nullptr; } if (outputRoot != nullptr) { free(outputRoot); outputRoot = nullptr; } if (inspectorLockDestroy(&guard) != inspectorSuccess) { INFO_INSPECTOR("NCCL Inspector inspectorDumpThread: couldn't destroy lock"); } } // Implementation of inspectorDumpThread methods FILE* inspectorDumpThread::getOrCreateFileHandle(const char* deviceUuidStr, const char* filename, uint64_t currentTime) { int flushIndex = -1; bool needsFlush = false; // Find existing entry for this device UUID for (size_t i = 0; i < deviceFlushEntries.size(); i++) { if (strncmp(deviceFlushEntries[i].deviceUuidStr, deviceUuidStr, sizeof(deviceFlushEntries[i].deviceUuidStr) - 1) == 0) { flushIndex = static_cast(i); // Check if we need to flush (clear) the file if (deviceFlushEntries[i].lastFlushTime == 0 || ((currentTime - deviceFlushEntries[i].lastFlushTime) >= sampleIntervalUsecs)) { needsFlush = true; } break; } } // If not found, add new entry if (flushIndex == -1) { deviceFlushInfo newEntry; strncpy(newEntry.deviceUuidStr, deviceUuidStr, sizeof(newEntry.deviceUuidStr)); newEntry.deviceUuidStr[sizeof(newEntry.deviceUuidStr) - 1] = '\0'; strncpy(newEntry.filename, filename, sizeof(newEntry.filename) - 1); newEntry.filename[sizeof(newEntry.filename) - 1] = '\0'; newEntry.lastFlushTime = 0; newEntry.fileHandle = NULL; newEntry.needsCreation = true; deviceFlushEntries.push_back(newEntry); flushIndex = static_cast(deviceFlushEntries.size() - 1); needsFlush = true; } // Close existing handle if we need to flush (recreate file) if (needsFlush && deviceFlushEntries[flushIndex].fileHandle) { fclose(deviceFlushEntries[flushIndex].fileHandle); deviceFlushEntries[flushIndex].fileHandle = NULL; } // Open/create file if needed if (!deviceFlushEntries[flushIndex].fileHandle) { // Create file if flushing, otherwise append const char* mode = needsFlush ? "w" : "a"; FILE* file = fopen(filename, mode); if (!file) { INFO_INSPECTOR("NCCL Inspector: Failed to open Prometheus file %s", filename); return NULL; } chmod(filename, 0777); deviceFlushEntries[flushIndex].fileHandle = file; if (needsFlush) { TRACE_INSPECTOR("NCCL Inspector: Created/flushed Prometheus file %s", filename); deviceFlushEntries[flushIndex].lastFlushTime = currentTime; } } return deviceFlushEntries[flushIndex].fileHandle; } void inspectorDumpThread::startThread() { inspectorLockWr(&guard); run = true; inspectorUnlockRWLock(&guard); if (pthread_create(&pthread, NULL, dumpMain, this) != 0) { INFO_INSPECTOR( "NCCL Inspector inspectorDumpThread: couldn't create dump thread!"); return; } TRACE_INSPECTOR("NCCL Inspector inspectorDumpThread: created"); } void inspectorDumpThread::stopThread() { INFO(NCCL_ENV, "NCCL Inspector Stopping Dump thread"); inspectorLockWr(&guard); run = false; inspectorUnlockRWLock(&guard); std::this_thread::sleep_for(std::chrono::milliseconds(1)); INFO_INSPECTOR( "NCCL Inspector inspectorDumpThread: stopped"); } inspectorResult_t inspectorDumpThread::inspectorStateDump(const char* output_root) { if (!ncclInspectorInit) { return inspectorUninitializedError; } if (!enableNcclInspector) { INFO_INSPECTOR( "NCCL Inspector is not enabled, will not do ncclAllCommTallyDump"); return inspectorDisabledError; } if (enableNcclInspectorPromDump) { return inspectorStateDumpProm(output_root); } else { return inspectorStateDumpJSON(output_root); } } inspectorResult_t inspectorDumpThread::inspectorStateDumpJSON(const char* output_root) { if (jfo == 0) { char hostname[256]; gethostname(hostname, 255); char tmp[2048]; snprintf(tmp, sizeof(tmp), "%s/%s-pid%d.log", output_root, hostname, getpid()); jsonResult_t result = jsonInitFileOutput(&jfo, tmp); if (jsonSuccess != result) { INFO_INSPECTOR("Cannot open %s for writing: %s", tmp, jsonErrorString(result)); return inspectorFileOpenError; } chmod(tmp, 0666); } if (jfo != nullptr) { inspectorCommInfoListDump(jfo, &g_state.liveComms); inspectorCommInfoListDump(jfo, &g_state.deletedComms); } if (g_state.deletedComms.ncomms > 0) { inspectorCommInfoListFinalize(&g_state.deletedComms); } return inspectorSuccess; } inspectorResult_t inspectorDumpThread::inspectorStateDumpProm(const char* output_root) { // Write communicators directly to files with per-device flushing handled inside inspectorResult_t dumpResult = inspectorPromCommInfoListDump(&g_state.liveComms, output_root, this); if (dumpResult != inspectorSuccess) { INFO_INSPECTOR("NCCL Inspector: Direct Prometheus dump failed: %s", inspectorErrorString(dumpResult)); return dumpResult; } // Finalize deleted communicators if (g_state.deletedComms.ncomms > 0) { inspectorCommInfoListFinalize(&g_state.deletedComms); } return inspectorSuccess; } void* inspectorDumpThread::dumpMain(void* arg) { inspectorDumpThread* dumper = (inspectorDumpThread*)arg; inspectorResult_t res = inspectorSuccess; while (dumper->run) { inspectorLockWr(&dumper->guard); if (!dumper->run) { inspectorUnlockRWLock(&dumper->guard); break; } res = dumper->inspectorStateDump(dumper->outputRoot); if (res == inspectorFileOpenError || res == inspectorDisabledError) { inspectorUnlockRWLock(&dumper->guard); break; } inspectorUnlockRWLock(&dumper->guard); std::this_thread::sleep_for(std::chrono::microseconds(dumper->sampleIntervalUsecs)); } return 0; } /* * Description: * * Starts the internal dump thread with the specified interval. * * Thread Safety: * Not thread-safe (should be called during initialization). * * Input: * uint64_t intervalUsecs - dump interval in microseconds. * * Output: * Dump thread is started if successful. * * Return: * inspectorResult_t - success or error code. */ static inspectorResult_t inspectorStartDumpThread(uint64_t intervalUsecs) { if (intervalUsecs == 0) { INFO_INSPECTOR( "NCCL Inspector: dump thread enabled but " "dump interval is 0; not starting internal dump thread."); return inspectorSuccess; } char* dumpdir; genDumpDir(&dumpdir); if (dumpdir != nullptr) { if (!ensureDir(dumpdir)) { free(dumpdir); INFO_INSPECTOR( "NCCL Inspector: failed to generate a dump dir; not " "starting internal dump thread."); return inspectorSuccess; } dumper = new inspectorDumpThread(dumpdir, intervalUsecs); INFO_INSPECTOR( "NCCL Inspector enabled with polling interval %lu us, " "output directory %s, format %s", intervalUsecs, dumpdir, enableNcclInspectorPromDump ? "Prometheus" : "JSON"); dumper->startThread(); free(dumpdir); } else { INFO_INSPECTOR( "NCCL Inspector: failed to generate a dump " "dir; not starting internal dump thread."); } return inspectorSuccess; } /* * Description: * * Shows the NCCL Inspector plugin version and configuration * environment variables in a structured format similar to NCCL's * showVersion function. * * Thread Safety: * Thread-safe (read-only environment variable access). * * Input: * None. * * Output: * Logs version and environment variables to debug output. * * Return: * None. */ static void showInspectorVersion() { VERSION("NCCL Inspector Plugin - Version: %s", get_git_version_info()); } /* * Description: * * Shows all NCCL Inspector environment variables and their values * in a structured format. * * Thread Safety: * Thread-safe (read-only environment variable access). * * Input: * None. * * Output: * Logs environment variables to debug output. * * Return: * None. */ static void showInspectorEnvVars() { struct { const char* name; const char* value; const char* defaultVal; const char* description; } envVars[] = { {"NCCL_INSPECTOR_ENABLE", getenv("NCCL_INSPECTOR_ENABLE"), "0", "Enable/disable inspector plugin"}, {"NCCL_INSPECTOR_DUMP_THREAD_ENABLE", getenv("NCCL_INSPECTOR_DUMP_THREAD_ENABLE"), "1", "Enable/disable dump thread"}, {"NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS", getenv("NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS"), "0", "Dump thread interval in microseconds"}, {"NCCL_INSPECTOR_DUMP_DIR", getenv("NCCL_INSPECTOR_DUMP_DIR"), "(auto-generated)", "Output directory for inspector logs"}, {"NCCL_INSPECTOR_DUMP_VERBOSE", getenv("NCCL_INSPECTOR_DUMP_VERBOSE"), "0", "Enable/disable verbose dumping (event_trace)"}, {"NCCL_INSPECTOR_PROM_DUMP", getenv("NCCL_INSPECTOR_PROM_DUMP"), "0", "Enable/disable Prometheus format output dump"} }; const int numEnvVars = sizeof(envVars) / sizeof(envVars[0]); VERSION("NCCL Inspector Environment Variables:"); for (int i = 0; i < numEnvVars; i++) { VERSION(" %s = %s%s%s", envVars[i].name, envVars[i].value ? envVars[i].value : "(not set)", envVars[i].value ? "" : ", default=", envVars[i].value ? "" : envVars[i].defaultVal); } } /* * Description: * * Initializes the global inspector state and starts the dump thread * if enabled. * * Thread Safety: * * Not thread-safe (should be called during initialization). * * Input: * None. * * Output: * Global state is initialized and dump thread may be started. * * Return: * inspectorResult_t - success or error code. */ inspectorResult_t inspectorGlobalInit(int rank) { TRACE_INSPECTOR("NCCL Inspector: inspectorGlobalInit"); const char* str = getenv("NCCL_INSPECTOR_ENABLE"); int enable = str ? atoi(str) : 0; // default disable enableNcclInspector = enable == 0 ? false : true; ncclInspectorInit = true; // Show version and environment configuration (similar to NCCL's showVersion) if (rank == 0) { showInspectorVersion(); showInspectorEnvVars(); } if (enableNcclInspector == false) { VERSION("NCCL Inspector Plugin DISABLED (NCCL_INSPECTOR_ENABLE=%s)", str ? str : "0"); return inspectorDisabledError; } // Initialize CUDA wrapper for inspector inspectorResult_t cudaInitResult = inspectorCudaWrapInit(); if (cudaInitResult != inspectorSuccess) { INFO_INSPECTOR("NCCL Inspector: Failed to initialize CUDA wrapper"); return cudaInitResult; } INS_CHK(inspectorGlobalStateInit()); str = getenv("NCCL_INSPECTOR_DUMP_THREAD_ENABLE"); enable = str ? atoi(str) : 1; // default enable enableNcclInspectorDumpThread = enable == 0 ? false : true; str = getenv("NCCL_INSPECTOR_DUMP_VERBOSE"); enable = str ? atoi(str) : 0; // default disable enableNcclInspectorDumpVerbose = enable == 0 ? false : true; // Check for Prometheus dump format str = getenv("NCCL_INSPECTOR_PROM_DUMP"); enable = str ? atoi(str) : 0; // default disable enableNcclInspectorPromDump = enable == 0 ? false : true; // Read and validate dump interval once str = getenv("NCCL_INSPECTOR_DUMP_THREAD_INTERVAL_MICROSECONDS"); ncclInspectorDumpIntervalUsecs = str ? strtoull(str, 0, 0) : 0; // Apply Prometheus-specific interval validation if enabled if (enableNcclInspectorPromDump && enableNcclInspectorDumpThread) { ncclInspectorDumpIntervalUsecs = inspectorPromValidateInterval(ncclInspectorDumpIntervalUsecs); } if (enableNcclInspectorDumpThread) { INS_CHK(inspectorStartDumpThread(ncclInspectorDumpIntervalUsecs)); } else { INFO_INSPECTOR( "NCCL Inspector: NCCL_INSPECTOR_DUMP_THREAD_ENABLE set to 0; not " "starting internal dump " "thread."); } return inspectorSuccess; } /* * Description: * * Returns a string describing the given inspectorResult_t error * code. * * Thread Safety: * Thread-safe (read-only operation). * * Input: * inspectorResult_t result - error code. * * Output: * None. * * Return: * const char* - error string. */ const char* inspectorErrorString(inspectorResult_t result) { switch (result) { case inspectorSuccess: return "Success"; case inspectorUninitializedError: return "Inspector is not initialized"; case inspectorMemoryError: return "Inspector encountered issue allocating memory"; case inspectorFileOpenError: return "Inspector could not open file"; case inspectorDisabledError: return "Inspector is disabled"; case inspectorLockError: return "Inspector encountered error with lock"; case inspectorPthreadError: return "Inspector encountered error with pthreads"; case inspectorJsonError: return "Inspector encountered error while emitting JSON"; case inspectorCudaError: return "Inspector encountered CUDA error"; case inspectorBadHash: return "Inspector encountered bad communicator hash"; case inspectorDeleteUnknownCommError: return "Inspector was asked to delete a communicator that it is not " "tracking"; case inspectorAddDuplicateCommError: return "Inspector was asked to add a communicator it was already " "tracking"; case inspectorNop: return "Inspector NOP"; case inspectorNullTally: return "Inspector encountered a null OpTally"; case inspectorGlobalInitError: return "Inspector encountered a repeated global init"; case inspectorReturn: return "Inspector Unconditional Return"; default: return "Unknown error"; } } /* * Description: * Converts a communicator hash to a string. * * Thread Safety: * Thread-safe (writes to provided buffer). * * Input: * uint64_t commHash - communicator hash. * char hashStr[NCCL_COMM_HASH_LENGTH] - output buffer. * * Output: * hashStr is set to the string representation of commHash. * * Return: * inspectorResult_t - success or error code. */ inspectorResult_t inspectorCommGetHashStr(uint64_t commHash, char hashStr[NCCL_COMM_HASH_LENGTH]) { snprintf(hashStr, NCCL_COMM_HASH_LENGTH, "0x%lx", commHash); return inspectorSuccess; } /* * Description: * Compares two communicator configurations for equality. * * Thread Safety: * Thread-safe (read-only comparison). * * Input: * uint64_t lCommHash - left communicator hash. * uint64_t rCommHash - right communicator hash. * int lRank - left rank. * int rRank - right rank. * * Output: * None. * * Return: * bool - true if communicators are equal (same hash and rank), false otherwise. */ static bool comm_eq(uint64_t lCommHash, uint64_t rCommHash, int lRank, int rRank) { return lCommHash == rCommHash && lRank == rRank; } /* * Description: * Initializes a communicator info structure with the provided parameters. * * Thread Safety: * Not thread-safe - should be called during communicator initialization. * * Input: * struct inspectorCommInfo* commInfo - communicator info structure to initialize (must not be NULL). * const char* commName - communicator name (can be NULL). * uint64_t commHash - communicator hash. * int nnodes - number of nodes (must be > 0). * int nranks - number of ranks (must be > 0). * int rank - rank (must be >= 0 and < nranks). * * Output: * commInfo is initialized with the provided parameters. * * Return: * inspectorResult_t - success or error code. * * Preconditions: * - commInfo must not be NULL * - nnodes must be positive * - nranks must be positive * - rank must be non-negative and less than nranks */ static inspectorResult_t inspectorFillCommInfo(struct inspectorCommInfo* commInfo, const char* commName, uint64_t commHash, int nnodes, int nranks, int rank) { commInfo->commName = commName; commInfo->commHash = commHash; inspectorCommGetHashStr(commHash, commInfo->commHashStr); commInfo->rank = rank; commInfo->nranks = nranks; commInfo->nnodes = nnodes; commInfo->dump = false; // Capture current CUDA device ID and convert to UUID string int cudaDeviceId = -1; cudaError_t err = cudaGetDevice(&cudaDeviceId); if (err != cudaSuccess) { INFO_INSPECTOR("Inspector: Failed to get CUDA device ID: %s", cudaGetErrorString(err)); return inspectorCudaError; } commInfo->cudaDeviceId = cudaDeviceId; // Get CUDA device handle for driver API CUdevice cuDevice; CUresult cuErr = INSPECTOR_CUPFN(cuDeviceGet)(&cuDevice, cudaDeviceId); if (cuErr != CUDA_SUCCESS) { INFO_INSPECTOR("Inspector: Failed to get CUDA device handle for device %d", cudaDeviceId); return inspectorCudaError; } // Get device UUID and convert to string CUuuid deviceUuid; cuErr = INSPECTOR_CUPFN(cuDeviceGetUuid)(&deviceUuid, cuDevice); if (cuErr != CUDA_SUCCESS) { INFO_INSPECTOR("Inspector: Failed to get device UUID for device %d", cudaDeviceId); return inspectorCudaError; } // Format UUID as string (standard UUID format) snprintf(commInfo->deviceUuidStr, sizeof(commInfo->deviceUuidStr), "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", (unsigned char)deviceUuid.bytes[0], (unsigned char)deviceUuid.bytes[1], (unsigned char)deviceUuid.bytes[2], (unsigned char)deviceUuid.bytes[3], (unsigned char)deviceUuid.bytes[4], (unsigned char)deviceUuid.bytes[5], (unsigned char)deviceUuid.bytes[6], (unsigned char)deviceUuid.bytes[7], (unsigned char)deviceUuid.bytes[8], (unsigned char)deviceUuid.bytes[9], (unsigned char)deviceUuid.bytes[10], (unsigned char)deviceUuid.bytes[11], (unsigned char)deviceUuid.bytes[12], (unsigned char)deviceUuid.bytes[13], (unsigned char)deviceUuid.bytes[14], (unsigned char)deviceUuid.bytes[15]); INS_CHK(inspectorLockInit(&commInfo->guard)); commInfo->next = nullptr; // Cache static Prometheus labels if Prometheus mode is enabled if (enableNcclInspectorPromDump) { INS_CHK(inspectorPromCacheStaticLabels(commInfo)); } return inspectorSuccess; } /* * Description: * Adds a communicator to the global state. * * Thread Safety: * Thread-safe (uses locks internally). * * Input: * struct inspectorCommInfo **commInfo - pointer to output struct (must not be NULL). * const char* commName - communicator name (can be NULL). * uint64_t commHash - communicator hash. * int nNodes - number of nodes (must be > 0). * int nranks - number of ranks (must be > 0). * int rank - rank (must be >= 0 and < nranks). * * Output: * commInfo is set to the new communicator struct. * * Return: * inspectorResult_t - success or error code. * * Preconditions: * - commInfo must not be NULL * - nNodes must be positive * - nranks must be positive * - rank must be non-negative and less than nranks */ inspectorResult_t inspectorAddComm(struct inspectorCommInfo **commInfo, const char* commName, uint64_t commHash, int nNodes, int nranks, int rank) { struct inspectorCommInfoList* liveCommInfoList = &g_state.liveComms; struct inspectorCommInfo* commInfoPtr = nullptr; inspectorResult_t res = inspectorSuccess; bool locked = false; INSPECTOR_LOCK_RD_FLAG(&liveCommInfoList->guard, locked, "inspectorAddComm: commList::guard -rd"); for (struct inspectorCommInfo* itr = liveCommInfoList->comms; itr != nullptr; itr = itr->next) { if (comm_eq(commHash, itr->commHash, rank, itr->rank)) { INFO_INSPECTOR("NCCL Inspector: comm 0x%lx already in tracker", commHash); res = inspectorAddDuplicateCommError; goto exit; } } INSPECTOR_UNLOCK_RW_LOCK_FLAG(&liveCommInfoList->guard, locked, "inspectorAddComm: commList::guard"); commInfoPtr = (struct inspectorCommInfo*)calloc(1, sizeof(struct inspectorCommInfo)); if (0 == commInfoPtr) { res = inspectorMemoryError; goto exit; } INS_CHK_GOTO(inspectorFillCommInfo(commInfoPtr, commName, commHash, nNodes, nranks, rank), res, fail); INSPECTOR_LOCK_WR_FLAG(&liveCommInfoList->guard, locked, "inspectorAddComm: commList::guard -wr"); ++liveCommInfoList->ncomms; commInfoPtr->next = liveCommInfoList->comms; liveCommInfoList->comms = commInfoPtr; exit: INSPECTOR_UNLOCK_RW_LOCK_FLAG(&liveCommInfoList->guard, locked, "inspectorAddComm: commList::guard"); *commInfo = commInfoPtr; return res; fail: if (commInfoPtr) { free(commInfoPtr); commInfoPtr = nullptr; } goto exit; } /* * Description: * * Removes a communicator from the global state and moves it to the * deleted list. * * Thread Safety: * Thread-safe (uses locks internally). * * Input: * struct inspectorCommInfo *commInfo - communicator to remove. * * Output: * Communicator is removed from live list and added to deleted list. * * Return: * inspectorResult_t - success or error code. */ inspectorResult_t inspectorDelComm(struct inspectorCommInfo *commInfo) { struct inspectorCommInfoList* liveCommInfoList = &g_state.liveComms; struct inspectorCommInfoList* deletedCommInfoList = &g_state.deletedComms; struct inspectorCommInfo* commInfoPtr = nullptr; bool locked = false; TRACE_INSPECTOR("NCCL Inspector: DelComm removing 0x%lx", commInfo->commHash); INSPECTOR_LOCK_WR_FLAG(&liveCommInfoList->guard, locked, "inspectorDelComm: liveCommInfoList::guard -wr"); struct inspectorCommInfo** prev_ptr = &liveCommInfoList->comms; for (struct inspectorCommInfo* itr = liveCommInfoList->comms; itr != nullptr; itr = itr->next) { if (comm_eq(commInfo->commHash, itr->commHash, commInfo->rank, itr->rank)) { *prev_ptr = itr->next; liveCommInfoList->ncomms--; commInfoPtr = itr; break; } prev_ptr = &itr->next; } INSPECTOR_UNLOCK_RW_LOCK_FLAG(&liveCommInfoList->guard, locked, "inspectorDelComm: liveCommInfoList::guard -unlock"); if (!commInfoPtr) { INFO_INSPECTOR("NCCL Inspector: DelComm can't remove 0x%lx, not present", commInfo->commHash); return inspectorDeleteUnknownCommError; } inspectorLockWr(&commInfoPtr->guard); commInfoPtr->dump = false; inspectorUnlockRWLock(&commInfoPtr->guard); INSPECTOR_LOCK_WR_FLAG(&deletedCommInfoList->guard, locked, "inspectorDelComm: deletedCommInfoList::guard -wr"); commInfoPtr->next = deletedCommInfoList->comms; deletedCommInfoList->comms = commInfoPtr; deletedCommInfoList->ncomms++; INSPECTOR_UNLOCK_RW_LOCK_FLAG(&deletedCommInfoList->guard, locked, "inspectorDelComm: deletedCommInfoList::guard -unlock"); return inspectorSuccess; } /* * Description: * * Computes the algorithmic and bus bandwidth (in GB/s) for a given * NCCL collective operation, based on the communication info and * completed collective details. The calculation uses the message * size, execution time, and the type of collective operation to * determine the effective bandwidths. The 'factor' variable adjusts * the bus bandwidth calculation according to the communication * pattern of each collective, as described in the NCCL performance * documentation: * https://github.com/NVIDIA/nccl-tests/blob/master/doc/PERFORMANCE.md * * Thread Safety: * * This function does not perform any locking and assumes the caller * ensures thread safety if required. * * Input: * * commInfo - Pointer to inspectorCommInfo structure containing * communicator details. * * completedColl- Pointer to inspectorCompletedCollInfo structure * containing completed collective info. * * collType - The type of NCCL collective operation (ncclFunc_t). * * Output: * Updates the algoBwGbs and busBwGbs fields of the completedColl * structure. * * Return: * N.A. (void function) */ void inspectorComputeCollBw(struct inspectorCommInfo *commInfo, struct inspectorCompletedCollInfo *completedColl, ncclFunc_t collType) { double timeInSec = completedColl->execTimeUsecs / 1000000.0; double factor = 0.0; double trafficSize = 0.0; switch (collType) { case ncclFuncReduce: case ncclFuncBroadcast: trafficSize = (double)completedColl->msgSizeBytes; factor = 1; break; case ncclFuncAllReduce: trafficSize = (double)completedColl->msgSizeBytes; factor = ((double)(2 * (commInfo->nranks - 1))) / ((double)commInfo->nranks); break; case ncclFuncReduceScatter: trafficSize = (double)(completedColl->msgSizeBytes * commInfo->nranks); factor = ((double)(commInfo->nranks - 1)) / ((double)commInfo->nranks); break; case ncclFuncAllGather: trafficSize = (double)(completedColl->msgSizeBytes * commInfo->nranks); factor = ((double)(commInfo->nranks - 1)) / ((double)commInfo->nranks); break; case ncclFuncSendRecv: case ncclFuncSend: case ncclFuncRecv: trafficSize = (double)completedColl->msgSizeBytes; factor = 1; break; default: trafficSize = 0; factor = 0.0; } completedColl->algoBwGbs = timeInSec != 0 ? (trafficSize / 1.0E9 / timeInSec) : 0; completedColl->busBwGbs = completedColl->algoBwGbs * factor; } /* * Description: * * Helper function to calculate kernel execution time using GPU * clock values. The GPU clock values are measured in nanoseconds * from the globaltimer register. * * Thread Safety: * Thread-safe (read-only operations on kernel info). * * Input: * struct inspectorKernelChInfo *kernelCh - kernel channel info * containing GPU clock values. * * Output: * None. * * Return: * uint64_t - execution time in microseconds, or 0 if invalid timing * data. */ static uint64_t calculateKernelGpuExecTimeUsecs(struct inspectorKernelChInfo *kernelCh) { if (kernelCh->startGpuClk != 0 && kernelCh->stopGpuClk != 0) { if (kernelCh->stopGpuClk > kernelCh->startGpuClk) { uint64_t execTimeNanosecs = kernelCh->stopGpuClk - kernelCh->startGpuClk; return execTimeNanosecs / 1000; } } return 0; } /* * Description: * * Calculates the maximum kernel execution time across all kernel * channels in a collective operation, using GPU clock values when * available and falling back to CPU timestamps when necessary. * * Thread Safety: * Thread-safe (read-only operations on collective info). * * Input: * struct inspectorCollInfo *collInfo - collective operation info * containing kernel channels. * inspectorTimingSource_t *timingSource - pointer to store the timing source used. * * Output: * timingSource is set to indicate whether GPU, CPU, or collective timing was used. * * Return: * * uint64_t - maximum execution time in microseconds across all * kernels, or collective execution time if no kernel * timing is available. * */ static uint64_t calculateMaxKernelExecTimeUsecs(struct inspectorCollInfo *collInfo, inspectorTimingSource_t *timingSource) { uint64_t maxKernelExecTimeUsecs = 0; inspectorTimingSource_t bestTimingSource = inspectorTimingSourceCollectiveCpu; for (uint32_t i = 0; i < collInfo->nChannels; i++) { struct inspectorKernelChInfo *kernelCh = &collInfo->kernelCh[i]; uint64_t gpuExecTimeUsecs = calculateKernelGpuExecTimeUsecs(kernelCh); if (gpuExecTimeUsecs > 0) { if (gpuExecTimeUsecs > maxKernelExecTimeUsecs) { maxKernelExecTimeUsecs = gpuExecTimeUsecs; bestTimingSource = inspectorTimingSourceKernelGpu; } } else { if (kernelCh->tsCompletedUsec > kernelCh->tsStartUsec) { uint64_t cpuExecTimeUsecs = kernelCh->tsCompletedUsec - kernelCh->tsStartUsec; if (cpuExecTimeUsecs > maxKernelExecTimeUsecs) { maxKernelExecTimeUsecs = cpuExecTimeUsecs; bestTimingSource = inspectorTimingSourceKernelCpu; } } } } if (maxKernelExecTimeUsecs > 0) { *timingSource = bestTimingSource; return maxKernelExecTimeUsecs; } else { *timingSource = inspectorTimingSourceCollectiveCpu; return collInfo->tsCompletedUsec - collInfo->tsStartUsec; } } /* * Description: * * Updates the performance information for a completed collective * operation. * * Thread Safety: * Thread-safe (uses locks internally). * * Input: * struct inspectorCommInfo *commInfo - communicator info. * struct inspectorCollInfo *collInfo - completed collective info. * * Output: * commInfo is updated with completed collective info. * * Return: * None. * */ void inspectorUpdateCollPerf(struct inspectorCompletedCollInfo *completedColl, struct inspectorCollInfo *collInfo) { completedColl->func = ncclStringToFunc(collInfo->func); completedColl->sn = collInfo->sn; completedColl->msgSizeBytes = collInfo->msgSizeBytes; completedColl->execTimeUsecs = calculateMaxKernelExecTimeUsecs(collInfo, &completedColl->timingSource); completedColl->collEvtTrk = collInfo->collEvtTrk; } /* * Description: * * Finalizes the global inspector state and stops the dump thread if * running. * * Thread Safety: * Not thread-safe (should be called during teardown). * * Input: * None. * * Output: * Global state is finalized and dump thread is stopped. * * Return: * inspectorResult_t - success or error code. * */ inspectorResult_t inspectorGlobalFinalize() { // Cleanup CUDA wrapper inspectorCudaWrapCleanup(); if (dumper) { dumper->stopThread(); delete dumper; dumper = nullptr; } return inspectorSuccess; } nccl-2.29.7-1/plugins/profiler/inspector/inspector.h000066400000000000000000000240051515037102200223550ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef INSPECTOR_INSPECTOR_H_ #define INSPECTOR_INSPECTOR_H_ #include #include #include #include #include #include "json.h" #include "common.h" #include "version.h" #define MAX_CHANNELS 64 #define INS_CHK(call) \ do { \ inspectorResult_t res = call; \ if (inspectorSuccess != res) { \ INFO_INSPECTOR("%s:%d -> error %d: %s", __FILE__, __LINE__, res, \ inspectorErrorString(res)); \ return res; \ } \ } while (0); #define INS_CHK_GOTO(call, res, label) \ do { \ res = call; \ if (inspectorSuccess != res) { \ INFO_INSPECTOR("%s:%d -> error %d: %s", __FILE__, __LINE__, res, \ inspectorErrorString(res)); \ goto label; \ } \ } while (0); // Lock convenience macros #define INSPECTOR_LOCK_RD_FLAG(lockRef, lockFlag, debug) \ do { \ if (!lockFlag) { \ INS_CHK(inspectorLockRd(lockRef)); \ } \ lockFlag = true; \ } while (0); #define INSPECTOR_LOCK_WR_FLAG(lockRef, lockFlag, debug) \ do { \ if (!lockFlag) { \ INS_CHK(inspectorLockWr(lockRef)); \ } \ lockFlag = true; \ } while (0); #define INSPECTOR_UNLOCK_RW_LOCK_FLAG(lockRef, lockFlag, debug) \ do { \ if (lockFlag) { \ INS_CHK(inspectorUnlockRWLock(lockRef)); \ } \ lockFlag = false; \ } while (0); typedef enum { ncclFuncBroadcast = 0, ncclFuncReduce = 1, ncclFuncAllGather = 2, ncclFuncReduceScatter = 3, ncclFuncAllReduce = 4, ncclFuncSendRecv = 5, ncclFuncSend = 6, ncclFuncRecv = 7, ncclNumFuncs = 8 } ncclFunc_t; typedef enum { inspectorSuccess = 0, inspectorUninitializedError, inspectorMemoryError, inspectorFileOpenError, inspectorDisabledError, inspectorLockError, inspectorPthreadError, inspectorJsonError, inspectorCudaError, inspectorBadHash, inspectorDeleteUnknownCommError, inspectorAddDuplicateCommError, inspectorNop, inspectorNullTally, inspectorGlobalInitError, inspectorReturn, } inspectorResult_t; typedef enum { inspectorTimingSourceKernelGpu = 0, inspectorTimingSourceKernelCpu = 1, inspectorTimingSourceCollectiveCpu = 2, } inspectorTimingSource_t; struct inspectorEventTraceInfo { uint64_t ts; uint64_t sn; }; typedef enum { NCCL_INSP_EVT_TRK_COLL_START = 0, NCCL_INSP_EVT_TRK_COLL_STOP = 1, NCCL_INSP_EVT_TRK_COLL_NEVT = 2, } inspectorEventTrkColl_t; typedef enum { NCCL_INSP_EVT_TRK_KERNEL_START = 0, NCCL_INSP_EVT_TRK_KERNEL_STOP = 1, NCCL_INSP_EVT_TRK_KERNEL_RECORD = 2, NCCL_INSP_EVT_TRK_KERNEL_NEVT = 3, } inspectorEventTrkKernel_t; struct inspectorEventTrkKernelInfo { struct inspectorEventTraceInfo evntTrace[NCCL_INSP_EVT_TRK_KERNEL_NEVT]; }; struct inspectorEventTrkCollInfo { int sn; uint32_t nChannels; struct inspectorEventTraceInfo evntTrace[NCCL_INSP_EVT_TRK_COLL_NEVT]; struct inspectorEventTrkKernelInfo kernelCh[MAX_CHANNELS]; }; struct inspectorCompletedCollInfo { ncclFunc_t func; uint64_t sn; size_t msgSizeBytes; uint64_t execTimeUsecs; inspectorTimingSource_t timingSource; double algoBwGbs; double busBwGbs; // Event trace information struct inspectorEventTrkCollInfo collEvtTrk; }; enum { NCCL_COMM_HASH_LENGTH = 17 }; struct inspectorCommInfo { struct inspectorCommInfo* next; const char* commName; uint64_t commHash; char commHashStr[NCCL_COMM_HASH_LENGTH]; int rank; int nranks; int nnodes; int cudaDeviceId; // CUDA device ID for this communicator char deviceUuidStr[37]; // Pre-computed device UUID string for filename generation char cachedStaticLabels[256]; // Cached static parts of Prometheus labels (hostname, job, comm, rank, etc.) bool dump; struct inspectorCompletedCollInfo completedCollInfo; pthread_rwlock_t guard; }; // Structure to track flush times and file handles per device UUID struct deviceFlushInfo { char deviceUuidStr[37]; uint64_t lastFlushTime; FILE* fileHandle; // Open file handle for this device bool needsCreation; // Whether file needs to be created/flushed char filename[1024]; // Store filename for cleanup }; struct inspectorDumpThread { bool run{false}; jsonFileOutput* jfo; char* outputRoot; uint64_t sampleIntervalUsecs; std::vector deviceFlushEntries; pthread_t pthread; pthread_rwlock_t guard; // Constructor and destructor implemented in inspector.cc where dependencies are available inspectorDumpThread(const char* _outputRoot, uint64_t _sampleIntervalUsecs); ~inspectorDumpThread(); /* * Gets or creates a file handle for a device UUID, handling flushing as needed. */ FILE* getOrCreateFileHandle(const char* deviceUuidStr, const char* filename, uint64_t currentTime); void startThread(); void stopThread(); inspectorResult_t inspectorStateDump(const char* output_root); inspectorResult_t inspectorStateDumpJSON(const char* output_root); inspectorResult_t inspectorStateDumpProm(const char* output_root); static void* dumpMain(void* arg); }; struct inspectorKernelChInfo { uint64_t type; int refCount; /*unused*/ struct inspectorCollInfo *collInfo; uint8_t channelId; uint64_t tsStartUsec; uint64_t tsCompletedUsec; uint64_t startGpuClk; uint64_t stopGpuClk; }; struct inspectorCollInfo { uint64_t type; int refCount; struct inspectorCommInfo *commInfo; const char* func; uint64_t sn; size_t msgSizeBytes; uint64_t tsStartUsec; uint64_t tsCompletedUsec; uint32_t nChannels; uint32_t nKernelChStarted; uint32_t nKernelChCompleted; pthread_rwlock_t guard; struct inspectorKernelChInfo kernelCh[MAX_CHANNELS]; struct inspectorEventTrkCollInfo collEvtTrk; }; struct inspectorCommInfoList { struct inspectorCommInfo* comms; uint32_t ncomms; pthread_rwlock_t guard; }; struct inspectorState { struct inspectorCommInfoList liveComms; struct inspectorCommInfoList deletedComms; }; extern ncclDebugLogger_t logFn; #define VERSION(...) logFn(NCCL_LOG_VERSION, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__) #define INFO(FLAGS, ...) logFn(NCCL_LOG_INFO, (FLAGS), __func__, __LINE__, __VA_ARGS__) // Use NCCL_PROFILE for inspector messages so they can be filtered with NCCL_DEBUG_SUBSYS=PROFILE #define INFO_INSPECTOR(...) logFn(NCCL_LOG_INFO, NCCL_PROFILE, __func__, __LINE__, __VA_ARGS__) #define TRACE_INSPECTOR(...) logFn(NCCL_LOG_TRACE, NCCL_PROFILE, __func__, __LINE__, __VA_ARGS__) inline int ncclTypeSize(ncclDataType_t type) { switch (type) { case ncclInt8: case ncclUint8: case ncclFloat8e4m3: case ncclFloat8e5m2: return 1; case ncclFloat16: case ncclBfloat16: return 2; case ncclInt32: case ncclUint32: case ncclFloat32: return 4; case ncclInt64: case ncclUint64: case ncclFloat64: return 8; default: return -1; } } const char* inspectorErrorString(inspectorResult_t result); inspectorResult_t inspectorLockInit(pthread_rwlock_t* lockRef); inspectorResult_t inspectorLockDestroy(pthread_rwlock_t* lockRef); inspectorResult_t inspectorLockRd(pthread_rwlock_t* lockRef); inspectorResult_t inspectorLockWr(pthread_rwlock_t* lockRef); inspectorResult_t inspectorUnlockRWLock(pthread_rwlock_t* lockRef); inspectorResult_t inspectorGlobalInit(int rank); inspectorResult_t inspectorGlobalFinalize(); uint64_t inspectorGetTime(); inspectorResult_t inspectorGetTimeUTC(char* buffer, size_t bufferSize); inspectorResult_t inspectorAddComm(struct inspectorCommInfo **commInfo, const char* commName, uint64_t commHash, int nNodes, int nranks, int rank); inspectorResult_t inspectorDelComm(struct inspectorCommInfo *commInfo); void inspectorUpdateCollPerf(struct inspectorCompletedCollInfo *completedColl, struct inspectorCollInfo *collInfo); ncclDataType_t inspectorStringToDatatype(const char* str); void inspectorComputeCollBw(struct inspectorCommInfo *commInfo, struct inspectorCompletedCollInfo *completedColl, ncclFunc_t collType); // Utility functions exposed for Prometheus module const char* inspectorTimingSourceToString(inspectorTimingSource_t timingSource); inspectorResult_t inspectorCommInfoListFinalize(struct inspectorCommInfoList* commList); const char* ncclFuncToString(ncclFunc_t fn); // Global state extern struct inspectorState g_state; #endif // INSPECTOR_INSPECTOR_H_ nccl-2.29.7-1/plugins/profiler/inspector/inspector_cudawrap.cc000066400000000000000000000067021515037102200244050ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "inspector_cudawrap.h" #include #include #include // Function pointer storage PFN_cuGetErrorString pfn_cuGetErrorString = nullptr; PFN_cuDeviceGet pfn_cuDeviceGet = nullptr; PFN_cuDeviceGetUuid pfn_cuDeviceGetUuid = nullptr; // Handle to the CUDA driver library static void* cudaDriverLib = nullptr; /* * Description: * * Loads a CUDA driver function symbol from the library. * * Thread Safety: * Not thread-safe (should be called during initialization). * * Input: * const char* symbol_name - name of the symbol to load. * * Output: * None. * * Return: * void* - pointer to the loaded symbol, or nullptr on failure. */ static void* loadCudaSymbol(const char* symbol_name) { if (!cudaDriverLib) { return nullptr; } void* symbol = dlsym(cudaDriverLib, symbol_name); if (!symbol) { INFO_INSPECTOR("Inspector: Failed to load CUDA symbol '%s': %s", symbol_name, dlerror()); return nullptr; } return symbol; } /* * Description: * * Initializes the CUDA wrapper by loading the CUDA driver library * and resolving required function pointers. * * Thread Safety: * Not thread-safe (should be called once during initialization). * * Input: * None. * * Output: * None. * * Return: * inspectorResult_t - success or error code. */ inspectorResult_t inspectorCudaWrapInit(void) { // Clear any previous dlopen errors dlerror(); // Try to load CUDA driver library cudaDriverLib = dlopen("libcuda.so", RTLD_LAZY); if (!cudaDriverLib) { // Try alternative name cudaDriverLib = dlopen("libcuda.so.1", RTLD_LAZY); if (!cudaDriverLib) { INFO_INSPECTOR("Inspector: Failed to load CUDA driver library: %s", dlerror()); return inspectorCudaError; } } // Load required CUDA driver functions pfn_cuGetErrorString = (PFN_cuGetErrorString)loadCudaSymbol("cuGetErrorString"); if (!pfn_cuGetErrorString) { INFO_INSPECTOR("Inspector: Failed to load cuGetErrorString"); inspectorCudaWrapCleanup(); return inspectorCudaError; } pfn_cuDeviceGet = (PFN_cuDeviceGet)loadCudaSymbol("cuDeviceGet"); if (!pfn_cuDeviceGet) { INFO_INSPECTOR("Inspector: Failed to load cuDeviceGet"); inspectorCudaWrapCleanup(); return inspectorCudaError; } pfn_cuDeviceGetUuid = (PFN_cuDeviceGetUuid)loadCudaSymbol("cuDeviceGetUuid"); if (!pfn_cuDeviceGetUuid) { INFO_INSPECTOR("Inspector: Failed to load cuDeviceGetUuid"); inspectorCudaWrapCleanup(); return inspectorCudaError; } INFO(NCCL_INIT, "Inspector: CUDA wrapper initialized successfully"); return inspectorSuccess; } /* * Description: * * Cleans up the CUDA wrapper by closing the driver library. * * Thread Safety: * Not thread-safe (should be called during cleanup). * * Input: * None. * * Output: * None. * * Return: * None. */ void inspectorCudaWrapCleanup(void) { // Clear function pointers pfn_cuGetErrorString = nullptr; pfn_cuDeviceGet = nullptr; pfn_cuDeviceGetUuid = nullptr; // Close library handle if (cudaDriverLib) { dlclose(cudaDriverLib); cudaDriverLib = nullptr; } } nccl-2.29.7-1/plugins/profiler/inspector/inspector_cudawrap.h000066400000000000000000000024761515037102200242530ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef INSPECTOR_CUDAWRAP_H_ #define INSPECTOR_CUDAWRAP_H_ #include #include // Inspector-specific CUDA wrapper for standalone compilation // Include inspector.h for proper type definitions #include "inspector.h" // Function pointer types for CUDA driver API functions typedef CUresult (*PFN_cuGetErrorString)(CUresult error, const char **pStr); typedef CUresult (*PFN_cuDeviceGet)(CUdevice *device, int ordinal); typedef CUresult (*PFN_cuDeviceGetUuid)(CUuuid *uuid, CUdevice dev); // Function pointers - externally defined extern PFN_cuGetErrorString pfn_cuGetErrorString; extern PFN_cuDeviceGet pfn_cuDeviceGet; extern PFN_cuDeviceGetUuid pfn_cuDeviceGetUuid; // Convenience macro for calling function pointers #define INSPECTOR_CUPFN(symbol) pfn_##symbol // Initialize the CUDA wrapper (load function pointers) inspectorResult_t inspectorCudaWrapInit(void); // Cleanup the CUDA wrapper void inspectorCudaWrapCleanup(void); #endif // INSPECTOR_CUDAWRAP_H_ nccl-2.29.7-1/plugins/profiler/inspector/inspector_plugin.cc000066400000000000000000000402511515037102200240720ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include #include #include #include #include #include #include #include #include "profiler.h" #include "inspector.h" #define __hidden __attribute__ ((visibility("hidden"))) static int gInitialized; static pthread_mutex_t gLock = PTHREAD_MUTEX_INITIALIZER; /* * Description: * Records an event trace with timestamp and sequence number * * Thread Safety: * Not thread-safe - must be called with proper locking. This function * is designed to be called from within locked sections where the * collective info structure is already protected. * * Input: * struct inspectorEventTraceInfo* evtTrace - event trace array * int eventIndex - index in the event trace array (must be valid) * struct inspectorCollInfo* collInfo - collective info structure (must not be NULL) * * Output: * Event trace is updated with current timestamp and next sequence * number from collective * * Return: * uint64_t - the sequence number assigned to this event * * Preconditions: * - collInfo must not be NULL * - eventIndex must be within valid bounds for evtTrace array * - Function must be called from within a locked section */ static uint64_t inspectorRecordEventTrace(struct inspectorEventTraceInfo* evtTrace, int eventIndex, struct inspectorCollInfo* collInfo) { evtTrace[eventIndex].ts = inspectorGetTime(); evtTrace[eventIndex].sn = ++collInfo->collEvtTrk.sn; // Increment coll sequence counter return evtTrace[eventIndex].sn; } /* * Description: * * Initializes the NCCL Inspector plugin and global state for a * communicator. * * Thread Safety: * Thread-safe (uses mutex for initialization). * * Input: * void** context - pointer to plugin context. * int* eActivationMask - pointer to activation mask output. * const char* commName - communicator name. * uint64_t commHash - communicator hash. * int nNodes - number of nodes. * int nranks - number of ranks. * int rank - rank. * ncclDebugLogger_t logfn - logger function pointer. * * Output: * context is set to plugin context; eActivationMask is set. * * Return: * ncclResult_t - success or error code. * */ __hidden ncclResult_t inspectorPluginInit(void** context, uint64_t commHash, int* eActivationMask, const char* commName, int nNodes, int nranks, int rank, ncclDebugLogger_t logfn) { inspectorResult_t res = inspectorSuccess; *context = nullptr; logFn = logfn; pthread_mutex_lock(&gLock); if (++gInitialized == 1) { res = inspectorGlobalInit(rank); if (res != inspectorSuccess) { INFO_INSPECTOR("Inspector Init Failed %s:%d -> error %d: %s",__FILE__, __LINE__, res, inspectorErrorString(res)); gInitialized = 0; pthread_mutex_unlock(&gLock); return ncclSuccess; } } pthread_mutex_unlock(&gLock); res = inspectorAddComm((struct inspectorCommInfo **)context, commName, commHash, nNodes, nranks, rank); if (res != inspectorSuccess) { INFO_INSPECTOR("%s:%d -> error %d: %s", __FILE__, __LINE__, res, inspectorErrorString(res)); return ncclSuccess; } *eActivationMask = ncclProfileColl | ncclProfileKernelCh; INFO(NCCL_INIT, "PROFILER/Plugin: init commName: %s commHash: %lu nranks: %d rank: %d", commName ? commName : "", commHash, nranks, rank); return ncclSuccess; } /* * Description: * * Finalizes the NCCL Inspector plugin and global state for a * communicator. * * Thread Safety: * Thread-safe (uses mutex for finalization). * * Input: * void* context - plugin context. * * Output: * Plugin context is finalized and cleaned up. * * Return: * ncclResult_t - success or error code. * */ __hidden ncclResult_t inspectorPluginFinalize(void* context) { inspectorDelComm((struct inspectorCommInfo *)context); pthread_mutex_lock(&gLock); if (--gInitialized == 0) { inspectorGlobalFinalize(); } pthread_mutex_unlock(&gLock); return ncclSuccess; } inspectorResult_t inspectorPluginCollInfoRef(struct inspectorCollInfo *collInfo) { collInfo->refCount += 1; return inspectorSuccess; } inspectorResult_t inspectorPluginCollInfoRefSafe(struct inspectorCollInfo *collInfo) { inspectorLockWr(&collInfo->guard); inspectorPluginCollInfoRef(collInfo); inspectorUnlockRWLock(&collInfo->guard); return inspectorSuccess; } inspectorResult_t inspectorPluginCollInfoDeRef(struct inspectorCollInfo *collInfo) { collInfo->refCount -= 1; if (collInfo->refCount == 0) { inspectorLockDestroy(&collInfo->guard); memset(collInfo, 0, sizeof(struct inspectorCollInfo)); free(collInfo); return inspectorReturn; } return inspectorSuccess; } inspectorResult_t inspectorPluginCollInfoDeRefSafe(struct inspectorCollInfo *collInfo) { inspectorLockWr(&collInfo->guard); inspectorResult_t res = inspectorPluginCollInfoDeRef(collInfo); inspectorUnlockRWLock(&collInfo->guard); return res; } /* * Description: * Initializes a new inspectorCollInfo structure for a collective * event. * * Thread Safety: * Not thread-safe (allocates and initializes a new collective info * structure). * * Input: * * struct inspectorCollInfo **collInfo - pointer to output * collective info struct. * ncclProfilerEventDescr_t *eDescr - event descriptor. * * Output: * collInfo is set to the new collective info struct. * * Return: * None. */ static void inspectorPluginCollInfoInit(struct inspectorCollInfo **collInfo, ncclProfilerEventDescr_t *eDescr, struct inspectorCommInfo *commInfo) { struct inspectorCollInfo *collInfoPtr = (struct inspectorCollInfo*)calloc(1, sizeof(struct inspectorCollInfo)); if (collInfoPtr == nullptr) { INFO_INSPECTOR("Inspector: Failed to allocate memory for collective info structure"); *collInfo = nullptr; return; } collInfoPtr->type = ncclProfileColl; collInfoPtr->refCount = 0; inspectorPluginCollInfoRef(collInfoPtr); //self ref; no locks needed collInfoPtr->func = eDescr->coll.func; collInfoPtr->sn = eDescr->coll.seqNumber; collInfoPtr->nChannels = eDescr->coll.nChannels; if (collInfoPtr->nChannels > 0) { inspectorPluginCollInfoRef(collInfoPtr); //extra ref for kernel completion } collInfoPtr->tsStartUsec = inspectorGetTime(); collInfoPtr->msgSizeBytes = ncclTypeSize(inspectorStringToDatatype(eDescr->coll.datatype)) * eDescr->coll.count; collInfoPtr->commInfo = commInfo; collInfoPtr->collEvtTrk.sn = 0; collInfoPtr->collEvtTrk.nChannels = collInfoPtr->nChannels; inspectorRecordEventTrace(collInfoPtr->collEvtTrk.evntTrace, NCCL_INSP_EVT_TRK_COLL_START, collInfoPtr); inspectorLockInit(&collInfoPtr->guard); *collInfo = collInfoPtr; } /* * Description: * * Initializes a new inspectorKernelChInfo structure for a kernel * channel event. * * Thread Safety: * Not thread-safe (initializes kernel channel info within a * collective info structure). * * Input: * struct inspectorKernelChInfo **kernelChInfo - pointer to output * kernel channel info struct. * ncclProfilerEventDescr_t *eDescr - event descriptor. * * Output: * * kernelChInfo is set to the new kernel channel info struct. * * Return: * None. */ static void inspectorPluginKernelChInfoInit(struct inspectorKernelChInfo **kernelChInfo, ncclProfilerEventDescr_t *eDescr) { if (eDescr->parentObj) { uint64_t parentType=*(uint64_t*)eDescr->parentObj; if (parentType == ncclProfileColl) { struct inspectorCollInfo *collInfo = (struct inspectorCollInfo*)eDescr->parentObj; if (collInfo && collInfo->type == ncclProfileColl) { inspectorLockWr(&collInfo->guard); struct inspectorEventTraceInfo *krnlEvtTrk = collInfo->collEvtTrk.kernelCh[eDescr->kernelCh.channelId].evntTrace; inspectorRecordEventTrace(krnlEvtTrk, NCCL_INSP_EVT_TRK_KERNEL_START, collInfo); struct inspectorKernelChInfo *kernelChInfoPtr = &collInfo->kernelCh[eDescr->kernelCh.channelId]; kernelChInfoPtr->type = ncclProfileKernelCh; kernelChInfoPtr->channelId = eDescr->kernelCh.channelId; kernelChInfoPtr->startGpuClk = eDescr->kernelCh.pTimer; if (kernelChInfoPtr->stopGpuClk == 0) { inspectorPluginCollInfoRef(collInfo); //Pairs with Record Kernel Stop event } kernelChInfoPtr->tsStartUsec = inspectorGetTime(); if (collInfo->nKernelChStarted == 0) { collInfo->tsStartUsec = kernelChInfoPtr->tsStartUsec; } collInfo->nKernelChStarted += 1; inspectorPluginCollInfoRef(collInfo); //Pairs with Stop Kernel Event kernelChInfoPtr->collInfo = collInfo; *kernelChInfo = kernelChInfoPtr; inspectorUnlockRWLock(&collInfo->guard); } } } } /* * Description: * * Starts a profiling event for the NCCL Inspector plugin. * * Thread Safety: * Thread-safe (allocates and initializes event structures). * * Input: * void* context - plugin context. * void** eHandle - pointer to event handle output. * ncclProfilerEventDescr_t* eDescr - event descriptor. * * Output: * eHandle is set to the new event structure. * * Return: * ncclResult_t - success or error code. * */ __hidden ncclResult_t inspectorPluginStartEvent(void* context, void** eHandle, ncclProfilerEventDescr_t* eDescr) { if (context == nullptr || eDescr == nullptr) { INFO(NCCL_INIT, "Profiler/Plugin: context/eDescr NULL for start event %s", __func__); return ncclSuccess; } *eHandle = nullptr; if (eDescr->type == ncclProfileColl) { struct inspectorCollInfo *collEvent = nullptr; struct inspectorCommInfo *commInfoCtx = (struct inspectorCommInfo*)context; inspectorPluginCollInfoInit(&collEvent, eDescr, commInfoCtx); *eHandle = collEvent; } else if (eDescr->type == ncclProfileKernelCh) { struct inspectorKernelChInfo *kernelChEvent = nullptr; inspectorPluginKernelChInfoInit(&kernelChEvent, eDescr); *eHandle = kernelChEvent; } else { return ncclSuccess; } return ncclSuccess; } /* * Description: * * Stops a profiling event for the NCCL Inspector plugin. * * Thread Safety: * * Thread-safe (updates event state and performance info). * * Input: * * void *eHandle - event handle. * * Output: * * Event is stopped and performance info may be updated. * * Return: * ncclResult_t - success or error code. * */ __hidden ncclResult_t inspectorPluginStopEvent(void *eHandle) { if (eHandle == nullptr) { INFO(NCCL_INIT, "Profiler/Plugin: Event Handle NULL for start event %s", __func__); return ncclSuccess; } uint64_t type = *(uint64_t *)eHandle; inspectorResult_t res = inspectorSuccess; if (type == ncclProfileColl) { struct inspectorCollInfo *collInfo = (struct inspectorCollInfo *)eHandle; // Record collective stop event inspectorLockWr(&collInfo->guard); inspectorRecordEventTrace(collInfo->collEvtTrk.evntTrace, NCCL_INSP_EVT_TRK_COLL_STOP, collInfo); res = inspectorPluginCollInfoDeRef(collInfo); if (res == inspectorReturn) { // WARN("NCCL Inspector unnatural return: inspectorPluginStopEvent:ncclProfileColl"); return ncclSuccess; } inspectorUnlockRWLock(&collInfo->guard); return ncclSuccess; } else if (type == ncclProfileKernelCh) { struct inspectorKernelChInfo *kernelChInfo = (struct inspectorKernelChInfo *)eHandle; struct inspectorCollInfo *collInfo = kernelChInfo->collInfo; if (collInfo && collInfo->type == ncclProfileColl) { inspectorLockWr(&collInfo->guard); struct inspectorEventTraceInfo *krnlEvtTrk = collInfo->collEvtTrk.kernelCh[kernelChInfo->channelId].evntTrace; inspectorRecordEventTrace(krnlEvtTrk, NCCL_INSP_EVT_TRK_KERNEL_STOP, collInfo); kernelChInfo->tsCompletedUsec = inspectorGetTime(); collInfo->nKernelChCompleted += 1; res = inspectorPluginCollInfoDeRef(collInfo); if (res == inspectorReturn) { INFO_INSPECTOR("NCCL Inspector unnatural return: inspectorPluginStopEvent:ncclProfileKernelCh"); return ncclSuccess; } if ((collInfo->nKernelChCompleted == collInfo->nKernelChStarted) && (collInfo->nKernelChCompleted == collInfo->nChannels)) { struct inspectorCompletedCollInfo completedColl; struct inspectorCommInfo *commInfo = collInfo->commInfo; collInfo->tsCompletedUsec = kernelChInfo->tsCompletedUsec; inspectorUpdateCollPerf(&completedColl, collInfo); res = inspectorPluginCollInfoDeRef(collInfo); if (res != inspectorReturn) { inspectorUnlockRWLock(&collInfo->guard); } if (commInfo != nullptr) { inspectorLockWr(&commInfo->guard); inspectorComputeCollBw(commInfo, &completedColl, completedColl.func); memcpy(&commInfo->completedCollInfo, &completedColl, sizeof(struct inspectorCompletedCollInfo)); commInfo->dump = true; inspectorUnlockRWLock(&commInfo->guard); } return ncclSuccess; } inspectorUnlockRWLock(&collInfo->guard); } return ncclSuccess; } return ncclSuccess; } /* * Description: * * Records the state of a profiling event for the NCCL Inspector * plugin. * * Thread Safety: * * Thread-safe (updates event state as needed). * * Input: * void* eHandle - event handle. * ncclProfilerEventState_t eState - event state. * ncclProfilerEventStateArgs_t* eStateArgs - event state arguments. * * Output: * Event state is updated as needed. * * Return: * ncclResult_t - success or error code. * */ __hidden ncclResult_t inspectorPluginRecordEventState(void* eHandle, ncclProfilerEventState_t eState, ncclProfilerEventStateArgs_t* eStateArgs) { if (eHandle == nullptr || eStateArgs == nullptr) return ncclSuccess; uint64_t type = *(uint64_t *)eHandle; if (type == ncclProfileKernelCh && eState == ncclProfilerKernelChStop) { struct inspectorKernelChInfo *kernelChInfo = (struct inspectorKernelChInfo *)eHandle; struct inspectorCollInfo *collInfo = kernelChInfo->collInfo; inspectorResult_t res = inspectorSuccess; if (collInfo && collInfo->type == ncclProfileColl) { inspectorLockWr(&collInfo->guard); struct inspectorEventTraceInfo *krnlEvtTrk = collInfo->collEvtTrk.kernelCh[kernelChInfo->channelId].evntTrace; inspectorRecordEventTrace(krnlEvtTrk, NCCL_INSP_EVT_TRK_KERNEL_RECORD, collInfo); kernelChInfo->stopGpuClk = eStateArgs->kernelCh.pTimer; if (kernelChInfo->startGpuClk != 0) { res = inspectorPluginCollInfoDeRef(collInfo); if (res == inspectorReturn) { INFO_INSPECTOR("NCCL Inspector unnatural return: inspectorPluginRecordEventState"); return ncclSuccess; } } inspectorUnlockRWLock(&collInfo->guard); } } return ncclSuccess; } ncclProfiler_t ncclProfiler_v5 = { "Inspector", inspectorPluginInit, inspectorPluginStartEvent, inspectorPluginStopEvent, inspectorPluginRecordEventState, inspectorPluginFinalize, }; nccl-2.29.7-1/plugins/profiler/inspector/inspector_prom.cc000066400000000000000000000314701515037102200235540ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "inspector_prom.h" #include "inspector.h" #include "inspector_cudawrap.h" #include #include #include #include #include #include #include #include // External references from inspector.cc extern struct inspectorState g_state; extern inspectorResult_t inspectorCommInfoListFinalize(struct inspectorCommInfoList* commList); extern const char* inspectorTimingSourceToString(inspectorTimingSource_t timingSource); extern const char* ncclFuncToString(ncclFunc_t fn); /* * Description: * * Converts bytes to human-readable format (KB, MB, GB, etc.). * * Thread Safety: * * Not thread-safe. Onus of thread safety is on the caller/owner of * the buffer. * * Input: * size_t bytes - number of bytes. * char* output - output buffer for formatted string. * size_t outputSize - size of output buffer. * * Output: * Human-readable size string is written to buffer. * * Return: * None. */ static void inspectorFormatHumanReadableSize(size_t bytes, char* output, size_t outputSize) { const char* units[] = {"B", "KB", "MB", "GB", "TB"}; int unitIndex = 0; double size = (double)bytes; while (size >= 1024.0 && unitIndex < 4) { size /= 1024.0; unitIndex++; } if (unitIndex == 0) { // For bytes, show as integer snprintf(output, outputSize, "%zuB", bytes); } else { // For larger units, show with decimal precision snprintf(output, outputSize, "%.2f%s", size, units[unitIndex]); } } /* * Description: * * Caches the static parts of Prometheus labels (hostname, job info, comm info) * for a communicator. This should be called once when the communicator is created. * * Thread Safety: * Not thread-safe (should be called during comm initialization). * * Input: * struct inspectorCommInfo* commInfo - communicator info. * * Output: * commInfo->cachedStaticLabels is populated. * * Return: * inspectorResult_t - success or error code. */ inspectorResult_t inspectorPromCacheStaticLabels(struct inspectorCommInfo* commInfo) { char hostname[256]; const char* jobId = getenv("SLURM_JOB_ID"); const char* jobName = getenv("SLURM_JOB_NAME"); gethostname(hostname, sizeof(hostname)-1); hostname[sizeof(hostname)-1] = '\0'; char gpuDeviceStr[16]; snprintf(gpuDeviceStr, sizeof(gpuDeviceStr), "GPU%d", commInfo->cudaDeviceId); int ret = snprintf(commInfo->cachedStaticLabels, sizeof(commInfo->cachedStaticLabels), "comm_id=\"%s\",hostname=\"%s\",rank=\"%d\"," "slurm_job=\"%s\",slurm_job_id=\"%s\",nranks=\"%d\"," "n_nodes=\"%d\",gpu_device_id=\"%s\"", commInfo->commHashStr, hostname, commInfo->rank, jobName ? jobName : "unknown", jobId ? jobId : "unknown", commInfo->nranks, commInfo->nnodes, gpuDeviceStr); if (ret < 0 || (size_t)ret >= sizeof(commInfo->cachedStaticLabels)) { return inspectorMemoryError; } return inspectorSuccess; } /* * Description: * * Formats labels for Prometheus metrics from communicator and collective info. * Uses cached static labels and only adds dynamic parts (collective, timestamp, etc). * * Thread Safety: * * Not thread-safe. Onus of thread safety is on the caller/owner of * the buffer. * * Input: * char* labels - output buffer for formatted labels. * size_t labelSize - size of labels buffer. * struct inspectorCommInfo* commInfo - communicator info (with cached static labels). * struct inspectorCompletedCollInfo* collInfo - completed collective info. * * Output: * Formatted labels string is written to buffer. * * Return: * inspectorResult_t - success or error code. */ static inspectorResult_t inspectorPromGetLabels(char* labels, size_t labelSize, struct inspectorCommInfo* commInfo, struct inspectorCompletedCollInfo* collInfo) { char datetimeStr[32]; INS_CHK(inspectorGetTimeUTC(datetimeStr, sizeof(datetimeStr))); char msgSizeStr[32]; inspectorFormatHumanReadableSize(collInfo->msgSizeBytes, msgSizeStr, sizeof(msgSizeStr)); int ret = snprintf(labels, labelSize, "%s,collective=\"%s\",coll_sn=\"%lu\",timestamp=\"%s\",message_size=\"%s\"", commInfo->cachedStaticLabels, ncclFuncToString(collInfo->func), collInfo->sn, datetimeStr, msgSizeStr); if (ret < 0 || (size_t)ret >= labelSize) { return inspectorMemoryError; } return inspectorSuccess; } /* * Description: * * Generates GPU-specific Prometheus filename using pre-computed * device UUID. Each GPU gets its own file containing metrics from * all communicators using that GPU. * * Thread Safety: * * Not thread-safe. Onus of thread safety is on the caller/owner of * the buffer. * * Input: * const char* baseFilename - base output file path. * const char* deviceUuidStr - pre-computed device UUID string. * char* output - output buffer for filename. * size_t outputSize - size of output buffer. * * Output: * UUID-based filename is written to output buffer. * * Return: * inspectorResult_t - success or error code. */ static inspectorResult_t inspectorPromGetFilename(const char* baseFilename, const char* deviceUuidStr, char* output, size_t outputSize) { snprintf(output, outputSize, "%s/nccl_inspector_metrics_%s.prom", baseFilename, deviceUuidStr); return inspectorSuccess; } /* * Description: * * Writes Prometheus metrics for a completed collective to a file. * * Thread Safety: * * Not thread-safe. Onus of thread safety is on the caller/owner of * the file handle. * * Input: * const char* filename - output file path. * struct inspectorCommInfo* commInfo - communicator info. * struct inspectorCompletedCollInfo* collInfo - completed collective info. * * Output: * Metrics are written to file. * * Return: * inspectorResult_t - success or error code. */ static inspectorResult_t inspectorPromWriteCollInfo(FILE* file, struct inspectorCommInfo* commInfo, struct inspectorCompletedCollInfo* collInfo) { if (!file) { return inspectorFileOpenError; } char labels[512]; memset(labels, 0, sizeof(labels)); INS_CHK(inspectorPromGetLabels(labels, sizeof(labels), commInfo, collInfo)); char buffer[2048]; int written = snprintf(buffer, sizeof(buffer), "nccl_algorithm_bandwidth_gbs{%s} %.6g\n" "nccl_bus_bandwidth_gbs{%s} %.6g\n" "nccl_collective_exec_time_microseconds{%s} %.6g\n", labels, collInfo->algoBwGbs, labels, collInfo->busBwGbs, labels, (double)collInfo->execTimeUsecs); if (written < 0 || (size_t)written >= sizeof(buffer)) { return inspectorMemoryError; } if (fwrite(buffer, 1, written, file) != (size_t)written) { return inspectorFileOpenError; } fflush(file); return inspectorSuccess; } /* * Description: * * Dumps the state of a single communicator to Prometheus format. * * Thread Safety: * Not thread-safe (should be called with proper locking). * * Input: * struct inspectorCommInfo* commInfo - communicator info. * const char* filename - output filename. * bool* needs_writing - set to true if output was written. * * Output: * Prometheus metrics are written to file if needed. * * Return: * inspectorResult_t - success or error code. */ static inspectorResult_t inspectorPromCommInfoDump(struct inspectorCommInfo* commInfo, FILE* file, bool* needs_writing) { *needs_writing = false; if (commInfo == nullptr || file == nullptr) { return inspectorSuccess; } struct inspectorCompletedCollInfo collInfo; memset(&collInfo, 0, sizeof(struct inspectorCompletedCollInfo)); inspectorLockWr(&commInfo->guard); if (commInfo->dump) { *needs_writing = true; memcpy(&collInfo, &commInfo->completedCollInfo, sizeof(struct inspectorCompletedCollInfo)); commInfo->dump = false; // Clear flag after reading } inspectorUnlockRWLock(&commInfo->guard); if (*needs_writing) { TRACE_INSPECTOR("NCCL Inspector: Writing metrics for comm %s directly", commInfo->commHashStr); INS_CHK(inspectorPromWriteCollInfo(file, commInfo, &collInfo)); } return inspectorSuccess; } /* * Description: * * Dumps the state of all communicators in a commList to Prometheus format. * * Thread Safety: * Thread-safe - acquires necessary locks to iterate through communicators. * * Input: * struct inspectorCommInfoList* commList - list of communicators. * const char* output_root - base output directory. * * Output: * Prometheus metrics are written to UUID-named file. * * Return: * inspectorResult_t - success or error code. */ inspectorResult_t inspectorPromCommInfoListDump(struct inspectorCommInfoList* commList, const char* output_root, struct inspectorDumpThread* dumpThread) { INS_CHK(inspectorLockRd(&commList->guard)); inspectorResult_t res = inspectorSuccess; if (commList->ncomms > 0) { uint32_t processed = 0; uint64_t currentTime = inspectorGetTime(); for (struct inspectorCommInfo* itr = commList->comms; itr != nullptr; itr = itr->next) { bool needs_writing; // Get filename for this specific communicator's device char filename[1024]; INS_CHK_GOTO(inspectorPromGetFilename(output_root, itr->deviceUuidStr, filename, sizeof(filename)), res, exit); FILE* file = dumpThread ? dumpThread->getOrCreateFileHandle(itr->deviceUuidStr, filename, currentTime) : NULL; if (!file) { INFO_INSPECTOR("NCCL Inspector: Failed to get file handle for device UUID %s, file %s", itr->deviceUuidStr, filename); continue; } INS_CHK_GOTO(inspectorPromCommInfoDump(itr, file, &needs_writing), res, exit); if (needs_writing) { processed++; TRACE_INSPECTOR( "NCCL Inspector: Processed comm %u for CUDA device (rank %d)", processed, itr->rank); } } TRACE_INSPECTOR( "NCCL Inspector: Completed dump across devices, flushed %u/%u communicators", processed, commList->ncomms); } exit: INS_CHK(inspectorUnlockRWLock(&commList->guard)); return res; } /* * Description: * * Validates and adjusts the dump interval for Prometheus-specific requirements. * Prometheus requires a minimum 30-second interval to match node exporter poll interval. * * Thread Safety: * Thread-safe. * * Input: * uint64_t interval - raw interval in microseconds from environment variable. * * Output: * None. * * Return: * uint64_t - validated interval in microseconds. */ uint64_t inspectorPromValidateInterval(uint64_t interval) { const uint64_t MIN_PROM_INTERVAL = 30000000; if (interval > 0 && interval < MIN_PROM_INTERVAL) { INFO_INSPECTOR( "NCCL Inspector: Prometheus dump requires minimum interval of %lu microseconds " "to match node exporter poll interval, but got %lu. Setting to minimum.", MIN_PROM_INTERVAL, interval); return MIN_PROM_INTERVAL; } else if (interval == 0) { INFO_INSPECTOR( "NCCL Inspector: Using default interval of %lu microseconds for Prometheus dump", MIN_PROM_INTERVAL); return MIN_PROM_INTERVAL; } return interval; } nccl-2.29.7-1/plugins/profiler/inspector/inspector_prom.h000066400000000000000000000020501515037102200234060ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef INSPECTOR_INSPECTOR_PROM_H_ #define INSPECTOR_INSPECTOR_PROM_H_ #include #include "inspector.h" // Forward declarations struct inspectorCommInfoList; struct inspectorDumpThread; // Prometheus-related function declarations inspectorResult_t inspectorPromCacheStaticLabels(struct inspectorCommInfo* commInfo); inspectorResult_t inspectorPromCommInfoListDump(struct inspectorCommInfoList* commList, const char* output_root, struct inspectorDumpThread* dumpThread); // Prometheus-specific configuration uint64_t inspectorPromValidateInterval(uint64_t interval); #endif // INSPECTOR_INSPECTOR_PROM_H_ nccl-2.29.7-1/plugins/profiler/inspector/json.cc000066400000000000000000000314031515037102200214560ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "json.h" #include #include #include #include #include #include const char* jsonErrorString(jsonResult_t res) { switch (res) { case jsonSuccess: return "jsonSuccess"; case jsonFileError: return "jsonFileError"; case jsonUnknownStateError: return "jsonUnknownStateError"; case jsonEmptyStateError: return "jsonEmptyStateError"; case jsonExpectedNonNoneStateError: return "jsonExpectedNonNoneStateError"; case jsonMemoryError: return "jsonMemoryError"; case jsonStringOverflowError: return "jsonStringOverflowError"; case jsonStringBadChar: return "jsonStringBadChar"; case jsonLockError: return "jsonLockError"; default: return "unknown json error"; } } // We use these statics to mantain a stack of states where we are writing. typedef struct jsonFileOutput { jsonState_t* states; size_t state_cap; // Allocated stack capacity size_t state_n; // # of items in the stack. FILE* fp; pthread_mutex_t mutex; } jsonFileOutput; jsonResult_t jsonInitFileOutput(jsonFileOutput** jfo, const char* outfile) { jsonFileOutput* new_jfo = (jsonFileOutput*)malloc(sizeof(jsonFileOutput)); if (new_jfo == NULL) { return jsonMemoryError; } if (pthread_mutex_init(&new_jfo->mutex, NULL) != 0) { free(new_jfo); *jfo = 0; return jsonLockError; } new_jfo->states = NULL; new_jfo->state_cap = 0; new_jfo->state_n = 0; new_jfo->fp = fopen(outfile, "w"); if (new_jfo->fp == NULL) { free(new_jfo); *jfo = 0; return jsonFileError; } *jfo = new_jfo; return jsonSuccess; } jsonResult_t jsonNewline(jsonFileOutput* jfo) { fprintf(jfo->fp, "\n"); return jsonSuccess; } jsonResult_t jsonFlushOutput(jsonFileOutput* jfo) { fflush(jfo->fp); return jsonSuccess; } jsonResult_t jsonLockOutput(jsonFileOutput* jfo) { if (pthread_mutex_lock(&jfo->mutex) != 0) { return jsonLockError; } return jsonSuccess; } jsonResult_t jsonUnlockOutput(jsonFileOutput* jfo) { if (pthread_mutex_unlock(&jfo->mutex) != 0) { return jsonLockError; } return jsonSuccess; } jsonResult_t jsonFinalizeFileOutput(jsonFileOutput* jfo) { // Really should probably complain if we aren't in a valid state if (pthread_mutex_destroy(&jfo->mutex) != 0) { free(jfo); return jsonLockError; } if (jfo->states != NULL) { free(jfo->states); } jfo->states = NULL; jfo->state_cap = 0; jfo->state_n = 0; if (jfo->fp) { fclose(jfo->fp); jfo->fp = 0; } free(jfo); return jsonSuccess; } static int utf8copy(unsigned char* out, int out_lim, const unsigned char* in) { int copy_len; if ((in[0] & 0xE0) == 0xC0) { // 2-byte sequence if ((in[1] & 0xC0) != 0x80 || out_lim < 2) { return 0; } copy_len = 2; } else if ((in[0] & 0xF0) == 0xE0) { // 3-byte sequence if ((in[1] & 0xC0) != 0x80 || (in[2] & 0xC0) != 0x80 || out_lim < 3) { return 0; } copy_len = 3; } else if ((in[0] & 0xF8) == 0xF0) { // 4-byte sequence if ((in[1] & 0xC0) != 0x80 || (in[2] & 0xC0) != 0x80 || (in[3] & 0xC0) != 0x80 || out_lim < 4) { return 0; } copy_len = 4; } else { // Invalid start byte return 0; } for (int i = 0; i < copy_len; ++i) { out[i] = in[i]; } return copy_len; } // This tries to sanitize/quote a string from 'in' into 'out', // assuming 'out' has length 'lim'. We mainly quote ",/,\,\t,\n, and // bail if we encounter non-printable stuff or non-ASCII stuff. // 'in' should be null-terminated, of course. // // We return false if we were not able to copy all of 'in', either for // length reasons or for unhandled characters. static jsonResult_t sanitizeJson(unsigned char out[], int lim, const unsigned char* in) { int c = 0; while (*in) { if (c + 1 >= lim) { out[c] = 0; return jsonStringOverflowError; } switch (*in) { case '"': case '\\': case '/': case '\t': case '\n': if (c + 2 > lim) { out[c] = 0; return jsonStringOverflowError; } out[c++] = '\\'; if (*in == '\n') { out[c++] = 'n'; } else if (*in == '\t') { out[c++] = 't'; } else { out[c++] = *in; } ++in; break; default: if (*in <= 0x1F) { out[c] = 0; return jsonStringBadChar; } else if (*in <= 0x7F) { out[c++] = *in; ++in; } else { const int utf8len = utf8copy(out + c, lim - c - 1, in); if (utf8len == 0) { out[c] = 0; return jsonStringBadChar; } c += utf8len; in += utf8len; } break; } } out[c] = 0; return jsonSuccess; } static size_t max(size_t a, size_t b) { if (a < b) { return b; } return a; } // Push state onto the state stack. Reallocate for extra storage if needed. // Because JSON_NONE is a pseudo-state, don't allow it to be pushed. static jsonResult_t jsonPushState(jsonFileOutput* jfo, jsonState_t state) { if (state == JSON_NONE) { return jsonExpectedNonNoneStateError; } if (jfo->state_cap <= (jfo->state_n + 1)) { jfo->state_cap = max((size_t)16, jfo->state_cap * 2); jfo->states = (jsonState_t*)realloc(jfo->states, sizeof(jsonState_t) * jfo->state_cap); if (jfo->states == 0) { return jsonMemoryError; } } jfo->states[jfo->state_n++] = state; return jsonSuccess; } // Return the current state at the top of the stack static jsonState_t jsonCurrState(const jsonFileOutput* jfo) { if (jfo->state_n == 0) { return JSON_NONE; } return jfo->states[jfo->state_n - 1]; } // Replace the stack with state (equivalent to a pop & push if stack is not empty) static jsonResult_t jsonReplaceState(jsonFileOutput* jfo, jsonState_t state) { if (state == JSON_NONE) { return jsonExpectedNonNoneStateError; } if (jfo->state_n == 0) { return jsonEmptyStateError; } jfo->states[jfo->state_n - 1] = state; return jsonSuccess; } // Pop the top state off the stack, or return that the state is empty static jsonState_t jsonPopState(jsonFileOutput* jfo) { if (jfo->state_n == 0) { return JSON_NONE; } return jfo->states[--jfo->state_n]; } // Emit a key and separator. Santize the key. // This is only acceptable if the top state is an object // Emit a ',' separator of we aren't the first item. jsonResult_t jsonKey(jsonFileOutput* jfo, const char* name) { switch (jsonCurrState(jfo)) { case JSON_OBJECT_EMPTY: jsonReplaceState(jfo, JSON_OBJECT_SOME); break; case JSON_OBJECT_SOME: fprintf(jfo->fp, ","); break; default: return jsonUnknownStateError; } unsigned char tmp[2048]; const jsonResult_t res = sanitizeJson(tmp, sizeof(tmp), (const unsigned char*)name); if (res != jsonSuccess) { return res; } fprintf(jfo->fp, "\"%s\":", tmp); jsonPushState(jfo, JSON_KEY); return jsonSuccess; } // Helper function for inserting values. // Only acceptable after keys, top-level, or in lists. // Emit preceeding ',' if in a list and not first item. static jsonResult_t jsonValHelper(jsonFileOutput* jfo) { switch (jsonCurrState(jfo)) { case JSON_LIST_EMPTY: jsonReplaceState(jfo, JSON_LIST_SOME); break; case JSON_LIST_SOME: fprintf(jfo->fp, ","); break; case JSON_KEY: jsonPopState(jfo); break; case JSON_NONE: break; default: return jsonUnknownStateError; } return jsonSuccess; } // Start an object jsonResult_t jsonStartObject(jsonFileOutput* jfo) { const jsonResult_t res = jsonValHelper(jfo); if (res != jsonSuccess) { return res; } fprintf(jfo->fp, "{"); return jsonPushState(jfo, JSON_OBJECT_EMPTY); } // Close an object jsonResult_t jsonFinishObject(jsonFileOutput* jfo) { switch (jsonPopState(jfo)) { case JSON_OBJECT_EMPTY: case JSON_OBJECT_SOME: break; default: return jsonUnknownStateError; } fprintf(jfo->fp, "}"); return jsonSuccess; } // Start a list jsonResult_t jsonStartList(jsonFileOutput* jfo) { const jsonResult_t res = jsonValHelper(jfo); if (res != jsonSuccess) { return res; } fprintf(jfo->fp, "["); return jsonPushState(jfo, JSON_LIST_EMPTY); } // Close a list jsonResult_t jsonFinishList(jsonFileOutput* jfo) { switch (jsonPopState(jfo)) { case JSON_LIST_EMPTY: case JSON_LIST_SOME: break; default: return jsonUnknownStateError; } fprintf(jfo->fp, "]"); return jsonSuccess; } // Write a null value jsonResult_t jsonNull(jsonFileOutput* jfo) { const jsonResult_t res = jsonValHelper(jfo); if (res != jsonSuccess) { return res; } fprintf(jfo->fp, "null"); return jsonSuccess; } // Write a (sanititzed) string jsonResult_t jsonStr(jsonFileOutput* jfo, const char* str) { if (str == NULL) { jsonNull(jfo); return jsonSuccess; } const jsonResult_t res = jsonValHelper(jfo); if (res != jsonSuccess) { return res; } unsigned char tmp[2048]; const jsonResult_t san_res = sanitizeJson(tmp, sizeof(tmp), (const unsigned char*)str); if (san_res != jsonSuccess) { return san_res; } fprintf(jfo->fp, "\"%s\"", tmp); return jsonSuccess; } // Write a bool as "true" or "false" strings. jsonResult_t jsonBool(jsonFileOutput* jfo, bool val) { return jsonStr(jfo, val ? "true" : "false"); } // Write an integer value jsonResult_t jsonInt(jsonFileOutput* jfo, const int val) { const jsonResult_t res = jsonValHelper(jfo); if (res != jsonSuccess) { return res; } fprintf(jfo->fp, "%d", val); return jsonSuccess; } // Write an integer value jsonResult_t jsonUint32(jsonFileOutput* jfo, const uint32_t val) { const jsonResult_t res = jsonValHelper(jfo); if (res != jsonSuccess) { return res; } fprintf(jfo->fp, "%u", val); return jsonSuccess; } // Write an integer value jsonResult_t jsonUint64(jsonFileOutput* jfo, const uint64_t val) { const jsonResult_t res = jsonValHelper(jfo); if (res != jsonSuccess) { return res; } fprintf(jfo->fp, "%lu", val); return jsonSuccess; } // Write a size_t value jsonResult_t jsonSize_t(jsonFileOutput* jfo, const size_t val) { const jsonResult_t res = jsonValHelper(jfo); if (res != jsonSuccess) { return res; } fprintf(jfo->fp, "%zu", val); return jsonSuccess; } // Write a double value jsonResult_t jsonDouble(jsonFileOutput* jfo, const double val) { const jsonResult_t res = jsonValHelper(jfo); if (res != jsonSuccess) { return res; } if (val != val) { fprintf(jfo->fp, "\"nan\""); } else { fprintf(jfo->fp, "%lf", val); } return jsonSuccess; } #ifdef DO_JSON_TEST // compile with // gcc json.cc -Iinclude/ -DDO_JSON_TEST -o json_test // run with: // ./json_test // if something fails, it will print out the error // if it all works, print out "output matches reference" #define JSONCHECK(expr) \ do { \ const jsonResult_t res = (expr); \ if (res != jsonSuccess) { \ fprintf(stderr, "jsonError: %s\n", jsonErrorString(res)); \ exit(1); \ } \ } while (0) int main() { const char refstr[] = "{\"number\":123,\"utfstring\":\"∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ∀x∈ℝ: ⌈x⌉ = −⌊−x⌋, α ∧ " "¬β = ¬(¬α ∨ β),\",\"list\":[\"true\",null,9423812381231,3123111,0.694234]}"; jsonFileOutput* jfo; JSONCHECK(jsonInitFileOutput(&jfo, "test.json")); JSONCHECK(jsonStartObject(jfo)); JSONCHECK(jsonKey(jfo, "number")); JSONCHECK(jsonInt(jfo, 123)); JSONCHECK(jsonKey(jfo, "utfstring")); JSONCHECK( jsonStr(jfo, "∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ∀x∈ℝ: ⌈x⌉ = −⌊−x⌋, α ∧ ¬β = ¬(¬α ∨ β),")); JSONCHECK(jsonKey(jfo, "list")); JSONCHECK(jsonStartList(jfo)); JSONCHECK(jsonBool(jfo, true)); JSONCHECK(jsonNull(jfo)); JSONCHECK(jsonUint64(jfo, 9423812381231ULL)); JSONCHECK(jsonSize_t(jfo, 3123111)); JSONCHECK(jsonDouble(jfo, 0.69423413)); JSONCHECK(jsonFinishList(jfo)); JSONCHECK(jsonFinishObject(jfo)); JSONCHECK(jsonFinalizeFileOutput(jfo)); FILE* fp = fopen("test.json", "r"); const size_t reflen = sizeof(refstr) / sizeof(char); char buffer[reflen]; fread(buffer, sizeof(char), reflen, fp); fclose(fp); if (memcmp(buffer, refstr, reflen) == 0) { printf("output matches reference\n"); } else { printf("output %s\nreference %s\n", buffer, refstr); return 1; } return 0; } #endif nccl-2.29.7-1/plugins/profiler/inspector/json.h000066400000000000000000000050231515037102200213170ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef INSPECTOR_JSON_H_ #define INSPECTOR_JSON_H_ #include #include #include typedef enum { JSON_NONE, // A pseudo-state meaning that the document is empty JSON_KEY, JSON_OBJECT_EMPTY, JSON_OBJECT_SOME, JSON_LIST_EMPTY, JSON_LIST_SOME, } jsonState_t; typedef enum { jsonSuccess, jsonFileError, jsonUnknownStateError, jsonEmptyStateError, jsonExpectedNonNoneStateError, jsonStringOverflowError, jsonStringBadChar, jsonMemoryError, jsonLockError, } jsonResult_t; const char *jsonErrorString(jsonResult_t res); typedef struct jsonFileOutput jsonFileOutput; jsonResult_t jsonLockOutput(jsonFileOutput *jfo); jsonResult_t jsonUnlockOutput(jsonFileOutput *jfo); jsonResult_t jsonInitFileOutput(jsonFileOutput **jfo, const char *outfile); jsonResult_t jsonFinalizeFileOutput(jsonFileOutput *jfo); jsonResult_t jsonNewline(jsonFileOutput *jfo); jsonResult_t jsonFlushOutput(jsonFileOutput *jfo); // Emit a key and separator. Santize the key. // This is only acceptable if the top state is an object // Emit a ',' separator of we aren't the first item. jsonResult_t jsonKey(jsonFileOutput *jfo, const char *name); // Start an object jsonResult_t jsonStartObject(jsonFileOutput *jfo); // Close an object jsonResult_t jsonFinishObject(jsonFileOutput *jfo); // Start a list jsonResult_t jsonStartList(jsonFileOutput *jfo); // Close a list jsonResult_t jsonFinishList(jsonFileOutput *jfo); // Emit a null value jsonResult_t jsonNull(jsonFileOutput *jfo); // Write a (sanititzed) string jsonResult_t jsonStr(jsonFileOutput *jfo, const char *str); // Write a bool as "true" or "false" strings. jsonResult_t jsonBool(jsonFileOutput *jfo, bool val); // Write an integer value jsonResult_t jsonInt(jsonFileOutput *jfo, const int val); //Write an unsigned int value jsonResult_t jsonUint32(jsonFileOutput *jfo, const uint32_t val); // Write an integer value jsonResult_t jsonUint64(jsonFileOutput *jfo, const uint64_t val); // Write a size_t value jsonResult_t jsonSize_t(jsonFileOutput *jfo, const size_t val); // Write a double value jsonResult_t jsonDouble(jsonFileOutput *jfo, const double val); #endif // INSPECTOR_JSON_H_ nccl-2.29.7-1/plugins/profiler/inspector/nccl/000077500000000000000000000000001515037102200211145ustar00rootroot00000000000000nccl-2.29.7-1/plugins/profiler/inspector/nccl/common.h000066400000000000000000000047531515037102200225660ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef COMMON_H_ #define COMMON_H_ /* typedef enum {NCCL_LOG_NONE=0, NCCL_LOG_VERSION=1, NCCL_LOG_WARN=2, NCCL_LOG_INFO=3, NCCL_LOG_ABORT=4, NCCL_LOG_TRACE=5} ncclDebugLogLevel; */ /* typedef enum {NCCL_INIT=1, NCCL_COLL=2, NCCL_P2P=4, NCCL_SHM=8, NCCL_NET=16, NCCL_GRAPH=32, NCCL_TUNING=64, NCCL_ENV=128, NCCL_ALLOC=256, NCCL_CALL=512, NCCL_PROXY=1024, NCCL_NVLS=2048, NCCL_BOOTSTRAP=4096, NCCL_REG=8192, NCCL_ALL=~0} ncclDebugLogSubSys; */ /* Data types */ typedef enum { ncclInt8 = 0, ncclChar = 0, ncclUint8 = 1, ncclInt32 = 2, ncclInt = 2, ncclUint32 = 3, ncclInt64 = 4, ncclUint64 = 5, ncclFloat16 = 6, ncclHalf = 6, ncclFloat32 = 7, ncclFloat = 7, ncclFloat64 = 8, ncclDouble = 8, ncclBfloat16 = 9, ncclFloat8e4m3 = 10, ncclFloat8e5m2 = 11, ncclNumTypes = 12 } ncclDataType_t; typedef enum { NCCL_LOG_NONE = 0, NCCL_LOG_VERSION = 1, NCCL_LOG_WARN = 2, NCCL_LOG_INFO = 3, NCCL_LOG_ABORT = 4, NCCL_LOG_TRACE = 5 } ncclDebugLogLevel; typedef enum { ncclSuccess = 0, ncclUnhandledCudaError = 1, ncclSystemError = 2, ncclInternalError = 3, ncclInvalidArgument = 4, ncclInvalidUsage = 5, ncclRemoteError = 6, ncclInProgress = 7, ncclNumResults = 8 } ncclResult_t; typedef enum { NCCL_INIT = 0x1, NCCL_COLL = 0x2, NCCL_P2P = 0x4, NCCL_SHM = 0x8, NCCL_NET = 0x10, NCCL_GRAPH = 0x20, NCCL_TUNING = 0x40, NCCL_ENV = 0x80, NCCL_ALLOC = 0x100, NCCL_CALL = 0x200, NCCL_PROXY = 0x400, NCCL_NVLS = 0x800, NCCL_BOOTSTRAP = 0x1000, NCCL_REG = 0x2000, NCCL_PROFILE = 0x4000, NCCL_RAS = 0x8000, NCCL_ALL = ~0 } ncclDebugLogSubSys; typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags, const char *file, int line, const char *fmt, ...); #endif nccl-2.29.7-1/plugins/profiler/inspector/nccl/profiler.h000066400000000000000000000065741515037102200231230ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_H_ #define PROFILER_H_ #include #include #include "common.h" enum { ncclProfileGroup = (1 << 0), // group event type ncclProfileColl = (1 << 1), // host collective call event type ncclProfileP2p = (1 << 2), // host point-to-point call event type ncclProfileProxyOp = (1 << 3), // proxy operation event type ncclProfileProxyStep = (1 << 4), // proxy step event type ncclProfileProxyCtrl = (1 << 5), // proxy control event type ncclProfileKernelCh = (1 << 6), // kernel channel event type ncclProfileNetPlugin = (1 << 7), // network plugin-defined, events ncclProfileGroupApi = (1 << 8), // Group API events ncclProfileCollApi = (1 << 9), // Collective API events ncclProfileP2pApi = (1 << 10), // Point-to-Point API events ncclProfileKernelLaunch = (1 << 11), // Kernel launch events }; typedef enum { ncclProfilerProxyOpSendPosted = 0, // deprecated in v4 ncclProfilerProxyOpSendRemFifoWait = 1, // deprecated in v4 ncclProfilerProxyOpSendTransmitted = 2, // deprecated in v4 ncclProfilerProxyOpSendDone = 3, // deprecated in v4 ncclProfilerProxyOpRecvPosted = 4, // deprecated in v4 ncclProfilerProxyOpRecvReceived = 5, // deprecated in v4 ncclProfilerProxyOpRecvTransmitted = 6, // deprecated in v4 ncclProfilerProxyOpRecvDone = 7, // deprecated in v4 ncclProfilerProxyOpInProgress_v4 = 19, /* Legacy proxy profiler states */ ncclProfilerProxyStepSendGPUWait = 8, ncclProfilerProxyStepSendPeerWait_v4 = 20, ncclProfilerProxyStepSendWait = 9, ncclProfilerProxyStepRecvWait = 10, ncclProfilerProxyStepRecvFlushWait = 11, ncclProfilerProxyStepRecvGPUWait = 12, /* Legacy proxy control states */ ncclProfilerProxyCtrlIdle = 13, ncclProfilerProxyCtrlActive = 14, ncclProfilerProxyCtrlSleep = 15, ncclProfilerProxyCtrlWakeup = 16, ncclProfilerProxyCtrlAppend = 17, ncclProfilerProxyCtrlAppendEnd = 18, /* Network defined events states */ ncclProfilerNetPluginUpdate = 21, /* Kernel event states */ ncclProfilerKernelChStop = 22, /* Group API States */ ncclProfilerEndGroupApiStart = 23, ncclProfilerBeginGroupApiEnd = 24 } ncclProfilerEventState_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v1_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v2_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v3_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v4_t; typedef ncclProfilerEventState_t ncclProfilerEventState_v5_t; #include "profiler_v5.h" #include "profiler_v4.h" #include "profiler_v3.h" #include "profiler_v2.h" #include "profiler_v1.h" #include "profiler_net.h" typedef ncclProfiler_v5_t ncclProfiler_t; typedef ncclProfilerEventDescr_v5_t ncclProfilerEventDescr_t; typedef ncclProfilerEventStateArgs_v5_t ncclProfilerEventStateArgs_t; #endif // end include guard nccl-2.29.7-1/plugins/profiler/inspector/nccl/profiler_net.h000066400000000000000000000013741515037102200237620ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_NET_H_ #define PROFILER_NET_H_ #define NCCL_PROFILER_NET_VER_BITS (16) #define NCCL_PROFILER_NET_VER_MASK (~0U >> NCCL_PROFILER_NET_VER_BITS) #define NCCL_PROFILER_NET_TYPE_MASK (~0U << NCCL_PROFILER_NET_VER_BITS) typedef enum { NCCL_PROFILER_NET_TYPE_IB = (1U << NCCL_PROFILER_NET_VER_BITS), NCCL_PROFILER_NET_TYPE_SOCK = (2U << NCCL_PROFILER_NET_VER_BITS), } ncclProfilerNetType; #endif nccl-2.29.7-1/plugins/profiler/inspector/nccl/profiler_v1.h000066400000000000000000000066331515037102200235250ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V1_H_ #define PROFILER_V1_H_ #include #include #include typedef struct { uint8_t type; // event type descriptor: ncclProfileColl, ... void* parentObj; // pointer to the profiler parent object (for coll is the group) int rank; // originating rank union { struct { const char* name; uint64_t commHash; uint64_t seqNumber; uint8_t func; void const* sendBuff; void* recvBuff; size_t count; int root; uint8_t datatype; uint32_t op; size_t trafficBytes; uint8_t nMaxChannels; uint8_t nWarps; uint8_t algo; uint8_t proto; int isCollnet; int isNvls; } coll; struct { const char* name; uint64_t commHash; uint8_t func; void* buff; uint8_t datatype; size_t count; int peer; } p2p; struct { pid_t pid; // pid of the originating process uint8_t channelId; // channel id for this proxy operation int peer; // remote rank for send/recv int nSteps; // number of steps for this proxy operation int chunkSize; // amount of data transferred by this proxy operation int isSend; } proxyOp; struct { int step; } proxyStep; }; } ncclProfilerEventDescr_v1_t; typedef union { struct { size_t transSize; int steps; } proxyOp; struct { int appendedProxyOps; } proxyCtrl; } ncclProfilerEventStateArgs_v1_t; typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, int* eActivationMask); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v1_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v1_t eState, ncclProfilerEventStateArgs_v1_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v1_t; #endif nccl-2.29.7-1/plugins/profiler/inspector/nccl/profiler_v2.h000066400000000000000000000065701515037102200235260ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V2_H_ #define PROFILER_V2_H_ #include #include #include typedef struct { uint8_t type; // event type descriptor: ncclProfileColl, ... void* parentObj; // pointer to the profiler parent object (for coll is the group) int rank; // originating rank union { struct { const char* name; uint64_t commHash; uint64_t seqNumber; const char* func; void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; size_t trafficBytes; uint8_t nMaxChannels; uint8_t nWarps; const char* algo; const char* proto; } coll; struct { const char* name; uint64_t commHash; const char* func; void* buff; const char* datatype; size_t count; int peer; } p2p; struct { pid_t pid; // pid of the originating process uint8_t channelId; // channel id for this proxy operation int peer; // remote rank for send/recv int nSteps; // number of steps for this proxy operation int chunkSize; // amount of data transferred by this proxy operation int isSend; } proxyOp; struct { int step; } proxyStep; }; } ncclProfilerEventDescr_v2_t; typedef union { struct { size_t transSize; int steps; } proxyOp; struct { int appendedProxyOps; } proxyCtrl; } ncclProfilerEventStateArgs_v2_t; typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, int* eActivationMask); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v2_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v2_t eState, ncclProfilerEventStateArgs_v2_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v2_t; #endif nccl-2.29.7-1/plugins/profiler/inspector/nccl/profiler_v3.h000066400000000000000000000067271515037102200235330ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V3_H_ #define PROFILER_V3_H_ #include #include #include typedef struct { uint8_t type; // event type descriptor: ncclProfileColl, ... void* parentObj; // pointer to the profiler parent object (for coll is the group) int rank; // originating rank union { struct { const char* name; uint64_t commHash; uint64_t seqNumber; const char* func; void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; uint8_t nMaxChannels; uint8_t nWarps; const char* algo; const char* proto; } coll; struct { const char* name; uint64_t commHash; const char* func; void* buff; const char* datatype; size_t count; int peer; } p2p; struct { pid_t pid; // pid of the originating process uint8_t channelId; // channel id for this proxy operation int peer; // remote rank for send/recv int nSteps; // number of steps for this proxy operation int chunkSize; // amount of data transferred by this proxy operation int isSend; } proxyOp; struct { int step; } proxyStep; struct { uint8_t channelId; } kernelCh; struct { int64_t id; void* data; } netPlugin; }; } ncclProfilerEventDescr_v3_t; typedef union { struct { size_t transSize; int steps; } proxyOp; struct { int appendedProxyOps; } proxyCtrl; } ncclProfilerEventStateArgs_v3_t; typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, int* eActivationMask); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v3_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v3_t eState, ncclProfilerEventStateArgs_v3_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v3_t; #endif nccl-2.29.7-1/plugins/profiler/inspector/nccl/profiler_v4.h000066400000000000000000000076721515037102200235340ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V4_H_ #define PROFILER_V4_H_ #include #include #include typedef struct { uint8_t type; // event type descriptor: ncclProfileColl, ... void* parentObj; // pointer to the profiler parent object (for coll is the group) int rank; // originating rank union { struct { uint64_t seqNumber; const char* func; void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; uint8_t nChannels; uint8_t nWarps; const char* algo; const char* proto; } coll; struct { const char* func; void* buff; const char* datatype; size_t count; int peer; uint8_t nChannels; } p2p; struct { pid_t pid; // pid of the originating process uint8_t channelId; // channel id for this proxy operation int peer; // remote rank for send/recv int nSteps; // number of steps for this proxy operation int chunkSize; // amount of data transferred by this proxy operation int isSend; } proxyOp; struct { int step; } proxyStep; struct { uint8_t channelId; uint64_t pTimer; // start timestamp from GPU globaltimer } kernelCh; struct { int64_t id; void* data; } netPlugin; }; } ncclProfilerEventDescr_v4_t; typedef union { struct { size_t transSize; } proxyStep; struct { int appendedProxyOps; } proxyCtrl; struct { void* data; } netPlugin; struct { uint64_t pTimer; } kernelCh; } ncclProfilerEventStateArgs_v4_t; typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // - commName : user assigned communicator name // - commHash : communicator id // - nNodes : number of nodes in communicator // - nranks : number of ranks in communicator // - rank : rank identifier in communicator // - logfn : logger function // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, int* eActivationMask, const char* commName, uint64_t commHash, int nNodes, int nranks, int rank, ncclDebugLogger_t logfn); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v4_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v4_t eState, ncclProfilerEventStateArgs_v4_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v4_t; #endif nccl-2.29.7-1/plugins/profiler/inspector/nccl/profiler_v5.h000066400000000000000000000106571515037102200235320ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef PROFILER_V5_H_ #define PROFILER_V5_H_ typedef struct { uint64_t type; // event type descriptor: ncclProfileColl, ... void* parentObj; // pointer to the profiler parent object (for coll is the group) int rank; // originating rank union { struct { bool graphCaptured; int groupDepth; } groupApi; struct { const char* func; size_t count; const char* datatype; int root; void* stream; bool graphCaptured; } collApi; struct { const char* func; size_t count; const char* datatype; void* stream; bool graphCaptured; } p2pApi; struct { void* stream; } kernelLaunch; struct { uint64_t seqNumber; const char* func; void const* sendBuff; void* recvBuff; size_t count; int root; const char* datatype; uint8_t nChannels; uint8_t nWarps; const char* algo; const char* proto; void* parentGroup; // for backward compatibility with v4 } coll; struct { const char* func; void* buff; const char* datatype; size_t count; int peer; uint8_t nChannels; void* parentGroup; // for backward compatibility with v4 } p2p; struct { pid_t pid; // pid of the originating process uint8_t channelId; // channel id for this proxy operation int peer; // remote rank for send/recv int nSteps; // number of steps for this proxy operation int chunkSize; // amount of data transferred by this proxy operation int isSend; } proxyOp; struct { int step; } proxyStep; struct { uint8_t channelId; uint64_t pTimer; // start timestamp from GPU globaltimer } kernelCh; struct { int64_t id; void* data; } netPlugin; }; } ncclProfilerEventDescr_v5_t; typedef union { struct { size_t transSize; } proxyStep; struct { int appendedProxyOps; } proxyCtrl; struct { void* data; } netPlugin; struct { uint64_t pTimer; } kernelCh; } ncclProfilerEventStateArgs_v5_t; typedef struct { const char* name; // init - initialize the profiler plugin // Input // - context : opaque profiler context object for separating profiler behavior across comms // - commId : communicator id // - commName : user assigned communicator name // - nNodes : number of nodes in communicator // - nranks : number of ranks in communicator // - rank : rank identifier in communicator // - logfn : logger function // Output // - eActivationMask: bitmask of active events set by the plugin ncclResult_t (*init)(void** context, uint64_t commId, int* eActivationMask, const char* commName, int nNodes, int nranks, int rank, ncclDebugLogger_t logfn); // startEvent - initialize and start a new event for the supplied event descriptor inside the eventset // Input // - context: opaque profiler context object // - eDescr : pointer to ncclProfilerEventDescr_t object // Output // - eHandle: return event handle for supplied event descriptor object ncclResult_t (*startEvent)(void* context, void** eHandle, ncclProfilerEventDescr_v5_t* eDescr); // stopEvent - stop/finalize an event inside and event set // Input // - eHandle: handle to event object ncclResult_t (*stopEvent)(void* eHandle); // recordEventState - record event state transitions and event attribute updates // Input // - eHandle : handle to event object created through startEvent // - eStateArgs: optional argument used to capture event attribute updates associated with the state transition // - eState : event state transition ncclResult_t (*recordEventState)(void* eHandle, ncclProfilerEventState_v5_t eState, ncclProfilerEventStateArgs_v5_t* eStateArgs); // finalize - finalize the profiler plugin // Input // - context: opaque profiler context object ncclResult_t (*finalize)(void* context); } ncclProfiler_v5_t; #endif nccl-2.29.7-1/plugins/profiler/inspector/nccl/types.h000066400000000000000000000015651515037102200224400ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_TYPES_H_ #define NCCL_TYPES_H_ /* Data types */ typedef enum { ncclInt8 = 0, ncclChar = 0, ncclUint8 = 1, ncclInt32 = 2, ncclInt = 2, ncclUint32 = 3, ncclInt64 = 4, ncclUint64 = 5, ncclFloat16 = 6, ncclHalf = 6, ncclFloat32 = 7, ncclFloat = 7, ncclFloat64 = 8, ncclDouble = 8, ncclBfloat16 = 9, } ncclDataType_t; #endif nccl-2.29.7-1/plugins/profiler/inspector/version.h000066400000000000000000000010151515037102200220300ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef VERSION_H #define VERSION_H #ifdef __cplusplus extern "C" { #endif const char* get_git_version_info(); #ifdef __cplusplus } /* extern "C" */ #endif #endif // VERSION_H nccl-2.29.7-1/plugins/tuner/000077500000000000000000000000001515037102200155025ustar00rootroot00000000000000nccl-2.29.7-1/plugins/tuner/README.md000066400000000000000000000142421515037102200167640ustar00rootroot00000000000000# NCCL Tuner Plugin Development This directory contains resources and examples for developing NCCL tuner plugins. Tuner plugins allow you to customize NCCL's algorithm and protocol selection behavior to optimize performance for specific workloads and hardware configurations. ## Overview NCCL tuner plugins provide a way to influence NCCL's automatic algorithm and protocol selection by modifying the cost tables that NCCL uses to make decisions. This allows you to: - Override default algorithm/protocol combinations for specific collective operations - Customize tuning based on message size, topology, and other parameters - Implement sophisticated tuning strategies without recompiling NCCL - Optimize performance for specific hardware configurations or workloads ## Tuner Plugin Interface NCCL tuner plugins must implement the `ncclTuner_t` interface defined in `nccl_tuner.h` within `nccl/src/include/plugin`. These definitions have been forked to `tuner.h` in each example plugin, and it is expected that any plugin implementor forks the internal NCCL definitions as well. The current interface includes: ```c // Initialize the tuner plugin ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context); // Get and modify collective operation cost information ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff, int* nChannels); // Clean up plugin resources ncclResult_t (*destroy)(void* context); ``` ## Development Guidelines ### 1. Plugin Structure A typical tuner plugin should: - Include the necessary forked NCCL headers (`tuner.h`) - Implement all required interface functions - Export the plugin structure with appropriate version - Handle all input parameters gracefully ### 2. Cost Table Modification The `getCollInfo` function receives a cost table that maps algorithm/protocol combinations to performance costs. Lower costs indicate preferred combinations. You can: - Set costs to `0.0` to make combinations highly preferred - Set costs to `NCCL_ALGO_PROTO_IGNORE` to disable combinations - Use relative costs to create preferences between options ### 3. Channel Management The `nChannels` parameter allows you to: - Set a specific number of channels to use - Return the original value to preserve NCCL's default behavior - Implement dynamic channel selection based on message size or topology ### 4. Error Handling Always return appropriate `ncclResult_t` values: - `ncclSuccess` for successful or ignored operations - `ncclInternalError` for plugin-specific errors. Returning an error is only advisable on plugin initialization and destruction, as the penalty users can pay for the overhead of a failed plugin call can be immense. - Other NCCL error codes as appropriate ## Getting Started ### Option 1: Start with the Example Plugin If you're new to tuner plugin development, start with the `example/` directory: ```bash cd example/ make ``` This provides a CSV-based configuration system that you can customize or use as a template. ## Building and Testing ### Build Requirements - GCC or compatible C compiler - NCCL headers (included in `nccl/` subdirectories) - Make ## Option 2: Use the Basic Plugin For more customized tuning needs, you might want to start with a clean baseline. In that case, base off the basic plugin in the `basic/` directory: ```bash cd basic/ make ``` ### Build Process Each plugin directory contains a Makefile: ```bash cd basic/ # or example/ make ``` This generates a shared library (`.so` file) that can be loaded by NCCL. ### Loading the Plugin Set the `LD_LIBRARY_PATH` to include your plugin directory: ```bash export LD_LIBRARY_PATH=/path/to/your/plugin:$LD_LIBRARY_PATH ``` Set `NCCL_TUNER_PLUGIN` to either the plugin name, or the absolute path to the plugin file. Any of the below can work: ```bash export NCCL_TUNER_PLUGIN=example export NCCL_TUNER_PLUGIN=libnccl-tuner-example.so export NCCL_TUNER_PLUGIN=/path/to/your/plugin/libnccl-tuner-example.so ``` NCCL will automatically discover and load the plugin based on the exported symbol names. ## Advanced Topics ### Plugin Versioning NCCL supports multiple plugin interface versions. Make sure your plugin exports the correct version: ```c const ncclTuner_v4_t ncclTunerPlugin_v4 = { .name = "YourPluginName", .init = yourInitFunction, .getCollInfo = yourGetCollInfoFunction, .destroy = yourDestroyFunction }; ``` ### Multi-GPU and Multi-Node Considerations Your plugin receives topology information (`nRanks`, `nNodes`) during initialization. Use this to: - Implement topology-aware tuning strategies - Handle single-node vs. multi-node optimizations differently - Scale channel counts based on available hardware ### Performance Optimization - Keep plugin logic lightweight to avoid impacting NCCL performance - Cache expensive computations when possible - Use the logging system for debugging but avoid excessive output in production ## Debugging and Logging Use NCCL's debug logging system: ```bash export NCCL_DEBUG=INFO # General information export NCCL_DEBUG_SUBSYS=TUNING ``` Within your plugin, use the provided `ncclDebugLogger_t` function for consistent logging. ## Best Practices 1. **Test thoroughly**: Verify your plugin works with various message sizes and topologies 2. **Handle edge cases**: Ensure your plugin behaves correctly with unusual input parameters 3. **Document your approach**: Clearly document your tuning strategy and configuration options 4. **Version your plugin**: Use meaningful version numbers and maintain backward compatibility 5. **Performance validation**: Measure the impact of your tuning decisions on real workloads ## Contributing When developing new tuner plugins: - Follow the existing code style and structure - Include comprehensive documentation - Add example configurations and test cases - Consider contributing useful plugins back to the community ## Resources - [NCCL Documentation](https://docs.nvidia.com/deeplearning/nccl/) - Example plugin implementations in this directory For questions and support, refer to the NCCL community resources and documentation. nccl-2.29.7-1/plugins/tuner/basic/000077500000000000000000000000001515037102200165635ustar00rootroot00000000000000nccl-2.29.7-1/plugins/tuner/basic/Makefile000066400000000000000000000011751515037102200202270ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # .DEFAULT_GOAL: build include ../../../makefiles/common.mk SRCDIR ?= $(abspath ../..) BUILDDIR ?= . NCCLDIR := $(BUILDDIR) SRC_FILES := $(wildcard *.c) DST_DIR := $(BUILDDIR)/test/unit/plugins build: ${BUILDDIR}/libnccl-tuner-basic.so ${BUILDDIR}/libnccl-tuner-basic.so: ${SRC_FILES} @printf "Compiling %-35s > %s\n" $< $@ @mkdir -p ${BUILDDIR} $(CC) -Inccl -fPIC -shared -o $@ $^ clean: rm -f ${BUILDDIR}/libnccl-tuner-basic.so nccl-2.29.7-1/plugins/tuner/basic/README.md000066400000000000000000000137121515037102200200460ustar00rootroot00000000000000# Basic NCCL Tuner Plugin This directory contains a minimal placeholder implementation of an NCCL tuner plugin. It serves as a starting point for developing custom tuner plugins by providing the essential function stubs and interface structure required by NCCL. ## Purpose This basic plugin is designed to: - Provide a minimal working example of the NCCL tuner plugin interface - Serve as a template for developing custom tuner plugins - Demonstrate the required function signatures and structure - Implement placeholder functionality that can be extended ## Implementation Details The plugin implements the following functions: ### `pluginInit` ```c ncclResult_t pluginInit(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context) ``` - **Purpose**: Initialize the plugin with communicator information - **Current Implementation**: Simple placeholder that returns success - **Parameters**: - `nRanks`: Total number of ranks in the communicator - `nNodes`: Total number of nodes in the communicator - `logFunction`: NCCL debug logging function - `context`: Plugin context pointer (output) ### `pluginGetCollInfo` ```c ncclResult_t pluginGetCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff, int* nChannels) ``` - **Purpose**: Modify cost tables for collective operations - **Current Implementation**: - Sets RING+SIMPLE algorithm to cost 0.0 (highest preference) - Sets channel count to 1 - **Parameters**: - `context`: Plugin context from init - `collType`: Type of collective operation - `nBytes`: Message size in bytes - `numPipeOps`: Number of pipeline operations - `collCostTable`: Cost table to modify - `numAlgo`: Number of algorithms - `numProto`: Number of protocols - `regBuff`: Whether buffer can be registered - `nChannels`: Number of channels to use (output) ### `pluginDestroy` ```c ncclResult_t pluginDestroy(void* context) ``` - **Purpose**: Clean up plugin resources - **Current Implementation**: Simple placeholder that returns success ## Cost Table Structure The plugin demonstrates how to modify NCCL's cost tables: ```c float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable; ``` The cost table is a 2D array where: - First dimension: Algorithm index (e.g., `NCCL_ALGO_RING`) - Second dimension: Protocol index (e.g., `NCCL_PROTO_SIMPLE`) - Values: Cost for that algorithm/protocol combination ### Cost Values - **0.0**: Highest preference (lowest cost) - **Positive values**: Relative costs (lower is better) - **`NCCL_ALGO_PROTO_IGNORE`**: Disable this combination ## Building ```bash make ``` This creates `libnccl-tuner-basic.so` which can be loaded by NCCL. ## Usage ### Loading the Plugin ```bash export LD_LIBRARY_PATH=/path/to/basic:$LD_LIBRARY_PATH mpirun -np 4 your_nccl_application ``` ```bash export NCCL_TUNER_PLUGIN=basic export NCCL_TUNER_PLUGIN=libnccl-tuner-basic.so export NCCL_TUNER_PLUGIN=/path/to/your/plugin/libnccl-tuner-basic.so ``` ### Verifying Plugin Loading Enable NCCL debug output to see if the plugin is loaded: ```bash export NCCL_DEBUG=INFO ``` You should see messages indicating the tuner plugin is being used. ## Extending the Plugin This basic plugin provides a foundation that you can extend: ### 1. Add Configuration Logic Modify `pluginGetCollInfo` to implement your tuning strategy: ```c __hidden ncclResult_t pluginGetCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff, int* nChannels) { // Your custom tuning logic here if (nBytes < 1024) { // Small message optimization table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE] = 0.0; } else { // Large message optimization table[NCCL_ALGO_RING][NCCL_PROTO_LL128] = 0.0; } // Dynamic channel selection *nChannels = (nBytes > 1024*1024) ? 4 : 1; return ncclSuccess; } ``` ### 2. Add Context Management Use the context pointer to store plugin state: ```c struct pluginContext { int initialized; size_t nRanks; size_t nNodes; // Add your plugin-specific data here }; ``` ### 3. Add File-Based Configuration Read configuration from files, environment variables, or other sources. ### 4. Add Topology Awareness Use the `nRanks` and `nNodes` parameters to implement topology-specific tuning. ## File Structure ``` basic/ ├── README.md # This file ├── plugin.c # Plugin implementation ├── Makefile # Build configuration └── nccl/ # NCCL header files └── tuner.h # Tuner plugin interface definitions ``` ## Next Steps 1. **Understand the Interface**: Study the function signatures and parameters 2. **Implement Your Logic**: Add your tuning strategy to `pluginGetCollInfo` 3. **Test Thoroughly**: Verify your plugin works with different message sizes and topologies 4. **Add Error Handling**: Implement proper error checking and resource management 5. **Document Your Changes**: Update this README with your specific implementation details ## Comparison with Example Plugin - **Basic Plugin**: Minimal implementation, good for learning and simple use cases - **Example Plugin**: Full-featured CSV-based configuration system, good for production use Choose the basic plugin if you want to: - Learn the tuner plugin interface - Implement simple, hardcoded tuning strategies - Build a custom plugin from scratch Choose the example plugin if you want: - File-based configuration - Complex tuning strategies - Production-ready features ## Resources - [Parent Directory README](../README.md) - General tuner plugin development guide - [Example Plugin](../example/README.md) - Fully featured implementation This basic plugin provides the foundation you need to start developing custom NCCL tuner plugins. Extend it with your specific tuning logic and requirements. nccl-2.29.7-1/plugins/tuner/basic/nccl/000077500000000000000000000000001515037102200175025ustar00rootroot00000000000000nccl-2.29.7-1/plugins/tuner/basic/nccl/common.h000066400000000000000000000016271515037102200211510ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef COMMON_H_ #define COMMON_H_ typedef enum {NCCL_LOG_NONE=0, NCCL_LOG_VERSION=1, NCCL_LOG_WARN=2, NCCL_LOG_INFO=3, NCCL_LOG_ABORT=4, NCCL_LOG_TRACE=5} ncclDebugLogLevel; typedef enum {NCCL_INIT=1, NCCL_COLL=2, NCCL_P2P=4, NCCL_SHM=8, NCCL_NET=16, NCCL_GRAPH=32, NCCL_TUNING=64, NCCL_ENV=128, NCCL_ALLOC=256, NCCL_CALL=512, NCCL_PROXY=1024, NCCL_NVLS=2048, NCCL_BOOTSTRAP=4096, NCCL_REG=8192, NCCL_ALL=~0} ncclDebugLogSubSys; typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags, const char *file, int line, const char *fmt, ...); #endif nccl-2.29.7-1/plugins/tuner/basic/nccl/err.h000066400000000000000000000014171515037102200204460ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_ERR_H_ #define NCCL_ERR_H_ /* Error type for plugins */ typedef enum { ncclSuccess = 0, ncclUnhandledCudaError = 1, ncclSystemError = 2, ncclInternalError = 3, ncclInvalidArgument = 4, ncclInvalidUsage = 5, ncclRemoteError = 6 } ncclResult_t; #endif nccl-2.29.7-1/plugins/tuner/basic/nccl/tuner.h000066400000000000000000000065211515037102200210140ustar00rootroot00000000000000/************************************************************************* * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. * Copyright (c) 2023, Meta Platforms, Inc. and affiliates. * * See LICENSE.txt for license information ************************************************************************/ #ifndef NCCL_TUNER_H_ #define NCCL_TUNER_H_ #include #include #include "common.h" #include "err.h" #define NCCL_NUM_FUNCTIONS 5 // Send/Recv not included for now typedef enum { ncclFuncBroadcast = 0, ncclFuncReduce = 1, ncclFuncAllGather = 2, ncclFuncReduceScatter = 3, ncclFuncAllReduce = 4, ncclFuncSendRecv = 5, ncclFuncSend = 6, ncclFuncRecv = 7, ncclNumFuncs = 8 } ncclFunc_t; #define NCCL_NUM_ALGORITHMS 7 // Tree/Ring/CollNet* #define NCCL_ALGO_UNDEF -1 #define NCCL_ALGO_TREE 0 #define NCCL_ALGO_RING 1 #define NCCL_ALGO_COLLNET_DIRECT 2 #define NCCL_ALGO_COLLNET_CHAIN 3 #define NCCL_ALGO_NVLS 4 #define NCCL_ALGO_NVLS_TREE 5 #define NCCL_ALGO_PAT 6 #define NCCL_NUM_PROTOCOLS 3 // Simple/LL/LL128 #define NCCL_PROTO_UNDEF -1 #define NCCL_PROTO_LL 0 #define NCCL_PROTO_LL128 1 #define NCCL_PROTO_SIMPLE 2 #define NCCL_ALGO_PROTO_IGNORE -1.0 // API to be implemented by external tuner typedef struct { // Name of the tuner const char* name; // Initializes tuner states. // Inputs: // - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner. // - nNodes: number of nodes in current communicator. // - logFunction: a logFunction can be useful to integrate logging together with NCCL core. // Outputs: // - context: tuner context object ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context); // Gets info (algo, protocol, number of ctas and threads) for a given collective. // Inputs: // - context: tuner context object // - collType: collective type , e.g., allreduce, allgather… // - nBytes: collective size in bytes // - numPipeOps: number of operations in the group // - numAlgo: number of algorithms in collCostTable // - numProto: number of protocols in collCostTable // - regBuff: can register user buffer // // Outputs: // - nChannels: number of channels (hence SMs) to be used. // // InOut: // - collCostTable: collective cost table, generated by NCCL core, containing algo|proto|time entries for collType. // NCCL core sets ignored algo/proto cost table entries to -1.0 (NCCL_ALGO_PROTO_IGNORE). // // If getCollInfo() does not return ncclSuccess, NCCL will fall back to the // default tuning for the given collective. // Also, the plugin is allowed to not set any output, or set only the // algorithm and protocol, but not only the algorithm or only the protocol. // Unset fields will be set automatically by NCCL. ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff, int* nChannels); // Terminates the plugin and cleans up any resources that the plugin allocated. // context: tuner context object ncclResult_t (*destroy)(void* context); } ncclTuner_v4_t; typedef ncclTuner_v4_t ncclTuner_t; #define NCCL_TUNER_PLUGIN_SYMBOL "ncclTunerPlugin_v4" #endif nccl-2.29.7-1/plugins/tuner/basic/plugin.c000066400000000000000000000026451515037102200202340ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "tuner.h" #define __hidden __attribute__ ((visibility("hidden"))) __hidden ncclResult_t pluginInit(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context) { return ncclSuccess; } __hidden ncclResult_t pluginGetCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff, int* nChannels) { // Update NCCL core generated cost table. Updated table will be evaluated by NCCL to pick the best algo/proto combo float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable; if (table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] != NCCL_ALGO_PROTO_IGNORE) { table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] = 0.0; } *nChannels = 1; return ncclSuccess; } __hidden ncclResult_t pluginDestroy(void* context) { return ncclSuccess; } #define PLUGIN_NAME "Basic" const ncclTuner_v4_t ncclTunerPlugin_v4 = { .name = PLUGIN_NAME, .init = pluginInit, .getCollInfo = pluginGetCollInfo, .destroy = pluginDestroy }; nccl-2.29.7-1/plugins/tuner/example/000077500000000000000000000000001515037102200171355ustar00rootroot00000000000000nccl-2.29.7-1/plugins/tuner/example/.gitignore000066400000000000000000000010251515037102200211230ustar00rootroot00000000000000# Compiled shared objects and binaries *.so *.o *.a *.out *.exe *.dll *.dylib *.bin *.elf # Python cache __pycache__/ *.pyc *.pyo # Build and test artifacts /build/ *.log *.tmp *.swp # Ignore all CSV files except scripts/sample_performance_data.csv *.csv !scripts/sample_performance_data.csv # Ignore all .conf files except nccl_tuner.conf *.conf !nccl_tuner.conf my_configs # Ignore test binary test/test_plugin # Editor/OS files .DS_Store Thumbs.db # Backup files *~ *.bak # Ignore by convention *.old *.orig # Git .git/ nccl-2.29.7-1/plugins/tuner/example/CMakeLists.txt000066400000000000000000000014321515037102200216750ustar00rootroot00000000000000# Find all C source files in current directory set(SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/plugin.c ) # Create shared library add_library(nccl-tuner-example SHARED ${SRC_FILES}) # Set include directories target_include_directories(nccl-tuner-example PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/nccl ) # Set output name to match Makefile set_target_properties(nccl-tuner-example PROPERTIES OUTPUT_NAME "nccl-tuner-example" PREFIX "lib" POSITION_INDEPENDENT_CODE ON LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test/unit/plugins ) # Add custom target for clean (equivalent to Makefile clean target) add_custom_target(clean-tuner-lib COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/libnccl-tuner-example.so COMMENT "Cleaning libnccl-tuner-example.so" ) nccl-2.29.7-1/plugins/tuner/example/Makefile000066400000000000000000000030151515037102200205740ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # .DEFAULT_GOAL: build PLUGIN_SO:=libnccl-tuner-example.so include ../../../makefiles/common.mk SRCDIR ?= $(abspath ../../..) BUILDDIR ?= . NCCLDIR := $(BUILDDIR) SRC_FILES := $(wildcard *.c) DST_DIR := $(BUILDDIR)/test/unit/plugins default: ${BUILDDIR}/$(PLUGIN_SO) build: ${BUILDDIR}/$(PLUGIN_SO) ${BUILDDIR}/$(PLUGIN_SO): plugin.c @printf "Compiling %-35s > %s\n" $< $@ @mkdir -p ${BUILDDIR} $(CC) -Inccl $(INC) -fPIC -shared -o $@ -Wl,-soname,$(PLUGIN_SO) $^ # Test targets - delegate to test directory test: $(MAKE) -C test test TEST_CASE=$(TEST_CASE) test-verbose: $(MAKE) -C test test-verbose TEST_CASE=$(TEST_CASE) # Build tests test-build: $(MAKE) -C test all # Optimize configurations from performance data optimize-config: @if [ -z "$(CSV_FILE)" ]; then \ echo "Usage: make optimize-config CSV_FILE=path/to/data.csv [OUTPUT=config.conf] [METRIC=latency_us]"; \ echo "Example: make optimize-config CSV_FILE=scripts/sample_performance_data.csv"; \ exit 1; \ fi python3 scripts/optimize_config.py $(CSV_FILE) \ $(if $(OUTPUT),-o $(OUTPUT)) \ $(if $(METRIC),-m $(METRIC)) \ $(if $(SIZE_RANGES),--size-ranges $(SIZE_RANGES)) \ $(if $(DRY_RUN),--dry-run) \ $(if $(NO_HEADER),--no-header) clean: rm -f ${BUILDDIR}/$(PLUGIN_SO) $(MAKE) -C test clean .PHONY: test test-verbose test-build optimize-config clean nccl-2.29.7-1/plugins/tuner/example/README.md000066400000000000000000000153601515037102200204210ustar00rootroot00000000000000# NCCL Example Tuner Plugin This example plugin shows a practical example of a CSV file-based tuning approach, allowing selective overrides for tuning parameters based on all tuning inputs without recompiling. ## Features - **File-based Configuration**: Read tuning parameters from a CSV configuration file - **Size-based Tuning**: Specify different configurations based on message size ranges - **Dimension-aware Tuning**: Match configurations based on number of nodes and ranks - **Optional Channels Configuration**: Set specific channel counts or use -1 to keep NCCL's default - **Environment Variable Support**: Specify config file location via `NCCL_TUNER_CONFIG_FILE` - **Fallback Behavior**: Gracefully handles missing config files and invalid entries ## Building ```bash make ``` This will create `libnccl-tuner-example.so` that can be loaded by NCCL. ## Configuration File Format The configuration file uses CSV (Comma-Separated Values) format with one configuration per line: ``` collective_type,min_bytes,max_bytes,algorithm,protocol,channels,nNodes,nRanks,numPipeOps,regBuff ``` ### Parameters - **collective_type**: The collective operation type - `broadcast`, `reduce`, `allgather`, `reducescatter`, `allreduce` - **min_bytes/max_bytes**: The message size range (in bytes) for which this config applies - Use `0` for minimum and `4294967295` for maximum (covers all sizes) - **algorithm**: The NCCL algorithm to use - `tree`, `ring`, `collnet_direct`, `collnet_chain`, `nvls`, `nvls_tree`, `pat` - **protocol**: The NCCL protocol to use - `ll`, `ll128`, `simple` - **channels**: Number of channels (SMs) to use - Use a positive integer to specify exact channel count - Use `-1` to keep NCCL's default channel selection - **nNodes**: Number of nodes to match - Use a positive integer to match specific node count - Use `-1` to match any number of nodes - **nRanks**: Number of ranks to match - Use a positive integer to match specific rank count - Use `-1` to match any number of ranks - **numPipeOps**: Number of pipeline operations to match (optional) - Use a positive integer to match specific pipeline operation count - Use `-1` to match any number of pipeline operations - If omitted, configuration will match any numPipeOps value - **regBuff**: Whether user buffer can be registered (optional) - Use `0` to match only non-registered buffers - Use `1` to match only registered buffers - Use `-1` to match either registered or non-registered buffers - If omitted, configuration will match any regBuff value ### Example Configuration ```csv # Single-node, small allreduce: use tree algorithm, registered buffers only allreduce,0,65536,tree,simple,2,1,-1,-1,1 # 4-node, 32-rank setup: medium allreduce, single pipeline op, non-registered buffers allreduce,65537,1048576,ring,simple,4,4,32,1,0 # Any topology: large allreduce with LL128, multiple pipeline ops, any buffer type allreduce,1048577,4294967295,ring,ll128,-1,-1,-1,4,-1 # Single-node broadcast: prefer tree, any pipeOps, registered buffers (backward compatible) broadcast,0,32768,tree,simple,-1,1,-1 # Multi-node broadcast: optimized for non-registered buffers, single pipeline op broadcast,32769,4294967295,ring,simple,2,-1,-1,1,0 ``` Comments start with `#` and empty lines are ignored. The CSV format makes it easy to edit configurations in spreadsheet applications like Excel, Google Sheets, or LibreOffice Calc. ### Backward Compatibility Configurations without the numPipeOps and/or regBuff parameters are fully supported: - 8 fields: matches any numPipeOps and regBuff values - 9 fields: matches any regBuff value - 10 fields: full parameter specification This ensures existing configuration files continue to work without modification. ## Usage ### Method 1: Default Config File Place your configuration in `nccl_tuner.conf` in the current working directory. ### Method 2: Environment Variable Set the `NCCL_TUNER_CONFIG_FILE` environment variable to specify the config file path: ```bash export NCCL_TUNER_CONFIG_FILE=/path/to/your/tuner.conf mpirun -np 4 your_nccl_application ``` ## Editing Configuration Files ### Generating Configuration Files from Raw Data A python script to generate valid CSV configs has been provided. [Using optimize_config.py](scripts/README.md). ### Spreadsheet Tips: - Use column headers: `collective_type,min_bytes,max_bytes,algorithm,protocol,channels,nNodes,nRanks,numPipeOps,regBuff` - Save as CSV format (not Excel format) for the plugin to read - Use data validation to prevent typos in algorithm/protocol names ## Logging The plugin uses NCCL's logging system. To see tuner-related messages: ```bash export NCCL_DEBUG=INFO ``` This will show when configurations are loaded and applied, including the topology information. For detailed debugging output during tuning decisions: ```bash export NCCL_DEBUG=TRACE ``` This will show verbose information about which configurations are being evaluated and matched. ## Dimension Matching Configurations are only applied when the topology matches: - **Exact Match**: Configuration specifies `nNodes=4,nRanks=32`, only applied when communicator has exactly 4 nodes and 32 ranks - **Wildcard Nodes**: Configuration specifies `nNodes=-1,nRanks=8`, applied to any topology with exactly 8 ranks - **Wildcard Ranks**: Configuration specifies `nNodes=2,nRanks=-1`, applied to any 2-node topology regardless of ranks per node - **Wildcard Both**: Configuration specifies `nNodes=-1,nRanks=-1`, applied to any topology This allows you to create specialized configurations for different cluster setups while maintaining flexibility. ## Default Behavior If no configuration file is found or no matching configuration exists for a collective operation, the plugin falls back to preferring the ring algorithm with simple protocol. All configured algorithm/protocol combinations are given a low cost (0.0) to make them preferred by NCCL's selection logic. When channels is set to `-1`, NCCL's default channel selection logic is preserved, allowing the system to automatically determine the optimal number of channels based on hardware and message size. ## Troubleshooting 1. **Config file not found**: Check the file path and permissions 2. **Configurations not applied**: Verify the collective type, size ranges, algorithm/protocol names, and topology parameters 3. **Plugin not loaded**: Ensure `LD_LIBRARY_PATH` includes the plugin directory and that `NCCL_TUNER_PLUGIN` either specifies the plugin name, or an absolute path to the plugin shared library. 4. **No effect on performance**: Check that NCCL is actually using the tuner plugin with `NCCL_DEBUG=INFO` 5. **Topology mismatch**: Verify that nNodes and nRanks match your actual setup, or use -1 for wildcards 6. **CSV parsing errors**: Ensure no spaces after commas, or quote fields containing spaces nccl-2.29.7-1/plugins/tuner/example/nccl/000077500000000000000000000000001515037102200200545ustar00rootroot00000000000000nccl-2.29.7-1/plugins/tuner/example/nccl/common.h000066400000000000000000000016271515037102200215230ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef COMMON_H_ #define COMMON_H_ typedef enum {NCCL_LOG_NONE=0, NCCL_LOG_VERSION=1, NCCL_LOG_WARN=2, NCCL_LOG_INFO=3, NCCL_LOG_ABORT=4, NCCL_LOG_TRACE=5} ncclDebugLogLevel; typedef enum {NCCL_INIT=1, NCCL_COLL=2, NCCL_P2P=4, NCCL_SHM=8, NCCL_NET=16, NCCL_GRAPH=32, NCCL_TUNING=64, NCCL_ENV=128, NCCL_ALLOC=256, NCCL_CALL=512, NCCL_PROXY=1024, NCCL_NVLS=2048, NCCL_BOOTSTRAP=4096, NCCL_REG=8192, NCCL_ALL=~0} ncclDebugLogSubSys; typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags, const char *file, int line, const char *fmt, ...); #endif nccl-2.29.7-1/plugins/tuner/example/nccl/err.h000066400000000000000000000014171515037102200210200ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_ERR_H_ #define NCCL_ERR_H_ /* Error type for plugins */ typedef enum { ncclSuccess = 0, ncclUnhandledCudaError = 1, ncclSystemError = 2, ncclInternalError = 3, ncclInvalidArgument = 4, ncclInvalidUsage = 5, ncclRemoteError = 6 } ncclResult_t; #endif nccl-2.29.7-1/plugins/tuner/example/nccl/tuner.h000066400000000000000000000115031515037102200213620ustar00rootroot00000000000000/************************************************************************* * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. * Copyright (c) 2023, Meta Platforms, Inc. and affiliates. * * See LICENSE.txt for license information ************************************************************************/ #ifndef NCCL_TUNER_H_ #define NCCL_TUNER_H_ #include #include #include "common.h" #include "err.h" #define NCCL_NUM_FUNCTIONS 5 // Send/Recv not included for now typedef enum { ncclFuncBroadcast = 0, ncclFuncReduce = 1, ncclFuncAllGather = 2, ncclFuncReduceScatter = 3, ncclFuncAllReduce = 4, ncclFuncSendRecv = 5, ncclFuncSend = 6, ncclFuncRecv = 7, ncclNumFuncs = 8 } ncclFunc_t; #define NCCL_NUM_ALGORITHMS 7 // Tree/Ring/CollNet* #define NCCL_ALGO_UNDEF -1 #define NCCL_ALGO_TREE 0 #define NCCL_ALGO_RING 1 #define NCCL_ALGO_COLLNET_DIRECT 2 #define NCCL_ALGO_COLLNET_CHAIN 3 #define NCCL_ALGO_NVLS 4 #define NCCL_ALGO_NVLS_TREE 5 #define NCCL_ALGO_PAT 6 #define NCCL_NUM_PROTOCOLS 3 // Simple/LL/LL128 #define NCCL_PROTO_UNDEF -1 #define NCCL_PROTO_LL 0 #define NCCL_PROTO_LL128 1 #define NCCL_PROTO_SIMPLE 2 #define NCCL_ALGO_PROTO_IGNORE -1.0 #define NCCL_HW_NVLINK 0 #define NCCL_HW_PCI 1 #define NCCL_HW_NET 2 #define NCCL_NUM_HW_LINKS 3 #define NCCL_VOLTA_COMPCAP_IDX 0 #define NCCL_AMPERE_COMPCAP_IDX 1 #define NCCL_HOPPER_COMPCAP_IDX 2 #define NCCL_BLACKWELL_COMPCAP_IDX 3 #define NCCL_NUM_COMPCAPS 4 #define NCCL_TUNING_SCALE_1NODE 0 #define NCCL_TUNING_SCALE_2NODES 1 #define NCCL_TUNING_SCALE_4NODES 2 #define NCCL_NUM_TUNING_SCALES 3 typedef struct { int nNvlDomains; // number of NVLink domains int minRanksPerNvlDomain; // minimum ranks across all NVLink domains int maxRanksPerNvlDomain; // maximum ranks across all NVLink domains } ncclNvlDomainInfo_v5_t; typedef struct { double baseLatencies [NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; double hwLatencies [NCCL_NUM_HW_LINKS][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; double llMaxBws [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES]; double perChMaxRingLL128Bws [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES]; double perChMaxTreeLL128Bws [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES]; double perChMaxTreeBws [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES]; } ncclTunerConstants_v5_t; // API to be implemented by external tuner typedef struct { // Name of the tuner const char* name; // Initializes tuner states. // Inputs: // - commId: communicator identifier // - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner. // - nNodes: number of nodes in current communicator. // - logFunction: a logFunction can be useful to integrate logging together with NCCL core. // - nvlDomainInfo: NVL domain information struct // Outputs: // - context: tuner context object // Input/Output: // - constants: tuner constants ncclResult_t (*init)(void** ctx, uint64_t commId, size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, ncclNvlDomainInfo_v5_t* nvlDomainInfo, ncclTunerConstants_v5_t* constants); // Gets info (algo, protocol, number of ctas and threads) for a given collective. // Inputs: // - context: tuner context object // - collType: collective type , e.g., allreduce, allgather… // - nBytes: collective size in bytes // - numPipeOps: number of operations in the group // - numAlgo: number of algorithms in collCostTable // - numProto: number of protocols in collCostTable // - regBuff: can register user buffer // // Outputs: // - nChannels: number of channels (hence SMs) to be used. // // InOut: // - collCostTable: collective cost table, generated by NCCL core, containing algo|proto|time entries for collType. // NCCL core sets ignored algo/proto cost table entries to -1.0 (NCCL_ALGO_PROTO_IGNORE). // // If getCollInfo() does not return ncclSuccess, NCCL will fall back to the // default tuning for the given collective. // Also, the plugin is allowed to not set any output, or set only the // algorithm and protocol, but not only the algorithm or only the protocol. // Unset fields will be set automatically by NCCL. ncclResult_t (*getCollInfo)(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff, int* nChannels); // Terminates the plugin and cleans up any resources that the plugin allocated. // context: tuner context object ncclResult_t (*finalize)(void* context); } ncclTuner_v5_t; typedef ncclTuner_v5_t ncclTuner_t; typedef ncclNvlDomainInfo_v5_t ncclNvlDomainInfo_t; typedef ncclTunerConstants_v5_t ncclTunerConstants_t; #define NCCL_TUNER_PLUGIN_SYMBOL "ncclTunerPlugin_v5" #endif nccl-2.29.7-1/plugins/tuner/example/nccl_tuner.conf000066400000000000000000000043601515037102200221430ustar00rootroot00000000000000# NCCL Tuner Configuration File (CSV Format) # Format: collective_type,min_bytes,max_bytes,algorithm,protocol,channels,nNodes,nRanks,numPipeOps,regBuff # # Collective types: broadcast, reduce, allgather, reducescatter, allreduce # Algorithms: tree, ring, collnet_direct, collnet_chain, nvls, nvls_tree, pat # Protocols: ll, ll128, simple # Channels: number of channels to use, or -1 to keep default # nNodes: number of nodes to match, or -1 for any number of nodes # nRanks: number of ranks to match, or -1 for any number of ranks # numPipeOps: number of pipeline operations to match, or -1 for any number (optional) # regBuff: whether user buffer can be registered (0=no, 1=yes, -1=any) (optional) # # Note: numPipeOps and regBuff parameters are optional - configurations without them will match any value # # Examples: # For single-node configurations with registered buffers # Small allreduce operations on single node - use tree algorithm, registered buffers allreduce,0,65536,tree,simple,2,1,-1,-1,1 # For multi-node configurations with 4 nodes, 32 total ranks, single pipeline op, non-registered buffers # Medium allreduce operations - use ring algorithm allreduce,65537,1048576,ring,simple,4,4,32,1,0 # For any topology - large allreduce operations with LL128 protocol, multiple pipeline ops, any buffer type allreduce,1048577,4294967295,ring,ll128,-1,-1,-1,4,-1 # Broadcast operations - different configs for different topologies, pipeline complexity, and buffer types # Single node broadcast - prefer tree, any pipeOps, registered buffers only broadcast,0,32768,tree,simple,-1,1,-1,-1,1 # Multi-node broadcast with single pipeline operation, non-registered buffers - use ring broadcast,32769,4294967295,ring,simple,2,-1,-1,1,0 # AllGather operations - optimized for 2-node configurations, any pipeOps, any buffer type allgather,0,4294967295,ring,simple,4,2,-1 # ReduceScatter operations # Small messages on single node, single pipeline op, registered buffers reducescatter,0,131072,tree,simple,2,1,-1,1,1 # Large messages on any topology, multiple pipeline ops, non-registered buffers reducescatter,131073,4294967295,ring,simple,-1,-1,-1,2,0 # Reduce operations - any topology, keep default channels, any pipeOps, any buffer type reduce,0,4294967295,tree,simple,-1,-1,-1 nccl-2.29.7-1/plugins/tuner/example/plugin.c000066400000000000000000000471221515037102200206050ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "tuner.h" #include #include #include #define __hidden __attribute__ ((visibility("hidden"))) #define MAX_LINE_LENGTH 256 // CSV field indices for configuration parsing // Format: colltype,minbytes,maxbytes,algorithm,protocol,channels,nNodes,nRanks,numPipeOps,regBuff #define CONFIG_FIELD_COLLTYPE 0 #define CONFIG_FIELD_MINBYTES 1 #define CONFIG_FIELD_MAXBYTES 2 #define CONFIG_FIELD_ALGORITHM 3 #define CONFIG_FIELD_PROTOCOL 4 #define CONFIG_FIELD_CHANNELS 5 #define CONFIG_FIELD_NNODES 6 #define CONFIG_FIELD_NRANKS 7 #define CONFIG_FIELD_PIPEOPS 8 // Optional field #define CONFIG_FIELD_REGBUFF 9 // Optional field // Field count constants #define CONFIG_FIELDS_REQUIRED 8 // Minimum required fields (up to nRanks) #define CONFIG_FIELDS_WITH_PIPEOPS 9 // Fields including numPipeOps #define CONFIG_FIELDS_WITH_REGBUFF 10 // Fields including both numPipeOps and regBuff #define CONFIG_FIELDS_MAX 10 // Maximum number of fields supported typedef struct { ncclFunc_t collType; size_t minBytes; size_t maxBytes; int algorithm; int protocol; int nChannels; int nNodes; int nRanks; int numPipeOps; int regBuff; } TuningConfig; typedef struct { TuningConfig* configs; // Changed from static array to dynamic pointer int numConfigs; int maxConfigs; // Added to track allocated size size_t nRanks; size_t nNodes; ncclDebugLogger_t logFunction; ncclNvlDomainInfo_v5_t nvlDomainInfo; } TunerContext; // Parse collective type from string static ncclFunc_t parseCollType(const char* str) { if (strcmp(str, "broadcast") == 0) return ncclFuncBroadcast; if (strcmp(str, "reduce") == 0) return ncclFuncReduce; if (strcmp(str, "allgather") == 0) return ncclFuncAllGather; if (strcmp(str, "reducescatter") == 0) return ncclFuncReduceScatter; if (strcmp(str, "allreduce") == 0) return ncclFuncAllReduce; return ncclFuncAllReduce; // default } // Convert collective type to string static const char* collTypeToString(ncclFunc_t collType) { switch (collType) { case ncclFuncBroadcast: return "broadcast"; case ncclFuncReduce: return "reduce"; case ncclFuncAllGather: return "allgather"; case ncclFuncReduceScatter: return "reducescatter"; case ncclFuncAllReduce: return "allreduce"; default: return "unknown"; } } // Parse algorithm from string static int parseAlgorithm(const char* str) { if (strcmp(str, "tree") == 0) return NCCL_ALGO_TREE; if (strcmp(str, "ring") == 0) return NCCL_ALGO_RING; if (strcmp(str, "collnet_direct") == 0) return NCCL_ALGO_COLLNET_DIRECT; if (strcmp(str, "collnet_chain") == 0) return NCCL_ALGO_COLLNET_CHAIN; if (strcmp(str, "nvls") == 0) return NCCL_ALGO_NVLS; if (strcmp(str, "nvls_tree") == 0) return NCCL_ALGO_NVLS_TREE; if (strcmp(str, "pat") == 0) return NCCL_ALGO_PAT; return NCCL_ALGO_RING; // default } // Convert algorithm to string static const char* algorithmToString(int algorithm) { switch (algorithm) { case NCCL_ALGO_TREE: return "tree"; case NCCL_ALGO_RING: return "ring"; case NCCL_ALGO_COLLNET_DIRECT: return "collnet_direct"; case NCCL_ALGO_COLLNET_CHAIN: return "collnet_chain"; case NCCL_ALGO_NVLS: return "nvls"; case NCCL_ALGO_NVLS_TREE: return "nvls_tree"; case NCCL_ALGO_PAT: return "pat"; default: return "unknown"; } } // Parse protocol from string static int parseProtocol(const char* str) { if (strcmp(str, "ll") == 0) return NCCL_PROTO_LL; if (strcmp(str, "ll128") == 0) return NCCL_PROTO_LL128; if (strcmp(str, "simple") == 0) return NCCL_PROTO_SIMPLE; return NCCL_PROTO_SIMPLE; // default } // Convert protocol to string static const char* protocolToString(int protocol) { switch (protocol) { case NCCL_PROTO_LL: return "ll"; case NCCL_PROTO_LL128: return "ll128"; case NCCL_PROTO_SIMPLE: return "simple"; default: return "unknown"; } } // Helper function to count valid configuration lines in file static int countConfigLines(const char* filename) { FILE* file = fopen(filename, "r"); if (!file) { return 0; } char line[MAX_LINE_LENGTH]; int count = 0; while (fgets(line, sizeof(line), file)) { // Skip comments and empty lines if (line[0] == '#' || line[0] == '\n') continue; // Remove trailing newline line[strcspn(line, "\n")] = 0; // Check if line has content if (strlen(line) > 0) { count++; } } fclose(file); return count; } // Load configuration from file static ncclResult_t loadConfig(TunerContext* ctx, const char* filename) { FILE* file = fopen(filename, "r"); if (!file) { if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Config file %s not found, using defaults", filename); } return ncclSuccess; // Not finding config file is not an error } // First pass: count valid configuration lines int configCount = countConfigLines(filename); if (configCount == 0) { if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: No valid configurations found in %s", filename); } fclose(file); return ncclSuccess; } // Allocate memory for configurations based on actual count ctx->configs = (TuningConfig*)malloc(configCount * sizeof(TuningConfig)); if (!ctx->configs) { if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Failed to allocate memory for %d configurations", configCount); } fclose(file); return ncclSystemError; } ctx->maxConfigs = configCount; ctx->numConfigs = 0; if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Allocated memory for %d configurations", configCount); } // Reset file pointer to beginning fseek(file, 0, SEEK_SET); char line[MAX_LINE_LENGTH]; while (fgets(line, sizeof(line), file) && ctx->numConfigs < ctx->maxConfigs) { // Skip comments and empty lines if (line[0] == '#' || line[0] == '\n') continue; // Remove trailing newline line[strcspn(line, "\n")] = 0; // Parse CSV format: colltype,minbytes,maxbytes,algorithm,protocol,channels,nNodes,nRanks,numPipeOps,regBuff char* token; char* tokens[CONFIG_FIELDS_MAX]; int tokenCount = 0; // Make a copy of the line for tokenizing char lineCopy[MAX_LINE_LENGTH]; strncpy(lineCopy, line, sizeof(lineCopy)); lineCopy[sizeof(lineCopy) - 1] = '\0'; // Tokenize by comma token = strtok(lineCopy, ","); while (token != NULL && tokenCount < CONFIG_FIELDS_MAX) { // Trim whitespace while (*token == ' ' || *token == '\t') token++; char* end = token + strlen(token) - 1; while (end > token && (*end == ' ' || *end == '\t')) { *end = '\0'; end--; } tokens[tokenCount++] = token; token = strtok(NULL, ","); } // Validate field count: support required fields (8), with pipeOps (9), or with regBuff (10) if (tokenCount >= CONFIG_FIELDS_REQUIRED && tokenCount <= CONFIG_FIELDS_MAX) { TuningConfig* config = &ctx->configs[ctx->numConfigs]; config->collType = parseCollType(tokens[CONFIG_FIELD_COLLTYPE]); config->minBytes = (size_t)strtoull(tokens[CONFIG_FIELD_MINBYTES], NULL, 10); config->maxBytes = (size_t)strtoull(tokens[CONFIG_FIELD_MAXBYTES], NULL, 10); config->algorithm = parseAlgorithm(tokens[CONFIG_FIELD_ALGORITHM]); config->protocol = parseProtocol(tokens[CONFIG_FIELD_PROTOCOL]); config->nChannels = atoi(tokens[CONFIG_FIELD_CHANNELS]); config->nNodes = atoi(tokens[CONFIG_FIELD_NNODES]); config->nRanks = atoi(tokens[CONFIG_FIELD_NRANKS]); // numPipeOps is optional (9th field, index 8) if (tokenCount >= CONFIG_FIELDS_WITH_PIPEOPS) { config->numPipeOps = atoi(tokens[CONFIG_FIELD_PIPEOPS]); } else { config->numPipeOps = -1; // -1 means match any numPipeOps } // regBuff is optional (10th field, index 9) if (tokenCount >= CONFIG_FIELDS_WITH_REGBUFF) { config->regBuff = atoi(tokens[CONFIG_FIELD_REGBUFF]); } else { config->regBuff = -1; // -1 means match any regBuff value } ctx->numConfigs++; if (ctx->logFunction) { if (config->numPipeOps == -1 && config->regBuff == -1) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Loaded config: %s [%zu-%zu] %s/%s channels=%d nodes=%d ranks=%d pipeOps=any regBuff=any", tokens[CONFIG_FIELD_COLLTYPE], config->minBytes, config->maxBytes, tokens[CONFIG_FIELD_ALGORITHM], tokens[CONFIG_FIELD_PROTOCOL], config->nChannels, config->nNodes, config->nRanks); } else if (config->regBuff == -1) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Loaded config: %s [%zu-%zu] %s/%s channels=%d nodes=%d ranks=%d pipeOps=%d regBuff=any", tokens[CONFIG_FIELD_COLLTYPE], config->minBytes, config->maxBytes, tokens[CONFIG_FIELD_ALGORITHM], tokens[CONFIG_FIELD_PROTOCOL], config->nChannels, config->nNodes, config->nRanks, config->numPipeOps); } else if (config->numPipeOps == -1) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Loaded config: %s [%zu-%zu] %s/%s channels=%d nodes=%d ranks=%d pipeOps=any regBuff=%d", tokens[CONFIG_FIELD_COLLTYPE], config->minBytes, config->maxBytes, tokens[CONFIG_FIELD_ALGORITHM], tokens[CONFIG_FIELD_PROTOCOL], config->nChannels, config->nNodes, config->nRanks, config->regBuff); } else { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Loaded config: %s [%zu-%zu] %s/%s channels=%d nodes=%d ranks=%d pipeOps=%d regBuff=%d", tokens[CONFIG_FIELD_COLLTYPE], config->minBytes, config->maxBytes, tokens[CONFIG_FIELD_ALGORITHM], tokens[CONFIG_FIELD_PROTOCOL], config->nChannels, config->nNodes, config->nRanks, config->numPipeOps, config->regBuff); } } } } fclose(file); if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Loaded %d tuning configurations from %s", ctx->numConfigs, filename); } return ncclSuccess; } __hidden ncclResult_t pluginInit(void** context, uint64_t commId, size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, ncclNvlDomainInfo_v5_t* nvlDomainInfo, ncclTunerConstants_v5_t* constants) { if (NULL != constants) { // NCCL constants tuning // Tune NCCL's internal tuning model to improve base algo/proto selection. // Note: Example numbers are for reference only. // Actual numbers may vary depending on the hardware and network topology. // These numbers are not guaranteed to be optimal for all cases. // Limit the tree bandwidth to 15GB/s constants->perChMaxTreeBws[NCCL_BLACKWELL_COMPCAP_IDX][NCCL_TUNING_SCALE_4NODES] = 15.0; // Limit the ring bandwidth to 20GB/s constants->perChMaxRingLL128Bws[NCCL_BLACKWELL_COMPCAP_IDX][NCCL_TUNING_SCALE_4NODES] = 20.0; // Set NVLSTree base network latency to 24us constants->hwLatencies[NCCL_HW_NET][NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] = 24.0; } TunerContext* ctx = (TunerContext*)malloc(sizeof(TunerContext)); if (!ctx) return ncclSystemError; ctx->configs = NULL; // Initialize to NULL ctx->numConfigs = 0; ctx->maxConfigs = 0; // Initialize to 0 ctx->nRanks = nRanks; ctx->nNodes = nNodes; ctx->logFunction = logFunction; if (nvlDomainInfo) { ctx->nvlDomainInfo = *nvlDomainInfo; } else { memset(&ctx->nvlDomainInfo, 0, sizeof(ncclNvlDomainInfo_v5_t)); } if (logFunction) { logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Initializing tuner for %zu nodes, %zu ranks, %d NVL domains", nNodes, nRanks, ctx->nvlDomainInfo.nNvlDomains); } // Try to load config file from environment variable or default location const char* configFile = getenv("NCCL_TUNER_CONFIG_FILE"); if (!configFile) { configFile = "nccl_tuner.conf"; // default config file name } ncclResult_t result = loadConfig(ctx, configFile); if (result != ncclSuccess) { if (ctx->configs) { free(ctx->configs); // Clean up allocated memory on error } free(ctx); return result; } *context = ctx; return ncclSuccess; } __hidden ncclResult_t pluginGetCollInfo(void* context, ncclFunc_t collType, size_t nBytes, int numPipeOps, float** collCostTable, int numAlgo, int numProto, int regBuff, int* nChannels) { TunerContext* ctx = (TunerContext*)context; if (!ctx) return ncclInternalError; // Default channels *nChannels = 1; if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_TRACE, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: pluginGetCollInfo called - collType=%s, nBytes=%zu, numPipeOps=%d, regBuff=%d, numConfigs=%d", collTypeToString(collType), nBytes, numPipeOps, regBuff, ctx->numConfigs); } // Look for matching configuration for (int i = 0; i < ctx->numConfigs; i++) { TuningConfig* config = &ctx->configs[i]; if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_TRACE, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Checking config %d - collType=%s, minBytes=%zu, maxBytes=%zu, algo=%s, proto=%s, nNodes=%d, nRanks=%d, numPipeOps=%d, regBuff=%d", i, collTypeToString(config->collType), config->minBytes, config->maxBytes, algorithmToString(config->algorithm), protocolToString(config->protocol), config->nNodes, config->nRanks, config->numPipeOps, config->regBuff); } // Check if this config matches the current collective, size range, topology, pipeline ops, and regBuff if (config->collType == collType && nBytes >= config->minBytes && nBytes <= config->maxBytes && (config->nNodes == -1 || config->nNodes == (int)ctx->nNodes) && (config->nRanks == -1 || config->nRanks == (int)ctx->nRanks) && (config->numPipeOps == -1 || config->numPipeOps == numPipeOps) && (config->regBuff == -1 || config->regBuff == regBuff)) { if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_TRACE, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Config matches. Applying algo=%s, proto=%s, channels=%d", algorithmToString(config->algorithm), protocolToString(config->protocol), config->nChannels); } // Check bounds if (config->algorithm < numAlgo && config->protocol < numProto) { if (collCostTable[config->algorithm][config->protocol] != NCCL_ALGO_PROTO_IGNORE) { if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_TRACE, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Setting cost table[%s][%s] (%p) = 0.0 (was %.1f)", algorithmToString(config->algorithm), protocolToString(config->protocol), &collCostTable[config->algorithm][config->protocol], collCostTable[config->algorithm][config->protocol]); } collCostTable[config->algorithm][config->protocol] = 0.0; // Set low cost to prefer this configuration // Only override channels if not set to -1 (keep default) if (config->nChannels != -1) { *nChannels = config->nChannels; } if (ctx->logFunction) { if (config->nChannels == -1) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Applied config for collType=%s, bytes=%zu, pipeOps=%d, regBuff=%d: algo=%s, proto=%s, channels=default (nodes=%d, ranks=%d)", collTypeToString(config->collType), nBytes, numPipeOps, regBuff, algorithmToString(config->algorithm), protocolToString(config->protocol), config->nNodes, config->nRanks); } else { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Applied config for collType=%s, bytes=%zu, pipeOps=%d, regBuff=%d: algo=%s, proto=%s, channels=%d (nodes=%d, ranks=%d)", collTypeToString(config->collType), nBytes, numPipeOps, regBuff, algorithmToString(config->algorithm), protocolToString(config->protocol), config->nChannels, config->nNodes, config->nRanks); } } return ncclSuccess; } else { if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Algorithm/protocol combination [%s][%s] is marked as IGNORE", algorithmToString(config->algorithm), protocolToString(config->protocol)); } } } else { if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Algorithm/protocol out of bounds - algo=%s (max %d), proto=%s (max %d)", algorithmToString(config->algorithm), numAlgo, protocolToString(config->protocol), numProto); } } } else { if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: Config does not match - collType match=%d, size match=%d, nodes match=%d, ranks match=%d, pipeOps match=%d, regBuff match=%d", config->collType == collType, (nBytes >= config->minBytes && nBytes <= config->maxBytes), (config->nNodes == -1 || config->nNodes == (int)ctx->nNodes), (config->nRanks == -1 || config->nRanks == (int)ctx->nRanks), (config->numPipeOps == -1 || config->numPipeOps == numPipeOps), (config->regBuff == -1 || config->regBuff == regBuff)); } } } // If no specific config found, apply default behavior if (ctx->logFunction) { ctx->logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__, "TUNER/ExamplePlugin: No matching config found"); } return ncclSuccess; } __hidden ncclResult_t pluginFinalize(void* context) { if (context) { TunerContext* ctx = (TunerContext*)context; if (ctx->configs) { free(ctx->configs); // Free dynamically allocated configs array } free(context); } return ncclSuccess; } #define PLUGIN_NAME "Example" const ncclTuner_v5_t ncclTunerPlugin_v5 = { .name = PLUGIN_NAME, .init = pluginInit, .getCollInfo = pluginGetCollInfo, .finalize = pluginFinalize }; nccl-2.29.7-1/plugins/tuner/example/scripts/000077500000000000000000000000001515037102200206245ustar00rootroot00000000000000nccl-2.29.7-1/plugins/tuner/example/scripts/README.md000066400000000000000000000067701515037102200221150ustar00rootroot00000000000000# NCCL Tuner Configuration Scripts This directory contains scripts for optimizing NCCL tuner configurations based on performance data. ## optimize_config.py A Python script that reads performance data from CSV files and generates optimal NCCL tuner configurations. ### Usage ```bash python scripts/optimize_config.py [options] ``` ### Options - `-o, --output FILE`: Output NCCL tuner config file (default: `nccl_tuner.conf`) - `-m, --metric METRIC`: Optimization metric (`cost_metric`, `bandwidth_gbps`, `latency_us`) - `--no-header`: Don't add header comments to output file - `--dry-run`: Print configurations without writing to file ### CSV Input Format The input CSV file should have the following columns: ```csv collective,size_bytes,algorithm,protocol,channels,nodes,ranks,pipeOps,regBuff,cost_metric,bandwidth_gbps,latency_us ``` **Required columns:** - `collective`: NCCL collective type (`allreduce`, `broadcast`, `reduce`, etc.) - `size_bytes`: Message size in bytes - `algorithm`: NCCL algorithm (`tree`, `ring`, `nvls`, etc.) - `protocol`: NCCL protocol (`simple`, `ll`, `ll128`) - `channels`: Number of channels (or `-1` for default) - `nodes`: Number of nodes (or `-1` for any) - `ranks`: Number of ranks (or `-1` for any) - `pipeOps`: Number of pipeline operations (or `-1` for any) - `regBuff`: Registered buffer flag (`0`, `1`, or `-1` for any) **Optional metrics (must have at least one present):** - `bandwidth_gbps`: Bandwidth in GB/s (higher is better) - `latency_us`: Latency in microseconds (lower is better) ### Examples **Basic usage with cost optimization:** ```bash python scripts/optimize_config.py sample_performance_data.csv ``` **Optimize for bandwidth and write to custom file:** ```bash python scripts/optimize_config.py -m bandwidth_gbps -o my_tuner.conf performance_data.csv ``` **Preview configurations without writing:** ```bash python scripts/optimize_config.py --dry-run performance_data.csv ``` ### How It Works 1. **Data Loading**: Reads CSV performance data and validates format 2. **Grouping**: Groups data by collective type, topology (nodes/ranks), and other parameters 3. **Size Ranges**: Automatically bins data into size ranges for optimization 4. **Optimization**: Finds the best performing configuration for each group/size combination 5. **Output**: Generates NCCL tuner config format and appends to specified file ### Default Size Ranges The script uses these default size ranges (in bytes): - Small: 0 - 1,024 - Medium: 1,025 - 65,536 - Large: 65,537 - 1,048,576 - XLarge: 1,048,577 - 16,777,216 - XXLarge: 16,777,217 - 4,294,967,295 ### Sample Data See `sample_performance_data.csv` for an example of the expected input format. ### Integration with NCCL The generated configuration file can be used directly with the NCCL tuner plugin: ```bash export NCCL_TUNER_CONFIG_FILE=/path/to/optimized_config.conf export NCCL_TUNER_PLUGIN=/path/to/libnccl-tuner.so mpirun -np 8 your_nccl_application ``` ### Performance Data Collection To collect performance data for optimization, you can: 1. **Use NCCL benchmarks** with different algorithm/protocol combinations 2. **Profile your applications** with various tuner settings 3. **Run systematic sweeps** across parameter combinations 4. **Use NCCL debug output** to collect timing information The key is to have comprehensive data covering: - Different message sizes (small to large) - Various topologies (single node, multi-node) - All relevant algorithm/protocol combinations - Different channel counts and pipeline configurations nccl-2.29.7-1/plugins/tuner/example/scripts/optimize_config.py000066400000000000000000000463241515037102200243740ustar00rootroot00000000000000#!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information """ NCCL Tuner Configuration Optimizer Reads a CSV file containing performance data across different tuning parameters and generates optimal NCCL tuner configurations based on the best performing combinations. By default, creates growing size ranges that interpolate between the actual data sizes for each unique dimension (node count, rank count combination). This ensures that different cluster configurations get their own optimized size boundaries, as performance characteristics often vary significantly between topologies. Each dimension gets its own set of ranges starting from 0 and extending to the maximum size for that dimension, with boundaries at midpoints between consecutive data sizes. CSV Input Format: collective,size_bytes,algorithm,protocol,channels,nodes,ranks,pipeOps,regBuff,bandwidth_gbps,latency_us Output Format (NCCL Tuner Config): collective_type,min_bytes,max_bytes,algorithm,protocol,channels,nNodes,nRanks,numPipeOps,regBuff Usage Examples: # Auto-create dimension-specific interpolated ranges (default) python3 optimize_config.py data.csv # Use custom size ranges (applied to all topologies) python3 optimize_config.py data.csv --size-ranges "0-1024,1025-65536,65537-1048576" # Use hardcoded default ranges (applied to all topologies) python3 optimize_config.py data.csv --no-auto-ranges """ import csv import argparse import sys import os from collections import defaultdict from typing import Dict, List, Tuple, Any class PerformanceData: def __init__(self, row: Dict[str, str]): self.collective = row['collective'] self.size_bytes = int(row['size_bytes']) self.algorithm = row['algorithm'] self.protocol = row['protocol'] self.channels = int(row['channels']) if row['channels'] != '-1' else -1 self.nodes = int(row['nodes']) if row['nodes'] != '-1' else -1 self.ranks = int(row['ranks']) if row['ranks'] != '-1' else -1 self.pipeOps = int(row['pipeOps']) if row['pipeOps'] != '-1' else -1 self.regBuff = int(row['regBuff']) if row['regBuff'] != '-1' else -1 # Performance metrics self.bandwidth_gbps = float(row.get('bandwidth_gbps', 0)) # Higher is better self.latency_us = float(row.get('latency_us', 0)) # Lower is better def get_config_key(self) -> Tuple: """Generate a key for grouping similar configurations""" return (self.collective, self.nodes, self.ranks, self.pipeOps, self.regBuff) def get_size_range_key(self, topology_size_ranges: Dict[Tuple[int, int], List[Tuple[int, int]]]) -> Tuple[int, int]: """Find which size range this data point belongs to for its dimension""" topology_key = (self.nodes, self.ranks) # Get size ranges for this dimension, or fall back to default if topology_key in topology_size_ranges: size_ranges = topology_size_ranges[topology_key] elif (-1, -1) in topology_size_ranges: size_ranges = topology_size_ranges[(-1, -1)] else: # Fallback to first available dimension ranges size_ranges = next(iter(topology_size_ranges.values())) for min_size, max_size in size_ranges: if min_size <= self.size_bytes <= max_size: return (min_size, max_size) # If no range found, create a single-point range return (self.size_bytes, self.size_bytes) class ConfigOptimizer: def __init__(self, optimization_metric: str = 'latency_us'): self.optimization_metric = optimization_metric # Default size ranges - will be overridden by auto-detection self.size_ranges = [ (0, 1024), (1025, 64*1024), (64*1024+1, 1024*1024), (1024*1024+1, 16*1024*1024), (16*1024*1024+1, 4*1024*1024*1024-1) ] self.auto_size_ranges = True def set_size_ranges(self, ranges: List[Tuple[int, int]]): """Set custom size ranges for optimization""" self.size_ranges = ranges self.auto_size_ranges = False def auto_determine_size_ranges(self, data: List[PerformanceData]) -> Dict[Tuple[int, int], List[Tuple[int, int]]]: """Create growing size ranges for each unique (nodes, ranks) dimension""" if not data: return {(-1, -1): self.size_ranges} # Group data by dimension (nodes, ranks) topology_data = defaultdict(list) for item in data: topology_key = (item.nodes, item.ranks) topology_data[topology_key].append(item) topology_ranges = {} for topology_key, items in topology_data.items(): nodes, ranks = topology_key # Extract unique sizes for this dimension and sort them unique_sizes = sorted(set(item.size_bytes for item in items)) if len(unique_sizes) <= 1: # Only one size, create a single range from 0 to that size size = unique_sizes[0] if unique_sizes else 0 ranges = [(0, size)] else: # Create growing ranges that interpolate between data points ranges = [] for i, size in enumerate(unique_sizes): if i == 0: # First range: 0 to midpoint between first and second size if len(unique_sizes) > 1: next_size = unique_sizes[i + 1] max_size = (size + next_size) // 2 else: max_size = size min_size = 0 elif i == len(unique_sizes) - 1: # Last range: previous max + 1 to current size (and beyond) min_size = ranges[-1][1] + 1 max_size = size else: # Intermediate ranges: previous max + 1 to midpoint with next size min_size = ranges[-1][1] + 1 next_size = unique_sizes[i + 1] max_size = (size + next_size) // 2 ranges.append((min_size, max_size)) topology_ranges[topology_key] = ranges print(f"Dimension {nodes} nodes, {ranks} ranks: {len(ranges)} size ranges from {len(unique_sizes)} unique sizes:") for i, (min_size, max_size) in enumerate(ranges): # Count data points that fall in this range for this dimension count = sum(1 for item in items if min_size <= item.size_bytes <= max_size) actual_sizes = sorted(set(item.size_bytes for item in items if min_size <= item.size_bytes <= max_size)) if actual_sizes: size_list = ', '.join(f"{s:,}" for s in actual_sizes[:3]) if len(actual_sizes) > 3: size_list += f", ... (+{len(actual_sizes)-3} more)" print(f" Range {i+1}: {min_size:,} - {max_size:,} bytes ({count} data points, sizes: {size_list})") return topology_ranges def load_data(self, csv_file: str) -> List[PerformanceData]: """Load performance data from CSV file""" data = [] try: with open(csv_file, 'r') as f: reader = csv.DictReader(f) for row in reader: try: data.append(PerformanceData(row)) except (ValueError, KeyError) as e: print(f"Warning: Skipping invalid row: {row} - {e}") except FileNotFoundError: print(f"Error: File {csv_file} not found") sys.exit(1) except Exception as e: print(f"Error reading {csv_file}: {e}") sys.exit(1) print(f"Loaded {len(data)} performance data points") # Auto-determine size ranges if enabled if self.auto_size_ranges and data: self.topology_size_ranges = self.auto_determine_size_ranges(data) else: # Use default ranges for all topologies self.topology_size_ranges = {(-1, -1): self.size_ranges} return data def is_better(self, new_data: PerformanceData, current_best: PerformanceData) -> bool: """Determine if new_data is better than current_best""" if self.optimization_metric == 'bandwidth_gbps': return new_data.bandwidth_gbps > current_best.bandwidth_gbps elif self.optimization_metric == 'latency_us': return new_data.latency_us < current_best.latency_us else: # Default to latency return new_data.latency_us < current_best.latency_us def optimize_configurations(self, data: List[PerformanceData]) -> List[str]: """Find optimal configurations and return as NCCL config strings""" # Group data by configuration key and size range grouped_data = defaultdict(lambda: defaultdict(list)) for item in data: config_key = item.get_config_key() size_range = item.get_size_range_key(self.topology_size_ranges) grouped_data[config_key][size_range].append(item) # Store optimal configurations before combining ranges optimal_configs = [] for config_key, size_ranges_dict in grouped_data.items(): collective, nodes, ranks, pipeOps, regBuff = config_key for (min_size, max_size), items in size_ranges_dict.items(): if not items: continue # Find the best performing configuration for this size range best_item = items[0] for item in items[1:]: if self.is_better(item, best_item): best_item = item # Store the optimal configuration with its range optimal_configs.append({ 'collective': collective, 'min_size': min_size, 'max_size': max_size, 'algorithm': best_item.algorithm, 'protocol': best_item.protocol, 'channels': best_item.channels, 'nodes': best_item.nodes, 'ranks': best_item.ranks, 'pipeOps': best_item.pipeOps, 'regBuff': best_item.regBuff, 'metric_value': getattr(best_item, self.optimization_metric) }) # Combine sequential ranges with identical tunings combined_configs = self.combine_sequential_ranges(optimal_configs) # Generate config strings configs = [] for config in combined_configs: config_str = f"{config['collective']},{config['min_size']},{config['max_size']},{config['algorithm']},{config['protocol']},{config['channels']},{config['nodes']},{config['ranks']},{config['pipeOps']},{config['regBuff']}" configs.append(config_str) print(f"Optimal for {config['collective']} [{config['min_size']}-{config['max_size']}] nodes={config['nodes']} ranks={config['ranks']}: " f"{config['algorithm']}/{config['protocol']} channels={config['channels']} " f"({self.optimization_metric}={config['metric_value']:.3f})") return configs def combine_sequential_ranges(self, configs: List[Dict]) -> List[Dict]: """Combine sequential ranges that have identical tuning parameters""" if not configs: return configs # Group by collective and topology (nodes, ranks) topology_groups = defaultdict(list) for config in configs: topology_key = (config['collective'], config['nodes'], config['ranks'], config['pipeOps'], config['regBuff']) topology_groups[topology_key].append(config) combined_configs = [] for topology_key, topology_configs in topology_groups.items(): # Sort by min_size to ensure proper ordering topology_configs.sort(key=lambda x: x['min_size']) # Group by tuning parameters (algorithm, protocol, channels) tuning_groups = defaultdict(list) for config in topology_configs: tuning_key = (config['algorithm'], config['protocol'], config['channels']) tuning_groups[tuning_key].append(config) # For each tuning group, combine sequential ranges for tuning_key, tuning_configs in tuning_groups.items(): if not tuning_configs: continue # Sort by min_size tuning_configs.sort(key=lambda x: x['min_size']) # Combine sequential ranges current_config = tuning_configs[0].copy() for next_config in tuning_configs[1:]: # Check if ranges are adjacent or overlapping if current_config['max_size'] + 1 >= next_config['min_size']: # Extend the current range current_config['max_size'] = max(current_config['max_size'], next_config['max_size']) # Update metric value to the better one if self.optimization_metric == 'bandwidth_gbps': if next_config['metric_value'] > current_config['metric_value']: current_config['metric_value'] = next_config['metric_value'] else: # latency_us or default if next_config['metric_value'] < current_config['metric_value']: current_config['metric_value'] = next_config['metric_value'] else: # Gap between ranges, save current and start new one combined_configs.append(current_config) current_config = next_config.copy() # Add the last configuration combined_configs.append(current_config) # Sort final configs by collective, nodes, ranks, then min_size combined_configs.sort(key=lambda x: (x['collective'], x['nodes'], x['ranks'], x['min_size'])) original_count = len(configs) combined_count = len(combined_configs) if combined_count < original_count: print(f"Combined {original_count} ranges into {combined_count} ranges " f"(reduced by {original_count - combined_count})") return combined_configs def append_to_config_file(self, configs: List[str], config_file: str, add_header: bool = True): """Append optimized configurations to NCCL tuner config file""" try: # Create directory if it doesn't exist config_dir = os.path.dirname(config_file) if config_dir and not os.path.exists(config_dir): os.makedirs(config_dir) print(f"Created directory: {config_dir}") # Check if file exists and has content file_exists = os.path.exists(config_file) add_separator = False if file_exists: with open(config_file, 'r') as f: content = f.read().strip() add_separator = len(content) > 0 print(f"Appending to existing file: {config_file}") else: print(f"Creating new file: {config_file}") with open(config_file, 'a') as f: if add_separator: f.write("\n\n") if add_header: f.write(f"# Optimized configurations generated by optimize_config.py\n") f.write(f"# Optimization metric: {self.optimization_metric}\n") f.write(f"# Format: collective_type,min_bytes,max_bytes,algorithm,protocol,channels,nNodes,nRanks,numPipeOps,regBuff\n") for config in configs: f.write(f"{config}\n") if file_exists: print(f"Appended {len(configs)} optimized configurations to {config_file}") else: print(f"Created {config_file} with {len(configs)} optimized configurations") except PermissionError: print(f"Error: Permission denied writing to {config_file}") print("Try running with appropriate permissions or choose a different output location") sys.exit(1) except OSError as e: print(f"Error: Cannot create/write to {config_file}: {e}") print("Check that the path is valid and you have write permissions") sys.exit(1) except Exception as e: print(f"Unexpected error writing to {config_file}: {e}") sys.exit(1) def main(): parser = argparse.ArgumentParser(description="Optimize NCCL tuner configurations from performance data") parser.add_argument("csv_file", help="Input CSV file with performance data") parser.add_argument("-o", "--output", default="nccl_tuner.conf", help="Output NCCL tuner config file (default: nccl_tuner.conf)") parser.add_argument("-m", "--metric", choices=['bandwidth_gbps', 'latency_us'], default='latency_us', help="Optimization metric (default: latency_us)") parser.add_argument("--no-header", action="store_true", help="Don't add header comments to output file") parser.add_argument("--dry-run", action="store_true", help="Print configurations without writing to file") parser.add_argument("--no-auto-ranges", action="store_true", help="Disable automatic size range determination (use default ranges)") parser.add_argument("--size-ranges", type=str, help="Custom size ranges as comma-separated pairs: 'min1-max1,min2-max2,...'") args = parser.parse_args() optimizer = ConfigOptimizer(args.metric) # Handle size range configuration if args.size_ranges: # Parse custom size ranges try: ranges = [] for range_str in args.size_ranges.split(','): min_size, max_size = map(int, range_str.split('-')) ranges.append((min_size, max_size)) optimizer.set_size_ranges(ranges) print(f"Using custom size ranges: {ranges}") except ValueError: print("Error: Invalid size ranges format. Use 'min1-max1,min2-max2,...'") sys.exit(1) elif args.no_auto_ranges: # Disable auto-ranging optimizer.auto_size_ranges = False print("Using default hardcoded size ranges") else: # Auto-ranging is enabled by default - creates one bucket per unique size optimizer.auto_size_ranges = True print("Auto-ranging enabled: will create one bucket per unique size in data") # Load and optimize data data = optimizer.load_data(args.csv_file) if not data: print("No valid data found in CSV file") sys.exit(1) configs = optimizer.optimize_configurations(data) if args.dry_run: print("\nGenerated configurations:") for config in configs: print(config) else: optimizer.append_to_config_file(configs, args.output, not args.no_header) if __name__ == "__main__": main() nccl-2.29.7-1/plugins/tuner/example/scripts/sample_performance_data.csv000066400000000000000000000025661515037102200262050ustar00rootroot00000000000000collective,size_bytes,algorithm,protocol,channels,nodes,ranks,pipeOps,regBuff,cost_metric,bandwidth_gbps,latency_us allreduce,1024,tree,simple,2,1,8,-1,-1,0.15,45.2,12.5 allreduce,1024,ring,simple,4,1,8,-1,-1,0.12,52.1,10.8 allreduce,1024,tree,ll,2,1,8,-1,-1,0.18,41.3,15.2 allreduce,1024,ring,ll,4,1,8,-1,-1,0.14,48.7,12.1 allreduce,32768,tree,simple,2,1,8,-1,-1,0.25,156.8,25.3 allreduce,32768,ring,simple,4,1,8,-1,-1,0.18,189.2,18.4 allreduce,32768,ring,ll128,8,1,8,-1,-1,0.16,201.5,16.2 allreduce,1048576,ring,simple,4,1,8,-1,-1,0.45,425.6,45.1 allreduce,1048576,ring,ll128,8,1,8,-1,-1,0.38,482.3,38.7 allreduce,1048576,nvls,simple,16,1,8,-1,-1,0.32,551.2,32.1 broadcast,1024,tree,simple,2,1,8,-1,-1,0.08,89.4,8.2 broadcast,1024,ring,simple,4,1,8,-1,-1,0.12,71.3,12.1 broadcast,32768,tree,simple,2,1,8,-1,-1,0.18,234.7,18.5 broadcast,32768,ring,ll128,4,1,8,-1,-1,0.15,267.8,15.2 broadcast,1048576,ring,simple,4,1,8,-1,-1,0.35,612.4,35.1 broadcast,1048576,ring,ll128,8,1,8,-1,-1,0.28,702.1,28.3 allreduce,1024,tree,simple,2,2,16,-1,-1,0.22,38.1,22.4 allreduce,1024,ring,simple,4,2,16,-1,-1,0.19,42.7,19.6 allreduce,32768,ring,simple,4,2,16,-1,-1,0.28,145.2,28.1 allreduce,32768,ring,ll128,8,2,16,-1,-1,0.24,167.8,24.3 allreduce,1048576,ring,simple,4,2,16,-1,-1,0.58,387.5,58.2 allreduce,1048576,ring,ll128,8,2,16,-1,-1,0.48,456.9,48.1 allreduce,1048576,nvls,simple,16,2,16,-1,-1,0.42,512.6,42.3 nccl-2.29.7-1/plugins/tuner/example/test/000077500000000000000000000000001515037102200201145ustar00rootroot00000000000000nccl-2.29.7-1/plugins/tuner/example/test/Makefile000066400000000000000000000014121515037102200215520ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # # # Makefile for NCCL Tuner Plugin Unit Tests # CC := gcc CFLAGS := -Wall -Wextra -g -std=c99 -fPIC INC := -I. -I../nccl TARGET := test_plugin SOURCES := test_plugin.c # Default target all: $(TARGET) # Build the test executable $(TARGET): $(SOURCES) $(CC) $(CFLAGS) $(INC) -o $(TARGET) $(SOURCES) # Run the tests test: $(TARGET) ./$(TARGET) $(TEST_CASE) # Run tests with verbose output test-verbose: $(TARGET) NCCL_DEBUG=INFO ./$(TARGET) $(TEST_CASE) # Clean build artifacts clean: rm -f $(TARGET) *.o *.gcov *.gcda *.gcno test_*.conf .PHONY: all test test-verbose clean nccl-2.29.7-1/plugins/tuner/example/test/README.md000066400000000000000000000113241515037102200213740ustar00rootroot00000000000000# NCCL Tuner Plugin Unit Tests This directory contains comprehensive unit tests for the NCCL tuner plugin. The tests verify all major functionality including configuration parsing, matching logic, and cost table updates. ## Test Structure ``` test/ ├── test_plugin.c # Main unit test file ├── Makefile # Build system for tests └── README.md # This file ``` ## Building and Running Tests ### Quick Start ```bash # Build and run all tests make test # Or step by step make # Build test executable ./test_plugin # Run tests ``` ### Advanced Testing ```bash # Run with memory leak detection (requires valgrind) make test-memory # Run with verbose logging make test-verbose # Generate code coverage report (requires gcov) make coverage # Create sample test configuration files make test-configs ``` ## Test Coverage The unit tests cover the following functionality: ### 1. **Plugin Initialization (`test_plugin_init`)** - Tests successful plugin initialization - Verifies context allocation - Tests cleanup on destroy ### 2. **Configuration Parsing (`test_config_parsing_valid`, `test_config_parsing_invalid`)** - Valid CSV format parsing - Comment and empty line handling - Invalid format graceful handling - Environment variable configuration ### 3. **Collective Type Matching (`test_collective_matching`)** - Correct matching of allreduce, broadcast, etc. - Algorithm/protocol selection - Channel configuration ### 4. **Size Range Matching (`test_size_matching`)** - Small, medium, large message size handling - Proper range boundary checking - Multiple size-based configurations ### 5. **Topology Matching (`test_topology_matching`)** - Single-node vs multi-node configurations - Exact nNodes/nRanks matching - Wildcard matching (-1 values) ### 6. **Default Channels (`test_default_channels`)** - Proper handling of -1 channel specification - Preservation of NCCL default behavior ### 7. **Registered Buffer Matching (`test_regbuff_matching`)** - Configurations based on regBuff parameter - Registered vs non-registered buffer handling - Backward compatibility with configs missing regBuff ### 8. **Pipeline Operations Matching (`test_pipeops_matching`)** - Configurations based on numPipeOps parameter - Single vs multiple pipeline operation handling - Backward compatibility with configs missing numPipeOps ### 9. **Fallback Behavior (`test_no_match_fallback`)** - Default behavior when no config matches - Ring/Simple algorithm fallback ## Test Output Successful test run: ``` Running NCCL Tuner Plugin Unit Tests ===================================== PASS: test_plugin_init PASS: test_config_parsing_valid PASS: test_config_parsing_invalid PASS: test_collective_matching PASS: test_size_matching PASS: test_topology_matching PASS: test_default_channels PASS: test_regbuff_matching PASS: test_pipeops_matching PASS: test_no_match_fallback ===================================== Test Results: 9/9 tests passed All tests PASSED! ``` Failed test example: ``` FAIL: test_collective_matching - Tree/Simple should have low cost Test Results: 8/9 tests passed Some tests FAILED! ``` ## Mock NCCL Implementation The tests use the actual NCCL header files from the `../nccl/` directory: - `tuner.h` - Complete NCCL tuner interface and type definitions - `common.h` - Common NCCL types and logging functions - `err.h` - NCCL error codes This allows testing with the real NCCL interface definitions while still being able to run tests without the full NCCL library installation. ## Integration with CI/CD ```bash # Install tests for CI/CD pipeline make install-test # Run as part of automated testing make test && echo "Tests passed" || echo "Tests failed" ``` ## Memory Testing The tests can be run with valgrind for memory leak detection: ```bash make test-memory ``` This will detect: - Memory leaks - Invalid memory access - Use of uninitialized memory ## Code Coverage Generate code coverage reports to ensure comprehensive testing: ```bash make coverage # Creates test_plugin.c.gcov with line-by-line coverage ``` ## Adding New Tests To add a new test: 1. Create a new test function in `test_plugin.c`: ```c int test_new_feature() { // Test setup TEST_ASSERT(condition, "description"); // Test cleanup TEST_PASS(); } ``` 2. Add the test to the main function: ```c total++; passed += test_new_feature(); ``` 3. Rebuild and run: ```bash make test ``` ## Debugging Tests For debugging failed tests: ```bash # Compile with debug symbols make CFLAGS="-g -O0 -DDEBUG" # Run with gdb gdb ./test_plugin ``` ## Cleaning Up ```bash # Remove all build artifacts and temporary files make clean ``` This comprehensive test suite ensures the NCCL tuner plugin works correctly across all supported configurations and edge cases. nccl-2.29.7-1/plugins/tuner/example/test/test_plugin.c000066400000000000000000001117711515037102200226250ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ /************************************************************************* * Unit tests for NCCL Tuner Plugin ************************************************************************/ #define _GNU_SOURCE // Enable setenv/unsetenv and other GNU extensions #include #include #include #include #include #include #include // Include NCCL tuner header (which includes common.h and err.h) #include "tuner.h" // Include plugin source for testing #include "../plugin.c" // Test framework macros #define TEST_ASSERT(condition, message) \ do { \ if (!(condition)) { \ printf("FAIL: %s - %s\n", __func__, message); \ return 0; \ } \ } while(0) #define TEST_PASS() \ do { \ printf("PASS: %s\n", __func__); \ return 1; \ } while(0) // Global test state static int test_log_count = 0; // Mock logger function void mock_logger(ncclDebugLogLevel level, unsigned long flags, const char* file, int line, const char* fmt, ...) { (void)flags; // Suppress unused parameter warning test_log_count++; // Check if we should print based on NCCL_DEBUG level const char* debug_level = getenv("NCCL_DEBUG"); int should_print = 0; if (debug_level) { if (strcmp(debug_level, "TRACE") == 0) { should_print = 1; // Print everything } else if (strcmp(debug_level, "INFO") == 0 && level <= NCCL_LOG_INFO) { should_print = 1; // Print INFO and below } else if (strcmp(debug_level, "WARN") == 0 && level <= NCCL_LOG_WARN) { should_print = 1; // Print WARN and below } } if (!should_print) return; // Convert log level to string const char* level_str; switch(level) { case NCCL_LOG_NONE: level_str = "NONE"; break; case NCCL_LOG_VERSION: level_str = "VERSION"; break; case NCCL_LOG_WARN: level_str = "WARN"; break; case NCCL_LOG_INFO: level_str = "INFO"; break; case NCCL_LOG_ABORT: level_str = "ABORT"; break; case NCCL_LOG_TRACE: level_str = "TRACE"; break; default: level_str = "UNKNOWN"; break; } // Print log header printf("[TUNER:%s:%s:%d] ", level_str, file, line); // Print formatted message va_list args; va_start(args, fmt); vprintf(fmt, args); va_end(args); printf("\n"); } // Helper function to create test config file void create_test_config(const char* filename, const char* content) { FILE* f = fopen(filename, "w"); if (f) { fprintf(f, "%s", content); fclose(f); } } // Test 1: Plugin initialization int test_plugin_init() { void* context = NULL; // Test successful initialization ncclResult_t result = pluginInit(&context, 0, 8, 2, mock_logger, NULL, NULL); TEST_ASSERT(result == ncclSuccess, "Plugin init should succeed"); TEST_ASSERT(context != NULL, "Context should be allocated"); // Clean up pluginFinalize(context); TEST_PASS(); } // Test 2: Configuration file parsing - valid CSV int test_config_parsing_valid() { const char* test_config = "# Test configuration\n" "allreduce,0,65536,tree,simple,2,1,-1,-1,-1\n" "broadcast,0,32768,ring,ll128,4,2,16,-1,-1\n" "# Comment line\n" "\n" // Empty line "reduce,1024,2048,tree,simple,-1,-1,-1,-1,-1\n"; create_test_config("test_valid.conf", test_config); // Set environment variable to use our test config setenv("NCCL_TUNER_CONFIG_FILE", "test_valid.conf", 1); void* context = NULL; ncclResult_t result = pluginInit(&context, 0, 16, 2, mock_logger, NULL, NULL); TEST_ASSERT(result == ncclSuccess, "Plugin init with valid config should succeed"); // Clean up pluginFinalize(context); unlink("test_valid.conf"); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 3: Configuration file parsing - invalid CSV int test_config_parsing_invalid() { const char* test_config = "allreduce,0,65536,tree,simple,2,1 # Missing nRanks and other fields\n" "invalid_collective,0,1024,ring,simple,1,1,1,-1,-1\n" "broadcast,abc,def,ring,simple,1,1,1,-1,-1\n"; // Invalid numbers create_test_config("test_invalid.conf", test_config); setenv("NCCL_TUNER_CONFIG_FILE", "test_invalid.conf", 1); void* context = NULL; ncclResult_t result = pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL); // Should still succeed but with no valid configs loaded TEST_ASSERT(result == ncclSuccess, "Plugin init should succeed even with invalid config"); // Clean up pluginFinalize(context); unlink("test_invalid.conf"); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 4: Collective type matching int test_collective_matching() { const char* test_config = "allreduce,0,65536,tree,simple,8,1,-1,-1,-1\n" "broadcast,0,32768,ring,ll128,4,-1,-1,-1,-1\n"; create_test_config("test_match.conf", test_config); setenv("NCCL_TUNER_CONFIG_FILE", "test_match.conf", 1); void* context = NULL; pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL); // Create mock cost table float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float* cost_table_ptr[NCCL_NUM_ALGORITHMS]; for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { cost_table_ptr[i] = cost_table[i]; for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; // Default high cost } } int nChannels; // Test allreduce matching (should match first config) ncclResult_t result = pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); TEST_ASSERT(result == ncclSuccess, "GetCollInfo should succeed"); mock_logger(NCCL_LOG_INFO, NCCL_ALL, __FILE__, __LINE__, "DEBUG: Checking cost_table[TREE][SIMPLE] (%p) = %.1f (expecting 0.0)", &cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE], cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE]); TEST_ASSERT(cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE] == 0.0, "Tree/Simple should have low cost"); TEST_ASSERT(nChannels == 8, "Should set 8 channels"); // Test broadcast matching (should match second config) for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; // Reset costs } } result = pluginGetCollInfo(context, ncclFuncBroadcast, 16384, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); TEST_ASSERT(result == ncclSuccess, "GetCollInfo should succeed"); mock_logger(NCCL_LOG_INFO, NCCL_ALL, __FILE__, __LINE__, "DEBUG: Checking cost_table[RING][LL128] (%p) = %.1f (expecting 0.0)", &cost_table[NCCL_ALGO_RING][NCCL_PROTO_LL128], cost_table[NCCL_ALGO_RING][NCCL_PROTO_LL128]); TEST_ASSERT(cost_table[NCCL_ALGO_RING][NCCL_PROTO_LL128] == 0.0, "Ring/LL128 should have low cost"); TEST_ASSERT(nChannels == 4, "Should set 4 channels"); // Clean up pluginFinalize(context); unlink("test_match.conf"); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 5: Size range matching int test_size_matching() { const char* test_config = "allreduce,0,1024,tree,simple,2,-1,-1,-1,-1\n" "allreduce,1025,65536,ring,simple,4,-1,-1,-1,-1\n" "allreduce,65537,4294967295,ring,ll128,8,-1,-1,-1,-1\n"; create_test_config("test_size.conf", test_config); setenv("NCCL_TUNER_CONFIG_FILE", "test_size.conf", 1); void* context = NULL; pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL); float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float* cost_table_ptr[NCCL_NUM_ALGORITHMS]; for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { cost_table_ptr[i] = cost_table[i]; for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } int nChannels = 1; pluginGetCollInfo(context, ncclFuncAllReduce, 512, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); mock_logger(NCCL_LOG_INFO, NCCL_ALL, __FILE__, __LINE__, "DEBUG: Small message - checking cost_table[TREE][SIMPLE] (%p) = %.1f (expecting 0.0)", &cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE], cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE]); TEST_ASSERT(cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE] == 0.0, "Small: Tree/Simple should have low cost"); TEST_ASSERT(nChannels == 2, "Small: Should set 2 channels"); // Test medium message (should match second config) for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); mock_logger(NCCL_LOG_INFO, NCCL_ALL, __FILE__, __LINE__, "DEBUG: Medium message - checking cost_table[RING][SIMPLE] (%p) = %.1f (expecting 0.0)", &cost_table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE], cost_table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE]); TEST_ASSERT(cost_table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] == 0.0, "Medium: Ring/Simple should have low cost"); TEST_ASSERT(nChannels == 4, "Medium: Should set 4 channels"); // Test large message (should match third config) for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } pluginGetCollInfo(context, ncclFuncAllReduce, 1048576, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); mock_logger(NCCL_LOG_INFO, NCCL_ALL, __FILE__, __LINE__, "DEBUG: Large message - checking cost_table[RING][LL128] (%p) = %.1f (expecting 0.0)", &cost_table[NCCL_ALGO_RING][NCCL_PROTO_LL128], cost_table[NCCL_ALGO_RING][NCCL_PROTO_LL128]); TEST_ASSERT(cost_table[NCCL_ALGO_RING][NCCL_PROTO_LL128] == 0.0, "Large: Ring/LL128 should have low cost"); TEST_ASSERT(nChannels == 8, "Large: Should set 8 channels"); // Clean up pluginFinalize(context); unlink("test_size.conf"); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 6: Topology matching int test_topology_matching() { const char* test_config = "allreduce,0,65536,tree,simple,2,1,-1,-1,-1\n" // Single node only "allreduce,0,65536,ring,simple,4,4,32,-1,-1\n" // 4 nodes, 32 ranks exactly "allreduce,0,65536,ring,ll128,8,-1,-1,-1,-1\n"; // Any topology create_test_config("test_topo.conf", test_config); setenv("NCCL_TUNER_CONFIG_FILE", "test_topo.conf", 1); // Test with single node setup void* context1 = NULL; pluginInit(&context1, 0, 8, 1, mock_logger, NULL, NULL); // 8 ranks, 1 node float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float* cost_table_ptr[NCCL_NUM_ALGORITHMS]; for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { cost_table_ptr[i] = cost_table[i]; for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } int nChannels; pluginGetCollInfo(context1, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); TEST_ASSERT(cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE] == 0.0, "Single node: Should match tree config"); TEST_ASSERT(nChannels == 2, "Single node: Should set 2 channels"); pluginFinalize(context1); // Test with 4 nodes, 32 ranks setup void* context2 = NULL; pluginInit(&context2, 0, 32, 4, mock_logger, NULL, NULL); // 32 ranks, 4 nodes for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } pluginGetCollInfo(context2, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); TEST_ASSERT(cost_table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] == 0.0, "4-node: Should match ring/simple config"); TEST_ASSERT(nChannels == 4, "4-node: Should set 4 channels"); // Clean up unlink("test_topo.conf"); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 7: Default channels behavior (-1) int test_default_channels() { const char* test_config = "allreduce,0,65536,tree,simple,-1,-1,-1,-1,-1\n"; // Use default channels create_test_config("test_default.conf", test_config); setenv("NCCL_TUNER_CONFIG_FILE", "test_default.conf", 1); void* context = NULL; pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL); float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float* cost_table_ptr[NCCL_NUM_ALGORITHMS]; for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { cost_table_ptr[i] = cost_table[i]; for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } int nChannels = 99; // Set to known value pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); TEST_ASSERT(cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE] == 0.0, "Should apply algorithm/protocol"); TEST_ASSERT(nChannels == 1, "Should keep default channels (1) when config has -1"); // Clean up pluginFinalize(context); unlink("test_default.conf"); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 8: regBuff matching int test_regbuff_matching() { const char* test_config = "allreduce,0,65536,tree,simple,2,-1,-1,-1,1\n" // Registered buffers only "allreduce,0,65536,ring,simple,4,-1,-1,-1,0\n" // Non-registered buffers only "allreduce,0,65536,ring,ll128,8,-1,-1,-1,-1\n"; // Any buffer type (backward compatible) create_test_config("test_regbuff.conf", test_config); setenv("NCCL_TUNER_CONFIG_FILE", "test_regbuff.conf", 1); void* context = NULL; pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL); float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float* cost_table_ptr[NCCL_NUM_ALGORITHMS]; for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { cost_table_ptr[i] = cost_table[i]; } int nChannels; // Test registered buffer (should match first config) for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 1, &nChannels); // regBuff = 1 (registered) TEST_ASSERT(cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE] == 0.0, "Registered buffer: Tree/Simple should have low cost"); TEST_ASSERT(nChannels == 2, "Registered buffer: Should set 2 channels"); // Test non-registered buffer (should match second config) for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); // regBuff = 0 (non-registered) TEST_ASSERT(cost_table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] == 0.0, "Non-registered buffer: Ring/Simple should have low cost"); TEST_ASSERT(nChannels == 4, "Non-registered buffer: Should set 4 channels"); // Test backward compatibility - config without regBuff should match any regBuff value for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } // First try with regBuff=2 (unusual value, should match third config) pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 2, &nChannels); // regBuff = 2 (only third config should match) TEST_ASSERT(cost_table[NCCL_ALGO_RING][NCCL_PROTO_LL128] == 0.0, "Any regBuff: Ring/LL128 should have low cost"); TEST_ASSERT(nChannels == 8, "Any regBuff: Should set 8 channels"); // Clean up pluginFinalize(context); unlink("test_regbuff.conf"); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 9: numPipeOps matching int test_pipeops_matching() { const char* test_config = "allreduce,0,65536,tree,simple,2,-1,-1,1,-1\n" // Single pipeline op "allreduce,0,65536,ring,simple,4,-1,-1,4,-1\n" // Multiple pipeline ops "allreduce,0,65536,ring,ll128,8,-1,-1,-1,-1\n"; // Any pipeline ops (backward compatible) create_test_config("test_pipeops.conf", test_config); setenv("NCCL_TUNER_CONFIG_FILE", "test_pipeops.conf", 1); void* context = NULL; pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL); float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float* cost_table_ptr[NCCL_NUM_ALGORITHMS]; for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { cost_table_ptr[i] = cost_table[i]; } int nChannels; // Test single pipeline op (should match first config) for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); TEST_ASSERT(cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE] == 0.0, "Single pipeOp: Tree/Simple should have low cost"); TEST_ASSERT(nChannels == 2, "Single pipeOp: Should set 2 channels"); // Test multiple pipeline ops (should match second config) for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 4, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); TEST_ASSERT(cost_table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] == 0.0, "Multiple pipeOps: Ring/Simple should have low cost"); TEST_ASSERT(nChannels == 4, "Multiple pipeOps: Should set 4 channels"); // Test different number of pipeline ops (should match third config - backward compatible) for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 2, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); TEST_ASSERT(cost_table[NCCL_ALGO_RING][NCCL_PROTO_LL128] == 0.0, "Any pipeOps: Ring/LL128 should have low cost"); TEST_ASSERT(nChannels == 8, "Any pipeOps: Should set 8 channels"); // Clean up pluginFinalize(context); unlink("test_pipeops.conf"); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 10: No matching configuration (fallback behavior) int test_no_match_fallback() { const char* test_config = "broadcast,0,1024,tree,simple,2,-1,-1,-1,-1\n"; // Only broadcast config create_test_config("test_fallback.conf", test_config); setenv("NCCL_TUNER_CONFIG_FILE", "test_fallback.conf", 1); void* context = NULL; pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL); float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float* cost_table_ptr[NCCL_NUM_ALGORITHMS]; for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { cost_table_ptr[i] = cost_table[i]; for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } int nChannels; // Try allreduce (should not match, use fallback) pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); mock_logger(NCCL_LOG_INFO, NCCL_ALL, __FILE__, __LINE__, "DEBUG: Fallback test - checking cost_table[RING][SIMPLE] (%p) = %.1f (expecting 0.0)", &cost_table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE], cost_table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE]); TEST_ASSERT(cost_table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] == 1.0, "Should use pass through unmodified"); TEST_ASSERT(nChannels == 1, "Should use default channels"); // Clean up pluginFinalize(context); unlink("test_fallback.conf"); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 11: Large configuration files (testing dynamic allocation) int test_large_config() { const char* large_config_file = "test_large.conf"; // Create a large configuration file with many entries // This tests the dynamic allocation functionality FILE* f = fopen(large_config_file, "w"); TEST_ASSERT(f != NULL, "Should be able to create large config file"); // Write header comment fprintf(f, "# Large configuration file for testing dynamic allocation\n"); fprintf(f, "# This file contains many configurations to test memory allocation\n"); // Generate a large number of configurations (much more than the old MAX_CONFIGS=100) const int num_configs = 500; // 5x the old static limit const char* collectives[] = {"allreduce", "broadcast", "reduce", "allgather", "reducescatter"}; const char* algorithms[] = {"tree", "ring", "collnet_direct", "nvls"}; const char* protocols[] = {"simple", "ll", "ll128"}; for (int i = 0; i < num_configs; i++) { // Vary the configurations to create realistic test data const char* coll = collectives[i % 5]; const char* algo = algorithms[i % 4]; const char* proto = protocols[i % 3]; size_t min_bytes = (i * 1024) % 1048576; // Vary from 0 to 1MB size_t max_bytes = min_bytes + 65536; // 64KB range int channels = (i % 8) + 1; // 1-8 channels int nodes = (i % 4) == 0 ? -1 : (i % 4); // Mix of -1 and 1-3 nodes int ranks = (i % 8) == 0 ? -1 : (i % 32) + 1; // Mix of -1 and 1-32 ranks int pipeOps = (i % 3) == 0 ? -1 : (i % 4) + 1; // Mix of -1 and 1-4 pipeOps int regBuff = (i % 3) == 0 ? -1 : (i % 2); // Mix of -1, 0, 1 fprintf(f, "%s,%zu,%zu,%s,%s,%d,%d,%d,%d,%d\n", coll, min_bytes, max_bytes, algo, proto, channels, nodes, ranks, pipeOps, regBuff); } fclose(f); // Set environment to use our large config file setenv("NCCL_TUNER_CONFIG_FILE", large_config_file, 1); // Initialize plugin with large config void* context = NULL; ncclResult_t result = pluginInit(&context, 0, 16, 4, mock_logger, NULL, NULL); TEST_ASSERT(result == ncclSuccess, "Plugin init with large config should succeed"); TEST_ASSERT(context != NULL, "Context should be allocated"); // Verify that configurations were loaded TunerContext* ctx = (TunerContext*)context; TEST_ASSERT(ctx->numConfigs == num_configs, "Should load all configurations from large file"); TEST_ASSERT(ctx->maxConfigs == num_configs, "maxConfigs should match allocated size"); TEST_ASSERT(ctx->configs != NULL, "Configs array should be dynamically allocated"); // Test that we can access configurations throughout the array // (This would have failed with the old static MAX_CONFIGS=100 limit) for (int i = 0; i < ctx->numConfigs; i++) { TuningConfig* config = &ctx->configs[i]; // Basic sanity checks on the loaded configurations TEST_ASSERT(config->collType >= ncclFuncBroadcast && config->collType <= ncclFuncAllReduce, "Collective type should be valid"); TEST_ASSERT(config->maxBytes >= config->minBytes, "maxBytes should be >= minBytes"); TEST_ASSERT(config->nChannels > 0, "nChannels should be positive"); } // Test specific configuration access at various indices // Index 0 (first config) TuningConfig* first_config = &ctx->configs[0]; TEST_ASSERT(first_config != NULL, "First config should be accessible"); // Index in middle TuningConfig* mid_config = &ctx->configs[num_configs / 2]; TEST_ASSERT(mid_config != NULL, "Middle config should be accessible"); // Index near end (this would have crashed with static array of 100) TuningConfig* late_config = &ctx->configs[num_configs - 1]; TEST_ASSERT(late_config != NULL, "Last config should be accessible"); // Test memory allocation size - verify we didn't over-allocate mock_logger(NCCL_LOG_INFO, NCCL_ALL, __FILE__, __LINE__, "Successfully loaded %d configurations (dynamic allocation)", ctx->numConfigs); mock_logger(NCCL_LOG_INFO, NCCL_ALL, __FILE__, __LINE__, "Memory allocated for %d configurations (%zu bytes total)", ctx->maxConfigs, ctx->maxConfigs * sizeof(TuningConfig)); // Test that the plugin can still find matching configurations from the large set float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float* cost_table_ptr[NCCL_NUM_ALGORITHMS]; for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { cost_table_ptr[i] = cost_table[i]; for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; // Default high cost } } int nChannels; // Try to find a matching configuration - should work with large config set result = pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); TEST_ASSERT(result == ncclSuccess, "GetCollInfo should work with large config set"); // Clean up pluginFinalize(context); unlink(large_config_file); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 12: Very large configuration stress test int test_very_large_config_stress() { const char* stress_config_file = "test_stress.conf"; // Create an even larger configuration file to stress test the implementation FILE* f = fopen(stress_config_file, "w"); TEST_ASSERT(f != NULL, "Should be able to create stress test config file"); fprintf(f, "# Stress test configuration with very large number of entries\n"); // Generate an extremely large number of configurations const int stress_configs = 2000; // 20x the old static limit for (int i = 0; i < stress_configs; i++) { // Create varied but valid configurations fprintf(f, "allreduce,%d,%d,ring,simple,4,-1,-1,-1,-1\n", i * 512, (i * 512) + 1024); } fclose(f); setenv("NCCL_TUNER_CONFIG_FILE", stress_config_file, 1); // Test initialization with stress config void* context = NULL; ncclResult_t result = pluginInit(&context, 0, 8, 2, mock_logger, NULL, NULL); TEST_ASSERT(result == ncclSuccess, "Plugin should handle very large config files"); TunerContext* ctx = (TunerContext*)context; TEST_ASSERT(ctx->numConfigs == stress_configs, "Should load all stress test configurations"); TEST_ASSERT(ctx->configs != NULL, "Stress test configs should be allocated"); mock_logger(NCCL_LOG_INFO, NCCL_ALL, __FILE__, __LINE__, "Stress test - loaded %d configurations successfully", stress_configs); mock_logger(NCCL_LOG_INFO, NCCL_ALL, __FILE__, __LINE__, "Memory usage: %zu bytes for configuration array", stress_configs * sizeof(TuningConfig)); // Verify we can access configurations throughout the entire range for (int i = 0; i < stress_configs; i += 100) { // Sample every 100th config TuningConfig* config = &ctx->configs[i]; TEST_ASSERT(config->collType == ncclFuncAllReduce, "Config should have correct collective type"); TEST_ASSERT(config->minBytes == (size_t)(i * 512), "Config should have correct minBytes"); } // Clean up pluginFinalize(context); unlink(stress_config_file); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test 13: Edge case - empty config file int test_empty_config() { const char* empty_config_file = "test_empty.conf"; // Create empty config file (only comments) create_test_config(empty_config_file, "# Empty configuration file\n" "# No actual configurations\n" "\n" "\n"); setenv("NCCL_TUNER_CONFIG_FILE", empty_config_file, 1); void* context = NULL; ncclResult_t result = pluginInit(&context, 0, 8, 2, mock_logger, NULL, NULL); TEST_ASSERT(result == ncclSuccess, "Plugin should handle empty config files"); TunerContext* ctx = (TunerContext*)context; TEST_ASSERT(ctx->numConfigs == 0, "Should have zero configurations"); TEST_ASSERT(ctx->maxConfigs == 0, "Should have zero max configurations"); TEST_ASSERT(ctx->configs == NULL, "Should not allocate memory for empty config"); // Test that plugin still works with no configurations (fallback behavior) float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float* cost_table_ptr[NCCL_NUM_ALGORITHMS]; for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) { cost_table_ptr[i] = cost_table[i]; for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) { cost_table[i][j] = 1.0; } } int nChannels; result = pluginGetCollInfo(context, ncclFuncAllReduce, 32768, 1, cost_table_ptr, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, 0, &nChannels); TEST_ASSERT(result == ncclSuccess, "GetCollInfo should work with empty config"); // Clean up pluginFinalize(context); unlink(empty_config_file); unsetenv("NCCL_TUNER_CONFIG_FILE"); TEST_PASS(); } // Test NVLink domain info handling int test_nvl_domain_info() { printf("Testing NVLink domain info handling...\n"); // Test NVLink domain structure with min/max ranks per domain ncclNvlDomainInfo_v5_t nvl_domain = { .nNvlDomains = 2, // 2 nodes = 2 domains .minRanksPerNvlDomain = 3, // minimum ranks across all domains (bottleneck) .maxRanksPerNvlDomain = 5 // maximum ranks across all domains (capacity) }; void* context = NULL; ncclResult_t result = pluginInit(&context, 0, 8, 2, mock_logger, &nvl_domain, NULL); TEST_ASSERT(result == ncclSuccess, "Plugin init with NVLink domains should succeed"); // Validate NVLD info structure TEST_ASSERT(nvl_domain.nNvlDomains == 2, "Should have 2 domains (nodes)"); TEST_ASSERT(nvl_domain.minRanksPerNvlDomain == 3, "Should have minimum 3 ranks per domain"); TEST_ASSERT(nvl_domain.maxRanksPerNvlDomain == 5, "Should have maximum 5 ranks per domain"); // Clean up pluginFinalize(context); printf("NVLink domain info test passed!\n"); TEST_PASS(); } int test_tuner_constants() { // Initialize constants to -1.0 for testing purposes ncclTunerConstants_v5_t constants = { // Base latencies: [NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] .baseLatencies = { {-1.0, -1.0, -1.0}, // NCCL_ALGO_TREE: LL, LL128, Simple {-1.0, -1.0, -1.0}, // NCCL_ALGO_RING: LL, LL128, Simple {-1.0, -1.0, -1.0}, // NCCL_ALGO_COLLNET_DIRECT {-1.0, -1.0, -1.0}, // NCCL_ALGO_COLLNET_CHAIN {-1.0, -1.0, -1.0}, // NCCL_ALGO_NVLS {-1.0, -1.0, -1.0}, // NCCL_ALGO_NVLS_TREE {-1.0, -1.0, -1.0} // NCCL_ALGO_PAT }, // Hardware latencies: [NCCL_NUM_HW_LINKS][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS] .hwLatencies = { // NCCL_HW_NVLINK { {-1.0, -1.0, -1.0}, // TREE {-1.0, -1.0, -1.0}, // RING {-1.0, -1.0, -1.0}, // COLLNET_DIRECT {-1.0, -1.0, -1.0}, // COLLNET_CHAIN {-1.0, -1.0, -1.0}, // NVLS {-1.0, -1.0, -1.0}, // NVLS_TREE {-1.0, -1.0, -1.0} // PAT }, // NCCL_HW_PCI { {-1.0, -1.0, -1.0}, // TREE {-1.0, -1.0, -1.0}, // RING {-1.0, -1.0, -1.0}, // COLLNET_DIRECT {-1.0, -1.0, -1.0}, // COLLNET_CHAIN {-1.0, -1.0, -1.0}, // NVLS {-1.0, -1.0, -1.0}, // NVLS_TREE {-1.0, -1.0, -1.0} // PAT }, // NCCL_HW_NET { {-1.0, -1.0, -1.0}, // TREE {-1.0, -1.0, -1.0}, // RING {-1.0, -1.0, -1.0}, // COLLNET_DIRECT {-1.0, -1.0, -1.0}, // COLLNET_CHAIN {-1.0, -1.0, -1.0}, // NVLS {-1.0, -1.0, -1.0}, // NVLS_TREE {-1.0, -1.0, -1.0} // PAT } }, // LL maximum bandwidths: [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES] .llMaxBws = { {-1.0, -1.0, -1.0}, // Volta: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0}, // Ampere: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0}, // Hopper: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0} // Blackwell: 1node, 2nodes, 4nodes }, // Per-channel maximum Ring LL128 bandwidths: [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES] .perChMaxRingLL128Bws = { {-1.0, -1.0, -1.0}, // Volta: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0}, // Ampere: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0}, // Hopper: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0} // Blackwell: 1node, 2nodes, 4nodes }, // Per-channel maximum Tree LL128 bandwidths: [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES] .perChMaxTreeLL128Bws = { {-1.0, -1.0, -1.0}, // Volta: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0}, // Ampere: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0}, // Hopper: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0} // Blackwell: 1node, 2nodes, 4nodes }, // Per-channel maximum Tree bandwidths: [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES] .perChMaxTreeBws = { {-1.0, -1.0, -1.0}, // Volta: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0}, // Ampere: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0}, // Hopper: 1node, 2nodes, 4nodes {-1.0, -1.0, -1.0} // Blackwell: 1node, 2nodes, 4nodes } }; void* context = NULL; ncclResult_t result = pluginInit(&context, 0, 8, 2, mock_logger, NULL, &constants); TEST_ASSERT(result == ncclSuccess, "Plugin init with constants should succeed"); // Test that the constants were set correctly TEST_ASSERT(constants.perChMaxTreeBws[NCCL_BLACKWELL_COMPCAP_IDX][NCCL_TUNING_SCALE_4NODES] == 15.0, "Tree bandwidth should be 15GB/s"); TEST_ASSERT(constants.perChMaxRingLL128Bws[NCCL_BLACKWELL_COMPCAP_IDX][NCCL_TUNING_SCALE_4NODES] == 20.0, "Ring bandwidth should be 20GB/s"); TEST_ASSERT(constants.hwLatencies[NCCL_HW_NET][NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] == 24.0, "NVLSTree base network latency should be 24us"); // Clean up pluginFinalize(context); TEST_PASS(); } // Test runner function pointer type typedef int (*TestFunction)(void); // Test registry typedef struct { const char* name; TestFunction func; const char* description; } TestCase; // All available tests TestCase test_cases[] = { {"init", test_plugin_init, "Plugin initialization"}, {"config-valid", test_config_parsing_valid, "Valid configuration parsing"}, {"config-invalid", test_config_parsing_invalid, "Invalid configuration parsing"}, {"collective", test_collective_matching, "Collective type matching"}, {"size", test_size_matching, "Size range matching"}, {"topology", test_topology_matching, "Topology matching"}, {"channels", test_default_channels, "Default channels behavior"}, {"regbuff", test_regbuff_matching, "Registered buffer matching"}, {"pipeops", test_pipeops_matching, "Pipeline operations matching"}, {"fallback", test_no_match_fallback, "Fallback behavior"}, {"large-config", test_large_config, "Large configuration files (dynamic allocation)"}, {"stress-config", test_very_large_config_stress, "Very large configuration stress test"}, {"empty-config", test_empty_config, "Empty configuration file handling"}, {"nvl-domain", test_nvl_domain_info, "NVL domain info handling"}, {"constants", test_tuner_constants, "Tuner constants initialization"}, {NULL, NULL, NULL} // End marker }; // Show help/usage information void show_help(const char* program_name) { printf("Usage: %s [test_name ...]\n\n", program_name); printf("Available tests:\n"); for (int i = 0; test_cases[i].name != NULL; i++) { printf(" %-15s - %s\n", test_cases[i].name, test_cases[i].description); } printf("\nExamples:\n"); printf(" %s # Run all tests\n", program_name); printf(" %s init # Run only initialization test\n", program_name); printf(" %s init collective # Run initialization and collective tests\n", program_name); printf(" %s --help # Show this help\n", program_name); } // Find test by name TestFunction find_test(const char* name) { for (int i = 0; test_cases[i].name != NULL; i++) { if (strcmp(test_cases[i].name, name) == 0) { return test_cases[i].func; } } return NULL; } // Main test runner int main(int argc, char* argv[]) { int passed = 0, total = 0; // Check for help if (argc > 1 && (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0)) { show_help(argv[0]); return 0; } printf("Running NCCL Tuner Plugin Unit Tests\n"); printf("=====================================\n"); if (argc == 1) { // No arguments - run all tests for (int i = 0; test_cases[i].name != NULL; i++) { printf("Running test: %s\n", test_cases[i].name); total++; passed += test_cases[i].func(); } } else { // Run specific tests for (int arg = 1; arg < argc; arg++) { TestFunction test_func = find_test(argv[arg]); if (test_func) { total++; passed += test_func(); } else { printf("ERROR: Unknown test '%s'\n", argv[arg]); printf("Use --help to see available tests\n"); return 1; } } } printf("\n=====================================\n"); printf("Test Results: %d/%d tests passed\n", passed, total); if (passed == total) { printf("All tests PASSED!\n"); return 0; } else { printf("Some tests FAILED!\n"); return 1; } } nccl-2.29.7-1/src/000077500000000000000000000000001515037102200134535ustar00rootroot00000000000000nccl-2.29.7-1/src/CMakeLists.txt000066400000000000000000000176671515037102200162340ustar00rootroot00000000000000# Source files set(LIBSRCFILES bootstrap.cc channel.cc ce_coll.cc collectives.cc debug.cc enqueue.cc group.cc init.cc proxy.cc transport.cc mnnvl.cc allocator.cc sym_kernels.cc dev_runtime.cc mem_manager.cc ) # Add NVTX support if enabled if(NVTX) list(APPEND LIBSRCFILES init_nvtx.cc) endif() # Add compatibility shim if using static cudart if(CUDARTLIB STREQUAL "cudart_static") list(APPEND LIBSRCFILES enhcompat.cc) endif() # Configure pkg-config file configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/nccl.pc.in ${CMAKE_BINARY_DIR}/lib/pkgconfig/nccl.pc @ONLY ) # Add files from subdirectories add_subdirectory(transport) add_subdirectory(misc) add_subdirectory(register) add_subdirectory(graph) add_subdirectory(plugin) add_subdirectory(device) add_subdirectory(nccl_device) add_subdirectory(ras) add_subdirectory(scheduler) add_subdirectory(os) add_subdirectory(gin) add_subdirectory(rma) if(NCCL_OS_LINUX) add_compile_options(-fmacro-prefix-map=${CMAKE_CURRENT_SOURCE_DIR}/=) endif() # Add all source files list(APPEND LIBSRCFILES ${TRANSPORT_SOURCES} ${MISC_SOURCES} ${REGISTER_SOURCES} ${GRAPH_SOURCES} ${PLUGIN_SOURCES} ${RAS_SOURCES} ${SYM_SOURCES} ${SCHEDULER_SOURCES} ${OS_SOURCES} ${GIN_SOURCES} ${DOCA_SOURCES} ${RMA_SOURCES} ) ###################### Create a shared NCCL library ############################ add_library(nccl SHARED) target_sources(nccl PRIVATE ${LIBSRCFILES}) target_sources(nccl PRIVATE $) # Include directories target_include_directories(nccl PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/device ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/include/plugin ${DOCA_HOME}/include ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_INCLUDE_DIRS}/cccl ${CMAKE_BINARY_DIR}/obj/include ) add_custom_command( OUTPUT ${CMAKE_BINARY_DIR}/include/nccl.h COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/include COMMAND sed -e "s/\\\$$\\{nccl:Major\\}/${NCCL_MAJOR}/g" -e "s/\\\$$\\{nccl:Minor\\}/${NCCL_MINOR}/g" -e "s/\\\$$\\{nccl:Patch\\}/${NCCL_PATCH}/g" -e "s/\\\$$\\{nccl:Suffix\\}/${NCCL_SUFFIX}/g" -e "s/\\\$$\\{nccl:Version\\}/${NCCL_VERSION_CODE}/g" ${CMAKE_CURRENT_SOURCE_DIR}/nccl.h.in > ${CMAKE_BINARY_DIR}/include/nccl.h DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/nccl.h.in ) file(GLOB_RECURSE SRC_DEVICE_HEADERS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include/nccl_device/*.h) # Copy all device header files to the destination foreach(HEADER_FILE ${SRC_DEVICE_HEADERS}) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${HEADER_FILE} ${CMAKE_BINARY_DIR}/${HEADER_FILE} COPYONLY) list(APPEND DEVICE_HEADERS ${CMAKE_BINARY_DIR}/${HEADER_FILE}) endforeach() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/nccl_device.h ${CMAKE_BINARY_DIR}/include/nccl_device.h COPYONLY) add_custom_target(nccl_header DEPENDS ${CMAKE_BINARY_DIR}/include/nccl.h ${CMAKE_BINARY_DIR}/include/nccl_device.h ${DEVICE_HEADERS} ${DEVICE_DOCA_HEADERS} ) add_dependencies(nccl nccl_header) add_dependencies(nccl_device nccl_header) # Set version and output name set_target_properties(nccl PROPERTIES VERSION ${NCCL_MAJOR}.${NCCL_MINOR}.${NCCL_PATCH} SOVERSION ${NCCL_MAJOR} OUTPUT_NAME "nccl" PREFIX "lib" ) # # Generate nccl_git_version.h # set(GIT_VERSION_FILE "${CMAKE_BINARY_DIR}/obj/include/nccl_git_version.h") file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/obj/include") add_custom_target(generate_git_version ALL COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/src/misc/generate_git_version.py ${GIT_VERSION_FILE} COMMENT "Checking git version" BYPRODUCTS ${GIT_VERSION_FILE} ) add_dependencies(nccl generate_git_version) # Git version overrides (set via cmake -DNCCL_GIT_BRANCH=xxx -DNCCL_GIT_COMMIT_HASH=yyy) set(NCCL_GIT_BRANCH "" CACHE STRING "Override git branch name") set(NCCL_GIT_COMMIT_HASH "" CACHE STRING "Override git commit hash") if(NCCL_GIT_BRANCH) target_compile_definitions(nccl PRIVATE NCCL_GIT_BRANCH="${NCCL_GIT_BRANCH}") endif() if(NCCL_GIT_COMMIT_HASH) target_compile_definitions(nccl PRIVATE NCCL_GIT_COMMIT_HASH="${NCCL_GIT_COMMIT_HASH}") endif() # Set CUDA specific flags set_target_properties(nccl PROPERTIES CUDA_SEPARABLE_COMPILATION ON CUDA_RESOLVE_DEVICE_SYMBOLS ON CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}" POSITION_INDEPENDENT_CODE ON ) # Link libraries target_link_libraries(nccl PRIVATE Threads::Threads ${CMAKE_DL_LIBS} CUDA::cudart ${EXTRA_LIBS} ) # Add version script for symbol visibility control target_link_options(nccl PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/libnccl.map" ) # Set output directories for nccl shared library set_target_properties(nccl PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" ) ###################### Create a ras binary executable ############################ set(RAS_BINSRCFILES ras/client.cc) add_executable(ncclras ${RAS_BINSRCFILES}) target_include_directories(ncclras PUBLIC $ $ PRIVATE ${CUDAToolkit_INCLUDE_DIRS} ) add_dependencies(ncclras nccl_header) target_link_libraries(ncclras PRIVATE pthread ${CMAKE_DL_LIBS} ) # Set output directory for ncclras executable set_target_properties(ncclras PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" ) ###################### Create a static NCCL library ############################ add_library(nccl_static STATIC ${LIBSRCFILES}) target_sources(nccl_static PRIVATE $) # Include directories target_include_directories(nccl_static PUBLIC $ $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/device ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/include/plugin transport/net_ib/gdaki/doca-gpunetio/include ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_INCLUDE_DIRS}/cccl ${CMAKE_BINARY_DIR}/obj/include ) # Add dependency on nccl_header add_dependencies(nccl_static nccl_header) add_dependencies(nccl_static generate_git_version) # Git version overrides for static lib if(NCCL_GIT_BRANCH) target_compile_definitions(nccl_static PRIVATE NCCL_GIT_BRANCH="${NCCL_GIT_BRANCH}") endif() if(NCCL_GIT_COMMIT_HASH) target_compile_definitions(nccl_static PRIVATE NCCL_GIT_COMMIT_HASH="${NCCL_GIT_COMMIT_HASH}") endif() # Link libraries target_link_libraries(nccl_static PUBLIC Threads::Threads ${CMAKE_DL_LIBS} CUDA::cudart ${EXTRA_LIBS} ) # Set CUDA specific flags set_target_properties(nccl_static PROPERTIES CUDA_SEPARABLE_COMPILATION ON CUDA_RESOLVE_DEVICE_SYMBOLS ON CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}" POSITION_INDEPENDENT_CODE ON ) # Set output directory for nccl_static library set_target_properties(nccl_static PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" ) ###################### Install targets / headers / pkg-config ############################ install(TARGETS nccl nccl_static EXPORT NCCLTargets RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" ) # Install tools, but do not export them as linkable imported targets. install(TARGETS ncclras RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" ) # Install generated + copied public headers from the build tree. install(DIRECTORY "${CMAKE_BINARY_DIR}/include/" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" ) # Install pkg-config metadata generated during configure. install(FILES "${CMAKE_BINARY_DIR}/lib/pkgconfig/nccl.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" ) nccl-2.29.7-1/src/Makefile000066400000000000000000000213651515037102200151220ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # include ../makefiles/common.mk include ../makefiles/version.mk ##### src files INCEXPORTS := nccl.h nccl_device.h \ $(patsubst include/%,%,$(wildcard include/nccl_device/*.h include/nccl_device/*/*.h include/nccl_device/*/*/*.h)) LIBSRCFILES := \ bootstrap.cc channel.cc collectives.cc debug.cc enqueue.cc group.cc \ init.cc proxy.cc transport.cc mnnvl.cc allocator.cc dev_runtime.cc sym_kernels.cc ce_coll.cc mem_manager.cc \ $(wildcard graph/*.cc) \ $(wildcard misc/*.cc) \ $(wildcard transport/*.cc) \ $(wildcard transport/net_ib/*.cc) \ $(wildcard transport/net_ib/gdaki/*.cc) \ $(wildcard register/*.cc) \ $(wildcard plugin/*.cc) \ $(wildcard plugin/net/*.cc) \ $(wildcard plugin/gin/*.cc) \ $(wildcard plugin/tuner/*.cc) \ $(wildcard plugin/profiler/*.cc) \ $(wildcard plugin/env/*.cc) \ $(wildcard nccl_device/*.cc) \ $(wildcard scheduler/*.cc) \ $(wildcard gin/*.cc) \ $(wildcard rma/*.cc) \ $(filter-out ras/client.cc,$(wildcard ras/*.cc)) BINSRCFILES := ras/client.cc ifeq ($(NCCL_OS_LINUX), 1) LIBSRCFILES += os/linux.cc else ifeq ($(NCCL_OS_WINDOWS), 1) LIBSRCFILES += os/windows.cc endif ifneq ($(NVTX), 0) LIBSRCFILES += init_nvtx.cc endif ##### lib files LIBNAME := libnccl.so STATICLIBNAME := libnccl_static.a ##### binaries BINNAME := ncclras ##### pkgconfig files PKGCONFIGFILE := nccl.pc ##### dirs BUILDDIR ?= $(abspath ../build) INCDIR := $(BUILDDIR)/include LIBDIR := $(BUILDDIR)/lib OBJDIR := $(BUILDDIR)/obj PKGDIR := $(BUILDDIR)/lib/pkgconfig BINDIR := $(BUILDDIR)/bin ##### target files CUDARTLIB ?= cudart_static # Use compatibility shim only with static cudart; see https://github.com/NVIDIA/nccl/issues/658 ifeq ($(CUDARTLIB), cudart_static) LIBSRCFILES += enhcompat.cc endif INCTARGETS := $(INCEXPORTS:%=$(INCDIR)/%) LIBSONAME := $(LIBNAME:%=%.$(NCCL_MAJOR)) LIBTARGET := $(LIBNAME:%=%.$(NCCL_MAJOR).$(NCCL_MINOR).$(NCCL_PATCH)) STATICLIBTARGET := $(STATICLIBNAME) PKGTARGET := $(PKGCONFIGFILE) LIBOBJ := $(LIBSRCFILES:%.cc=$(OBJDIR)/%.o) BINOBJ := $(BINSRCFILES:%.cc=$(OBJDIR)/%.o) DEPFILES := $(LIBOBJ:%.o=%.d) $(BINOBJ:%.o=%.d) LDFLAGS += -L${CUDA_LIB} -l$(CUDARTLIB) -lpthread -lrt -ldl INCPLUGIN := include/plugin DEVMANIFEST := $(BUILDDIR)/obj/device/manifest GIT_VERSION_FILE := $(OBJDIR)/include/nccl_git_version.h $(GIT_VERSION_FILE): ALWAYS_REBUILD @mkdir -p $(dir $@) @./misc/generate_git_version.py $@ # DOCA GPUNetIO definitions DOCA_HOME ?= transport/net_ib/gdaki/doca-gpunetio DOCA_INC_INSTALL := $(INCDIR)/nccl_device/gin/gdaki/doca_gpunetio DOCA_OBJDIR := $(OBJDIR)/transport/net_ib/gdaki/doca-gpunetio DOCA_INCLUDES := $(DOCA_HOME)/include/doca_gpunetio_device.h $(wildcard $(DOCA_HOME)/include/common/*.h) $(wildcard $(DOCA_HOME)/include/device/*.cuh) DOCA_INCTARGETS := $(DOCA_INCLUDES:$(DOCA_HOME)/include/%=$(DOCA_INC_INSTALL)/%) INCTARGETS += $(DOCA_INCTARGETS) DOCA_LIBSRC := doca_verbs_qp.cpp doca_verbs_cq.cpp doca_verbs_device_attr.cpp doca_verbs_umem.cpp doca_verbs_srq.cpp doca_verbs_uar.cpp doca_gpunetio.cpp doca_gpunetio_log.cpp doca_gpunetio_high_level.cpp doca_verbs_cuda_wrapper.cpp doca_verbs_mlx5dv_wrapper.cpp doca_verbs_ibv_wrapper.cpp doca_gpunetio_gdrcopy.cpp DOCA_LIBOBJ := $(DOCA_LIBSRC:%.cpp=$(DOCA_OBJDIR)/%.o) LIBOBJ += $(DOCA_LIBOBJ) ##### rules build : lib staticlib binary lib : $(INCTARGETS) $(LIBDIR)/$(LIBTARGET) $(PKGDIR)/$(PKGTARGET) staticlib : $(LIBDIR)/$(STATICLIBTARGET) binary : $(BINDIR)/$(BINNAME) $(DEVMANIFEST): ALWAYS_REBUILD $(INCTARGETS) $(MAKE) -C ./device # Empty target to force rebuild .PHONY: ALWAYS_REBUILD -include $(DEPFILES) $(LIBDIR)/$(LIBTARGET) $(LIBDIR)/$(STATICLIBTARGET) : $(LIBOBJ) $(INCDIR)/nccl.h : nccl.h.in ../makefiles/version.mk # NCCL_VERSION(X,Y,Z) ((X) * 10000 + (Y) * 100 + (Z)) @$(eval NCCL_VERSION := $(shell printf "%d%02d%02d" $(NCCL_MAJOR) $(NCCL_MINOR) $(NCCL_PATCH))) mkdir -p $(INCDIR) @printf "Generating %-35s > %s\n" $< $@ sed -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \ -e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \ -e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \ -e "s/\$${nccl:Suffix}/$(NCCL_SUFFIX)/g" \ -e "s/\$${nccl:Version}/$(NCCL_VERSION)/g" \ $< > $@ $(LIBDIR)/$(LIBTARGET): $(LIBOBJ) $(DEVMANIFEST) @printf "Linking %-35s > %s\n" $(LIBTARGET) $@ mkdir -p $(LIBDIR) $(CXX) $(CXXFLAGS) -shared -Wl,--no-as-needed -Wl,-soname,$(LIBSONAME) -o $@ $(LIBOBJ) $$(cat $(DEVMANIFEST)) $(LDFLAGS) -Wl,--version-script=libnccl.map ln -sf $(LIBSONAME) $(LIBDIR)/$(LIBNAME) ln -sf $(LIBTARGET) $(LIBDIR)/$(LIBSONAME) $(LIBDIR)/$(STATICLIBTARGET): $(LIBOBJ) $(DEVMANIFEST) @printf "Archiving %-35s > %s\n" $(STATICLIBTARGET) $@ mkdir -p $(LIBDIR) ar cr $@ $(LIBOBJ) $$(cat $(DEVMANIFEST)) $(BINDIR)/$(BINNAME): $(BINOBJ) @printf "Linking %-35s > %s\n" $(BINNAME) $@ mkdir -p $(BINDIR) $(CXX) $(CXXFLAGS) $^ -o $@ $(PKGDIR)/nccl.pc : nccl.pc.in mkdir -p $(PKGDIR) @printf "Generating %-35s > %s\n" $< $@ sed -e 's|$${nccl:Prefix}|\$(PREFIX)|g' \ -e "s/\$${nccl:Major}/$(NCCL_MAJOR)/g" \ -e "s/\$${nccl:Minor}/$(NCCL_MINOR)/g" \ -e "s/\$${nccl:Patch}/$(NCCL_PATCH)/g" \ $< > $@ $(INCDIR)/%.h : %.h @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(INCDIR) install -m 644 $< $@ $(INCDIR)/nccl_%.h : include/nccl_%.h @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(INCDIR) install -m 644 $< $@ $(INCDIR)/nccl_device/%.h: include/nccl_device/%.h @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(INCDIR)/nccl_device install -m 644 $< $@ $(INCDIR)/nccl_device/impl/%.h: include/nccl_device/impl/%.h @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(INCDIR)/nccl_device/impl install -m 644 $< $@ $(INCDIR)/nccl_device/gin/%.h: include/nccl_device/gin/%.h @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(INCDIR)/nccl_device/gin install -m 644 $< $@ $(INCDIR)/nccl_device/gin/gdaki/%.h: include/nccl_device/gin/gdaki/%.h @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(INCDIR)/nccl_device/gin/gdaki install -m 644 $< $@ $(INCDIR)/nccl_device/gin/proxy/%.h: include/nccl_device/gin/proxy/%.h @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(INCDIR)/nccl_device/gin/proxy install -m 644 $< $@ $(DOCA_INC_INSTALL)/%.h: $(DOCA_HOME)/include/%.h @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(DOCA_INC_INSTALL) install -m 644 $< $@ $(DOCA_INC_INSTALL)/common/%.h: $(DOCA_HOME)/include/common/%.h @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(DOCA_INC_INSTALL)/common install -m 644 $< $@ $(DOCA_INC_INSTALL)/device/%.cuh: $(DOCA_HOME)/include/device/%.cuh @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(DOCA_INC_INSTALL)/device install -m 644 $< $@ $(PKGDIR)/%.pc : %.pc @printf "Grabbing %-35s > %s\n" $< $@ mkdir -p $(PKGDIR) install -m 644 $< $@ $(OBJDIR)/%.o : %.cc $(INCTARGETS) @printf "Compiling %-35s > %s\n" $< $@ mkdir -p `dirname $@` $(CXX) -I. -I$(INCDIR) -I$(OBJDIR)/include $(CXXFLAGS) -Iinclude -I$(INCPLUGIN) -I$(DOCA_HOME)/include -c $< -o $@ @$(CXX) -I. -I$(INCDIR) -I$(OBJDIR)/include $(CXXFLAGS) -Iinclude -I$(INCPLUGIN) -I$(DOCA_HOME)/include -M $< > $(@:%.o=%.d.tmp) @sed "0,/^.*:/s//$(subst /,\/,$@):/" $(@:%.o=%.d.tmp) > $(@:%.o=%.d) @sed -e 's/.*://' -e 's/\\$$//' < $(@:%.o=%.d.tmp) | fmt -1 | \ sed -e 's/^ *//' -e 's/$$/:/' >> $(@:%.o=%.d) @rm -f $(@:%.o=%.d.tmp) $(OBJDIR)/misc/git_version.o: $(GIT_VERSION_FILE) $(DOCA_OBJDIR)/%.o : $(DOCA_HOME)/src/%.cpp @printf "Compiling %-35s > %s\n" $< $@ mkdir -p `dirname $@` $(CXX) -I$(DOCA_HOME)/src -I$(DOCA_HOME)/include $(CXXFLAGS) -c $< -o $@ @$(CXX) -I$(DOCA_HOME)/src -I$(DOCA_HOME)/include $(CXXFLAGS) -M $< > $(@:%.o=%.d.tmp) @sed "0,/^.*:/s//$(subst /,\/,$@):/" $(@:%.o=%.d.tmp) > $(@:%.o=%.d) @sed -e 's/.*://' -e 's/\\$$//' < $(@:%.o=%.d.tmp) | fmt -1 | \ sed -e 's/^ *//' -e 's/$$/:/' >> $(@:%.o=%.d) @rm -f $(@:%.o=%.d.tmp) clean : $(MAKE) -C device clean rm -rf ${BINDIR} ${INCDIR} ${LIBDIR} ${PKGDIR} ${OBJDIR} install : build mkdir -p $(PREFIX)/lib mkdir -p $(PREFIX)/lib/pkgconfig mkdir -p $(PREFIX)/include mkdir -p $(PREFIX)/bin cp -P -v $(BUILDDIR)/lib/lib* $(PREFIX)/lib/ cp -P -v $(BUILDDIR)/lib/pkgconfig/* $(PREFIX)/lib/pkgconfig/ cp -v -r $(BUILDDIR)/include/* $(PREFIX)/include/ cp -v $(BUILDDIR)/bin/ncclras $(PREFIX)/bin/ FILESTOFORMAT := $(shell find . -name ".\#*" -prune -o \( -name "*.cc" -o -name "*.h" \) -print | grep -v -E 'ibvwrap.h|nvmlwrap.h|gdrwrap.h|nccl.h') # Note that formatting.mk defines a new target so in order to not overwrite the default target, # it shouldn't be included at the top. Also, it uses the above definition of FILESTOFORMAT as well # as the BUILDDIR variable. include ../makefiles/formatting.mk nccl-2.29.7-1/src/allocator.cc000066400000000000000000000376361515037102200157610ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "comm.h" #include "transport.h" #include "group.h" #include "nvtx.h" #include "utils.h" NCCL_API(ncclResult_t, ncclMemAlloc, void **ptr, size_t size); ncclResult_t ncclMemAlloc(void **ptr, size_t size) { NCCL_NVTX3_FUNC_RANGE; ncclResult_t ret = ncclSuccess; #if CUDART_VERSION >= 11030 size_t memGran = 0; CUdevice currentDev; CUmemAllocationProp memprop = {}; CUmemAccessDesc accessDesc = {}; CUmemGenericAllocationHandle handle = (CUmemGenericAllocationHandle)-1; int cudaDev; int flag; int dcnt; if (ptr == NULL || size == 0) goto fallback; if (ncclCudaLibraryInit() != ncclSuccess) goto fallback; CUDACHECK(cudaGetDevice(&cudaDev)); CUCHECK(cuDeviceGet(¤tDev, cudaDev)); if (ncclCuMemEnable()) { size_t handleSize = size; int requestedHandleTypes = CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR; #if CUDART_VERSION >= 12030 // Query device to see if FABRIC handle support is available flag = 0; (void) CUPFN(cuDeviceGetAttribute(&flag, CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_FABRIC_SUPPORTED, currentDev)); if (flag) requestedHandleTypes |= CU_MEM_HANDLE_TYPE_FABRIC; #endif memprop.type = CU_MEM_ALLOCATION_TYPE_PINNED; memprop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; memprop.requestedHandleTypes = (CUmemAllocationHandleType) requestedHandleTypes; memprop.location.id = currentDev; // Query device to see if RDMA support is available flag = 0; CUCHECK(cuDeviceGetAttribute(&flag, CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WITH_CUDA_VMM_SUPPORTED, currentDev)); if (flag) memprop.allocFlags.gpuDirectRDMACapable = 1; CUCHECK(cuMemGetAllocationGranularity(&memGran, &memprop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED)); CUDACHECK(cudaGetDeviceCount(&dcnt)); ALIGN_SIZE(handleSize, memGran); #if CUDART_VERSION >= 12030 if (requestedHandleTypes & CU_MEM_HANDLE_TYPE_FABRIC) { /* First try cuMemCreate() with FABRIC handle support and then remove if it fails */ CUresult err = CUPFN(cuMemCreate(&handle, handleSize, &memprop, 0)); if (err == CUDA_ERROR_NOT_PERMITTED || err == CUDA_ERROR_NOT_SUPPORTED) { requestedHandleTypes &= ~CU_MEM_HANDLE_TYPE_FABRIC; memprop.requestedHandleTypes = (CUmemAllocationHandleType) requestedHandleTypes; /* Allocate the physical memory on the device */ CUCHECK(cuMemCreate(&handle, handleSize, &memprop, 0)); } else if (err != CUDA_SUCCESS) { // Catch and report any error from above CUCHECK(cuMemCreate(&handle, handleSize, &memprop, 0)); } } else #endif { /* Allocate the physical memory on the device */ CUCHECK(cuMemCreate(&handle, handleSize, &memprop, 0)); } /* Reserve a virtual address range */ CUCHECK(cuMemAddressReserve((CUdeviceptr*)ptr, handleSize, memGran, 0, 0)); /* Map the virtual address range to the physical allocation */ CUCHECK(cuMemMap((CUdeviceptr)*ptr, handleSize, 0, handle, 0)); /* Now allow RW access to the newly mapped memory */ for (int i = 0; i < dcnt; ++i) { int p2p = 0; if (i == cudaDev || (CUDASUCCESS(cudaDeviceCanAccessPeer(&p2p, i, cudaDev)) && p2p)) { accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; accessDesc.location.id = i; accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; CUCHECK(cuMemSetAccess((CUdeviceptr)*ptr, handleSize, &accessDesc, 1)); } if (0 == p2p && i != cudaDev) INFO(NCCL_ALLOC, "P2P not supported between GPU%d and GPU%d", cudaDev, i); } goto exit; } fallback: #endif // Coverity is right to complain that we may pass a NULL ptr to cudaMalloc. That's deliberate though: // we want CUDA to return an error to the caller. // coverity[var_deref_model] CUDACHECKGOTO(cudaMalloc(ptr, size), ret, fail); exit: return ret; fail: goto exit; } NCCL_API(ncclResult_t, ncclMemFree, void *ptr); ncclResult_t ncclMemFree(void *ptr) { NCCL_NVTX3_FUNC_RANGE; ncclResult_t ret = ncclSuccess; int saveDevice; CUDACHECK(cudaGetDevice(&saveDevice)); #if CUDART_VERSION >= 11030 CUdevice ptrDev = 0; if (ptr == NULL) goto fallback; if (ncclCudaLibraryInit() != ncclSuccess) goto fallback; CUCHECKGOTO(cuPointerGetAttribute((void*)&ptrDev, CU_POINTER_ATTRIBUTE_DEVICE_ORDINAL, (CUdeviceptr)ptr), ret, fail); CUDACHECKGOTO(cudaSetDevice((int)ptrDev), ret, fail); if (ncclCuMemEnable()) { NCCLCHECKGOTO(ncclCuMemFree(ptr, nullptr), ret, fail); // User facing API, memManager does not need to track user memory. Same as ncclMemAlloc goto exit; } fallback: #endif CUDACHECKGOTO(cudaFree(ptr), ret, fail); exit: CUDACHECK(cudaSetDevice(saveDevice)); return ret; fail: goto exit; } //////////////////////////////////////////////////////////////////////////////// // ncclSpace: // // This datastructure "cuts" the line of non-negative integers into segments // which alternate between "full" (allocated) and "empty" (not allocated). The // cuts are sorted ascending. The segment after the last cut must be empty // (the unallocated frontier). Knwoing this we can deduce whether the segment // ending at cut[i] is full or empty with this formula: // isFull(i) = (i%2 != ncuts%2) void ncclSpaceConstruct(struct ncclSpace* a) { memset(a, 0, sizeof(*a)); } void ncclSpaceDestruct(struct ncclSpace* a) { free(a->cuts); } static void insertSegment(struct ncclSpace* a, int index, int64_t lo, int64_t hi) { // Insert space for two cuts in `a->cuts[]` before `index`. if (a->count + 2 > a->capacity) { a->capacity *= 2; if (a->capacity == 0) a->capacity = 16; int64_t* cuts1 = (int64_t*)malloc(a->capacity*sizeof(int64_t)); for (int i=0; i < index; i++) cuts1[i] = a->cuts[i]; for (int i=index; i < a->count; i++) cuts1[i+2] = a->cuts[i]; free(a->cuts); a->cuts = cuts1; } else { for (int i=a->count-1; index <= i; i--) a->cuts[i+2] = a->cuts[i]; } a->cuts[index+0] = lo; a->cuts[index+1] = hi; a->count += 2; // Filter pairs of adjacent repeated values from cuts[]. Since these mark // boundaries where segments transition between full<->empty, dropping such a // pair fuses two adjacent segments together. Examples: // [1,2,3,3,4] -> [1,2,4] // [1,2,3,3,3,4] -> [1,2,3,4] // have to leave one 3 because its a full<->empty transition // [1,2,3,3,3,3,4] -> [1,2,4] // Leading zeros don't have to be in pairs, they are always dropped: // [0,1,2] -> [1,2] // [0,0,1,2] -> [1,2] int r = index, w = index; // Read and write cursors. int64_t prev = r==0 ? 0 : a->cuts[r-1]; while (r < a->count) { int64_t cur = a->cuts[r++]; a->cuts[w++] = cur; if (prev == cur) { // Repeated value is an empty segment which can be deleted. // Erase last two cuts or just one if we're at the start. w -= w==1 ? 1 : 2; // Zeros can only occur at the beginning (due to being sorted). We want to // drop any number of zeros, but only even numbers of other repeated values. // So set to zero here, which will make prev=0, thus if next value is zero // it will be dropped but if its not zero then it will need to begin a new // pair to be dropped. cur = 0; } prev = cur; } a->count = w; } ncclResult_t ncclSpaceAlloc( struct ncclSpace* a, int64_t limit, int64_t size, int align, int64_t* outOffset ) { // When allocating we try to locate the first empty segment which can hold // the allocation and move its lower cut upward. int i = a->count%2; // First empty segment ends at cuts[i] size_t off; while (i <= a->count) { size_t lo = i == 0 ? 0 : a->cuts[i-1]; size_t hi = i == a->count ? limit : a->cuts[i]; off = alignUp(lo, align); if (off + size <= hi) { *outOffset = off; if (i == 0 || off + size == hi) { // Slow path required. insertSegment(a, i, off, off+size); } else { // We can just append to the end of a full segment. a->cuts[i-1] = off + size; } return ncclSuccess; } i += 2; // Next empty segment } WARN("Allocation failed. No suitable space found to accommodate size=0x%lx within limit=0x%lx", (long)size, (long)limit); return ncclInternalError; } ncclResult_t ncclSpaceFree(struct ncclSpace* a, int64_t offset, int64_t size) { if (a->count == 0 || a->cuts[a->count-1] <= offset) { WARN("No allocation found at offset=0x%lx", (long)offset); return ncclInternalError; } // This could be binary search, but since allocate is linear there's no point. int i = 1 - a->count%2; // First full segment ends at cuts[i] while (a->cuts[i] <= offset) i += 2; int64_t lo = i==0 ? 0 : a->cuts[i-1]; int64_t hi = a->cuts[i]; if (offset < lo || hi < offset + size) { WARN("Given size=0x%lx extends beyond allocation.", (long)size); return ncclInternalError; } // First try the two fast cases which just shrink a segment from one side. if (i != 0 && lo == offset && offset + size != hi) { a->cuts[i-1] = offset + size; // Bring bottom up. } else if (lo != offset && offset + size == hi) { a->cuts[i] = offset; // Bring top down. } else { // Slow path. insertSegment(a, i, offset, offset+size); } return ncclSuccess; } //////////////////////////////////////////////////////////////////////////////// // ncclShadowPool: struct ncclShadowPage { // A contiguous block of (at most) 64 objects struct ncclShadowPage* next; int objSize; uint64_t freeMask; void* devObjs; }; struct ncclShadowObject { struct ncclShadowObject* next; void* devObj; void* hostObj; struct ncclShadowPage* page; // null if not allocated in page but directly in CUDA mempool. }; void ncclShadowPoolConstruct(struct ncclShadowPool* pool) { pool->hbits = 0; pool->count = 0; pool->table = nullptr; pool->pages = nullptr; } ncclResult_t ncclShadowPoolDestruct(struct ncclShadowPool* pool) { if (pool->hbits != 0) { cudaStream_t stream; CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); if (pool->count != 0) { for (int i=0; i < 1<hbits; i++) { struct ncclShadowObject* obj = pool->table[i]; while (obj != nullptr) { struct ncclShadowPage* page = obj->page; if (page != nullptr) { if (page->freeMask == 0) { // Put full pages back into page list. page->freeMask = 1; page->next = pool->pages; pool->pages = page; } } else { cudaFreeAsync(obj->devObj, stream); } struct ncclShadowObject* next = obj->next; free(obj); obj = next; } } } free(pool->table); while (pool->pages != nullptr) { cudaFreeAsync(pool->pages->devObjs, stream); struct ncclShadowPage* next = pool->pages->next; free(pool->pages); pool->pages = next; } cudaStreamSynchronize(stream); cudaStreamDestroy(stream); cudaMemPoolDestroy(pool->memPool); } return ncclSuccess; } static void hashInsert(struct ncclShadowPool* pool, struct ncclShadowObject* obj) { uint64_t b = ncclHashPointer(pool->hbits, obj->devObj); obj->next = pool->table[b]; pool->table[b] = obj; } ncclResult_t ncclShadowPoolAlloc( struct ncclShadowPool* pool, size_t size, void** outDevObj, void** outHostObj, cudaStream_t stream ) { if (size == 0) { if (outDevObj) *outDevObj = nullptr; if (outHostObj) *outHostObj = nullptr; return ncclSuccess; } int hbits = pool->hbits; if (hbits == 0) { cudaMemPoolProps props = {}; props.allocType = cudaMemAllocationTypePinned; props.handleTypes = cudaMemHandleTypeNone; props.location.type = cudaMemLocationTypeDevice; cudaGetDevice(&props.location.id); CUDACHECK(cudaMemPoolCreate(&pool->memPool, &props)); pool->hbits = hbits = 4; pool->table = (struct ncclShadowObject**)malloc(sizeof(struct ncclShadowObject*)<table[i] = nullptr; } // Check for hash table size increase before inserting. Maintain 2:1 object:bucket ratio. if (pool->count+1 > 2<table; struct ncclShadowObject** table1 = (struct ncclShadowObject**)malloc(sizeof(struct ncclShadowObject*)<<(hbits+1)); pool->table = table1; pool->hbits = hbits+1; for (int i1=0; i1 < 2<next; hashInsert(pool, obj); obj = next; } } hbits += 1; // match pool->hbits free(table0); } struct ncclShadowPage* page; void *devObj; if ((64<<10)/size >= 3) { int shift = std::max(0, (int)log2Down(size) + 1 - 4); int pageObjSize = ((size + (1<>shift)<pages; while (true) { page = *pagePtr; if (page == nullptr) { size_t pageSize = std::min(64<<10, 64*pageObjSize); page = (struct ncclShadowPage*)malloc(sizeof(struct ncclShadowPage)); page->objSize = pageObjSize; page->freeMask = uint64_t(-1)>>(64 - pageSize/pageObjSize); page->next = pool->pages; pool->pages = page; CUDACHECK(cudaMallocFromPoolAsync(&page->devObjs, pageSize, pool->memPool, stream)); CUDACHECK(cudaMemsetAsync(page->devObjs, 0, pageSize, stream)); // fall through... } if (page->objSize == pageObjSize) { int slot = popFirstOneBit(&page->freeMask); devObj = (char*)page->devObjs + slot*pageObjSize; if (page->freeMask == 0) *pagePtr = page->next; // Remove full page from list. break; } pagePtr = &page->next; } } else { page = nullptr; CUDACHECK(cudaMallocFromPoolAsync(&devObj, size, pool->memPool, stream)); CUDACHECK(cudaMemsetAsync(devObj, 0, size, stream)); } struct ncclShadowObject* obj = (struct ncclShadowObject*)malloc( sizeof(struct ncclShadowObject) + /*padding=*/alignof(max_align_t)-1 + size ); obj->page = page; obj->devObj = devObj; obj->hostObj = alignUp((char*)(obj+1), alignof(max_align_t)); memset(obj->hostObj, 0, size); hashInsert(pool, obj); pool->count += 1; if (outDevObj) *outDevObj = devObj; if (outHostObj) *outHostObj = obj->hostObj; return ncclSuccess; } ncclResult_t ncclShadowPoolFree(struct ncclShadowPool* pool, void* devObj, cudaStream_t stream) { if (devObj == nullptr) return ncclSuccess; uint64_t b = ncclHashPointer(pool->hbits, devObj); struct ncclShadowObject** pobj = &pool->table[b]; while (true) { if (*pobj == nullptr) { WARN("Device object does not exist in shadow pool."); return ncclInternalError; } if ((*pobj)->devObj == devObj) break; pobj = &(*pobj)->next; } struct ncclShadowObject* obj = *pobj; *pobj = obj->next; if (obj->page != nullptr) { if (obj->page->freeMask == 0) { obj->page->next = pool->pages; pool->pages = obj->page; } int slot = ((char*)obj->devObj - (char*)obj->page->devObjs)/obj->page->objSize; obj->page->freeMask |= uint64_t(1)<count -= 1; return ncclSuccess; } ncclResult_t ncclShadowPoolToHost(struct ncclShadowPool* pool, void* devObj, void** hostObj) { if (devObj == nullptr) { *hostObj = nullptr; return ncclSuccess; } uint64_t b = ncclHashPointer(pool->hbits, devObj); struct ncclShadowObject* obj = pool->table[b]; while (true) { if (obj == nullptr) { WARN("Device object does not exist in shadow pool."); return ncclInternalError; } if (obj->devObj == devObj) break; obj = obj->next; } *hostObj = obj->hostObj; return ncclSuccess; } nccl-2.29.7-1/src/bootstrap.cc000066400000000000000000001555411515037102200160120ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "nccl.h" #include "core.h" #include "utils.h" #include "bootstrap.h" #include "net.h" #include #include #include "proxy.h" #include "param.h" #include "ras.h" #include #include "os.h" #include #include #define BOOTSTRAP_N_CHECK_ABORT 10000 #define BOOTSTRAP_TAG_CONNECT (0x1 << 31) #define BOOTSTRAP_TAG_ALLGATHER (0x1 << 30) #define BOOTSTRAP_TAG_COMMSPLIT (0x1 << 29) #define BOOTSTRAP_TAG_INTRANODE_ALLGATHER (0x1 << 28) #define BOOTSTRAP_TAG_GROW_BOUNDARY (0x1 << 27) #define BOOTSTRAP_INIT_TIME_CREATE 0 #define BOOTSTRAP_INIT_TIME_SEND 1 #define BOOTSTRAP_INIT_TIME_RECV 2 #define BOOTSTRAP_INIT_TIME_RING 3 #define BOOTSTRAP_INIT_TIME_TOTAL 4 #define BOOTSTRAP_INIT_TIME_DELAY 5 #define BOOTSTRAP_INIT_TIME_N 6 #define BOOTSTRAP_INIT_ROOT_WAIT 0 #define BOOTSTRAP_INIT_ROOT_SEND 1 #define BOOTSTRAP_INIT_ROOT_RECV 2 #define BOOTSTRAP_INIT_ROOT_N 3 #define BOOTSTRAP_PROF_OPEN(time) \ do { \ time = clockNano(); \ } while (0) #define BOOTSTRAP_PROF_CLOSE(time) \ do { \ time = clockNano() - time; \ } while (0) #define BOOTSTRAP_PID(i, n) (((i) + (n)) % (n)) // returns the first rank associated to the root. must have root >=0 // if root >= n_roots, it does NOT assume periodicity static int firstRankFromRoot(int root, int n_ranks, int nRoots,int offset) { if (root == -1) return 0; // only distribute the n_ranks - offset on the roots n_ranks -= offset; return offset + root * (n_ranks / nRoots) + std::min(root, n_ranks % nRoots); } // returns the root of a rank, must have rank >=0 // if rank >= n_ranks, it does NOT assume periodicity static int rootIdFromRank(int rank, int nRanks, int nRoots, int offset) { // ranks < offset have no root (id = -1), ranks above the offset will get assigned to their respective root if(nRoots == 0 || rank < offset) return -1; nRanks -= offset; rank -= offset; int rmr = nRanks % nRoots; // rank mod root int rpr = nRanks / nRoots; // rank per root int D = rmr * (rpr + 1); if (rank < D) return rank / (rpr + 1); else return (rank - D) / rpr + rmr; } // return the number of child for a root, root will be periodized static int nRankFromRoot(int root, int nRanks, int nRoots, int offset) { if(root == -1) return 0; nRanks -= offset; int ir = BOOTSTRAP_PID(root, nRoots); int rmr = nRanks % nRoots; // rank mod root int rpr = nRanks / nRoots; // rank per root return rpr + ((ir < rmr) ? 1 : 0); } // return the local id of a given rank for a given root // root will be periodize, rank will not static int localIdFromRoot(int rank, int root, int nRanks, int nRoots, int offset) { // any rank for root -1 has a local id that is the rank id if(root == -1) return rank; int ir = BOOTSTRAP_PID(root, nRoots); return rank - firstRankFromRoot(ir, nRanks, nRoots, offset); } // Check if the given rank is the first rank from the root static int isFirstFromRoot(int rank, int root, int nRanks, int nRoots, int offset) { return (rank == firstRankFromRoot(root, nRanks, nRoots, offset)); } struct bootstrapRootArgs { struct ncclSocket* listenSock; uint64_t magic; }; /* Init functions */ static char bootstrapNetIfName[MAX_IF_NAME_SIZE+1]; static union ncclSocketAddress bootstrapNetIfAddr; static int bootstrapNetInitDone = 0; static std::mutex bootstrapNetMutex; NCCL_PARAM(BootstrapNetEnable,"OOB_NET_ENABLE", 0); ncclResult_t bootstrapNetInit() { if (bootstrapNetInitDone == 0) { std::lock_guard lock(bootstrapNetMutex); if (bootstrapNetInitDone == 0) { const char* env = ncclGetEnv("NCCL_COMM_ID"); int nIfs = 0; if (env) { union ncclSocketAddress remoteAddr; if (ncclSocketGetAddrFromString(&remoteAddr, env) != ncclSuccess) { WARN("Invalid NCCL_COMM_ID, please use format: : or []: or :"); return ncclInvalidArgument; } NCCLCHECK(ncclFindInterfaceMatchSubnet(bootstrapNetIfName, &bootstrapNetIfAddr, &remoteAddr, MAX_IF_NAME_SIZE, &nIfs)); if (nIfs <= 0) { WARN("NET/Socket : No usable listening interface found"); return ncclSystemError; } } else { NCCLCHECK(ncclFindInterfaces(bootstrapNetIfName, &bootstrapNetIfAddr, MAX_IF_NAME_SIZE, 1, &nIfs)); if (nIfs <= 0) { WARN("Bootstrap : no socket interface found"); return ncclInvalidUsage; } } char line[SOCKET_NAME_MAXLEN+MAX_IF_NAME_SIZE+2]; snprintf(line, sizeof(line), " %s:", bootstrapNetIfName); ncclSocketToString(&bootstrapNetIfAddr, line+strlen(line)); INFO(NCCL_BOOTSTRAP, "Bootstrap: Using%s", line); bootstrapNetInitDone = 1; } } return ncclSuccess; } /* Socket Interface Selection type */ enum bootstrapInterface_t { findSubnetIf = -1, dontCareIf = -2 }; // check abort function static ncclResult_t checkAbort(volatile uint32_t* flag, int* cntr) { if ((*cntr % BOOTSTRAP_N_CHECK_ABORT) == 0) { if (flag && COMPILER_ATOMIC_LOAD(flag, std::memory_order_acquire)) { TRACE(NCCL_BOOTSTRAP, "bootstrap: abort called"); return ncclInternalError; } } *cntr = (*cntr + 1) % BOOTSTRAP_N_CHECK_ABORT; return ncclSuccess; } // send/recv functions static ncclResult_t netReg(ncclNet_t* net, void* comm, void* data, int size, void** handle) { NCCLCHECK(net->regMr(comm, data, size, NCCL_PTR_HOST, handle)); return ncclSuccess; } static ncclResult_t netDereg(ncclNet_t* net, void* comm, void** handle) { NCCLCHECK(net->deregMr(comm, *handle)); *handle = NULL; return ncclSuccess; } static ncclResult_t netIsend(ncclNet_t* net, void* sendComm, void* data, int size, void* dataHandle, int tag, void** sendReq, int* done) { if (*done) return ncclSuccess; if (!*sendReq) { NCCLCHECK(net->isend(sendComm, data, (size_t)size, tag, dataHandle, NULL, sendReq)); } if (*sendReq) { NCCLCHECK(net->test(*sendReq, done, NULL)); if (*done) { *sendReq = NULL; } } return ncclSuccess; } static ncclResult_t netIrecv(ncclNet_t* net, void* recvComm, void* data, int size, void* dataHandle, int tag, void** recvReq, int* done) { if (*done) return ncclSuccess; if (!*recvReq) { size_t size64 = size; NCCLCHECK(net->irecv(recvComm, 1, &data, &size64, &tag, &dataHandle, NULL, recvReq)); } if (*recvReq) { NCCLCHECK(net->test(*recvReq, done, NULL)); if (*done) { *recvReq = NULL; } } return ncclSuccess; } static ncclResult_t netSendRecv(ncclNet_t* net, void* sendComm, void* sendData, int sendSize, void* sendDataHandle, void* recvComm, void* recvData, int recvSize, void* recvDataHandle, int tag, volatile uint32_t* abortFlag) { int abortCounter = 0; int doneSend = 0, doneRecv = 0; void *sendReq = NULL, *recvReq = NULL; do { NCCLCHECK(checkAbort(abortFlag, &abortCounter)); if (!doneRecv) { NCCLCHECK(netIrecv(net, recvComm, recvData, recvSize, recvDataHandle, tag, &recvReq, &doneRecv)); } if (!doneSend) { NCCLCHECK(netIsend(net, sendComm, sendData, sendSize, sendDataHandle, tag, &sendReq, &doneSend)); } } while (!doneSend || !doneRecv); return ncclSuccess; } // Additional socket based functions, first send the size, then send the message static ncclResult_t socketSend(struct ncclSocket* sock, void* data, int size) { NCCLCHECK(ncclSocketSend(sock, &size, sizeof(int))); if (size > 0) NCCLCHECK(ncclSocketSend(sock, data, size)); return ncclSuccess; } static ncclResult_t socketRecv(struct ncclSocket* sock, void* data, int size) { int recvSize; NCCLCHECK(ncclSocketRecv(sock, &recvSize, sizeof(int))); if (recvSize > size) { WARN("Message truncated : received %d bytes instead of %d", recvSize, size); return ncclInternalError; } int actualSize = std::min(recvSize, size); if (actualSize > 0) NCCLCHECK(ncclSocketRecv(sock, data, actualSize)); return ncclSuccess; } static ncclResult_t socketSendRecv(struct ncclSocket* sendSock, void* sendData, int sendSize, struct ncclSocket* recvSock, void* recvData, int recvSize) { int senderRecvSize; NCCLCHECK(ncclSocketSendRecv(sendSock, &sendSize, sizeof(int), recvSock, &senderRecvSize, sizeof(int))); if (senderRecvSize > recvSize) { WARN("Message truncated : received %d bytes instead of %d", senderRecvSize, recvSize); return ncclInternalError; } NCCLCHECK(ncclSocketSendRecv(sendSock, sendData, sendSize, recvSock, recvData, std::min(recvSize, senderRecvSize))); return ncclSuccess; } static ncclResult_t socketDoubleSendRecv(struct ncclSocketOp ops[4]) { // ops synchronously exchange size then asynchronously exchange data in send->recv->send->recv order int senderRecvSize1, senderRecvSize2; NCCLCHECK(ncclSocketSendRecv(ops[0].sock, &ops[0].size, sizeof(int), ops[1].sock, &senderRecvSize1, sizeof(int))); NCCLCHECK(ncclSocketSendRecv(ops[2].sock, &ops[2].size, sizeof(int), ops[3].sock, &senderRecvSize2, sizeof(int))); if (senderRecvSize1 > ops[1].size || senderRecvSize2 > ops[3].size) { WARN("Message truncated : received %d,%d bytes instead of %d,%d", senderRecvSize1, senderRecvSize2, ops[1].size, ops[3].size); return ncclInternalError; } ops[1].size = std::min(ops[1].size, senderRecvSize1); ops[3].size = std::min(ops[3].size, senderRecvSize2); NCCLCHECK(ncclSocketMultiOp(ops, 4)); return ncclSuccess; } union ringConnectInfo { union ncclSocketAddress addr; char handle[NCCL_NET_HANDLE_MAXSIZE]; }; struct extInfo { int rank; // rank of the process reaching out int nranks; // total number of ranks int iroot; // current root index int nroots; // total number of roots int offset; // offset for rank distribution union ncclSocketAddress listenRootAddress; // address of my listenSocket for the root union ringConnectInfo connectInfo; }; #define NET_HANDLE(h, rank) ((h) + (rank * NCCL_NET_HANDLE_MAXSIZE)) #define BOOTSTRAP_HANDLE(h, i) ((struct ncclBootstrapHandle*)((char*)h + i * NCCL_UNIQUE_ID_BYTES)) #include static ncclResult_t setFilesLimit() { struct rlimit filesLimit; SYSCHECK(getrlimit(RLIMIT_NOFILE, &filesLimit), "getrlimit"); filesLimit.rlim_cur = filesLimit.rlim_max; SYSCHECK(setrlimit(RLIMIT_NOFILE, &filesLimit), "setrlimit"); return ncclSuccess; } static ncclResult_t rootSend(union ncclSocketAddress* addr, uint64_t magic, union ringConnectInfo* info) { ncclResult_t res = ncclSuccess; struct ncclSocket sock; NCCLCHECKGOTO(ncclSocketInit(&sock, addr, magic, ncclSocketTypeBootstrap), res, fail); NCCLCHECKGOTO(ncclSocketConnect(&sock), res, fail); NCCLCHECKGOTO(socketSend(&sock, info, sizeof(union ringConnectInfo)), res, fail); NCCLCHECK(ncclSocketClose(&sock)); return res; fail: (void)ncclSocketClose(&sock); return res; } static void* bootstrapRoot(void* rargs) { uint64_t timers[BOOTSTRAP_INIT_ROOT_N] = {0}; struct bootstrapRootArgs* args = (struct bootstrapRootArgs*)rargs; struct ncclSocket* listenSock = args->listenSock; uint64_t magic = args->magic; ncclResult_t res = ncclSuccess; int nranks = 0, c = 0; int iroot = 0, nroots = 0, localId = 0; int nrecv = 0, n2send = 0, offset = 0; struct extInfo info; union ringConnectInfo* rankInfo = NULL; union ncclSocketAddress* rankAddressesRoot = NULL; // for initial rank <-> root information exchange // get zeros for comparison char zeroHandle[NCCL_NET_HANDLE_MAXSIZE]; union ncclSocketAddress zeroAddress; union ringConnectInfo zeroInfo; memset(&zeroAddress, 0, sizeof(union ncclSocketAddress)); memset(&zeroHandle, 0, NCCL_NET_HANDLE_MAXSIZE); memset(&zeroInfo, 0, sizeof(union ringConnectInfo)); setFilesLimit(); TRACE(NCCL_BOOTSTRAP, "BEGIN"); BOOTSTRAP_PROF_OPEN(timers[BOOTSTRAP_INIT_ROOT_WAIT]); /* Receive addresses from all ranks */ do { struct ncclSocket sock; NCCLCHECKGOTO(ncclSocketInit(&sock), res, out); NCCLCHECKGOTO(ncclSocketAccept(&sock, listenSock), res, out); NCCLCHECKGOTO(socketRecv(&sock, &info, sizeof(info)), res, out); NCCLCHECKGOTO(ncclSocketClose(&sock), res, out); if (c == 0) { BOOTSTRAP_PROF_CLOSE(timers[BOOTSTRAP_INIT_ROOT_WAIT]); BOOTSTRAP_PROF_OPEN(timers[BOOTSTRAP_INIT_ROOT_RECV]); nranks = info.nranks; iroot = info.iroot; nroots = info.nroots; offset = info.offset; // if the number of root > 1, we will receive one extra info from the first local_id of the next root n2send = nRankFromRoot(iroot, nranks, nroots, offset); // offset>0 automatically means that we need to switch to the multiroot logic nrecv = n2send + ((offset > 0 || nroots > 1) ? 1 : 0); NCCLCHECKGOTO(ncclCalloc(&rankInfo, nrecv), res, out); NCCLCHECKGOTO(ncclCalloc(&rankAddressesRoot, nrecv), res, out); } if (nranks != info.nranks || nroots != info.nroots || iroot != info.iroot || offset != info.offset) { WARN("Bootstrap Root : mismatch in info from procs, nranks %d vs %d, nroots %d vs %d, iroot %d vs %d, offset %d vs %d", nranks, info.nranks, nroots, info.nroots, iroot, info.iroot, offset, info.offset); goto out; } localId = localIdFromRoot(info.rank, iroot, nranks, nroots, offset); if (localId < 0 || localId >= nrecv) { WARN("Bootstrap Root : localId %d is out of range", localId); goto out; } if (memcmp(&zeroAddress, &rankAddressesRoot[localId], sizeof(union ncclSocketAddress)) != 0 || memcmp(&zeroInfo, &rankInfo[localId], sizeof(union ringConnectInfo)) != 0) { WARN("Bootstrap Root : rank %d of %d ranks has already checked in", info.rank, nranks); goto out; } // if the previous has already checked in, send the newly received handle, if not save the handle for later // if we have more than 1 root, I do not own the previous of local_id = 0 // if we have prev > n2send, we do not send anything int prev = (nroots > 1) ? (localId - 1) : BOOTSTRAP_PID(localId - 1, nrecv); if (prev >= 0 && prev < n2send && memcmp(&zeroAddress, &rankAddressesRoot[prev], sizeof(union ncclSocketAddress)) != 0) { NCCLCHECKGOTO(rootSend(&rankAddressesRoot[prev], magic, &info.connectInfo), res, out); } else { memcpy(&rankInfo[localId], &info.connectInfo, sizeof(union ringConnectInfo)); } // if the next rank has checked in, send the newly received info, if not save the addr for later // for nroots >=1, I will always own the information of the next connection // if the local_id id must be [0 ; n2send[ otherwise we do not answer int next = BOOTSTRAP_PID(localId + 1, nrecv); if (localId >= 0 && localId < n2send && memcmp(&zeroInfo, &rankInfo[next], sizeof(union ringConnectInfo)) != 0) { NCCLCHECKGOTO(rootSend(&info.listenRootAddress, magic, &rankInfo[next]), res, out); } else { memcpy(rankAddressesRoot + localId, &info.listenRootAddress, sizeof(union ncclSocketAddress)); } ++c; TRACE(NCCL_BOOTSTRAP, "Received connect from rank %d total %d/%d", info.rank, c, nrecv); } while (c < nrecv); TRACE(NCCL_BOOTSTRAP, "COLLECTED ALL %d HANDLES", nrecv); BOOTSTRAP_PROF_CLOSE(timers[BOOTSTRAP_INIT_ROOT_RECV]); // send the remaining info to the ranks who haven't received anything BOOTSTRAP_PROF_OPEN(timers[BOOTSTRAP_INIT_ROOT_SEND]); // here we need to send info only to my own local process for (int r = 0; r < n2send; ++r) { // use nrecv to periodize: if 1 root, we will send the first one to the last one, if >1 roots we will send the additional one we have received int next = BOOTSTRAP_PID(r + 1, nrecv); if (memcmp(&zeroAddress, &rankAddressesRoot[r], sizeof(union ncclSocketAddress)) != 0 && memcmp(&zeroInfo, &rankInfo[next], sizeof(union ringConnectInfo)) != 0) { NCCLCHECKGOTO(rootSend(&rankAddressesRoot[r], magic, &rankInfo[next]), res, out); } } BOOTSTRAP_PROF_CLOSE(timers[BOOTSTRAP_INIT_ROOT_SEND]); TRACE(NCCL_BOOTSTRAP | NCCL_PROFILE, "Root timings (wait %f, recv %f, send %f)", timers[BOOTSTRAP_INIT_ROOT_WAIT] / 1e9, timers[BOOTSTRAP_INIT_ROOT_RECV] / 1e9, timers[BOOTSTRAP_INIT_ROOT_SEND] / 1e9); out: if (listenSock != NULL) { (void)ncclSocketClose(listenSock); free(listenSock); } if (rankInfo) free(rankInfo); if (rankAddressesRoot) free(rankAddressesRoot); free(rargs); TRACE(NCCL_BOOTSTRAP, "DONE"); return NULL; } ncclResult_t bootstrapCreateRoot(struct ncclBootstrapHandle* handle, bool idFromEnv) { ncclResult_t ret = ncclSuccess; struct ncclSocket* listenSock = NULL; struct bootstrapRootArgs* args = NULL; std::thread thread; NCCLCHECK(ncclCalloc(&listenSock, 1)); NCCLCHECKGOTO(ncclSocketInit(listenSock, &handle->addr, handle->magic, ncclSocketTypeBootstrap, NULL, 0), ret, fail); NCCLCHECKGOTO(ncclSocketListen(listenSock), ret, fail); NCCLCHECKGOTO(ncclSocketGetAddr(listenSock, &handle->addr), ret, fail); NCCLCHECKGOTO(ncclCalloc(&args, 1), ret, fail); args->listenSock = listenSock; args->magic = handle->magic; thread = std::thread(bootstrapRoot, args); ncclSetThreadName(thread, "NCCL BootstrapR"); thread.detach(); exit: return ret; fail: if (listenSock) free(listenSock); if (args) free(args); goto exit; } ncclResult_t bootstrapGetUniqueId(struct ncclBootstrapHandle* handle, struct ncclComm* comm) { memset(handle, 0, sizeof(ncclBootstrapHandle)); const char* env = ncclGetEnv("NCCL_COMM_ID"); if (env) { // If comm is provided (grow operation), NCCL_COMM_ID should not be set if (comm) { WARN("ncclCommGetUniqueId should not be called when NCCL_COMM_ID is set"); return ncclInvalidUsage; } // Normal init: use NCCL_COMM_ID from environment INFO(NCCL_ENV, "NCCL_COMM_ID set by environment to %s", env); if (ncclSocketGetAddrFromString(&handle->addr, env) != ncclSuccess) { WARN("Invalid NCCL_COMM_ID, please use format: : or []: or :"); return ncclInvalidArgument; } handle->magic = NCCL_MAGIC; } else { if (comm) { // comm->childCount will be increment in ncclCommGrow for all existing ranks, use +1 here handle->magic = hashCombine(comm->magic, comm->childCount + 1); } else { NCCLCHECK(getRandomData(&handle->magic, sizeof(handle->magic))); } handle->nRanks = comm ? comm->nRanks : 0; memcpy(&handle->addr, &bootstrapNetIfAddr, sizeof(union ncclSocketAddress)); NCCLCHECK(bootstrapCreateRoot(handle, false)); } return ncclSuccess; } ncclResult_t bcastGrowHandle(struct ncclBootstrapHandle* handle, struct ncclComm* parent, bool isRoot) { if (!parent || !handle) { WARN("bcastGrowHandle: parent comm and handle must be provided"); return ncclInvalidArgument; } // Single rank parent already has the handle, no need to broadcast if (parent->nRanks == 1) return ncclSuccess; if (isRoot) { NCCLCHECK(bootstrapSend(parent->bootstrap, 0, BOOTSTRAP_TAG_GROW_BOUNDARY, handle, sizeof(struct ncclBootstrapHandle))); NCCLCHECK(bootstrapSend(parent->bootstrap, parent->nRanks - 1, BOOTSTRAP_TAG_GROW_BOUNDARY, handle, sizeof(struct ncclBootstrapHandle))); } else { NCCLCHECK(bootstrapRecv(parent->bootstrap, -1, BOOTSTRAP_TAG_GROW_BOUNDARY, handle, sizeof(struct ncclBootstrapHandle))); } return ncclSuccess; } struct unexConn { int peer; int tag; struct ncclSocket sock; struct unexConn* next; }; struct bootstrapRing_t { union { struct { void *sendComm, *recvComm; ncclNetDeviceHandle_t *sendDevHandle, *recvDevHandle; } net; struct { struct ncclSocket recv; struct ncclSocket send; } socket; }; }; struct bootstrapListen_t { struct ncclSocket peerSocket; // socket for peers to contact me in P2P union { struct { int dev; void* comm; char handle[NCCL_NET_HANDLE_MAXSIZE]; } net; struct ncclSocket socket; // socket to be used for the ring }; }; struct bootstrapState { struct bootstrapRing_t ring; struct bootstrapListen_t listen; ncclNet_t* net; uint64_t* peerProxyAddressesUDS; union ncclSocketAddress* peerProxyAddresses; union ncclSocketAddress* peerP2pAddresses; struct unexConn* unexpectedConnections; int cudaDev; int rank; int nranks; uint64_t magic; volatile uint32_t* abortFlag; }; #define STATE_RING(s, f) (s->ring.f) #define STATE_LISTEN(s, f) (s->listen.f) // helper functions static ncclResult_t createListenSocket(struct ncclComm* comm, uint64_t magic, struct ncclSocket* socket, union ncclSocketAddress* addr, ncclSocketType type) { NCCLCHECK(ncclSocketInit(socket, &bootstrapNetIfAddr, magic, type, comm->abortFlag)); NCCLCHECK(ncclSocketListen(socket)); NCCLCHECK(ncclSocketGetAddr(socket, addr)); return ncclSuccess; } static ncclResult_t getUDS(uint64_t* peerUDS) { uint64_t randId; NCCLCHECK(getRandomData(&randId, sizeof(randId))); *peerUDS = getPidHash() + randId; return ncclSuccess; } #define MAX_OOB_DEVS 16 static ncclResult_t netGetDevice(int rank, struct ncclComm* comm, int* dev) { static int devOOB = -1; if (devOOB < 0) { std::lock_guard lock(bootstrapNetMutex); if (devOOB < 0) { const char* userIfEnv = ncclGetEnv("NCCL_OOB_NET_IFNAME"); if (userIfEnv && strlen(userIfEnv) > 0) { INFO(NCCL_BOOTSTRAP | NCCL_ENV, "NCCL_OOB_NET_IFNAME set to %s", userIfEnv); bool searchNot = userIfEnv && userIfEnv[0] == '^'; if (searchNot) userIfEnv++; bool searchExact = userIfEnv && userIfEnv[0] == '='; if (searchExact) userIfEnv++; struct netIf userIfs[MAX_OOB_DEVS]; int nUserIfs = parseStringList(userIfEnv, userIfs, MAX_OOB_DEVS); // loop over the device and return the first one matching int nDev = 0; NCCLCHECK(comm->ncclNet->devices(&nDev)); int devId = 0; while (devId < nDev) { ncclNetProperties_t props; comm->ncclNet->getProperties(devId, &props); // check against user specified HCAs/ports if (matchIfList(props.name, props.port, userIfs, nUserIfs, searchExact) ^ searchNot) { // All plain physical devices have been initialized at this point devOOB = devId; break; } devId++; } if (devOOB == -1) { if (!searchNot) WARN("no device found matching %s%s, verify NCCL_OOB_NET_IFNAME", searchExact ? "exactly " : "", userIfEnv); else WARN("no device found after excluding %s%s, verify NCCL_OOB_NET_IFNAME", searchExact ? "exactly " : "", userIfEnv); return ncclInvalidArgument; } } else { // default choice is device 0 devOOB = 0; } // display info on the chosen device ncclNetProperties_t props; ncclResult_t res = comm->ncclNet->getProperties(devOOB, &props); bool hasProp = res == ncclSuccess; INFO(NCCL_BOOTSTRAP, "Bootstrap: Using %s:%d", (hasProp) ? props.name : "N/A", (hasProp) ? props.port : -1); } } *dev = devOOB; return ncclSuccess; } static ncclResult_t netRingConnect(void* ctx, ncclNet_t* net, struct bootstrapListen_t* listen, char peerHandle[NCCL_NET_HANDLE_MAXSIZE], void** sendComm, ncclNetDeviceHandle_t** sendDevHandle, void** recvComm, ncclNetDeviceHandle_t** recvDevHandle, volatile uint32_t* abortFlag) { int abortCounter = 0; do { NCCLCHECK(checkAbort(abortFlag, &abortCounter)); if (!*sendComm) NCCLCHECK(net->connect(ctx, listen->net.dev, peerHandle, sendComm, sendDevHandle)); if (!*recvComm) NCCLCHECK(net->accept(listen->net.comm, recvComm, recvDevHandle)); } while (!*sendComm || !*recvComm); return ncclSuccess; } static ncclResult_t socketRingConnect(ncclSocketAddress* addr, struct ncclSocket* sendSocket, struct ncclSocket* listenSock, struct ncclSocket* recvSocket, uint64_t magic, volatile uint32_t* abortFlag) { NCCLCHECK(ncclSocketInit(sendSocket, addr, magic, ncclSocketTypeBootstrap, abortFlag)); NCCLCHECK(ncclSocketConnect(sendSocket)); NCCLCHECK(ncclSocketInit(recvSocket)); NCCLCHECK(ncclSocketAccept(recvSocket, listenSock)); return ncclSuccess; } static ncclResult_t ringAllInfo(struct ncclComm* comm, struct bootstrapState* state, union ncclSocketAddress* peerAddresss, union ncclSocketAddress* peerProxy, uint64_t* peerUDS, struct rasRankInit* rasRanks) { ncclResult_t res = ncclSuccess; int rank = comm->rank; int nRanks = comm->nRanks; struct bootstrapRingData { union ncclSocketAddress peerAddress; union ncclSocketAddress peerProxy; uint64_t peerUDS; struct rasRankInit rasRank; }* ringData = NULL; NCCLCHECK(ncclCalloc(&ringData, nRanks)); // pack if (peerAddresss) memcpy(&(ringData[rank].peerAddress), peerAddresss + rank, sizeof(union ncclSocketAddress)); if (peerProxy) memcpy(&(ringData[rank].peerProxy), peerProxy + rank, sizeof(union ncclSocketAddress)); if (peerUDS) memcpy(&(ringData[rank].peerUDS), peerUDS + rank, sizeof(uint64_t)); if (rasRanks) memcpy(&(ringData[rank].rasRank), rasRanks + rank, sizeof(*rasRanks)); // allgather NCCLCHECKGOTO(bootstrapAllGather(state, ringData, sizeof(struct bootstrapRingData)), res, exit); // unpack for (int irank = 0; irank < nRanks; ++irank) { if (peerAddresss) memcpy(peerAddresss + irank, &(ringData[irank].peerAddress), sizeof(union ncclSocketAddress)); if (peerProxy) memcpy(peerProxy + irank, &(ringData[irank].peerProxy), sizeof(union ncclSocketAddress)); if (peerUDS) memcpy(peerUDS + irank, &(ringData[irank].peerUDS), sizeof(uint64_t)); if (rasRanks) memcpy(rasRanks + irank, &(ringData[irank].rasRank), sizeof(*rasRanks)); } exit: free(ringData); return ncclSuccess; } static ncclResult_t sendToRoot(struct ncclBootstrapHandle* handle, struct ncclComm* comm, struct extInfo* info) { ncclResult_t ret = ncclSuccess; struct ncclSocket sock; NCCLCHECK(ncclSocketInit(&sock, &handle->addr, handle->magic, ncclSocketTypeBootstrap, comm->abortFlag)); NCCLCHECKGOTO(ncclSocketConnect(&sock), ret, fail); NCCLCHECKGOTO(socketSend(&sock, info, sizeof(struct extInfo)), ret, fail); NCCLCHECK(ncclSocketClose(&sock)); return ret; fail: (void)ncclSocketClose(&sock); return ret; } NCCL_PARAM(StaggerRate, "UID_STAGGER_RATE", 7000); NCCL_PARAM(StaggerThreshold, "UID_STAGGER_THRESHOLD", 256); NCCL_PARAM(RasEnable, "RAS_ENABLE", 1); ncclResult_t bootstrapInit(int nHandles, void* handles, struct ncclComm* comm, struct ncclComm* parent) { ncclResult_t result = ncclSuccess; int rank = comm->rank; int nranks = comm->nRanks; // char nextPeerHandle[NCCL_NET_HANDLE_MAXSIZE]; struct bootstrapState* state; struct ncclSocket* proxySocket; struct ncclSocket sock, listenSockRoot; struct extInfo info = {0}; union ringConnectInfo nextPeer; bool performRasAddRanks = true; struct rasRankInit* rasRanks = nullptr; uint64_t timers[BOOTSTRAP_INIT_TIME_N] = {0}; NCCLCHECK(ncclCalloc(&state, 1)); state->rank = rank; state->nranks = nranks; state->cudaDev = comm->cudaDev; state->abortFlag = comm->abortFlag; state->net = comm->ncclNet; comm->bootstrap = state; // Set magic: for grow existing ranks, receive from coordinator; otherwise use handle magic. // This is consistent with the magic created in ncclCommGetUniqueId. if (handles != NULL) { comm->magic = state->magic = BOOTSTRAP_HANDLE(handles, 0)->magic; // state and comm magic set to the first magic ID } else if (parent != NULL) { comm->magic = state->magic = hashCombine(parent->magic, parent->childCount); } else { WARN("bootstrapInit: handles and parent are NULL"); return ncclSystemError; } TRACE(NCCL_BOOTSTRAP, "rank %d nranks %d", rank, nranks); BOOTSTRAP_PROF_OPEN(timers[BOOTSTRAP_INIT_TIME_TOTAL]); // fill up the info info.nranks = nranks; info.nroots = nHandles; // get the ring connection info memset(&nextPeer, 0, sizeof(union ringConnectInfo)); BOOTSTRAP_PROF_OPEN(timers[BOOTSTRAP_INIT_TIME_CREATE]); if (ncclParamBootstrapNetEnable()) { // Create net interface for other ranks to contact me (all gather) NCCLCHECK(netGetDevice(rank, comm, &STATE_LISTEN(state, net.dev))); NCCLCHECK(state->net->listen(comm->netContext, STATE_LISTEN(state, net.dev), STATE_LISTEN(state, net.handle), &STATE_LISTEN(state, net.comm))); memcpy(info.connectInfo.handle, STATE_LISTEN(state, net.handle), NCCL_NET_HANDLE_MAXSIZE); } else { // create socket for ring neightbor to contact mee NCCLCHECK(createListenSocket(comm, comm->magic, &STATE_LISTEN(state, socket), &info.connectInfo.addr, ncclSocketTypeBootstrap)); } // Create socket for root to contact me using the root's magic // For grow operations, offset is parent->nRanks - 1 (last existing rank joins the root) // For normal init, offset is 0 int offset = 0; if(comm->isGrow) { if(parent != NULL) { offset = parent->nRanks - 1; } else { if(handles != NULL) { offset = BOOTSTRAP_HANDLE(handles, 0)->nRanks - 1; } else { WARN("bootstrapInit: handles and parent are NULL"); return ncclSystemError; } } } int curr_root = rootIdFromRank(rank, nranks, nHandles, offset); if(curr_root >= 0) NCCLCHECK(createListenSocket(comm, BOOTSTRAP_HANDLE(handles, curr_root)->magic, &listenSockRoot, &info.listenRootAddress, ncclSocketTypeBootstrap)); BOOTSTRAP_PROF_CLOSE(timers[BOOTSTRAP_INIT_TIME_CREATE]); // stagger connection times to avoid an overload of the root BOOTSTRAP_PROF_OPEN(timers[BOOTSTRAP_INIT_TIME_DELAY]); int nRankRoot = nRankFromRoot(curr_root, nranks, nHandles, offset); if (nRankRoot > ncclParamStaggerThreshold()) { // for socket the message rate in microsec double msg_rate = ncclParamStaggerRate() / 1.0e6; long musec = localIdFromRoot(rank, curr_root, nranks, nHandles, offset) / msg_rate; TRACE(NCCL_BOOTSTRAP, "rank %d delaying connection to root by %ld microsec", rank, musec); std::this_thread::sleep_for(std::chrono::microseconds(musec)); } BOOTSTRAP_PROF_CLOSE(timers[BOOTSTRAP_INIT_TIME_DELAY]); // send info on my listening socket to root BOOTSTRAP_PROF_OPEN(timers[BOOTSTRAP_INIT_TIME_SEND]); // send contact info to my own root info.rank = rank; info.iroot = curr_root; info.offset = offset; if(curr_root >= 0) NCCLCHECK(sendToRoot(BOOTSTRAP_HANDLE(handles, curr_root), comm, &info)); if(parent && comm->isGrow && rank != 0) { // Grow: Ranks 1 to N-1 use the parent bootstrap to send connection information to the previous rank NCCLCHECK(bootstrapSend(parent->bootstrap, rank - 1, 0, &info.connectInfo, sizeof(info.connectInfo))); } // if needed, send the connection info to the previous root // commGrow with more than = 1 rank in the parent comm is a special case of multiroot if (((comm->isGrow && parent && (parent->nRanks > 1)) || nHandles > 1) && isFirstFromRoot(rank, curr_root, nranks, nHandles, offset)) { int prev_rank = BOOTSTRAP_PID(rank - 1, nranks); int prev_root = rootIdFromRank(prev_rank, nranks, nHandles, offset); info.rank = prev_rank + 1; // my rank as seen by the previous root info.iroot = prev_root; // only send if the root is valid, existing rank N-1 will use the bootstrapSend just above if(prev_root >= 0) NCCLCHECK(sendToRoot(BOOTSTRAP_HANDLE(handles, prev_root), comm, &info)); } BOOTSTRAP_PROF_CLOSE(timers[BOOTSTRAP_INIT_TIME_SEND]); // get info on my "next" rank in the bootstrap ring from root BOOTSTRAP_PROF_OPEN(timers[BOOTSTRAP_INIT_TIME_RECV]); if(curr_root >= 0){ NCCLCHECK(ncclSocketInit(&sock)); NCCLCHECK(ncclSocketAccept(&sock, &listenSockRoot)); NCCLCHECK(socketRecv(&sock, &nextPeer, sizeof(nextPeer))); NCCLCHECK(ncclSocketClose(&sock)); NCCLCHECK(ncclSocketClose(&listenSockRoot)); } if (parent && comm->isGrow && rank != parent->nRanks - 1) { // Grow: Ranks 0 to N-2 use the parent bootstrap to recv connection information to the next rank. This is consistent with the bootstrapSend above. NCCLCHECK(bootstrapRecv(parent->bootstrap, rank + 1, 0, &nextPeer, sizeof(nextPeer))); } BOOTSTRAP_PROF_CLOSE(timers[BOOTSTRAP_INIT_TIME_RECV]); // accept and connect the ring network if (ncclParamBootstrapNetEnable()) { NCCLCHECK(netRingConnect(comm->netContext, state->net, &state->listen, nextPeer.handle, &STATE_RING(state, net.sendComm), &STATE_RING(state, net.sendDevHandle), &STATE_RING(state, net.recvComm), &STATE_RING(state, net.recvDevHandle), state->abortFlag)); } else { NCCLCHECK(socketRingConnect(&nextPeer.addr, &STATE_RING(state, socket.send), &STATE_LISTEN(state, socket), &STATE_RING(state, socket.recv), comm->magic, state->abortFlag)); } // AllGather all listen handlers // in case of failure, those resources will be free'd when calling bootstrapDestroy, so we can return immediatly NCCLCHECK(ncclCalloc(&state->peerProxyAddresses, nranks)); NCCLCHECK(ncclCalloc(&proxySocket, 1)); NCCLCHECKGOTO(createListenSocket(comm, comm->magic, proxySocket, state->peerProxyAddresses + rank, ncclSocketTypeProxy), result, fail); NCCLCHECKGOTO(ncclCalloc(&state->peerProxyAddressesUDS, nranks), result, fail); NCCLCHECKGOTO(getUDS(state->peerProxyAddressesUDS + rank), result, fail); // create a socket for others to reach out (P2P) union ncclSocketAddress peerSocketAddress; NCCLCHECKGOTO(createListenSocket(comm, comm->magic, &STATE_LISTEN(state, peerSocket), &peerSocketAddress, ncclSocketTypeBootstrap), result, fail); NCCLCHECKGOTO(ncclCalloc(&state->peerP2pAddresses, nranks), result, fail); memcpy(state->peerP2pAddresses + rank, &peerSocketAddress, sizeof(union ncclSocketAddress)); // Initialize RAS if (ncclParamRasEnable() == 1) { // The RAS thread will take care of freeing the memory allocated below. NCCLCHECK(ncclCalloc(&rasRanks, nranks)); memcpy(&rasRanks[rank].addr, &bootstrapNetIfAddr, sizeof(rasRanks[rank].addr)); rasRanks[rank].pid = ncclOsGetPid(); rasRanks[rank].cudaDev = comm->cudaDev; rasRanks[rank].nvmlDev = comm->nvmlDev; rasRanks[rank].hostHash = getHostHash(); rasRanks[rank].pidHash = getPidHash(); if (ncclRasCommInit(comm, rasRanks+rank) != ncclSuccess) { INFO(NCCL_INIT|NCCL_RAS, "Continuing in spite of a RAS initialization error"); // We should still participate in the ringAllInfo below as the peers will be waiting for us. // Just make sure that the address is clearly invalid... memset(rasRanks+rank, '\0', sizeof(*rasRanks)); performRasAddRanks = false; } } BOOTSTRAP_PROF_OPEN(timers[BOOTSTRAP_INIT_TIME_RING]); NCCLCHECKGOTO(ringAllInfo(comm, state, state->peerP2pAddresses, state->peerProxyAddresses, state->peerProxyAddressesUDS, rasRanks), result, fail); BOOTSTRAP_PROF_CLOSE(timers[BOOTSTRAP_INIT_TIME_RING]); // Create the service proxy and get the UDS NCCLCHECKGOTO(ncclProxyInit(comm, proxySocket, state->peerProxyAddresses, state->peerProxyAddressesUDS), result, fail); if (ncclParamRasEnable() == 1 && performRasAddRanks) { if (ncclRasAddRanks(rasRanks, nranks) != ncclSuccess) INFO(NCCL_INIT|NCCL_RAS, "Continuing in spite of a RAS initialization error"); } BOOTSTRAP_PROF_CLOSE(timers[BOOTSTRAP_INIT_TIME_TOTAL]); TRACE(NCCL_BOOTSTRAP, "rank %d nranks %d - DONE", rank, nranks); INFO(NCCL_BOOTSTRAP | NCCL_PROFILE, "Bootstrap timings total %f (create %f, send %f, recv %f, ring %f, delay %f)", timers[BOOTSTRAP_INIT_TIME_TOTAL] / 1e9, timers[BOOTSTRAP_INIT_TIME_CREATE] / 1e9, timers[BOOTSTRAP_INIT_TIME_SEND] / 1e9, timers[BOOTSTRAP_INIT_TIME_RECV] / 1e9, timers[BOOTSTRAP_INIT_TIME_RING] / 1e9, timers[BOOTSTRAP_INIT_TIME_DELAY] / 1e9); exit: return result; fail: free(proxySocket); goto exit; } ncclResult_t bootstrapSplit(uint64_t magic, struct ncclComm* comm, struct ncclComm* parent, int color, int key, int* parentRanks) { ncclResult_t ret = ncclSuccess; int rank = comm->rank; int nranks = comm->nRanks; int prev, next; union ringConnectInfo info; union ringConnectInfo nextPeer; struct ncclSocket* proxySocket = NULL; struct bootstrapState* state; NCCLCHECKGOTO(ncclCalloc(&state, 1), ret, fail); state->rank = rank; state->nranks = nranks; state->cudaDev = comm->cudaDev; state->abortFlag = comm->abortFlag; state->net = comm->ncclNet; comm->bootstrap = state; comm->magic = state->magic = magic; prev = parentRanks[(rank - 1 + nranks) % nranks]; next = parentRanks[(rank + 1) % nranks]; // create a handle for the others to reach out to me if (ncclParamBootstrapNetEnable()) { NCCLCHECKGOTO(netGetDevice(rank, comm, &STATE_LISTEN(state, net.dev)), ret, fail); NCCLCHECKGOTO(state->net->listen(comm->netContext, STATE_LISTEN(state, net.dev), STATE_LISTEN(state, net.handle), &STATE_LISTEN(state, net.comm)), ret, fail); memcpy(info.handle, STATE_LISTEN(state, net.handle), NCCL_NET_HANDLE_MAXSIZE); } else { // create socket for ring neightbor to contact mee NCCLCHECK(createListenSocket(comm, comm->magic, &STATE_LISTEN(state, socket), &info.addr, ncclSocketTypeBootstrap)); } // create a socket for others to reach out (P2P) union ncclSocketAddress peerSocketAddress; NCCLCHECK(createListenSocket(comm, comm->magic, &STATE_LISTEN(state, peerSocket), &peerSocketAddress, ncclSocketTypeBootstrap)); if (ncclParamRasEnable() == 1) { if (ncclRasCommInit(comm, nullptr) != ncclSuccess) INFO(NCCL_INIT|NCCL_RAS, "Continuing in spite of a RAS initialization error"); } // Get addr from next rank using the parent's connections NCCLCHECKGOTO(bootstrapSend(parent->bootstrap, prev, BOOTSTRAP_TAG_COMMSPLIT, &info, sizeof(union ringConnectInfo)), ret, fail); NCCLCHECKGOTO(bootstrapRecv(parent->bootstrap, next, BOOTSTRAP_TAG_COMMSPLIT, &nextPeer, sizeof(union ringConnectInfo)), ret, fail); if (ncclParamBootstrapNetEnable()) { NCCLCHECKGOTO(netRingConnect(comm->netContext, state->net, &state->listen, nextPeer.handle, &STATE_RING(state, net.sendComm), &STATE_RING(state, net.sendDevHandle), &STATE_RING(state, net.recvComm), &STATE_RING(state, net.recvDevHandle), state->abortFlag), ret, fail); } else { NCCLCHECK(socketRingConnect(&nextPeer.addr, &STATE_RING(state, socket.send), &STATE_LISTEN(state, socket), &STATE_RING(state, socket.recv), comm->magic, state->abortFlag)); } NCCLCHECKGOTO(ncclCalloc(&state->peerP2pAddresses, nranks), ret, fail); memcpy(state->peerP2pAddresses + rank, &peerSocketAddress, sizeof(union ncclSocketAddress)); if (parent->shareResources) { /* map local rank to top parent local rank. */ for (int i = 0; i < nranks; ++i) { comm->topParentRanks[i] = parent->topParentRanks[parentRanks[i]]; } NCCLCHECKGOTO(ringAllInfo(comm, state, state->peerP2pAddresses, NULL, NULL, NULL), ret, fail); } else { NCCLCHECKGOTO(ncclCalloc(&state->peerProxyAddresses, nranks), ret, fail); NCCLCHECKGOTO(ncclCalloc(&state->peerProxyAddressesUDS, nranks), ret, fail); // Create the service proxy and get the UDS NCCLCHECKGOTO(ncclCalloc(&proxySocket, 1), ret, fail); NCCLCHECKGOTO(getUDS(state->peerProxyAddressesUDS + rank), ret, fail); NCCLCHECKGOTO(createListenSocket(comm, comm->magic, proxySocket, state->peerProxyAddresses + rank, ncclSocketTypeProxy), ret, fail); NCCLCHECKGOTO(ringAllInfo(comm, state, state->peerP2pAddresses, state->peerProxyAddresses, state->peerProxyAddressesUDS, NULL), ret, fail); NCCLCHECKGOTO(ncclProxyInit(comm, proxySocket, state->peerProxyAddresses, state->peerProxyAddressesUDS), ret, fail); } TRACE(NCCL_BOOTSTRAP, "bootstrapSplit: comm %p parent %p rank %d nranks %d color %d key %d prev %d next %d - DONE", comm, parent, rank, nranks, color, key, prev, next); exit: return ret; fail: free(proxySocket); goto exit; } struct socketAckInfo { int rank; int tag; }; static ncclResult_t socketConnect(void* commState, int peer, int tag, struct ncclSocket* sock) { ncclResult_t ret = ncclSuccess; struct bootstrapState* state = (struct bootstrapState*)commState; struct socketAckInfo ack = (struct socketAckInfo){.rank = state->rank, .tag = tag}; NCCLCHECKGOTO(ncclSocketInit(sock, state->peerP2pAddresses + peer, state->magic, ncclSocketTypeBootstrap, state->abortFlag), ret, fail); NCCLCHECKGOTO(ncclSocketConnect(sock), ret, fail); NCCLCHECKGOTO(socketSend(sock, &ack, sizeof(struct socketAckInfo)), ret, fail); return ncclSuccess; fail: (void)ncclSocketClose(sock); return ret; } ncclResult_t bootstrapSend(void* commState, int peer, int tag, void* data, int size) { ncclResult_t ret = ncclSuccess; struct ncclSocket sock; TRACE(NCCL_BOOTSTRAP, "Sending to peer=%d tag=%d size=%d", peer, tag, size); NCCLCHECK(socketConnect(commState, peer, tag, &sock)); NCCLCHECKGOTO(socketSend(&sock, data, size), ret, fail); TRACE(NCCL_BOOTSTRAP, "Sent to peer=%d tag=%d size=%d", peer, tag, size); NCCLCHECK(ncclSocketClose(&sock)); return ret; fail: (void)ncclSocketClose(&sock); return ret; } // Bootstrap send/receive functions static ncclResult_t unexpectedEnqueue(struct bootstrapState* state, int peer, int tag, struct ncclSocket* sock) { // New unex struct unexConn* unex; NCCLCHECK(ncclCalloc(&unex, 1)); unex->peer = peer; unex->tag = tag; memcpy(&unex->sock, sock, sizeof(struct ncclSocket)); // Enqueue struct unexConn* list = state->unexpectedConnections; if (list == NULL) { state->unexpectedConnections = unex; return ncclSuccess; } while (list->next) list = list->next; list->next = unex; return ncclSuccess; } static ncclResult_t unexpectedDequeue(struct bootstrapState* state, int peer, int tag, struct ncclSocket* sock, int* found) { struct unexConn* elem = state->unexpectedConnections; struct unexConn* prev = NULL; *found = 0; while (elem) { // peer < 0 means wildcard (accept from any peer) if ((peer < 0 || elem->peer == peer) && elem->tag == tag) { if (prev == NULL) { state->unexpectedConnections = elem->next; } else { prev->next = elem->next; } memcpy(sock, &elem->sock, sizeof(struct ncclSocket)); free(elem); *found = 1; return ncclSuccess; } prev = elem; elem = elem->next; } return ncclSuccess; } static void unexpectedFree(struct bootstrapState* state) { struct unexConn* elem = state->unexpectedConnections; struct unexConn* prev = NULL; while (elem) { prev = elem; elem = elem->next; free(prev); } return; } // We can't know who we'll receive from, so we need to receive everything at once static ncclResult_t socketAccept(void* commState, int peer, int tag, struct ncclSocket* sock) { ncclResult_t ret = ncclSuccess; struct bootstrapState* state = (struct bootstrapState*)commState; // Search unexpected connections first int found; NCCLCHECK(unexpectedDequeue(state, peer, tag, sock, &found)); if (found) return ncclSuccess; // Then look for new connections while (1) { struct socketAckInfo ack = {0}; NCCLCHECKGOTO(ncclSocketInit(sock), ret, fail); NCCLCHECKGOTO(ncclSocketAccept(sock, &STATE_LISTEN(state, peerSocket)), ret, fail); NCCLCHECKGOTO(socketRecv(sock, &ack, sizeof(struct socketAckInfo)), ret, fail); // Match: tag must match, and peer must match (peer < 0 means wildcard) if (ack.tag == tag && (peer < 0 || ack.rank == peer)) return ncclSuccess; // No match: queue for later and try next connection NCCLCHECKGOTO(unexpectedEnqueue(state, ack.rank, ack.tag, sock), ret, fail); } return ncclSuccess; fail: (void)ncclSocketClose(sock); return ret; } // We can't know who we'll receive from, so we need to receive everything at once ncclResult_t bootstrapRecv(void* commState, int peer, int tag, void* data, int size) { ncclResult_t ret; struct ncclSocket sock; NCCLCHECK(socketAccept(commState, peer, tag, &sock)); TRACE(NCCL_BOOTSTRAP, "Receiving tag=%d peer=%d size=%d", tag, peer, size); NCCLCHECKGOTO(socketRecv(&sock, ((char*)data), size), ret, fail); NCCLCHECKGOTO(ncclSocketClose(&sock, /*wait*/true), ret, fail); return ret; fail: (void)ncclSocketClose(&sock); return ret; } static ncclResult_t netRingAllGather(ncclNet_t* net, void* sendComm, void* recvComm, int rank, int nranks, char* data, int size, volatile uint32_t* abortFlag) { ncclResult_t res; uint64_t tFirst = 0, tRest = 0; void* sendDataHandle = NULL; void* recvDataHandle = NULL; NCCLCHECKGOTO(netReg(net, sendComm, data, nranks * size, &sendDataHandle), res, exit); NCCLCHECKGOTO(netReg(net, recvComm, data, nranks * size, &recvDataHandle), res, exit); /* Simple ring based AllGather * At each step i receive data from (rank-i-1) from prev * and send previous step's data from (rank-i) to next */ TRACE(NCCL_BOOTSTRAP, "NetRingAllGather started"); BOOTSTRAP_PROF_OPEN(tFirst); for (int i = 0; i < nranks - 1; i++) { int tag = i; size_t rslice = (rank - i - 1 + nranks) % nranks; size_t sslice = (rank - i + nranks) % nranks; void* recv_data = data + rslice * size; void* send_data = data + sslice * size; NCCLCHECKGOTO(netSendRecv(net, sendComm, send_data, size, sendDataHandle, recvComm, recv_data, size, recvDataHandle, tag, abortFlag), res, exit); if (i == 0) { BOOTSTRAP_PROF_CLOSE(tFirst); BOOTSTRAP_PROF_OPEN(tRest); } } BOOTSTRAP_PROF_CLOSE(tRest); TRACE(NCCL_BOOTSTRAP | NCCL_PROFILE, "netRingAllGather first message in %f (%f MB/sec), rest in %f (%f MB/sec)", tFirst / 1e9, (size / 1e6) / (tFirst / 1e9), tRest / 1e9, (nranks - 1) * (size / 1e6) / (tRest / 1e9)); exit: // do not fail in case of error, try to deregister as much as possible if (sendDataHandle) netDereg(net, sendComm, &sendDataHandle); if (recvDataHandle) netDereg(net, recvComm, &recvDataHandle); return res; } static ncclResult_t socketRingAllGather(struct ncclSocket* nextSock, struct ncclSocket* prevSock, int rank, int nranks, char* data, int size) { ncclResult_t res = ncclSuccess; uint64_t tFirst = 0, tRest = 0; /* Simple ring based AllGather * At each step i receive data from (rank-i-1) from prev * and send previous step's data from (rank-i) to next */ TRACE(NCCL_BOOTSTRAP, "socketRingAllGather started: rank=%d nranks=%d", rank, nranks); int totalSteps = nranks / 2; TRACE(NCCL_BOOTSTRAP, "bidirectional bootstrap: totalSteps=%d", totalSteps); BOOTSTRAP_PROF_OPEN(tFirst); for (int step = 0; step < totalSteps; step++) { // N ranks requires (N-1)/2 steps for the double ring algorithm. If N is even, the last step is requires a single send/recv bool isFinalUnidirectional = (step == totalSteps - 1) && (nranks % 2 == 0); // Ring0: ring from previous to next int sendSliceRing0 = (rank - step + nranks) % nranks; // Send this slice to next neighbor int recvSliceRing0 = (rank - step - 1 + nranks) % nranks; // Receive this slice from prev neighbor // Ring1: ring from next to previous int sendSliceRing1 = (rank + step) % nranks; // Send this slice to prev neighbor int recvSliceRing1 = (rank + step + 1) % nranks; // Receive this slice from next neighbor if (isFinalUnidirectional) { // Final unidirectional step, only Ring0 is used NCCLCHECKGOTO(socketSendRecv(nextSock, data + sendSliceRing0 * size, size, prevSock, data + recvSliceRing0 * size, size), res, exit); } else { // Bidirectional step: Ring0 and Ring1 are used simultaneously struct ncclSocketOp ops[4] = { {NCCL_SOCKET_SEND, nextSock, data + sendSliceRing0 * size, size, 0}, // Ring0: send to next {NCCL_SOCKET_RECV, prevSock, data + recvSliceRing0 * size, size, 0}, // Ring0: recv from prev {NCCL_SOCKET_SEND, prevSock, data + sendSliceRing1 * size, size, 0}, // Ring1: send to prev {NCCL_SOCKET_RECV, nextSock, data + recvSliceRing1 * size, size, 0} // Ring1: recv from next }; NCCLCHECKGOTO(socketDoubleSendRecv(ops), res, exit); } if (step == 0) { BOOTSTRAP_PROF_CLOSE(tFirst); BOOTSTRAP_PROF_OPEN(tRest); } } BOOTSTRAP_PROF_CLOSE(tRest); TRACE(NCCL_BOOTSTRAP | NCCL_PROFILE, "socketRingAllGather first message in %f (%f MB/sec), rest in %f (%f MB/sec)", tFirst / 1e9, (size / 1e6) / (tFirst / 1e9), tRest / 1e9, (nranks - 1) * (size / 1e6) / (tRest / 1e9)); exit: return res; } ncclResult_t bootstrapAllGather(void* commState, void* allData, int size) { ncclResult_t res = ncclSuccess; struct bootstrapState* state = (struct bootstrapState*)commState; int rank = state->rank; int nranks = state->nranks; TRACE(NCCL_BOOTSTRAP, "rank %d nranks %d size %d - AllGather", rank, nranks, size); uint64_t time = 0; BOOTSTRAP_PROF_OPEN(time); if (ncclParamBootstrapNetEnable()) { NCCLCHECKGOTO(netRingAllGather(state->net, STATE_RING(state, net.sendComm), STATE_RING(state, net.recvComm), rank, nranks, (char*)allData, size, state->abortFlag), res, exit); } else { NCCLCHECKGOTO(socketRingAllGather(&STATE_RING(state, socket.send), &STATE_RING(state, socket.recv), rank, nranks, (char*)allData, size), res, exit); } exit: BOOTSTRAP_PROF_CLOSE(time); TRACE(NCCL_BOOTSTRAP | NCCL_PROFILE, "bootstrapAllGather for %d B done in %f sec: %f MB/sec", size, time / 1e9, (nranks * size / 1e6) / (time / 1e9)); TRACE(NCCL_BOOTSTRAP, "rank %d nranks %d size %d - AllGather DONE", rank, nranks, size); return res; } static ncclResult_t bootstrapP2PBarrier(void* commState, int* ranks, int rank, int nranks, int tag) { if (nranks == 1) return ncclSuccess; /* Simple [intra] process barrier * * Based on the dissemination algorithm by Debra Hensgen, Raphael Finkel, and Udi Manbet, * "Two Algorithms for Barrier Synchronization," International Journal of Parallel Programming, 17(1):1-17, 1988" */ int data[1] = {0}; for (int mask = 1; mask < nranks; mask <<= 1) { int src = (rank - mask + nranks) % nranks; int dst = (rank + mask) % nranks; NCCLCHECK(bootstrapSend(commState, ranks ? ranks[dst] : dst, tag, data, sizeof(data))); NCCLCHECK(bootstrapRecv(commState, ranks ? ranks[src] : src, tag, data, sizeof(data))); } return ncclSuccess; } ncclResult_t bootstrapIntraNodeBarrier(void* commState, int* ranks, int rank, int nranks, int tag) { uint64_t time = 0; BOOTSTRAP_PROF_OPEN(time); NCCLCHECK(bootstrapP2PBarrier(commState, ranks, rank, nranks, tag)); BOOTSTRAP_PROF_CLOSE(time); TRACE(NCCL_BOOTSTRAP | NCCL_PROFILE, "bootstrapIntraNodeBarrier done in %f sec", time / 1e9); return ncclSuccess; } ncclResult_t bootstrapBarrier(void* commState, int rank, int nranks, int tag) { uint64_t time = 0; BOOTSTRAP_PROF_OPEN(time); NCCLCHECK(bootstrapP2PBarrier(commState, NULL, rank, nranks, tag)); BOOTSTRAP_PROF_CLOSE(time); TRACE(NCCL_BOOTSTRAP | NCCL_PROFILE, "bootstrapBarrier done in %f sec", time / 1e9); return ncclSuccess; } ncclResult_t bootstrapIntraNodeAllGather(void* commState, int* ranks, int rank, int nranks, void* allData, int size) { if (nranks == 1) return ncclSuccess; TRACE(NCCL_INIT, "rank %d nranks %d size %d - ENTER", rank, nranks, size); int prevRank = ranks[(rank - 1 + nranks) % nranks]; int nextRank = ranks[(rank + 1) % nranks]; // intraNode bootstrap is done defacto using the socket-based implementation struct ncclSocket recvSocket, sendSocket; NCCLCHECK(socketConnect(commState, nextRank, BOOTSTRAP_TAG_INTRANODE_ALLGATHER, &sendSocket)); NCCLCHECK(socketAccept(commState, prevRank, BOOTSTRAP_TAG_INTRANODE_ALLGATHER, &recvSocket)); NCCLCHECK(socketRingAllGather(&sendSocket, &recvSocket, rank, nranks, (char*)allData, size)); NCCLCHECK(ncclSocketClose(&sendSocket)); NCCLCHECK(ncclSocketClose(&recvSocket)); TRACE(NCCL_INIT, "rank %d nranks %d size %d - DONE", rank, nranks, size); return ncclSuccess; } // [IntraNode] in-place Broadcast static ncclResult_t bootstrapP2PBroadcast(void* commState, int* ranks, int rank, int nranks, int root, void* bcastData, int size) { if (nranks == 1) return ncclSuccess; if (rank == root) { for (int i = 0; i < nranks; i++) { if (i != root) NCCLCHECK(bootstrapSend(commState, ranks ? ranks[i] : i, /*tag=*/ranks ? ranks[i] : i, bcastData, size)); } } else { NCCLCHECK(bootstrapRecv(commState, ranks ? ranks[root] : root, /*tag=*/ranks ? ranks[rank] : rank, bcastData, size)); } return ncclSuccess; } ncclResult_t bootstrapIntraNodeBroadcast(void* commState, int* ranks, int rank, int nranks, int root, void* bcastData, int size) { uint64_t time = 0; BOOTSTRAP_PROF_OPEN(time); NCCLCHECK(bootstrapP2PBroadcast(commState, ranks, rank, nranks, root, bcastData, size)); BOOTSTRAP_PROF_CLOSE(time); TRACE(NCCL_BOOTSTRAP | NCCL_PROFILE, "bootstrapIntraNodeBroadcast for %d B done in %f sec: %f MB/sec", size, time / 1e9, (nranks * size / 1e6) / (time / 1e9)); return ncclSuccess; } ncclResult_t bootstrapBroadcast(void* commState, int rank, int nranks, int root, void* bcastData, int size) { uint64_t time = 0; BOOTSTRAP_PROF_OPEN(time); NCCLCHECK(bootstrapP2PBroadcast(commState, NULL, rank, nranks, root, bcastData, size)); BOOTSTRAP_PROF_CLOSE(time); TRACE(NCCL_BOOTSTRAP | NCCL_PROFILE, "bootstrapBroadcast done in %f sec", time / 1e9); return ncclSuccess; } ncclResult_t bootstrapClose(void* commState) { if (commState == NULL) return ncclSuccess; struct bootstrapState* state = (struct bootstrapState*)commState; // close unexpected and return an error if we are not aborting and still operations in the pipe if (state->unexpectedConnections != NULL) { unexpectedFree(state); if (COMPILER_ATOMIC_LOAD(state->abortFlag, std::memory_order_acquire) == 0) { WARN("Unexpected connections are not empty"); return ncclInternalError; } } if (ncclParamBootstrapNetEnable()) { NCCLCHECK(state->net->closeSend(STATE_RING(state, net.sendComm))); NCCLCHECK(state->net->closeRecv(STATE_RING(state, net.recvComm))); NCCLCHECK(state->net->closeListen(STATE_LISTEN(state, net.comm))); } else { NCCLCHECK(ncclSocketClose(&STATE_RING(state, socket.send))); NCCLCHECK(ncclSocketClose(&STATE_RING(state, socket.recv))); NCCLCHECK(ncclSocketClose(&STATE_LISTEN(state, socket))); } // close the p2p socket NCCLCHECK(ncclSocketClose(&STATE_LISTEN(state, peerSocket))); // proxy things are free'd elsewhere free(state->peerP2pAddresses); free(state); return ncclSuccess; } ncclResult_t bootstrapAbort(void* commState) { if (commState == NULL) return ncclSuccess; struct bootstrapState* state = (struct bootstrapState*)commState; // when aborting we need to close the proxy here (maybe?) free(state->peerProxyAddresses); free(state->peerProxyAddressesUDS); NCCLCHECK(bootstrapClose(commState)); return ncclSuccess; } nccl-2.29.7-1/src/ce_coll.cc000066400000000000000000000660141515037102200153710ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "comm.h" #include "register_inline.h" #include #include "cudawrap.h" #include "ce_coll.h" #include "alloc.h" // Static constant for graph synchronization static const uint32_t GRAPH_SYNC_VALUE = 1; // Static constants for intra-batch synchronization to improve CE collective performance with large scale // Frequency of intra-batch synchronization static const uint32_t CE_COLL_INTRA_BATCH_SYNC_FREQ = 8; // Message threshold for intra-batch synchronization static const uint64_t CE_COLL_INTRA_BATCH_SYNC_MSG_THRESHOLD = 512*1024*1024; ncclResult_t ncclCeInit(struct ncclComm* comm) { ncclResult_t ret = ncclSuccess; uint8_t* ceDevBase = nullptr; size_t ceDevBaseSize = alignUp(comm->nRanks*sizeof(uint32_t), 16) * 2; ncclWindow_vidmem* ceWinDev = nullptr; ncclWindow_vidmem* ceWinDevHost = nullptr; // Ensure symmetric memory runtime is initialized NCCLCHECKGOTO(ncclDevrInitOnce(comm), ret, fail); // Allocate and register memory for the symmetric memory NCCLCHECKGOTO(ncclMemAlloc((void**)&ceDevBase, ceDevBaseSize), ret, fail); NCCLCHECKGOTO(ncclDevrWindowRegisterInGroup(comm, ceDevBase, ceDevBaseSize, NCCL_WIN_COLL_SYMMETRIC, &ceWinDev), ret, fail); NCCLCHECKGOTO(ncclShadowPoolToHost(&comm->devrState.shadows, ceWinDev, &ceWinDevHost), ret, fail); // Get the ncclDevrWindow from the winHost field comm->ceColl.ceSyncWin = (struct ncclDevrWindow*)ceWinDevHost->winHost; comm->ceColl.baseUCSymReadyOffset = 0; comm->ceColl.baseUCSymComplOffset = alignUp(comm->nRanks*sizeof(uint32_t), 16); comm->ceColl.baseUCSymReadyPtr = (uint8_t*)comm->ceColl.ceSyncWin->userPtr + comm->ceColl.baseUCSymReadyOffset; comm->ceColl.baseUCSymComplPtr = (uint8_t*)comm->ceColl.ceSyncWin->userPtr + comm->ceColl.baseUCSymComplOffset; comm->ceColl.ceSeqNum = 0; comm->ceColl.useCompletePtr = false; comm->ceColl.intraBatchSyncFreq = CE_COLL_INTRA_BATCH_SYNC_FREQ; comm->ceColl.intraBatchSyncMsgThreshold = CE_COLL_INTRA_BATCH_SYNC_MSG_THRESHOLD; INFO(NCCL_INIT, "Init CE, rank %d baseUCSymReadyPtr %p, baseUCSymComplPtr %p, seq num %d", comm->rank, comm->ceColl.baseUCSymReadyPtr, comm->ceColl.baseUCSymComplPtr, comm->ceColl.ceSeqNum); exit: return ret; fail: // Clean up partial initialization - both functions handle null safely ncclCommWindowDeregister(comm, ceWinDev); ncclMemFree(ceDevBase); goto exit; } ncclResult_t ncclCeFinalize(struct ncclComm* comm) { ncclResult_t ret = ncclSuccess; // Clean up ceInitTaskQueue while (!ncclIntruQueueEmpty(&comm->ceInitTaskQueue)) { struct ncclCeInitTask* task = ncclIntruQueueDequeue(&comm->ceInitTaskQueue); free(task); } // Clean up CE resources - continue cleanup even on errors to avoid leaks // Note: both functions handle null safely NCCLCHECKIGNORE(ncclCommWindowDeregister(comm, comm->ceColl.ceSyncWin ? comm->ceColl.ceSyncWin->vidmem : nullptr), ret); NCCLCHECKIGNORE(ncclMemFree(comm->ceColl.baseUCSymReadyPtr), ret); comm->ceColl.baseUCSymReadyPtr = nullptr; comm->ceColl.baseUCSymComplPtr = nullptr; comm->ceColl.ceSyncWin = nullptr; return ret; } bool ncclCeImplemented(ncclFunc_t coll, int/*ncclDevRedOp_t*/ red, ncclDataType_t ty) { int driverVersion; if (ncclCudaDriverVersion(&driverVersion) != ncclSuccess) return false; // CE is supported in CUDA 12.5 and later if (driverVersion >= 12050) { switch (coll) { case ncclFuncAllGather: case ncclFuncAlltoAll: case ncclFuncScatter: case ncclFuncGather: return true; default: return false; } } return false; } bool ncclCeAvailable(struct ncclComm* comm, ncclFunc_t coll, int/*ncclDevRedOp_t*/ red, ncclDataType_t ty, ncclSymRegType_t winRegType) { if (!ncclCeImplemented(coll, red, ty)) { TRACE(NCCL_TUNING, "Skipping CE collective: not implemented"); return false; } if (comm->nNodes > 1) { TRACE(NCCL_TUNING, "Skipping CE collective: comm is not a single node"); return false; } if (!comm->symmetricSupport) { TRACE(NCCL_TUNING, "Skipping CE collective: symmetric support is not enabled"); return false; } if (winRegType != ncclSymSendRegRecvReg && winRegType != ncclSymSendNonregRecvReg) { TRACE(NCCL_TUNING, "Skipping CE collective: window registration type %d is not supported", winRegType); return false; } return true; } ncclResult_t ncclPrepMCSync(struct ncclComm* comm, bool isComplete, CUstreamBatchMemOpParams* batchParams, size_t* opIdx, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; uint32_t* readyPtrs = (uint32_t*)comm->ceColl.baseUCSymReadyPtr; uint32_t* completePtrs = (uint32_t*)comm->ceColl.baseUCSymComplPtr; bool capturing = ncclCudaGraphValid(comm->planner.capturingGraph); uint32_t currentSeq = ++comm->ceColl.ceSeqNum; // Source pointer is either the constant graph sync value or the sequence number void* srcPtr = capturing ? (void*)&GRAPH_SYNC_VALUE : (void*)¤tSeq; // Wait value is either the constant graph sync value or the sequence number uint32_t waitValue = capturing ? GRAPH_SYNC_VALUE : currentSeq; // Use multi-cast address as destination pointer void* mcDstPtr; void* dstPtr = isComplete ? (void*)&completePtrs[comm->rank] : (void*)&readyPtrs[comm->rank]; size_t offset = (uint8_t*)dstPtr - (uint8_t*)comm->ceColl.ceSyncWin->userPtr; NCCLCHECKGOTO(ncclDevrGetLsaTeamPtrMC(comm, comm->ceColl.ceSyncWin, offset, ncclTeamLsa(comm), &mcDstPtr), ret, fail); // Write our own ready/complete flag to the multi-cast address CUDACHECKGOTO(cudaMemcpyAsync( mcDstPtr, srcPtr, sizeof(uint32_t), cudaMemcpyHostToDevice, stream), ret, fail); // Add local wait operations for every other rank for (int r = 0; r < comm->nRanks; ++r) { if (r == comm->rank) continue; batchParams[*opIdx] = {}; batchParams[*opIdx].waitValue.operation = CU_STREAM_MEM_OP_WAIT_VALUE_32; batchParams[*opIdx].waitValue.address = (CUdeviceptr)(isComplete ? (void*)&completePtrs[r] : (void*)&readyPtrs[r]); batchParams[*opIdx].waitValue.value = waitValue; batchParams[*opIdx].waitValue.flags = CU_STREAM_WAIT_VALUE_EQ; (*opIdx)++; } exit: return ret; fail: goto exit; } ncclResult_t ncclPrepUCSync(struct ncclComm* comm, bool isComplete, CUstreamBatchMemOpParams* batchParams, size_t* opIdx, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; uint32_t* readyPtrs = (uint32_t*)comm->ceColl.baseUCSymReadyPtr; uint32_t* completePtrs = (uint32_t*)comm->ceColl.baseUCSymComplPtr; bool capturing = ncclCudaGraphValid(comm->planner.capturingGraph); uint32_t currentSeq = ++comm->ceColl.ceSeqNum; // Write our own ready/complete flag to remote ranks using cudaMemcpyAsync for (int r = 0; r < comm->nRanks; ++r) { if (r == comm->rank) continue; void * peerDstPtr; void* dstPtr = isComplete ? (void*)&completePtrs[comm->rank] : (void*)&readyPtrs[comm->rank]; size_t offset = (uint8_t*)dstPtr - (uint8_t*)comm->ceColl.ceSyncWin->userPtr; NCCLCHECKGOTO(ncclDevrGetLsaRankPtr(comm, comm->ceColl.ceSyncWin, offset, r, &peerDstPtr), ret, fail); if (capturing) { CUDACHECKGOTO(cudaMemcpyAsync(peerDstPtr, &GRAPH_SYNC_VALUE, sizeof(uint32_t), cudaMemcpyHostToDevice, stream), ret, fail); } else { CUDACHECKGOTO(cudaMemcpyAsync(peerDstPtr, ¤tSeq, sizeof(uint32_t), cudaMemcpyHostToDevice, stream), ret, fail); } } // Add local wait operations for every other rank for (int r = 0; r < comm->nRanks; ++r) { if (r == comm->rank) continue; batchParams[*opIdx] = {}; batchParams[*opIdx].waitValue.operation = CU_STREAM_MEM_OP_WAIT_VALUE_32; batchParams[*opIdx].waitValue.address = (CUdeviceptr)(isComplete ? (void*)&completePtrs[r] : (void*)&readyPtrs[r]); batchParams[*opIdx].waitValue.value = capturing ? GRAPH_SYNC_VALUE : currentSeq; batchParams[*opIdx].waitValue.flags = CU_STREAM_WAIT_VALUE_EQ; (*opIdx)++; } exit: return ret; fail: goto exit; } ncclResult_t ncclMemOpSync(struct ncclComm* comm, struct ncclCeCollArgs* args, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; void* ceSyncHandle = NULL; // Get pointers to the ready and complete synchronization arrays uint32_t* readyPtrs = (uint32_t*)comm->ceColl.baseUCSymReadyPtr; uint32_t* completePtrs = (uint32_t*)comm->ceColl.baseUCSymComplPtr; // Allocate enough slots for all possible ops size_t batchSize = (comm->nvlsSupport ? NCCL_CE_SYNC_OPS_PER_RANK_MC : NCCL_CE_SYNC_OPS_PER_RANK_UC) * comm->nRanks; size_t opIdx = 0; CUstreamBatchMemOpParams* batchParams = nullptr; // Start CE sync profiling NCCLCHECKGOTO(ncclProfilerStartCeSyncEvent(comm, args, stream, &ceSyncHandle), ret, fail); // Prepare batch memory operations for synchronization NCCLCHECKGOTO(ncclCalloc(&batchParams, batchSize), ret, fail); if (comm->nvlsSupport) { NCCLCHECKGOTO(ncclPrepMCSync(comm, comm->ceColl.useCompletePtr, batchParams, &opIdx, stream), ret, fail); } else { NCCLCHECKGOTO(ncclPrepUCSync(comm, comm->ceColl.useCompletePtr, batchParams, &opIdx, stream), ret, fail); } // For CUDA graph capture, add reset operation if (ncclCudaGraphValid(comm->planner.capturingGraph)) { for (int i = 0; i < comm->nRanks; i++) { batchParams[opIdx] = {}; batchParams[opIdx].writeValue.operation = CU_STREAM_MEM_OP_WRITE_VALUE_32; batchParams[opIdx].writeValue.address = (CUdeviceptr)(comm->ceColl.useCompletePtr ? (void*)&completePtrs[i] : (void*)&readyPtrs[i]); batchParams[opIdx].writeValue.value = 0; batchParams[opIdx].writeValue.flags = CU_STREAM_WRITE_VALUE_DEFAULT; opIdx++; } } // Execute all memory operations in a single batch NCCLCHECKGOTO(ncclCuStreamBatchMemOp(stream, opIdx, batchParams), ret, fail); // Toggle the flag for next call comm->ceColl.useCompletePtr = !comm->ceColl.useCompletePtr; exit: // Stop CE sync profiling - always attempt if started, even on error ncclProfilerStopCeSyncEvent(comm, ceSyncHandle, stream); if (batchParams) free(batchParams); return ret; fail: goto exit; } ncclResult_t ncclCeInitBatchOpsParams(struct ncclCeBatchOpsParams* params, int nRanks) { ncclResult_t ret = ncclSuccess; void** srcs = nullptr; void** dsts = nullptr; size_t* sizes = nullptr; #if CUDART_VERSION >= 12080 cudaMemcpyAttributes* attrs = nullptr; size_t* attrIdxs = nullptr; #endif NCCLCHECKGOTO(ncclCalloc(&srcs, nRanks), ret, fail); NCCLCHECKGOTO(ncclCalloc(&dsts, nRanks), ret, fail); NCCLCHECKGOTO(ncclCalloc(&sizes, nRanks), ret, fail); #if CUDART_VERSION >= 12080 NCCLCHECKGOTO(ncclCalloc(&attrs, nRanks), ret, fail); NCCLCHECKGOTO(ncclCalloc(&attrIdxs, nRanks), ret, fail); #endif exit: params->srcs = srcs; params->dsts = dsts; params->sizes = sizes; params->numOps = 0; params->intraBatchSync = false; #if CUDART_VERSION >= 12080 params->attrs = attrs; params->attrIdxs = attrIdxs; params->numAttrs = 0; #endif return ret; fail: if (srcs) free(srcs); srcs = nullptr; if (dsts) free(dsts); dsts = nullptr; if (sizes) free(sizes); sizes = nullptr; #if CUDART_VERSION >= 12080 if (attrs) free(attrs); attrs = nullptr; if (attrIdxs) free(attrIdxs); attrIdxs = nullptr; #endif goto exit; } void ncclCeFreeBatchOpsParams(struct ncclCeBatchOpsParams* params) { if (params->srcs) free(params->srcs); params->srcs = nullptr; if (params->dsts) free(params->dsts); params->dsts = nullptr; if (params->sizes) free(params->sizes); params->sizes = nullptr; params->numOps = 0; params->intraBatchSync = false; #if CUDART_VERSION >= 12080 if (params->attrs) free(params->attrs); params->attrs = nullptr; if (params->attrIdxs) free(params->attrIdxs); params->attrIdxs = nullptr; params->numAttrs = 0; #endif } ncclResult_t ncclCeLaunchBatchOps(struct ncclComm* comm, struct ncclCeCollArgs* args, struct ncclCeBatchOpsParams* params, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; bool capturing; int driverVersion; void* ceBatchHandle = NULL; // cudaMemcpyBatchAsync does not accept the legacy null stream (e.g. PyTorch null stream). // Fall back to cudaMemcpyAsync per-op when stream is NULL. bool isLegacyStream; NCCLCHECKGOTO(ncclCudaStreamIsLegacyNull(stream, &isLegacyStream), ret, fail); // Start CE batch profiling NCCLCHECKGOTO(ncclProfilerStartCeBatchEvent(comm, args, params, stream, &ceBatchHandle), ret, fail); // Check if there are any operations to perform if (params->numOps == 0) goto exit; // Check if we are in a CUDA graph capture capturing = ncclCudaGraphValid(comm->planner.capturingGraph); NCCLCHECKGOTO(ncclCudaDriverVersion(&driverVersion), ret, fail); //--------------Graph capture / legacy stream-------------- // cudaMemcpyBatchAsync is not supported during CUDA graph capture or with legacy stream if (capturing || isLegacyStream) { for (int i =0; i < params->numOps; i++) { CUDACHECKGOTO(cudaMemcpyAsync( (void*)params->dsts[i], (void*)params->srcs[i], params->sizes[i], cudaMemcpyDeviceToDevice, stream), ret, fail); if (params->intraBatchSync && ((i+1) % comm->ceColl.intraBatchSyncFreq == 0) && ((i+1) < params->numOps)) { NCCLCHECKGOTO(ncclMemOpSync(comm, args, stream), ret, fail); } } } //--------------No graph capture / not legacy stream-------------- else { if (CUDART_VERSION >= 12080 && driverVersion >= 12080) { #if CUDART_VERSION >= 12080 // For CUDA 12.8+, use batch memory copy for better performance params->attrs[0] = {}; params->attrs[0].srcAccessOrder = cudaMemcpySrcAccessOrderStream; params->attrs[0].flags = cudaMemcpyFlagPreferOverlapWithCompute; params->attrIdxs[0] = 0; params->numAttrs = 1; if (params->intraBatchSync) { // Find the maximum transfer size to determine number of rounds size_t maxSize = 0; size_t totalSize = 0; for (int i = 0; i < params->numOps; i++) { if (params->sizes[i] > maxSize) { maxSize = params->sizes[i]; } totalSize += params->sizes[i]; } size_t chunkSize = comm->ceColl.intraBatchSyncMsgThreshold / params->numOps; int numRounds = (maxSize + chunkSize - 1) / chunkSize; size_t numTmpOps = params->numOps * numRounds; // Allocate temporary arrays for all chunked operations // Use ncclUniqueArrayPtr for automatic cleanup on any exit path ncclUniqueArrayPtr tmpDsts{nullptr}; ncclUniqueArrayPtr tmpSrcs{nullptr}; ncclUniqueArrayPtr tmpSizes{nullptr}; NCCLCHECKGOTO(ncclCalloc(tmpDsts, numTmpOps), ret, fail); NCCLCHECKGOTO(ncclCalloc(tmpSrcs, numTmpOps), ret, fail); NCCLCHECKGOTO(ncclCalloc(tmpSizes, numTmpOps), ret, fail); int opIdx = 0; for (int round = 0; round < numRounds; round++) { size_t offset = round * chunkSize; // Prepare chunk transfers for this round for (int i = 0; i < params->numOps; i++) { int index = (i+round) % params->numOps; if (offset < params->sizes[index]) { size_t remainingSize = params->sizes[index] - offset; size_t currentChunkSize = (remainingSize > chunkSize) ? chunkSize : remainingSize; tmpDsts[opIdx] = (void*)((uint8_t*)params->dsts[index] + offset); tmpSrcs[opIdx] = (void*)((uint8_t*)params->srcs[index] + offset); tmpSizes[opIdx] = currentChunkSize; opIdx++; } } } // Launch a single batch for all chunks if (opIdx > 0) { #if CUDART_VERSION >= 13000 CUDACHECKGOTO(cudaMemcpyBatchAsync( tmpDsts.get(), tmpSrcs.get(), tmpSizes.get(), opIdx, params->attrs, params->attrIdxs, params->numAttrs, stream), ret, fail); #else CUDACHECKGOTO(cudaMemcpyBatchAsync( tmpDsts.get(), tmpSrcs.get(), tmpSizes.get(), opIdx, params->attrs, params->attrIdxs, params->numAttrs, nullptr, stream), ret, fail); #endif } } else { // Use single batch for all operations #if CUDART_VERSION >= 13000 CUDACHECKGOTO(cudaMemcpyBatchAsync( params->dsts, params->srcs, params->sizes, params->numOps, params->attrs, params->attrIdxs, params->numAttrs, stream), ret, fail); #else CUDACHECKGOTO(cudaMemcpyBatchAsync( params->dsts, params->srcs, params->sizes, params->numOps, params->attrs, params->attrIdxs, params->numAttrs, nullptr, stream), ret, fail); #endif } #endif } else { // For older CUDA versions, fall back to individual transfers for (int i = 0; i < params->numOps; i++) { CUDACHECKGOTO(cudaMemcpyAsync( (void*)params->dsts[i], (void*)params->srcs[i], params->sizes[i], cudaMemcpyDeviceToDevice, stream), ret, fail); if (params->intraBatchSync && ((i+1) % comm->ceColl.intraBatchSyncFreq == 0) && ((i+1) < params->numOps)) { NCCLCHECKGOTO(ncclMemOpSync(comm, args, stream), ret, fail); } } } } exit: // Stop CE batch profiling - always attempt if started, even on error ncclProfilerStopCeBatchEvent(comm, ceBatchHandle, stream); return ret; fail: goto exit; } ncclResult_t ncclCeAllGather(struct ncclComm* comm, struct ncclCeCollArgs* args, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; const size_t chunkBytes = args->nElts * args->eltSize; uint8_t* mySendBuff = (uint8_t*)args->sendBuff; uint8_t* myRecvBuff = (uint8_t*)args->recvBuff + comm->rank * chunkBytes; void* peerRecvBuff; size_t offset; struct ncclCeBatchOpsParams batchOpsParams = {}; NCCLCHECKGOTO(ncclCeInitBatchOpsParams(&batchOpsParams, comm->nRanks), ret, fail); // Ensure all ranks are ready before starting transfers NCCLCHECKGOTO(ncclMemOpSync(comm, args, stream), ret, fail); // Copy own data to receive buffer if operation is out-of-place if (myRecvBuff != mySendBuff) { batchOpsParams.srcs[batchOpsParams.numOps] = (void*)mySendBuff; batchOpsParams.dsts[batchOpsParams.numOps] = (void*)myRecvBuff; batchOpsParams.sizes[batchOpsParams.numOps] = chunkBytes; batchOpsParams.numOps++; } // Copy data to other ranks for (int r = 1; r < comm->nRanks; r++) { int targetRank = (comm->rank + r) % comm->nRanks; offset = myRecvBuff - (uint8_t*)args->recvWin->userPtr; NCCLCHECKGOTO(ncclDevrGetLsaRankPtr(comm, args->recvWin, offset, targetRank, &peerRecvBuff), ret, fail); batchOpsParams.srcs[batchOpsParams.numOps] = (void*)mySendBuff; batchOpsParams.dsts[batchOpsParams.numOps] = (void*)peerRecvBuff; batchOpsParams.sizes[batchOpsParams.numOps] = chunkBytes; batchOpsParams.numOps++; } // Check if we need to perform intra-batch synchronization batchOpsParams.intraBatchSync = (batchOpsParams.numOps > comm->ceColl.intraBatchSyncFreq && chunkBytes*batchOpsParams.numOps >= comm->ceColl.intraBatchSyncMsgThreshold); // Launch the batch operations NCCLCHECKGOTO(ncclCeLaunchBatchOps(comm, args, &batchOpsParams, stream), ret, fail); // Ensure all transfers are complete across all ranks NCCLCHECKGOTO(ncclMemOpSync(comm, args, stream), ret, fail); exit: ncclCeFreeBatchOpsParams(&batchOpsParams); return ret; fail: goto exit; } ncclResult_t ncclCeAlltoAll(struct ncclComm* comm, struct ncclCeCollArgs* args, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; // Calculate the size of data each rank sends to every other rank const size_t chunkBytes = args->nElts * args->eltSize; uint8_t* mySendBuff = (uint8_t*)args->sendBuff; uint8_t* myRecvBuff = (uint8_t*)args->recvBuff; void* peerRecvBuff; size_t offset; struct ncclCeBatchOpsParams batchOpsParams = {}; NCCLCHECKGOTO(ncclCeInitBatchOpsParams(&batchOpsParams, comm->nRanks), ret, fail); // Ensure all ranks are ready before starting transfers NCCLCHECKGOTO(ncclMemOpSync(comm, args, stream), ret, fail); // Copy data to other ranks: send data chunk for each destination rank for (int r = 0; r < comm->nRanks; r++) { int dstRank = (comm->rank + r) % comm->nRanks; uint8_t* srcPtr = mySendBuff + dstRank * chunkBytes; uint8_t* dstPtr = myRecvBuff + comm->rank * chunkBytes; if (dstRank == comm->rank) { // Local copy for own data batchOpsParams.srcs[batchOpsParams.numOps] = (void*)srcPtr; batchOpsParams.dsts[batchOpsParams.numOps] = (void*)dstPtr; batchOpsParams.sizes[batchOpsParams.numOps] = chunkBytes; batchOpsParams.numOps++; } else { // Remote copy to other ranks: send to rank dstRank's receive buffer at position comm->rank offset = dstPtr - (uint8_t*)args->recvWin->userPtr; NCCLCHECKGOTO(ncclDevrGetLsaRankPtr(comm, args->recvWin, offset, dstRank, &peerRecvBuff), ret, fail); batchOpsParams.srcs[batchOpsParams.numOps] = (void*)srcPtr; batchOpsParams.dsts[batchOpsParams.numOps] = (void*)peerRecvBuff; batchOpsParams.sizes[batchOpsParams.numOps] = chunkBytes; batchOpsParams.numOps++; } } // Check if we need to perform intra-batch synchronization batchOpsParams.intraBatchSync = (batchOpsParams.numOps > comm->ceColl.intraBatchSyncFreq && chunkBytes*batchOpsParams.numOps >= comm->ceColl.intraBatchSyncMsgThreshold); // Launch the batch operations NCCLCHECKGOTO(ncclCeLaunchBatchOps(comm, args, &batchOpsParams, stream), ret, fail); // Ensure all transfers are complete across all ranks NCCLCHECKGOTO(ncclMemOpSync(comm, args, stream), ret, fail); exit: ncclCeFreeBatchOpsParams(&batchOpsParams); return ret; fail: goto exit; } ncclResult_t ncclCeScatter(struct ncclComm* comm, struct ncclCeCollArgs* args, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; // Calculate the size of data each rank sends to every other rank const size_t chunkBytes = args->nElts * args->eltSize; uint8_t* mySendBuff = (uint8_t*)args->sendBuff; uint8_t* myRecvBuff = (uint8_t*)args->recvBuff; int rootRank = args->rootRank; void* peerDstPtr; size_t offset; struct ncclCeBatchOpsParams batchOpsParams = {}; NCCLCHECKGOTO(ncclCeInitBatchOpsParams(&batchOpsParams, comm->nRanks), ret, fail); // Ensure all ranks are ready before starting transfers NCCLCHECKGOTO(ncclMemOpSync(comm, args, stream), ret, fail); if (comm->rank == rootRank) { // Check if this is an in-place scatter operation bool isInPlace = (myRecvBuff == mySendBuff + comm->rank * chunkBytes); // Copy root's own data first if not in-place if (!isInPlace) { uint8_t* srcPtr = mySendBuff + comm->rank * chunkBytes; uint8_t* dstPtr = myRecvBuff; batchOpsParams.srcs[batchOpsParams.numOps] = (void*)srcPtr; batchOpsParams.dsts[batchOpsParams.numOps] = (void*)dstPtr; batchOpsParams.sizes[batchOpsParams.numOps] = chunkBytes; batchOpsParams.numOps++; } // Root rank distributes data to other ranks for (int r = 1; r < comm->nRanks; r++) { int dstRank = (comm->rank + r) % comm->nRanks; uint8_t* srcPtr = mySendBuff + dstRank * chunkBytes; uint8_t* dstPtr = isInPlace ? myRecvBuff + dstRank * chunkBytes : myRecvBuff; offset = dstPtr - (uint8_t*)args->recvWin->userPtr; NCCLCHECKGOTO(ncclDevrGetLsaRankPtr(comm, args->recvWin, offset, dstRank, &peerDstPtr), ret, fail); batchOpsParams.srcs[batchOpsParams.numOps] = (void*)srcPtr; batchOpsParams.dsts[batchOpsParams.numOps] = (void*)peerDstPtr; batchOpsParams.sizes[batchOpsParams.numOps] = chunkBytes; batchOpsParams.numOps++; } } // Non-root ranks don't need to perform any copy operations // Launch the batch operations NCCLCHECKGOTO(ncclCeLaunchBatchOps(comm, args, &batchOpsParams, stream), ret, fail); // Ensure all transfers are complete across all ranks NCCLCHECKGOTO(ncclMemOpSync(comm, args, stream), ret, fail); exit: ncclCeFreeBatchOpsParams(&batchOpsParams); return ret; fail: goto exit; } ncclResult_t ncclCeGather(struct ncclComm* comm, struct ncclCeCollArgs* args, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; // Calculate the size of data each rank sends to every other rank const size_t chunkBytes = args->nElts * args->eltSize; uint8_t* mySendBuff = (uint8_t*)args->sendBuff; uint8_t* myRecvBuff = (uint8_t*)args->recvBuff; int rootRank = args->rootRank; void* peerRecvBuff; size_t offset; struct ncclCeBatchOpsParams batchOpsParams = {}; NCCLCHECKGOTO(ncclCeInitBatchOpsParams(&batchOpsParams, 1), ret, fail); // Ensure all ranks are ready before starting transfers NCCLCHECKGOTO(ncclMemOpSync(comm, args, stream), ret, fail); if (comm->rank == rootRank) { // Root rank copies its own data to the correct position in receive buffer uint8_t* dstPtr = myRecvBuff + comm->rank * chunkBytes; if (mySendBuff != dstPtr) { batchOpsParams.srcs[batchOpsParams.numOps] = (void*)mySendBuff; batchOpsParams.dsts[batchOpsParams.numOps] = (void*)dstPtr; batchOpsParams.sizes[batchOpsParams.numOps] = chunkBytes; batchOpsParams.numOps++; } } else { // Non-root ranks send their data to root's receive buffer uint8_t* rootRecvPtr = (uint8_t*)args->recvBuff + comm->rank * chunkBytes; offset = rootRecvPtr - (uint8_t*)args->recvWin->userPtr; NCCLCHECKGOTO(ncclDevrGetLsaRankPtr(comm, args->recvWin, offset, rootRank, &peerRecvBuff), ret, fail); batchOpsParams.srcs[batchOpsParams.numOps] = (void*)mySendBuff; batchOpsParams.dsts[batchOpsParams.numOps] = (void*)peerRecvBuff; batchOpsParams.sizes[batchOpsParams.numOps] = chunkBytes; batchOpsParams.numOps++; } // Launch the batch operations NCCLCHECKGOTO(ncclCeLaunchBatchOps(comm, args, &batchOpsParams, stream), ret, fail); // Ensure all transfers are complete across all ranks NCCLCHECKGOTO(ncclMemOpSync(comm, args, stream), ret, fail); exit: ncclCeFreeBatchOpsParams(&batchOpsParams); return ret; fail: goto exit; } ncclResult_t ncclLaunchCeColl(struct ncclComm* comm, struct ncclKernelPlan* plan) { ncclResult_t ret = ncclSuccess; cudaStream_t stream = comm->planner.streams->stream; struct ncclCeCollArgs* args = plan->ceCollArgs; // Start CE collective profiling NCCLCHECKGOTO(ncclProfilerStartCeCollEvent(comm, args, stream), ret, fail); switch (args->func) { case ncclFuncAllGather: NCCLCHECKGOTO(ncclCeAllGather(comm, args, stream), ret, fail); break; case ncclFuncAlltoAll: NCCLCHECKGOTO(ncclCeAlltoAll(comm, args, stream), ret, fail); break; case ncclFuncScatter: NCCLCHECKGOTO(ncclCeScatter(comm, args, stream), ret, fail); break; case ncclFuncGather: NCCLCHECKGOTO(ncclCeGather(comm, args, stream), ret, fail); break; default: ret = ncclInvalidUsage; } exit: // Stop CE collective profiling - always attempt if started, even on error ncclProfilerStopCeCollEvent(comm, args, stream); return ret; fail: goto exit; } nccl-2.29.7-1/src/channel.cc000066400000000000000000000206401515037102200153740ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "channel.h" #include "param.h" #include "gdrwrap.h" #include "transport.h" ncclResult_t initChannel(struct ncclComm* comm, int channelId) { struct ncclChannel* channel = &comm->channels[channelId]; if (channel->id != -1) return ncclSuccess; int nRanks = comm->nRanks; int nvlsRanks = comm->localRanks; int nPeers = nRanks + 1 /* Collnet */ + nvlsRanks /* NVLS */; channel->id = channelId; channel->workFifoProduced = 0; struct ncclSharedResources* sharedRes = comm->sharedRes; cudaStream_t deviceStream; NCCLCHECK(ncclStrongStreamAcquire(ncclCudaGraphNone(comm->config.graphUsageMode), &sharedRes->deviceStream, /*concurrent=*/false, &deviceStream)); if (channel->peers == NULL) { // The extra on nRanks+1 is for collnet root (i.e. network) // Allocate everything related to sharedRes with ncclCalloc as this can be // shared between communicators hence should not be tied to comm. if (sharedRes->peers[channelId] == NULL) { NCCLCHECK(ncclCalloc(sharedRes->peers + channelId, sharedRes->tpNRanks)); } channel->peers = ncclMemoryStackAlloc(&comm->memPermanent, nPeers); for (int r = 0; r < nRanks; r++) { channel->peers[r] = comm->sharedRes->peers[channelId] + comm->topParentRanks[r]; ncclAtomicRefCountIncrement(&channel->peers[r]->refCount); } } if (channel->devPeers == NULL) { if (sharedRes->devPeers[channelId] == NULL) { NCCLCHECK(ncclCudaCallocAsync(sharedRes->devPeers + channelId, sharedRes->tpNRanks, deviceStream, comm->memManager)); } /* channel->devPeers is not shared, so just free it when calling commFree() */ NCCLCHECK(ncclCudaCallocAsync(&channel->devPeers, nPeers, deviceStream, comm->memManager)); ncclCommPushCudaFree(comm, channel->devPeers); NCCLCHECK(ncclCalloc(&channel->devPeersHostPtr, nPeers)); for (int r = 0; r < nRanks; r++) { uintptr_t addr = (uintptr_t)(comm->sharedRes->devPeers[channelId] + comm->topParentRanks[r]); NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + r), (uintptr_t*)&addr, 1, deviceStream)); channel->devPeersHostPtr[r] = (struct ncclDevChannelPeer*)addr; } } channel->ring.userRanks = ncclMemoryStackAlloc(&comm->memPermanent, nRanks); channel->ring.rankToIndex = ncclMemoryStackAlloc(&comm->memPermanent, nRanks); NCCLCHECK(ncclCudaCallocAsync(&channel->devRingUserRanks, nRanks, deviceStream, comm->memManager)); ncclCommPushCudaFree(comm, channel->devRingUserRanks); /* guarantee addr has been copied into channel->devPeers */ NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(comm->config.graphUsageMode), &sharedRes->deviceStream, /*concurrent=*/false)); NCCLCHECK(ncclStrongStreamSynchronize(&sharedRes->deviceStream)); return ncclSuccess; } ncclResult_t initNvlsChannel(struct ncclComm* comm, int channelId, struct ncclComm* parent, bool share) { struct ncclChannel* channel = &comm->channels[channelId]; struct ncclSharedResources* sharedRes = comm->sharedRes; cudaStream_t deviceStream; if (channel->nvlsPeers != NULL) return ncclSuccess; if (channel->id == -1) NCCLCHECK(initChannel(comm, channelId)); NCCLCHECK(ncclStrongStreamAcquire(ncclCudaGraphNone(comm->config.graphUsageMode), &sharedRes->deviceStream, /*concurrent=*/false, &deviceStream)); int nvlsRanks = comm->localRanks; if (share) { channel->nvlsPeers = parent->channels[channelId].nvlsPeers; channel->nvlsDevPeers = parent->channels[channelId].nvlsDevPeers; for (int r = 0; r < nvlsRanks; ++r) { int tr = comm->topParentLocalRanks[r]; uintptr_t addr = (uintptr_t)(parent->channels[channelId].nvlsDevPeers + tr); channel->peers[comm->nRanks + 1 + r] = parent->channels[channelId].nvlsPeers + tr; NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks + 1 + r), (uintptr_t*)&addr, 1, deviceStream)); channel->devPeersHostPtr[comm->nRanks + 1 + r] = (struct ncclDevChannelPeer*)addr; ncclAtomicRefCountIncrement(&parent->channels[channelId].nvlsPeers[tr].refCount); } } else { NCCLCHECK(ncclCalloc(&channel->nvlsPeers, nvlsRanks)); NCCLCHECK(ncclCudaCallocAsync(&channel->nvlsDevPeers, nvlsRanks, deviceStream, comm->memManager)); for (int r = 0; r < nvlsRanks; ++r) { uintptr_t addr = (uintptr_t)(channel->nvlsDevPeers + r); channel->peers[comm->nRanks + 1 + r] = channel->nvlsPeers + r; NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks + 1 + r), (uintptr_t*)&addr, 1, deviceStream)); channel->devPeersHostPtr[comm->nRanks + 1 + r] = (struct ncclDevChannelPeer*)addr; ncclAtomicRefCountIncrement(&channel->nvlsPeers[r].refCount); } } NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(comm->config.graphUsageMode), &sharedRes->deviceStream, /*concurrent=*/false)); NCCLCHECK(ncclStrongStreamSynchronize(&sharedRes->deviceStream)); return ncclSuccess; } ncclResult_t initCollnetChannel(struct ncclComm* comm, int channelId, struct ncclComm* parent, bool share) { struct ncclChannel* channel = &comm->channels[channelId]; struct ncclSharedResources* sharedRes = comm->sharedRes; uintptr_t addr; cudaStream_t deviceStream; if (channel->collnetPeers != NULL) return ncclSuccess; if (channel->id == -1) NCCLCHECK(initChannel(comm, channelId)); NCCLCHECK(ncclStrongStreamAcquire(ncclCudaGraphNone(comm->config.graphUsageMode), &sharedRes->deviceStream, /*concurrent=*/false, &deviceStream)); if (share) { channel->collnetPeers = parent->channels[channelId].collnetPeers; channel->collnetDevPeers = parent->channels[channelId].collnetDevPeers; addr = (uintptr_t)parent->channels[channelId].collnetDevPeers; channel->peers[comm->nRanks] = parent->channels[channelId].collnetPeers; NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks), (uintptr_t*)&addr, 1, deviceStream)); channel->devPeersHostPtr[comm->nRanks] = (struct ncclDevChannelPeer*)addr; ncclAtomicRefCountIncrement(&parent->channels[channelId].collnetPeers->refCount); } else { NCCLCHECK(ncclCalloc(&channel->collnetPeers, 1)); NCCLCHECK(ncclCudaCallocAsync(&channel->collnetDevPeers, 1, deviceStream, comm->memManager)); addr = (uintptr_t)channel->collnetDevPeers; channel->peers[comm->nRanks] = channel->collnetPeers; NCCLCHECK(ncclCudaMemcpyAsync((uintptr_t*)(channel->devPeers + comm->nRanks), (uintptr_t*)&addr, 1, deviceStream)); channel->devPeersHostPtr[comm->nRanks] = (struct ncclDevChannelPeer*)addr; ncclAtomicRefCountIncrement(&channel->collnetPeers->refCount); } NCCLCHECK(ncclStrongStreamRelease(ncclCudaGraphNone(comm->config.graphUsageMode), &sharedRes->deviceStream, /*concurrent=*/false)); NCCLCHECK(ncclStrongStreamSynchronize(&sharedRes->deviceStream)); return ncclSuccess; } ncclResult_t freeChannel(struct ncclChannel* channel, int nRanks, int collnetNRanks, int nvlsNRanks, struct ncclComm* comm) { int nPeers = nRanks + collnetNRanks + nvlsNRanks; /* channel peers are only valid when async init thread completes commAlloc() and * the channel is initialized with initChannel(); if either is not done, this channel * should never be free. */ if (channel->id == -1 || channel->peers == NULL) return ncclSuccess; // Free transport proxy resources // Note: free all send resources first due to CollNet arrangement for (int r = 0; r < nPeers; r++) { struct ncclChannelPeer* peer = channel->peers[r]; if (peer) { if (ncclAtomicRefCountDecrement(&peer->refCount) == 0) { for (int b=0; bsend[b].transportComm) NCCLCHECK(peer->send[b].transportComm->free(comm, peer->send+b)); if (peer->recv[b].transportComm) NCCLCHECK(peer->recv[b].transportComm->free(comm, peer->recv+b)); } if (r == nRanks) { free(channel->collnetPeers); ncclCudaFree(channel->collnetDevPeers, comm->memManager); } else if (r == nPeers - 1) { free(channel->nvlsPeers); ncclCudaFree(channel->nvlsDevPeers, comm->memManager); } } } } free(channel->devPeersHostPtr); return ncclSuccess; } nccl-2.29.7-1/src/collectives.cc000066400000000000000000000276761515037102200163200ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "argcheck.h" // Need some checks here since we access comm #include "collectives.h" #include "enqueue.h" #include "nccl.h" #include "nvtx_payload_schemas.h" const char* ncclFuncToString(ncclFunc_t fn) { switch (fn) { case ncclFuncAllGather: return "AllGather"; case ncclFuncAllReduce: return "AllReduce"; case ncclFuncAlltoAll: return "AlltoAll"; case ncclFuncBroadcast: return "Broadcast"; case ncclFuncGather: return "Gather"; case ncclFuncRecv: return "Recv"; case ncclFuncReduce: return "Reduce"; case ncclFuncReduceScatter: return "ReduceScatter"; case ncclFuncScatter: return "Scatter"; case ncclFuncSendRecv: return "SendRecv"; case ncclFuncSend: return "Send"; case ncclFuncPutSignal: return "PutSignal"; case ncclFuncSignal: return "Signal"; case ncclFuncWaitSignal: return "WaitSignal"; default: return "Invalid"; } } const char* ncclDevRedOpToString(ncclDevRedOp_t op) { switch (op) { case ncclDevSum: return "Sum"; case ncclDevProd: return "Prod"; case ncclDevMinMax: return "MinMax"; case ncclDevPreMulSum: return "PreMulSum"; case ncclDevSumPostDiv: return "SumPostDiv"; default: return "Unknown"; } } const char* ncclDatatypeToString(ncclDataType_t type) { switch (type) { case ncclInt8: return "ncclInt8"; case ncclInt32: return "ncclInt32"; case ncclUint32: return "ncclUint32"; case ncclInt64: return "ncclInt64"; case ncclUint64: return "ncclUint64"; case ncclFloat16: return "ncclFloat16"; case ncclFloat32: return "ncclFloat32"; case ncclFloat64: return "ncclFloat64"; case ncclBfloat16: return "ncclBfloat16"; case ncclFloat8e4m3: return "ncclFloat8e4m3"; case ncclFloat8e5m2: return "ncclFloat8e5m2"; default: return "Unknown"; } } const char* ncclAlgoToString(int algo) { switch (algo) { case NCCL_ALGO_TREE: return "TREE"; case NCCL_ALGO_RING: return "RING"; case NCCL_ALGO_COLLNET_DIRECT: return "COLLNET_DIRECT"; case NCCL_ALGO_COLLNET_CHAIN: return "COLLNET_CHAIN"; case NCCL_ALGO_NVLS: return "NVLS"; case NCCL_ALGO_NVLS_TREE: return "NVLS_TREE"; case NCCL_ALGO_PAT: return "PAT"; default: return "Unknown"; } } const char* ncclProtoToString(int proto) { switch (proto) { case NCCL_PROTO_LL: return "LL"; case NCCL_PROTO_LL128: return "LL128"; case NCCL_PROTO_SIMPLE: return "SIMPLE"; default: return "Unknown"; } } NCCL_API(ncclResult_t, ncclAllGather, const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream); ncclResult_t ncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount, ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) { // Just pass the size of one message and not the total bytes sent/received. NVTX3_FUNC_WITH_PARAMS(AllGather, NcclNvtxParamsAllGather, NVTX3_PAYLOAD(comm ? comm->commHash : 0, sendcount * ncclTypeSize(datatype))); struct ncclInfo info = { ncclFuncAllGather, "AllGather", sendbuff, recvbuff, sendcount, datatype, ncclSum, 0, comm, stream, /* Args */ ALLGATHER_CHUNKSTEPS, ALLGATHER_SLICESTEPS }; return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclAlltoAll, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm* comm, cudaStream_t stream); ncclResult_t ncclAlltoAll(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm* comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(AlltoAll, NcclNvtxParamsAlltoAll, NVTX3_PAYLOAD(comm ? comm->commHash : 0, count * ncclTypeSize(datatype))); struct ncclInfo info = { ncclFuncAlltoAll, "AlltoAll", sendbuff, recvbuff, count, datatype, ncclSum, 0, comm, stream, /* Args */ ALLTOALL_CHUNKSTEPS, ALLTOALL_SLICESTEPS }; return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclAllReduce, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm* comm, cudaStream_t stream); ncclResult_t ncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, ncclComm* comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(AllReduce, NcclNvtxParamsAllReduce, NVTX3_PAYLOAD(comm ? comm->commHash : 0, count * ncclTypeSize(datatype), op)); struct ncclInfo info = { ncclFuncAllReduce, "AllReduce", sendbuff, recvbuff, count, datatype, op, 0, comm, stream, /* Args */ ALLREDUCE_CHUNKSTEPS, ALLREDUCE_SLICESTEPS }; return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclBroadcast, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream); ncclResult_t ncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(Broadcast, NcclNvtxParamsBroadcast, NVTX3_PAYLOAD(comm ? comm->commHash : 0, count * ncclTypeSize(datatype), root)); struct ncclInfo info = { ncclFuncBroadcast, "Broadcast", sendbuff, recvbuff, count, datatype, ncclSum, root, comm, stream, /* Args */ BROADCAST_CHUNKSTEPS, BROADCAST_SLICESTEPS }; return ncclEnqueueCheck(&info); } /* Deprecated original "in place" function, similar to MPI */ NCCL_API(ncclResult_t, ncclBcast, void* buff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream); ncclResult_t ncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root, ncclComm_t comm, cudaStream_t stream) { return ncclBroadcast(buff, buff, count, datatype, root, comm, stream); } NCCL_API(ncclResult_t, ncclGather, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm* comm, cudaStream_t stream); ncclResult_t ncclGather(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm* comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(Gather, NcclNvtxParamsGather, NVTX3_PAYLOAD(comm ? comm->commHash : 0, count * ncclTypeSize(datatype), root)); struct ncclInfo info = { ncclFuncGather, "Gather", sendbuff, recvbuff, count, datatype, ncclSum, root, comm, stream, /* Args */ GATHER_CHUNKSTEPS, GATHER_SLICESTEPS }; return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclReduce, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream); ncclResult_t ncclReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(Reduce, NcclNvtxParamsReduce, NVTX3_PAYLOAD(comm ? comm->commHash : 0, count * ncclTypeSize(datatype), root, op)); struct ncclInfo info = { ncclFuncReduce, "Reduce", sendbuff, recvbuff, count, datatype, op, root, comm, stream, /* Args */ REDUCE_CHUNKSTEPS, REDUCE_SLICESTEPS }; return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclReduceScatter, const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm* comm, cudaStream_t stream); ncclResult_t ncclReduceScatter(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm* comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(ReduceScatter, NcclNvtxParamsReduceScatter, NVTX3_PAYLOAD(comm ? comm->commHash : 0, recvcount * ncclTypeSize(datatype), op)); struct ncclInfo info = { ncclFuncReduceScatter, "ReduceScatter", sendbuff, recvbuff, recvcount, datatype, op, 0, comm, stream, /* Args */ REDUCESCATTER_CHUNKSTEPS, REDUCESCATTER_SLICESTEPS }; return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclScatter, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm* comm, cudaStream_t stream); ncclResult_t ncclScatter(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, ncclComm* comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(Scatter, NcclNvtxParamsScatter, NVTX3_PAYLOAD(comm ? comm->commHash : 0, count * ncclTypeSize(datatype), root)); struct ncclInfo info = { ncclFuncScatter, "Scatter", sendbuff, recvbuff, count, datatype, ncclSum, root, comm, stream, /* Args */ SCATTER_CHUNKSTEPS, SCATTER_SLICESTEPS }; return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclSend, const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream); ncclResult_t ncclSend(const void* sendbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(Send, NcclNvtxParamsSendRecv, NVTX3_PAYLOAD(comm ? comm->commHash : 0, count * ncclTypeSize(datatype), peer)); struct ncclInfo info = { ncclFuncSend, "Send", NULL, (void*)sendbuff, count, datatype, ncclSum, peer, comm, stream, /* Args */ 1, 1 }; return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclRecv, void* recvbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream); ncclResult_t ncclRecv(void* recvbuff, size_t count, ncclDataType_t datatype, int peer, ncclComm_t comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(Recv, NcclNvtxParamsSendRecv, NVTX3_PAYLOAD(comm ? comm->commHash : 0, count * ncclTypeSize(datatype), peer)); struct ncclInfo info = { ncclFuncRecv, "Recv", NULL, recvbuff, count, datatype, ncclSum, peer, comm, stream, /* Args */ 1, 1 }; return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclPutSignal, const void* localbuff, size_t count, ncclDataType_t datatype, int peer, ncclWindow_t peerWin, size_t peerWinOffset, int sigIdx, int ctx, unsigned int flags, ncclComm_t comm, cudaStream_t stream); ncclResult_t ncclPutSignal(const void* localbuff, size_t count, ncclDataType_t datatype, int peer, ncclWindow_t peerWin, size_t peerWinOffset, int sigIdx, int ctx, unsigned int flags, ncclComm_t comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(PutSignal, NcclNvtxParamsPut, NVTX3_PAYLOAD(comm ? comm->commHash : 0, count * ncclTypeSize(datatype), peer, ctx)); struct ncclInfo info = { ncclFuncPutSignal, "PutSignal", localbuff, NULL, count, datatype, ncclSum, peer, comm, stream, /* Args */ 1, 1, /* chunkSteps, sliceSteps */ peerWinOffset, peerWin, sigIdx, ctx, flags, /* peerWinOffset, peerWin, sigIdx, ctx, flags */ 0, NULL }; /* nDesc, signalDescs */ return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclSignal, int peer, int sigIdx, int ctx, unsigned int flags, ncclComm_t comm, cudaStream_t stream); ncclResult_t ncclSignal(int peer, int sigIdx, int ctx, unsigned int flags, ncclComm_t comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(Signal, NcclNvtxParamsSignal, NVTX3_PAYLOAD(comm ? comm->commHash : 0, peer, ctx)); struct ncclInfo info = { ncclFuncSignal, "Signal", NULL, NULL, 0, ncclInt8, ncclSum, peer, comm, stream, /* Args */ 1, 1, /* chunkSteps, sliceSteps */ 0, NULL, sigIdx, ctx, flags, /* peerWinOffset, peerWin, sigIdx, ctx, flags */ 0, NULL }; /* nDesc, signalDescs */ return ncclEnqueueCheck(&info); } NCCL_API(ncclResult_t, ncclWaitSignal, int nDesc, ncclWaitSignalDesc_t* signalDescs, ncclComm_t comm, cudaStream_t stream); ncclResult_t ncclWaitSignal(int nDesc, ncclWaitSignalDesc_t* signalDescs, ncclComm_t comm, cudaStream_t stream) { NVTX3_FUNC_WITH_PARAMS(WaitSignal, NcclNvtxParamsWaitSignal, NVTX3_PAYLOAD(comm ? comm->commHash : 0, nDesc, 0)); struct ncclInfo info = { ncclFuncWaitSignal, "WaitSignal", NULL, NULL, 0, ncclInt32, ncclSum, 0, comm, stream, /* Args */ 1, 1, /* chunkSteps, sliceSteps */ 0, NULL, 0, 0, 0, /* peerWinOffset, peerWin, sigIdx, ctx, flags */ nDesc, signalDescs }; /* nDesc, signalDescs */ return ncclEnqueueCheck(&info); } nccl-2.29.7-1/src/debug.cc000066400000000000000000000414361515037102200150600ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "core.h" #include "nccl_net.h" #include #include #include #include #include #include #include #include #include "param.h" #include #include "os.h" #include "utils.h" #include "env.h" #define NCCL_DEBUG_RESET_TRIGGERED (-2) int ncclDebugLevel = -1; static uint32_t ncclDebugTimestampLevels = 0; // bitmaps of levels that have timestamps turned on static char ncclDebugTimestampFormat[256]; // with space for subseconds static int ncclDebugTimestampSubsecondsStart; // index where the subseconds starts static uint64_t ncclDebugTimestampMaxSubseconds; // Max number of subseconds plus 1, used in duration ratio static int ncclDebugTimestampSubsecondDigits; // Number of digits to display static int pid = -1; static char hostname[1024]; thread_local int ncclDebugNoWarn = 0; char ncclLastError[1024] = ""; // Global string for the last error in human readable form uint64_t ncclDebugMask = 0; FILE *ncclDebugFile = stdout; static std::mutex ncclDebugMutex; static std::chrono::steady_clock::time_point ncclEpoch; static bool ncclWarnSetDebugInfo = false; static thread_local int tid = -1; typedef const char* (*ncclGetEnvFunc_t)(const char*); static ncclResult_t getHostNameForLog(char* hostname, int maxlen, const char delim) { ncclResult_t ret = getHostName(hostname, maxlen, delim); if (ret != ncclSuccess) return ret; for (int i = 0; i < maxlen-1 && hostname[i]; ++i) { // Replace special characters in hostnames with dashes switch (hostname[i]) { case '%': case '/': hostname[i] = '-'; break; default: break; } } return ncclSuccess; } // This function must be called with ncclDebugLock locked! static void ncclDebugInit() { ncclGetEnvFunc_t getEnvFunc = ncclEnvPluginInitialized() ? ncclGetEnv : (ncclGetEnvFunc_t)std::getenv; const char* nccl_debug = getEnvFunc("NCCL_DEBUG"); int tempNcclDebugLevel = -1; uint64_t tempNcclDebugMask = NCCL_INIT | NCCL_BOOTSTRAP | NCCL_ENV; // Default debug sub-system mask if (ncclDebugLevel == NCCL_DEBUG_RESET_TRIGGERED && ncclDebugFile != stdout) { // Finish the reset initiated via ncclResetDebugInit(). fclose(ncclDebugFile); ncclDebugFile = stdout; } if (nccl_debug == NULL) { tempNcclDebugLevel = NCCL_LOG_NONE; } else if (strcasecmp(nccl_debug, "VERSION") == 0) { tempNcclDebugLevel = NCCL_LOG_VERSION; } else if (strcasecmp(nccl_debug, "WARN") == 0) { tempNcclDebugLevel = NCCL_LOG_WARN; } else if (strcasecmp(nccl_debug, "INFO") == 0) { tempNcclDebugLevel = NCCL_LOG_INFO; } else if (strcasecmp(nccl_debug, "ABORT") == 0) { tempNcclDebugLevel = NCCL_LOG_ABORT; } else if (strcasecmp(nccl_debug, "TRACE") == 0) { tempNcclDebugLevel = NCCL_LOG_TRACE; } /* Parse the NCCL_DEBUG_SUBSYS env var * This can be a comma separated list such as INIT,COLL * or ^INIT,COLL etc */ const char* ncclDebugSubsysEnv = getEnvFunc("NCCL_DEBUG_SUBSYS"); if (ncclDebugSubsysEnv != NULL) { int invert = 0; if (ncclDebugSubsysEnv[0] == '^') { invert = 1; ncclDebugSubsysEnv++; } tempNcclDebugMask = invert ? ~0ULL : 0ULL; char *ncclDebugSubsys = strdup(ncclDebugSubsysEnv); char *subsys = strtok(ncclDebugSubsys, ","); while (subsys != NULL) { uint64_t mask = 0; if (strcasecmp(subsys, "INIT") == 0) { mask = NCCL_INIT; } else if (strcasecmp(subsys, "COLL") == 0) { mask = NCCL_COLL; } else if (strcasecmp(subsys, "P2P") == 0) { mask = NCCL_P2P; } else if (strcasecmp(subsys, "SHM") == 0) { mask = NCCL_SHM; } else if (strcasecmp(subsys, "NET") == 0) { mask = NCCL_NET; } else if (strcasecmp(subsys, "GRAPH") == 0) { mask = NCCL_GRAPH; } else if (strcasecmp(subsys, "TUNING") == 0) { mask = NCCL_TUNING; } else if (strcasecmp(subsys, "ENV") == 0) { mask = NCCL_ENV; } else if (strcasecmp(subsys, "ALLOC") == 0) { mask = NCCL_ALLOC; } else if (strcasecmp(subsys, "CALL") == 0) { mask = NCCL_CALL; } else if (strcasecmp(subsys, "PROXY") == 0) { mask = NCCL_PROXY; } else if (strcasecmp(subsys, "NVLS") == 0) { mask = NCCL_NVLS; } else if (strcasecmp(subsys, "BOOTSTRAP") == 0) { mask = NCCL_BOOTSTRAP; } else if (strcasecmp(subsys, "REG") == 0) { mask = NCCL_REG; } else if (strcasecmp(subsys, "PROFILE") == 0) { mask = NCCL_PROFILE; } else if (strcasecmp(subsys, "RAS") == 0) { mask = NCCL_RAS; } else if (strcasecmp(subsys, "ALL") == 0) { mask = NCCL_ALL; } if (mask) { if (invert) tempNcclDebugMask &= ~mask; else tempNcclDebugMask |= mask; } subsys = strtok(NULL, ","); } free(ncclDebugSubsys); } const char* ncclWarnSetDebugInfoEnv = getEnvFunc("NCCL_WARN_ENABLE_DEBUG_INFO"); if (ncclWarnSetDebugInfoEnv != NULL && strlen(ncclWarnSetDebugInfoEnv) > 0) { int64_t value; errno = 0; value = strtoll(ncclWarnSetDebugInfoEnv, NULL, 0); if (!errno) ncclWarnSetDebugInfo = value; } // Determine which debug levels will have timestamps. const char* timestamps = getEnvFunc("NCCL_DEBUG_TIMESTAMP_LEVELS"); if (timestamps == nullptr) { ncclDebugTimestampLevels = (1< sizeof(ncclDebugTimestampFormat) - 1) { // Won't fit; fall back on the default. break; } ncclDebugTimestampSubsecondsStart = i; ncclDebugTimestampMaxSubseconds = 1; memcpy(ncclDebugTimestampFormat, tsFormat, i); for (int j=0; j VERSION */ const char* ncclDebugFileEnv = getEnvFunc("NCCL_DEBUG_FILE"); if (tempNcclDebugLevel > NCCL_LOG_VERSION && ncclDebugFileEnv != NULL) { int c = 0; char debugFn[PATH_MAX+1] = ""; char *dfn = debugFn; while (ncclDebugFileEnv[c] != '\0' && (dfn - debugFn) < PATH_MAX) { if (ncclDebugFileEnv[c++] != '%') { *dfn++ = ncclDebugFileEnv[c-1]; continue; } switch (ncclDebugFileEnv[c++]) { case '%': // Double % *dfn++ = '%'; break; case 'h': // %h = hostname dfn += snprintf(dfn, PATH_MAX + 1 - (dfn - debugFn), "%s", hostname); break; case 'p': // %p = pid dfn += snprintf(dfn, PATH_MAX + 1 - (dfn - debugFn), "%d", pid); break; default: // Echo everything we don't understand *dfn++ = '%'; if ((dfn - debugFn) < PATH_MAX) { *dfn++ = ncclDebugFileEnv[c-1]; } break; } if ((dfn - debugFn) > PATH_MAX) { // snprintf wanted to overfill the buffer: set dfn to the end // of the buffer (for null char) and it will naturally exit // the loop. dfn = debugFn + PATH_MAX; } } *dfn = '\0'; if (debugFn[0] != '\0') { FILE *file = fopen(debugFn, "w"); if (file != nullptr) { setlinebuf(file); // disable block buffering ncclDebugFile = file; } } } ncclEpoch = std::chrono::steady_clock::now(); ncclDebugMask = tempNcclDebugMask; COMPILER_ATOMIC_STORE(&ncclDebugLevel, tempNcclDebugLevel, std::memory_order_release); } /* Common logging function used by the INFO, WARN and TRACE macros * Also exported to the dynamically loadable Net transport modules so * they can share the debugging mechanisms and output files */ void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char *filefunc, int line, const char *fmt, ...) { int gotLevel = COMPILER_ATOMIC_LOAD(&ncclDebugLevel, std::memory_order_acquire); if (ncclDebugNoWarn != 0 && level == NCCL_LOG_WARN) { level = NCCL_LOG_INFO; flags = ncclDebugNoWarn; } // Save the last error (WARN) as a human readable string if (level == NCCL_LOG_WARN) { std::lock_guard lock(ncclDebugMutex); va_list vargs; va_start(vargs, fmt); (void) vsnprintf(ncclLastError, sizeof(ncclLastError), fmt, vargs); va_end(vargs); } if (gotLevel >= 0 && (gotLevel < level || (flags & ncclDebugMask) == 0)) { return; } std::lock_guard lock(ncclDebugMutex); if (ncclDebugLevel < 0) ncclDebugInit(); if (ncclDebugLevel < level || ((flags & ncclDebugMask) == 0)) { return; } if (tid == -1) { tid = ncclOsGetTid(); } char buffer[1024]; size_t len = 0; // WARNs come with an extra newline at the beginning. if (level == NCCL_LOG_WARN) { buffer[len++] = '\n'; }; // Add the timestamp to the buffer if they are turned on for this level. if (ncclDebugTimestampLevels & (1<>(delta).count()*1000; len += snprintf(buffer+len, sizeof(buffer)-len, "[%d] %f %s:%d NCCL TRACE %s\n", cudaDev, timestamp, filefunc, line, fmt); } else { len += snprintf(buffer+len, sizeof(buffer)-len, "%s\n", fmt); } // If the prefixed format string overflows, make sure it is still terminated with a newline. if (len > sizeof(buffer)-1) { // snprintf already placed a \0 at sizeof(buffer)-1 buffer[sizeof(buffer)-2] = '\n'; } // Add the message as given by the call site. // The call site's format string has been incorporated into `buffer` along with our prefix. va_list vargs; va_start(vargs, fmt); (void) vfprintf(ncclDebugFile, buffer, vargs); va_end(vargs); } // Non-deprecated version for internal use. extern "C" __attribute__ ((visibility("default"))) void ncclResetDebugInitInternal() { // Cleans up from a previous ncclDebugInit() and reruns. // Use this after changing NCCL_DEBUG and related parameters in the environment. std::lock_guard lock(ncclDebugMutex); // Let ncclDebugInit() know to complete the reset. COMPILER_ATOMIC_STORE(&ncclDebugLevel, NCCL_DEBUG_RESET_TRIGGERED, std::memory_order_release); } // In place of: NCCL_API(void, ncclResetDebugInit); __attribute__ ((visibility("default"))) __attribute__ ((alias("ncclResetDebugInit"))) void pncclResetDebugInit(); extern "C" __attribute__ ((visibility("default"))) __attribute__ ((weak)) __attribute__ ((deprecated("ncclResetDebugInit is not supported as part of the NCCL API and will be removed in the future"))) void ncclResetDebugInit(); void ncclResetDebugInit() { // This is now deprecated as part of the NCCL API. It will be removed // from the API in the future. It is still available as an // exported symbol. ncclResetDebugInitInternal(); } NCCL_PARAM(SetThreadName, "SET_THREAD_NAME", 0); void ncclSetThreadName(std::thread& thread, const char *fmt, ...) { // pthread_setname_np is nonstandard GNU extension // needs the following feature test macro #ifdef _GNU_SOURCE if (ncclParamSetThreadName() != 1) return; char threadName[NCCL_THREAD_NAMELEN]; va_list vargs; va_start(vargs, fmt); vsnprintf(threadName, NCCL_THREAD_NAMELEN, fmt, vargs); va_end(vargs); pthread_setname_np(thread.native_handle(), threadName); #endif } nccl-2.29.7-1/src/dev_runtime.cc000066400000000000000000001752601515037102200163160ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "dev_runtime.h" #include "comm.h" #include "nccl_device/core.h" #include "rma/rma.h" #include "device.h" #include "sym_kernels.h" #include "transport.h" #include "group.h" #include "nccl_device.h" #include "utils.h" #include "gin/gin_host.h" #include "argcheck.h" #include NCCL_PARAM(WinStride, "WIN_STRIDE", -1); NCCL_PARAM(EnableVersionCheck, "ENABLE_VERSION_CHECK", 1); // Global window map using intrusive address map // Uses ncclDevrWindow directly (vidmem as key, next pointer embedded in struct) static std::mutex ncclWindowMapMutex; static ncclIntruAddressMap ncclWindowMap; static ncclResult_t symWindowDestroy(struct ncclComm* comm, struct ncclWindow_vidmem* winDev, cudaStream_t stream); // Complete types from src/include/dev_runtime.h struct ncclDevrMemory { int refCount; struct ncclDevrMemory* next; CUmemGenericAllocationHandle memHandle; void* primaryAddr; // What we hope is the VA of this memory's first mapping. size_t size; size_t bigOffset; // offset in big VA space void* ginHostWins[NCCL_GIN_MAX_CONNECTIONS]; ncclGinWindow_t ginDevWins[NCCL_GIN_MAX_CONNECTIONS]; void* rmaHostWins[NCCL_GIN_MAX_CONNECTIONS]; ncclGinWindow_t rmaDevWins[NCCL_GIN_MAX_CONNECTIONS]; int winFlags; }; struct ncclDevrWindowSorted { uintptr_t userAddr; size_t size; struct ncclDevrWindow* win; }; struct ncclDevrTeam { struct ncclDevrTeam* next; struct ncclTeam team; CUmemGenericAllocationHandle mcHandle; void* mcBasePtr; int worldRankList[]; }; //////////////////////////////////////////////////////////////////////////////// // Helpers at the bottom: // Find least index such that `arg < sorted[i].key` (least upper bound) template static int listFindSortedLub(Key Obj::*key, Obj* sorted, int count, Key arg); template static void listInsert(Obj** list, int* capacity, int* count, int index, Obj val); template static void listRemove(Obj* list, int* count, int index); //////////////////////////////////////////////////////////////////////////////// NCCL_PARAM(LsaTeamSize, "LSA_TEAM_SIZE", 0) ncclResult_t ncclDevrInitOnce(struct ncclComm* comm) { ncclResult_t ret = ncclSuccess; struct ncclDevrState* devr = &comm->devrState; if (devr->bigSize != 0) return ncclSuccess; // LSA needs to be the same size for all ranks, and it needs to represent // a consecutive set of ranks. int lsaSize = ncclParamLsaTeamSize(); int nodeSize = 1; for (int r=1; r < comm->nRanks; r++) { if (comm->rankToNode[r] == comm->rankToNode[r-1]) { nodeSize += 1; } else { lsaSize = gcd(lsaSize, nodeSize); nodeSize = 1; } } lsaSize = gcd(lsaSize, nodeSize); devr->lsaSize = lsaSize; devr->lsaSelf = comm->rank % lsaSize; devr->lsaRankList = (int*)malloc(devr->lsaSize*sizeof(int)); for (int i=0; i < devr->lsaSize; i++) { devr->lsaRankList[i] = comm->rank + (i - devr->lsaSelf); } devr->nLsaTeams = comm->nRanks / devr->lsaSize; CUmemAllocationProp memProp = {}; memProp.type = CU_MEM_ALLOCATION_TYPE_PINNED; memProp.location.type = CU_MEM_LOCATION_TYPE_DEVICE; memProp.requestedHandleTypes = ncclCuMemHandleType; memProp.location.id = comm->cudaDev; CUCHECKGOTO(cuMemGetAllocationGranularity(&devr->granularity, &memProp, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED), ret, fail_lsaRankList); devr->bigSize = ncclParamWinStride(); if (-devr->bigSize <= 1) { devr->bigSize = 1; for (int r=0; r < comm->nRanks; ++r) { devr->bigSize = std::max(devr->bigSize, comm->peerInfo[r].totalGlobalMem); } } devr->bigSize = alignUp(devr->bigSize, size_t(1)<<32); INFO(NCCL_INIT, "Symmetric VA size=%ldGB", (long)devr->bigSize>>30); ncclSpaceConstruct(&devr->bigSpace); ncclShadowPoolConstruct(&devr->shadows); return ncclSuccess; fail_lsaRankList: free(devr->lsaRankList); return ret; } static void symTeamDestroyAll(struct ncclComm* comm); // Further down ncclResult_t ncclDevrFinalize(struct ncclComm* comm) { struct ncclDevrState* devr = &comm->devrState; cudaStream_t stream; ncclResult_t ret = ncclSuccess; if (devr->bigSize == 0) return ncclSuccess; while (!ncclIntruQueueEmpty(&devr->regTaskQueue)) { struct ncclDevrRegTask* task = ncclIntruQueueDequeue(&devr->regTaskQueue); free(task); } // During abort or any other cases, users might not call deregister API for // symmetric window objects, we need to destroy all remaining window objects // that are not deregistered by user to avoid memory leaks here. CUDACHECKIGNORE(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); while (devr->winSortedCount > 0) { struct ncclDevrWindow* win = devr->winSorted[0].win; NCCLCHECKIGNORE(symWindowDestroy(comm, win->vidmem, stream), ret); } CUDACHECKIGNORE(cudaStreamSynchronize(stream)); CUDACHECKIGNORE(cudaStreamDestroy(stream)); symTeamDestroyAll(comm); { // delete windowTable cudaStream_t stream; if (CUDASUCCESS(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking))) { struct ncclDevCommWindowTable* tableDev = devr->windowTable; while (tableDev != nullptr) { struct ncclDevCommWindowTable* tableHost; if (ncclSuccess != ncclShadowPoolToHost(&devr->shadows, tableDev, &tableHost)) break; struct ncclDevCommWindowTable* next = tableHost->next; ncclShadowPoolFree(&devr->shadows, tableDev, stream); tableDev = next; } cudaStreamSynchronize(stream); cudaStreamDestroy(stream); } } if (devr->lsaFlatBase != nullptr) { CUdeviceptr flatAddr = reinterpret_cast(devr->lsaFlatBase); CUCHECKIGNORE(cuMemUnmap(flatAddr, devr->lsaSize*devr->bigSize)); CUCHECKIGNORE(cuMemAddressFree(flatAddr, devr->lsaSize*devr->bigSize)); } ncclShadowPoolDestruct(&devr->shadows); ncclSpaceDestruct(&devr->bigSpace); free(devr->lsaRankList); free(devr->winSorted); return ncclSuccess; } //////////////////////////////////////////////////////////////////////////////// static ncclResult_t symMemoryMapLsaTeam( struct ncclComm* comm, CUmemGenericAllocationHandle memHandle, size_t size, size_t bigOffset ) { ncclResult_t ret = ncclSuccess; struct ncclDevrState* devr = &comm->devrState; CUmemAccessDesc accessDesc = {}; union Message { CUmemGenericAllocationHandle memHandle; CUmemFabricHandle fabricHandle; }; Message* messages = (Message*)calloc(devr->lsaSize, sizeof(Message)); if (ncclCuMemHandleType == CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR) { messages[devr->lsaSelf].memHandle = memHandle; } else { CUCHECKGOTO(cuMemExportToShareableHandle(&messages[devr->lsaSelf].fabricHandle, memHandle, ncclCuMemHandleType, 0), ret, fail); } NCCLCHECKGOTO(bootstrapIntraNodeAllGather(comm->bootstrap, devr->lsaRankList, devr->lsaSelf, devr->lsaSize, messages, sizeof(Message)), ret, fail); if (devr->lsaFlatBase == nullptr) { // Create on first need. CUdeviceptr addr; CUCHECKGOTO(cuMemAddressReserve(&addr, devr->lsaSize*devr->bigSize, NCCL_MAX_PAGE_SIZE, 0, 0), ret, fail); devr->lsaFlatBase = reinterpret_cast(addr); } accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; accessDesc.location.id = comm->cudaDev; accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; for (int r = 0; r < devr->lsaSize; r++) { CUmemGenericAllocationHandle impHandle; if (r == devr->lsaSelf) { impHandle = memHandle; } else { if (ncclCuMemHandleType == CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR) { int fd = -1; NCCLCHECKGOTO(ncclProxyClientGetFdBlocking(comm, devr->lsaRankList[r], &messages[r], &fd), ret, fail); CUCHECKGOTO(cuMemImportFromShareableHandle(&impHandle, reinterpret_cast((uintptr_t)fd), ncclCuMemHandleType), ret, fail); SYSCHECKGOTO(close(fd), "close", ret, fail); } else { CUCHECKGOTO(cuMemImportFromShareableHandle(&impHandle, (void*)&messages[r].fabricHandle, ncclCuMemHandleType), ret, fail); } } CUdeviceptr addr = reinterpret_cast((char*)devr->lsaFlatBase + r*devr->bigSize + bigOffset); CUCHECKGOTO(cuMemMap(addr, size, 0, impHandle, 0), ret, fail); CUCHECKGOTO(cuMemSetAccess(addr, size, &accessDesc, 1), ret, fail); if (r != devr->lsaSelf) { CUCHECKGOTO(cuMemRelease(impHandle), ret, fail); } } // Ensure everyone has imported my mem handle. NCCLCHECKGOTO(bootstrapIntraNodeBarrier(comm->bootstrap, devr->lsaRankList, devr->lsaSelf, devr->lsaSize, 0xbeef), ret, fail); leave: free(messages); return ret; fail: goto leave; } static ncclResult_t symBindTeamMemory( struct ncclComm* comm, struct ncclDevrTeam* tm, struct ncclDevrMemory* mem ) { if (comm->nvlsSupport && tm->mcBasePtr != nullptr) { #if CUDART_VERSION >= 12010 INFO(NCCL_NVLS, "Binding multicast memory at big=%lx to team {%d x %d}", mem->bigOffset, tm->team.nRanks, tm->team.stride); CUCHECK(cuMulticastBindMem(tm->mcHandle, mem->bigOffset, mem->memHandle, 0, mem->size, 0)); #endif } return ncclSuccess; } static ncclResult_t symUnbindTeamMemory( struct ncclComm* comm, struct ncclDevrTeam* tm, struct ncclDevrMemory* mem ) { if (comm->nvlsSupport && tm->mcBasePtr != nullptr) { #if CUDART_VERSION >= 12010 CUCHECK(cuMulticastUnbind(tm->mcHandle, comm->cudaDev, mem->bigOffset, mem->size)); #endif } return ncclSuccess; } // Caller must barrier the team afterward. static ncclResult_t symTeamObtain( struct ncclComm* comm, struct ncclTeam team, bool multimem, struct ncclDevrTeam** outTeam ) { ncclResult_t ret = ncclSuccess; struct ncclDevrState* devr = &comm->devrState; struct ncclDevrTeam* t = devr->teamHead; bool teamIsNew = false; while (true) { if (t == nullptr) { teamIsNew = true; t = (struct ncclDevrTeam*)malloc(sizeof(struct ncclDevrTeam) + team.nRanks*sizeof(int)); t->team = team; t->mcHandle = 0x0; t->mcBasePtr = nullptr; for (int i=0; i < team.nRanks; i++) { t->worldRankList[i] = comm->rank + (i - team.rank)*team.stride; } break; } else if (t->team.rank == team.rank && t->team.nRanks == team.nRanks && t->team.stride == team.stride) { if (!multimem || t->mcBasePtr != nullptr) { // Matching team is sufficient if (outTeam) *outTeam = t; return ncclSuccess; } break; // Need to enable multimem } } if (multimem) { if (!comm->nvlsSupport) { WARN("Multicast support requested for team but none available on system."); ret = ncclInvalidArgument; goto fail; } else { #if CUDART_VERSION >= 12010 CUmemGenericAllocationHandle mcHandle = 0; CUdeviceptr mcAddr = 0; CUmulticastObjectProp mcProp = {}; char shareableHandle[NVLS_HANDLE_SIZE]; mcProp.numDevices = team.nRanks; mcProp.handleTypes = ncclCuMemHandleType; mcProp.flags = 0; mcProp.size = devr->bigSize; if (team.rank == 0) { NCCLCHECKGOTO(ncclNvlsGroupCreate(comm, &mcProp, team.rank, team.nRanks, &mcHandle, shareableHandle), ret, fail); NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, t->worldRankList, team.rank, team.nRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail_mcHandle); } else { NCCLCHECKGOTO(bootstrapIntraNodeBroadcast(comm->bootstrap, t->worldRankList, team.rank, team.nRanks, 0, shareableHandle, NVLS_HANDLE_SIZE), ret, fail); NCCLCHECKGOTO(ncclNvlsGroupConnect(comm, shareableHandle, t->worldRankList[0], &mcHandle), ret, fail); } CUCHECKGOTO(cuMulticastAddDevice(mcHandle, comm->cudaDev), ret, fail_mcHandle); CUCHECKGOTO(cuMemAddressReserve(&mcAddr, devr->bigSize, NCCL_MAX_PAGE_SIZE, 0, 0), ret, fail_mcHandle); CUCHECKGOTO(cuMemMap(mcAddr, devr->bigSize, 0, mcHandle, 0), ret, fail_mcHandle_mcAddr); { CUmemAccessDesc accessDesc = {}; accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; accessDesc.location.id = comm->cudaDev; accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; CUCHECKGOTO(cuMemSetAccess(mcAddr, devr->bigSize, &accessDesc, 1), ret, fail_mcHandle_mcAddr_unmap); } t->mcHandle = mcHandle; t->mcBasePtr = reinterpret_cast(mcAddr); // Bind new team with all existing memories. for (struct ncclDevrMemory* mem = devr->memHead; mem != nullptr; mem = mem->next) { NCCLCHECKGOTO(symBindTeamMemory(comm, t, mem), ret, fail_mcHandle_mcAddr_unmap_mems); } if (false) { // Error labels: fail_mcHandle_mcAddr_unmap_mems: for (struct ncclDevrMemory* mem = devr->memHead; mem != nullptr; mem = mem->next) { symUnbindTeamMemory(comm, t, mem); } fail_mcHandle_mcAddr_unmap: CUCHECKIGNORE(cuMemUnmap(mcAddr, devr->bigSize)); goto fail_mcHandle_mcAddr; // silence unused label warning fail_mcHandle_mcAddr: CUCHECKIGNORE(cuMemAddressFree(mcAddr, devr->bigSize)); goto fail_mcHandle; // silence unused label warning fail_mcHandle: CUCHECKIGNORE(cuMemRelease(mcHandle)); goto fail; // silence unused label warning } #else goto fail; // silence unused label warning #endif } } if (teamIsNew) { // Add to list t->next = devr->teamHead; devr->teamHead = t; } if (outTeam) *outTeam = t; return ret; fail: if (teamIsNew) free(t); return ret; } static void symTeamDestroyAll(struct ncclComm* comm) { struct ncclDevrState* devr = &comm->devrState; while (devr->teamHead != nullptr) { struct ncclDevrTeam* t = devr->teamHead; devr->teamHead = t->next; if (t->mcBasePtr != nullptr) { for (struct ncclDevrMemory* m = devr->memHead; m != nullptr; m = m->next) { symUnbindTeamMemory(comm, t, m); } CUdeviceptr mcAddr = reinterpret_cast(t->mcBasePtr); CUCHECKIGNORE(cuMemUnmap(mcAddr, devr->bigSize)); CUCHECKIGNORE(cuMemAddressFree(mcAddr, devr->bigSize)); CUCHECKIGNORE(cuMemRelease(t->mcHandle)); } free(t); } } static ncclResult_t symMemoryRegisterGin(struct ncclComm* comm, struct ncclDevrMemory* mem) { NCCLCHECK(ncclGinConnectOnce(comm, comm->globalGinSupport, 0)); // Will allocate the default number of contexts if needed. NCCLCHECK(ncclGinRegister(comm, mem->primaryAddr, mem->size, mem->ginHostWins, mem->ginDevWins, mem->winFlags)); return ncclSuccess; } static ncclResult_t symMemoryRegisterRma(struct ncclComm* comm, struct ncclDevrMemory* mem) { NCCLCHECK(ncclRmaProxyConnectOnce(comm)); NCCLCHECK(ncclRmaProxyRegister(comm, mem->primaryAddr, mem->size, mem->rmaHostWins, mem->rmaDevWins)); return ncclSuccess; } // On success we take caller's reference on memHandle. // Due to multicast binds for each pre-exiting team, this function requires // caller do a world barrier before returning to user. static ncclResult_t symMemoryObtain( struct ncclComm* comm, CUmemGenericAllocationHandle memHandle, void* memAddr, size_t size, int winFlags, struct ncclDevrMemory** outMem ) { ncclResult_t ret = ncclSuccess; struct ncclDevrState* devr = &comm->devrState; int64_t bigOffset = 0; struct ncclDevrMemory* mem = devr->memHead; while (mem != nullptr) { if (mem->memHandle == memHandle) { CUCHECKIGNORE(cuMemRelease(memHandle)); goto leave; } mem = mem->next; } // New memory. mem = (struct ncclDevrMemory*)malloc(sizeof(struct ncclDevrMemory)); mem->refCount = 0; mem->memHandle = memHandle; mem->primaryAddr = memAddr; mem->size = size; mem->winFlags = winFlags; // Grab offset in the big space. NCCLCHECKGOTO(ncclSpaceAlloc(&devr->bigSpace, devr->bigSize, size, devr->granularity, &bigOffset), ret, fail_mem); mem->bigOffset = bigOffset; // Map unicast addresses into flat VA space for lsa team. NCCLCHECKGOTO(symMemoryMapLsaTeam(comm, memHandle, size, bigOffset), ret, fail_mem_space); // If our caller doesn't have a VA then we'll use the LSA mapping. if (mem->primaryAddr == nullptr) { mem->primaryAddr = (char*)devr->lsaFlatBase + devr->lsaSelf*devr->bigSize + mem->bigOffset; } // Bind new memory with each existing team. for (struct ncclDevrTeam* t = devr->teamHead; t != nullptr; t = t->next) { NCCLCHECKGOTO(symBindTeamMemory(comm, t, mem), ret, fail_mem_space_teams); } if (devr->ginEnabled) { NCCLCHECKGOTO(symMemoryRegisterGin(comm, mem), ret, fail_mem_space_teams); } // ginEnabled is set in ncclDevrCommCreateInternal, which might not be called for RMA proxy // so we introduce rmaProxyEnabled to track if RMA proxy is enabled devr->rmaProxyEnabled = devr->nLsaTeams > 1 && comm->config.numRmaCtx > 0 && comm->globalRmaProxySupport; if (devr->rmaProxyEnabled) { NCCLCHECKGOTO(symMemoryRegisterRma(comm, mem), ret, fail_mem_space_teams); } // Add to list of mems. mem->next = devr->memHead; devr->memHead = mem; leave: mem->refCount += 1; *outMem = mem; return ret; fail_mem_space_teams: for (struct ncclDevrTeam* t = devr->teamHead; t != nullptr; t = t->next) { symUnbindTeamMemory(comm, t, mem); } fail_mem_space: ncclSpaceFree(&devr->bigSpace, bigOffset, size); fail_mem: free(mem); //fail: return ret; } static void symMemoryDropRef( struct ncclComm* comm, struct ncclDevrMemory* mem ) { if (mem != nullptr && 0 == --mem->refCount) { struct ncclDevrState* devr = &comm->devrState; if (devr->ginEnabled) { ncclGinDeregister(comm, mem->ginHostWins); } if (devr->rmaProxyEnabled) { ncclRmaProxyDeregister(comm, mem->rmaHostWins); } for (struct ncclDevrTeam* t = devr->teamHead; t != nullptr; t = t->next) { symUnbindTeamMemory(comm, t, mem); } for (int r = 0; r < devr->lsaSize; r++) { CUdeviceptr addr = reinterpret_cast((char*)devr->lsaFlatBase + r*devr->bigSize + mem->bigOffset); CUCHECKIGNORE(cuMemUnmap(addr, mem->size)); } ncclSpaceFree(&devr->bigSpace, mem->bigOffset, mem->size); CUCHECKIGNORE(cuMemRelease(mem->memHandle)); struct ncclDevrMemory** ptr = &devr->memHead; while (*ptr != mem) ptr = &(*ptr)->next; *ptr = mem->next; // Remove from list. free(mem); } } static ncclResult_t symWindowTableInitOnce(struct ncclComm* comm, cudaStream_t stream) { struct ncclDevrState* devr = &comm->devrState; struct ncclDevCommWindowTable* tableDev = devr->windowTable; if (tableDev == nullptr) { // Create on first need. NCCLCHECK(ncclShadowPoolAlloc(&devr->shadows, &tableDev, nullptr, stream)); devr->windowTable = tableDev; } return ncclSuccess; } // On success we take callers reference on `mem`. static ncclResult_t symWindowCreate( struct ncclComm* comm, struct ncclDevrMemory* mem, size_t memOffset, void* userPtr, size_t userSize, int winFlags, void* localReg, struct ncclWindow_vidmem** outWinDev, struct ncclDevrWindow** outWin, cudaStream_t stream ) { uintptr_t userAddr = reinterpret_cast(userPtr); struct ncclDevrState* devr = &comm->devrState; struct ncclDevrWindow* win; win = (struct ncclDevrWindow*)malloc(sizeof(struct ncclDevrWindow)); memset(win, 0, sizeof(*win)); win->memory = mem; win->size = userSize; win->bigOffset = mem->bigOffset + memOffset; win->winFlags = winFlags; win->localRegHandle = localReg; if (userPtr == nullptr) { // Null means caller has no VA and will use the lsa team flat VA address. win->userPtr = userPtr = (char*)devr->lsaFlatBase + (devr->lsaSelf*devr->bigSize) + mem->bigOffset; userAddr = reinterpret_cast(userPtr); } else { win->userPtr = userPtr; } struct ncclWindow_vidmem* winDev; struct ncclWindow_vidmem* winDevHost; NCCLCHECK(ncclShadowPoolAlloc(&devr->shadows, &winDev, &winDevHost, stream)); win->vidmem = winDev; winDevHost->lsaFlatBase = (char*)devr->lsaFlatBase + win->bigOffset; winDevHost->mcOffset4K = win->bigOffset>>12; winDevHost->stride4G = devr->bigSize>>32; winDevHost->lsaRank = devr->lsaSelf; winDevHost->worldRank = comm->rank; winDevHost->winHost = (void*)win; winDevHost->ginOffset4K = memOffset>>12; for (int i=0; i < NCCL_GIN_MAX_CONNECTIONS; i++) { winDevHost->ginWins[i] = mem->ginDevWins[i]; } CUDACHECK(cudaMemcpyAsync(winDev, winDevHost, sizeof(struct ncclWindow_vidmem), cudaMemcpyHostToDevice, stream)); NCCLCHECK(symWindowTableInitOnce(comm, stream)); // ensure devr->windowTable exists struct ncclDevCommWindowTable* tableDev = devr->windowTable; while (true) { struct ncclDevCommWindowTable* tableHost; NCCLCHECK(ncclShadowPoolToHost(&devr->shadows, tableDev, &tableHost)); int i = 0; while (i < 32 && tableHost->entries[i].window != nullptr) i += 1; if (i < 32) { tableHost->entries[i].base = userAddr; tableHost->entries[i].size = userSize; tableHost->entries[i].window = winDev; CUDACHECK(cudaMemcpyAsync(&tableDev->entries[i], &tableHost->entries[i], sizeof(tableHost->entries[i]), cudaMemcpyHostToDevice, stream)); break; } if (tableHost->next == nullptr) { NCCLCHECK(ncclShadowPoolAlloc(&devr->shadows, &tableHost->next, nullptr, stream)); CUDACHECK(cudaMemcpyAsync(&tableDev->next, &tableHost->next, sizeof(tableHost->next), cudaMemcpyHostToDevice, stream)); } tableDev = tableHost->next; } { // insert into winSorted[] int i = listFindSortedLub(&ncclDevrWindowSorted::userAddr, devr->winSorted, devr->winSortedCount, userAddr); struct ncclDevrWindowSorted winSort; winSort.userAddr = userAddr; winSort.size = userSize; winSort.win = win; listInsert(&devr->winSorted, &devr->winSortedCapacity, &devr->winSortedCount, i, winSort); } if (outWinDev) *outWinDev = winDev; if (outWin) *outWin = win; return ncclSuccess; } static ncclResult_t symWindowDestroy(struct ncclComm* comm, struct ncclWindow_vidmem* winDev, cudaStream_t stream) { ncclResult_t ret = ncclSuccess; struct ncclDevrState* devr = &comm->devrState; struct ncclWindow_vidmem* winDevHost; struct ncclDevrWindow* winHost; NCCLCHECKGOTO(ncclShadowPoolToHost(&devr->shadows, winDev, &winDevHost), ret, fail); winHost = (struct ncclDevrWindow*)winDevHost->winHost; symMemoryDropRef(comm, winHost->memory); { struct ncclDevCommWindowTable* tableDev = devr->windowTable; while (true) { struct ncclDevCommWindowTable* tableHost; NCCLCHECKGOTO(ncclShadowPoolToHost(&devr->shadows, tableDev, &tableHost), ret, remove_winSorted); int i = 0; while (i < 32 && tableHost->entries[i].window != winDev) i += 1; if (i < 32) { memset(&tableHost->entries[i], 0, sizeof(tableHost->entries[i])); CUDACHECKGOTO(cudaMemsetAsync(&tableDev->entries[i], 0, sizeof(tableDev->entries[i]), stream), ret, remove_winSorted); break; } if (tableHost->next == nullptr) break; // Error didn't find window in table tableDev = tableHost->next; } } NCCLCHECKGOTO(ncclShadowPoolFree(&devr->shadows, winDev, stream), ret, remove_winSorted); NCCLCHECKGOTO(ncclCommDeregister(comm, winHost->localRegHandle), ret, remove_winSorted); remove_winSorted: { int i = listFindSortedLub(&ncclDevrWindowSorted::userAddr, devr->winSorted, devr->winSortedCount, reinterpret_cast(winHost->userPtr)); i -= 1; // least upper bound is just after ours. listRemove(devr->winSorted, &devr->winSortedCount, i); } // Remove the just deallocated window from the table storing the communicator pointer { std::lock_guard lock(ncclWindowMapMutex); NCCLCHECKGOTO(ncclIntruAddressMapRemove(&ncclWindowMap, winDev), ret, fail); } free(winHost); fail: return ret; } ncclResult_t ncclDevrWindowRegisterInGroup( struct ncclComm* comm, void* userPtr, size_t userSize, int winFlags, ncclWindow_t* outWinDev ) { ncclResult_t ret = ncclSuccess; CUdeviceptr memAddr = 0; size_t memSize = 0; CUmemGenericAllocationHandle memHandle = 0x0ULL; size_t memOffset; struct ncclDevrMemory* mem = nullptr; cudaStream_t stream = nullptr; void* localRegHandle = nullptr; struct ncclDevrWindow* winHost = nullptr; int numSegments = 0; NCCLCHECKGOTO(ncclCommRegister(comm, userPtr, userSize, &localRegHandle), ret, fail); if (winFlags & NCCL_WIN_COLL_SYMMETRIC) { // Defer symmetric kernel init until at least one window with that flag exists. NCCLCHECKGOTO(ncclSymkInitOnce(comm), ret, fail_locReg); } // Get underlying cumem base address and number of mapped physical segments that userPtr spans NCCLCHECKGOTO(ncclCuMemGetAddressRange(reinterpret_cast(userPtr), userSize, &memAddr, &memSize, &numSegments), ret, fail_locReg); if (numSegments > 1) { WARN("Window registration of addresses that span multiple physical segments is currently not supported."); ret = ncclInvalidArgument; goto fail_locReg; } memOffset = reinterpret_cast(userPtr) - memAddr; if (memOffset%NCCL_WIN_REQUIRED_ALIGNMENT != 0) { WARN("Window address must be suitably aligned."); ret = ncclInvalidArgument; goto fail_locReg; } CUCHECKGOTO(cuMemRetainAllocationHandle(&memHandle, reinterpret_cast(memAddr)), ret, fail_locReg); // Trade cumem handle for ncclDevrMemory* NCCLCHECKGOTO(symMemoryObtain(comm, memHandle, (void*)memAddr, memSize, winFlags, &mem), ret, fail_locReg_memHandle); memHandle = 0x0; // symMemoryObtain took our reference CUDACHECKGOTO(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), ret, fail_locReg_memHandle_mem); NCCLCHECKGOTO(symWindowCreate( comm, mem, memOffset, userPtr, userSize, winFlags, localRegHandle, outWinDev, &winHost, stream ), ret, fail_locReg_memHandle_mem_stream); mem = nullptr; // symWindowCreate took our reference CUDACHECKGOTO(cudaStreamSynchronize(stream), ret, fail_locReg_memHandle_mem_stream_win); // symWindowCreate needs barrier. NCCLCHECKGOTO(bootstrapBarrier(comm->bootstrap, comm->rank, comm->nRanks, 0xbeef), ret, fail_locReg_memHandle_mem_stream_win); { std::lock_guard lock(ncclWindowMapMutex); // Set intrusive map fields directly on winHost winHost->comm = comm; winHost->next = nullptr; // Initialize next pointer // Since windows are unique and belong to a single communicator, // it is not necessary to check if the insert would overwrite existing entries NCCLCHECKGOTO(ncclIntruAddressMapInsert(&ncclWindowMap, *outWinDev, winHost), ret, fail_locReg_memHandle_mem_stream_win); INFO(NCCL_ALLOC, "Inserted window %p into address map, ret=%d", *outWinDev, ret); } cudaStreamDestroy(stream); return ret; fail_locReg_memHandle_mem_stream_win: symWindowDestroy(comm, *outWinDev, stream); *outWinDev = nullptr; cudaStreamSynchronize(stream); fail_locReg_memHandle_mem_stream: cudaStreamDestroy(stream); fail_locReg_memHandle_mem: symMemoryDropRef(comm, mem); fail_locReg_memHandle: if (memHandle != 0x0ULL) { CUCHECKIGNORE(cuMemRelease(memHandle)); } fail_locReg: ncclCommDeregister(comm, localRegHandle); fail: *outWinDev = nullptr; return ret; } static ncclResult_t deepCopyDevCommRequirements( struct ncclDevCommRequirements const* src, struct ncclDevCommRequirements** dst ) { ncclResult_t ret = ncclSuccess; struct ncclDevResourceRequirements **dstRes; struct ncclTeamRequirements **dstTeam; NCCLCHECK(ncclCalloc(dst, 1)); /* copy the entire struct now and update linked lists later */ **dst = *src; dstRes = &(*dst)->resourceRequirementsList; for (struct ncclDevResourceRequirements* rr = src->resourceRequirementsList; rr != nullptr; rr = rr->next) { NCCLCHECKGOTO(ncclCalloc(dstRes, 1), ret, fail); (*dstRes)->bufferSize = rr->bufferSize; (*dstRes)->bufferAlign = rr->bufferAlign; (*dstRes)->outBufferHandle = rr->outBufferHandle; dstRes = &(*dstRes)->next; } dstTeam = &(*dst)->teamRequirementsList; for (struct ncclTeamRequirements* tr = src->teamRequirementsList; tr != nullptr; tr = tr->next) { NCCLCHECKGOTO(ncclCalloc(dstTeam, 1), ret, fail); (*dstTeam)->team = tr->team; (*dstTeam)->multimem = tr->multimem; (*dstTeam)->outMultimemHandle = tr->outMultimemHandle; dstTeam = &(*dstTeam)->next; } exit: return ret; fail: freeDevCommRequirements(*dst); *dst = nullptr; goto exit; } void freeDevCommRequirements( struct ncclDevCommRequirements* reqs ) { if (reqs) { while (reqs->resourceRequirementsList) { struct ncclDevResourceRequirements* rr_next = reqs->resourceRequirementsList->next; free(reqs->resourceRequirementsList); reqs->resourceRequirementsList = rr_next; } while (reqs->teamRequirementsList) { struct ncclTeamRequirements* tr_next = reqs->teamRequirementsList->next; free(reqs->teamRequirementsList); reqs->teamRequirementsList = tr_next; } free(reqs); } } bool ncclGinResourcesRequested(struct ncclDevCommRequirements const* reqs) { bool requestedGinResources = reqs->ginSignalCount > 0 || reqs->ginCounterCount > 0 || reqs->barrierCount > 0 || reqs->railGinBarrierCount > 0; struct ncclDevResourceRequirements* node = reqs->resourceRequirementsList; while (!requestedGinResources && node != nullptr) { requestedGinResources = node->ginSignalCount > 0 || node->ginCounterCount > 0; node = node->next; } return requestedGinResources; } NCCL_PARAM(GinExclusiveContexts, "GIN_EXCLUSIVE_CONTEXTS", -1); ncclResult_t ncclDevrCommCreateInternal( struct ncclComm* comm, struct ncclDevCommRequirements const* reqs, struct ncclDevComm* outDevComm, bool isInternal, ncclResult_t (*outDevCommCopyCB)(struct ncclDevComm const* tmpDevComm, void* out) ) { ncclResult_t ret = ncclSuccess; struct ncclDevrState* devr = &comm->devrState; struct ncclTeam world = ncclTeamWorld(comm); struct ncclTeam lsa = ncclTeamInnerFactor(world, devr->lsaSize); bool ginActivated = false; struct ncclDevrTeam* tmLsa; size_t bufSizeTotal; int nGinConnections = 0; int nGinContexts = 0; int ginSignalTotal = 0, ginCounterTotal = 0; struct ncclDevResourceRequirements* resReqsHead = reqs->resourceRequirementsList; struct ncclDevResourceRequirements lsaBarReq; cudaStream_t stream = nullptr; struct ncclDevResourceRequirements railGinBarrierReq; CUmemGenericAllocationHandle memHandle = 0x0; struct ncclDevrMemory* mem = nullptr; struct ncclDevrWindow* win = nullptr; struct ncclWindow_vidmem* winHost = nullptr; size_t ginSignalShadowsOffset = 0; bool ginExclusiveContexts = false; void* outDevCommPreserve; struct ncclDevComm outDevCommTmp; ncclGinConnectionType_t requestedConnectionType = reqs->ginConnectionType; if (reqs->ginForceEnable) { INFO(NCCL_INIT, "ginForceEnable set to true, defaulting ginConnectionType to NCCL_GIN_CONNECTION_FULL"); INFO(NCCL_INIT, "ginForceEnable is being deprecated in favor of explicitly setting ginConnectionType!"); requestedConnectionType = NCCL_GIN_CONNECTION_FULL; } bool requestedGinResources = ncclGinResourcesRequested(reqs); if (requestedGinResources && requestedConnectionType == NCCL_GIN_CONNECTION_NONE) { WARN("User requested GIN resources but did not request GIN to be enabled!"); return ncclInvalidArgument; } if (requestedConnectionType != NCCL_GIN_CONNECTION_NONE) { if (comm->globalGinSupport == NCCL_GIN_CONNECTION_NONE) { WARN("User requested GIN but not all ranks in the communicator support GIN"); return ncclInvalidArgument; } if (requestedConnectionType == NCCL_GIN_CONNECTION_FULL) { if (comm->globalGinSupport == NCCL_GIN_CONNECTION_RAIL) { WARN("User requested GIN connection type NCCL_GIN_CONNECTION_FULL but the communicator supports only NCCL_GIN_CONNECTION_RAIL"); return ncclInvalidArgument; } if (comm->sharedRes->ginState.ginConnectionType == NCCL_GIN_CONNECTION_RAIL) { WARN("User requested GIN connection type NCCL_GIN_CONNECTION_FULL but the communicator is already connected with NCCL_GIN_CONNECTION_RAIL"); return ncclInvalidArgument; } } ginActivated = !devr->ginEnabled; devr->ginEnabled = true; } if (ginActivated) { NCCLCHECKGOTO(ncclGinConnectOnce(comm, requestedConnectionType, reqs->ginContextCount, reqs->ginQueueDepth), ret, fail); // Register all preexisting memories with GIN. Update the windows later when // we have a stream. for (struct ncclDevrMemory* mem = devr->memHead; mem != nullptr; mem = mem->next) { NCCLCHECKGOTO(symMemoryRegisterGin(comm, mem), ret, fail); } } if (devr->ginEnabled) { nGinConnections = comm->sharedRes->ginState.ginCommCount; if (ncclParamGinExclusiveContexts() != -1) ginExclusiveContexts = ncclParamGinExclusiveContexts(); else ginExclusiveContexts = reqs->ginExclusiveContexts; if (ginExclusiveContexts) { int unallocated = comm->sharedRes->ginState.ctxLastExclusive - comm->sharedRes->ginState.ctxFirstAvailable; nGinContexts = reqs->ginContextCount; if (nGinContexts > unallocated) { WARN("Requested number of exclusive GIN contexts (%d) exceeds the unallocated count (%d). Use NCCL_GIN_NCONTEXTS to increase the limit", nGinContexts, unallocated); ret = ncclInvalidArgument; goto fail; } } else { nGinContexts = std::min(reqs->ginContextCount, comm->sharedRes->ginState.ctxLastExclusive); if (nGinContexts == 0) { WARN("No shared contexts are available (%d requested) as all have been allocated for exclusive use. Use NCCL_GIN_NCONTEXTS to increase the limit", reqs->ginContextCount); ret = ncclInvalidArgument; goto fail; } if (nGinContexts < reqs->ginContextCount) { INFO(NCCL_INIT|NCCL_NET, "Capping the number of GIN contexts to %d (%d requested). Use NCCL_GIN_NCONTEXTS to increase the limit", nGinContexts, reqs->ginContextCount); } } } // If we have a copy callback for backwards compatibility, we use a temporary buffer for the devComm and, once we're // finished, we let the callback copy the data over. if (outDevCommCopyCB) { outDevCommPreserve = outDevComm; outDevComm = &outDevCommTmp; } memset(outDevComm, 0, sizeof(*outDevComm)); outDevComm->rank = comm->rank; outDevComm->nRanks = comm->nRanks; outDevComm->nRanks_rcp32 = idivRcp32(comm->nRanks); outDevComm->lsaRank = devr->lsaSelf; outDevComm->lsaSize = devr->lsaSize; outDevComm->lsaSize_rcp32 = idivRcp32(devr->lsaSize); outDevComm->ginIsRailed = requestedConnectionType == NCCL_GIN_CONNECTION_RAIL; // false if FULL or NONE if (isInternal) outDevComm->abortFlag = comm->abortFlagDev; NCCLCHECKGOTO(symTeamObtain(comm, lsa, /*multicast=*/reqs->lsaMultimem, &tmLsa), ret, fail); outDevComm->lsaMultimem.mcBasePtr = tmLsa->mcBasePtr; { struct ncclTeamRequirements* tr = reqs->teamRequirementsList; while (tr != nullptr) { if (tr->multimem) { struct ncclDevrTeam* tm; NCCLCHECKGOTO(symTeamObtain(comm, tr->team, tr->multimem, &tm), ret, fail); if (tr->outMultimemHandle != nullptr) tr->outMultimemHandle->mcBasePtr = tm->mcBasePtr; } tr = tr->next; } } resReqsHead = reqs->resourceRequirementsList; ncclLsaBarrierCreateRequirement(lsa, std::max(reqs->barrierCount, reqs->lsaBarrierCount), &outDevComm->lsaBarrier, &lsaBarReq); lsaBarReq.next = resReqsHead; resReqsHead = &lsaBarReq; ncclGinBarrierCreateRequirement(comm, ncclTeamRail(comm), std::max(reqs->barrierCount, reqs->railGinBarrierCount), &outDevComm->railGinBarrier, &railGinBarrierReq); railGinBarrierReq.next = resReqsHead; resReqsHead = &railGinBarrierReq; { struct ncclDevResourceRequirements* rr = resReqsHead; bufSizeTotal = 0; ginSignalTotal = reqs->ginSignalCount; ginCounterTotal = reqs->ginCounterCount; while (rr != nullptr) { bufSizeTotal = alignUp(bufSizeTotal, std::max(128, rr->bufferAlign)); if (rr->outBufferHandle != nullptr) *rr->outBufferHandle = bufSizeTotal/128; if (rr->outGinSignalStart != nullptr) *rr->outGinSignalStart = ginSignalTotal; if (rr->outGinCounterStart != nullptr) *rr->outGinCounterStart = ginCounterTotal; bufSizeTotal += rr->bufferSize; ginSignalTotal += rr->ginSignalCount; ginCounterTotal += rr->ginCounterCount; rr = rr->next; } bufSizeTotal= alignUp(bufSizeTotal, 128); ginSignalShadowsOffset = bufSizeTotal; bufSizeTotal += nGinContexts*ginSignalTotal*sizeof(uint64_t); // include signal shadows bufSizeTotal = alignUp(bufSizeTotal, devr->granularity); } CUDACHECKGOTO(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), ret, fail); if (ginActivated) { // Now update the GIN handles in all existing windows. Registration of memories happened above. for (int i=0; i < devr->winSortedCount; i++) { struct ncclDevrWindow* win = devr->winSorted[i].win; struct ncclWindow_vidmem* winHost; NCCLCHECKGOTO(ncclShadowPoolToHost(&devr->shadows, win->vidmem, &winHost), ret, fail_stream); winHost->ginOffset4K = (win->bigOffset - win->memory->bigOffset)>>12; for (int i=0; i < NCCL_GIN_MAX_CONNECTIONS; i++) { winHost->ginWins[i] = win->memory->ginDevWins[i]; } CUDACHECKGOTO(cudaMemcpyAsync(win->vidmem, winHost, sizeof(struct ncclWindow_vidmem), cudaMemcpyHostToDevice, stream), ret, fail_stream); } } NCCLCHECKGOTO(symWindowTableInitOnce(comm, stream), ret, fail_stream); // ensure devr->windowTable exists outDevComm->windowTable = devr->windowTable; if (bufSizeTotal == 0) { outDevComm->resourceWindow = nullptr; outDevComm->resourceWindow_inlined = {}; } else { CUmemAllocationProp memProp = {}; memProp.type = CU_MEM_ALLOCATION_TYPE_PINNED; memProp.location.type = CU_MEM_LOCATION_TYPE_DEVICE; memProp.requestedHandleTypes = ncclCuMemHandleType; // We have to assume that if GIN is possible it might be requested in the future, // even on single node. memProp.allocFlags.gpuDirectRDMACapable = comm->sharedRes->ginState.ncclGin != nullptr ? 1 : 0; memProp.location.id = comm->cudaDev; CUCHECKGOTO(cuMemCreate(&memHandle, bufSizeTotal, &memProp, 0), ret, fail_stream); NCCLCHECKGOTO(symMemoryObtain(comm, memHandle, NULL, bufSizeTotal, /*winFlags=*/0, &mem), ret, fail_stream_mem); memHandle = 0x0; // Reference given to symMemoryObtain NCCLCHECKGOTO(symWindowCreate( // Requires world barrier afterward. comm, mem, /*memOffset=*/0, nullptr, bufSizeTotal, /*winFlags=*/0, /*localReg=*/nullptr, &outDevComm->resourceWindow, &win, stream), ret, fail_stream_mem); mem = nullptr; // Reference given to symWindowCreate NCCLCHECKGOTO(ncclShadowPoolToHost(&devr->shadows, win->vidmem, &winHost), ret, fail_stream_mem_win); outDevComm->resourceWindow_inlined = *winHost; outDevComm->ginSignalShadows = (uint64_t*)add4G((char*)winHost->lsaFlatBase + ginSignalShadowsOffset, winHost->lsaRank*winHost->stride4G); CUDACHECKGOTO(cudaMemsetAsync(win->userPtr, 0, bufSizeTotal, stream), ret, fail_stream_mem_win); } if (devr->ginEnabled) { outDevComm->ginConnectionCount = nGinConnections; outDevComm->ginContextCount = nGinContexts; outDevComm->ginSignalCount = ginSignalTotal; outDevComm->ginCounterCount = ginCounterTotal; NCCLCHECKGOTO(ncclGinAllocSignalsCounters(comm, ginSignalTotal, &outDevComm->ginSignalBase, ginCounterTotal, &outDevComm->ginCounterBase ), ret, fail_stream_mem_win); if (ginExclusiveContexts) { comm->sharedRes->ginState.ctxLastExclusive -= nGinContexts; outDevComm->ginContextBase = comm->sharedRes->ginState.ctxLastExclusive; } else { comm->sharedRes->ginState.ctxFirstAvailable = std::max(comm->sharedRes->ginState.ctxFirstAvailable, nGinContexts); outDevComm->ginContextBase = 0; } INFO(NCCL_INIT|NCCL_NET, "Initialized a devComm with %d GIN connections, %d %s contexts (base %d), %d signals (base %d), %d counters (base %d)", nGinConnections, nGinContexts, (ginExclusiveContexts ? "exclusive" : "shared"), outDevComm->ginContextBase, ginSignalTotal, outDevComm->ginSignalBase, ginCounterTotal, outDevComm->ginCounterBase); for (int connectionId=0; connectionId < nGinConnections; connectionId++) { outDevComm->ginNetDeviceTypes[connectionId] = (int)comm->sharedRes->ginState.ginDevHandles[connectionId]->netDeviceType; outDevComm->ginHandles[connectionId] = comm->sharedRes->ginState.ginDevHandles[connectionId]->handle; } } CUDACHECKGOTO(cudaStreamSynchronize(stream), ret, fail_stream_mem_win_signals); NCCLCHECKGOTO(bootstrapBarrier(comm->bootstrap, comm->rank, comm->nRanks, 0xbeef), ret, fail_stream_mem_win_signals); CUDACHECKGOTO(cudaStreamDestroy(stream), ret, fail_stream_mem_win_signals); if (outDevCommCopyCB) NCCLCHECKGOTO(outDevCommCopyCB(outDevComm, outDevCommPreserve), ret, fail_stream_mem_win_signals); return ret; fail_stream_mem_win_signals: if (devr->ginEnabled) { ncclGinFreeSignalsCounters(comm, outDevComm->ginSignalBase, outDevComm->ginSignalCount, outDevComm->ginCounterBase, outDevComm->ginCounterCount ); } fail_stream_mem_win: symWindowDestroy(comm, win->vidmem, stream); cudaStreamSynchronize(stream); fail_stream_mem: if (memHandle != 0x0) { CUCHECKIGNORE(cuMemRelease(memHandle)); } symMemoryDropRef(comm, mem); fail_stream: cudaStreamDestroy(stream); fail: return ret; } //////////////////////////////////////////////////////////////////////////////// NCCL_API(ncclResult_t, ncclCommWindowRegister, ncclComm_t comm, void* buff, size_t size, ncclWindow_t* win, int winFlags); ncclResult_t ncclCommWindowRegister(ncclComm_t comm, void* buff, size_t size, ncclWindow_t* win, int winFlags) { NCCLCHECK(CommCheck(comm, __func__, "comm")); NCCLCHECK(PtrCheck(win, __func__, "win")); *win = nullptr; if (buff == nullptr || size <= 0) { WARN("%s: invalid pointer %p / size %zu", __func__, buff, size); return ncclInvalidArgument; } if (!comm->symmetricSupport) { return ncclSuccess; } ncclResult_t ret = ncclSuccess; int saveDev; struct ncclDevrRegTask* task; CUDACHECK(cudaGetDevice(&saveDev)); NCCLCHECK(ncclGroupStartInternal()); NCCLCHECKGOTO(ncclCommEnsureReady(comm), ret, fail); CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), ret, fail); NCCLCHECKGOTO(ncclDevrInitOnce(comm), ret, fail); NCCLCHECKGOTO(ncclCalloc(&task, 1), ret, fail); task->userPtr = buff; task->userSize = size; task->winFlags = winFlags; task->outWinDev = win; ncclIntruQueueEnqueue(&comm->devrState.regTaskQueue, task); ncclGroupCommJoin(comm, ncclGroupTaskTypeSymRegister); exit: ncclGroupErrCheck(ret); NCCLCHECK(ncclGroupEndInternal()); cudaSetDevice(saveDev); return ret; fail: goto exit; } NCCL_API(ncclResult_t, ncclCommWindowDeregister, ncclComm_t comm, ncclWindow_t win); ncclResult_t ncclCommWindowDeregister(struct ncclComm* comm, struct ncclWindow_vidmem* winDev) { NCCLCHECK(CommCheck(comm, __func__, "comm")); ncclResult_t ret = ncclSuccess; int saveDev; cudaStream_t stream; if (winDev == nullptr) goto exit; CUDACHECKGOTO(cudaGetDevice(&saveDev), ret, fail); CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), ret, fail); CUDACHECKGOTO(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), ret, fail_dev); NCCLCHECKGOTO(symWindowDestroy(comm, winDev, stream), ret, fail_dev_stream); fail_dev_stream: cudaStreamSynchronize(stream); cudaStreamDestroy(stream); fail_dev: cudaSetDevice(saveDev); fail: exit: return ret; } ncclResult_t ncclDevrFindWindow( struct ncclComm* comm, void const* userPtr, struct ncclDevrWindow** outWin ) { struct ncclDevrState* devr = &comm->devrState; uintptr_t userAddr = reinterpret_cast(userPtr); int i = listFindSortedLub(&ncclDevrWindowSorted::userAddr, devr->winSorted, devr->winSortedCount, userAddr); if (0 < i && (userAddr - devr->winSorted[i-1].userAddr < devr->winSorted[i-1].size)) { *outWin = devr->winSorted[i-1].win; } else { *outWin = nullptr; } return ncclSuccess; } // Returns ncclInvalidUsage if the compiled version is greater than the runtime version and NCCL_ENABLE_VERSION_CHECK=0 is not set static ncclResult_t validateNcclVersion(int compiledVersion, int minSupportedVersion = -1) { int runtimeVersion; if (ncclParamEnableVersionCheck() == 0) return ncclSuccess; NCCLCHECK(ncclGetVersion(&runtimeVersion)); if (compiledVersion > runtimeVersion) { WARN("NCCL library version is too old. This application was compiled with NCCL version %d, but is running with NCCL library version %d.", compiledVersion, runtimeVersion); return ncclInvalidUsage; } if (minSupportedVersion > 0 && compiledVersion < minSupportedVersion) { WARN("The application was compiled with too old version of NCCL. It was compiled with NCCL version %d, but is running with NCCL library version %d. It needs to be recompiled with at least NCCL version %d.", compiledVersion, runtimeVersion, minSupportedVersion); return ncclInvalidUsage; } return ncclSuccess; } typedef enum : uint8_t { NCCL_GIN_TYPE_NONE_v22902 = 0, NCCL_GIN_TYPE_PROXY_v22902 = 2, NCCL_GIN_TYPE_GDAKI_v22902 = 3, } ncclGinType_t_v22902; struct ncclCommProperties_v22902 { size_t size; unsigned int magic; unsigned int version; int rank; int nRanks; int cudaDev; int nvmlDev; bool deviceApiSupport; bool multimemSupport; ncclGinType_t_v22902 ginType; }; static ncclResult_t ncclCommQueryProperties_v22902(ncclComm_t comm, struct ncclCommProperties_v22902* props) { ncclCommProperties_t newProps = NCCL_COMM_PROPERTIES_INITIALIZER; NCCLCHECK(ncclCommQueryProperties(comm, &newProps)); props->rank = newProps.rank; props->nRanks = newProps.nRanks; props->cudaDev = newProps.cudaDev; props->nvmlDev = newProps.nvmlDev; // We don't provide backwards compatibility for GIN with 2.29.2. If a communicator needs it, we disable Device API. props->deviceApiSupport = (newProps.deviceApiSupport && ncclTeamLsa(comm).nRanks == comm->nRanks); props->multimemSupport = newProps.multimemSupport; props->ginType = NCCL_GIN_TYPE_NONE_v22902; return ncclSuccess; } NCCL_API(ncclResult_t, ncclCommQueryProperties, ncclComm_t, ncclCommProperties_t*); ncclResult_t ncclCommQueryProperties(ncclComm_t comm, ncclCommProperties_t* props) { NCCLCHECK(CommCheck(comm, __func__, "comm")); NCCLCHECK(PtrCheck(props, __func__, "props")); NCCLCHECK(ncclCommEnsureReady(comm)); if (props->magic != NCCL_API_MAGIC) { WARN("Cannot get communicator properties: ncclCommProperties_t argument must be initialized via NCCL_COMM_PROPERTIES_INITIALIZER"); return ncclInvalidUsage; } NCCLCHECK(validateNcclVersion(props->version)); if (props->version >= NCCL_VERSION(2, 29, 2) && props->version <= NCCL_VERSION(2, 29, 3)) { NCCLCHECK(ncclCommQueryProperties_v22902(comm, (struct ncclCommProperties_v22902*)props)); return ncclSuccess; } props->rank = comm->rank; props->nRanks = comm->nRanks; props->cudaDev = comm->cudaDev; props->nvmlDev = comm->nvmlDev; props->deviceApiSupport = comm->symmetricSupport; props->multimemSupport = comm->nvlsSupport; props->hostRmaSupport = comm->hostRmaSupport; NCCLCHECK(getGlobalGinType(comm, &props->ginType)); NCCLCHECK(getGlobalRailedGinType(comm, &props->railedGinType)); // Preferring to call ncclDevrInitOnce directly instead to calling ncclTeam* functions because // we can propagate the result of ncclDevrInitOnce back to the caller. NCCLCHECK(ncclDevrInitOnce(comm)); props->nLsaTeams = comm->devrState.nLsaTeams; return ncclSuccess; } struct ncclDevCommRequirements_v22902 { size_t size; unsigned int magic; unsigned int version; // These two structures are unchanged. ncclDevResourceRequirements_t* resourceRequirementsList; ncclTeamRequirements_t* teamRequirementsList; bool lsaMultimem; int barrierCount; int lsaBarrierCount; int railGinBarrierCount; int lsaLLA2ABlockCount, lsaLLA2ASlotCount; bool ginForceEnable; int ginContextCount; int ginSignalCount; int ginCounterCount; }; struct ncclDevComm_v22902 { int rank, nRanks; uint32_t nRanks_rcp32; int lsaRank, lsaSize; uint32_t lsaSize_rcp32; // This structure is unchanged. struct ncclDevCommWindowTable* windowTable; // The ncclWindow_vidmem structure is unchanged, and ncclWindow_t is just a (device) pointer to it. ncclWindow_t resourceWindow; struct ncclWindow_vidmem resourceWindow_inlined; // ncclMultimemHandle_t, ncclLsaBarrierHandle_t, and ncclGinBarrierHandle_t are unchanged. ncclMultimemHandle_t lsaMultimem; ncclLsaBarrierHandle_t lsaBarrier; ncclGinBarrierHandle_t railGinBarrier; uint8_t ginContextCount; uint8_t ginNetDeviceTypes[4]; void* ginHandles[4]; uint32_t ginSignalBase; int ginSignalCount; uint32_t ginCounterBase; int ginCounterCount; uint64_t* ginSignalShadows; }; static ncclResult_t ncclDevCommCreateCopyCB_v22902(struct ncclDevComm const* tmpDevComm, void* out) { struct ncclDevComm_v22902* outDevComm = (struct ncclDevComm_v22902*)out; outDevComm->rank = tmpDevComm->rank; outDevComm->nRanks = tmpDevComm->nRanks; outDevComm->nRanks_rcp32 = tmpDevComm->nRanks_rcp32; outDevComm->lsaRank = tmpDevComm->lsaRank; outDevComm->lsaSize = tmpDevComm->lsaSize; outDevComm->lsaSize_rcp32 = tmpDevComm->lsaSize_rcp32; outDevComm->windowTable = tmpDevComm->windowTable; outDevComm->resourceWindow = tmpDevComm->resourceWindow; outDevComm->resourceWindow_inlined = tmpDevComm->resourceWindow_inlined; outDevComm->lsaMultimem = tmpDevComm->lsaMultimem; outDevComm->lsaBarrier = tmpDevComm->lsaBarrier; // No need to copy GIN-specific fields since this is used only if GIN has not been requested. return ncclSuccess; } static ncclResult_t ncclDevCommCreateCommon( ncclComm_t comm, struct ncclDevCommRequirements const* reqs, struct ncclDevComm* outDevComm, ncclResult_t (*outDevCommCopyCB)(struct ncclDevComm const* tmpDevComm, void* out) ) { ncclResult_t ret = ncclSuccess; int saveDev; struct ncclDevrCommCreateTask* task = nullptr; CUDACHECK(cudaGetDevice(&saveDev)); NCCLCHECK(ncclGroupStartInternal()); if (!comm->symmetricSupport) { WARN("Communicator does not support symmetric memory!"); ret = ncclInvalidUsage; goto fail; } NCCLCHECKGOTO(ncclCommEnsureReady(comm), ret, fail); CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), ret, fail); NCCLCHECKGOTO(ncclDevrInitOnce(comm), ret, fail); NCCLCHECKGOTO(ncclCalloc(&task, 1), ret, fail); // reqs must be deep copied to the task so background threads can safely access it NCCLCHECKGOTO(deepCopyDevCommRequirements(reqs, &task->reqs), ret, fail); task->outDevComm = outDevComm; task->outDevCommCopyCB = outDevCommCopyCB; ncclIntruQueueEnqueue(&comm->devrState.commCreateTaskQueue, task); ncclGroupCommJoin(comm, ncclGroupTaskTypeSymRegister); exit: ncclGroupErrCheck(ret); NCCLCHECK(ncclGroupEndInternal()); cudaSetDevice(saveDev); return ret; fail: free(task); goto exit; } static ncclResult_t ncclDevCommCreate_v22902( ncclComm_t comm, struct ncclDevCommRequirements_v22902 const* reqs, struct ncclDevComm_v22902* outDevComm ) { ncclDevCommRequirements_t newReqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER; bool userRequestedGin = reqs->ginForceEnable || reqs->ginSignalCount > 0 || reqs->ginCounterCount > 0; { struct ncclDevResourceRequirements* rr = reqs->resourceRequirementsList; while (!userRequestedGin && rr != nullptr) { userRequestedGin = rr->ginSignalCount > 0 || rr->ginCounterCount > 0; rr = rr->next; } } if (userRequestedGin) { int runtimeVersion; NCCLCHECK(ncclGetVersion(&runtimeVersion)); WARN("The application was compiled with too old version of NCCL. It was compiled with NCCL version %d, but is running with NCCL library version %d. Because of its use of GIN device kernels, it needs to be recompiled, preferably with the same NCCL version that it will be running with.", reqs->version, runtimeVersion); return ncclInvalidUsage; } newReqs.resourceRequirementsList = reqs->resourceRequirementsList; newReqs.teamRequirementsList = reqs->teamRequirementsList; newReqs.lsaMultimem = reqs->lsaMultimem; // Prior to 2.29.4, a non-zero barrierCount did not imply GIN, but it does since, so we can't just copy it over. newReqs.lsaBarrierCount = std::max(reqs->lsaBarrierCount, reqs->barrierCount); newReqs.lsaLLA2ABlockCount = reqs->lsaLLA2ABlockCount; newReqs.lsaLLA2ASlotCount = reqs->lsaLLA2ASlotCount; // No need to copy GIN-specific fields since we established above that it's not being requested. memset(outDevComm, '\0', sizeof(*outDevComm)); NCCLCHECK(ncclDevCommCreateCommon(comm, &newReqs, (struct ncclDevComm*)outDevComm, ncclDevCommCreateCopyCB_v22902)); return ncclSuccess; } NCCL_API(ncclResult_t, ncclDevCommCreate, ncclComm_t comm, ncclDevCommRequirements_t const* reqs, ncclDevComm_t* outDevComm); ncclResult_t ncclDevCommCreate( ncclComm_t comm, struct ncclDevCommRequirements const* reqs, struct ncclDevComm* outDevComm ) { NCCLCHECK(CommCheck(comm, __func__, "comm")); NCCLCHECK(PtrCheck(reqs, __func__, "reqs")); if (reqs->magic != NCCL_API_MAGIC) { WARN("Cannot create device communicator: ncclDevCommRequirements_t argument must be initialized via NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER"); return ncclInvalidUsage; } NCCLCHECK(validateNcclVersion(reqs->version)); if (reqs->version >= NCCL_VERSION(2, 29, 2) && reqs->version <= NCCL_VERSION(2, 29, 3)) { NCCLCHECK(ncclDevCommCreate_v22902(comm, (const struct ncclDevCommRequirements_v22902*)reqs, (struct ncclDevComm_v22902*)outDevComm)); return ncclSuccess; } return ncclDevCommCreateCommon(comm, reqs, outDevComm, /*outDevCommCopyCB=*/nullptr); } NCCL_API(ncclResult_t, ncclDevCommDestroy, ncclComm_t comm, ncclDevComm_t const* devComm); ncclResult_t ncclDevCommDestroy( struct ncclComm* comm, struct ncclDevComm const* devComm ) { NCCLCHECK(CommCheck(comm, __func__, "comm")); NCCLCHECK(PtrCheck(devComm, __func__, "devComm")); struct ncclDevrState* devr = &comm->devrState; if (devr->ginEnabled) { NCCLCHECK(ncclGinResetSignalsAndCounters(comm, devComm)); ncclGinFreeSignalsCounters(comm, devComm->ginSignalBase, devComm->ginSignalCount, devComm->ginCounterBase, devComm->ginCounterCount ); if (devComm->ginContextBase == comm->sharedRes->ginState.ctxLastExclusive) { // Since we don't track the shared/exclusive state of each context individually, we can't support the general // case of release. However, we support the release of contexts of the most recently created exclusive devComm, // as it doesn't require any additional tracking. comm->sharedRes->ginState.ctxLastExclusive += devComm->ginContextCount; } } if (devComm->resourceWindow != nullptr) { NCCLCHECK(ncclCommWindowDeregister(comm, devComm->resourceWindow)); } return ncclSuccess; } NCCL_API(ncclResult_t, ncclWinGetUserPtr, ncclComm_t comm, ncclWindow_t win, void** outUserPtr); ncclResult_t ncclWinGetUserPtr(struct ncclComm* comm, struct ncclWindow_vidmem* win, void** outUserPtr) { NCCLCHECK(CommCheck(comm, __func__, "comm")); NCCLCHECK(PtrCheck(win, __func__, "win")); NCCLCHECK(PtrCheck(outUserPtr, __func__, "outUserPtr")); if (!comm->symmetricSupport) { INFO(NCCL_INIT, "Symmetric registration is not supported in this communicator."); *outUserPtr = nullptr; return ncclSuccess; } struct ncclDevrWindow* winHost = NULL; struct ncclWindow_vidmem* winDevHost = NULL; NCCLCHECK(ncclShadowPoolToHost(&comm->devrState.shadows, win, &winDevHost)); winHost = (struct ncclDevrWindow*)winDevHost->winHost; if (winHost == nullptr) { WARN("window has a NULL user pointer"); return ncclInternalError; } *outUserPtr = winHost->userPtr; return ncclSuccess; } // Get the corresponding pointer in another lsa rank's symmetric memory window ncclResult_t ncclDevrGetLsaRankPtr(struct ncclComm* comm, struct ncclDevrWindow* winHost, size_t offset, int lsaRank, void** outPtr) { NCCLCHECK(CommCheck(comm, __func__, "comm")); NCCLCHECK(PtrCheck(outPtr, __func__, "outPtr")); struct ncclDevrState* devr = &comm->devrState; // Validate lsaRank is within bounds if (lsaRank < 0 || lsaRank >= devr->lsaSize) { return ncclInvalidArgument; } // Validate offset is within bounds if (offset < 0 || offset >= winHost->size) { return ncclInvalidArgument; } // Calculate the address with offset for the specified lsa rank *outPtr = (void*)((uintptr_t)devr->lsaFlatBase + lsaRank * devr->bigSize + winHost->bigOffset + offset); return ncclSuccess; } // Get the RMA device window handle for a specific context ncclGinWindow_t ncclDevrGetRmaDevWin(struct ncclDevrWindow* winHost, int ctx) { if (winHost == nullptr || winHost->memory == nullptr) { return nullptr; } if (ctx < 0 || ctx >= NCCL_GIN_MAX_CONNECTIONS) { return nullptr; } return winHost->memory->rmaDevWins[ctx]; } // Get the multicast address for a given team ncclResult_t ncclDevrGetLsaTeamPtrMC(struct ncclComm* comm, struct ncclDevrWindow* winHost, size_t offset, struct ncclTeam lsaTeam, void** outPtr){ if (winHost == nullptr || outPtr == nullptr) return ncclInternalError; if (!comm->nvlsSupport) { WARN("Multimem pointer requested but system does not support multimem."); return ncclInvalidUsage; } bool multimem = true; struct ncclDevrTeam* tm; NCCLCHECK(symTeamObtain(comm, lsaTeam, multimem, &tm)); // Return the base multicast address for this team with offset *outPtr = (void*)((uintptr_t)tm->mcBasePtr + winHost->bigOffset + offset); return ncclSuccess; } static ncclResult_t findCommAndHostWindowFromDeviceWindow(ncclWindow_t devWindow, ncclComm_t* foundComm, ncclDevrWindow** hostWindow) { struct ncclDevrWindow* winHost = nullptr; std::lock_guard lock(ncclWindowMapMutex); NCCLCHECK(ncclIntruAddressMapFind(&ncclWindowMap, devWindow, &winHost)); if (winHost == nullptr) { WARN("Could not find communicator matching window %p (map hbits=%d count=%d)", devWindow, ncclWindowMap.base.hbits, ncclWindowMap.base.count); return ncclInvalidArgument; } *foundComm = winHost->comm; *hostWindow = winHost; return ncclSuccess; } NCCL_API(ncclResult_t, ncclGetMultimemDevicePointer, ncclWindow_t window, size_t offset, ncclMultimemHandle multimem, void** outPtr); ncclResult_t ncclGetMultimemDevicePointer (ncclWindow_t window, size_t offset, ncclMultimemHandle multimem, void** outPtr) { NCCLCHECK(PtrCheck(window, __func__, "window")); NCCLCHECK(PtrCheck(outPtr, __func__, "outPtr")); if (multimem.mcBasePtr == nullptr) { WARN("MCBasePtr %p needs to be valid.", multimem.mcBasePtr); return ncclInvalidArgument; } ncclComm_t comm = nullptr; struct ncclDevrWindow* winHost = nullptr; NCCLCHECK(findCommAndHostWindowFromDeviceWindow(window, &comm, &winHost)); if (!comm->nvlsSupport) { *outPtr = nullptr; return ncclSuccess; } *outPtr = (void*)((uintptr_t)multimem.mcBasePtr + winHost->bigOffset + offset); return ncclSuccess; } NCCL_API(ncclResult_t, ncclGetLsaMultimemDevicePointer, ncclWindow_t window, size_t offset, void** outPtr); ncclResult_t ncclGetLsaMultimemDevicePointer(ncclWindow_t window, size_t offset, void** outPtr) { NCCLCHECK(PtrCheck(window, __func__, "window")); NCCLCHECK(PtrCheck(outPtr, __func__, "outPtr")); ncclComm_t comm = nullptr; struct ncclDevrWindow* winHost = nullptr; NCCLCHECK(findCommAndHostWindowFromDeviceWindow(window, &comm, &winHost)); if (comm->nvlsSupport == 0) { *outPtr = nullptr; return ncclSuccess; } NCCLCHECK(ncclDevrGetLsaTeamPtrMC(comm, winHost, offset, ncclTeamLsa(comm), outPtr)); return ncclSuccess; } NCCL_API(ncclResult_t, ncclGetLsaDevicePointer, ncclWindow_t window, size_t offset, int lsaRank, void** outPtr); ncclResult_t ncclGetLsaDevicePointer(ncclWindow_t window, size_t offset, int lsaRank, void** outPtr) { NCCLCHECK(PtrCheck(window, __func__, "window")); NCCLCHECK(PtrCheck(outPtr, __func__, "outPtr")); ncclComm_t comm = nullptr; struct ncclDevrState* devr; struct ncclDevrWindow* winHost = nullptr; // Get the host version of the device window NCCLCHECK(findCommAndHostWindowFromDeviceWindow(window, &comm, &winHost)); devr = &comm->devrState; if (lsaRank < 0 || lsaRank >= devr->lsaSize) { WARN("The provided lsaRank %d is not in the valid lsaSize of [0,%d] for the provided window %p.", lsaRank, devr->lsaSize, window); return ncclInvalidArgument; // In this case the user should know what the lsa size is. } NCCLCHECK(ncclDevrGetLsaRankPtr(comm, winHost, offset, lsaRank, outPtr)); return ncclSuccess; } NCCL_API(ncclResult_t, ncclGetPeerDevicePointer, ncclWindow_t window, size_t offset, int peer, void** outPtr); ncclResult_t ncclGetPeerDevicePointer(ncclWindow_t window, size_t offset, int peer, void** outPtr) { NCCLCHECK(PtrCheck(window, __func__, "window")); NCCLCHECK(PtrCheck(outPtr, __func__, "outPtr")); ncclComm_t comm = nullptr; struct ncclDevrState* devr; struct ncclDevrWindow* winHost = nullptr; int lsaRank; ncclTeam_t worldTeam; ncclTeam_t lsaTeam; // Get the host version of the device window NCCLCHECK(findCommAndHostWindowFromDeviceWindow(window, &comm, &winHost)); // Validate peer rank is within bounds if (peer < 0 || peer >= comm->nRanks) { WARN("peer %d is not within valid range of ranks %d.", peer, comm->nRanks); return ncclInvalidArgument; } devr = &comm->devrState; worldTeam = ncclTeamWorld(comm); lsaTeam = ncclTeamLsa(comm); // Convert world rank to LSA team rank lsaRank = ncclTeamRankToTeam(lsaTeam, worldTeam, peer); // Validate the converted LSA rank is within bounds if (lsaRank < 0 || lsaRank >= devr->lsaSize) { // We return a nullptr if peer is not reachable. Same as device side *outPtr = nullptr; return ncclSuccess; } NCCLCHECK(ncclDevrGetLsaRankPtr(comm, winHost, offset, lsaRank, outPtr)); return ncclSuccess; } //////////////////////////////////////////////////////////////////////////////// // Find the least index strictly greater than arg. template static int listFindSortedLub(Key Obj::*key, Obj* sorted, int count, Key arg) { int lo = 0, hi = count; while (lo + 16 < hi) { int i = (lo + hi)/2; if (sorted[i].*key <= arg) lo = i+1; else hi = i; } int i = lo; while (i < hi && sorted[i].*key <= arg) i++; return i; } template static void listInsert(Obj** list, int* capacity, int* count, int index, Obj val) { if (*capacity < *count + 1) { *capacity *= 2; if (*capacity == 0) *capacity = 16; *list = (Obj*)realloc(*list, (*capacity)*sizeof(Obj)); } for (int j = *count; j != index; j--) { (*list)[j] = (*list)[j-1]; } (*list)[index] = val; *count += 1; } template static void listRemove(Obj* list, int* count, int index) { for (int i = index; i+1 < *count; i++) { list[i] = list[i+1]; } *count -= 1; } nccl-2.29.7-1/src/device/000077500000000000000000000000001515037102200147125ustar00rootroot00000000000000nccl-2.29.7-1/src/device/CMakeLists.txt000066400000000000000000000043501515037102200174540ustar00rootroot00000000000000# Run the scripts once during configuration to get the file lists execute_process( COMMAND ${CMAKE_COMMAND} -E env NCCL_USE_CMAKE=1 ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate.py ${CMAKE_CURRENT_BINARY_DIR}/gensrc "${ONLY_FUNCS}" OUTPUT_VARIABLE files WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) string(STRIP "${files}" files) list(TRANSFORM files PREPEND ${CMAKE_CURRENT_BINARY_DIR}/gensrc/) execute_process( COMMAND ${CMAKE_COMMAND} -E env NCCL_USE_CMAKE=1 ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/symmetric/generate.py ${CMAKE_CURRENT_BINARY_DIR}/gensrc/symmetric "${ONLY_FUNCS}" OUTPUT_VARIABLE symmetric_files WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) string(STRIP "${symmetric_files}" symmetric_files) list(TRANSFORM symmetric_files PREPEND ${CMAKE_CURRENT_BINARY_DIR}/gensrc/symmetric/) # Create custom commands to generate source files with proper dependencies add_custom_command( OUTPUT ${files} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate.py ${CMAKE_CURRENT_BINARY_DIR}/gensrc "${ONLY_FUNCS}" DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/generate.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Generating device source files" ) add_custom_command( OUTPUT ${symmetric_files} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/symmetric/generate.py ${CMAKE_CURRENT_BINARY_DIR}/gensrc/symmetric "${ONLY_FUNCS}" DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/symmetric/generate.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Generating symmetric device source files" ) # Add library target add_library(nccl_device OBJECT ${files} ${symmetric_files} ${CMAKE_CURRENT_SOURCE_DIR}/common.cu ${CMAKE_CURRENT_SOURCE_DIR}/onerank.cu ) set_target_properties(nccl_device PROPERTIES CUDA_SEPARABLE_COMPILATION ON CUDA_RESOLVE_DEVICE_SYMBOLS ON ) # Set include directories for the target target_include_directories(nccl_device PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_BINARY_DIR}/include ${CMAKE_SOURCE_DIR}/src/include ${CMAKE_SOURCE_DIR}/src/include/plugin ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_INCLUDE_DIRS}/cccl ) add_dependencies(nccl_device nccl_header) nccl-2.29.7-1/src/device/Makefile000066400000000000000000000101151515037102200163500ustar00rootroot00000000000000# # SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information # SHELL := /usr/bin/env bash MAKEFLAGS += -r .SUFFIXES: .SECONDARY: NCCLDIR := ../.. include $(NCCLDIR)/makefiles/common.mk include $(NCCLDIR)/makefiles/version.mk BUILDDIR ?= $(abspath ../../build) OBJDIR := $(BUILDDIR)/obj/device MANIFEST := $(OBJDIR)/manifest DEVGLUE_OBJ := $(OBJDIR)/device_glue.o INCFLAGS = -I. -I.. -I$(BUILDDIR)/include -I../include -I../include/plugin NVCUFLAGS += $(INCFLAGS) --compiler-options "-fPIC -fvisibility=hidden" CXXFLAGS += $(INCFLAGS) NVCUFLAGS_SYM += -ccbin $(CXX) $(CXXSTD) --expt-extended-lambda -Xptxas -maxrregcount=128 -Xfatbin -compress-all NVCUFLAGS_SYM += $(INCFLAGS) --compiler-options "-fPIC -fvisibility=hidden" #NVCUFLAGS_SYM += --ptx SAY = @bash -c 'path="$$2"; [[ "$$(realpath "$$2")" =~ ^$(subst .,\.,$(abspath $(NCCLDIR)))/(.*)$$ ]] && path="$${BASH_REMATCH[1]}"; printf "%-15s %s\n" "$$1" "$$path"' SAY COMPILE.cu = $(NVCC) $(NVCUFLAGS) -dc $2 -o $1 COMPILE.kernel = $(NVCC) $(NVCUFLAGS) -dw $2 -o $1 COMPILE.cc = $(CXX) $(CXXFLAGS) -c $2 -o $1 define COMPILE @$(SAY) "Compiling" $2;\ mkdir -p $(dir $1);\ $(call COMPILE$(or $3,$(suffix $2)),$1,$2) endef ifeq ($(shell echo "$$((1000*$(CUDA_MAJOR) + 10*$(CUDA_MINOR) >= 12090))"),1) NVCC_GENCODE_LDMC_FP8 = -gencode=arch=compute_100f,code=sm_100f else ifeq ($(shell echo "$$((1000*$(CUDA_MAJOR) + 10*$(CUDA_MINOR) >= 12070))"),1) NVCC_GENCODE_LDMC_FP8 = -gencode=arch=compute_100a,code=sm_100a else NVCC_GENCODE_LDMC_FP8 = endif define COMPILE_SYM @$(SAY) "Compiling" $2;\ mkdir -p $(dir $1);\ if [[ -n "$3" ]]; then\ $(NVCC) $(NVCUFLAGS_SYM) $3 -dw $2 -o $1;\ else\ touch $2.empty.cu; $(NVCC) $(NVCUFLAGS_SYM) -dw $2.empty.cu -o $1; rm $2.empty.cu;\ fi endef DEPENDS.cu = $(NVCC) $(NVCUFLAGS) -M -dc $1 DEPENDS.cc = $(CXX) $(CXXFLAGS) -M -c $1 define DEPENDS @$(SAY) "Dependencies" $2;\ mkdir -p $(dir $1);\ mk=$$($(call DEPENDS$(suffix $2),$2));\ [[ $$mk =~ ^[^:]*:(.*)$$ ]];\ files=$${BASH_REMATCH[1]};\ files=$$(for x in $$files; do case "$$x" in '\'|$$'\t') ;; *) echo "$$x"; esac; done);\ files=$$(for x in $$files; do [[ "$$(realpath "$$x")" == "$$(realpath "$(NCCLDIR)")"* ]] && echo "$$x"; done);\ echo "$(patsubst %.d,%.o,$1) $1: " $$files > $1 endef all: $(MANIFEST) $(OBJDIR)/gensrc: generate.py @mkdir -p $@ (which python3 >/dev/null || \ (bar='!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'; \ printf "\n$${bar}\nERROR: Building NCCL requires a Python 3 installation invokable as 'python3'.\n$${bar}\n\n" 1>&2; \ exit 1)) \ && ./generate.py $@ "$(ONLY_FUNCS)" $(OBJDIR)/gensrc/symmetric: $(OBJDIR)/gensrc symmetric/generate.py @mkdir -p $@ ./symmetric/generate.py $@ # The trailing ";" is necessary to make this an "empty recipe": # https://www.gnu.org/software/make/manual/html_node/Empty-Recipes.html $(OBJDIR)/gensrc/rules.mk: $(OBJDIR)/gensrc ; $(OBJDIR)/gensrc/symmetric/rules.mk: $(OBJDIR)/gensrc/symmetric ; -include $(OBJDIR)/gensrc/rules.mk # "gensrc/rules.mk" populates $(LIB_OBJS_GEN) -include $(OBJDIR)/gensrc/symmetric/rules.mk # "gensrc/symmetric/rules.mk" populates $(LIB_OBJS_SYM_GEN) SRCS = common.cu onerank.cu LIB_OBJS = $(patsubst %, $(OBJDIR)/%.o, $(SRCS)) $(LIB_OBJS_GEN) $(LIB_OBJS_SYM_GEN) $(OBJDIR)/%.o: % $(OBJDIR)/%.d $(call COMPILE,$@,$<) $(OBJDIR)/genobj/%.o: $(OBJDIR)/gensrc $(OBJDIR)/genobj/%.d $(call COMPILE,$@,$(OBJDIR)/gensrc/$*) $(OBJDIR)/genobj/symmetric/%.o: $(OBJDIR)/gensrc/symmetric $(OBJDIR)/genobj/symmetric/%.d $(call COMPILE,$@,$(OBJDIR)/gensrc/symmetric/$*) $(OBJDIR)/%.d: % $(call DEPENDS,$@,$<) $(OBJDIR)/genobj/%.d: $(OBJDIR)/gensrc/% $(call DEPENDS,$@,$<) $(OBJDIR)/genobj/symmetric/%.d: $(OBJDIR)/gensrc/symmetric/% $(call DEPENDS,$@,$<) $(DEVGLUE_OBJ): $(LIB_OBJS) $(NVCC) $(NVCUFLAGS) -dlink $^ -o $@ $(MANIFEST): $(LIB_OBJS) $(DEVGLUE_OBJ) @echo $^ > $@ -include $(wildcard $(OBJDIR)/*.d) -include $(wildcard $(OBJDIR)/genobj/*.d) -include $(wildcard $(OBJDIR)/genobj/symmetric/*.d) .PHONY: clean clean: rm -rf $(OBJDIR) nccl-2.29.7-1/src/device/all_gather.h000066400000000000000000000636551515037102200172040ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "device.h" #include "collectives.h" #include "primitives.h" namespace { template __device__ __forceinline__ void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { ncclRing *ring = &ncclShmem.channel.ring; const int *ringRanks = ring->userRanks; const int nranks = ncclShmem.comm.nRanks; ssize_t count, partOffset, partCount, chunkCount; ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), &count, &partOffset, &partCount, &chunkCount); ssize_t offset; ssize_t dataOffset; int nelem; int rankDest; int workNthreads; T *inputBuf = (T*)work->sendbuff; T *outputBuf = (T*)work->recvbuff; // If isNetOffload == true, we only use 1 warp to drive Ring algo/network communication // and the rest of warps proceed to copy src data into dst buffer in parallel when AG // is not in-place. if (isNetOffload) { workNthreads = WARP_SIZE; chunkCount = NCCL_MAX_NET_SIZE; } else { workNthreads = nthreads; } if (tid < workNthreads) { // Coverity reports that the callee treats &ring->next as an array. However, due to the use of // FanSymmetric<1>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, 1, Proto, 0, isNetOffload> prims (tid, workNthreads, &ring->prev, &ring->next, inputBuf, outputBuf, work->redOpArg, 0, 0, 0, work, NULL, isNetOffload ? NCCL_MAX_NET_SIZE : 0); for (size_t elemOffset = 0; elemOffset < partCount; elemOffset += chunkCount) { /////////////// begin AllGather steps /////////////// nelem = min(chunkCount, partCount - elemOffset); dataOffset = partOffset + elemOffset; // step 0: push data to next GPU rankDest = ringRanks[0]; offset = dataOffset + rankDest * count; if ((inputBuf + dataOffset == outputBuf + offset) || isNetOffload) { // In place or onePPN prims.directSend(dataOffset, offset, nelem); } else { prims.directCopySend(dataOffset, offset, nelem); } // k-2 steps: copy to next GPU for (int j = 1; j < nranks - 1; ++j) { rankDest = ringRanks[nranks - j]; offset = dataOffset + rankDest * count; prims.directRecvCopyDirectSend(offset, offset, nelem); } // Make final copy from buffer to dest. rankDest = ringRanks[1]; offset = dataOffset + rankDest * count; // Final wait/copy. prims.directRecv(offset, nelem); } } else if (inputBuf != outputBuf + ringRanks[0] * count) { inputBuf = inputBuf + partOffset; outputBuf = outputBuf + partOffset + ringRanks[0] * count; reduceCopy (tid - workNthreads, nthreads - workNthreads, work->redOpArg, false, 1, (void**)&inputBuf, 1, (void**)&outputBuf, partCount); } // we have to wait for all warps before we can proceed to the next work; // otherwise, we can have contention if next work will use the outputBuf // in this work. We use bar 14 to avoid conflicts with prims barrier and // __syncthread(). if (isNetOffload) barrier_sync(14, nthreads); } } template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { bool isNetOffload = work->isOneRPN && work->netRegUsed; if (isNetOffload) runRing, true>(tid, nthreads, work); else runRing, false>(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { #if __CUDA_ARCH__ >= 600 using Proto = ProtoSimple<1, 1>; const int nranks = ncclShmem.comm.nRanks; const int rank = ncclShmem.comm.rank; size_t count, channelOffset, channelCount, chunkCount; ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), &count, &channelOffset, &channelCount, &chunkCount); static constexpr int nworkers = NCCL_PAT_NWORKERS; struct ncclPatShmem* shmem = (struct ncclPatShmem*)ncclScratchForWarp(0); uint64_t pollCount = 0; __syncthreads(); // Don't start using shared mem until everyone arrives for (int i=tid; ipatSteps[i].flags = 0; if (tid == 0) shmem->localAccSize = 0; if (tid == nworkers) shmem->parallelFactor = 0; __syncthreads(); if (tid == nworkers) { // Algo computation thread PatAGAlgorithm patAlgo(chunkCount*sizeof(T), NCCL_STEPS, NCCL_PAT_NWORKERS/WARP_SIZE, channelOffset, channelOffset + channelCount, count, chunkCount, rank, nranks); int parallelFactor = shmem->parallelFactor = patAlgo.getParallelFactor(); int step = 0; while (1) { struct ncclPatStep* ps = shmem->patSteps+(step%NCCL_SHMEM_PAT_STEPS); cuda::atomic_ref poll(ps->flags); while (poll.load(cuda::memory_order_acquire) != 0) pollCount++; // Wait for workers to be done with step 'step-NCCL_SHMEM_PAT_STEPS' patAlgo.getNextOp(ps); int last = ps->last; step++; if (last == 2) break; } } else if (tid < nworkers) { // Worker threads T *inputBuf = (T*)work->sendbuff; T *outputBuf = (T*)work->recvbuff; int parallelFactor = 0; volatile int* pfPtr = &shmem->parallelFactor; while (parallelFactor == 0) parallelFactor = *pfPtr; int groupSize = nworkers/(WARP_SIZE*parallelFactor) * WARP_SIZE; int group = tid / groupSize; int nGroups = nworkers / groupSize; int tidInGroup = tid - group*groupSize; // We don't use recvPeers/sendPeers so let's pass shmem structs instead Primitives, 0, Proto, 0> prims (tidInGroup, groupSize, (int*)shmem->recvDims, (int*)shmem->sendDims, inputBuf, outputBuf, work->redOpArg, group, 0, 0, nullptr, nullptr, 0, primsModePatAg); int step = group; while(1) { struct ncclPatStep* ps = shmem->patSteps+(step%NCCL_SHMEM_PAT_STEPS); cuda::atomic_ref poll(ps->flags); while (poll.load(cuda::memory_order_acquire) == 0) pollCount++; // Wait for compute thread int last = ps->last; prims.patCopy(ps, shmem); if (tidInGroup == 0) poll.store(0, cuda::memory_order_release); // Return element to compute thread if (last) break; step += nGroups; } } #endif } }; template struct RunWorkColl { template struct Scatterer { struct ncclDevWorkColl* work; ssize_t chunkSize; ssize_t railGridOffset; template __device__ __forceinline__ void operator()( int tid, int tn, int slice, int maxSliceSize, int nSrcs, void** srcPtrs, int nDsts, void** dstPtrs, int32_t* dstSizes, uint32_t sendDirectFlag, uint32_t recvDirectFlag ) { static_assert(SlicePerChunk==1, "require: SlicePerChunk==1"); static_assert(MaxDsts<=1 || MaxSrcs<=1, "require: MaxDsts<=1 || MaxSrcs<=1"); struct ncclNvls* nvls = &ncclShmem.channel.nvls; int nNodes = ncclShmem.comm.nNodes; int nRails = nvls->nHeads; int part = ncclShmem.channelId - work->channelLo; char* inbuf = (char*)work->sendbuff; char* outbuf = (char*)work->recvbuff; ssize_t countPerRank = work->collnet.count; bool inPlace = (inbuf == outbuf + ncclShmem.comm.rank * countPerRank); ssize_t railAllBeg = min(railGridOffset + part * chunkSize, nNodes * countPerRank); ssize_t railAllEnd = min(railAllBeg + chunkSize, nNodes * countPerRank); int railAllSize = railAllEnd - railAllBeg; int rail = 0; int src = 0; if (BcastSendNotRecv) { rail = nvls->headRank; } else { if (work->regUsed) return; rail = 0; } if (tid < nDsts) dstSizes[tid] = railAllSize; do { int node = railAllBeg / countPerRank; int railAllOffset = 0; while (railAllOffset < railAllSize) { ssize_t railOneBeg = node * countPerRank; ssize_t railOneEnd = railOneBeg + countPerRank; ssize_t railOneOffset = (railAllBeg + railAllOffset) - railOneBeg; int delta = min(railAllEnd, railOneEnd) - (railAllBeg + railAllOffset); int rank = ncclShmem.comm.collNetDenseToUserRank[node * nRails + rail]; ssize_t userOneBeg = rank * countPerRank + railOneOffset; int outIsDst = (inPlace && rank == ncclShmem.comm.rank) || BcastSendNotRecv || work->regUsed ? 0 : 1; if (nSrcs != 0 && outIsDst + nDsts != 0) { reduceCopy (tid, tn, 0, false, /*nSrcs=*/1, [=]__device__(int s/*==0*/) -> void* { return (char*)srcPtrs[src] + railAllOffset; }, /*nDsts=*/outIsDst + nDsts, [=]__device__(int d) -> void* { return d < outIsDst ? outbuf + userOneBeg : work->regUsed ? (char*)dstPtrs[d - outIsDst] + userOneBeg : (char*)dstPtrs[d - outIsDst] + railAllOffset; }, delta); } railAllOffset += delta; node += 1; } rail += 1; src += 1; } while (!BcastSendNotRecv && src < nRails); } }; __device__ __forceinline__ void run(int tid, int/*nthreads*/, struct ncclDevWorkColl* work) { struct ncclNvls* nvls = &ncclShmem.channel.nvls; int nelem; const int nThreadsNetSend = work->oneNode ? 0 : (work->netRegUsed ? WARP_SIZE : 6 * WARP_SIZE); const int nThreadsGather = work->regUsed ? roundUp(nvls->nHeads << 2, WARP_SIZE) : 8 * WARP_SIZE; const int nThreadsBcast = NCCL_MAX_NTHREADS - nThreadsNetSend - nThreadsGather; const int tidEndGather = nThreadsGather; const int tidEndNetSend = tidEndGather + nThreadsNetSend; const int tidEndBcast = tidEndNetSend + nThreadsBcast; if (work->oneNode) { const ssize_t rank = ncclShmem.comm.rank; size_t count, gridOffset, channelCount, offset, chunkCount; ncclCollCbdPart(work, ncclShmem.channelId, NCCL_PROTO_SIMPLE, sizeof(T), &count, &gridOffset, &channelCount, &chunkCount); if (!work->regUsed) { if (tid < tidEndGather) { // Gather using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid, nThreadsGather, nvls->up, NULL, NULL, work->recvbuff, work->redOpArg, 0 * Proto::MaxGroupWidth, 1, 1); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.gather(offset, nvls->nHeads * count, nelem, count, -1, 0); } // coverity[overrun-call] => Coverity think prims.index can be greater than 1 } else if (tid < tidEndBcast) { // Bcast through NVLS using Proto = ProtoSimple<1, 1, COLL_UNROLL, 0, 1>; Primitives, /*Direct=*/0, Proto, 0> prims(tid - tidEndGather, nThreadsBcast, NULL, &nvls->down, work->sendbuff, NULL, work->redOpArg, 3 * Proto::MaxGroupWidth, 0, 0); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.send(offset, nelem); } // coverity[overrun-call] => Coverity think prims.index can be greater than 1 } } else { if (tid < tidEndGather) { using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid, nThreadsGather, nvls->up, nvls->up, NULL, NULL, work->redOpArg, 0 * Proto::MaxGroupWidth, 1, 1); /* used as sync */ prims.scatter(0, 0, 0, 0, -1, 0); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { prims.gather(0, 0, 0, 0, -1, 0); } } else if (tid < tidEndBcast) { using Proto = ProtoSimple<1, 1, COLL_UNROLL, 0, 1>; Primitives, /*Direct=*/1, Proto, 0> prims(tid - tidEndGather, nThreadsBcast, &nvls->down, &nvls->down, work->sendbuff, NULL, work->redOpArg, 1 * Proto::MaxGroupWidth, 0, 0, work); /* used as sync */ prims.recv(0, 0); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { ssize_t inpOffset = gridOffset + elemOffset; ssize_t outOffset = inpOffset + rank * count; nelem = min(chunkCount, channelCount - elemOffset); prims.directSend(inpOffset, outOffset, nelem); } } } } else { // NVLS + IB SHARP int nNodes = ncclShmem.comm.nNodes; int part = ncclShmem.channelId - work->channelLo; ssize_t countPerRank = work->collnet.count; const int nChannels = work->channelHi - work->channelLo + 1; ssize_t chunkCount = work->collnet.chunkCount; if (tid < tidEndGather) { using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/1, Proto, 0> prims(tid, nThreadsGather, nvls->up, nullptr, nullptr, work->recvbuff, /*redOpArg=*/0, 1 * Proto::MaxGroupWidth, 1, 1, work); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkCount) { Scatterer scat; scat.work = work; scat.chunkSize = chunkCount; scat.railGridOffset = railGridOffset; prims.template process(scat); } } else { if (work->netRegUsed) { using ProtoSend = ProtoSimple<1, 1, COLL_UNROLL>; using ProtoBcast = ProtoSimple<1, 1, COLL_UNROLL, 0, 1>; int maxSteps = (int)divUp(nNodes * countPerRank, nChannels * chunkCount); int curSteps = -1; int postThread = tid - tidEndGather == 0 ? 1 : 0; // for UB, we need to control the send speed to avoid net congestion. // first unroll 2 steps, then unroll the rest steps when the data is received. if (postThread) { curSteps = min(2, maxSteps); Primitives, /*Direct=*/1, ProtoSend, 0>::sendPeerNotify(nvls->out, 1, curSteps); } Primitives, /*Direct=*/1, ProtoBcast, 0> prims(tid - tidEndGather, nThreadsNetSend + nThreadsBcast, &nvls->out, &nvls->down, nullptr, nullptr, /*redOpArg=*/0, 2 * ProtoBcast::MaxGroupWidth, 0, 0, work); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkCount) { Scatterer scat; scat.work = work; scat.chunkSize = chunkCount; scat.railGridOffset = railGridOffset; prims.template process(scat); if (postThread && curSteps < maxSteps) { curSteps++; Primitives, /*Direct=*/1, ProtoSend, 0>::sendPeerNotify(nvls->out, 1, 1); } } } else { if (tid < tidEndNetSend) { using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid - tidEndGather, nThreadsNetSend, nullptr, &nvls->out, work->sendbuff, nullptr, /*redOpArg=*/0, 0 * Proto::MaxGroupWidth, 1, 1); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkCount) { ssize_t railAllBeg = railGridOffset + part * chunkCount; ssize_t railAllEnd = min(railAllBeg + chunkCount, nNodes * countPerRank); ssize_t railOneBeg = ncclShmem.comm.node * countPerRank; ssize_t railOneEnd = railOneBeg + countPerRank; ssize_t beg = max(railAllBeg, railOneBeg); ssize_t end = min(railAllEnd, railOneEnd); prims.send(beg - railOneBeg, max(ssize_t(0), end - beg)); } } else { using Proto = ProtoSimple<1, 1, COLL_UNROLL, 0, 1>; Primitives, /*Direct=*/0, Proto, 0> prims(tid - tidEndNetSend, nThreadsBcast, &nvls->out, &nvls->down, nullptr, nullptr, /*redOpArg=*/0, 2 * Proto::MaxGroupWidth, 0, 0); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkCount) { Scatterer scat; scat.work = work; scat.chunkSize = chunkCount; scat.railGridOffset = railGridOffset; prims.template process(scat); } } } } } } }; template struct RunWorkColl { template struct Scatterer { struct ncclDevWorkColl* work; ssize_t chunkSize; ssize_t railGridOffset; template __device__ __forceinline__ void operator()( int tid, int tn, int slice, int maxSliceSize, int nSrcs, void** srcPtrs, int nDsts, void** dstPtrs, int32_t* dstSizes, uint32_t sendDirectFlag, uint32_t recvDirectFlag ) { static_assert(SlicePerChunk==1, "require: SlicePerChunk==1"); static_assert(MaxDsts<=1 || MaxSrcs<=1, "require: MaxDsts<=1 || MaxSrcs<=1"); struct ncclDirect* direct = &ncclShmem.channel.collnetDirect; int nNodes = ncclShmem.comm.nNodes; int nRails = direct->nHeads; int part = ncclShmem.channelId - work->channelLo; char* inbuf = (char*)work->sendbuff; char* outbuf = (char*)work->recvbuff; ssize_t countPerRank = work->collnet.count*sizeof(T); bool inPlace = (inbuf == outbuf + ncclShmem.comm.rank*countPerRank); ssize_t railAllBeg = min(railGridOffset + part*chunkSize, nNodes*countPerRank); ssize_t railAllEnd = min(railAllBeg + chunkSize, nNodes*countPerRank); int railAllSize = railAllEnd - railAllBeg; if (tid < nDsts) dstSizes[tid] = railAllSize; int src = 0; int rail; if (BcastSendNotRecv) { rail = direct->headRank; } else { rail = direct->headRank+1; if (rail == nRails) rail = 0; } do { int node = railAllBeg/countPerRank; int railAllOffset = 0; while (railAllOffset < railAllSize) { ssize_t railOneBeg = node*countPerRank; ssize_t railOneEnd = railOneBeg + countPerRank; ssize_t railOneOffset = (railAllBeg+railAllOffset) - railOneBeg; int delta = min(railAllEnd, railOneEnd) - (railAllBeg+railAllOffset); int rank = ncclShmem.comm.collNetDenseToUserRank[node*nRails + rail]; ssize_t userOneBeg = rank*countPerRank + railOneOffset; int outIsDst = (inPlace && rank == ncclShmem.comm.rank) ? 0 : 1; if (nSrcs != 0 && outIsDst+nDsts != 0) { reduceCopy (tid, tn, 0, false, /*nSrcs=*/1, [=]__device__(int s/*==0*/) -> void* { return work->regUsed && (recvDirectFlag & NCCL_P2P_READ) ? (char*)srcPtrs[src] + userOneBeg : (char*)srcPtrs[src] + railAllOffset; }, /*nDsts=*/outIsDst+nDsts, [=]__device__(int d) -> void* { return d < outIsDst ? outbuf + userOneBeg : work->regUsed && (sendDirectFlag & NCCL_P2P_WRITE) ? (char*)dstPtrs[d-outIsDst] + userOneBeg : (char*)dstPtrs[d-outIsDst] + railAllOffset; }, delta); } railAllOffset += delta; node += 1; } src += 1; rail += 1; if (rail == nRails) rail = 0; } while (!BcastSendNotRecv && src < nRails-1); } }; __device__ __forceinline__ void run(int tid, int/*nthreads*/, struct ncclDevWorkColl* work) { const int part = ncclShmem.channelId - work->channelLo; const int nChannels = work->channelHi - work->channelLo + 1; struct ncclDirect* direct = &ncclShmem.channel.collnetDirect; int const &nNodes = ncclShmem.comm.nNodes; ssize_t countPerRank = work->collnet.count; size_t chunkSize = work->collnet.chunkCount; const int hasDn = (direct->down[0] >= 0) ? 1 : 0; bool isMultiRail = (direct->nHeads > 1); int nWarps1 = 1; int nWarps2 = (isMultiRail ? 2 : 1); int nWarps3 = (isMultiRail ? 2 : 0); float denom = float(work->nWarps)/float(nWarps1+nWarps2+nWarps3); nWarps3 = int(denom*nWarps3); nWarps2 = int(denom*nWarps2); nWarps1 = work->nWarps - (nWarps2+nWarps3); using Proto = ProtoSimple<1, 1>; int tn = nWarps1*WARP_SIZE; if (tid < tn) { if (work->netRegUsed) { if (tid == 0) { // If this rank has local peers (i.e, hasDn == true), we cannot offload all data to network. // In this case, steps should be computed based on chunkSize and so on; otherwise, we just // bump the step by 1 to kick off collnet progress. int steps = hasDn ? (int)divUp(nNodes * countPerRank, nChannels * chunkSize) : 1; Primitives, /*Direct=*/0, Proto, 0>::sendPeerNotify(direct->out, 1, steps); } __syncwarp(); } else { // Phase 1: send to network Primitives, /*Direct=*/0, Proto, 0> prims(tid, tn, nullptr, &direct->out, work->sendbuff, nullptr, /*redOpArg=*/0, 0 * Proto::MaxGroupWidth, 1, 1); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkSize) { ssize_t railAllBeg = railGridOffset + part * chunkSize; ssize_t railAllEnd = min(railAllBeg + chunkSize, nNodes * countPerRank); ssize_t railOneBeg = ncclShmem.comm.node * countPerRank; ssize_t railOneEnd = railOneBeg + countPerRank; ssize_t beg = max(railAllBeg, railOneBeg); ssize_t end = min(railAllEnd, railOneEnd); prims.send(beg - railOneBeg, max(ssize_t(0), end - beg)); } } return; } tid -= tn; tn = nWarps2*WARP_SIZE; if (tid < tn) { if (work->netRegUsed && !hasDn) { if (tid == 0) { Primitives, /*Direct=*/0, Proto, 0>::recvPeerNotify(direct->out, 0, 1); } __syncwarp(); } else { // Phase 2: Recv network -> deposit output + send to bcast Primitives, /*Direct=*/1, Proto, 0> prims(tid, tn, &direct->out, direct->heads + 1, nullptr, work->recvbuff, /*redOpArg=*/0, 1 * Proto::MaxGroupWidth, 0, 0, work); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkSize) { Scatterer scat; scat.work = work; scat.chunkSize = chunkSize; scat.railGridOffset = railGridOffset; prims.template process(scat, work->direct, 0); } } return; } tid -= tn; tn = nWarps3*WARP_SIZE; if (tid < tn) { // Phase 3: Recv bcast -> deposit output Primitives, /*Direct=*/1, Proto, 0> prims(tid, tn, direct->heads+1, nullptr, nullptr, work->recvbuff, /*redOpArg=*/0, 2*Proto::MaxGroupWidth, 0, 0, work); for (ssize_t railGridOffset=0; railGridOffset < nNodes*countPerRank; railGridOffset += nChannels*chunkSize) { Scatterer scat; scat.work = work; scat.chunkSize = chunkSize; scat.railGridOffset = railGridOffset; prims.template process(scat, 0, work->direct); } return; } } }; nccl-2.29.7-1/src/device/all_gather_v.h000066400000000000000000000066501515037102200175210ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "device.h" #include "collectives.h" #include "primitives.h" #include namespace { template __device__ __forceinline__ void setDataPtrsHelper(Primitives, 0, Proto, 0, 0>& prims, void const* srcBuf, void* dstBuf, uint64_t redOpArg) { prims.setDataPtrs(srcBuf, dstBuf); } template __device__ __forceinline__ void setDataPtrsHelper(Primitives, 0, ProtoSimple<1,1>, 0, 0>& prims, void const* srcBuf, void* dstBuf, uint64_t redOpArg) { prims.setDataPtrs(srcBuf, dstBuf, redOpArg, nullptr, 0, 0); } } template __device__ __forceinline__ void runAllGatherV() { int tid = threadIdx.x; int tn = blockDim.x; ncclRing* ring = &ncclShmem.channel.ring; ncclDevWorkBcast *works = (ncclDevWorkBcast*)ncclShmem.workStorage; Primitives, FanSymmetric<1>, /*Direct=*/0, Proto, 0> prims(tid, tn, &ring->prev, &ring->next, nullptr, nullptr, /*redOpArg=*/0); int w = 0; while (true) { int nWorks = ncclShmem.nWorks; int nRanks = ncclShmem.comm.nRanks; size_t bytes = works[w].bytes; int ringDepth = works[w].ringDepth; void* srcBuf = ringDepth==0 ? works[w].sendbuff : nullptr; void* dstBuf = works[w].recvbuff; bool inPlace = srcBuf == dstBuf; size_t offset = works[w].bytes_done; setDataPtrsHelper(prims, (void const*)srcBuf, (void *)dstBuf, 0); __syncthreads(); int wNext = (w+1 == nWorks) ? 0 : w+1; size_t chunkBytes = (size_t)works[w].chunkSize; size_t delta = min(bytes, chunkBytes); if (delta > 0) { if (ringDepth == 0) { if (inPlace) { prims.send(offset, delta); } else { prims.copySend(offset, offset, delta); } } else if (ringDepth == nRanks-1) { prims.recv(offset, delta); } else { prims.recvCopySend(offset, delta); } } if (tid == 0) { works[w].bytes -= delta; works[w].bytes_done += delta; } __syncthreads(); int nr_done = 0; for (int i = 0; i < ncclShmem.nWorks; i++) { if (works[i].bytes == 0) { nr_done += 1; } } if (nr_done == ncclShmem.nWorks) { break; } w = wNext; } } // Specialized for broadcast template struct RunWorkBatch { __device__ __forceinline__ void run() { using Proto = ProtoSimple<1,1>; runAllGatherV(); } }; template struct RunWorkBatch { __device__ __forceinline__ void run() { runAllGatherV(); } }; template struct RunWorkBatch { __device__ __forceinline__ void run() { runAllGatherV(); } }; nccl-2.29.7-1/src/device/all_reduce.h000066400000000000000000001155611515037102200171730ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "device.h" #include "collectives.h" #include "primitives.h" namespace { template __device__ __forceinline__ void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { ncclRing *ring = &ncclShmem.channel.ring; int ringIx = ring->index; const int nranks = ncclShmem.comm.nRanks; ssize_t gridOffset; ssize_t channelCount; ssize_t chunkCount; ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), (ssize_t*)nullptr, &gridOffset, &channelCount, &chunkCount); const ssize_t loopCount = nranks * chunkCount; ssize_t offset; int nelem; int chunk; // Coverity reports that the callee treats &ring->next as an array. However, due to the use of // FanSymmetric<1>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, 1, Proto, 0> prims (tid, nthreads, &ring->prev, &ring->next, work->sendbuff, work->recvbuff, work->redOpArg, 0, 0, 0, work); for (ssize_t elemOffset = 0; elemOffset < channelCount; elemOffset += loopCount) { ssize_t remCount = channelCount - elemOffset; ssize_t chunkOffset; if (remCount < loopCount) chunkCount = alignUp(divUp(remCount, nranks), 16/sizeof(T)); auto modRanks = [&]__device__(int r)->int { return r - (r >= nranks ? nranks : 0); }; // step 0: push data to next GPU chunk = modRanks(ringIx + nranks - 1); chunkOffset = chunk * chunkCount; offset = gridOffset + elemOffset + chunkOffset; nelem = (int)min(chunkCount, remCount - chunkOffset); prims.directSend(offset, offset, nelem); // k-2 steps: reduce and copy to next GPU for (int j = 2; j < nranks; ++j) { chunk = modRanks(ringIx + nranks - j); chunkOffset = chunk * chunkCount; offset = gridOffset + elemOffset + chunkOffset; nelem = (int)min(chunkCount, remCount - chunkOffset); prims.directRecvReduceDirectSend(offset, offset, nelem); } // step k-1: reduce this buffer and data, which will produce the final // result that we store in this data and push to the next GPU chunk = ringIx + 0; chunkOffset = chunk * chunkCount; offset = gridOffset + elemOffset + chunkOffset; nelem = (int)min(chunkCount, remCount - chunkOffset); prims.directRecvReduceCopyDirectSend(offset, offset, nelem, /*postOp=*/true); // k-2 steps: copy to next GPU for (int j = 1; j < nranks - 1; ++j) { chunk = modRanks(ringIx + nranks - j); chunkOffset = chunk * chunkCount; offset = gridOffset + elemOffset + chunkOffset; nelem = (int)min(chunkCount, remCount - chunkOffset); prims.directRecvCopyDirectSend(offset, offset, nelem); } // Make final copy from buffer to dest. chunk = modRanks(ringIx + 1); chunkOffset = chunk * chunkCount; offset = gridOffset + elemOffset + chunkOffset; nelem = (int)min(chunkCount, remCount - chunkOffset); prims.directRecv(offset, nelem); } } template __device__ __forceinline__ void runTreeUpDown(int tid, int nthreads, struct ncclDevWorkColl* work) { ncclTree *tree = &ncclShmem.channel.tree; size_t gridOffset; size_t channelCount; size_t chunkCount; ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), (size_t*)nullptr, &gridOffset, &channelCount, &chunkCount); size_t offset; int nelem; { // Reduce : max number of recv is 3, max number of send is 1 (binary tree + local) Primitives, /*Direct=*/1, Proto, 0> prims (tid, nthreads, tree->down, &tree->up, work->sendbuff, work->recvbuff, work->redOpArg, 0, 0, 0, work); if (tree->up == -1) { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directRecvReduceCopy(offset, offset, nelem, /*postOp=*/true); } } else if (tree->down[0] == -1) { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directSend(offset, offset, nelem); } } else { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directRecvReduceDirectSend(offset, offset, nelem); } } } { // Broadcast : max number of recv is 1, max number of send is 3 (binary tree + local) Primitives, /*Direct=*/1, Proto, 0> prims (tid, nthreads, &tree->up, tree->down, work->sendbuff, work->recvbuff, work->redOpArg, 0, 0, 0, work); if (tree->up == -1) { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directSendFromOutput(offset, nelem); } } else if (tree->down[0] == -1) { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directRecv(offset, nelem); } } else { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directRecvCopyDirectSend(offset, offset, nelem); } } } } template __device__ __forceinline__ void runTreeSplit(int tid, int nthreads, struct ncclDevWorkColl* work) { ncclTree *tree = &ncclShmem.channel.tree; size_t gridOffset; size_t channelCount; size_t chunkCount; ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), (size_t*)nullptr, &gridOffset, &channelCount, &chunkCount); size_t offset; int nelem; int nthreadsSplit; if (Proto::Id == NCCL_PROTO_SIMPLE) { nthreadsSplit = nthreads/2; if (nthreadsSplit >= 256) nthreadsSplit += 64; } else { // LL & LL128 // Receiving from up to 3 sources is more compute intensive than sending // to 3 dests. Use 70% for reduce and 30% for bcast. nthreadsSplit = (nthreads*7/(10*WARP_SIZE))*WARP_SIZE; } if (tree->up == -1) { // Reduce and broadcast. Max number of recv is 2, max number of send is 2 Primitives, /*Direct=*/1, Proto, 0> prims(tid, nthreads, tree->down, tree->down, work->sendbuff, work->recvbuff, work->redOpArg, 0, 0, 0, work); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directRecvReduceCopyDirectSend(offset, offset, nelem, /*doPost=*/true); } } else if (tid < nthreadsSplit) { /* Reduce up. Max number of recv is 3, max number of send is 1 (binary tree + local). * Why Direct=1???? * Answer: Because despite not performing any direct operations, the ctor * must assume Direct so that it can exchange direct pointers with remote ctors * that are Direct, otherwise it hangs. A cleaner solution would be to seperate * into DirectRecv and DirectSend capabilities, this ctor would have both=0, * but the ctor above for tree roots would be DirectRecv=0 DirectSend=1. */ // Coverity reports that the callee treats &tree->up as an array. However, due to the use of // FanAsymmetric, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, /*Direct=*/1, Proto, 0> prims(tid, nthreadsSplit, tree->down, &tree->up, work->sendbuff, work->recvbuff, work->redOpArg, 0*Proto::MaxGroupWidth, 0, 0, work); if (tree->down[0] == -1) { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directSend(offset, offset, nelem); } } else { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directRecvReduceDirectSend(offset, offset, nelem); } } } else { // Broadcast down. Max number of recv is 1, max number of send is 3 (binary tree + local) // Coverity reports that the callee treats &tree->up as an array. However, due to the use of // FanAsymmetric<1, n>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, /*Direct=*/1, Proto, 0> prims(tid-nthreadsSplit, nthreads-nthreadsSplit, &tree->up, tree->down, work->sendbuff, work->recvbuff, work->redOpArg, 1*Proto::MaxGroupWidth, 0, 0, work); if (tree->down[0] == -1) { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directRecv(offset, nelem); } } else { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.directRecvCopyDirectSend(offset, offset, nelem); } } } } } template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { using Proto = ProtoSimple; runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { #if CUDART_VERSION >= 11020 && CUDART_VERSION < 11040 && __CUDA_ARCH__ >= 800 runTreeUpDown>(tid, nthreads, work); #else runTreeSplit>(tid, nthreads, work); #endif } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int/*nthreads*/, struct ncclDevWorkColl* work) { static constexpr int COLLNET_COPY_THREADS = 96; const int bid = ncclShmem.channelId - work->channelLo; const int nChannels = work->channelHi - work->channelLo + 1; struct ncclDirect* direct = &ncclShmem.channel.collnetDirect; const ssize_t chunkSize = work->collnet.chunkCount; const ssize_t size = work->collnet.count; const ssize_t loopSize = nChannels*direct->nHeads*chunkSize; const int hasUp = (direct->up[0] >= 0) ? 1 : 0; const int hasDn = (direct->down[0] >= 0) ? 1 : 0; const int nThreadsScatter = WARP_SIZE + ((hasUp && hasDn) ? COLLNET_COPY_THREADS : hasUp ? 3*COLLNET_COPY_THREADS : 0); const int nThreadsGather = ((hasUp && hasDn) ? COLLNET_COPY_THREADS : hasUp ? 2*COLLNET_COPY_THREADS : 0); const int nThreadsBcast = WARP_SIZE + ((hasUp && hasDn) ? COLLNET_COPY_THREADS : hasUp ? 0 : 2*COLLNET_COPY_THREADS); const int nThreadsReduce = work->nWarps*WARP_SIZE - nThreadsScatter - nThreadsGather - nThreadsBcast; const int tidStartBcast = nThreadsGather; const int tidStartScatter = tidStartBcast + nThreadsBcast; const int tidStartReduce = tidStartScatter + nThreadsScatter; using Proto = ProtoSimple<1, 1>; if (tid >= tidStartScatter && tid < tidStartReduce && hasUp) { // Scatter Primitives, /*Direct=*/0, Proto, 0> prims(tid-tidStartScatter, nThreadsScatter, NULL, direct->up, work->sendbuff, work->recvbuff, work->redOpArg, 2*Proto::MaxGroupWidth, 1, 1, work); ssize_t offsetBase, peerOffset; ssize_t maxNelems; if (work->netRegUsed) { offsetBase = bid * chunkSize; maxNelems = size; // never be the min peerOffset = nChannels * chunkSize; } else { offsetBase = bid * direct->nHeads * chunkSize; maxNelems = direct->nHeads * chunkSize; peerOffset = chunkSize; } // For collnet UB case, we need to organize buffers differently for contiguous buffer access // across channels. This access pattern should be consistent with code in coll_net.cc for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + offsetBase; ssize_t nelem = min(maxNelems, size - offset); prims.scatter(offset, nelem, chunkSize, peerOffset, direct->headRank, direct->shift); } // Coverity complains about a possible overrun inside the destructor of "prims", but that's actually // a false positive. // coverity[overrun-call:FALSE] } else if (tid >= tidStartReduce && direct->out != -1) { if (hasDn) { // Reduce, send to network Primitives, /*Direct=*/0, Proto, 0> prims(tid-tidStartReduce, nThreadsReduce, direct->down, &direct->out, work->sendbuff, work->recvbuff, work->redOpArg, 3*Proto::MaxGroupWidth, 1, 1, work); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = work->netRegUsed ? gridOffset + (bid + direct->headRank * nChannels) * chunkSize : gridOffset + (bid * direct->nHeads + direct->headRank) * chunkSize; int nelem = min(chunkSize, size - offset); prims.recvReduceDirectSend(offset, offset, nelem); } } else { // Directly send to network if (work->netRegUsed) { if (tid == tidStartReduce) { Primitives, /*Direct=*/0, Proto, 0>::sendPeerNotify(direct->out, 1, 1); } __syncwarp(); } else { Primitives, /*Direct=*/0, Proto, 0> prims(tid-tidStartReduce, nThreadsReduce, nullptr, &direct->out, work->sendbuff, work->recvbuff, work->redOpArg, 3*Proto::MaxGroupWidth, 1, 1); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + (bid * direct->nHeads + direct->headRank) * chunkSize; int nelem = min(chunkSize, size - offset); prims.send(offset, nelem); } } } } else if (tid < tidStartBcast && hasUp) { // Gather Primitives, /*Direct=*/0, Proto, 0> prims(tid, nThreadsGather, direct->up, NULL, work->sendbuff, work->recvbuff, work->redOpArg, 0*Proto::MaxGroupWidth, 0, 0, work); ssize_t offsetBase, peerOffset; ssize_t maxNelems; if (work->netRegUsed) { offsetBase = bid * chunkSize; maxNelems = size; // never be the min peerOffset = nChannels * chunkSize; } else { offsetBase = bid * direct->nHeads * chunkSize; maxNelems = direct->nHeads * chunkSize; peerOffset = chunkSize; } for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + offsetBase; ssize_t nelem = min(maxNelems, size - offset); prims.directGather(offset, nelem, chunkSize, peerOffset, direct->headRank, direct->shift); } } else if (tid >= tidStartBcast && tid < tidStartScatter && direct->out != -1) { if (hasDn) { // Recv from network, broadcast // Coverity complains about a possible overrun inside the class below, but that's actually // a false positive. // coverity[identity_transfer:FALSE] Primitives, /*Direct=*/0, Proto, 0> prims(tid-tidStartBcast, nThreadsBcast, &direct->out, direct->down, work->sendbuff, work->recvbuff, work->redOpArg, 1*Proto::MaxGroupWidth, 0, 0, work); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = work->netRegUsed ? gridOffset + (bid + direct->headRank * nChannels) * chunkSize : gridOffset + (bid * direct->nHeads + direct->headRank) * chunkSize; int nelem = min(chunkSize, size - offset); prims.directRecvCopyDirectSend(offset, offset, nelem, /*postOp=*/true); } } else { if (work->netRegUsed) { if (tid == tidStartBcast) { Primitives, /*Direct=*/0, Proto, 0>::recvPeerNotify(direct->out, 0, 1); } __syncwarp(); } else { // Recv from network (no post thread needed) Primitives, /*Direct=*/0, Proto, 0> prims(tid - tidStartBcast, nThreadsBcast, &direct->out, nullptr, work->sendbuff, work->recvbuff, work->redOpArg, 1 * Proto::MaxGroupWidth, 0, 0); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + (bid * direct->nHeads + direct->headRank) * chunkSize; int nelem = min(chunkSize, size - offset); prims.recv(offset, nelem, /*postOp=*/true); } } } } } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int/*nthreads*/, struct ncclDevWorkColl* work) { struct ncclNvls* nvls = &ncclShmem.channel.nvls; const bool hasOut = nvls->out != -1; const int nranks = ncclShmem.comm.nRanks; const int totalWarps = NCCL_MAX_NTHREADS/WARP_SIZE; const int bcastWarps = hasOut ? (work->regUsed ? ((totalWarps - 2) >> 1) - 1 : 2) : 0; const int reduceWarps = work->regUsed ? (totalWarps - bcastWarps - 2) : (hasOut ? 3 : nranks <= 6 ? 7 : 5); const int scatterWarps = work->regUsed ? 1 : (totalWarps - reduceWarps - bcastWarps + 1) >> 1; const int gatherWarps = work->regUsed ? 1 : (totalWarps - reduceWarps - bcastWarps) >> 1; const int nThreadsScatter = scatterWarps*WARP_SIZE; const int nThreadsGather = gatherWarps*WARP_SIZE; const int nThreadsReduce = reduceWarps*WARP_SIZE; const int nThreadsBcast = (bcastWarps)*WARP_SIZE; const int tidEndScatter = nThreadsScatter; const int tidEndGather = tidEndScatter + nThreadsGather; const int tidEndReduce = tidEndGather + nThreadsReduce; const int tidEndBcast = tidEndReduce + nThreadsBcast; if (work->oneNode) { ssize_t gridOffset, channelCount, chunkSize; ncclCollCbdPart(work, ncclShmem.channelId, NCCL_PROTO_SIMPLE, sizeof(T), (ssize_t*)nullptr, &gridOffset, &channelCount, &chunkSize); const ssize_t loopCount = nvls->nHeads * chunkSize; int remCount = channelCount%(nvls->nHeads*chunkSize); int lastChunkSize = alignUp(divUp(remCount, nvls->nHeads), 16384/sizeof(T)); if (tid < tidEndScatter) { // Scatter using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid, nThreadsScatter, NULL, nvls->up, work->sendbuff, NULL, work->redOpArg, 0 * Proto::MaxGroupWidth, 1, 1); for (ssize_t elemOffset = 0; elemOffset < channelCount; elemOffset += loopCount) { if (channelCount - elemOffset < loopCount) chunkSize = lastChunkSize; ssize_t offset = gridOffset + elemOffset; int nelem = work->regUsed ? 0 : min(loopCount, channelCount - elemOffset); prims.scatter(offset, nelem, chunkSize, chunkSize, -1, 0); } } else if (tid < tidEndGather) { // Gather using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid - tidEndScatter, nThreadsGather, nvls->up, NULL, NULL, work->recvbuff, work->redOpArg, 1 * Proto::MaxGroupWidth, 1, 1); for (ssize_t elemOffset = 0; elemOffset < channelCount; elemOffset += loopCount) { if (channelCount - elemOffset < loopCount) chunkSize = lastChunkSize; ssize_t offset = gridOffset + elemOffset; int nelem = work->regUsed ? 0 : min(loopCount, channelCount - elemOffset); prims.gather(offset, nelem, chunkSize, chunkSize, -1, 0); } } else if (tid < tidEndReduce && nvls->headRank != -1) { // Reduce, broadcast through NVLS using Proto = ProtoSimple<1, 1, COLL_UNROLL, 1, 1>; Primitives, /*Direct=*/1, Proto, 0> prims(tid - tidEndGather, nThreadsReduce, &nvls->down, &nvls->down, NULL, NULL, work->redOpArg, 2 * Proto::MaxGroupWidth, 0, 0, work); for (ssize_t elemOffset = 0; elemOffset < channelCount; elemOffset += loopCount) { ssize_t chunkOffset, offset; int nelem; if (channelCount - elemOffset < loopCount) chunkSize = lastChunkSize; chunkOffset = elemOffset + nvls->headRank * chunkSize; offset = gridOffset + chunkOffset; nelem = min(chunkSize, channelCount - chunkOffset); prims.directRecvDirectSend(offset, offset, nelem); } } } else { const int bid = ncclShmem.channelId - work->channelLo; const int nChannels = work->channelHi - work->channelLo + 1; const ssize_t chunkSize = work->collnet.chunkCount; const ssize_t loopSize = nChannels * nvls->nHeads * chunkSize; const ssize_t size = work->collnet.count; if (tid < tidEndScatter) { // Scatter using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid, nThreadsScatter, NULL, nvls->up, work->sendbuff, NULL, work->redOpArg, 0 * Proto::MaxGroupWidth, 1, 1); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + bid * nvls->nHeads * chunkSize; int nelem = work->regUsed ? 0 : min(nvls->nHeads * chunkSize, size - offset); prims.scatter(offset, nelem, chunkSize, chunkSize, -1, 0); } // coverity[overrun-call] => Coverity think prims.index can be greater than 1 } else if (tid < tidEndGather) { // Gather using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid - tidEndScatter, nThreadsGather, nvls->up, NULL, NULL, work->recvbuff, work->redOpArg, 1 * Proto::MaxGroupWidth, 1, 1); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + bid * nvls->nHeads * chunkSize; int nelem = work->regUsed ? 0 : min(nvls->nHeads * chunkSize, size - offset); prims.gather(offset, nelem, chunkSize, chunkSize, -1, 0); } } else if (tid < tidEndReduce && nvls->headRank != -1) { // Reduce, send to network using Proto = ProtoSimple<1, 1, COLL_UNROLL, 1, 0>; // Coverity complains about a possible overrun inside the class below, but that's actually // a false positive. // coverity[identity_transfer:FALSE] Primitives, /*Direct=*/1, Proto, 0> prims(tid - tidEndGather, nThreadsReduce, &nvls->down, &nvls->out, NULL, work->recvbuff, work->redOpArg, 2 * Proto::MaxGroupWidth, 0, 1, work); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = work->regUsed && work->netRegUsed ? gridOffset + (nvls->headRank * nChannels + bid) * chunkSize : gridOffset + (bid * nvls->nHeads + nvls->headRank) * chunkSize; int nelem = min(chunkSize, size - offset); prims.directRecvDirectSend(offset, offset, nelem); } } else if (tid < tidEndBcast && nvls->headRank != -1) { // Recv from network, broadcast using Proto = ProtoSimple<1, 1, COLL_UNROLL, 0, 1>; // Coverity complains about a possible overrun inside the class below, but that's actually // a false positive. // coverity[identity_transfer:FALSE] Primitives, /*Direct=*/1, Proto, 0> prims(tid - tidEndReduce, nThreadsBcast, &nvls->out, &nvls->down, NULL, work->recvbuff, work->redOpArg, 3 * Proto::MaxGroupWidth, 0, 0, work); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = work->regUsed && work->netRegUsed ? gridOffset + (nvls->headRank * nChannels + bid) * chunkSize : gridOffset + (bid * nvls->nHeads + nvls->headRank) * chunkSize; int nelem = min(chunkSize, size - offset); prims.directRecvDirectSend(offset, offset, nelem); } } } } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int/*nthreads*/, struct ncclDevWorkColl* work) { struct ncclNvls* nvls = &ncclShmem.channel.nvls; const int treeUp = nvls->treeUp; const int* treeDown = nvls->treeDown; ssize_t gridOffset, channelCount, chunkCount; ncclCollCbdPart(work, ncclShmem.channelId, NCCL_PROTO_SIMPLE, sizeof(T), (ssize_t*)nullptr, &gridOffset, &channelCount, &chunkCount); const ssize_t loopCount = nvls->nHeads * chunkCount; const int nranks = ncclShmem.comm.nRanks; const bool hasUp = treeUp != -1; const int totalWarps = NCCL_MAX_NTHREADS/WARP_SIZE; const int bcastWarps = hasUp ? (work->regUsed ? ((totalWarps - 2) >> 1) - 1 : 4) : 0; const int reduceWarps = work->regUsed ? (totalWarps - bcastWarps - 2) : (hasUp ? 5 : nranks <= 6 ? 7 : 5); const int scatterWarps = work->regUsed ? 1 : (totalWarps - reduceWarps - bcastWarps + 1) >> 1; const int gatherWarps = work->regUsed ? 1 : (totalWarps - reduceWarps - bcastWarps) >> 1; ssize_t offset; int nelem; int remCount = channelCount%(nvls->nHeads*chunkCount); int lastChunkCount = alignUp(divUp(remCount, nvls->nHeads), 16/sizeof(T)); const int nThreadsScatter = scatterWarps*WARP_SIZE; const int nThreadsGather = gatherWarps*WARP_SIZE; const int nThreadsReduce = reduceWarps*WARP_SIZE; const int nThreadsBcast = (bcastWarps)*WARP_SIZE; const int tidEndScatter = nThreadsScatter; const int tidEndGather = tidEndScatter + nThreadsGather; const int tidEndReduce = tidEndGather + nThreadsReduce; const int tidEndBcast = tidEndReduce + nThreadsBcast; if (tid < tidEndScatter) { // Scatter using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid, nThreadsScatter, NULL, nvls->up, work->sendbuff, NULL, work->redOpArg, 0 * Proto::MaxGroupWidth, 1, 1); for (ssize_t elemOffset = 0; elemOffset < channelCount; elemOffset += loopCount) { if (channelCount - elemOffset < loopCount) chunkCount = lastChunkCount; offset = gridOffset + elemOffset; nelem = work->regUsed ? 0 : min(loopCount, channelCount - elemOffset); prims.scatter(offset, nelem, chunkCount, chunkCount, -1, 0); } } else if (tid < tidEndGather) { // Gather using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid - tidEndScatter, nThreadsGather, nvls->up, NULL, NULL, work->recvbuff, work->redOpArg, 1 * Proto::MaxGroupWidth, 1, 1); for (ssize_t elemOffset = 0; elemOffset < channelCount; elemOffset += loopCount) { if (channelCount - elemOffset < loopCount) chunkCount = lastChunkCount; offset = gridOffset + elemOffset; nelem = work->regUsed ? 0 : min(loopCount, channelCount - elemOffset); prims.gather(offset, nelem, chunkCount, chunkCount, -1, 0); } } else if (tid < tidEndReduce && nvls->headRank != -1) { if (!hasUp) { // Reduce and Broadcast using Proto = ProtoSimple<1, 1, COLL_UNROLL, 1, 1>; Primitives, /*Direct=*/1, Proto, 0> prims(tid - tidEndGather, nThreadsReduce, treeDown, treeDown, NULL, NULL, work->redOpArg, 2 * Proto::MaxGroupWidth, 0, 0, work); for (ssize_t elemOffset = 0; elemOffset < channelCount; elemOffset += loopCount) { ssize_t chunkOffset; if (channelCount - elemOffset < loopCount) chunkCount = lastChunkCount; chunkOffset = elemOffset + nvls->headRank * chunkCount; offset = gridOffset + chunkOffset; nelem = min(chunkCount, channelCount - chunkOffset); prims.directRecvDirectSend(offset, offset, nelem); } } else { // Reduce, send to network using Proto = ProtoSimple<1, 1, COLL_UNROLL, 1, 0>; // Coverity reports that the callee treats &treeUp as an array. However, due to the use of // FanAsymmetric<3, 1>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, /*Direct=*/1, Proto, 0> prims(tid - tidEndGather, nThreadsReduce, treeDown, &treeUp, NULL, NULL, work->redOpArg, 2 * Proto::MaxGroupWidth, 0, 0, work); for (ssize_t elemOffset = 0; elemOffset < channelCount; elemOffset += loopCount) { ssize_t chunkOffset; if (channelCount - elemOffset < loopCount) chunkCount = lastChunkCount; chunkOffset = elemOffset + nvls->headRank * chunkCount; offset = gridOffset + chunkOffset; nelem = min(chunkCount, channelCount - chunkOffset); prims.directRecvDirectSend(offset, offset, nelem); } } } else if (tid < tidEndBcast && nvls->headRank != -1) { // Recv from network, broadcast using Proto = ProtoSimple<1, 1, COLL_UNROLL, 0, 1>; // Coverity reports that the callee treats &treeUp as an array. However, due to the use of // FanAsymmetric<1, 3>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, /*Direct=*/1, Proto, 0> prims(tid - tidEndReduce, nThreadsBcast, &treeUp, treeDown, NULL, NULL, work->redOpArg, 3 * Proto::MaxGroupWidth, 0, 0, work); for (ssize_t elemOffset = 0; elemOffset < channelCount; elemOffset += loopCount) { ssize_t chunkOffset; if (channelCount - elemOffset < loopCount) chunkCount = lastChunkCount; chunkOffset = elemOffset + nvls->headRank * chunkCount; offset = gridOffset + chunkOffset; nelem = min(chunkCount, channelCount - chunkOffset); prims.directRecvDirectSend(offset, offset, nelem); } } } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { const int bid = ncclShmem.channelId - work->channelLo; const int nChannels = work->channelHi - work->channelLo + 1; ncclTree *tree = &ncclShmem.channel.collnetChain; ssize_t chunkSize = work->collnet.chunkCount; const ssize_t loopSize = int(nChannels*chunkSize); const int nranks = ncclShmem.comm.nRanks; const ssize_t size = work->collnet.count; int nthreadsSplit = nthreads/2; if (nthreadsSplit >= 256) nthreadsSplit += 64; int group, connIndex, send, recv, groupTid, groupNthreads; using Proto = ProtoSimple<1, 1>; if (tid < nthreadsSplit) { // Reduce up the chain group = 0; connIndex = 1; recv = tree->down[0]; send = tree->up; groupTid = tid; groupNthreads = nthreadsSplit; } else { // Broadcast down the chain group = 1; connIndex = 0; recv = tree->up; send = tree->down[0]; groupTid = tid - nthreadsSplit; groupNthreads = nthreads-nthreadsSplit; } if (tid < nthreadsSplit) { if (recv == -1) { if (work->netRegUsed) { if (groupTid == 0) { Primitives, /*Direct=*/1, Proto, 0>::sendPeerNotify(send, connIndex, 1); } __syncwarp(); } else { Primitives, /*Direct=*/1, Proto, 0> prims(groupTid, groupNthreads, &recv, &send, work->sendbuff, work->recvbuff, work->redOpArg, group * Proto::MaxGroupWidth, connIndex, connIndex, work); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + bid * int(chunkSize); int nelem = min(chunkSize, size - offset); // coverity[overrun-call] => Coverity think prims.index can be greater than 1 prims.directSend(offset, offset, nelem); } // coverity[overrun-call] => Coverity think prims.index can be greater than 1 } } else { Primitives, /*Direct=*/1, Proto, 0> prims(groupTid, groupNthreads, &recv, &send, work->sendbuff, work->recvbuff, work->redOpArg, group * Proto::MaxGroupWidth, connIndex, connIndex, work); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + bid * int(chunkSize); int nelem = min(chunkSize, size - offset); // coverity[overrun-call] => Coverity think prims.index can be greater than 1 prims.directRecvReduceDirectSend(offset, offset, nelem); } // coverity[overrun-call] => Coverity think prims.index can be greater than 1 } } else { if (recv == nranks) { // I'm the first in the broadcast chain, I need to perform the division (postOp) if (send == -1) { if (work->netRegUsed) { if (groupTid == 0) { Primitives, /*Direct=*/1, Proto, 0>::recvPeerNotify(recv, connIndex, 1); } __syncwarp(); } else { // Coverity reports that the callee treats &send as an array. However, due to the use of // FanSymmetric<1>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, /*Direct=*/1, Proto, 0> prims(groupTid, groupNthreads, &recv, &send, work->sendbuff, work->recvbuff, work->redOpArg, group * Proto::MaxGroupWidth, connIndex, connIndex, work); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + bid * int(chunkSize); int nelem = min(chunkSize, size - offset); prims.directRecv(offset, nelem, /*postOp*/true); } } } else { // Coverity reports that the callee treats &send as an array. However, due to the use of // FanSymmetric<1>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, /*Direct=*/1, Proto, 0> prims(groupTid, groupNthreads, &recv, &send, work->sendbuff, work->recvbuff, work->redOpArg, group * Proto::MaxGroupWidth, connIndex, connIndex, work); for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + bid * int(chunkSize); int nelem = min(chunkSize, size - offset); prims.directRecvCopyDirectSend(offset, offset, nelem, /*postOp*/true); } } } else { // Coverity reports that the callee treats &send as an array. However, due to the use of // FanSymmetric<1>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, /*Direct=*/1, Proto, 0> prims(groupTid, groupNthreads, &recv, &send, work->sendbuff, work->recvbuff, work->redOpArg, group * Proto::MaxGroupWidth, connIndex, connIndex, work); if (send == -1) { for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + bid*int(chunkSize); int nelem = min(chunkSize, size-offset); prims.directRecv(offset, nelem); } } else { for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { ssize_t offset = gridOffset + bid*int(chunkSize); int nelem = min(chunkSize, size-offset); prims.directRecvCopyDirectSend(offset, offset, nelem); } } } } } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runTreeSplit(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runTreeSplit(tid, nthreads, work); } }; nccl-2.29.7-1/src/device/broadcast.h000066400000000000000000000067441515037102200170400ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "device.h" #include "collectives.h" #include "primitives.h" namespace { template __device__ __forceinline__ void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { ncclRing *ring = &ncclShmem.channel.ring; const int rank = ring->userRanks[0]; const int nextRank = ring->userRanks[1]; const int root = work->root; ssize_t chunkCount; ssize_t channelCount; ssize_t gridOffset; ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), (ssize_t*)nullptr, &gridOffset, &channelCount, &chunkCount); size_t offset; int nelem; int workNthreads; bool isNetOffload = work->isOneRPN && work->netRegUsed; T *inputBuf = (T*)work->sendbuff; T *outputBuf = (T*)work->recvbuff; workNthreads = isNetOffload ? WARP_SIZE : nthreads; if (tid < workNthreads) { // Coverity reports that the callee treats &ring->next as an array. However, due to the use of // FanSymmetric<1>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, 1, Proto, 0> prims(tid, workNthreads, &ring->prev, &ring->next, inputBuf, outputBuf, work->redOpArg, 0, 0, 0, work); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); if (rank == root) { if (inputBuf == outputBuf || isNetOffload) { prims.directSend(offset, offset, nelem); } else { prims.directCopySend(offset, offset, nelem); } } else if (nextRank == root) { prims.directRecv(offset, nelem); } else { prims.directRecvCopyDirectSend(offset, offset, nelem); } } } else if (inputBuf != outputBuf && rank == root) { inputBuf = inputBuf + gridOffset; outputBuf = outputBuf + gridOffset; reduceCopy (tid - workNthreads, nthreads - workNthreads, work->redOpArg, false, 1, (void**)&inputBuf, 1, (void**)&outputBuf, channelCount); } if (isNetOffload) barrier_sync(14, nthreads); } } template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { using Proto = ProtoSimple; runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runRing(tid, nthreads, work); } }; nccl-2.29.7-1/src/device/common.cu000066400000000000000000000045021515037102200165340ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "device.h" #include "collectives.h" #include "common.h" #include "nccl_device.h" #include "comm.h" __shared__ ncclShmemData ncclShmem; #if __CUDA_ARCH__ < 700 __shared__ ulong2 ncclShmemPerWarp[ncclShmemScratchWarpSize()*(NCCL_MAX_NTHREADS/WARP_SIZE)/sizeof(ulong2)]; #endif struct RunWorkNop { __device__ void run() {} }; __global__ void ncclDevKernel_Generic(ncclDevKernelArgs4K NCCL_GRID_CONSTANT const args4K) { ncclKernelMain<-1, RunWorkNop>(&args4K.args); } __global__ void ncclDevKernelGinResetSignalsAndCounters(ncclDevComm devComm) { int tid = blockIdx.x * blockDim.x + threadIdx.x; int totalThreads = gridDim.x * blockDim.x; int signalCount = devComm.ginSignalCount; int counterCount = devComm.ginCounterCount; // Reset signals and counters for all contexts for (int contextIdx = 0; contextIdx < devComm.ginContextCount; contextIdx++) { ncclGin gin(devComm, contextIdx); for (int i = tid; i < signalCount; i += totalThreads) { gin.resetSignal(i); } for (int i = tid; i < counterCount; i += totalThreads) { gin.resetCounter(i); } } } ncclResult_t ncclGinResetSignalsAndCounters(struct ncclComm* comm, ncclDevComm_t const* devComm) { int deviceWork = std::max(devComm->ginSignalCount, devComm->ginCounterCount); if (deviceWork == 0) { return ncclSuccess; } // Ensure we run on the comm's device (important when called from async reclaim thread) CUDACHECK(cudaSetDevice(comm->cudaDev)); dim3 grid(1); dim3 block(ncclSymkMaxThreads); // NOTE: Use a dedicated stream so we only wait for the reset kernel, not other device work. cudaStream_t stream = nullptr; CUDACHECK(cudaStreamCreate(&stream)); void* args[] = { (void*)devComm }; CUDACHECK(cudaLaunchKernel((void*)ncclDevKernelGinResetSignalsAndCounters, grid, block, args, 0, stream)); CUDACHECK(cudaStreamSynchronize(stream)); CUDACHECK(cudaStreamDestroy(stream)); return ncclSuccess; } __device__ void ncclDevFunc_Nop() {} nccl-2.29.7-1/src/device/common.h000066400000000000000000000403671515037102200163650ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_DEVICE_COMMON_H_ #define NCCL_DEVICE_COMMON_H_ #include "collectives.h" #include "device.h" #include "op128.h" #include "reduce_kernel.h" #include "network/unpack/unpack_defs.h" #define COLL_UNROLL (ncclCollUnroll()) #if __CUDA_ARCH__ >= 700 // __grid_constant__ appears to break cuda-gdb #define NCCL_GRID_CONSTANT __grid_constant__ #else #define NCCL_GRID_CONSTANT #endif typedef void(*ncclDevFuncPtr_t)(); extern __device__ ncclDevFuncPtr_t const ncclDevFuncTable[]; struct ncclShmemGroup { ncclConnInfo *recvConns[NCCL_MAX_ARITY]; ncclConnInfo *sendConns[NCCL_MAX_ARITY]; void* userInput; void* userOutput; void* srcs[NCCL_MAX_ARITY+1]; void* dsts[NCCL_MAX_ARITY+1]; union { unpackGroupShmem unpack; } devicePlugin; int32_t dstSizes[NCCL_MAX_ARITY+1]; uint64_t redOpArgs; }; struct ncclShmemData { struct ncclDevKernelArgs args; int channelId; int aborted; alignas(16) struct ncclKernelComm comm; alignas(16) struct ncclDevChannel channel; int batchIx, nextBatchIx; enum ncclDevWorkType workType; uint8_t directMode; uint16_t funcId; int nWorks; int workSize; uint64_t workCounter; bool profilerEnabled; struct ncclShmemGroup groups[NCCL_MAX_GROUPS]; alignas(16) char workStorage[ncclMaxDevWorkBatchBytes()]; alignas(16) union { unpackShmem unpack; } devicePlugin; }; extern __shared__ ncclShmemData ncclShmem; #if __CUDA_ARCH__ >= 700 extern __shared__ ulong2 ncclShmemPerWarp[/*ncclShmemDynamicSize()/sizeof(ulong2)*/]; #else extern __shared__ ulong2 ncclShmemPerWarp[ncclShmemScratchWarpSize()*(NCCL_MAX_NTHREADS/WARP_SIZE)/sizeof(ulong2)]; #endif __device__ inline void* ncclScratchForWarp(int warp) { return (char*)ncclShmemPerWarp + warp*ncclShmemScratchWarpSize(); } __device__ inline void barrier_sync(int name) { #if 0 asm volatile("barrier.sync %0;" :: "r"(name) : "memory"); #else asm volatile("barrier.sync.aligned %0;" :: "r"(name) : "memory"); #endif } __device__ inline void barrier_sync(int name, int nThreads) { #if 0 asm volatile("barrier.sync %0, %1;" :: "r"(name), "r"(nThreads) : "memory"); #else asm volatile("barrier.sync.aligned %0, %1;" :: "r"(name), "r"(nThreads) : "memory"); #endif } __device__ inline void barrier_sync_aligned(int name) { asm volatile("barrier.sync.aligned %0;" :: "r"(name) : "memory"); } __device__ inline void barrier_sync_aligned(int name, int nThreads) { asm volatile("barrier.sync.aligned %0, %1;" :: "r"(name), "r"(nThreads) : "memory"); } __device__ inline bool barrier_red_or(bool vote, int name) { int ans; asm volatile("{ .reg .pred p;" " setp.ne.s32 p, %1, 0;" " barrier.red.or.pred p, %2, p; " " selp.s32 %0, 1, 0, p; }" : "=r"(ans) : "r"((int)vote), "r"(name) : "memory"); return bool(ans); } __device__ inline bool barrier_red_or(bool vote, int name, int nThreads) { int ans; asm volatile("{ .reg .pred p;" " setp.ne.s32 p, %1, 0;" " barrier.red.or.pred p, %2, %3, p; " " selp.s32 %0, 1, 0, p; }" : "=r"(ans) : "r"((int)vote), "r"(name), "r"(nThreads) : "memory"); return bool(ans); } // Copy 16-byte aligned data. You must call with at least `(bytes+15)/16` threads. inline __device__ void copyToShmem16(int tid, void* dst, void const* src, int bytes) { int offset = 16*tid; if (offset < bytes) { uint64_t a=0, b=0; asm volatile("ld.v2.u64 {%0,%1},[%2];" : "=l"(a),"=l"(b) : "l"((char const*)src + offset) : "memory"); uint32_t udst = (uint32_t)__cvta_generic_to_shared(dst); asm volatile("st.shared.v2.u64 [%0],{%1,%2};" :: "r"(udst + offset), "l"(a), "l"(b) : "memory"); } } // Must run with at least 64 threads __device__ __forceinline__ void loadWorkBatchToShmem( int tid, int tn, struct ncclDevKernelArgs const* args, int batchIx ) { int lane = tid%WARP_SIZE; int workCursor = 0; // num works written in previous loop iterations. while (true) { struct ncclDevWorkBatch batch = ((struct ncclDevWorkBatch*)(args+1))[batchIx]; // fnsOfBitset[n] = index of n'th set bit in batch.offsetBitset. // PTX has instruction "fns" (find n-th set) but it expands to a lot of SASS, // since we know all lanes will be querying the same bitmask we can compute // much faster using shared memory. uint8_t* fnsOfBitset = (uint8_t*)ncclScratchForWarp(threadIdx.x/WARP_SIZE); __syncwarp(); if (uint32_t(batch.offsetBitset) & (1u<>32) & (1u<>32) & ((1u<>32)); // add high 32 bits __syncwarp(); int workSize; int nPacks; // total number of packs loaded, each pack is 16 bytes int packInWork; // my pack index within work struct int dstWork; // my work index in contiguous destination shmem switch (batch.workType) { case (int)ncclDevWorkTypeP2p: workSize = sizeof(struct ncclDevWorkP2p); nPacks = nWorks*(workSize/16); packInWork = tid%(workSize/16); dstWork = tid/(workSize/16); break; case (int)ncclDevWorkTypeColl: workSize = sizeof(struct ncclDevWorkColl); nPacks = nWorks*(workSize/16); packInWork = tid%(workSize/16); dstWork = tid/(workSize/16); break; case (int)ncclDevWorkTypeBcast: workSize = sizeof(struct ncclDevWorkBcast); nPacks = nWorks*(workSize/16); packInWork = tid%(workSize/16); dstWork = tid/(workSize/16); break; case (int)ncclDevWorkTypeCollReg: default: workSize = sizeof(struct ncclDevWorkCollReg); nPacks = nWorks*(workSize/16); packInWork = tid%(workSize/16); dstWork = tid/(workSize/16); break; } if (tid == 0) { ncclShmem.workSize = workSize; } // We deliberately replicate these div and mod calculations into the case // blocks above so that they get constant divisor optimizations by the compiler. // packInWork = tid%(workSize/16); // dstWork = tid/(workSize/16); // We can only assume we have 64 threads, which means we can read at most 1024 bytes // here which is the per batch maximum. if (tid < nPacks) { int srcWork = fnsOfBitset[dstWork]; // find n'th set bit in batch.offsetBitset ulong2 tmp; // The loads done in these two cases must be kept separate since we are // relying on the compiler to use "ld.param" in the first one. The parameter // space is not generically addressable, so any attempt to load through // a pointer that *might* be parameter space backed will cause the // compiler to spill the parameter struct (4K!) to each thread's local space // before creating a pointer (to the spill) and decimate perf. // // An example of what not to do would be the following: // // if (condition) { // // The compiler could spill parameter_variable to local space and take // // the address of that, since when src is loaded below it could also // // be global space. // src = ¶meter_variable; // } else { // src = &global_variable; // } // memcpy(dst, src, n); if (ncclShmem.args.workStorageType == ncclDevWorkStorageTypeArgs) { char* src = (char*)args + (batch.offsetBase + srcWork*workSize + packInWork*16); tmp = *(ulong2*)src; // becomes ld.param.v2.u64 } else { char* src = (char*)ncclShmem.args.workBuf + ((batch.offsetBase + srcWork*workSize + packInWork*16) & ncclShmem.args.workMask); tmp = *(ulong2*)src; // becomes ld.v2.u64 } char* dst = ncclShmem.workStorage; dst += (workCursor + dstWork)*workSize + packInWork*16; *(ulong2*)dst = tmp; } workCursor += nWorks; if (batch.nextExtends) { batchIx += batch.nextJump; tid -= 64; // Rotate threads so we use the next two warps for next batch struct. if (tid < 0) tid += tn; } else { if (tid == 0) { ncclShmem.batchIx = batchIx; ncclShmem.nextBatchIx = (batch.nextJump == 0) ? -1 : batchIx + batch.nextJump; ncclShmem.workType = (enum ncclDevWorkType)batch.workType; ncclShmem.nWorks = workCursor; ncclShmem.funcId = batch.funcId; } break; } } } __device__ __forceinline__ unsigned long long int globaltimer() { unsigned long long int timer; asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(timer)); return timer; } template struct RunWorkColl { __device__ void run(int tid, int tn, struct ncclDevWorkColl* work) { // Put NOT IMPLEMENTED behavior here. } }; template struct RunWorkBatch; // Specialized for P2p in sendrecv.h template struct RunWorkBatch; template struct RunWorkBatch; // Specialized here for non-P2p (Coll and CollReg) template struct RunWorkBatch { // This __forceinline__ is necessary. The compiler was inserting a function call // here from the LL ncclKernel. __device__ __forceinline__ void run() { int tid = threadIdx.x; int tn = blockDim.x; if (RedOpArg::ArgUsed) { int nWorks = ncclShmem.nWorks; for (int w=tid; w < nWorks; w += tn) { struct ncclDevWorkColl* work = (ncclDevWorkColl*)(ncclShmem.workStorage + w*ncclShmem.workSize); if (work->redOpArgIsPtr) { work->redOpArg = RedOpArg::loadArg(reinterpret_cast(work->redOpArg)); } } __syncthreads(); } #pragma unroll 1 for (int w=0; w < ncclShmem.nWorks; w++) { struct ncclDevWorkColl* work = (struct ncclDevWorkColl*)(ncclShmem.workStorage + w*ncclShmem.workSize); if (w != 0) { struct ncclDevWorkColl* workPrev = (struct ncclDevWorkColl*)(ncclShmem.workStorage + (w-1)*ncclShmem.workSize); if (work->nWarps != workPrev->nWarps) __syncthreads(); } int subtn = work->nWarps*WARP_SIZE; // Coverity reports a possible thread divergence due to not all threads participating in the collective. // However, the code ensures that the participation is on a per-warp basis. // coverity[device_thread_diverged:FALSE] if (tid < subtn) RunWorkColl().run(tid, subtn, work); } } }; #define START 0 #define STOP 1 #define FINI 2 __device__ __forceinline__ bool profilerEnabled(int workItemIdx) { return (ncclShmem.workType == ncclDevWorkTypeP2p) ? ((struct ncclDevWorkP2p*)ncclShmem.workStorage)[workItemIdx].profilerEnabled : ((struct ncclDevWorkColl*)ncclShmem.workStorage)[workItemIdx].profilerEnabled; } __device__ __forceinline__ void profiler(int action) { if (threadIdx.x == 0) { int idx = 0; uint64_t wc = ncclShmem.channel.workCounter + 1; if (action == START) { for (; wc <= ncclShmem.channel.workCounter + ncclShmem.nWorks; wc++) { if (!profilerEnabled(idx++)) continue; ncclShmem.comm.workStarted[ncclShmem.channelId].data[wc%MAX_PROFILER_EVENTS_PER_CHANNEL].timestamp = globaltimer(); ncclShmem.comm.workStarted[ncclShmem.channelId].data[wc%MAX_PROFILER_EVENTS_PER_CHANNEL].counter = wc; } } else { for (; wc <= ncclShmem.channel.workCounter + ncclShmem.nWorks; wc++) { if (!profilerEnabled(idx++)) continue; ncclShmem.comm.workCompleted[ncclShmem.channelId].data[wc%MAX_PROFILER_EVENTS_PER_CHANNEL].timestamp = globaltimer(); ncclShmem.comm.workCompleted[ncclShmem.channelId].data[wc%MAX_PROFILER_EVENTS_PER_CHANNEL].counter = wc; } ncclShmem.channel.workCounter += ncclShmem.nWorks; if (action == FINI) ((ncclKernelCommAndChannels*)ncclShmem.args.comm)->channels[ncclShmem.channelId].workCounter = ncclShmem.channel.workCounter; } } } template __device__ __forceinline__ void ncclKernelMain(struct ncclDevKernelArgs const* args) { int tid = threadIdx.x; int tn = blockDim.x; // Copy kernel args to shmem and then only read those. Otherwise the compiler // will end up putting the args into thread local stack which is very wasteful. if (tid < sizeof(ncclDevKernelArgs)/sizeof(uint32_t)) { ((uint32_t*)&ncclShmem.args)[tid] = ((uint32_t*)args)[tid]; } // To map blockId to channelId, we need the n'th set bit of channelMask which // is the inverse of counting the number of set bits among the the first n. // PTX has the fns instruction which does this but is extremely slow. We can // do better when we know all threads are querying the same bitmask. if (tid < MAXCHANNELS && (args->channelMask & (1ull<channelMask & ((1ull<channels[ncclShmem.channelId].workCounter; } // Use first 2 warps to load comm and channel, and remaining load work batch. switch (tid/WARP_SIZE) { case 0: { void* dst = &ncclShmem.comm; void* src = ncclShmem.args.comm; int bytes = sizeof(ncclKernelComm); static_assert(sizeof(ncclKernelComm) <= 16*WARP_SIZE, "ncclKernelComm cannot be loaded by a single warp in one insn."); copyToShmem16(tid, dst, src, bytes); } break; case 1: { // Get address of channel without incurring indirect load from ncclKernelComm::channels void* dst = &ncclShmem.channel; void* src = &((ncclKernelCommAndChannels*)ncclShmem.args.comm)->channels[ncclShmem.channelId]; int bytes = sizeof(ncclDevChannel); static_assert(sizeof(ncclDevChannel) <= 16*WARP_SIZE, "ncclDevChannel cannot be loaded by a single warp in one insn."); copyToShmem16(tid-WARP_SIZE, dst, src, bytes); } break; default: { int subtid = tid - 2*WARP_SIZE; int subtn = tn - 2*WARP_SIZE; // Coverity reports a possible thread divergence due to not all threads participating in the collective. // However, the code ensures that the participation is on a per-warp basis. // coverity[device_thread_diverged:FALSE] loadWorkBatchToShmem(subtid, subtn, args, /*batchIx=*/blockIdx.x); } break; } __syncthreads(); // publish ncclShmem while (ncclShmem.aborted == 0) { profiler(START); if (0 <= SpecializedFnId && ncclShmem.funcId == (unsigned)SpecializedFnId) { SpecializedRunWorkBatch().run(); } else { ncclDevFuncTable[ncclShmem.funcId](); } if (ncclShmem.nextBatchIx == -1) break; int batchIx = ncclShmem.nextBatchIx; __syncthreads(); profiler(STOP); loadWorkBatchToShmem(tid, tn, args, batchIx); __syncthreads(); } profiler(FINI); } __global__ void ncclDevKernel_Generic(ncclDevKernelArgs4K NCCL_GRID_CONSTANT const args4K); __device__ void ncclDevFunc_Nop(); #define DEFINE_ncclDevKernel(suffix, coll, redop, ty, algo, proto, specializedFnId) \ __global__ void ncclDevKernel_##suffix(ncclDevKernelArgs4K NCCL_GRID_CONSTANT const args4K) { \ ncclKernelMain, algo, proto>>(&args4K.args); \ } #define DEFINE_ncclDevKernel_nop(suffix, coll, redop, ty, algo, proto, specializedFnId) \ __global__ void ncclDevKernel_##suffix(ncclDevKernelArgs4K NCCL_GRID_CONSTANT const args4K) {} #define DEFINE_ncclDevFunc(suffix, coll, redop, ty, algo, proto) \ __device__ void ncclDevFunc_##suffix() { \ RunWorkBatch, algo, proto>().run(); \ } #endif nccl-2.29.7-1/src/device/common_kernel.h000066400000000000000000000255531515037102200177250ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_COMMON_KERNEL_H_ #define NCCL_COMMON_KERNEL_H_ #include "device.h" #include "op128.h" #include "reduce_kernel.h" #include #include #include // Define min for ssize_t inline __device__ int min(int a, ssize_t b) { return (a < b) ? a : b; } inline __device__ int loadInt(int* ptr) { int v; asm volatile("ld.volatile.global.u32 %0, [%1];" : "=r"(v) : "l"(ptr) : "memory"); return v; } template __device__ __forceinline__ void reduceCopyPacks( int nThreads, int &thread, uint64_t redArg, bool postOp, int nSrcs, SrcPtrFn const &srcPtrFn, int nDsts, DstPtrFn const &dstPtrFn, IntBytes &nBytesBehind, IntBytes &nBytesAhead ) { static_assert(std::is_signed::value, "IntBytes must be a signed integral type."); if (BytePerPack == 0) __trap(); // A hunk is the amount of contiguous data a warp consumes per loop iteration // assuming all threads partake. constexpr int BytePerHunk = Unroll*WARP_SIZE*BytePerPack; int nWarps = nThreads/WARP_SIZE; int warp = thread/WARP_SIZE; int lane = thread%WARP_SIZE; // This thread's initial position. IntBytes threadBytesBehind = nBytesBehind + (warp*BytePerHunk + lane*BytePerPack); IntBytes threadBytesAhead = nBytesAhead - (warp*BytePerHunk + lane*BytePerPack); // Number of hunks to be consumed over all warps. IntBytes nHunksAhead = nBytesAhead/(BytePerHunk + !BytePerHunk); // Advance collective position. nBytesBehind += nHunksAhead*BytePerHunk; nBytesAhead -= nHunksAhead*BytePerHunk; if (Unroll==1 && BytePerPack <= nBytesAhead) { // Only Unroll=1 can do partial hunks (where not all threads partake). nHunksAhead += 1; nBytesBehind += nBytesAhead - (nBytesAhead%(BytePerPack + !BytePerPack)); nBytesAhead = nBytesAhead%(BytePerPack + !BytePerPack); } nHunksAhead -= warp; RedFn redFn(redArg); uintptr_t minSrcs[MinSrcs + !MinSrcs]; uintptr_t minDsts[MinDsts + !MinDsts]; #pragma unroll for (int s=0; s < MinSrcs; s++) { minSrcs[s] = cvta_to_global(srcPtrFn(s)) + threadBytesBehind; } #pragma unroll for (int d=0; d < MinDsts; d++) { // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] minDsts[d] = cvta_to_global(dstPtrFn(d)) + threadBytesBehind; } // We dictate loop termination condition according to whether partial hunks // can be handled or not. while (Unroll==1 ? (BytePerPack <= threadBytesAhead) : (0 < nHunksAhead)) { BytePack acc[Unroll]; // minSrcs[0] cannot be nullptr so we always process it { #pragma unroll Unroll for (int u=0; u < Unroll; u++) { if (0 < MultimemSrcs) { // applyLoadMultimem uses relaxed semantics for same reason we use volatile below. acc[u] = applyLoadMultimem(redFn, minSrcs[0]); } else { // Use volatile loads in case credits are polled for with volatile (instead of acquire). acc[u] = ld_volatile_global(minSrcs[0]); if (0 < PreOpSrcs) acc[u] = applyPreOp(redFn, acc[u]); } minSrcs[0] += WARP_SIZE*BytePerPack; } } #pragma unroll (MinSrcs-1 + !(MinSrcs-1)) for (int s=1; s < MinSrcs; s++) { // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_begin] BytePack tmp[Unroll]; // coverity[dead_error_line] #pragma unroll Unroll for (int u=0; u < Unroll; u++) { if (s < MultimemSrcs) { // applyLoadMultimem uses relaxed semantics for same reason we use volatile below. // coverity[dead_error_line] tmp[u] = applyLoadMultimem(redFn, minSrcs[s]); } else { // Use volatile loads in case credits are polled for with volatile (instead of acquire). tmp[u] = ld_volatile_global(minSrcs[s]); } minSrcs[s] += WARP_SIZE*BytePerPack; } #pragma unroll Unroll for (int u=0; u < Unroll; u++) { // coverity[dead_error_line] acc[u] = applyReduce(redFn, acc[u], tmp[u]); } } for (int s=MinSrcs; (MinSrcs < MaxSrcs) && (s < MaxSrcs) && (s < nSrcs); s++) { uintptr_t src = cvta_to_global(srcPtrFn(s)) + threadBytesBehind; BytePack tmp[Unroll]; // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] #pragma unroll Unroll for (int u=0; u < Unroll; u++) { // Use volatile loads in case credits are polled for with volatile (instead of acquire). tmp[u] = ld_volatile_global(src); src += WARP_SIZE*BytePerPack; } #pragma unroll Unroll for (int u=0; u < Unroll; u++) { // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] acc[u] = applyReduce(redFn, acc[u], tmp[u]); } } if (postOp) { #pragma unroll Unroll for (int u=0; u < Unroll; u++) acc[u] = applyPostOp(redFn, acc[u]); } #pragma unroll (MinDsts + !MinDsts) for (int d=0; d < MinDsts; d++) { #pragma unroll Unroll // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_begin] for (int u=0; u < Unroll; u++) { // coverity[dead_error_condition] if (d < MultimemDsts) { multimem_st_global(minDsts[d], acc[u]); } else { st_global(minDsts[d], acc[u]); } minDsts[d] += WARP_SIZE*BytePerPack; } } for (int d=MinDsts; (MinDsts < MaxDsts) && (d < MaxDsts) && (d < nDsts); d++) { uintptr_t dstPtr = cvta_to_global(dstPtrFn(d)); uintptr_t dst = dstPtr + threadBytesBehind; #pragma unroll Unroll for (int u=0; u < Unroll; u++) { st_global(dst, acc[u]); dst += WARP_SIZE*BytePerPack; } } nWarps = nThreads/WARP_SIZE; #pragma unroll for (int s=0; s < MinSrcs; s++) { minSrcs[s] += (nWarps-1)*BytePerHunk; } #pragma unroll // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] for (int d=0; d < MinDsts; d++) { minDsts[d] += (nWarps-1)*BytePerHunk; } threadBytesBehind += nWarps*BytePerHunk; threadBytesAhead -= nWarps*BytePerHunk; nHunksAhead -= nWarps; } nWarps = nThreads/WARP_SIZE; warp = thread/WARP_SIZE; lane = thread%WARP_SIZE; // The last loop iteration could have been partial, i.e. not taken by all // threads. The threads that weren't included need an extra subtraction to // make the value warp uniform. if (Unroll==1 && nHunksAhead > 0) nHunksAhead -= nWarps; // Rotate warps so the warp which got the least work here will be warp 0. // This effectively assigns: warp = (warp-nHunks+nWarps)%nWarps; warp = -nHunksAhead; thread = warp*WARP_SIZE + lane; } template __device__ __forceinline__ void reduceCopy( int thread, int nThreads, uint64_t redArg, bool postOp, int nSrcs, SrcPtrFn const &srcPtrFn, int nDsts, DstPtrFn const &dstPtrFn, IntBytes nElts ) { static_assert(MultimemSrcs <= MinSrcs && MultimemDsts <= MinDsts, "Multimem pointers cannot exceed respective Min values."); //int nWarps = nThreads/WARP_SIZE; //int warp = thread/WARP_SIZE; int lane = thread%WARP_SIZE; // If a multimem src is present then our biggest pack size is limited to what // is supported for this redfn/type. constexpr int BigPackSize = (MultimemSrcs == 0) ? 16 : LoadMultimem_BigPackSize::BigPackSize; if (MaxDsts==0) return; if (MinDsts==0 && nDsts==0) return; IntBytes nBytesBehind = 0; IntBytes nBytesAhead = nElts*sizeof(T); #if __cpp_if_constexpr if constexpr (BigPackSize > sizeof(T)) { #else if (BigPackSize > sizeof(T)) { #endif // Check that all pointers are BigPackSize aligned. bool aligned = true; if (lane < nSrcs) aligned &= 0 == cvta_to_global(srcPtrFn(lane)) % (BigPackSize + !BigPackSize); if (lane < nDsts) aligned &= 0 == cvta_to_global(dstPtrFn(lane)) % (BigPackSize + !BigPackSize); aligned = __all_sync(~0u, aligned); if (aligned) { reduceCopyPacks (nThreads, /*&*/thread, redArg, postOp, nSrcs, srcPtrFn, nDsts, dstPtrFn, /*&*/nBytesBehind, /*&*/nBytesAhead); if (nBytesAhead == 0) return; reduceCopyPacks (nThreads, /*&*/thread, redArg, postOp, nSrcs, srcPtrFn, nDsts, dstPtrFn, /*&*/nBytesBehind, /*&*/nBytesAhead); if (nBytesAhead == 0) return; } } reduceCopyPacks (nThreads, /*&*/thread, redArg, postOp, nSrcs, srcPtrFn, nDsts, dstPtrFn, /*&*/nBytesBehind, /*&*/nBytesAhead); if (nBytesAhead == 0) return; reduceCopyPacks (nThreads, /*&*/thread, redArg, postOp, nSrcs, srcPtrFn, nDsts, dstPtrFn, /*&*/nBytesBehind, /*&*/nBytesAhead); } template __device__ __forceinline__ void reduceCopy( int thread, int nThreads, uint64_t redArg, bool postOp, int nSrcs, void** srcPtrs, int nDsts, void** dstPtrs, IntBytes nElts ) { reduceCopy (thread, nThreads, redArg, postOp, nSrcs, [=]__device__(int i) { return srcPtrs[i]; }, nDsts, [=]__device__(int i) { return dstPtrs[i]; }, nElts); } #endif // COMMON_KERNEL_H_ nccl-2.29.7-1/src/device/generate.py000077500000000000000000000405301515037102200170630ustar00rootroot00000000000000#!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information import os import sys import shutil # Order of redops, tys, protos, algos must match src/include/device.h all_colls = ["Broadcast","Reduce","AllGather","AllGatherV", "ReduceScatter","AllReduce","SendRecv"] all_redops = ["Sum","Prod","MinMax","PreMulSum","SumPostDiv"] all_tys = ["i8","u8","i32","u32","i64","u64","f16","f32","f64","bf16","f8e4m3","f8e5m2"] all_protos = ["LL","LL128","SIMPLE"] all_algos = ["TREE","RING","COLLNET_DIRECT","COLLNET_CHAIN","NVLS","NVLS_TREE","PAT"] ################################################################################ # The first command line argument is the path to the directory to generate and # populate. gensrc = sys.argv[1] if os.path.exists(gensrc): for name in os.listdir(gensrc): path = os.path.join(gensrc, name) if os.path.isfile(path): os.remove(path) elif os.path.isdir(path): shutil.rmtree(path) else: os.mkdir(gensrc) ################################################################################ # The second command line argument is used as a regex to filter the functions # which make it into libnccl. This is helpful for reducing the binary when # developing device code. The regex supports non-space containing globs '*', # parentheses '(x)', and union 'a|b'. The string representing the function has # one of the forms: # # SendRecv # (AllGather|Broadcast) # (AlLReduce|Reduce|ReduceScatter) # # The possible values for redop, type, algo, proto can be found in the all_ # lists at the top of this file. # # Since the Makefile forwards this from the ONLY_FUNCS variable, useful command # line examples are given: """ # Only send/recv: make ONLY_FUNCS="SendRecv" # Only non-reductions: make ONLY_FUNCS="AllGather * *|Broadcast * *|SendRecv" # Only AllReduce sum f32 (but all algos, protos) make ONLY_FUNCS="AllReduce Sum f32 * *" # Only AllReduce minmax i32 NVLS (but all protos) make ONLY_FUNCS="AllReduce MinMax i32 NVLS *" # AllReduce sum RING LL128 make ONLY_FUNCS="AllReduce Sum f32 RING LL128" """ # Paste all non-None arguments together with `sep`. def paste(sep, *args): return sep.join(x for x in args if x is not None) func_pattern = sys.argv[2:3] if func_pattern and func_pattern[0]: import re func_pattern = func_pattern[0] func_pattern = func_pattern.replace("*", "[^ ]*") func_pattern += "$" def func_filter(*fn): return None is not re.match(func_pattern, paste(" ", *fn), flags=re.IGNORECASE) else: def func_filter(coll, redop, ty, algo, proto): return True ################################################################################ algos_of_coll = { "AllGather": ["RING","COLLNET_DIRECT","NVLS","PAT"], "AllGatherV": ["RING"], "AllReduce": ["TREE","RING","COLLNET_DIRECT","COLLNET_CHAIN","NVLS","NVLS_TREE"], "Broadcast": ["RING"], "Reduce": ["RING"], "ReduceScatter": ["RING","COLLNET_DIRECT","NVLS","PAT"], "SendRecv": [None] } coll_camel_to_lower = { "AllGather": "all_gather", "AllGatherV": "all_gather_v", "AllReduce": "all_reduce", "Broadcast": "broadcast", "Reduce": "reduce", "ReduceScatter": "reduce_scatter", "SendRecv": "sendrecv" } coll_lower_to_camel = {coll_camel_to_lower[x]: x for x in coll_camel_to_lower} ################################################################################ # Returns pair of minimum required values for (CUDART_VERSION, __CUDA_ARCH__) # or None if function is never supported. Note that (0, 0) encodes universal # support. def required_cuda(coll, redop, ty, algo, proto): cudart, arch = 0, 0 # kernels mapped to by coll="Nop" functions have coll="Generic" if coll in ("SendRecv", "Generic", "Nop"): return (cudart, arch) if proto!="SIMPLE" and algo not in ("RING","TREE"): return None if coll in ("AllReduce","Reduce","ReduceScatter"): if redop=="SumPostDiv" and ty[0] not in ("i","u"): return None if ty=="bf16": cudart = max(cudart, 11000) if ty.startswith("f8"): cudart = max(cudart, 11080) arch = max(arch, 900) if "NVLS" in algo: if coll in ("AllReduce","Reduce","ReduceScatter"): # Must match ncclNvlsSupported() in src/include/device.h nvls_ok = ((ty in ("i32","u32","i64","u64") and redop in ("Sum","MinMax")) or (ty in ("f32","f64") and redop=="Sum") or (ty in ("f16","bf16") and redop in ("Sum","MinMax"))) if not nvls_ok: return None cudart = max(cudart, 12010) arch = max(arch, 900) return (cudart, arch) # Maps functions to the chosen representative for the equivalence class it # belongs to. For instance (sum, signed int) maps to (sum, unsigned int). def equivalent_primary(coll, redop, ty, algo, proto): if coll in ("AllReduce", "Reduce", "ReduceScatter"): # map signed integer sum/prod to unsigned if redop in ("Sum","Prod","PreMulSum","SumPostDiv") and ty[0]=="i": return (coll, redop, "u"+ty[1:], algo, proto) # map signed integer min/max to unsigned for non-NVLS if redop=="MinMax" and ty[0]=="i" and ("NVLS" not in algo): return (coll, redop, "u"+ty[1:], algo, proto) return (coll, redop, ty, algo, proto) # Map to another func representing the best kernel to use. Every distinct value # returned will instantiate a ncclDevKernel specialized to run this func # without function call overhead. def best_kernel(coll, redop, ty, algo, proto): def best(coll, redop, ty, algo, proto): # Modify this logic to control how many kernels are specialized. if coll=="Nop": return ("Generic", None, None, None, None) if coll=="SendRecv": return ("SendRecv", None, None, None, None) if coll in ("AllGather","Broadcast","AllGatherV"): return (coll, None, None, "RING", "LL") return (coll, "Sum", ty, ("TREE" if algo=="TREE" else "RING"), "LL") # Need to ensure kernel is specialize for a primary function kfn = equivalent_primary(*best(coll, redop, ty, algo, proto)) # And isn't filtered out. if not func_filter(*kfn): return ("Generic", None, None, None, None) return kfn # Order rows are enumerated must match formula of `ncclDevFuncId()`: def enumerate_func_rows(): yield ("SendRecv", None, None, None, None) for coll in ("AllGather", "Broadcast", "AllGatherV"): algos = algos_of_coll[coll] for algo in algos: for proto in all_protos: yield (coll, None, None, algo, proto) for coll in ("AllReduce", "Reduce", "ReduceScatter"): algos = algos_of_coll[coll] for redop in all_redops: for ty in all_tys: for algo in algos: for proto in all_protos: yield (coll, redop, ty, algo, proto) ################################################################################ def is_built(coll, redop, ty, algo, proto): built = required_cuda(coll, redop, ty, algo, proto) built = built and func_filter(coll, redop, ty, algo, proto) return built # Returns None if required_cuda(...) is None. # Returns the coll="Nop" function if developer has filtered it out. # Otherwise just returns func it was given. def validate(coll, redop, ty, algo, proto): valid = required_cuda(coll, redop, ty, algo, proto) built = valid and func_filter(coll, redop, ty, algo, proto) if built: return (coll, redop, ty, algo, proto) if valid: return ("Nop", None, None, None, None) return None # Corresponds to ncclDevFuncRowToId[] func_rows = [validate(*fn) for fn in enumerate_func_rows()] # Corresponds to ncclDevFuncTable[] primary_funcs = sorted(set(equivalent_primary(*fn) for fn in func_rows if fn is not None)) # primary_to_index[primary_funcs[i]] == i primary_to_index = {fn: i for (i,fn) in zip(range(len(primary_funcs)), primary_funcs)} kernel_funcs = sorted(set(best_kernel(*fn) for fn in primary_funcs)) ################################################################################ # Generate /device_table.cu with open(os.path.join(gensrc, "device_table.cu"), "w") as f: out = f.write out('#include "common.h"\n') out("\n") for fn in primary_funcs: sym = paste("_", "ncclDevFunc", *fn) cudart, arch = required_cuda(*fn) if (cudart, arch) != (0, 0): out("#if CUDART_VERSION >= %d && __CUDA_ARCH__ >= %d\n" % (cudart, arch)) out("__device__ void %s();\n" % sym) if (cudart, arch) != (0, 0): out("#endif\n") out("\n") out("__device__ ncclDevFuncPtr_t const ncclDevFuncTable[] = {\n"); index = 0 for fn in primary_funcs: sym = paste("_", "ncclDevFunc", *fn) cudart, arch = required_cuda(*fn) if (cudart, arch) != (0, 0): out("#if CUDART_VERSION >= %d && __CUDA_ARCH__ >= %d\n" % (cudart ,arch)) out("/*%4d*/ %s,\n" % (index, sym)) if (cudart, arch) != (0, 0): out("#else\n" "/*%4d*/ nullptr,\n" "#endif\n" % index) index += 1 out("nullptr};\n") out("\n") out("// Workaround for https://reviews.llvm.org/D55580\n" "__device__ void ncclWorkaroundClangD55580() {}\n") # Generate /host_table.cc with open(os.path.join(gensrc, "host_table.cc"), "w") as f: out = f.write out('#include "device.h"\n') out("\n") out("extern int const ncclDevFuncIdCount = %d;\n" % len(primary_funcs)) # The mapping from function rows to valid primary function ids. out("extern int const ncclDevFuncRowToId[] = {\n") index = 0 for fn in func_rows: fn_id, comment = -1, "" if fn is not None: fn_id = primary_to_index[equivalent_primary(*fn)] comment = " // " + paste(" ", *fn) out("/*%4d*/ %d,%s\n" % (index, fn_id, comment)) index += 1 out("-1};\n") out("\n") # Forward declarations of kernels. for kfn in kernel_funcs: cudart, _ = required_cuda(*kfn) sym = paste("_", "ncclDevKernel", *kfn) if cudart != 0: out("#if CUDART_VERSION >= %d\n" % cudart) # __global__ below gets removed by the host compiler, which results in # Coverity diagnosing a specifiers inconsistency. out("// coverity[declaration]\n") out("__global__ void %s(ncclDevKernelArgs4K const);\n" % sym) if cudart != 0: out("#endif\n") out("\n") # List of all kernel function pointers. out("extern int const ncclDevKernelCount = %d;\n" % len(kernel_funcs)) out("void* ncclDevKernelList[] = {\n") index = 0 for kfn in kernel_funcs: cudart, _ = required_cuda(*kfn) sym = paste("_", "ncclDevKernel", *kfn) if cudart != 0: out("#if CUDART_VERSION >= %d\n" % cudart) out("/*%4d*/ (void*)%s,\n" % (index, sym)); if cudart != 0: out("#else\n" "/*%4d*/ nullptr,\n" "#endif\n" % index) index += 1 out("nullptr};\n") out("\n") out("int ncclDevKernelRequirements[] = {\n") for index,kfn in enumerate(kernel_funcs): cudart,_ = required_cuda(*kfn) sym = paste("_", "ncclDevKernel", *kfn) out(" %7d, /*%4d %s*/\n" % (cudart or 0, index, sym)); out("};\n") out("\n") # Maps primary id to kernel function pointer. out("extern void* const ncclDevKernelForFunc[] = {\n") index = 0 for fn in primary_funcs: kfn = best_kernel(*fn) sym = paste("_", "ncclDevKernel", *kfn) cudart, _ = required_cuda(*kfn) if cudart != 0: out("#if CUDART_VERSION >= %d\n" % cudart) out("/*%4d*/ (void*)%s,\n" % (index, sym)) if cudart != 0: out("#else\n" "/*%4d*/ nullptr,\n" "#endif\n" % index) index += 1 out("nullptr};\n") out("\n") # Does the prior map use an explicitly specialized kernel. out("extern bool const ncclDevKernelForFuncIsSpecialized[] = {\n") index = 0 for fn in primary_funcs: kfn = best_kernel(*fn) specialized = "1" if fn == kfn else "0" out("/*%4d*/ %s,\n" % (index, specialized)) index += 1 out("0};\n") # Maps to .cu filename which implements this func. The only constraint is that # "coll" is reflected in the name: formally that no two funcs having different # coll's map to the same filename. def impl_filename(coll, redop, ty, algo, proto): return "%s.cu" % paste("_", coll_camel_to_lower[coll], redop and redop.lower(), ty) # Partition the functions and kernels to the .cu filenames. The partition is # a dictionary mapping filename to (coll, func-tuple list) def partition_by_name(fns): ans = {} for fn in fns: name = impl_filename(*fn) coll = fn[0] if name not in ans: ans[name] = (coll, []) ans[name][1].append(fn) return ans name_to_funcs = partition_by_name(fn for fn in primary_funcs if fn[0]!="Nop") name_to_kernels = partition_by_name(kfn for kfn in kernel_funcs if kfn[0]!="Generic") files = "" for name in sorted(name_to_funcs.keys()): files += name + ";" files += "device_table.cu;" files += "host_table.cc" # Output file list for CMake (excludes rules.mk since it's not generated for CMake) if os.environ.get("NCCL_USE_CMAKE", "0") == "1": print(files) # Generate /rules.mk (only needed for Makefile builds, not CMake) if os.environ.get("NCCL_USE_CMAKE", "0") != "1": with open(os.path.join(gensrc, "rules.mk"), "w") as f: out = f.write impl_names = sorted(name_to_funcs.keys()) names = impl_names + ["host_table.cc", "device_table.cu"] out("LIB_OBJS_GEN = $(patsubst %,$(OBJDIR)/genobj/%.o,{names})\n" .format(names=" ".join(names))) out("\n") # For each __.cu compile to a .cu.o file. Notice the dependencies # come from the suffix-erased file (e.g. 'gensrc/all_reduce.cu') for name in impl_names: coll = name_to_funcs[name][0] out( "$(OBJDIR)/genobj/{name}.o: $(OBJDIR)/gensrc $(OBJDIR)/genobj/{lower_coll}.cu.d\n" "\t" "$(call COMPILE,$@,$(OBJDIR)/gensrc/{name})\n" "\n" .format(name=name, lower_coll=coll_camel_to_lower[coll]) ) # Add the suffix-erased .cu's which are used only for dependency scraping. for coll in set(coll for (coll,_,_,_,_) in primary_funcs if coll!="Nop"): name = impl_filename(coll, None, None, None, None) if name not in name_to_funcs: name_to_funcs[name] = (coll, []) redop_to_cxx = { None: "FuncCopy", "Sum": "FuncSum", "Prod": "FuncProd", "MinMax": "FuncMinMax", "PreMulSum": "FuncPreMulSum", "SumPostDiv": "FuncSumPostDiv" } ty_to_cxx = { None: "int8_t", "i8": "int8_t", "u8": "uint8_t", "i32": "int32_t", "u32": "uint32_t", "i64": "int64_t", "u64": "uint64_t", "f16": "half", "f32": "float", "f64": "double", "bf16": "__nv_bfloat16", "f8e4m3": "__nv_fp8_e4m3", "f8e5m2": "__nv_fp8_e5m2" } # Generate each /.cu: for name in name_to_funcs.keys(): (coll, fns) = name_to_funcs[name] with open(os.path.join(gensrc, name), "w") as f: out = f.write out( '#include "common.h"\n' '#include "{lower_coll}.h"\n' .format(lower_coll=coll_camel_to_lower[coll]) ) (_, kfns) = name_to_kernels.get(name) or (None, []) for kfn in kfns: (coll, redop, ty, algo, proto) = kfn sym = paste("_", coll, redop, ty, algo, proto) fn_id = primary_to_index[kfn] cudart, arch = required_cuda(*kfn) s = "DEFINE_ncclDevKernel({sym}, ncclFunc{coll}, {redop_cxx}, {ty_cxx}, NCCL_ALGO_{algo}, NCCL_PROTO_{proto}, {fn_id})\n" if (cudart, arch) != (0, 0): # Add conditional compilation logic around s. If CUDART_VERSION is satisfactory # we must compile a kernel regardless of __CUDA_ARCH__ since the host code has # to link against some stub. s = "#if CUDART_VERSION >= {cudart}\n" \ " #if __CUDA_ARCH__ < {arch}\n" \ " DEFINE_ncclDevKernel_nop({sym}, ncclFunc{coll}, {redop_cxx}, {ty_cxx}, NCCL_ALGO_{algo}, NCCL_PROTO_{proto}, {fn_id})\n" \ " #else\n" \ " " + s + \ " #endif\n" \ "#endif\n" out(s.format( cudart=cudart, arch=arch, sym=sym, coll=coll, redop_cxx=redop_to_cxx[redop], ty_cxx=ty_to_cxx[ty], algo=(algo or "RING"), proto=(proto or "SIMPLE"), fn_id=fn_id )) for fn in fns: (coll, redop, ty, algo, proto) = fn sym = paste("_", coll, redop, ty, algo, proto) cudart, arch = required_cuda(*fn) if (cudart, arch) != (0, 0): out("#if CUDART_VERSION >= %d && __CUDA_ARCH__ >= %d\n" % (cudart, arch)) out( "DEFINE_ncclDevFunc({sym}, ncclFunc{coll}, {redop_cxx}, {ty_cxx}, NCCL_ALGO_{algo}, NCCL_PROTO_{proto})\n" .format(sym=sym, coll=coll, redop_cxx=redop_to_cxx[redop], ty_cxx=ty_to_cxx[ty], algo=(algo or "RING"), proto=(proto or "SIMPLE")) ) if (cudart, arch) != (0, 0): out("#endif\n") nccl-2.29.7-1/src/device/network/000077500000000000000000000000001515037102200164035ustar00rootroot00000000000000nccl-2.29.7-1/src/device/network/unpack/000077500000000000000000000000001515037102200176645ustar00rootroot00000000000000nccl-2.29.7-1/src/device/network/unpack/unpack.h000066400000000000000000000264431515037102200213270ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2023 Google LLC. All rights reserved. * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 and BSD-3 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_DEVICE_UNPACK_H #define NET_DEVICE_UNPACK_H #include "unpack_defs.h" #include "op128.h" #include "bitops.h" #include "device.h" #include "common.h" // #define ALIGNED_LOAD inline __device__ void load64gpu(const uint64_t* ptr, uint64_t &v) { #if __CUDA_ARCH__ >= 700 asm volatile("ld.relaxed.gpu.u64 {%0}, [%1];" : "=l"(v) : "l"(ptr) : "memory"); #else asm volatile("ld.volatile.global.u64 {%0}, [%1];" : "=l"(v) : "l"(ptr) : "memory"); #endif } #define PAGE_META_SIZE 16 #define META_LOAD_SIZE 16 #define DATA_LOAD_SIZE 16 // Map internal association of handle with group and peer index (called once at init time) inline __device__ void ncclNetDeviceUnpackSetup(void* ohandle, const int group, const int index) { struct unpackNetDeviceHandle* handle = (struct unpackNetDeviceHandle*) ohandle; // coverity[index_parm:FALSE] ncclShmem.groups[group].devicePlugin.unpack.g_meta[index] = handle->meta; ncclShmem.devicePlugin.unpack.bounce_buf = handle->bounce_buf; // coverity[index_parm:FALSE] ncclShmem.groups[group].devicePlugin.unpack.head[index] = handle->head; } inline __device__ void ncclNetDeviceIncrementHead(const int group, const int index) { // coverity[index_parm:FALSE] ncclShmem.groups[group].devicePlugin.unpack.head[index]++; } inline __device__ void ncclNetDeviceSaveHead(void* ohandle, const int group, const int index) { struct unpackNetDeviceHandle* handle = (struct unpackNetDeviceHandle*) ohandle; // coverity[index_parm:FALSE] handle->head = ncclShmem.groups[group].devicePlugin.unpack.head[index]; } template inline __device__ void bulkLoad(const int t, const uint32_t len, char* cpy_src, char* cpy_dst, BytePack *reg, const int w, loadMeta* g_meta, loadMeta* s_meta, uint32_t src_off, uint64_t dst_off){ bulkLoad<1>(t, len, cpy_src, cpy_dst, reg, w, g_meta, s_meta, src_off, dst_off); } template <> inline __device__ void bulkLoad<1>(const int t, const uint32_t len, char* cpy_src, char* cpy_dst, BytePack<1> reg[16], const int w, loadMeta* g_meta, loadMeta* s_meta, uint32_t src_off, uint64_t dst_off){ uint64_t data_s; for (data_s = t * DATA_LOAD_SIZE; data_s + DATA_LOAD_SIZE - 1 < len; data_s += WARP_SIZE * DATA_LOAD_SIZE) { #ifdef ALIGNED_LOAD load128 ((uint64_t*)(cpy_src + data_s), reg.u64[0], reg.u64[1]); #else #pragma unroll for (int i=0; i<16; i++) { reg[i] = ld_volatile_global<1>((uintptr_t)((uint8_t*)(cpy_src + data_s) + i)); } #endif #pragma unroll for (int i=0; i<16; i++) { st_global<1>((uintptr_t)((uint8_t*)(cpy_dst + data_s) + i), reg[i]); } } } template <> inline __device__ void bulkLoad<2>(const int t, const uint32_t len, char* cpy_src, char* cpy_dst, BytePack<2> reg[8], const int w, loadMeta* g_meta, loadMeta* s_meta, uint32_t src_off, uint64_t dst_off){ uint64_t data_s; for (data_s = t * DATA_LOAD_SIZE; data_s + DATA_LOAD_SIZE - 1 < len; data_s += WARP_SIZE * DATA_LOAD_SIZE) { #ifdef ALIGNED_LOAD load128 ((uint64_t*)(cpy_src + data_s), reg.u64[0], reg.u64[1]); #else #pragma unroll for (int i=0; i<8; i++) { reg[i] = ld_volatile_global<2>((uintptr_t)((uint16_t*)(cpy_src + data_s) + i)); } #endif #pragma unroll for (int i=0; i<8; i++) { st_global<2>((uintptr_t)((uint16_t*)(cpy_dst + data_s) + i), reg[i]); } } } template <> inline __device__ void bulkLoad<4>(const int t, const uint32_t len, char* cpy_src, char* cpy_dst, BytePack<4> reg[4], const int w, loadMeta* g_meta, loadMeta* s_meta, uint32_t src_off, uint64_t dst_off){ uint64_t data_s; for (data_s = t * DATA_LOAD_SIZE; data_s + DATA_LOAD_SIZE - 1 < len; data_s += WARP_SIZE * DATA_LOAD_SIZE) { #ifdef ALIGNED_LOAD load128 ((uint64_t*)(cpy_src + data_s), reg.u64[0], reg.u64[1]); #else #pragma unroll for (int i=0; i<4; i++) { reg[i] = ld_volatile_global<4>((uintptr_t)((uint32_t *)(cpy_src + data_s) + i)); } #endif #pragma unroll for (int i=0; i<4; i++) { st_global<4>((uintptr_t)((uint32_t*)(cpy_dst + data_s) + i), reg[i]); } } } template <> inline __device__ void bulkLoad<8>(const int t, const uint32_t len, char* cpy_src, char* cpy_dst, BytePack<8> reg[2], const int w, loadMeta* g_meta, loadMeta* s_meta, uint32_t src_off, uint64_t dst_off){ uint64_t data_s; for (data_s = t * DATA_LOAD_SIZE; data_s + DATA_LOAD_SIZE - 1 < len; data_s += WARP_SIZE * DATA_LOAD_SIZE) { #ifdef ALIGNED_LOAD load128 ((uint64_t*)(cpy_src + data_s), reg.u64[0], reg.u64[1]); #else #pragma unroll for (int i=0; i<2; i++) { reg[i] = ld_volatile_global<8>((uintptr_t)((uint64_t*)(cpy_src + data_s) + i)); } #endif #pragma unroll for (int i=0; i<2; i++) { st_global<8>((uintptr_t)((uint64_t*)(cpy_dst + data_s) + i), reg[i]); } } } template <> inline __device__ void bulkLoad<16>(const int t, const uint32_t len, char* cpy_src, char* cpy_dst, BytePack<16> reg[1], const int w, loadMeta* g_meta, loadMeta* s_meta, uint32_t src_off, uint64_t dst_off){ uint64_t data_s; for (data_s = t * DATA_LOAD_SIZE; data_s + DATA_LOAD_SIZE - 1 < len; data_s += WARP_SIZE * DATA_LOAD_SIZE) { reg[0] = ld_volatile_global<16>((uintptr_t)(cpy_src + data_s)); st_global<16>((uintptr_t)(cpy_dst + data_s), reg[0]); } } #ifndef PAGE_SIZE #define PAGE_SIZE 4096 #endif inline __device__ int ppw(const int nbytes, int nw) { int v = DIVUP(nbytes, SLICE_PAGE_SIZE); v = DIVUP(v, nw); while (v > WARP_SHM_PAGE_CNT) { v = DIVUP(v, 2); } return v; } // This function is called by all threads // Pack data from the internal iovec to the supplied flat buffer using all the // threads template inline __device__ void ncclNetDeviceUnpack( const int tid, const int tidInBlock, const int nworkers, const int group, int mask, int Src, int workSize); template <> inline __device__ void ncclNetDeviceUnpack( const int tid, const int tidInBlock, const int nworkers, const int group, int mask, int Src, int workSize) { // send unpack empty } inline __device__ void ncclNetDeviceUnpackInner( const int tid, const int tidInBlock, const int nworkers, const int group, const int index, void *src, const int nbytes, const uint64_t step); template <> inline __device__ void ncclNetDeviceUnpack( const int tid, const int tidInBlock, const int nworkers, const int group, int mask, int Src, int workSize) { while (mask != 0) { int ix = __ffs(mask)-1; // Get the first set bit of the mask (this should correlate to a peer index) mask &= mask-1; // Drop the first set bit of the mask // Pack data from the internal iovec to the supplied flat srcs buffer using all the threads // + Src is necessary in the case of accessing the user buffer directly ncclNetDeviceUnpackInner(tid, tidInBlock, nworkers, group /* in case they need to use split warps shared memory partitioning*/, ix, ncclShmem.groups[group].srcs[ix + Src], workSize, ncclShmem.groups[group].devicePlugin.unpack.head[ix]); } } inline __device__ void ncclNetDeviceUnpackInner( const int tid, const int tidInBlock, const int nworkers, const int group, const int index, void *src, const int nbytes, const uint64_t step) { // from src/collectives/device/common_kernel.h const int w = tid / WARP_SIZE; // Warp number const int nw = nworkers / WARP_SIZE; // Number of warps const int t = tid % WARP_SIZE; // Thread (inside the warp) BytePack<16> reg; loadMeta meta; uint64_t head; struct netUnpackMeta* g_meta_struct; void* bounce_buf; loadMeta* g_meta; loadMeta* s_meta; uint64_t meta_cnt; // hack head use per-warp head = step; g_meta_struct = ncclShmem.groups[group].devicePlugin.unpack.g_meta[index]; bounce_buf = ncclShmem.devicePlugin.unpack.bounce_buf; __syncwarp(); head %= NCCL_NET_DEVICE_UNPACK_MAX_QUEUE_DEPTH; g_meta = g_meta_struct->mem[head]; // Currently, even/odd groups perform send/recv separately. We don't really need space for send side. // Total size is N page per warp * 16 B per page * 20 WARPS max = 320 * N bytes, N == WARP_SHM_PAGE_CNT static_assert(ncclShmemScratchWarpSize() >= WARP_SHM_SIZE, "Each warp must have enough scratch space"); s_meta = (loadMeta*) ncclScratchForWarp(tidInBlock / WARP_SIZE); // (loadMeta*) (ncclShmem.devicePlugin.unpack.meta + shm_off); load64gpu(g_meta_struct->cnt + head, meta_cnt); int PPW = ppw(nbytes, nw); // Coverity reports a potential overflow but in reality PPW is tiny so there's no need to store it in an uint64_t. // coverity[overflow_before_widen] for (uint64_t meta_s = w * PPW; meta_s < meta_cnt; meta_s += nw * PPW) { uint64_t iter_meta_cnt = meta_cnt - meta_s; iter_meta_cnt = iter_meta_cnt < PPW ? iter_meta_cnt : PPW; // TODO: this load size needs to work if not aligned, but since the two are both 16... if (t < PPW * PAGE_META_SIZE / META_LOAD_SIZE && t < iter_meta_cnt) { // avoid last iter load garbage data load128((const uint64_t*) (g_meta + (meta_s + t)), reg.u64[0], reg.u64[1]); storeShmem128(shmemCvtPtr((uint64_t *)(s_meta + (w * PPW + t))), reg.u64[0], reg.u64[1]); } __syncwarp(); for (int x = 0; x < iter_meta_cnt; x++) { int meta_idx = x + w * PPW; // load page offs loadShmem128(shmemCvtPtr((uint64_t*) (s_meta + meta_idx)), meta.r64[0], meta.r64[1]); if (meta.len >= DATA_LOAD_SIZE) { // fast path, but need to adapt to alignment issue // bulk copy data uint8_t align_off = (meta.src_off | meta.dst_off) % DATA_LOAD_SIZE; align_off = align_off & -align_off; // keep the lowest bit if (align_off == 0) { // 0x16 bulkLoad<16>(t, meta.len, (char*) bounce_buf + meta.src_off, (char*) src + meta.dst_off, ®, w, g_meta, s_meta, meta.src_off, meta.dst_off); } else if (align_off & 0x8) { bulkLoad<8>(t, meta.len, (char*) bounce_buf + meta.src_off, (char*) src + meta.dst_off, (BytePack<8>*) ®, w, g_meta, s_meta, meta.src_off, meta.dst_off); } else if (align_off & 0x4) { bulkLoad<4>(t, meta.len, (char*) bounce_buf + meta.src_off, (char*) src + meta.dst_off, (BytePack<4>*) ®, w, g_meta, s_meta, meta.src_off, meta.dst_off); } else if (align_off & 0x2) { bulkLoad<2>(t, meta.len, (char*) bounce_buf + meta.src_off, (char*) src + meta.dst_off, (BytePack<2>*) ®, w, g_meta, s_meta, meta.src_off, meta.dst_off); } else { // if (align_off & 0x1) bulkLoad<1>(t, meta.len, (char*) bounce_buf + meta.src_off, (char*) src + meta.dst_off, (BytePack<1>*) ®, w, g_meta, s_meta, meta.src_off, meta.dst_off); } } // must be less than 16 bytes if (t < meta.len % DATA_LOAD_SIZE) { volatile char* cpy_src = (char*) bounce_buf + meta.src_off + (meta.len / DATA_LOAD_SIZE) * DATA_LOAD_SIZE + t; volatile char* cpy_dst = (char*) src + meta.dst_off + (meta.len / DATA_LOAD_SIZE) * DATA_LOAD_SIZE + t; *cpy_dst = *cpy_src; } } __syncwarp(); } } #endif // NET_DEVICE_UNPACK_DEFS_H_ nccl-2.29.7-1/src/device/network/unpack/unpack_defs.h000066400000000000000000000040131515037102200223150ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2023 Google LLC. All rights reserved. * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 and BSD-3 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NET_DEVICE_UNPACK_DEFS_H #define NET_DEVICE_UNPACK_DEFS_H #include #include "device.h" #define NCCL_NET_DEVICE_UNPACK_MAX_QUEUE_DEPTH 16 union alignas(16) loadMeta { uint64_t r64[2]; struct { uint32_t src_off; uint32_t len; uint64_t dst_off; }; }; static_assert(sizeof(union loadMeta) == 16, "Must be 16-byte aligned"); /****** global memory ******/ #define NET_UNPACK_MAX_QUEUE_DEPTH 16 // MAX_REQUESTS #define NET_UNPACK_MAX_SLICE_SIZE 4194304 // 4MB per Irecv call #define SLICE_PAGE_SIZE 4096 #define NET_UNPACK_MAX_SLICE_PAGES \ (NET_UNPACK_MAX_SLICE_SIZE / SLICE_PAGE_SIZE * 2) // * 2 for slack, wasteful.. struct netUnpackMeta { loadMeta mem[NCCL_NET_DEVICE_UNPACK_MAX_QUEUE_DEPTH][NET_UNPACK_MAX_SLICE_PAGES]; uint64_t cnt[NCCL_NET_DEVICE_UNPACK_MAX_QUEUE_DEPTH]; }; struct unpackNetDeviceHandle { struct netUnpackMeta *meta; // mapped void* bounce_buf; uint64_t head; }; /****** shared memory ******/ #define NET_UNPACK_MAX_GROUPS 16 // Forked from NCCL_MAX_GROUPS in devcomm.h #define NET_UNPACK_MAX_NPEERS 2 // The most you should have is 2 network peers per-group (indexed by index) #define WARP_SHM_PAGE_CNT 4 #define WARP_SHM_SIZE (WARP_SHM_PAGE_CNT * sizeof(union loadMeta)) struct unpackShmem { void* bounce_buf; }; struct unpackGroupShmem { int unpackNetDeviceIndexMask; // We store a single unpackNetDeviceIndex because only one peer can be network recv uint64_t head[NET_UNPACK_MAX_NPEERS]; struct netUnpackMeta* g_meta[NET_UNPACK_MAX_NPEERS]; // head of handle to index into meta for meta copy }; #endif // NET_DEVICE_UNPACK_DEFS_H_ nccl-2.29.7-1/src/device/onerank.cu000066400000000000000000000070431515037102200167040ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2023-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "alloc.h" #include "collectives.h" #include "common_kernel.h" #include "common.h" #include namespace { template __global__ __launch_bounds__(512, 1) void oneRankReduce(void* dst, void* src, size_t nElts, uint64_t redOpArg, bool redOpArgIsPtr) { using T = typename RedOp::EltType; int tid = threadIdx.x; int tn = blockDim.x; int bid = blockIdx.x; int bn = gridDim.x; // each block/channel gets a roughly equal segment of 16 byte packs constexpr int EltPerPack = 16/sizeof(T); intptr_t i0 = (bid+0)*alignUp(divUp(nElts, bn), EltPerPack); intptr_t i1 = (bid+1)*alignUp(divUp(nElts, bn), EltPerPack); i0 = min(i0, nElts); i1 = min(i1, nElts); src = (T*)src + i0; dst = (T*)dst + i0; if (redOpArgIsPtr) { if (redOpArg%2 != 0) { redOpArg = *reinterpret_cast(redOpArg); } else if (redOpArg%4 != 0) { redOpArg = *reinterpret_cast(redOpArg); } else if (redOpArg%8 != 0) { redOpArg = *reinterpret_cast(redOpArg); } else { redOpArg = *reinterpret_cast(redOpArg); } } reduceCopy (tid, tn, redOpArg, true, 1, &src, 1, &dst, i1-i0); } } ncclResult_t ncclLaunchOneRank(void* dst, void const* src, size_t nElts, struct ncclDevRedOpFull redOp, ncclDataType_t eltType, cudaStream_t stream) { size_t eltSize = ncclTypeSize(eltType); if (redOp.op != ncclDevPreMulSum) { if (dst != src) { NCCLCHECK(ncclCudaMemcpyAsync((char*)dst, (char*)src, nElts*eltSize, stream)); } return ncclSuccess; } void const* kernel; switch (eltType) { case ncclInt8: kernel = (void const*)&oneRankReduce>; break; case ncclUint8: kernel = (void const*)&oneRankReduce>; break; case ncclInt32: kernel = (void const*)&oneRankReduce>; break; case ncclUint32: kernel = (void const*)&oneRankReduce>; break; case ncclInt64: kernel = (void const*)&oneRankReduce>; break; case ncclUint64: kernel = (void const*)&oneRankReduce>; break; #if defined(__CUDA_FP8_TYPES_EXIST__) && __CUDA_ARCH__ >= 900 case ncclFloat8e4m3: kernel = (void const*)&oneRankReduce>; break; case ncclFloat8e5m2: kernel = (void const*)&oneRankReduce>; break; #endif case ncclFloat16: kernel = (void const*)&oneRankReduce>; break; #if defined(__CUDA_BF16_TYPES_EXIST__) case ncclBfloat16: kernel = (void const*)&oneRankReduce>; break; #endif case ncclFloat32: kernel = (void const*)&oneRankReduce>; break; case ncclFloat64: kernel = (void const*)&oneRankReduce>; break; default: return ncclInvalidArgument; } dim3 grid = {0, 1, 1}; grid.x = std::min(32, (int)divUp(nElts*eltSize, 16<<10)); dim3 block = {512, 1, 1}; void* args[5] = {&dst, &src, &nElts, &redOp.scalarArg, &redOp.scalarArgIsPtr}; CUDACHECK(cudaLaunchKernel(kernel, grid, block, args, 0, stream)); return ncclSuccess; } nccl-2.29.7-1/src/device/op128.h000066400000000000000000000454401515037102200157430ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef OP128_H_ #define OP128_H_ #include inline __device__ void load128(const uint64_t* ptr, uint64_t &v0, uint64_t &v1) { asm volatile("ld.volatile.global.v2.u64 {%0,%1}, [%2];" : "=l"(v0), "=l"(v1) : "l"(ptr) : "memory"); } inline __device__ void store128(uint64_t* ptr, uint64_t v0, uint64_t v1) { asm volatile("st.volatile.global.v2.u64 [%2], {%0,%1};" :: "l"(v0), "l"(v1), "l"(ptr) : "memory"); } inline __device__ uint64_t* shmemCvtPtr(volatile uint64_t* shmemGenericPtr) { uint64_t* shmemAsmPtr; asm volatile("cvta.to.shared.u64 %0, %1;" : "=l"(shmemAsmPtr) : "l"(shmemGenericPtr) : "memory"); return shmemAsmPtr; } inline __device__ void loadShmem128(uint64_t* shmemAsmPtr, uint64_t &v0, uint64_t &v1) { asm volatile("ld.volatile.shared.v2.u64 {%0,%1}, [%2];" : "=l"(v0), "=l"(v1) : "l"(shmemAsmPtr) : "memory"); } inline __device__ void storeShmem128(uint64_t* shmemAsmPtr, uint64_t v0, uint64_t v1) { asm volatile("st.volatile.shared.v2.u64 [%2], {%0,%1};" :: "l"(v0), "l"(v1), "l"(shmemAsmPtr) : "memory"); } template inline __device__ void loadShmemMisaligned128(T *ptr, uint64_t &v0, uint64_t &v1) { union { uint32_t tmp4[4]; uint64_t tmp8[2]; }; if(sizeof(T) < 4) { uint32_t *ptr4 = reinterpret_cast(reinterpret_cast(ptr) & -uintptr_t(4)); #pragma unroll for(int e=0; e < 4; e++) { // Produce 4 bytes of sub-register type by reading 2 4-byte // aligned values and shifting. uint32_t lo, hi; asm volatile("ld.shared.b32 %0,[%1];" : "=r"(lo) : "l"(ptr4+e+0) : "memory"); asm volatile("ld.shared.b32 %0,[%1];" : "=r"(hi) : "l"(ptr4+e+1) : "memory"); tmp4[e] = __funnelshift_r(lo, hi, 8*(int(reinterpret_cast(ptr))%4)); } } else if(sizeof(T) == 4) { #pragma unroll for(int e=0; e < 4; e++) asm volatile("ld.shared.b32 %0,[%1];" : "=r"(tmp4[e]) : "l"(ptr+e) : "memory"); } else /*sizeof(T)==8*/ { #pragma unroll for(int e=0; e < 2; e++) asm volatile("ld.shared.b64 %0,[%1];" : "=l"(tmp8[e]) : "l"(ptr+e) : "memory"); } v0 = tmp8[0]; v1 = tmp8[1]; } template __device__ __forceinline__ uint32_t cvta_to_shared(T* ptr) { return (uint32_t)__cvta_generic_to_shared(ptr); } template __device__ __forceinline__ uintptr_t cvta_to_global(T* ptr) { return (uintptr_t)__cvta_generic_to_global(ptr); } template __device__ __forceinline__ T* cvta_from_shared(uint32_t shptr) { T* ans; asm("cvta.shared.u64 %0, %1;" : "=l"(ans) : "l"(uint64_t(shptr))); return ans; } template __device__ __forceinline__ T* cvta_from_global(uintptr_t gptr) { T* ans; asm("cvta.global.u64 %0, %1;" : "=l"(ans) : "l"(gptr)); return ans; } //////////////////////////////////////////////////////////////////////////////// // BytePack: struct of bytes. template union BytePack; template<> union BytePack<0> {}; template<> union BytePack<1> { uint8_t u8[1], native; }; template<> union BytePack<2> { BytePack<1> half[2]; BytePack<1> b1[2]; uint8_t u8[2]; uint16_t u16[1], native; }; template<> union BytePack<4> { BytePack<2> half[2]; BytePack<1> b1[4]; BytePack<2> b2[2]; uint8_t u8[4]; uint16_t u16[2]; uint32_t u32[1], native; }; template<> union BytePack<8> { BytePack<4> half[2]; BytePack<1> b1[8]; BytePack<2> b2[4]; BytePack<4> b4[2]; uint8_t u8[8]; uint16_t u16[4]; uint32_t u32[2]; uint64_t u64[1], native; }; template<> union alignas(16) BytePack<16> { BytePack<8> half[2]; BytePack<1> b1[16]; BytePack<2> b2[8]; BytePack<4> b4[4]; BytePack<8> b8[2]; uint8_t u8[16]; uint16_t u16[8]; uint32_t u32[4]; uint64_t u64[2]; ulong2 ul2[1], native; }; template union BytePack { BytePack half[2]; BytePack<1> b1[Size]; BytePack<2> b2[Size/2]; BytePack<4> b4[Size/4]; BytePack<8> b8[Size/8]; BytePack<16> b16[Size/16]; uint8_t u8[Size]; uint16_t u16[Size/2]; uint32_t u32[Size/4]; uint64_t u64[Size/8]; }; template struct BytePackOf { static constexpr int Size = sizeof(T); using Pack = BytePack; }; template<> struct BytePackOf> { static constexpr int Size = 0; using Pack = BytePack<0>; }; template __device__ __forceinline__ typename BytePackOf::Pack toPack(T value) { union { typename BytePackOf::Pack p; T v; }; // Coverity recommends the use of std::move here but, given that T is a POD // scalar, a plain copy will be just as efficient. // coverity[copy_assignment_call] v = value; return p; } template __device__ __forceinline__ T fromPack(typename BytePackOf::Pack pack) { union { typename BytePackOf::Pack p; T v; }; p = pack; return v; } //////////////////////////////////////////////////////////////////////////////// // Load/store of BytePack using integral addresses. template __device__ BytePack ld_global(uintptr_t addr); template __device__ BytePack ld_shared(uint32_t addr); template __device__ BytePack ld_volatile_global(uintptr_t addr); template __device__ BytePack ld_volatile_shared(uint32_t addr); template __device__ BytePack ld_relaxed_gpu_global(uintptr_t addr); template __device__ void st_global(uintptr_t addr, BytePack value); template __device__ void st_shared(uint32_t addr, BytePack value); template __device__ void st_relaxed_gpu_global(uintptr_t addr, BytePack value); template<> __device__ __forceinline__ BytePack<0> ld_global<0>(uintptr_t addr) { return {}; } template<> __device__ __forceinline__ BytePack<0> ld_shared<0>(uint32_t addr) { return {}; } template<> __device__ __forceinline__ BytePack<0> ld_volatile_global<0>(uintptr_t addr) { return {}; } template<> __device__ __forceinline__ BytePack<0> ld_volatile_shared<0>(uint32_t addr) { return {}; } template<> __device__ __forceinline__ BytePack<0> ld_relaxed_gpu_global<0>(uintptr_t addr) { return {}; } template<> __device__ __forceinline__ void st_global<0>(uintptr_t addr, BytePack<0> value) {} template<> __device__ __forceinline__ void st_shared<0>(uint32_t addr, BytePack<0> value) {} template<> __device__ __forceinline__ void st_relaxed_gpu_global<0>(uintptr_t addr, BytePack<0> value) {} // Used to define implementations for above prototypes. #define DEFINE_ld_st__size_space(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty, space, addr_cxx_ty, addr_reg_ty) \ template<> \ __device__ __forceinline__ BytePack ld_##space(addr_cxx_ty addr) { \ data_cxx_ty tmp; \ asm volatile("ld." #space "." #data_ptx_ty " %0, [%1];" : "="#data_reg_ty(tmp) : #addr_reg_ty(addr) : "memory"); \ BytePack ans; \ ans.native = tmp; \ return ans; \ } \ template<> \ __device__ __forceinline__ BytePack ld_volatile_##space(addr_cxx_ty addr) { \ data_cxx_ty tmp; \ asm volatile("ld.volatile." #space "." #data_ptx_ty " %0, [%1];" : "="#data_reg_ty(tmp) : #addr_reg_ty(addr) : "memory"); \ BytePack ans; \ ans.native = tmp; \ return ans; \ } \ template<> \ __device__ __forceinline__ void st_##space(addr_cxx_ty addr, BytePack value) { \ data_cxx_ty tmp = value.native; \ asm volatile("st." #space "." #data_ptx_ty " [%0], %1;" :: #addr_reg_ty(addr), #data_reg_ty(tmp) : "memory"); \ } #if __CUDA_ARCH__ >= 700 #define PTX_relaxed_gpu "relaxed.gpu" #else #define PTX_relaxed_gpu "volatile" #endif #define DEFINE_ld_st_gpu_relaxed__size(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty) \ template<> \ __device__ __forceinline__ BytePack ld_relaxed_gpu_global(uintptr_t addr) { \ data_cxx_ty tmp; \ asm volatile("ld." PTX_relaxed_gpu ".global." #data_ptx_ty " %0, [%1];" : "="#data_reg_ty(tmp) : "l"(addr) : "memory"); \ BytePack ans; \ ans.native = tmp; \ return ans; \ } \ template<> \ __device__ __forceinline__ void st_relaxed_gpu_global(uintptr_t addr, BytePack value) { \ data_cxx_ty tmp = value.native; \ asm volatile("st." PTX_relaxed_gpu ".global." #data_ptx_ty " [%0], %1;" :: "l"(addr), #data_reg_ty(tmp) : "memory"); \ } #define DEFINE_ld_st__size(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty) \ DEFINE_ld_st__size_space(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty, global, uintptr_t, l) \ DEFINE_ld_st__size_space(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty, shared, uint32_t, r) \ DEFINE_ld_st_gpu_relaxed__size(bytes, data_cxx_ty, data_ptx_ty, data_reg_ty) // Single-byte types use 4-byte registers since there is no 1-byte register // character for asm blocks. See https://docs.nvidia.com/cuda/inline-ptx-assembly/index.html#constraints DEFINE_ld_st__size(1, uint32_t, b8, r) DEFINE_ld_st__size(2, uint16_t, b16, h) DEFINE_ld_st__size(4, uint32_t, b32, r) DEFINE_ld_st__size(8, uint64_t, b64, l) #undef DEFINE_ld_st__size_space #undef DEFINE_ld_st__size #define DEFINE_ld_st_16__space(space, addr_cxx_ty, addr_reg_ty) \ template<> \ __device__ __forceinline__ BytePack<16> ld_##space<16>(addr_cxx_ty addr) { \ BytePack<16> ans; \ asm volatile("ld." #space ".v2.b64 {%0,%1}, [%2];" : "=l"(ans.u64[0]), "=l"(ans.u64[1]) : #addr_reg_ty(addr) : "memory"); \ return ans; \ } \ template<> \ __device__ __forceinline__ BytePack<16> ld_volatile_##space<16>(addr_cxx_ty addr) { \ BytePack<16> ans; \ asm volatile("ld.volatile." #space ".v2.b64 {%0,%1}, [%2];" : "=l"(ans.u64[0]), "=l"(ans.u64[1]) : #addr_reg_ty(addr) : "memory"); \ return ans; \ } \ template<> \ __device__ __forceinline__ void st_##space<16>(addr_cxx_ty addr, BytePack<16> value) { \ asm volatile("st." #space ".v2.b64 [%0], {%1,%2};" :: #addr_reg_ty(addr), "l"(value.u64[0]), "l"(value.u64[1]) : "memory"); \ } DEFINE_ld_st_16__space(global, uintptr_t, l) DEFINE_ld_st_16__space(shared, uint32_t, r) #undef DEFINE_ld_st_16 template<> __device__ __forceinline__ BytePack<16> ld_relaxed_gpu_global<16>(uintptr_t addr) { BytePack<16> ans; asm volatile("ld." PTX_relaxed_gpu ".global.v2.b64 {%0,%1}, [%2];" : "=l"(ans.u64[0]), "=l"(ans.u64[1]) : "l"(addr) : "memory"); return ans; } template<> __device__ __forceinline__ void st_relaxed_gpu_global<16>(uintptr_t addr, BytePack<16> value) { asm volatile("st." PTX_relaxed_gpu ".global.v2.b64 [%0], {%1,%2};" :: "l"(addr), "l"(value.u64[0]), "l"(value.u64[1]) : "memory"); } #undef PTX_relaxed_gpu //////////////////////////////////////////////////////////////////////////////// // Atomic load/store using c++ pointers. __device__ __forceinline__ uint64_t ld_volatile_global(uint64_t *ptr) { uint64_t ans; asm volatile("ld.volatile.global.u64 %0, [%1];" : "=l"(ans) : "l"(cvta_to_global(ptr)) : "memory"); return ans; } __device__ __forceinline__ uint64_t ld_relaxed_sys_global(uint64_t *ptr) { uint64_t ans; #if __CUDA_ARCH__ >= 700 asm volatile("ld.relaxed.sys.global.u64 %0, [%1];" : "=l"(ans) : "l"(cvta_to_global(ptr)) : "memory"); #else asm volatile("ld.volatile.global.u64 %0, [%1];" : "=l"(ans) : "l"(cvta_to_global(ptr)) : "memory"); #endif return ans; } __device__ __forceinline__ uint64_t ld_relaxed_gpu_global(uint64_t *ptr) { uint64_t ans; #if __CUDA_ARCH__ >= 700 asm volatile("ld.relaxed.gpu.global.u64 %0, [%1];" : "=l"(ans) : "l"(cvta_to_global(ptr)) : "memory"); #else asm volatile("ld.volatile.global.u64 %0, [%1];" : "=l"(ans) : "l"(cvta_to_global(ptr)) : "memory"); #endif return ans; } __device__ __forceinline__ uint64_t ld_acquire_sys_global(uint64_t *ptr) { uint64_t ans; #if __CUDA_ARCH__ >= 700 asm volatile("ld.acquire.sys.global.u64 %0, [%1];" : "=l"(ans) : "l"(cvta_to_global(ptr)) : "memory"); #else asm volatile("ld.volatile.sys.global.u64 %0, [%1]; membar.gl;" : "=l"(ans) : "l"(cvta_to_global(ptr)) : "memory"); #endif return ans; } __device__ __forceinline__ void st_volatile_global(uint64_t *ptr, uint64_t val) { asm volatile("st.volatile.global.u64 [%0], %1;" :: "l"(cvta_to_global(ptr)), "l"(val) : "memory"); } __device__ __forceinline__ void st_relaxed_sys_global(uint64_t *ptr, uint64_t val) { #if __CUDA_ARCH__ >= 700 asm volatile("st.relaxed.sys.global.u64 [%0], %1;" :: "l"(cvta_to_global(ptr)), "l"(val) : "memory"); #else asm volatile("st.volatile.global.u64 [%0], %1;" :: "l"(cvta_to_global(ptr)), "l"(val) : "memory"); #endif } __device__ __forceinline__ void st_release_sys_global(uint64_t *ptr, uint64_t val) { #if __CUDA_ARCH__ >= 700 asm volatile("st.release.sys.global.u64 [%0], %1;" :: "l"(cvta_to_global(ptr)), "l"(val) : "memory"); #else asm volatile("membar.sys; st.volatile.global.u64 [%0], %1;" :: "l"(cvta_to_global(ptr)), "l"(val) : "memory"); #endif } __device__ __forceinline__ void fence_acq_rel_sys() { #if __CUDA_ARCH__ >= 700 asm volatile("fence.acq_rel.sys;" ::: "memory"); #else asm volatile("membar.sys;" ::: "memory"); #endif } __device__ __forceinline__ void fence_acq_rel_gpu() { #if __CUDA_ARCH__ >= 700 asm volatile("fence.acq_rel.gpu;" ::: "memory"); #else asm volatile("membar.gl;" ::: "memory"); #endif } //////////////////////////////////////////////////////////////////////////////// // Multimem stores of BytePack. template __device__ __forceinline__ void multimem_st_global(uintptr_t addr, BytePack val); #if __CUDA_ARCH__ >= 900 && CUDART_VERSION >= 12010 template<> __device__ __forceinline__ void multimem_st_global<0>(uintptr_t addr, BytePack<0> val) { // nop } template<> __device__ __forceinline__ void multimem_st_global<1>(uintptr_t addr, BytePack<1> val) { asm volatile("st.global.b8 [%0], %1;" :: "l"(addr), "r"((uint32_t)val.native) : "memory"); } template<> __device__ __forceinline__ void multimem_st_global<2>(uintptr_t addr, BytePack<2> val) { asm volatile("st.global.b16 [%0], %1;" :: "l"(addr), "h"(val.native) : "memory"); } template<> __device__ __forceinline__ void multimem_st_global<4>(uintptr_t addr, BytePack<4> val) { asm volatile("multimem.st.global.b32 [%0], %1;" :: "l"(addr), "r"(val.native) : "memory"); } template<> __device__ __forceinline__ void multimem_st_global<8>(uintptr_t addr, BytePack<8> val) { asm volatile("multimem.st.global.b64 [%0], %1;" :: "l"(addr), "l"(val.native) : "memory"); } template<> __device__ __forceinline__ void multimem_st_global<16>(uintptr_t addr, BytePack<16> val) { asm volatile("multimem.st.global.v4.f32 [%0], {%1,%2,%3,%4};" :: "l"(addr), "r"(val.u32[0]), "r"(val.u32[1]), "r"(val.u32[2]), "r"(val.u32[3]) : "memory"); } #else template __device__ __forceinline__ void multimem_st_global(uintptr_t addr, BytePack val) { // nop } #endif // Load pack starting at index in array. Ignore elements past end (length of array). template __device__ __forceinline__ Pack loadPack(T* ptr, int ix, int end) { constexpr int Size = sizeof(Pack); ptr += ix; int n = end - ix; if (alignof(T) == Size && sizeof(T) == Size) { return *(Pack*)ptr; } else if ((Size+3)/4 + 1 < Size/sizeof(T)) { union { Pack ans; uint32_t part[Size/4]; }; int misalign = reinterpret_cast(ptr) % 4; uint32_t* down = reinterpret_cast(reinterpret_cast(ptr) & -uintptr_t(4)); int i; #pragma unroll for (i=0; i < Size/4; i++) { if (i*4/sizeof(T) < 1 || i*4/sizeof(T) < n) part[i] = down[i]; } uint32_t extra; if (misalign) extra = down[i]; #pragma unroll for (i=0; i < Size/4; i++) { part[i] = __funnelshift_r(part[i], part[i+1], 8*misalign); } if (misalign) part[i] = __funnelshift_r(part[i], extra, 8*misalign); return ans; } else { union { Pack ans; BytePack part[Size/sizeof(T)]; }; #pragma unroll for (int i=0; i < Size/sizeof(T); i++) { if (i < 1 || i < n) part[i] = ((BytePack*)ptr)[i]; } return ans; } } // Store pack starting at index in array. Ignore elements past end (length of array). template __device__ __forceinline__ void storePack(T* ptr, int ix, int end, Pack val) { constexpr int Size = sizeof(Pack); union { Pack tmp; BytePack part[Size/sizeof(T)]; }; tmp = val; ptr += ix; int n = end - ix; #pragma unroll for (int i=0; i < Size/sizeof(T); i++) { if (i < 1 || i < n) ((BytePack*)ptr)[i] = part[i]; } } // Warp-uniform memory copy from shared address (not generic) to global memory. // The number of bytes copied is `min(MaxBytes, nBytesAhead)`, a negative value // is interpeted as zero. EltSize is the guaranteed alignment of the addresses and sizes. template __device__ __forceinline__ void copyGlobalShared_WarpUnrolled( int lane, uintptr_t dstAddr, uint32_t srcAddr, IntBytes nBytesAhead ) { static_assert(std::is_signed::value, "`IntBytes` must be a signed integral type."); int nBytes = min(nBytesAhead, (IntBytes)MaxBytes); int nFrontBytes = min(nBytes, (16 - int(dstAddr%16))%16); int nMiddleBytes = (nBytes-nFrontBytes) & -16; int nBackBytes = (nBytes-nFrontBytes) % 16; { int backLane = WARP_SIZE-1 - lane; bool hasFront = lane*EltSize < nFrontBytes; bool hasBack = backLane*EltSize < nBackBytes; int offset = hasFront ? lane*EltSize : (nBytes - (backLane+1)*EltSize); if (hasFront | hasBack) { BytePack tmp = ld_shared(srcAddr+offset); // Can't use multimem_st since it doesn't support EltSize==2 st_global(dstAddr+offset, tmp); } } srcAddr += nFrontBytes; int srcMisalign = EltSize < 4 ? (srcAddr%4) : 0; srcAddr += -srcMisalign + lane*16; dstAddr += nFrontBytes + lane*16; nMiddleBytes -= lane*16; #pragma unroll for (int u=0; u < divUp(MaxBytes, WARP_SIZE*16); u++) { if (nMiddleBytes <= 0) break; union { BytePack<4> b4[4]; BytePack<16> b16; }; b4[0] = ld_shared<4>(srcAddr + 0*4); b4[1] = ld_shared<4>(srcAddr + 1*4); b4[2] = ld_shared<4>(srcAddr + 2*4); b4[3] = ld_shared<4>(srcAddr + 3*4); if (srcMisalign != 0) { BytePack<4> b4_4 = ld_shared<4>(srcAddr + 4*4); b4[0].native = __funnelshift_r(b4[0].native, b4[1].native, srcMisalign*8); b4[1].native = __funnelshift_r(b4[1].native, b4[2].native, srcMisalign*8); b4[2].native = __funnelshift_r(b4[2].native, b4[3].native, srcMisalign*8); b4[3].native = __funnelshift_r(b4[3].native, b4_4.native, srcMisalign*8); } if (Multimem) multimem_st_global<16>(dstAddr, b16); else st_global<16>(dstAddr, b16); srcAddr += WARP_SIZE*16; dstAddr += WARP_SIZE*16; nMiddleBytes -= WARP_SIZE*16; } } #endif nccl-2.29.7-1/src/device/primitives.h000066400000000000000000000147001515037102200172600ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_PRIMITIVES_H_ #define NCCL_PRIMITIVES_H_ #include #include "reduce_kernel.h" // for reduction funcs #include "common_kernel.h" #include "common.h" #define NCCL_SPINS_BEFORE_CHECK_ABORT 10000 /* Protocol classes: ProtoSimple, ProtoLL, ProtoLL128 * We use these as template args to the Primtiives class instead of integral * enums (e.g. NCCL_PROTO_LL) because for SIMPLE we need to carry a few extra * numbers. Also these types hold methods which let us compute numbers important * to how that protocol operates with a consistent interface so that our * algorithm code can operate protocol parametrically. */ template struct ProtoSimple { static constexpr int Id = NCCL_PROTO_SIMPLE; static constexpr int SlicePerChunk = SlicePerChunk_1; static constexpr int StepPerSlice = StepPerSlice_1; static constexpr int Unroll = Unroll_1; static constexpr int MultimemSrcs = MultimemSrcs_1; static constexpr int MultimemDsts = MultimemDsts_1; // Data bytes (no flags etc) in one step of the fifo queue. __device__ static int calcBytePerStep() { return ncclShmem.comm.buffSizes[NCCL_PROTO_SIMPLE]/NCCL_STEPS; } // Granularity of data bytes transferred per thread. __device__ static int calcBytePerGrain() { return sizeof(uint64_t); // Bogus value? Nobody queries this metric for simple. } // Group width is how many consecutive group values a subchannel occupies. static constexpr int MaxGroupWidth = 2; }; struct ProtoLL { static constexpr int Id = NCCL_PROTO_LL; // Data bytes (no flags etc) in one step of the fifo queue. __device__ static int calcBytePerStep() { return ncclShmem.comm.buffSizes[NCCL_PROTO_LL]/NCCL_STEPS/2; // Half is data } // Granularity of data bytes transferred per thread. __device__ static int calcBytePerGrain() { return sizeof(uint64_t); // One 16-byte line has 8-bytes of data } // Group width is how many consecutive group values a subchannel occupies. static constexpr int MaxGroupWidth = 1; }; struct ProtoLL128 { static constexpr int Id = NCCL_PROTO_LL128; // Data bytes (no flags etc) in one step of the fifo queue. __device__ static int calcBytePerStep() { return (ncclShmem.comm.buffSizes[NCCL_PROTO_LL128]/NCCL_STEPS)*NCCL_LL128_DATAELEMS/NCCL_LL128_LINEELEMS; } // Granularity of data bytes transferred per thread. __device__ static int calcBytePerGrain() { return NCCL_LL128_SHMEM_ELEMS_PER_THREAD*NCCL_LL128_DATAELEMS*sizeof(uint64_t)/NCCL_LL128_LINEELEMS; } // Group width is how many consecutive group values a subchannel occupies. static constexpr int MaxGroupWidth = 1; }; /* Fan (as in fan-in & fan-out) classes hold recv and send counts. The template * arguments are static bounds on the maximum values. Asymmetric counts are * independent. Symmetric is a static guarantee that nrecv==nsend, so it only * stores one value at runtime. This optimization save 32-bit register, but more * importantly uses fewer predicate registers when unrolling loops. */ template struct FanAsymmetric { static constexpr int MaxRecv = MaxRecv_, MaxSend = MaxSend_; int nr, ns; FanAsymmetric() = default; __device__ FanAsymmetric(int nrecv, int nsend): nr(nrecv), ns(nsend) { // assert(nrecv <= MaxRecv && nsend <= MaxSend); } __device__ int nrecv() const { return MaxRecv ? nr : 0; } __device__ int nsend() const { return MaxSend ? ns : 0; } }; template struct FanSymmetric { static constexpr int MaxRecv = MaxArity, MaxSend = MaxArity; int n; FanSymmetric() = default; __device__ FanSymmetric(int nrecv, int nsend): n(nrecv) { // assert(nrecv == nsend && nrecv <= MaxArity); } __device__ int nrecv() const { return n; } __device__ int nsend() const { return n; } }; // The primitives class. Specialized per protocol in the other headers. template class Primitives; // Used by LL & LL128 to implement direct members in the naive way. template struct PrimitivesWithoutDirect { __device__ void directSend(intptr_t inpIx, intptr_t outIx, int eltN) { static_cast(this)->send(inpIx, eltN); } __device__ void directSendFromOutput(intptr_t outIx, int eltN) { static_cast(this)->sendFromOutput(outIx, eltN); } __device__ void directRecv(intptr_t outIx, int eltN) { static_cast(this)->recv(outIx, eltN, /*postOp=*/false); } __device__ void directCopySend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { static_cast(this)->copySend(inpIx, outIx, eltN, postOp); } __device__ void directRecvCopyDirectSend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { static_cast(this)->recvCopySend(outIx, eltN, /*postOp=*/false); } __device__ void directRecvDirectSend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { return; } __device__ void recvReduceCopyDirectSend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { // Direct is only for the send part static_cast(this)->recvReduceCopySend(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void directRecvReduceDirectSend(intptr_t inpIx, intptr_t outIx, ssize_t eltN, bool postOp=false) { static_cast(this)->recvReduceSend(inpIx, eltN); } __device__ __forceinline__ void directRecvReduceCopyDirectSend(intptr_t inpIx, intptr_t outIx, ssize_t eltN, bool postOp=false) { static_cast(this)->recvReduceCopySend(inpIx, outIx, eltN, postOp); } }; __device__ inline int checkAbort(int &abortCache, const int abortValue, int &spins) { if (abortCache & abortValue) return 1; if (++spins < NCCL_SPINS_BEFORE_CHECK_ABORT) return 0; spins = 0; int abort = *ncclShmem.comm.abortFlag; if (abort) { ncclShmem.aborted = abort; abortCache |= abortValue; } return abort; } #include "prims_simple.h" #include "prims_ll.h" #include "prims_ll128.h" #endif nccl-2.29.7-1/src/device/prims_ll.h000066400000000000000000000354021515037102200167100ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ template class Primitives: public PrimitivesWithoutDirect> { // In the case of Fan::MaxRecv == 0, we need to force MaxRecv to 1 for this to compile // This is because of a recv buffer which is allocated to MaxRecv length in send-only cases static constexpr int MaxRecv = Fan::MaxRecv > 1 ? Fan::MaxRecv : 1; static constexpr int MaxSend = Fan::MaxSend; static constexpr int Input=0, Output=1; RedOp redOp; const int tid; const int nthreads; const int wid; const int group; const int stepLines; Fan fan; T *userBufs[2]; struct ncclConnInfo* recvConn = NULL; volatile uint64_t* recvConnHeadPtr = NULL; uint64_t recvConnHead; struct ncclConnInfo* sendConn = NULL; volatile struct ncclConnFifo* sendConnFifo = NULL; volatile uint64_t* sendConnHeadPtr = NULL; uint64_t sendConnHead; uint64_t sendConnHeadCache; // Cache last seen value uint64_t recvStep[MaxRecv]; uint64_t sendStep[MaxSend]; union ncclLLFifoLine* recvBuff[MaxRecv]; union ncclLLFifoLine* sendBuff[MaxSend]; inline __device__ int recvOffset(int i) { return (recvStep[i]%NCCL_STEPS)*stepLines; } inline __device__ int sendOffset(int i) { return (sendStep[i]%NCCL_STEPS)*stepLines; } inline __device__ union ncclLLFifoLine* recvPtr(int i) { return recvBuff[i]+recvOffset(i); } inline __device__ union ncclLLFifoLine* sendPtr(int i) { return sendBuff[i]+sendOffset(i); } inline __device__ uint32_t recvFlag(int i) { return NCCL_LL_FLAG(recvStep[i]+1); } inline __device__ uint32_t sendFlag(int i) { return NCCL_LL_FLAG(sendStep[i]+1); } inline __device__ void barrier() { if (nthreads == WARP_SIZE) { __syncwarp(); } else { barrier_sync(15-group, nthreads); } } int abort = 0; inline __device__ void waitSend(int nbytes) { if (sendConnHeadPtr) { int spins = 0; while (sendConnHeadCache + NCCL_STEPS < sendConnHead + 1) { sendConnHeadCache = *sendConnHeadPtr; if (checkAbort(abort, 1, spins)) break; } if (sendConnFifo) { int size = ((sendConnHead & NCCL_LL_CLEAN_MASK) == NCCL_LL_CLEAN_MASK) ? stepLines*sizeof(union ncclLLFifoLine) : nbytes; sendConnFifo[sendConnHead%NCCL_STEPS].size = size; } sendConnHead += 1; } barrier(); } inline __device__ void incRecv(int i) { recvStep[i] += 1; } inline __device__ void postRecv() { barrier(); if (recvConnHeadPtr) *recvConnHeadPtr = recvConnHead += 1; } inline __device__ void incSend(int i, int offset) { // LL Cleanup : write all flags in the slice to make sure we don't have // data corruption when flag loops over. if ((sendStep[i] & NCCL_LL_CLEAN_MASK) == NCCL_LL_CLEAN_MASK) { for (int o = offset; oi4) : "memory"); if (checkAbort(abort, 1, spins)) break; } while ((flag1 != flag) || (flag2 != flag)); uint64_t val64 = data1 + (((uint64_t)data2) << 32); return val64; } template __device__ void readLLBeginAll(int offset, ncclLLFifoLine(&line)[MaxRecv]) { #pragma unroll for (int i=BeginIx; i < MaxRecv; i++) { // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] if (i < fan.nrecv()) { union ncclLLFifoLine* src = recvPtr(i) + offset; asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" : "=r"(line[i].data1), "=r"(line[i].flag1), "=r"(line[i].data2), "=r"(line[i].flag2) : "l"(&src->i4) : "memory"); } } } __device__ uint64_t readLLFinish(int offset, ncclLLFifoLine(&line)[MaxRecv], int i) { union ncclLLFifoLine* src = recvPtr(i) + offset; uint32_t flag = recvFlag(i); int spins = 0; while (line[i].flag1 != flag || line[i].flag2 != flag) { asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" : "=r"(line[i].data1), "=r"(line[i].flag1), "=r"(line[i].data2), "=r"(line[i].flag2) : "l"(&src->i4) : "memory"); if (checkAbort(abort, 1, spins)) break; } uint64_t val64 = line[i].data1 + (((uint64_t)line[i].data2) << 32); return val64; } __device__ void storeLL(union ncclLLFifoLine* dst, uint64_t val, uint32_t flag) { asm volatile("st.volatile.global.v4.u32 [%0], {%1,%2,%3,%4};" :: "l"(&dst->i4), "r"((uint32_t)val), "r"(flag), "r"((uint32_t)(val >> 32)), "r"(flag) : "memory"); } static constexpr int EltPerLine = sizeof(uint64_t)/sizeof(T); template __device__ static U load(U *src) { union { U elt; uint16_t u2; uint32_t u4; uint64_t u8; }; if(sizeof(U) == 1) asm volatile("ld.volatile.global.b8 %0,[%1];" : "=r"(u4) : "l"(src) : "memory"); else if(sizeof(U) == 2) asm volatile("ld.volatile.global.b16 %0,[%1];" : "=h"(u2) : "l"(src) : "memory"); else if(sizeof(U) == 4) asm volatile("ld.volatile.global.b32 %0,[%1];" : "=r"(u4) : "l"(src) : "memory"); else asm volatile("ld.volatile.global.b64 %0,[%1];" : "=l"(u8) : "l"(src) : "memory"); return elt; } template __device__ static void store(U *dst, U val) { union { U elt; uint16_t u2; uint32_t u4; uint64_t u8; }; elt = val; if(sizeof(U) == 1) asm volatile("st.volatile.global.b8 [%0],%1;" :: "l"(dst), "r"(u4) : "memory"); else if(sizeof(U) == 2) asm volatile("st.volatile.global.b16 [%0],%1;" :: "l"(dst), "h"(u2) : "memory"); else if(sizeof(U) == 4) asm volatile("st.volatile.global.b32 [%0],%1;" :: "l"(dst), "r"(u4) : "memory"); else asm volatile("st.volatile.global.b64 [%0],%1;" :: "l"(dst), "l"(u8) : "memory"); } struct DataLoader { int misalign; union { uint32_t u4[sizeof(T) <= 2 ? 3 : 2]; uint64_t u8; T elt[EltPerLine]; }; __device__ void loadBegin(T *src, int eltN) { if (sizeof(T) <= 2) { misalign = reinterpret_cast(src)%4; uint32_t *p = reinterpret_cast(reinterpret_cast(src) & -uintptr_t(4)); u4[0] = load(p+0); u4[1] = misalign + eltN*sizeof(T) > 4 ? load(p+1) : 0; // u4[2] would be simpler, but that throws warnings on some compilers u4[sizeof(T) <= 2 ? 2 : 0] = misalign + eltN*sizeof(T) > 8 ? load(p+2) : 0; } else { #pragma unroll for(int i=0; i < EltPerLine; i++) { // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] if(i==0 || i < eltN) elt[i] = load(src + i); } } } __device__ uint64_t loadFinish() { if (sizeof(T) <= 2) { u4[0] = __funnelshift_r(u4[0], u4[1], 8*misalign); // u4[2] would be simpler, but that throws warnings on some compilers u4[1] = __funnelshift_r(u4[1], u4[sizeof(T) <= 2 ? 2 : 0], 8*misalign); } return u8; } }; __device__ void storeData(T *dst, uint64_t val, int eltN) { union { uint64_t u8; T elt[EltPerLine]; }; u8 = val; #pragma unroll for(int i=0; i < EltPerLine; i++) { // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] if (i==0 || i < eltN) //store(dst+i, elt[i]); dst[i] = elt[i]; } } template __device__ __forceinline__ void LLGenericOp(intptr_t srcIx, intptr_t dstIx, int nelem, bool postOp) { constexpr int SRC = SrcBuf != -1 ? 1 : 0; constexpr int DST = DstBuf != -1 ? 1 : 0; T *srcElts = SrcBuf == -1 ? nullptr : userBufs[SrcBuf] + srcIx; T *dstElts = DstBuf == -1 ? nullptr : userBufs[DstBuf] + dstIx; // Always waitSend in case of cleanup nelem = nelem < 0 ? 0 : nelem; if (SEND) waitSend(divUp(nelem, EltPerLine)*sizeof(ncclLLFifoLine)); nelem -= tid*EltPerLine; srcElts += tid*EltPerLine; dstElts += tid*EltPerLine; int offset = tid; int eltPerTrip = nthreads*EltPerLine; while (nelem > 0) { int eltInLine = EltPerLine < nelem ? EltPerLine : nelem; DataLoader dl; ncclLLFifoLine line[MaxRecv]; uint64_t data, peerData; if (SRC) { dl.loadBegin(srcElts, eltInLine); srcElts += eltPerTrip; } if (RECV) { readLLBeginAll<1>(offset, line); peerData = readLL(offset, 0); } if (SRC) { data = dl.loadFinish(); if (SrcBuf == Input) data = applyPreOp(redOp, data); } if (RECV) { data = !SRC ? peerData : applyReduce(redOp, peerData, data); #pragma unroll MaxRecv // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] for (int i=1; i < MaxRecv && i < fan.nrecv(); i++) { peerData = readLLFinish(offset, line, i); data = applyReduce(redOp, peerData, data); } } if (postOp) data = applyPostOp(redOp, data); // Send : inter-node, then intra-node, then local if (SEND) { // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] for (int i=1; i < MaxSend && i < fan.nsend(); i++) storeLL(sendPtr(i)+offset, data, sendFlag(i)); storeLL(sendPtr(0)+offset, data, sendFlag(0)); } if (DST) { storeData(dstElts, data, eltInLine); dstElts += eltPerTrip; } nelem -= eltPerTrip; offset += nthreads; } if (RECV) { for (int i=0; i < MaxRecv; i++) incRecv(i); postRecv(); } if (SEND) { // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] for (int i=1; i < MaxSend && i < fan.nsend(); i++) incSend(i, offset); incSend(0, offset); } } __device__ __forceinline__ void loadRecvConn(struct ncclConnInfo* conn, int i) { recvBuff[i] = (union ncclLLFifoLine*)conn->buffs[NCCL_PROTO_LL]; recvStep[i] = conn->step; if (wid == i) recvConn = conn; } __device__ __forceinline__ void loadRecvSync() { if (tid >= nthreads-WARP_SIZE && wid < fan.nrecv()) { recvConnHeadPtr = recvConn->head; recvConnHead = recvConn->step; } } __device__ __forceinline__ void loadSendConn(struct ncclConnInfo* conn, int i) { sendBuff[i] = (union ncclLLFifoLine*)conn->buffs[NCCL_PROTO_LL]; sendStep[i] = conn->step; if (wid == i) sendConn = conn; } __device__ __forceinline__ void loadSendSync() { if (tid < fan.nsend()) { sendConnHeadPtr = sendConn->head; sendConnHeadCache = *sendConnHeadPtr; sendConnHead = sendConn->step; sendConnFifo = sendConn->connFifo; } } public: __device__ Primitives( const int tid, const int nthreads, int const *recvPeers, int const *sendPeers, void const *inputBuf, void *outputBuf, uint64_t redOpArg, uint8_t group=0, uint8_t connIndexRecv=0, uint8_t connIndexSend=0, struct ncclDevWorkColl* e = nullptr, bool ipcReg = false, bool netReg = false, int stepSize_ = 0 ): redOp(redOpArg), tid(tid), nthreads(nthreads), wid(tid%WARP_SIZE), group(group), stepLines(ncclShmem.comm.buffSizes[NCCL_PROTO_LL]/NCCL_STEPS/sizeof(ncclLLFifoLine)) { auto *channel = &ncclShmem.channel; // If we are going to support oneshot collNet + LL, then we would need to add connector index here int nrecv=0, nsend=0; // We compare with Fan::MaxRecv here because this->MaxRecv is always at least 1 // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] while (nrecv < Fan::MaxRecv && recvPeers[nrecv] >= 0) { loadRecvConn(&channel->peers[recvPeers[nrecv]]->recv[connIndexRecv], nrecv); nrecv++; } // coverity[dead_error_line] while (nsend < MaxSend && sendPeers[nsend] >= 0) { loadSendConn(&channel->peers[sendPeers[nsend]]->send[connIndexSend], nsend); nsend++; } this->fan = Fan(nrecv, nsend); // Coverity reports recvConn and sendConn being possibly NULL at this point but that won't actually // happen given the two "while" loops just above. // coverity[var_deref_model:FALSE] loadRecvSync(); // coverity[var_deref_model:FALSE] loadSendSync(); setDataPtrs(inputBuf, outputBuf); } __device__ ~Primitives() { // Save steps for the next operation if (tid >= nthreads-WARP_SIZE && wid < fan.nrecv()) recvConn->step = recvConnHead; if (tid < fan.nsend()) sendConn->step = sendConnHead; // Ensure all steps written back barrier(); } __device__ void setDataPtrs(void const *inputBuf, void *outputBuf) { userBufs[Input] = (T*)inputBuf; userBufs[Output] = (T*)outputBuf; } __device__ void moveDataPtrs(intptr_t delta) { userBufs[Input] += delta; userBufs[Output] += delta; } __device__ void send(intptr_t inpIx, int eltN) { return LLGenericOp<0, 1, Input, -1>(inpIx, -1, eltN, false); } __device__ void sendFromOutput(intptr_t outIx, int eltN) { return LLGenericOp<0, 1, Output, -1>(outIx, -1, eltN, false); } __device__ void recv(intptr_t outIx, int eltN, bool postOp=false) { return LLGenericOp<1, 0, -1, Output>(-1, outIx, eltN, postOp); } __device__ void recvReduceSend(intptr_t inpIx, int eltN) { return LLGenericOp<1, 1, Input, -1>(inpIx, -1, eltN, false); } __device__ void recvReduceCopy(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { return LLGenericOp<1, 0, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ void copySend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { return LLGenericOp<0, 1, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ void recvCopySend(intptr_t outIx, int eltN, bool postOp=false) { return LLGenericOp<1, 1, -1, Output>(-1, outIx, eltN, postOp); } __device__ void recvReduceCopySend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { return LLGenericOp<1, 1, Input, Output>(inpIx, outIx, eltN, postOp); } }; nccl-2.29.7-1/src/device/prims_ll128.h000066400000000000000000000400101515037102200171320ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "op128.h" #define NCCL_LL128_FLAGTHREAD (NCCL_LL128_LINEELEMS-1) template class Primitives: public PrimitivesWithoutDirect> { static constexpr int MaxRecv = Fan::MaxRecv, MaxSend = Fan::MaxSend; static constexpr int Input=0, Output=1; RedOp redOp; const int tid; // thread index in primitives group const int nthreads; // thread count in primitives group const int wid; // lane index in warp const int stepSize; const int warp; // warp index in primitives group const int warpInBlock; // warp index in thread block const bool flagThread; const int group; Fan fan; T *userBufs[2]; struct ncclConnInfo* recvConn = NULL; volatile uint64_t* recvConnHeadPtr = NULL; uint64_t recvConnHead; struct ncclConnInfo* sendConn = NULL; volatile struct ncclConnFifo* sendConnFifo = NULL; volatile uint64_t* sendConnTailPtr = NULL; uint64_t sendConnTail; volatile uint64_t* sendConnHeadPtr = NULL; uint64_t sendConnHead; uint64_t sendConnHeadCache; // Cache last seen value uint64_t recvStep[MaxRecv]; uint64_t sendStep[MaxSend]; uint64_t* recvBuff[MaxRecv]; uint64_t* sendBuff[MaxSend]; inline __device__ int recvOffset(int i) { return (recvStep[i]%NCCL_STEPS)*stepSize; } inline __device__ int sendOffset(int i) { return (sendStep[i]%NCCL_STEPS)*stepSize; } inline __device__ uint64_t* recvPtr(int i) { return recvBuff[i]+recvOffset(i); } inline __device__ uint64_t* sendPtr(int i) { return sendBuff[i]+sendOffset(i); } inline __device__ uint64_t recvFlag(int i) { return recvStep[i]+1; } inline __device__ uint64_t sendFlag(int i) { return sendStep[i]+1; } inline __device__ void barrier() { barrier_sync(15-group, nthreads); } int abort = 0; inline __device__ void waitSend(int nbytes) { if (sendConnHeadPtr) { int spins = 0; while (sendConnHeadCache + NCCL_STEPS < sendConnHead + 1) { sendConnHeadCache = *sendConnHeadPtr; if (checkAbort(abort, 1, spins)) break; } if (sendConnFifo) { sendConnFifo[sendStep[wid]%NCCL_STEPS].size = nbytes; } sendConnHead += 1; } } inline __device__ void postRecv() { if (recvConnHeadPtr) *recvConnHeadPtr = recvConnHead += 1; } inline __device__ void postSend() { if (sendConnTailPtr) { #if __CUDA_ARCH__ >= 900 __threadfence_system(); #else __threadfence(); #endif *sendConnTailPtr = sendConnTail += 1; } } template __device__ __forceinline__ void loadRegsBegin(uint64_t(®s)[WordPerThread], T const *src, int eltN) { constexpr int EltPer16B = 16/sizeof(T); if(reinterpret_cast(src)%16 == 0) { /* We are aligned to 16 bytes, so load directly to registers no shmem. * Flag threads load half as much data which gets shuffled to the even * registers during Finish. The point of splitting into two phases is to * defer that shuffle, which incurs a dependency stall, until after other * memops are launched by the caller. */ #pragma unroll for(int g=0; g < WordPerThread/2; g++) { int ix = g*WARP_SIZE - 4*(g/2) + wid - (g%2)*(wid/8); if(!flagThread || g%2==0) { if(ix*EltPer16B < eltN) load128((uint64_t*)(src + ix*EltPer16B), regs[2*g+0], regs[2*g+1]); } } } else { // Not aligned. Stage the smallest 16 byte aligned region subsuming the // buffer into shmem. int misalignment = reinterpret_cast(src) % 16; uint64_t *src8 = reinterpret_cast(reinterpret_cast(src) & -uintptr_t(16)); uint64_t *shm8 = shmemCvtPtr((uint64_t*)ncclScratchForWarp(warpInBlock)); #pragma unroll for(int g=0; g < WordPerThread/2; g++) if((g*WARP_SIZE + wid)*16 < misalignment + eltN*sizeof(T)) load128(src8 + 2*(g*WARP_SIZE + wid), regs[2*g+0], regs[2*g+1]); #pragma unroll for(int g=0; g < WordPerThread/2; g++) storeShmem128(shm8 + 2*(g*WARP_SIZE + wid), regs[2*g+0], regs[2*g+1]); __syncwarp(); // Now load from shmem stage to regs. Preserve the same pre-shuffled layout // as the aligned case since Finish() will be applied regardless. T *shm = (T*)shm8 + misalignment/sizeof(T); #pragma unroll for(int g=0; g < WordPerThread/2; g++) { int ix = g*WARP_SIZE - 4*(g/2) + wid - (g%2)*(wid/8); if(!flagThread || g%2==0) { if(ix*EltPer16B < eltN) loadShmemMisaligned128(shm + ix*EltPer16B, regs[2*g+0], regs[2*g+1]); } } } } template __device__ __forceinline__ void loadRegsFinish(uint64_t(®s)[WordPerThread]) { // Move data out of flag registers into the vacant registers. #pragma unroll for (int g=1; g < WordPerThread/2; g+=2) { if (flagThread) regs[2*g] = regs[2*g-1]; } } template __device__ __forceinline__ void storeRegs(T *dst, uint64_t(®s)[WordPerThread], int eltN) { constexpr int EltPer16B = 16/sizeof(T); // Reverse Finish() register permuatation. #pragma unroll for (int g=1; g < WordPerThread/2; g+=2) { if (flagThread) regs[2*g-1] = regs[2*g]; } // Write to dst if 16-byte aligned, shmem otherwise. int misalignment = reinterpret_cast(dst)%16; uint64_t *shm8 = shmemCvtPtr((uint64_t*)ncclScratchForWarp(warpInBlock)); #pragma unroll for(int g=0; g < WordPerThread/2; g++) { int ix = g*WARP_SIZE - 4*(g/2) + wid - (g%2)*(wid/8); if (!flagThread || g%2==0) { if(misalignment == 0 && (ix+1)*EltPer16B <= eltN) store128((uint64_t*)(dst + ix*EltPer16B), regs[2*g+0], regs[2*g+1]); else storeShmem128(shm8+2*ix, regs[2*g+0], regs[2*g+1]); } } __syncwarp(); // Write rest from shmem to dst. No need to coalesce stores to 16-bytes, // the hardware keeps up fine. T *shm = (T*)ncclScratchForWarp(warpInBlock); int skip = misalignment == 0 ? eltN & -EltPer16B : 0; for(int i=skip+wid; i < eltN; i += WARP_SIZE) dst[i] = shm[i]; } #define WARP_MASK 0xffffffff template __device__ __forceinline__ void recvReduceSendCopy(uint64_t(&v)[ELEMS_PER_THREAD], int ll128Offset, bool postOp) { constexpr int SRC = SrcBuf != -1 ? 1 : 0; uint64_t vr[ELEMS_PER_THREAD]; __syncwarp(); /************************ Wait first recv ********************/ if (RECV) { uint64_t* ptr = recvPtr(0)+ll128Offset; uint64_t flag = recvFlag(0); bool needReload; int spins = 0; do { needReload = false; #pragma unroll for (int u=0; u __device__ __forceinline__ void GenericOp(intptr_t srcIx, intptr_t dstIx, int nelem, bool postOp) { constexpr int SRC = SrcBuf != -1 ? 1 : 0; constexpr int DST = DstBuf != -1 ? 1 : 0; T const *srcPtr = SrcBuf == -1 ? nullptr : userBufs[SrcBuf] + srcIx; T *dstPtr = DstBuf == -1 ? nullptr : userBufs[DstBuf] + dstIx; int wireOffset = WireWordPerSlice*warp + 2*wid; const int nwarps = nthreads/WARP_SIZE; nelem = nelem < 0 ? 0 : nelem; if (SEND) waitSend(divUp(nelem, DataEltPerSlice)*WireWordPerSlice*sizeof(uint64_t)); barrier(); nelem -= DataEltPerSlice*warp; srcPtr += DataEltPerSlice*warp; dstPtr += DataEltPerSlice*warp; while (nelem > 0) { const int eltInSlice = min(nelem, DataEltPerSlice); uint64_t regs[NCCL_LL128_SHMEM_ELEMS_PER_THREAD]; if (SRC) loadRegsBegin(regs, srcPtr, eltInSlice); recvReduceSendCopy(regs, wireOffset, postOp); if (DST) storeRegs(dstPtr, regs, eltInSlice); wireOffset += WireWordPerSlice*nwarps; srcPtr += DataEltPerSlice*nwarps; dstPtr += DataEltPerSlice*nwarps; nelem -= DataEltPerSlice*nwarps; } barrier(); if (SEND) for (int i=0; i < MaxSend; i++) sendStep[i] += 1; if (SEND) postSend(); if (RECV) for (int i=0; i < MaxRecv; i++) recvStep[i] += 1; if (RECV) postRecv(); } __device__ __forceinline__ void loadRecvConn(struct ncclConnInfo* conn, int i) { recvBuff[i] = (uint64_t*)conn->buffs[NCCL_PROTO_LL128]; recvStep[i] = conn->step; if (wid == i) recvConn = conn; } __device__ __forceinline__ void loadRecvSync() { if (tid >= nthreads-WARP_SIZE && wid < fan.nrecv()) { recvConnHeadPtr = recvConn->head; recvConnHead = recvConn->step; } } __device__ __forceinline__ void loadSendConn(struct ncclConnInfo* conn, int i) { sendBuff[i] = (uint64_t*)conn->buffs[NCCL_PROTO_LL128]; sendStep[i] = conn->step; if (wid == i) sendConn = conn; } __device__ __forceinline__ void loadSendSync() { if (tid < fan.nsend()) { sendConnHeadPtr = sendConn->head; sendConnHeadCache = *sendConnHeadPtr; sendConnHead = sendConn->step; sendConnFifo = sendConn->connFifo; } if (tid >= nthreads-WARP_SIZE && widconnFifo) { sendConnTailPtr = sendConn->tail; sendConnTail = sendConn->step; } } } public: __device__ Primitives( const int tid, const int nthreads, int const *recvPeers, int const *sendPeers, void const *inputBuf, void *outputBuf, uint64_t redOpArg, uint8_t group=0, uint8_t connIndexRecv=0, uint8_t connIndexSend=0, struct ncclDevWorkColl* e = nullptr, bool ipcReg = false, bool netReg = false, int stepSize_ = 0 ): redOp(redOpArg), tid(tid), nthreads(nthreads), wid(tid%WARP_SIZE), warp(tid/WARP_SIZE), warpInBlock(threadIdx.x/WARP_SIZE), flagThread((tid%8)==7), group(group), stepSize(ncclShmem.comm.buffSizes[NCCL_PROTO_LL128]/NCCL_STEPS/sizeof(uint64_t)) { auto *channel = &ncclShmem.channel; int nrecv=0, nsend=0; while (nrecv < MaxRecv && recvPeers[nrecv] >= 0) { loadRecvConn(&channel->peers[recvPeers[nrecv]]->recv[connIndexRecv], nrecv); nrecv++; } while (nsend < MaxSend && sendPeers[nsend] >= 0) { loadSendConn(&channel->peers[sendPeers[nsend]]->send[connIndexSend], nsend); nsend++; } this->fan = Fan(nrecv, nsend); // Coverity reports recvConn and sendConn being possibly NULL at this point but that won't actually // happen given the two "while" loops just above. // coverity[var_deref_model:FALSE] loadRecvSync(); // coverity[var_deref_model:FALSE] loadSendSync(); setDataPtrs(inputBuf, outputBuf); } __device__ ~Primitives() { // Save steps for the next operation if (tid >= nthreads-WARP_SIZE && wid < fan.nrecv()) recvConn->step = recvConnHead; if (tid < fan.nsend()) sendConn->step = sendConnHead; // Ensure all steps written back barrier(); } __device__ void setDataPtrs(void const *inputBuf, void *outputBuf) { userBufs[Input] = (T*)inputBuf; userBufs[Output] = (T*)outputBuf; } __device__ void moveDataPtrs(intptr_t delta) { userBufs[Input] += delta; userBufs[Output] += delta; } __device__ void send(intptr_t inpIx, int eltN) { return GenericOp<0, 1, Input, -1>(inpIx, -1, eltN, false); } __device__ void sendFromOutput(intptr_t outIx, int eltN) { return GenericOp<0, 1, Output, -1>(outIx, -1, eltN, false); } __device__ void recv(intptr_t outIx, int eltN, bool postOp=false) { return GenericOp<1, 0, -1, Output>(-1, outIx, eltN, postOp); } __device__ void recvReduceSend(intptr_t inpIx, int eltN) { return GenericOp<1, 1, Input, -1>(inpIx, -1, eltN, false); } __device__ void recvReduceCopy(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { return GenericOp<1, 0, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ void copySend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { return GenericOp<0, 1, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ void recvCopySend(intptr_t outIx, int eltN, bool postOp=false) { return GenericOp<1, 1, -1, Output>(-1, outIx, eltN, postOp); } __device__ void recvReduceCopySend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { return GenericOp<1, 1, Input, Output>(inpIx, outIx, eltN, postOp); } }; nccl-2.29.7-1/src/device/prims_simple.h000066400000000000000000001434651515037102200176030ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "network/unpack/unpack.h" #include enum primsMode { primsModeDefault = 0, primsModePatRs = 1, primsModePatAg = 2 }; template class Primitives< T, RedOp, Fan, Direct, ProtoSimple, P2p, isNetOffload > { static constexpr int MaxRecv = Fan::MaxRecv, MaxSend = Fan::MaxSend; static constexpr int Input=0, Output=1; static constexpr int RoleInput = 0x01, RoleOutput = 0x02, RoleWaitRecv = 0x04, RoleWaitSend = 0x08, RolePostSend = 0x10, RolePostRecv = 0x20, Aborted = 0x40, NetRegMode = 0x80, ConnFifoEnabled = 0x100, DirectWrite = 0x200, DirectRead = 0x400, PatMode = 0x800, NvlsMinPolling = 0x1000, NetDeviceUnpack = 0x2000, AnyNetDeviceUnpack = 0x4000; const int tid, tidInBlock; const int nthreads; int nworkers; const int stepSize; Fan fan; int index; // Peer index I'm responsible for int flags; int group; uint64_t step; struct ncclConnInfo* conn = NULL; struct ncclConnFifo* connFifo = NULL; T* connEltsFifo; T* directBuff = NULL; uint64_t *connStepPtr; uint64_t connStepCache; // Cache last seen value of (*connStepPtr) int connStepSize; // Connection step size void* netDeviceHandle; uint64_t accSize; // Don't use barrier 0 as it's used by the final sync __device__ void barrier() { if (nthreads == WARP_SIZE) __syncwarp(); else { int bar = 15-group; barrier_sync(bar, nthreads); } } __device__ void subBarrier() { if (nworkers == WARP_SIZE) __syncwarp(); else { int bar = 15-group - (nworkers!=nthreads ? 1 : 0); barrier_sync(bar, nworkers); } } // PAT uses a single barrier across all groups __device__ void patBarrier() { barrier_sync(15, NCCL_PAT_NWORKERS); } __device__ bool barrierAny(int vote) { if (nthreads == WARP_SIZE) { return __any_sync(~0u, vote); } else { int name = 15-group; return barrier_red_or(vote, name, nthreads); } } __device__ bool subBarrierAny(int vote) { if (nworkers == WARP_SIZE) { return __any_sync(~0u, vote); } else { int name = 15-group - (nworkers!=nthreads ? 1 : 0); return barrier_red_or(vote, name, nworkers); } } inline __device__ uint64_t loadStepValue(uint64_t* ptr) { #if __CUDA_ARCH__ >= 900 && CUDART_VERSION >= 12010 if (flags & NvlsMinPolling) { uint64_t ans; asm volatile("multimem.ld_reduce.acquire.sys.global.min.u64 %0, [%1];" : "=l"(ans) : "l"(cvta_to_global(ptr)) : "memory"); return ans; } #endif // volatile is faster than acquire but not as correct. Make sure reduceCopy // loads data using volatile so it doesn't see stale data in L1. return ld_volatile_global(ptr); } template __device__ __forceinline__ void waitPeer(intptr_t srcIx, intptr_t dstIx, int offset, int nelts) { const bool isSendNotRecv = (Send && Recv) ? (flags & RoleWaitSend) : Send; // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] if ((flags & (Recv * RoleWaitRecv)) || (flags & (Send * RoleWaitSend))) { int spins = 0; while (connStepCache + (isSendNotRecv ? NCCL_STEPS : 0) < step + StepPerSlice) { connStepCache = loadStepValue(connStepPtr); if (checkAbort(flags, Aborted, spins)) break; //if (spins == 0) printf("r=%d b=%d t=%d SPUN OUT got=%d want=%d\n", ncclShmem.comm.rank, blockIdx.x, threadIdx.x, int(connStepCache + (isSendNotRecv ? NCCL_STEPS : 0)), int(step+StepPerSlice)); } } if (flags & (Recv*RoleWaitRecv | Send*RoleWaitSend)) { if ((flags & ConnFifoEnabled) && (flags & (Send * RoleWaitSend))) connFifo[step%NCCL_STEPS].size = nelts*sizeof(T); void **ptrs = isSendNotRecv ? (ncclShmem.groups[group].dsts + Dst) : (ncclShmem.groups[group].srcs + Src); if ((flags & NetRegMode) && ((!isSendNotRecv && DirectRecv) || (isSendNotRecv && DirectSend))) { if (P2p) { ptrs[index] = NULL; } else { if (isSendNotRecv) { if (!Recv) ptrs[index] = NULL; else ptrs[index] = (T*)ncclShmem.groups[group].userOutput + dstIx + offset; } else { ptrs[index] = (T*)ncclShmem.groups[group].userOutput + srcIx + offset; } } } else if ((flags & ConnFifoEnabled) && connFifo[step%NCCL_STEPS].mode == NCCL_MODE_OFFSET) { ptrs[index] = connEltsFifo + loadInt(&connFifo[step%NCCL_STEPS].offset)/sizeof(T); } else if (isSendNotRecv && DirectSend) { if (flags & DirectWrite) { ptrs[index] = directBuff + dstIx + offset; } else if (flags & DirectRead) { // empty send ptrs[index] = nullptr; } else { ptrs[index] = connEltsFifo + (step%NCCL_STEPS)*connStepSize; } } else if (!isSendNotRecv && DirectRecv) { if (flags & DirectRead) { ptrs[index] = directBuff + srcIx + offset; } else if (flags & DirectWrite) { ptrs[index] = directBuff + dstIx + offset; // send to next from my output buffer } else { ptrs[index] = connEltsFifo + (step%NCCL_STEPS)*connStepSize; } } else { // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] ptrs[index] = connEltsFifo + (step%NCCL_STEPS)*connStepSize; } if (flags & NetDeviceUnpack) { ncclNetDeviceIncrementHead(group, index); } step += StepPerSlice; } } template inline __device__ void postPeer(bool dataStored) { if (flags & (Recv*RolePostRecv | Send*RolePostSend)) { step += StepPerSlice; if (Send && (flags & RolePostSend) && (dataStored||(flags&ConnFifoEnabled))) { fence_acq_rel_sys(); } st_relaxed_sys_global(connStepPtr, step); } } template __device__ __forceinline__ void genericOp( intptr_t srcIx, intptr_t dstIx, int nelem, bool postOp ) { constexpr int DirectRecv = 1 && Direct && DirectRecv1; constexpr int DirectSend = 1 && Direct && DirectSend1; constexpr int Src = SrcBuf != -1; constexpr int Dst = DstBuf != -1; nelem = nelem < 0 ? 0 : nelem; int sliceSize = stepSize*StepPerSlice; sliceSize = max(divUp(nelem, 16*SlicePerChunk)*16, sliceSize/32); int slice = 0; int offset = 0; if (tid < nworkers && offset < nelem && !isNetOffload) { // Worker-only loop for non-empty slices. Non-workers and empty slices are // processed in the loop following this if block. The benefit of splitting // the loop like this is we pull two branches out of the critical path. // Using "number of branch insns (taken or not) encountered dynamically" // as the performance metric, then: // perf_orig = 2*numslices // perf_new = 2+numslices // So the new code and old code behave the same for numslices=2, and for // numslices>2 the new code is superior. And note that in the case // numslices=1, the loop is trivially unrollable (single iteration) so we // don't incur that that tail branch and we still have perf_new=2. // // ORIGINAL CODE: // unrolled for(slices) { // if(worker) { // This branch removed // wait(); // subBarrier(); // if(slice not empty) // This branch removed // ReduceCopyMulti(); // } // barrier(); // post(); // } // Since we no longer unroll, new branch added here #if __CUDA_ARCH__ < 700 // Above doesn't matter on older hardware. #pragma unroll SlicePerChunk #else #pragma unroll 1 #endif do { sliceSize = sliceSize < nelem-offset ? sliceSize : nelem-offset; if (tid == 0) { T* userInput = (T*)ncclShmem.groups[group].userInput; T* userOutput = (T*)ncclShmem.groups[group].userOutput; if (Src) ncclShmem.groups[group].srcs[0] = (SrcBuf==Input ? userInput : userOutput) + srcIx + offset; if (Dst) ncclShmem.groups[group].dsts[0] = (DstBuf==Input ? userInput : userOutput) + dstIx + offset; } waitPeer(srcIx, dstIx, offset, sliceSize); subBarrier(); /* if user abort the kernel, we don't need to actually perform copy/reduce; just set size * to 0 to avoid unnecessary workload. */ int workSize = ncclShmem.aborted ? 0 : sliceSize; if (flags & AnyNetDeviceUnpack) { ncclNetDeviceUnpack(tid, tidInBlock, nworkers, group, ncclShmem.groups[group].devicePlugin.unpack.unpackNetDeviceIndexMask, Src, workSize); // Sync here to make sure all workers are reading from the updated srcs) subBarrier(); } if (DirectRecv && ncclShmem.groups[group].srcs[0] == ncclShmem.groups[group].dsts[0] /* NVLS can have srcs[0] == dsts[0], but we cannot enter this "if branch", * so we need to check whether MultimemSrcs and MultimemDsts are 0. */ && MultimemSrcs == 0 && MultimemDsts == 0 && !Src) { // We can only have one direct receive. Since srcs[0] == dstPtr+offset, skip one copy if (Send && Dst && ncclShmem.groups[group].srcs[0] != ncclShmem.groups[group].dsts[1]) { reduceCopy (tid, nworkers, /*redArg*/0, /*postOp*/false, 1, ncclShmem.groups[group].srcs, fan.nsend(), ncclShmem.groups[group].dsts+1, workSize); } } else if (DirectSend && !DirectRecv && SrcBuf != Input && ncclShmem.groups[group].dsts[Dst] == nullptr) { // For broadcast in CollNet to do empty send reduceCopy (tid, nworkers, ncclShmem.groups[group].redOpArgs, postOp, Recv, ncclShmem.groups[group].srcs, Dst, ncclShmem.groups[group].dsts, workSize); } else if (ncclShmem.groups[group].srcs[0] && ncclShmem.groups[group].dsts[0]) { constexpr int PreOpSrcs = SrcBuf != Input ? 0 : 1; if (Send && Dst && ncclShmem.groups[group].dsts[1] == nullptr) { // this case should only be directCopySend() with registered buffers and send to net peer reduceCopy (tid, nworkers, ncclShmem.groups[group].redOpArgs, postOp, Recv * fan.nrecv() + Src, ncclShmem.groups[group].srcs, 1, ncclShmem.groups[group].dsts, workSize); } else { reduceCopy (tid, nworkers, ncclShmem.groups[group].redOpArgs, postOp, Recv * fan.nrecv() + Src, ncclShmem.groups[group].srcs, Send * fan.nsend() + Dst, ncclShmem.groups[group].dsts, workSize); } } else { // we will come here when calling prims.directSend with net peer, // in this case, ncclShmem.groups[group].dsts[0] == NULL, so we // skip data flush. workSize = 0; } barrier(); // This barrier has a counterpart in following loop postPeer(0 < workSize); offset += sliceSize; slice += 1; // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] } while (slice < SlicePerChunk && offset < nelem); } // Non-workers come straight here. Workers too but only once the remaining // slices are all empty. Since empty slices are the uncommon case, and // worker perf is the limiter, perf-wise this loop is effectively unentered, // hence just a single branch insn. #pragma unroll 1 while (slice < SlicePerChunk) { sliceSize = sliceSize < nelem-offset ? sliceSize : nelem-offset; { // Only workers could have Wait roles so we know the slice must be empty // since we've exited the loop above. waitPeer(0, 0, 0, sliceSize); } barrier(); // Has couterpart in preceding worker-only loop. int workSize = ncclShmem.aborted ? 0 : sliceSize; postPeer(0 < workSize); offset += sliceSize; slice += 1; } } public: static inline __device__ void sendPeerNotify(int peer, int connIndex, int steps) { ncclDevChannelPeer* peerPtr = ncclShmem.channel.peers[peer]; peerPtr->send[connIndex].step += steps; st_relaxed_sys_global(peerPtr->send[connIndex].tail, peerPtr->send[connIndex].step); } static inline __device__ void recvPeerNotify(int peer, int connIndex, int steps) { int spins = 0; ncclDevChannelPeer* peerPtr = ncclShmem.channel.peers[peer]; peerPtr->recv[connIndex].step += steps; st_relaxed_sys_global(peerPtr->recv[connIndex].head, peerPtr->recv[connIndex].step); while (ld_volatile_global(peerPtr->recv[connIndex].tail) < peerPtr->recv[connIndex].step) { int abort = 0; if (checkAbort(abort, 1, spins)) break; } } template __device__ __forceinline__ void process(Fn &&fn, uint32_t sendDirectFlag = 0, uint32_t recvDirectFlag = 0) { #pragma unroll 1 for (int slice=0; slice < SlicePerChunk; slice++) { if (tid < nworkers) { int nsend, nrecv; if (flags & (Recv*RoleWaitRecv | Send*RoleWaitSend)) { const bool isSendNotRecv = (Send && Recv) ? (flags & RoleWaitSend) : Send; int spins = 0; while (connStepCache + (isSendNotRecv ? NCCL_STEPS : 0) < step + StepPerSlice) { connStepCache = loadStepValue(connStepPtr); if (checkAbort(flags, Aborted, spins)) break; } void **ptrs = isSendNotRecv ? ncclShmem.groups[group].dsts : ncclShmem.groups[group].srcs; if ((flags & ConnFifoEnabled) && connFifo[step%NCCL_STEPS].mode == NCCL_MODE_OFFSET) { int offset = loadInt(&connFifo[step%NCCL_STEPS].offset); ptrs[index] = connEltsFifo + offset/sizeof(T); } else if (Direct && fn.work->regUsed) { if (isSendNotRecv) { if (flags & DirectWrite) { ptrs[index] = directBuff; } else if (flags & DirectRead) { // empty send ptrs[index] = nullptr; } else { ptrs[index] = connEltsFifo + (step%NCCL_STEPS)*connStepSize; } } else { if (flags & DirectRead) { ptrs[index] = directBuff; } else if (flags & DirectWrite) { if (Send) ptrs[index] = directBuff; // send to next from my output buffer else ptrs[index] = nullptr; } else { ptrs[index] = connEltsFifo + (step%NCCL_STEPS)*connStepSize; } } } else { ptrs[index] = connEltsFifo + (step%NCCL_STEPS)*connStepSize; } } subBarrier(); if (Recv == 0 || ncclShmem.groups[group].srcs[0] == nullptr) { nrecv = 0; } else { nrecv = fan.nrecv(); } if (Send == 0 || ncclShmem.groups[group].dsts[0] == nullptr) { nsend = 0; } else { nsend = fan.nsend(); } fn.template operator() (tid, nworkers, slice, stepSize * StepPerSlice, nrecv, ncclShmem.groups[group].srcs, nsend, ncclShmem.groups[group].dsts, ncclShmem.groups[group].dstSizes, sendDirectFlag, recvDirectFlag); } barrier(); int32_t dstSize = 0; if (flags & Send*RolePostSend) { // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_begin] dstSize = ncclShmem.groups[group].dstSizes[index]; ncclShmem.groups[group].dstSizes[index] = 0; if (flags & ConnFifoEnabled) connFifo[step%NCCL_STEPS].size = dstSize*sizeof(T); } barrier(); if (flags & (Recv*(RoleWaitRecv|RolePostRecv) | Send*(RoleWaitSend|RolePostSend))) { step += StepPerSlice; } if (flags & (Recv*RolePostRecv | Send*RolePostSend)) { if (Send && (!Recv || (flags & RolePostSend)) && (dstSize!=0 || (flags&ConnFifoEnabled))) { fence_acq_rel_sys(); } st_relaxed_sys_global(connStepPtr, step); } } } private: // Scatter/Gather generic op // skip: my own rank order in the buffer chunks // shift: peer offset to avoid all ranks sending to or receiving from same peer template __device__ __forceinline__ void ScatterGatherOp(intptr_t inpIx, intptr_t outIx, ssize_t totalElem, int peerElem, ssize_t peerOffset, int skip, int shift, bool postOp) { constexpr int DirectRecv = 1 && Direct && DirectRecv1; constexpr int DirectSend = 1 && Direct && DirectSend1; int offset = 0; // slice offset int sliceSize = stepSize*StepPerSlice; int dataSize = max(DIVUP(peerElem, 16*SlicePerChunk)*16, sliceSize/32); // per-peer slice size #pragma unroll for (int slice=0; slice(0, inpIx, offset, realSize); subBarrier(); #pragma unroll // Loop over peers for (int j=0; j= 0 && i >= skip) pOffset += peerOffset; void* src0 = (T*)ncclShmem.groups[group].srcs[0] + pOffset; ssize_t realPeerSize = min(realSize, totalElem-pOffset); if (realPeerSize > 0 && ncclShmem.groups[group].dsts[i] != nullptr) { reduceCopy(tid, nworkers, ncclShmem.groups[group].redOpArgs, false, 1, &src0, 1, ncclShmem.groups[group].dsts+i, realPeerSize); // Mark for threadfence at the end fenceNeeded |= true; } } } else if (Recv) { if (tid==0) ncclShmem.groups[group].dsts[0] = (T*)ncclShmem.groups[group].userOutput + outIx + offset; ssize_t pOffset = index*peerOffset; if (skip >= 0 && index >= skip) pOffset += peerOffset; // Adjust remote index with peer offset in case we are directly pulling from peer's output buffer waitPeer(outIx+pOffset, outIx+pOffset, offset, realSize); subBarrier(); #pragma unroll for (int j=0; j= 0 && i >= skip) pOffset += peerOffset; void* dst0 = (T*)ncclShmem.groups[group].dsts[0] + pOffset; ssize_t realPeerSize = min(realSize, totalElem-pOffset); if (DirectRecv && ncclShmem.groups[group].srcs[i] == dst0) realPeerSize = 0; if (realPeerSize > 0) reduceCopy(tid, nworkers, ncclShmem.groups[group].redOpArgs, postOp, 1, ncclShmem.groups[group].srcs+i, 1, &dst0, realPeerSize); } } } fenceNeeded = barrierAny(fenceNeeded); postPeer(fenceNeeded); offset += realSize; } } __device__ __forceinline__ void loadRecvConn(ncclDevChannelPeer *peer, int connIndex, uint32_t direct, int ipcRegFlag, int netRegFlag) { conn = &peer->recv[connIndex]; if (conn->netDeviceHandle.netDeviceType == NCCL_NET_DEVICE_UNPACK) { // handle must be a device ptr netDeviceHandle = conn->netDeviceHandle.handle; // Cache the handle ncclNetDeviceUnpackSetup(netDeviceHandle, group, index); flags |= NetDeviceUnpack; } step = conn->step; step = roundUp(step, SlicePerChunk*StepPerSlice); if (flags & RolePostRecv) { connStepPtr = conn->head; *connStepPtr = step; // Return credits in case we rounded up. } if (flags & RoleWaitRecv) { if ((flags & PatMode) == 0) ncclShmem.groups[group].recvConns[index] = conn; // WaitRecv role saves since that's who needs it in setDataPtrs() flags |= (conn->flags & NCCL_NVLS_MIN_POLL) ? NvlsMinPolling : 0; connStepPtr = conn->tail; connStepCache = loadStepValue(connStepPtr); connStepSize = conn->stepSize/sizeof(T); connEltsFifo = (T*)conn->buffs[NCCL_PROTO_SIMPLE]; if (conn->connFifo != nullptr) { flags |= ConnFifoEnabled; connFifo = conn->connFifo; } if (Direct) { if (ipcRegFlag) { // User buffers have been registered if (conn->flags & (NCCL_P2P_READ | NCCL_P2P_WRITE)) { if (P2p) { flags |= conn->flags & NCCL_P2P_WRITE ? DirectWrite : DirectRead; } else if (connIndex == 1 && direct) { flags |= DirectRead; } else { flags |= direct & NCCL_P2P_READ ? DirectRead : DirectWrite; } } else if ((conn->flags & NCCL_NVLS_MIN_POLL)) { /* NVLS direct */ flags |= DirectRead; } } if (netRegFlag) { if (conn->flags & NCCL_DIRECT_NIC) { flags |= NetRegMode; connFifo[step % NCCL_STEPS].size = 0; } } } } } __device__ __forceinline__ void loadSendConn(ncclDevChannelPeer *peer, int connIndex, uint32_t direct, int ipcRegFlag, int netRegFlag) { conn = &peer->send[connIndex]; step = conn->step; step = roundUp(step, SlicePerChunk*StepPerSlice); connFifo = conn->connFifo; if (connFifo != nullptr) flags |= ConnFifoEnabled; if (flags & RolePostSend) { connStepPtr = conn->tail; connEltsFifo = (T*)conn->buffs[NCCL_PROTO_SIMPLE]; } if (flags & RoleWaitSend) { if ((flags & PatMode) == 0) ncclShmem.groups[group].sendConns[index] = conn; // WaitSend role saves since that's who needs it in setDataPtrs() flags |= (conn->flags & NCCL_NVLS_MIN_POLL) ? NvlsMinPolling : 0; connStepPtr = conn->head; connStepCache = loadStepValue(connStepPtr); connStepSize = conn->stepSize/sizeof(T); connEltsFifo = (T*)conn->buffs[NCCL_PROTO_SIMPLE]; if (Direct) { if (ipcRegFlag) { // User buffers have been registered if (conn->flags & (NCCL_P2P_WRITE | NCCL_P2P_READ)) { if (P2p) { flags |= conn->flags & NCCL_P2P_WRITE ? DirectWrite : DirectRead; } else if (connIndex == 1 && direct) { flags |= DirectRead; // scatter-reduce use direct pull } else { flags |= direct & NCCL_P2P_READ ? DirectRead : DirectWrite; } } else if ((conn->flags & NCCL_NVLS_MIN_POLL)) { /* NVLS direct */ flags |= DirectWrite; } } if (netRegFlag) { if (conn->flags & NCCL_DIRECT_NIC) { flags |= NetRegMode; } } } } } public: __device__ Primitives( int tid, int nthreads, int const *recvPeers, int const *sendPeers, void const *inputBuf, void *outputBuf, uint64_t redOpArg, uint8_t group=0, uint8_t connIndexRecv = 0, uint8_t connIndexSend = 0, struct ncclDevWorkColl* collWork = nullptr, struct ncclDevWorkP2p* p2pWork = nullptr, int stepSize_ = 0, int mode = primsModeDefault ): tid(tid), nthreads(nthreads), tidInBlock(threadIdx.x), group(group), stepSize(stepSize_ == 0 ? ncclShmem.comm.buffSizes[NCCL_PROTO_SIMPLE]/NCCL_STEPS/sizeof(T) : stepSize_) { int peer = -1; flags = 0; index = -1; if (mode == primsModeDefault) { // Connect to ranks in sendPeers/recvPeers // For send operations, we need an extra warp to overlap the threadfence and the copy this->nworkers = nthreads - (MaxSend > 0 && nthreads >= NCCL_SIMPLE_EXTRA_GROUP_IF_NTHREADS_GE ? WARP_SIZE : 0); int nrecv=0, nsend=0; // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_line] while (nrecv < MaxRecv && recvPeers[nrecv] != -1) nrecv++; // coverity[dead_error_line] while (nsend < MaxSend && sendPeers[nsend] != -1) nsend++; this->fan = Fan(nrecv, nsend); constexpr int ThreadPerSync = MaxSend >= 16 || MaxRecv >= 16 ? 32 : // NVLS may have an arity > 8. In that case increase the size of the groups MaxSend >= 8 || MaxRecv >= 8 ? 16 : 8; // Allows for all roles (WaitRecv/WaitSend/PostRecv/PostSend) within a single warp static_assert(MaxSend <= ThreadPerSync && MaxRecv <= ThreadPerSync, "Not enough threads to cover all peers"); assert(2*(nrecv+nsend) <= nthreads); // Ensure no thread is assigned more than one role. // Coverity assumes that index will equal tid based on the line below, but it doesn't consider the setting // of flags. This results in multiple false positive overruns being reported here and in all_reduce.h. // Unfortunately, we've been unsuccessful in trying to silence them with a single directive here so // instead it's being done at the callers. // coverity[assignment:FALSE] if (tid < nrecv) { flags |= RoleWaitRecv; index = tid; } // Yes, for some template arguments this code will be unreachable. That's fine. // coverity[dead_error_begin] else if (tid < nrecv+nsend) { flags |= RoleWaitSend; index = tid-nrecv; } else if (nthreads-nsend <= tid) { flags |= RolePostSend; index = tid-(nthreads-nsend); } else if (nthreads-nrecv-nsend <= tid) { flags |= RolePostRecv; index = tid-(nthreads-nrecv-nsend); } if (flags & (RoleWaitRecv|RolePostRecv)) peer = recvPeers[index]; if (flags & (RoleWaitSend|RolePostSend)) peer = sendPeers[index]; // Coverity thinks that index could be -1 here but that's not actually the case. // coverity[negative_returns:FALSE] int sendIpcReg; int recvIpcReg; int sendNetReg; int recvNetReg; if (P2p) { sendIpcReg = p2pWork ? p2pWork->sendIpcReg : 0; recvIpcReg = p2pWork ? p2pWork->recvIpcReg : 0; sendNetReg = p2pWork ? p2pWork->sendNetReg : 0; recvNetReg = p2pWork ? p2pWork->recvNetReg : 0; } else { recvIpcReg = sendIpcReg = collWork ? collWork->regUsed : 0; recvNetReg = sendNetReg = collWork ? collWork->netRegUsed : 0; } // coverity[overrun-call] => Coverity think prims.index can be greater than 1 if (flags & (RoleWaitRecv|RolePostRecv)) loadRecvConn(ncclShmem.channel.peers[peer], connIndexRecv, collWork ? collWork->direct : 0, recvIpcReg, recvNetReg); // coverity[overrun-call] => Coverity think prims.index can be greater than 1 if (flags & (RoleWaitSend|RolePostSend)) loadSendConn(ncclShmem.channel.peers[peer], connIndexSend, collWork ? collWork->direct : 0, sendIpcReg, sendNetReg); // coverity[negative_returns:FALSE] => coverity thinks that index could be -1 but that's not actually the case // coverity[var_deref_model] => coverity thinks work can dereferenced if NULL but this is not the case setDataPtrs(inputBuf, outputBuf, redOpArg, (struct ncclDevWorkCollReg*)collWork, sendIpcReg || recvIpcReg, peer); // coverity[uninit_member] => coverity thinks fan.n is not initialized if (barrierAny(flags & NetDeviceUnpack)) { flags |= AnyNetDeviceUnpack; // RoleWaitRecv starts at tid=0, so this creates the bitmask of which recv peers // have NetDeviceUnpack. uint32_t mask = __ballot_sync(~0u, ((flags & RoleWaitRecv) && (flags & NetDeviceUnpack)) ? 1 : 0); if (tid == 0) { ncclShmem.groups[this->group].devicePlugin.unpack.unpackNetDeviceIndexMask = mask; } } } else if (mode == primsModePatRs || mode == primsModePatAg) { // Connect to all ranks +/- 2^n flags |= PatMode; const int roles[5] = { RoleWaitRecv, RolePostRecv, RoleWaitSend, RolePostSend, RoleInput | RoleOutput }; if (tid < 5) flags |= roles[tid]; int nranks = ncclShmem.comm.nRanks; if (tid < 32 && ((1UL<conn = ncclShmem.channel.peers[recvPeer]->recv+connIndexRecv; peer->step = conn->step; peer->buff = conn->buffs[NCCL_PROTO_SIMPLE]; peer->stepCache = loadStepValue(peer->tailPtr = conn->tail); peer->headPtr = conn->head; peer->accSize = 0; peer->connStepSize = conn->stepSize/sizeof(T); // Load send peer int sendPeer = mode == primsModePatAg ? (rank - delta + nranks) % nranks : (rank + delta) % nranks; peer = ((struct ncclPatPeer*)sendPeers)+tid; conn = peer->conn = ncclShmem.channel.peers[sendPeer]->send+connIndexSend; peer->step = conn->step; peer->connFifo = conn->connFifo; peer->buff = conn->buffs[NCCL_PROTO_SIMPLE]; peer->stepCache = loadStepValue(peer->headPtr = conn->head); peer->tailPtr = conn->tail; peer->accSize = 0; peer->connStepSize = conn->stepSize/sizeof(T); } if (tid==0) { ncclShmem.groups[group].userInput = (void*)inputBuf; ncclShmem.groups[group].userOutput = (void*)outputBuf; ncclShmem.groups[group].redOpArgs = redOpArg; // scaler for local input } patBarrier(); } } __device__ ~Primitives() { if (flags&PatMode) return; // Save steps for the next operation if (flags & (RolePostSend|RolePostRecv)) conn->step = step; if ((flags & NetRegMode) && (flags & RoleWaitSend)) { // Make sure we wait until the proxy has sent data before we return. // We don't want the next CUDA kernel to overwrite the send buffer which // was accessed directly. uint64_t prevStep = step - StepPerSlice; volatile ssize_t* ptr = &(connFifo[prevStep%NCCL_STEPS].size); int spins = 0; while (*ptr != -1) if (checkAbort(flags, Aborted, spins)) break; } if (flags & NetDeviceUnpack) { ncclNetDeviceSaveHead(netDeviceHandle, group, index); } // Make sure all threads are done writing back conn->step and done using // ncclShmem.groups[group] barrier(); if ((flags & DirectRead) && (flags & RoleWaitSend) && P2p) { // For sendrecv DirectRead, sender needs to wait for receiver reading data from src. // This has to be done after barrier() since post thread might have contention with // this check. int spins = 0; volatile uint64_t* tail = conn->tail; volatile uint64_t* head = conn->head; while (*tail > *head) if (checkAbort(flags, Aborted, spins)) break; } } __device__ void setDataPtrs(void const *inputBuf, void *outputBuf, uint64_t redOpArg, struct ncclDevWorkCollReg* work, uint8_t ipcReg, int peer) { if (tid==0) { ncclShmem.groups[group].userInput = (void*)inputBuf; ncclShmem.groups[group].userOutput = (void*)outputBuf; ncclShmem.groups[group].redOpArgs = redOpArg; // scaler for local input } if (Direct && ipcReg) { bool recvProvider = (flags & RoleWaitRecv) && (flags & DirectWrite); bool sendAcceptor = (flags & RoleWaitSend) && (flags & DirectWrite); bool sendProvider = (flags & RoleWaitSend) && (flags & DirectRead); // sender provides direct buffer (to be fetched) bool recvAcceptor = (flags & RoleWaitRecv) && (flags & DirectRead); // receiver accepts direct buffer if (recvProvider) { int spins = 0; void* volatile* slot = ncclShmem.groups[group].recvConns[index]->ptrExchange; // Wait for consumer to consume previous value before trampling it. if (slot) { T* exchgPtr; directBuff = (T*)outputBuf; while (*slot != nullptr && !checkAbort(flags, Aborted, spins)); if (P2p) { exchgPtr = (T*)outputBuf; } else { int localPeer = ncclShmem.comm.rankToLocalRank[peer]; // coverity[deref_parm:FALSE] => work cannot be NULL if ipcReg != NULL exchgPtr = (T*)(work->coll.recvbuffOffset + work->coll.recvbuffRmtAddrs[localPeer]); } *slot = reinterpret_cast(exchgPtr); } } if (sendAcceptor) { int spins = 0; void* volatile* slot = ncclShmem.groups[group].sendConns[index]->ptrExchange; void* ptr; while (slot) { ptr = *slot; if (ptr != nullptr || checkAbort(flags, Aborted, spins)) break; } if (slot) { directBuff = reinterpret_cast(ptr); *slot = nullptr; } else { // coverity[var_deref_op] directBuff = (T*)work->dnOutputs[index]; } } if (sendProvider) { int spins = 0; void* volatile* slot = ncclShmem.groups[group].sendConns[index]->ptrExchange; // Wait for consumer to consume previous value before trampling it. if (slot) { T* exchgPtr; while ((*slot != nullptr) && !checkAbort(flags, Aborted, spins)); // If there is no recv, then we are directly pulling from input buffer (e.g. directScatter) // Otherwise, we are pulling from output buffer (e.g. recvCopyDirectSend) directBuff = MaxRecv == 0 ? (T*)inputBuf : (T*)outputBuf; if (P2p) { exchgPtr = MaxRecv == 0 ? (T*)inputBuf : (T*)outputBuf; } else { int localPeer = ncclShmem.comm.rankToLocalRank[peer]; if (MaxRecv == 0) // coverity[var_deref_op] exchgPtr = (T*)(work->coll.sendbuffOffset + work->coll.sendbuffRmtAddrs[localPeer]); else // coverity[var_deref_op] exchgPtr = (T*)(work->coll.recvbuffOffset + work->coll.recvbuffRmtAddrs[localPeer]); } // Exchange pre-scalers for use in direct pull *slot = reinterpret_cast(exchgPtr); } } if (recvAcceptor) { int spins = 0; void* volatile* slot = ncclShmem.groups[group].recvConns[index]->ptrExchange; void* ptr; while (slot) { ptr = *slot; if (ptr != nullptr || checkAbort(flags, Aborted, spins)) break; } if (slot) { directBuff = reinterpret_cast(ptr); *slot = nullptr; } else { // Coverity complains about work being possibly NULL below. However, slot // being NULL means that the NVLS buffer is registered (regUsed == 1) // so work can't be NULL in this code path. // coverity[var_deref_op] directBuff = (T*)work->dnInputs[index]; } } } } __device__ void moveDataPtrs(intptr_t delta) { if (tid==0) { ncclShmem.groups[group].userInput = (T*)ncclShmem.groups[group].userInput + delta; ncclShmem.groups[group].userOutput = (T*)ncclShmem.groups[group].userOutput + delta; } } __device__ __forceinline__ void send(intptr_t inpIx, int eltN) { genericOp<0, 0, 0, 1, Input, -1>(inpIx, -1, eltN, false); } __device__ __forceinline__ void sendFromOutput(intptr_t outIx, int eltN) { genericOp<0, 0, 0, 1, Output, -1>(outIx, -1, eltN, false); } __device__ __forceinline__ void directSend(intptr_t inpIx, intptr_t outIx, int eltN) { genericOp<0, 1, 0, 1, Input, -1>(inpIx, outIx, eltN, false); } __device__ __forceinline__ void directSendFromOutput(intptr_t outIx, int eltN) { genericOp<0, 1, 0, 1, Output, -1>(outIx, outIx, eltN, false); } __device__ __forceinline__ void recv(intptr_t outIx, int eltN, bool postOp=false) { genericOp<0, 0, 1, 0, -1, Output>(-1, outIx, eltN, postOp); } __device__ __forceinline__ void directRecv(intptr_t outIx, int eltN, bool postOp=false) { genericOp<1, 0, 1, 0, -1, Output>(outIx, outIx, eltN, postOp); } __device__ __forceinline__ void directRecvCopy(intptr_t inpIx, intptr_t outIx, int eltN) { genericOp<1, 0, 1, 0, -1, Output>(inpIx, outIx, eltN, /*postOp=*/false); } __device__ __forceinline__ void copySend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { genericOp<0, 0, 0, 1, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void directCopySend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { genericOp<0, 1, 0, 1, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void recvSend(int eltN, bool postOp=false) { genericOp<0, 0, 1, 1, -1, -1>(-1, -1, eltN, postOp); } __device__ __forceinline__ void recvCopySend(intptr_t outIx, int eltN, bool postOp=false) { genericOp<0, 0, 1, 1, -1, Output>(-1, outIx, eltN, postOp); } __device__ __forceinline__ void directRecvCopyDirectSend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { genericOp<1, 1, 1, 1, -1, Output>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void directRecvDirectSend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { genericOp<1, 1, 1, 1, -1, -1>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void recvDirectSend(intptr_t outIx, int eltN, bool postOp=false) { genericOp<0, 1, 1, 1, -1, -1>(-1, outIx, eltN, postOp); } __device__ __forceinline__ void directRecvSend(intptr_t outIx, int eltN, bool postOp=false) { genericOp<1, 0, 1, 1, -1, -1>(outIx, outIx, eltN, postOp); } __device__ __forceinline__ void recvCopyDirectSend(intptr_t outIx, int eltN, bool postOp=false) { genericOp<0, 1, 1, 1, -1, Output>(-1, outIx, eltN, postOp); } __device__ __forceinline__ void recvReduceCopy(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { genericOp<0, 0, 1, 0, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void directRecvReduceCopy(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { genericOp<1, 0, 1, 0, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void recvReduceSend(intptr_t inpIx, int eltN, bool postOp=false) { genericOp<0, 0, 1, 1, Input, -1>(inpIx, -1, eltN, postOp); } __device__ __forceinline__ void directRecvReduceSend(intptr_t inpIx, int eltN, bool postOp=false) { genericOp<1, 0, 1, 1, Input, -1>(inpIx, -1, eltN, postOp); } __device__ __forceinline__ void recvReduceDirectSend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { genericOp<0, 1, 1, 1, Input, -1>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void directRecvReduceDirectSend(intptr_t inpIx, intptr_t outIx, ssize_t eltN, bool postOp=false) { genericOp<1, 1, 1, 1, Input, -1>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void recvReduceCopySend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { genericOp<0, 0, 1, 1, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void recvReduceCopyDirectSend(intptr_t inpIx, intptr_t outIx, int eltN, bool postOp=false) { // Direct is only for the send part genericOp<0, 1, 1, 1, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void directRecvReduceCopyDirectSend(intptr_t inpIx, intptr_t outIx, ssize_t eltN, bool postOp=false) { genericOp<1, 1, 1, 1, Input, Output>(inpIx, outIx, eltN, postOp); } __device__ __forceinline__ void scatter(intptr_t inpIx, ssize_t totalElem, int peerElem, ssize_t peerOffset, int skip, int shift) { ScatterGatherOp<0, 0, 0, 1>(inpIx, -1, totalElem, peerElem, peerOffset, skip, shift, /*postOp=*/false); } __device__ __forceinline__ void directScatter(intptr_t inpIx, ssize_t totalElem, int peerElem, ssize_t peerOffset, int skip, int shift) { ScatterGatherOp<0, 1, 0, 1>(inpIx, -1, totalElem, peerElem, peerOffset, skip, shift, /*postOp=*/false); } __device__ __forceinline__ void gather(intptr_t outIx, ssize_t totalElem, int peerElem, ssize_t peerOffset, int skip, int shift, bool postOp=false) { ScatterGatherOp<0, 0, 1, 0>(-1, outIx, totalElem, peerElem, peerOffset, skip, shift, postOp); } __device__ __forceinline__ void directGather(intptr_t outIx, ssize_t totalElem, int peerElem, ssize_t peerOffset, int skip, int shift) { ScatterGatherOp<1, 0, 1, 0>(-1, outIx, totalElem, peerElem, peerOffset, skip, shift, /*postOp=*/false); } __device__ __forceinline__ void patReduce(struct ncclPatStep* ps, struct ncclPatShmem* shmem) { if (ps->flags & PatSkipped) { patBarrier(); patBarrier(); return; } // Skipped int nelem = ps->nelem < 0 ? 0 : ps->nelem; T* userInput = (T*)ncclShmem.groups[group].userInput; T* userOutput = (T*)ncclShmem.groups[group].userOutput; bool recv = ps->recvDim >= 0 && (flags & (RolePostRecv|RoleWaitRecv)); bool send = ps->sendDim >= 0 && (flags & (RolePostSend|RoleWaitSend)); bool postRecv = ps->postRecv && recv; bool postSend = ps->postSend && send; struct ncclPatPeer* peer = NULL; if (recv) { peer = shmem->recvDims+ps->recvDim; step = peer->step; } if (send) { peer = shmem->sendDims+ps->sendDim; step = peer->step; } if (recv && (flags & RoleWaitRecv)) { ncclShmem.groups[group].srcs[0] = ((T*)peer->buff) + (step%NCCL_STEPS)*peer->connStepSize + ps->recvOffset; int spins = 0; while (peer->stepCache < step + StepPerSlice) { peer->stepCache = loadStepValue(peer->tailPtr); if (checkAbort(flags, Aborted, spins)) break; } } if (send && (flags & RoleWaitSend)) { int spins = 0; while (peer->stepCache + NCCL_STEPS < step + ps->stepOffset + StepPerSlice) { peer->stepCache = loadStepValue(peer->headPtr); if (checkAbort(flags, Aborted, spins)) break; } ncclShmem.groups[group].dsts[0] = ((T*)peer->buff) + ((step+ps->stepOffset)%NCCL_STEPS)*peer->connStepSize + ps->sendOffset; if (peer->accSize < ps->sendOffset + nelem + (step+ps->stepOffset)*peer->connStepSize) { // New data, add our own data to it. ncclShmem.groups[group].srcs[1] = userInput + ps->inpIx; } else { // There is already data in there, accumulate instead of writing to it. ncclShmem.groups[group].srcs[1] = ncclShmem.groups[group].dsts[0]; } } long long int localAccSize = shmem->localAccSize; if (ps->sendDim < 0 && (flags & RoleOutput)) { // Destination is our own local buffer ncclShmem.groups[group].dsts[0] = userOutput + ps->outIx; if (localAccSize < ps->outIx + nelem) { // New data, add our own data to it. ncclShmem.groups[group].srcs[1] = userInput + ps->inpIx; localAccSize = ps->outIx + nelem; } else { // There is already data in there, accumulate instead of writing to it. ncclShmem.groups[group].srcs[1] = ncclShmem.groups[group].dsts[0]; } } patBarrier(); int nSrcs = 2; void** srcs = ncclShmem.groups[group].srcs; if (ps->recvDim < 0) { srcs++; nSrcs--; } // No peer to receive from, remove one source int workSize = ncclShmem.aborted ? 0 : nelem; reduceCopy (tid, nthreads, ncclShmem.groups[group].redOpArgs, /*postOp=*/false, nSrcs, srcs, 1, ncclShmem.groups[group].dsts, workSize); // Store conn step here inside the two barriers to make sure next reload will see the update. if (postSend && (flags & RolePostSend)) { if (peer->connFifo) { peer->connFifo[step%NCCL_STEPS].size = (ps->sendOffset + nelem)*sizeof(T); } peer->step = step += StepPerSlice; st_relaxed_sys_global(&peer->conn->step, step); } if (postRecv && (flags & RolePostRecv)) { peer->step = step += StepPerSlice; st_relaxed_sys_global(&peer->conn->step, step); // Also save in global mem for next op } // Update accSize if (ps->sendDim < 0 && (flags & RoleOutput)) atomicMax(&shmem->localAccSize, localAccSize); if (ps->sendDim >= 0 && (flags & RoleWaitSend)) atomicMax(&peer->accSize, ps->sendOffset + nelem + (step+ps->stepOffset)*peer->connStepSize); patBarrier(); if (postSend && (flags & RolePostSend)) { if (nelem > 0 || peer->connFifo) fence_acq_rel_sys(); st_relaxed_sys_global(peer->tailPtr, step); } if (postRecv && (flags & RolePostRecv)) { st_relaxed_sys_global(peer->headPtr, step); } } __device__ __forceinline__ void patCopy(struct ncclPatStep* ps, struct ncclPatShmem* shmem) { if (ps->flags & PatSkipped) { patBarrier(); patBarrier(); return; } // Skipped int nelem = ps->nelem < 0 ? 0 : ps->nelem; T* userInput = (T*)ncclShmem.groups[group].userInput; T* userOutput = (T*)ncclShmem.groups[group].userOutput; bool recv = ps->recvDim >= 0 && (flags & (RolePostRecv|RoleWaitRecv)); bool send = ps->sendDim >= 0 && (flags & (RolePostSend|RoleWaitSend)); bool postRecv = ps->postRecv && recv; bool postSend = ps->postSend && send; struct ncclPatPeer* peer = NULL; if (recv) { peer = shmem->recvDims+ps->recvDim; step = peer->step; } if (send) { peer = shmem->sendDims+ps->sendDim; step = peer->step; } if (recv && (flags & RoleWaitRecv)) { ncclShmem.groups[group].srcs[0] = ((T*)peer->buff) + ((step+ps->stepOffset)%NCCL_STEPS)*peer->connStepSize + ps->recvOffset; int spins = 0; while (peer->stepCache < step + ps->stepOffset + StepPerSlice) { peer->stepCache = loadStepValue(peer->tailPtr); if (checkAbort(flags, Aborted, spins)) break; } if (peer->accSize < ps->recvOffset + nelem + (step+ps->stepOffset)*peer->connStepSize) { // New data, copy to our output buffer. ncclShmem.groups[group].dsts[1] = userOutput + ps->outIx; } else { ncclShmem.groups[group].dsts[1] = ncclShmem.groups[group].srcs[0]; // Already done } } if (send && (flags & RoleWaitSend)) { int spins = 0; while (peer->stepCache + NCCL_STEPS < step + StepPerSlice) { peer->stepCache = loadStepValue(peer->headPtr); if (checkAbort(flags, Aborted, spins)) break; } ncclShmem.groups[group].dsts[0] = ((T*)peer->buff) + (step%NCCL_STEPS)*peer->connStepSize + ps->sendOffset; } long long int localAccSize = shmem->localAccSize; if (ps->recvDim < 0 && (flags & RoleInput)) { // Source is our own local buffer ncclShmem.groups[group].srcs[0] = userInput + ps->inpIx; if (localAccSize < ps->inpIx + nelem) { // New data, copy to our output buffer. ncclShmem.groups[group].dsts[1] = userOutput + ps->outIx; localAccSize = ps->inpIx + nelem; } else { // Already done ncclShmem.groups[group].dsts[1] = ncclShmem.groups[group].srcs[0]; } } patBarrier(); int nDsts = 2; void** dsts = ncclShmem.groups[group].dsts; if (ps->sendDim < 0) { dsts++; nDsts--; } // No peer to send to, remove one dest if (ncclShmem.groups[group].srcs[0] == ncclShmem.groups[group].dsts[1]) nDsts--; // In-place or already done. int workSize = ncclShmem.aborted ? 0 : nelem; reduceCopy (tid, nthreads, ncclShmem.groups[group].redOpArgs, /*postOp=*/false, 1, ncclShmem.groups[group].srcs, nDsts, dsts, workSize); // Store conn step here inside the two barriers to make sure next reload will see the update. if (postSend && (flags & RolePostSend)) { if (peer->connFifo) { peer->connFifo[step%NCCL_STEPS].size = (ps->sendOffset + nelem)*sizeof(T); } peer->step = step += StepPerSlice; st_relaxed_sys_global(&peer->conn->step, step); } if (postRecv && (flags & RolePostRecv)) { peer->step = step += StepPerSlice; st_relaxed_sys_global(&peer->conn->step, step); // Also save in global mem for next op } // Update accSize if (ps->recvDim < 0 && (flags & RoleInput)) atomicMax(&shmem->localAccSize, localAccSize); if (ps->recvDim >= 0 && (flags & RoleWaitRecv)) atomicMax(&peer->accSize, ps->recvOffset + nelem + (step+ps->stepOffset)*peer->connStepSize); patBarrier(); if (postSend && (flags & RolePostSend)) { if (nelem > 0 || peer->connFifo) fence_acq_rel_sys(); st_relaxed_sys_global(peer->tailPtr, step); } if (postRecv && (flags & RolePostRecv)) { st_relaxed_sys_global(peer->headPtr, step); } } }; nccl-2.29.7-1/src/device/reduce.h000066400000000000000000000061211515037102200163320ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "device.h" #include "collectives.h" #include "primitives.h" namespace { template __device__ __forceinline__ void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { ncclRing *ring = &ncclShmem.channel.ring; const int nranks = ncclShmem.comm.nRanks; const int rank = ncclShmem.comm.rank; const int prevRank = ring->userRanks[nranks-1]; const int root = work->root; size_t chunkCount; size_t channelCount; size_t gridOffset; ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), (size_t*)nullptr, &gridOffset, &channelCount, &chunkCount); size_t offset; int nelem; // Coverity reports that the callee treats &ring->next as an array. However, due to the use of // FanSymmetric<1>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, 0, Proto, 0> prims(tid, nthreads, &ring->prev, &ring->next, work->sendbuff, work->recvbuff, work->redOpArg); if (prevRank == root) { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.send(offset, nelem); } } else if (rank == root) { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.recvReduceCopy(offset, offset, nelem, /*postOp=*/true); } } else { for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.recvReduceSend(offset, nelem); } } } } template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { using Proto = ProtoSimple; runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runRing(tid, nthreads, work); } }; nccl-2.29.7-1/src/device/reduce_kernel.h000066400000000000000000001177621515037102200177100ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_REDUCE_KERNEL_H_ #define NCCL_REDUCE_KERNEL_H_ #include "op128.h" #include #include template struct IsFloatingPoint: std::false_type {}; template<> struct IsFloatingPoint: std::true_type {}; #if defined(__CUDA_BF16_TYPES_EXIST__) template<> struct IsFloatingPoint<__nv_bfloat16>: std::true_type {}; #endif #if defined(__CUDA_FP8_TYPES_EXIST__) template<> struct IsFloatingPoint<__nv_fp8_e4m3>: std::true_type {}; template<> struct IsFloatingPoint<__nv_fp8_e5m2>: std::true_type {}; #endif template<> struct IsFloatingPoint: std::true_type {}; template<> struct IsFloatingPoint: std::true_type {}; //////////////////////////////////////////////////////////////////////////////// // The reduction function classes. All classes must: // 1. Expose the `EltType` typedef. // 2. Have constructor taking no arguments (default constructible). // 3. Have constructor taking `uint64_t opArg`. template struct FuncCopy { using EltType = T; __device__ __forceinline__ FuncCopy(uint64_t opArg=0) {}; }; template struct FuncSum { using EltType = T; __device__ __forceinline__ FuncSum(uint64_t opArg=0) {}; }; template struct FuncProd { using EltType = T; __device__ __forceinline__ FuncProd(uint64_t opArg=0) {}; }; template struct FuncMinMax { using EltType = T; BytePack xormask; // only used by integers bool isMinNotMax; // only used by floats __device__ __forceinline__ FuncMinMax(uint64_t opArg=0) { xormask.native = opArg; isMinNotMax = (opArg&1)==0; } }; template struct FuncPreMulSum; template struct FuncSumPostDiv; //////////////////////////////////////////////////////////////////////////////// // Trait class for handling the reduction argument. template struct RedOpArg { // default case: no argument static constexpr bool ArgUsed = false; __device__ __forceinline__ static uint64_t loadArg(void *ptr) { return 0; } }; template struct RedOpArg> { static constexpr bool ArgUsed = true; __device__ __forceinline__ static uint64_t loadArg(void *ptr) { union { uint64_t u64; T val; }; u64 = 0; val = *(T*)ptr; return u64; } }; //////////////////////////////////////////////////////////////////////////////// // Trait classes for reduction functions. Given a function (FuncSum, etc.) // and a number of elements in a pack, will reduce, preOp, or postOp a pack // of elements. These classes are intended to be specialized for specific // combinations of reduction function and pack size. template struct Apply_Cast/*{ static BytePack cast(BytePack a); }*/; template struct Apply_Reduce /*{ static BytePack reduce( Fn fn, BytePack a, BytePack b ); }*/; template struct Apply_PreOp/*{ static constexpr bool IsIdentity; static BytePack preOp(Fn fn, BytePack a); }*/; template struct Apply_PostOp/*{ static constexpr bool IsIdentity; static BytePack postOp(Fn fn, BytePack a); }*/; template struct LoadMultimem_BigPackSize/*{ // If non-zero, then this and sizeof(T) are valid pack sizes for LoadMultimem, // otherwise there are no valid pack sizes for LoadMultimem. static constexpr int BigPackSize = 0; }*/; template struct Apply_LoadMultimem/*{ static BytePack load(Fn fn, uintptr_t addr); }*/; // Helpers for dealing with BytePack<0>'s template struct Apply_Cast_MaybeEmpty: Apply_Cast {}; template struct Apply_Cast_MaybeEmpty { __device__ constexpr static BytePack<0> cast(BytePack<0> a) { return {}; } }; template struct Apply_Reduce_MaybeEmpty: Apply_Reduce {}; template struct Apply_Reduce_MaybeEmpty { __device__ constexpr static BytePack<0> reduce(Fn fn, BytePack<0> a, BytePack<0> b) { return {}; } }; template struct Apply_PreOp_MaybeEmpty: Apply_PreOp {}; template struct Apply_PreOp_MaybeEmpty { static constexpr bool IsIdentity = true; __device__ constexpr static BytePack<0> preOp(Fn fn, BytePack<0> a) { return {}; } }; template struct Apply_PostOp_MaybeEmpty: Apply_PostOp {}; template struct Apply_PostOp_MaybeEmpty { static constexpr bool IsIdentity = true; __device__ constexpr static BytePack<0> postOp(Fn fn, BytePack<0> a) { return {}; } }; template struct Apply_LoadMultimem_MaybeEmpty: Apply_LoadMultimem {}; template struct Apply_LoadMultimem_MaybeEmpty { __device__ constexpr static BytePack<0> load(Fn fn, uintptr_t addr) { return {}; } }; //////////////////////////////////////////////////////////////////////////////// // Public API for calling the trait classes. These take the data elements as a // pack of any type, which could be a BytePack or any integral type (uint64_t, // uint32_t, etc.), and will return a new pack where each element has been // transformed appropriately. template __device__ __forceinline__ BytePack::Size*sizeof(B)/sizeof(A)> applyCast(PackA a) { return Apply_Cast_MaybeEmpty::Size/sizeof(A)>::cast(toPack(a)); } template __device__ __forceinline__ Pack applyReduce(Fn fn, Pack a, Pack b) { return fromPack( Apply_Reduce_MaybeEmpty::Size/sizeof(typename Fn::EltType)> ::reduce(fn, toPack(a), toPack(b)) ); } template __device__ __forceinline__ Pack applyPreOp(Fn fn, Pack a) { return fromPack( Apply_PreOp_MaybeEmpty::Size/sizeof(typename Fn::EltType)> ::preOp(fn, toPack(a)) ); } template __device__ __forceinline__ Pack applyPostOp(Fn fn, Pack a) { return fromPack( Apply_PostOp_MaybeEmpty::Size/sizeof(typename Fn::EltType)> ::postOp(fn, toPack(a)) ); } template __device__ __forceinline__ BytePack applyLoadMultimem(Fn fn, uintptr_t addr) { return Apply_LoadMultimem_MaybeEmpty::load(fn, addr); } //////////////////////////////////////////////////////////////////////////////// // Apply_Cast template struct Apply_Cast { __device__ __forceinline__ static BytePack cast(BytePack a) { BytePack b; b.half[0] = Apply_Cast::cast(a.half[0]); b.half[1] = Apply_Cast::cast(a.half[1]); return b; } }; template struct Apply_Cast { __device__ __forceinline__ static BytePack cast(BytePack a) { return toPack(B(fromPack(a))); } }; template<> struct Apply_Cast<__half, float, /*EltPerPack=*/1> { __device__ __forceinline__ static BytePack cast(BytePack a) { return toPack(__half2float(fromPack<__half>(a))); } }; template<> struct Apply_Cast { __device__ __forceinline__ static BytePack cast(BytePack a) { return toPack(__float2half_rn(fromPack(a))); } }; template<> struct Apply_Cast<__half, float, /*EltPerPack=*/2> { __device__ __forceinline__ static BytePack<4*2> cast(BytePack<2*2> a) { return toPack(__half22float2(fromPack<__half2>(a))); } }; template<> struct Apply_Cast { __device__ __forceinline__ static BytePack<2*2> cast(BytePack<4*2> a) { return toPack(__float22half2_rn(fromPack(a))); } }; #if defined(__CUDA_BF16_TYPES_EXIST__) && (CUDART_RUNTIME >= 12000 || __CUDA_ARCH__ >= 800) template<> struct Apply_Cast<__nv_bfloat16, float, /*EltPerPack=*/2> { __device__ __forceinline__ static BytePack<4*2> cast(BytePack<2*2> a) { return toPack(__bfloat1622float2(fromPack<__nv_bfloat162>(a))); } }; template<> struct Apply_Cast { __device__ __forceinline__ static BytePack<2*2> cast(BytePack<4*2> a) { return toPack(__float22bfloat162_rn(fromPack(a))); } }; #endif #define EASY_CAST(A, B, EltPerPack, VecA, VecB) \ template<> \ struct Apply_Cast { \ __device__ __forceinline__ static BytePack cast(BytePack a) { \ return toPack(VecB(fromPack(a))); \ } \ }; \ template<> \ struct Apply_Cast { \ __device__ __forceinline__ static BytePack cast(BytePack b) { \ return toPack(VecA(fromPack(b))); \ } \ }; #if defined(__CUDA_FP8_TYPES_EXIST__) EASY_CAST(__nv_fp8_e5m2, float, 2, __nv_fp8x2_e5m2, float2) EASY_CAST(__nv_fp8_e5m2, float, 4, __nv_fp8x4_e5m2, float4) EASY_CAST(__nv_fp8_e4m3, float, 2, __nv_fp8x2_e4m3, float2) EASY_CAST(__nv_fp8_e4m3, float, 4, __nv_fp8x4_e4m3, float4) #endif #undef EASY_CAST //////////////////////////////////////////////////////////////////////////////// // Apply_Reduce // Nonsensical base case template struct Apply_Reduce { __device__ __forceinline__ static BytePack<0> reduce(Fn fn, BytePack<0> a, BytePack<0> b) { return {}; } }; // General recursive definition (EltPerPack > 1). This is how we iterate over // all elements in a pack of any size, by breaking it into halves. Eventually // we'll hit a base case (a more specific template specialization which takes // precedence). template struct Apply_Reduce { template __device__ __forceinline__ static BytePack reduce(Fn fn, BytePack a, BytePack b) { a.half[0] = Apply_Reduce::reduce(fn, a.half[0], b.half[0]); a.half[1] = Apply_Reduce::reduce(fn, a.half[1], b.half[1]); return a; } }; // Base case definitions (EltPerPack == 1) template struct Apply_Reduce, /*EltPerPack=*/1> { __device__ __forceinline__ static BytePack reduce(FuncCopy fn, BytePack a, BytePack b) { return a; } }; template struct Apply_Reduce, /*EltPerPack=*/1> { __device__ __forceinline__ static BytePack reduce(FuncSum fn, BytePack a, BytePack b) { return toPack(fromPack(a) + fromPack(b)); } }; template struct Apply_Reduce, /*EltPerPack=*/1> { __device__ __forceinline__ static BytePack reduce(FuncProd fn, BytePack a, BytePack b) { return toPack(fromPack(a) * fromPack(b)); } }; template struct Apply_Reduce, /*EltPerPack=*/1> { __device__ __forceinline__ static BytePack reduce(FuncMinMax fn, BytePack a, BytePack b) { return (a.native ^ fn.xormask.native) < (b.native ^ fn.xormask.native) ? a : b; } }; // Optimizations for specfic types and element count combinations: template<> struct Apply_Reduce, /*EltPerPack=*/4> { __device__ __forceinline__ static BytePack<4> reduce(FuncSum fn, BytePack<4> a, BytePack<4> b) { constexpr uint32_t even = 0x00ff00ffu; uint32_t x = (a.native & even) + (b.native & even); uint32_t y = (a.native & ~even) + (b.native & ~even); //a.native = (x & even) | (y & ~even); a.native = __byte_perm(x, y, 0x7250); return a; } }; template<> struct Apply_Reduce, /*EltPerPack=*/4> { __device__ static BytePack<4> reduce(FuncMinMax fn, BytePack<4> a, BytePack<4> b) { constexpr uint32_t ones = 0x01010101u; constexpr uint32_t even = 0x00ff00ffu; // even byte mask // Replicate xormask to all bytes uint32_t x = fn.xormask.native * ones; // Transform inputs by xormask uint32_t ax = a.native ^ x; uint32_t bx = b.native ^ x; // Use 9-bit arithmetic to compute d=a-b uint32_t d0 = (ax & even) + (~bx & even) + ones; uint32_t d1 = (ax>>8 & even) + (~(bx>>8) & even) + ones; // Move sign bit of each 9-bit delta into the least bit of origin byte //uint32_t s = (d0>>8 & ones & even) | (d1 & ones & ~even); uint32_t s = __byte_perm(d0, d1, 0x7351) & ones; // Broadcast least bit across whole byte s *= 0xffu; // Compose result by selecting bytes via: signbit(a-b)==1 ? a : b a.native = (a.native & s) | (b.native & ~s); return a; } }; template<> struct Apply_Reduce, /*EltPerPack=*/4> { __device__ __forceinline__ static BytePack<4> reduce(FuncProd fn, BytePack<4> apack, BytePack<4> bpack) { uint32_t a = apack.native; uint32_t b = bpack.native; uint32_t ab0 = (a*b) & 0xffu; asm volatile("mad.lo.u32 %0, %1, %2, %0;" : "+r"(ab0) : "r"(a&0xff00u), "r"(b&0xff00u)); uint32_t ab1; asm volatile("mul.hi.u32 %0, %1, %2;" : "=r"(ab1) : "r"(a&0xff0000), "r"(b&0xff0000)); asm volatile("mad.hi.u32 %0, %1, %2, %0;" : "+r"(ab1) : "r"(a&0xff000000u), "r"(b&0xff000000u)); apack.native = __byte_perm(ab0, ab1, 0x6420); return apack; } }; #define SPECIALIZE_REDUCE(Fn, T, EltPerPack, Vec, expr_of_fn_x_y) \ template<> \ struct Apply_Reduce, EltPerPack> { \ __device__ __forceinline__ static BytePack reduce( \ Fn fn, BytePack a, BytePack b \ ) { \ Vec x = fromPack(a); \ Vec y = fromPack(b); \ return toPack(expr_of_fn_x_y); \ } \ }; SPECIALIZE_REDUCE(FuncMinMax, float, 1, float, fn.isMinNotMax ? fminf(x, y) : fmaxf(x, y)) SPECIALIZE_REDUCE(FuncMinMax, double, 1, double, fn.isMinNotMax ? fmin(x, y) : fmax(x, y)) #if __CUDA_ARCH__ >= 530 && __CUDA_ARCH__ != 610 SPECIALIZE_REDUCE(FuncSum, half, 1, half, __hadd(x, y)) // Coverity recommends the use of std::move here but, given that half is a scalar, // a plain copy will be just as efficient. // coverity[copy_constructor_call] SPECIALIZE_REDUCE(FuncSum, half, 2, half2, __hadd2(x, y)) SPECIALIZE_REDUCE(FuncProd, half, 1, half, __hmul(x, y)) // coverity[copy_constructor_call] SPECIALIZE_REDUCE(FuncProd, half, 2, half2, __hmul2(x, y)) #else SPECIALIZE_REDUCE(FuncSum, half, 1, half, __float2half(__half2float(x) + __half2float(y))) SPECIALIZE_REDUCE(FuncProd, half, 1, half, __float2half(__half2float(x) * __half2float(y))) #endif #if __CUDA_ARCH__ >= 800 SPECIALIZE_REDUCE(FuncMinMax, half, 1, half, fn.isMinNotMax ? __hmin(x, y) : __hmax(x, y)) // coverity[copy_constructor_call] SPECIALIZE_REDUCE(FuncMinMax, half, 2, half2, fn.isMinNotMax ? __hmin2(x, y) : __hmax2(x, y)) #else SPECIALIZE_REDUCE(FuncMinMax, half, 1, half, __float2half(fn.isMinNotMax ? fminf(__half2float(x), __half2float(y)) : fmaxf(__half2float(x), __half2float(y)))) #endif #if defined(__CUDA_BF16_TYPES_EXIST__) #if __CUDA_ARCH__ >= 800 SPECIALIZE_REDUCE(FuncSum, __nv_bfloat16, 1, __nv_bfloat16, __hadd(x, y)) // coverity[copy_constructor_call] SPECIALIZE_REDUCE(FuncSum, __nv_bfloat16, 2, __nv_bfloat162, __hadd2(x, y)) SPECIALIZE_REDUCE(FuncProd, __nv_bfloat16, 1, __nv_bfloat16, __hmul(x, y)) // coverity[copy_constructor_call] SPECIALIZE_REDUCE(FuncProd, __nv_bfloat16, 2, __nv_bfloat162, __hmul2(x, y)) SPECIALIZE_REDUCE(FuncMinMax, __nv_bfloat16, 1, __nv_bfloat16, fn.isMinNotMax ? __hmin(x, y) : __hmax(x, y)) // coverity[copy_constructor_call] SPECIALIZE_REDUCE(FuncMinMax, __nv_bfloat16, 2, __nv_bfloat162, fn.isMinNotMax ? __hmin2(x, y) : __hmax2(x, y)) #else SPECIALIZE_REDUCE(FuncSum, __nv_bfloat16, 1, __nv_bfloat16, __float2bfloat16(__bfloat162float(x) + __bfloat162float(y))) SPECIALIZE_REDUCE(FuncProd, __nv_bfloat16, 1, __nv_bfloat16, __float2bfloat16(__bfloat162float(x) * __bfloat162float(y))) SPECIALIZE_REDUCE(FuncMinMax, __nv_bfloat16, 1, __nv_bfloat16, __float2bfloat16(fn.isMinNotMax ? fminf(__bfloat162float(x), __bfloat162float(y)) : fmaxf(__bfloat162float(x), __bfloat162float(y)))) #endif #endif #if defined(__CUDA_FP8_TYPES_EXIST__) #if __CUDA_ARCH__ >= 900 SPECIALIZE_REDUCE(FuncSum, __nv_fp8_e4m3, 1, __nv_fp8_e4m3, __nv_fp8_e4m3(__hadd(__half(x),__half(y)))) SPECIALIZE_REDUCE(FuncSum, __nv_fp8_e4m3, 2, __nv_fp8x2_e4m3, __nv_fp8x2_e4m3(__hadd2(__half2(x),__half2(y)))) SPECIALIZE_REDUCE(FuncProd, __nv_fp8_e4m3, 1, __nv_fp8_e4m3, __nv_fp8_e4m3(__hmul(__half(x),__half(y)))) SPECIALIZE_REDUCE(FuncProd, __nv_fp8_e4m3, 2, __nv_fp8x2_e4m3, __nv_fp8x2_e4m3(__hmul2(__half2(x),__half2(y)))) SPECIALIZE_REDUCE(FuncMinMax, __nv_fp8_e4m3, 1, __nv_fp8_e4m3, __nv_fp8_e4m3(fn.isMinNotMax ? __hmin(__half(x),__half(y)) : __hmax(__half(x),__half(y)))) SPECIALIZE_REDUCE(FuncMinMax, __nv_fp8_e4m3, 2, __nv_fp8x2_e4m3, __nv_fp8x2_e4m3(fn.isMinNotMax ? __hmin2(__half2(x),__half2(y)) : __hmax2(__half2(x),__half2(y)))) SPECIALIZE_REDUCE(FuncSum, __nv_fp8_e5m2, 1, __nv_fp8_e5m2, __nv_fp8_e5m2(__hadd(__half(x),__half(y)))) SPECIALIZE_REDUCE(FuncSum, __nv_fp8_e5m2, 2, __nv_fp8x2_e5m2, __nv_fp8x2_e5m2(__hadd2(__half2(x),__half2(y)))) SPECIALIZE_REDUCE(FuncProd, __nv_fp8_e5m2, 1, __nv_fp8_e5m2, __nv_fp8_e5m2(__hmul(__half(x),__half(y)))) SPECIALIZE_REDUCE(FuncProd, __nv_fp8_e5m2, 2, __nv_fp8x2_e5m2, __nv_fp8x2_e5m2(__hmul2(__half2(x),__half2(y)))) SPECIALIZE_REDUCE(FuncMinMax, __nv_fp8_e5m2, 1, __nv_fp8_e5m2, __nv_fp8_e5m2(fn.isMinNotMax ? __hmin(__half(x), __half(y)) : __hmax(__half(x), __half(y)))) SPECIALIZE_REDUCE(FuncMinMax, __nv_fp8_e5m2, 2, __nv_fp8x2_e5m2, __nv_fp8x2_e5m2(fn.isMinNotMax ? __hmin2(__half2(x), __half2(y)) : __hmax2(__half2(x), __half2(y)))) #endif #endif #undef SPECIALIZE_REDUCE //////////////////////////////////////////////////////////////////////////////// // Apply_PreOp // General recursive definition (EltPerPack > 1) template struct Apply_PreOp { static constexpr bool IsIdentity = Apply_PreOp::IsIdentity; template __device__ __forceinline__ static BytePack preOp(Fn fn, BytePack a) { #if __cpp_if_constexpr if constexpr(!IsIdentity) { #else if (!IsIdentity) { #endif // The `if (!IsIdentity)` condition is not strictly necessary, but it may help // compiler in that it won't have to tear a register apart for no reason // just to put it back together again. a.half[0] = Apply_PreOp::preOp(fn, a.half[0]); a.half[1] = Apply_PreOp::preOp(fn, a.half[1]); } return a; } }; // Base case definition (EltPerPack == 1), by default is identity function. template struct Apply_PreOp { static constexpr bool IsIdentity = true; template __device__ __forceinline__ static BytePack preOp(Fn fn, BytePack a) { return a; } }; // Base case definition (EltPerPack == 0), is nonsense! template struct Apply_PreOp { static constexpr bool IsIdentity = true; __device__ __forceinline__ static BytePack<0> preOp(Fn fn, BytePack<0> a) { return {}; } }; //////////////////////////////////////////////////////////////////////////////// // Apply_PostOp // General recursive definition (EltPerPack > 1) template struct Apply_PostOp { static constexpr bool IsIdentity = Apply_PostOp::IsIdentity; template __device__ __forceinline__ static BytePack postOp(Fn fn, BytePack a) { #if __cpp_if_constexpr if constexpr(!IsIdentity) { #else if (!IsIdentity) { #endif // The `if (!IsIdentity)` condition is not strictly necessary, but it may help // compiler in that it won't have to tear a register apart for no reason // just to put it back together again. a.half[0] = Apply_PostOp::postOp(fn, a.half[0]); a.half[1] = Apply_PostOp::postOp(fn, a.half[1]); } return a; } }; // Base case definition (EltPerPack == 1), by default is identity function. template struct Apply_PostOp { static constexpr bool IsIdentity = true; template __device__ __forceinline__ static BytePack postOp(Fn fn, BytePack a) { return a; } }; // Base case definition (EltPerPack == 0), is nonsense! template struct Apply_PostOp { static constexpr bool IsIdentity = true; __device__ __forceinline__ static BytePack<0> postOp(Fn fn, BytePack<0> a) { return {}; } }; //////////////////////////////////////////////////////////////////////////////// // FuncPreMulSum template struct RedOpArg> { static constexpr bool ArgUsed = true; __device__ __forceinline__ static uint64_t loadArg(void *ptr) { union { uint64_t u64; T val; }; u64 = 0; val = *(T*)ptr; return u64; } }; // General definition for all integral types, float, and double. template struct FuncPreMulSum { using EltType = T; T scalar; __device__ __forceinline__ FuncPreMulSum(uint64_t opArg=0) { union { uint64_t u64; T val; }; u64 = opArg; scalar = val; } }; template<> // Coverity recommends the users of this type to use std::move in certain cases but, // given that half is a scalar, a plain copy will be just as efficient. // coverity[moveable_type] struct FuncPreMulSum { using EltType = half; #if __CUDA_ARCH__ >= 530 && __CUDA_ARCH__ != 610 __half2 scalar; __device__ __forceinline__ FuncPreMulSum(uint64_t opArg=0) { union { uint64_t u64; __half val; }; u64 = opArg; scalar.x = val; scalar.y = val; } #else float scalar; __device__ __forceinline__ FuncPreMulSum(uint64_t opArg=0) { union { uint64_t u64; __half val; }; u64 = opArg; scalar = (float)val; } #endif }; #if defined(__CUDA_BF16_TYPES_EXIST__) template<> // Coverity recommends the users of this type to use std::move in certain cases but, // given that __nv_bfloat16 is a scalar, a plain copy will be just as efficient. // coverity[moveable_type] struct FuncPreMulSum<__nv_bfloat16> { using EltType = __nv_bfloat16; #if __CUDA_ARCH__ >= 800 __nv_bfloat162 scalar; __device__ __forceinline__ FuncPreMulSum(uint64_t opArg=0) { union { uint64_t u64; __nv_bfloat16 val; }; u64 = opArg; scalar.x = val; scalar.y = val; } #else float scalar; __device__ __forceinline__ FuncPreMulSum(uint64_t opArg=0) { union { uint64_t u64; __nv_bfloat16 val; }; u64 = opArg; scalar = __bfloat162float(val); } #endif }; #endif #if defined(__CUDA_FP8_TYPES_EXIST__) #if __CUDA_ARCH__ >= 900 template<> struct FuncPreMulSum<__nv_fp8_e4m3> { using EltType = __nv_fp8_e4m3; __half2 scalar2; __device__ __forceinline__ FuncPreMulSum(uint64_t opArg) { union { uint64_t u64; __nv_fp8_storage_t val; }; u64 = opArg; scalar2.x = __half(__nv_cvt_fp8_to_halfraw(val, __NV_E4M3)); scalar2.y = scalar2.x; } }; template<> struct FuncPreMulSum<__nv_fp8_e5m2> { using EltType = __nv_fp8_e5m2; __half2 scalar2; __device__ __forceinline__ FuncPreMulSum(uint64_t opArg) { union { uint64_t u64; __nv_fp8_storage_t val; }; u64 = opArg; scalar2.x = __half(__nv_cvt_fp8_to_halfraw(val, __NV_E5M2)); scalar2.y = scalar2.x; } }; #endif #endif template struct Apply_Reduce, EltPerPack> { __device__ __forceinline__ static BytePack reduce(FuncPreMulSum fn, BytePack a, BytePack b) { // FuncPreMulSum reduce dispatches to FuncSum. return Apply_Reduce, EltPerPack>::reduce(FuncSum(), a, b); } }; // PreOp of FuncPreMulSum for integral types, float, and double. template struct Apply_PreOp, /*EltPerPack=*/1> { static constexpr bool IsIdentity = false; __device__ __forceinline__ static BytePack preOp(FuncPreMulSum fn, BytePack a) { return toPack(fromPack(a) * fn.scalar); } }; //////////////////////////////////////////////////////////////////////////////// // Apply_PreOp of FuncPreMulSum for float16. template<> struct Apply_PreOp, /*EltPerPack=*/1> { static constexpr bool IsIdentity = false; __device__ __forceinline__ static BytePack preOp(FuncPreMulSum fn, BytePack a) { #if __CUDA_ARCH__ >= 530 && __CUDA_ARCH__ != 610 return toPack(__hmul(fromPack(a), fn.scalar.x)); #else return toPack(__float2half(__half2float(fromPack(a)) * fn.scalar)); #endif } }; #if __CUDA_ARCH__ >= 530 && __CUDA_ARCH__ != 610 template<> struct Apply_PreOp, /*EltPerPack=*/2> { static constexpr bool IsIdentity = false; __device__ __forceinline__ static BytePack preOp(FuncPreMulSum fn, BytePack a) { return toPack(__hmul2(fromPack(a), fn.scalar)); } }; #endif //////////////////////////////////////////////////////////////////////////////// // Apply_PreOp of FuncPreMulSum for bfloat16. #if defined(__CUDA_BF16_TYPES_EXIST__) template<> struct Apply_PreOp, /*EltPerPack=*/1> { static constexpr bool IsIdentity = false; __device__ __forceinline__ static BytePack preOp( FuncPreMulSum<__nv_bfloat16> fn, BytePack a ) { #if __CUDA_ARCH__ >= 800 return toPack<__nv_bfloat16>(__hmul(fromPack<__nv_bfloat16>(a), fn.scalar.x)); #else return toPack<__nv_bfloat16>(__float2bfloat16(__bfloat162float(fromPack<__nv_bfloat16>(a)) * fn.scalar)); #endif } }; #if __CUDA_ARCH__ >= 800 template<> struct Apply_PreOp, /*EltPerPack=*/2> { static constexpr bool IsIdentity = false; __device__ __forceinline__ static BytePack preOp( FuncPreMulSum<__nv_bfloat16> fn, BytePack a ) { return toPack<__nv_bfloat162>(__hmul2(fromPack<__nv_bfloat162>(a), fn.scalar)); } }; #endif #endif //////////////////////////////////////////////////////////////////////////////// // Apply_PreOp of FuncPreMulSum for fp8. #if defined(__CUDA_FP8_TYPES_EXIST__) #if __CUDA_ARCH__ >= 900 template<> struct Apply_PreOp, /*EltPerPack=*/1> { static constexpr bool IsIdentity = false; __device__ __forceinline__ static BytePack preOp( FuncPreMulSum<__nv_fp8_e4m3> fn, BytePack a ) { return toPack<__nv_fp8_e4m3>(__nv_fp8_e4m3(__hmul(__half(fromPack<__nv_fp8_e4m3>(a)), fn.scalar2.x))); } }; template<> struct Apply_PreOp, /*EltPerPack=*/2> { static constexpr bool IsIdentity = false; __device__ __forceinline__ static BytePack preOp( FuncPreMulSum<__nv_fp8_e4m3> fn, BytePack a ) { return toPack<__nv_fp8x2_e4m3>(__nv_fp8x2_e4m3(__hmul2(__half2(fromPack<__nv_fp8x2_e4m3>(a)), fn.scalar2))); } }; template<> struct Apply_PreOp, /*EltPerPack=*/1> { static constexpr bool IsIdentity = false; __device__ __forceinline__ static BytePack preOp( FuncPreMulSum<__nv_fp8_e5m2> fn, BytePack a ) { return toPack<__nv_fp8_e5m2>(__nv_fp8_e5m2(__hmul(__half(fromPack<__nv_fp8_e5m2>(a)), fn.scalar2.x))); } }; template<> struct Apply_PreOp, /*EltPerPack=*/2> { static constexpr bool IsIdentity = false; __device__ __forceinline__ static BytePack preOp( FuncPreMulSum<__nv_fp8_e5m2> fn, BytePack a ) { return toPack<__nv_fp8x2_e5m2>(__nv_fp8x2_e5m2(__hmul2(__half2(fromPack<__nv_fp8x2_e5m2>(a)), fn.scalar2))); } }; #endif #endif //////////////////////////////////////////////////////////////////////////////// // FuncSumPostDiv template struct RedOpArg> { static constexpr bool ArgUsed = true; __device__ __forceinline__ static uint64_t loadArg(void *ptr) { return *(uint64_t*)ptr; } }; template struct FuncSumPostDiv { static_assert(T(0) < T(-1), "FuncSumPostDiv is only for implementing ncclAvg on uint types."); using EltType = T; using UintType = typename std::conditional::type; uint32_t divisor:31, isSigned:1; UintType recip; __device__ __forceinline__ FuncSumPostDiv(uint64_t opArg=0) { isSigned = opArg & 1; divisor = opArg >> 1; recip = UintType(-1)/divisor; } __device__ __forceinline__ T divide(T x) { // x is negative iff we are in signed mode and the top bit is set bool xneg = isSigned && (x & ~(T(-1)>>1)); // Compute abs(x): // T(-x) vs -T(x) is critical. We have to negate then truncate the bits. Consider // if we are doing signed 8-bit types, thus T=uint8_t. The value -1 is encoded // as 0xff. -T(0xff) when promoted to 32-bit (which is implicit by compiler) // gives 0xffffff01, but T(-0xff) is 0x1, and that is the abs value we want. UintType xabs = xneg ? T(-x) : x; // Compute quotient by multiplying by reciprical. UintType q = sizeof(T)==8 ? __umul64hi(xabs, recip) : __umulhi(xabs, recip); // Quotient may be off by one so do a fixup. if (xabs - q*divisor >= divisor) q += 1; // If original x was negative then we have to negate it back since we were // working with its abs val. return xneg ? -T(q) : T(q); } }; template struct Apply_Reduce, EltPerPack>: Apply_Reduce, EltPerPack> { __device__ __forceinline__ static BytePack reduce(FuncSumPostDiv fn, BytePack a, BytePack b) { // FuncSumPostDiv reduce dispatches to FuncSum. return Apply_Reduce, EltPerPack>::reduce(FuncSum(), a, b); } }; template struct Apply_PostOp, /*EltPerPack=*/1> { static constexpr bool IsIdentity = false; __device__ __forceinline__ static BytePack postOp(FuncSumPostDiv fn, BytePack a) { return toPack(fn.divide(fromPack(a))); } }; //////////////////////////////////////////////////////////////////////////////// // Apply_LoadMultimem #define RegCode_for_size_1 "r" #define RegCode_for_size_2 "h" #define RegCode_for_size_4 "r" #define RegCode_for_size_8 "l" #define RegSize_for_size_1 4 #define RegSize_for_size_2 2 #define RegSize_for_size_4 4 #define RegSize_for_size_8 8 #define PtxAcc_for_u32 #define PtxAcc_for_s32 #define PtxAcc_for_s64 #define PtxAcc_for_u64 #define PtxAcc_for_f32 #define PtxAcc_for_f64 #if CUDART_VERSION >= 12020 #define PtxAcc_for_f16 ".acc::f32" #define PtxAcc_for_bf16 ".acc::f32" #define PtxAcc_for_f16x2 ".acc::f32" #define PtxAcc_for_bf16x2 ".acc::f32" #else #define PtxAcc_for_f16 #define PtxAcc_for_bf16 #define PtxAcc_for_f16x2 #define PtxAcc_for_bf16x2 #endif #define PtxAcc_for_e4m3 ".acc::f16" #define PtxAcc_for_e5m2 ".acc::f16" #define PtxAcc_for_e4m3x4 ".acc::f16" #define PtxAcc_for_e5m2x4 ".acc::f16" #define DEFINE_Apply_LoadMultimem_sum(T, ptx_ty, PackSize) \ template<> \ struct Apply_LoadMultimem, PackSize> { \ __device__ __forceinline__ static BytePack load(FuncSum fn, uintptr_t addr) { \ BytePack reg; \ asm volatile("multimem.ld_reduce.relaxed.sys.global.add" PtxAcc_for_##ptx_ty "." #ptx_ty " %0, [%1];" \ : "=" RegCode_for_size_##PackSize(reg.native) \ : "l"(addr) : "memory"); \ BytePack ans; \ ans.native = reg.native; \ return ans; \ } \ }; #define DEFINE_Apply_LoadMultimem_minmax(T, ptx_ty, PackSize) \ template<> \ struct Apply_LoadMultimem, PackSize> { \ __device__ __forceinline__ static BytePack load(FuncMinMax fn, uintptr_t addr) { \ BytePack reg; \ if (fn.isMinNotMax) { \ asm volatile("multimem.ld_reduce.relaxed.sys.global.min." #ptx_ty " %0, [%1];" \ : "=" RegCode_for_size_##PackSize(reg.native) \ : "l"(addr) : "memory"); \ } else { \ asm volatile("multimem.ld_reduce.relaxed.sys.global.max." #ptx_ty " %0, [%1];" \ : "=" RegCode_for_size_##PackSize(reg.native) \ : "l"(addr) : "memory"); \ } \ BytePack ans; \ ans.native = reg.native; \ return ans; \ } \ }; #define DEFINE_Apply_LoadMultimem_sum_v4(T, ptx_ty, VecEltSize) \ template<> \ struct Apply_LoadMultimem, 4*(VecEltSize)> { \ static constexpr int PackSize = 4*(VecEltSize); \ __device__ __forceinline__ static BytePack load(FuncSum fn, uintptr_t addr) { \ union { BytePack ans; BytePack elts[4]; }; \ asm volatile("multimem.ld_reduce.relaxed.sys.global.add" PtxAcc_for_##ptx_ty ".v4." #ptx_ty " {%0,%1,%2,%3}, [%4];" \ : "=" RegCode_for_size_##VecEltSize(elts[0].native), \ "=" RegCode_for_size_##VecEltSize(elts[1].native), \ "=" RegCode_for_size_##VecEltSize(elts[2].native), \ "=" RegCode_for_size_##VecEltSize(elts[3].native) \ : "l"(addr) : "memory"); \ return ans; \ } \ }; #define DEFINE_Apply_LoadMultimem_minmax_v4(T, ptx_ty, VecEltSize) \ template<> \ struct Apply_LoadMultimem, 4*(VecEltSize)> { \ static constexpr int PackSize = 4*(VecEltSize); \ __device__ __forceinline__ static BytePack load(FuncMinMax fn, uintptr_t addr) { \ union { BytePack ans; BytePack elts[4]; }; \ if (fn.isMinNotMax) { \ asm volatile("multimem.ld_reduce.relaxed.sys.global.min.v4." #ptx_ty " {%0,%1,%2,%3}, [%4];" \ : "=" RegCode_for_size_##VecEltSize(elts[0].native), \ "=" RegCode_for_size_##VecEltSize(elts[1].native), \ "=" RegCode_for_size_##VecEltSize(elts[2].native), \ "=" RegCode_for_size_##VecEltSize(elts[3].native) \ : "l"(addr) : "memory"); \ } else { \ asm volatile("multimem.ld_reduce.relaxed.sys.global.max.v4." #ptx_ty " {%0,%1,%2,%3}, [%4];" \ : "=" RegCode_for_size_##VecEltSize(elts[0].native), \ "=" RegCode_for_size_##VecEltSize(elts[1].native), \ "=" RegCode_for_size_##VecEltSize(elts[2].native), \ "=" RegCode_for_size_##VecEltSize(elts[3].native) \ : "l"(addr) : "memory"); \ } \ return ans; \ } \ }; #define DEFINE_Apply_LoadMultimem_sum_v4_and_xparts(T, ptx_ty, VecEltSize) \ DEFINE_Apply_LoadMultimem_sum_v4(T, ptx_ty, VecEltSize) \ template<> \ struct Apply_LoadMultimem, sizeof(T)> { \ __device__ __forceinline__ static BytePack load(FuncSum fn, uintptr_t addr) { \ union { BytePack tmp; BytePack elts[(VecEltSize)/sizeof(T)]; }; \ asm volatile("multimem.ld_reduce.relaxed.sys.global.add" PtxAcc_for_##ptx_ty "." #ptx_ty " %0, [%1];" \ : "=" RegCode_for_size_##VecEltSize(tmp.native) \ : "l"(addr & -uintptr_t(VecEltSize)) : "memory"); \ return elts[(addr/sizeof(T))%((VecEltSize)/sizeof(T))]; \ } \ }; #define DEFINE_Apply_LoadMultimem_minmax_v4_and_xparts(T, ptx_ty, VecEltSize) \ DEFINE_Apply_LoadMultimem_minmax_v4(T, ptx_ty, VecEltSize) \ template<> \ struct Apply_LoadMultimem, sizeof(T)> { \ __device__ __forceinline__ static BytePack load(FuncMinMax fn, uintptr_t addr) { \ union { BytePack tmp; BytePack elts[(VecEltSize)/sizeof(T)]; }; \ if (fn.isMinNotMax) { \ asm volatile("multimem.ld_reduce.relaxed.sys.global.min." #ptx_ty " %0, [%1];" \ : "=" RegCode_for_size_##VecEltSize(tmp.native) \ : "l"(addr & -uintptr_t(VecEltSize)) : "memory"); \ } else { \ asm volatile("multimem.ld_reduce.relaxed.sys.global.max." #ptx_ty " %0, [%1];" \ : "=" RegCode_for_size_##VecEltSize(tmp.native) \ : "l"(addr & -uintptr_t(VecEltSize)) : "memory"); \ } \ return elts[(addr/sizeof(T))%((VecEltSize)/sizeof(T))]; \ } \ }; template struct Apply_LoadMultimem { __device__ __forceinline__ static BytePack load(Fn fn, uintptr_t addr) { __trap(); return {}; } }; #if __CUDA_ARCH__ >= 900 && CUDART_VERSION >= 12010 template struct LoadMultimem_BigPackSize { using T = typename Fn::EltType; static constexpr bool IsSum = std::is_same>::value || std::is_same>::value || std::is_same>::value; static constexpr bool IsMinMax = std::is_same>::value; static constexpr bool IsFloat = IsFloatingPoint::value; static constexpr int BigPackSize = IsFloat && IsSum && sizeof(T) < 8 ? 16 : IsFloat && IsSum ? sizeof(T) : IsFloat && IsMinMax && sizeof(T)==2 ? 16 : !IsFloat && (IsSum||IsMinMax) && sizeof(T)>=4 ? sizeof(T) : /*multimem.ld_reduce not supported:*/ 0; }; DEFINE_Apply_LoadMultimem_sum(uint32_t, u32, 4) DEFINE_Apply_LoadMultimem_minmax(uint32_t, u32, 4) DEFINE_Apply_LoadMultimem_sum(int32_t, s32, 4) DEFINE_Apply_LoadMultimem_minmax(int32_t, s32, 4) DEFINE_Apply_LoadMultimem_sum(uint64_t, u64, 8) DEFINE_Apply_LoadMultimem_minmax(uint64_t, u64, 8) DEFINE_Apply_LoadMultimem_sum(int64_t, u64, 8) DEFINE_Apply_LoadMultimem_minmax(int64_t, s64, 8) DEFINE_Apply_LoadMultimem_sum(float, f32, 4) DEFINE_Apply_LoadMultimem_sum_v4(float, f32, 4) DEFINE_Apply_LoadMultimem_sum(double, f64, 8) DEFINE_Apply_LoadMultimem_sum_v4_and_xparts(half, f16x2, 4) DEFINE_Apply_LoadMultimem_minmax_v4_and_xparts(half, f16x2, 4) #if defined(__CUDA_BF16_TYPES_EXIST__) DEFINE_Apply_LoadMultimem_sum_v4_and_xparts(__nv_bfloat16, bf16x2, 4) DEFINE_Apply_LoadMultimem_minmax_v4_and_xparts(__nv_bfloat16, bf16x2, 4) #endif #if NCCL_CUDA_ARCH_SPECIFIC == 1000 || NCCL_CUDA_ARCH_SPECIFIC == 1010 || NCCL_CUDA_ARCH_FAMILY_SPECIFIC == 1000 || NCCL_CUDA_ARCH_FAMILY_SPECIFIC == 1010 || NCCL_CUDA_ARCH_SPECIFIC == 1200 || NCCL_CUDA_ARCH_SPECIFIC == 1210 DEFINE_Apply_LoadMultimem_sum_v4_and_xparts(__nv_fp8_e4m3, e4m3x4, 4) DEFINE_Apply_LoadMultimem_minmax_v4_and_xparts(__nv_fp8_e4m3, e4m3x4, 4) DEFINE_Apply_LoadMultimem_sum_v4_and_xparts(__nv_fp8_e5m2, e5m2x4, 4) DEFINE_Apply_LoadMultimem_minmax_v4_and_xparts(__nv_fp8_e5m2, e5m2x4, 4) #endif #else template struct LoadMultimem_BigPackSize { static constexpr int BigPackSize = 0; }; #endif #undef DEFINE_Apply_LoadMultimem #undef DEFINE_Apply_LoadMultimem_v4 #undef DEFINE_Apply_LoadMultimem_v4x2_and_subhalf #undef RegCode_for_size_2 #undef RegCode_for_size_4 #undef RegCode_for_size_8 #undef RegSize_for_size_1 #undef RegSize_for_size_2 #undef RegSize_for_size_4 #undef RegSize_for_size_8 #undef PtxAcc_for_u32 #undef PtxAcc_for_s32 #undef PtxAcc_for_s64 #undef PtxAcc_for_u64 #undef PtxAcc_for_f32 #undef PtxAcc_for_f64 #undef PtxAcc_for_f16 #undef PtxAcc_for_bf16 #undef PtxAcc_for_f16x2 #undef PtxAcc_for_bf16x2 #undef PtxAcc_for_e4m3 #undef PtxAcc_for_e5m2 #undef PtxAcc_for_e4m3x4 #undef PtxAcc_for_e5m2x4 #endif // REDUCE_KERNEL_H_ nccl-2.29.7-1/src/device/reduce_scatter.h000066400000000000000000000553711515037102200200720ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "device.h" #include "collectives.h" #include "primitives.h" namespace { template __device__ __forceinline__ void runRing(int tid, int nthreads, struct ncclDevWorkColl* work) { ncclRing *ring = &ncclShmem.channel.ring; int const *ringRanks = ring->userRanks; const int nranks = ncclShmem.comm.nRanks; size_t count; size_t gridOffset; size_t channelCount; size_t chunkCount; ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), &count, &gridOffset, &channelCount, &chunkCount); size_t offset; size_t dataOffset; uint32_t nelem; int rankDest; // Coverity reports that the callee treats &ring->next as an array. However, due to the use of // FanSymmetric<1>, only the first element is ever accessed, so it's fine. // coverity[callee_ptr_arith:FALSE] Primitives, 0, Proto, 0> prims(tid, nthreads, &ring->prev, &ring->next, work->sendbuff, work->recvbuff, work->redOpArg); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { nelem = min(chunkCount, channelCount - elemOffset); dataOffset = gridOffset + elemOffset; /////////////// begin ReduceScatter steps /////////////// // step 0: push data to next GPU rankDest = ringRanks[nranks-1]; offset = dataOffset + rankDest * count; prims.send(offset, nelem); // k-2 steps: reduce and copy to next GPU for (int j=2; j struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { using Proto = ProtoSimple; runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { runRing(tid, nthreads, work); } }; template struct RunWorkColl { __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { #if __CUDA_ARCH__ >= 600 using Proto = ProtoSimple<1, 1>; const int nranks = ncclShmem.comm.nRanks; const int rank = ncclShmem.comm.rank; size_t count, channelOffset, channelCount, chunkCount; ncclCollCbdPart(work, ncclShmem.channelId, Proto::Id, sizeof(T), &count, &channelOffset, &channelCount, &chunkCount); static constexpr int nworkers = NCCL_PAT_NWORKERS; struct ncclPatShmem* shmem = (struct ncclPatShmem*)ncclScratchForWarp(0); uint64_t pollCount = 0; __syncthreads(); // Don't start using shared mem until everyone arrives for (int i=tid; ipatSteps[i].flags = 0; if (tid == 0) shmem->localAccSize = 0; if (tid == nworkers) shmem->parallelFactor = 0; __syncthreads(); if (tid == nworkers) { // Algo computation thread PatRSAlgorithm patAlgo(chunkCount*sizeof(T), NCCL_STEPS, NCCL_PAT_NWORKERS/WARP_SIZE, channelOffset, channelOffset + channelCount, count, chunkCount, rank, nranks); int parallelFactor = shmem->parallelFactor = patAlgo.getParallelFactor(); int step = 0; while (1) { struct ncclPatStep* ps = shmem->patSteps+(step%NCCL_SHMEM_PAT_STEPS); cuda::atomic_ref poll(ps->flags); while (poll.load(cuda::memory_order_acquire) != 0) pollCount++; // Wait for workers to be done with step 'step-NCCL_SHMEM_PAT_STEPS' patAlgo.getNextOp(ps); int last = ps->last; step++; if (last == 2) break; } } else if (tid < nworkers) { // Worker threads T *inputBuf = (T*)work->sendbuff; T *outputBuf = (T*)work->recvbuff; int parallelFactor = 0; volatile int* pfPtr = &shmem->parallelFactor; while (parallelFactor == 0) parallelFactor = *pfPtr; int groupSize = nworkers/(WARP_SIZE*parallelFactor) * WARP_SIZE; int group = tid / groupSize; int nGroups = nworkers / groupSize; int tidInGroup = tid - group*groupSize; // We don't use recvPeers/sendPeers so let's pass shmem structs instead Primitives, 0, Proto, 0> prims (tidInGroup, groupSize, (int*)shmem->recvDims, (int*)shmem->sendDims, inputBuf, outputBuf, work->redOpArg, group, 0, 0, nullptr, nullptr, 0, primsModePatRs); int step = group; while(1) { struct ncclPatStep* ps = shmem->patSteps+(step%NCCL_SHMEM_PAT_STEPS); cuda::atomic_ref poll(ps->flags); while (poll.load(cuda::memory_order_acquire) == 0) pollCount++; // Wait for compute thread int last = ps->last; prims.patReduce(ps, shmem); if (tidInGroup == 0) poll.store(0, cuda::memory_order_release); // Return element to compute thread if (last) break; step += nGroups; } } #endif } }; template struct RunWorkColl { template struct Scatterer { struct ncclDevWorkColl* work; int chunkCount; ssize_t railGridOffset; template __device__ __forceinline__ void operator()( int tid, int tn, int slice, int maxSliceSize, int nSrcs, void** srcPtrs, int nDsts, void** dstPtrs, int32_t* dstSizes, uint32_t sendDirectFlag, uint32_t recvDirectFlag ) { static_assert(SlicePerChunk == 1, "require: SlicePerChunk==1"); static_assert(MaxDsts <= 1 || MaxSrcs <= 1, "require: MaxDsts<=1 || MaxSrcs<=1"); struct ncclNvls* nvls = &ncclShmem.channel.nvls; int nNodes = ncclShmem.comm.nNodes; int nRails = nvls->nHeads; int part = ncclShmem.channelId - work->channelLo; void* inbuf = (void*)work->sendbuff; ssize_t countPerRank = work->collnet.count; ssize_t railAllBeg = min(railGridOffset + part * chunkCount, nNodes * countPerRank); ssize_t railAllEnd = min(railAllBeg + chunkCount, nNodes * countPerRank); int railAllSize = railAllEnd - railAllBeg; int rail = nvls->headRank; int dst = 0; if (ReduceSendNotRecv) { if (work->regUsed) return; rail = 0; nSrcs = 1; } else { rail = nvls->headRank; } if (tid < nDsts) dstSizes[tid] = railAllSize; do { int node = railAllBeg / countPerRank; int railAllOffset = 0; while (railAllOffset < railAllSize) { ssize_t railOneBeg = node * countPerRank; ssize_t railOneEnd = railOneBeg + countPerRank; ssize_t railOneOffset = (railAllBeg + railAllOffset) - railOneBeg; int delta = min(railAllEnd, railOneEnd) - (railAllBeg + railAllOffset); int rank = ncclShmem.comm.collNetDenseToUserRank[node * nRails + rail]; ssize_t userOneBeg = rank * countPerRank + railOneOffset; if (nDsts != 0) { reduceCopy (tid, tn, work->redOpArg, false, /*nSrcs=*/nSrcs, [=]__device__(int s) { return work->regUsed ? (T*)srcPtrs[s] + userOneBeg : !ReduceSendNotRecv ? (T*)srcPtrs[s] + railAllOffset: (T*)inbuf + userOneBeg; }, /*nDsts=*/1, [=]__device__(int d/*==0*/) { return (T*)dstPtrs[dst] + railAllOffset; }, delta); } railAllOffset += delta; node += 1; } dst += 1; rail += 1; } while (ReduceSendNotRecv && dst < nRails); } }; __device__ __forceinline__ void run(int tid, int/*nthreads*/, struct ncclDevWorkColl* work) { struct ncclNvls* nvls = &ncclShmem.channel.nvls; int nelem; /* if we are direct NVLS, we only need to allocate 1 warp to scatter for sync; * if not, based on #ranks, we allocate 7 or 5 warps to reduce to saturate bandwidth * and the rest are allocated to scatter. */ const int nThreadsNetRecv = work->oneNode ? 0 : (work->netRegUsed ? WARP_SIZE : 6 * WARP_SIZE); const int nThreadsScatter = work->regUsed ? roundUp(nvls->nHeads << 2, WARP_SIZE) : 8 * WARP_SIZE; const int nThreadsReduce = NCCL_MAX_NTHREADS - nThreadsNetRecv - nThreadsScatter; const int tidEndNetRecv = nThreadsNetRecv; const int tidEndScatter = tidEndNetRecv + nThreadsScatter; const int tidEndReduce = tidEndScatter + nThreadsReduce; if (work->oneNode) { const int rank = ncclShmem.comm.rank; size_t offset; size_t count, gridOffset, channelCount, chunkCount; ncclCollCbdPart(work, ncclShmem.channelId, NCCL_PROTO_SIMPLE, sizeof(T), &count, &gridOffset, &channelCount, &chunkCount); if (!work->regUsed) { if (tid < tidEndScatter) { // Scatter using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid, nThreadsScatter, NULL, nvls->up, work->sendbuff, NULL, work->redOpArg, 0 * Proto::MaxGroupWidth, 1, 1); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.scatter(offset, nvls->nHeads * count, nelem, count, -1, 0); } // coverity[overrun-call] => Coverity think prims.index can be greater than 1 } else if (tid < tidEndReduce) { // Reduce through NVLS using Proto = ProtoSimple<1, 1, COLL_UNROLL, 1, 0>; Primitives, /*Direct=*/0, Proto, 0> prims(tid - tidEndScatter, nThreadsReduce, &nvls->down, NULL, NULL, work->recvbuff, work->redOpArg, 3 * Proto::MaxGroupWidth, 0, 0); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { offset = gridOffset + elemOffset; nelem = min(chunkCount, channelCount - elemOffset); prims.recv(offset, nelem); } } } else { if (tid < tidEndScatter) { // Scatter using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/0, Proto, 0> prims(tid, nThreadsScatter, nvls->up, nvls->up, NULL, NULL, work->redOpArg, 0 * Proto::MaxGroupWidth, 1, 1); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { prims.scatter(0, 0, 0, 0, -1, 0); } /* gather used as sync */ prims.gather(0, 0, 0, 0, -1, 0); } else if (tid < tidEndReduce) { // Reduce through NVLS using Proto = ProtoSimple<1, 1, COLL_UNROLL, 1, 0>; Primitives, /*Direct=*/1, Proto, 0> prims(tid - tidEndScatter, nThreadsReduce, &nvls->down, &nvls->down, NULL, work->recvbuff, work->redOpArg, 3 * Proto::MaxGroupWidth, 0, 0, work); for (size_t elemOffset = 0; elemOffset < channelCount; elemOffset += chunkCount) { size_t outOffset = gridOffset + elemOffset; size_t inpOffset = outOffset + rank * count; nelem = min(chunkCount, channelCount - elemOffset); // Coverity complains about a possible overrun inside the method invoked below, but that's actually // a false positive. // coverity[overrun-call:FALSE] prims.directRecvCopy(inpOffset, outOffset, nelem); } /* send for sync */ prims.send(0, 0); } } } else { // multi-node int nNodes = ncclShmem.comm.nNodes; int part = ncclShmem.channelId - work->channelLo; ssize_t countPerRank = work->collnet.count; const int nChannels = work->channelHi - work->channelLo + 1; ssize_t chunkCount = work->collnet.chunkCount; if (tid < tidEndNetRecv) { using Proto = ProtoSimple<1, 1, COLL_UNROLL>; if (work->netRegUsed) { if (tid == 0) { int steps = (int)divUp(nNodes * countPerRank, nChannels * chunkCount); Primitives, /*Direct=*/0, Proto, 0>::recvPeerNotify(nvls->out, 0, steps); } __syncwarp(); } else { Primitives, /*Direct=*/0, Proto, 0> prims(tid, nThreadsNetRecv, &nvls->out, nullptr, nullptr, work->recvbuff, work->redOpArg, 0 * Proto::MaxGroupWidth, 0, 0); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkCount) { ssize_t railAllBeg = railGridOffset + part * chunkCount; ssize_t railAllEnd = min(railAllBeg + chunkCount, nNodes * countPerRank); ssize_t railOneBeg = ncclShmem.comm.node * countPerRank; ssize_t railOneEnd = railOneBeg + countPerRank; ssize_t beg = max(railAllBeg, railOneBeg); ssize_t end = min(railAllEnd, railOneEnd); prims.recv(beg - railOneBeg, max(ssize_t(0), end - beg), /*postOp=*/true); } } } else { if (tid < tidEndScatter) { using Proto = ProtoSimple<1, 1, COLL_UNROLL>; Primitives, /*Direct=*/1, Proto, 0> prims(tid - tidEndNetRecv, nThreadsScatter, nullptr, nvls->up, work->sendbuff, nullptr, work->redOpArg, 1 * Proto::MaxGroupWidth, 1, 1, work); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkCount) { Scatterer scat; scat.work = work; scat.chunkCount = chunkCount; scat.railGridOffset = railGridOffset; prims.template process(scat); } } else if (tid < tidEndReduce) { using Proto = ProtoSimple<1, 1, COLL_UNROLL, 1, 0>; Primitives, /*Direct=*/1, Proto, 0> prims(tid - tidEndScatter, nThreadsReduce, &nvls->down, &nvls->out, nullptr, nullptr, work->redOpArg, 2 * Proto::MaxGroupWidth, 0, 1, work); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkCount) { Scatterer scat; scat.work = work; scat.chunkCount = chunkCount; scat.railGridOffset = railGridOffset; prims.template process(scat); } } } } } }; template struct RunWorkColl { template struct Scatterer { struct ncclDevWorkColl* work; int chunkSize; ssize_t railGridOffset; template __device__ __forceinline__ void operator()( int tid, int tn, int slice, int maxSliceSize, int nSrcs, void** srcPtrs, int nDsts, void** dstPtrs, int32_t* dstSizes, uint32_t sendDirectFlag, uint32_t recvDirectFlag ) { static_assert(SlicePerChunk==1, "require: SlicePerChunk==1"); static_assert(MaxDsts<=1 || MaxSrcs<=1, "require: MaxDsts<=1 || MaxSrcs<=1"); struct ncclDirect* direct = &ncclShmem.channel.collnetDirect; int nNodes = ncclShmem.comm.nNodes; int nRails = direct->nHeads; int part = ncclShmem.channelId - work->channelLo; void* inbuf = (void*)work->sendbuff; ssize_t countPerRank = work->collnet.count; ssize_t railAllBeg = min(railGridOffset + part*chunkSize, nNodes*countPerRank); ssize_t railAllEnd = min(railAllBeg + chunkSize, nNodes*countPerRank); int railAllSize = railAllEnd - railAllBeg; if (tid < nDsts) dstSizes[tid] = railAllSize; int dst = 0; int rail; if (!ReduceSendNotRecv) { rail = direct->headRank; } else { rail = direct->headRank+1; if (rail == nRails) rail = 0; } do { int node = railAllBeg/countPerRank; int railAllOffset = 0; while (railAllOffset < railAllSize) { ssize_t railOneBeg = node*countPerRank; ssize_t railOneEnd = railOneBeg + countPerRank; ssize_t railOneOffset = (railAllBeg+railAllOffset) - railOneBeg; int delta = min(railAllEnd, railOneEnd) - (railAllBeg+railAllOffset); int rank = ncclShmem.comm.collNetDenseToUserRank[node*nRails + rail]; ssize_t userOneBeg = rank*countPerRank + railOneOffset; if (nDsts != 0) { reduceCopy (tid, tn, work->redOpArg, false, /*nSrcs=*/1+nSrcs, [=]__device__(int s) { return s==0 ? (T*)inbuf + userOneBeg : work->regUsed && (recvDirectFlag & NCCL_P2P_READ) ? (T*)srcPtrs[s-1] + userOneBeg : (T*)srcPtrs[s-1] + railAllOffset; }, /*nDsts=*/1, [=]__device__(int d/*==0*/) { return (T*)dstPtrs[dst] + railAllOffset; }, delta); } railAllOffset += delta; node += 1; } dst += 1; rail += 1; if (rail == nRails) rail = 0; } while (ReduceSendNotRecv && dst < nRails-1); } }; __device__ __forceinline__ void run(int tid, int nthreads, struct ncclDevWorkColl* work) { const int part = ncclShmem.channelId - work->channelLo; const int nChannels = work->channelHi - work->channelLo + 1; struct ncclDirect* direct = &ncclShmem.channel.collnetDirect; int const &nNodes = ncclShmem.comm.nNodes; ssize_t chunkSize = int(work->collnet.chunkCount); ssize_t countPerRank = work->collnet.count; const int hasDn = (direct->down[0] >= 0) ? 1 : 0; if (direct->out == -1) __trap(); bool isMultiRail = (direct->nHeads > 1); int nWarps1 = (isMultiRail ? 2 : 0); int nWarps2 = (isMultiRail ? 2 : 1); int nWarps3 = 1; float denom = float(work->nWarps)/float(nWarps1+nWarps2+nWarps3); nWarps3 = int(denom*nWarps3); nWarps2 = int(denom*nWarps2); nWarps1 = work->nWarps - (nWarps2+nWarps3); using Proto = ProtoSimple<1, 1>; int tn = nWarps1*WARP_SIZE; if (tid < tn) { // Phase 1: Scatter inputs to peers Primitives, /*Direct=*/0, Proto, 0> prims(tid, tn, nullptr, direct->heads+1, work->sendbuff, nullptr, work->redOpArg, 0*Proto::MaxGroupWidth, 1, 1); for (ssize_t railGridOffset=0; railGridOffset < nNodes*countPerRank; railGridOffset += nChannels*chunkSize) { Scatterer scat; scat.work = work; scat.chunkSize = chunkSize; scat.railGridOffset = railGridOffset; prims.template process(scat, 0, 0); } return; } tid -= tn; tn = nWarps2*WARP_SIZE; if (tid < tn) { if (work->netRegUsed && !hasDn) { if (tid == 0) { Primitives, /*Direct=*/0, Proto, 0>::sendPeerNotify(direct->out, 1, 1); } __syncwarp(); } else { // Phase 2: Reduce from peers + local input -> send to network Primitives, /*Direct=*/0, Proto, 0> prims(tid, tn, direct->heads + 1, &direct->out, nullptr, nullptr, work->redOpArg, 1 * Proto::MaxGroupWidth, 1, 1); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkSize) { Scatterer scat; scat.work = work; scat.chunkSize = chunkSize; scat.railGridOffset = railGridOffset; prims.template process(scat, 0, 0); } } return; } tid -= tn; tn = nWarps3*WARP_SIZE; if (tid < tn) { if (work->netRegUsed) { if (tid == 0) { int steps = hasDn ? (int)divUp(nNodes * countPerRank, nChannels * chunkSize) : 1; Primitives, /*Direct=*/0, Proto, 0>::recvPeerNotify(direct->out, 0, steps); } __syncwarp(); } else { // Phase 3: recv from network Primitives, /*Direct=*/0, Proto, 0> prims(tid, tn, &direct->out, nullptr, nullptr, work->recvbuff, work->redOpArg, 2 * Proto::MaxGroupWidth, 0, 0); for (ssize_t railGridOffset = 0; railGridOffset < nNodes * countPerRank; railGridOffset += nChannels * chunkSize) { ssize_t railAllBeg = railGridOffset + part * chunkSize; ssize_t railAllEnd = min(railAllBeg + chunkSize, nNodes * countPerRank); ssize_t railOneBeg = ncclShmem.comm.node * countPerRank; ssize_t railOneEnd = railOneBeg + countPerRank; ssize_t beg = max(railAllBeg, railOneBeg); ssize_t end = min(railAllEnd, railOneEnd); prims.recv(beg - railOneBeg, max(ssize_t(0), end - beg), /*postOp=*/true); } } return; } } }; nccl-2.29.7-1/src/device/sendrecv.h000066400000000000000000000170131515037102200166760ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "device.h" #include "collectives.h" #include "primitives.h" template struct RunWorkBatch { static_assert(sizeof(T)==1, "SendRecv only works on single byte types T."); template __device__ void runSend(int tid, int tn, int group, struct ncclDevWorkP2p* work) { size_t bytes = work->sendBytes; bool useLargeChunk = (work->sendIpcReg && ncclShmem.comm.isAllNvlink) || work->sendNetReg; int chunkSize = useLargeChunk ? NCCL_MAX_NET_SIZE : u32fp8Decode(work->sendChunkSize_u32fp8); int stepSize = useLargeChunk ? NCCL_MAX_NET_SIZE : ncclShmem.comm.p2pChunkSize; Primitives, 1, Proto, 1> prims(tid, tn, nullptr, &work->sendRank, work->sendAddr, nullptr, /*redOpArg(ignored)=*/0, group, 1, 1, nullptr, work, stepSize); size_t cursor = 0; do { int n = min(size_t(chunkSize), bytes-cursor); prims.directSend(cursor, cursor, n); cursor += n; } while (cursor < bytes); } template __device__ void runRecv(int tid, int tn, int group, struct ncclDevWorkP2p* work) { size_t bytes = work->recvBytes; bool useLargeChunk = (work->recvIpcReg && ncclShmem.comm.isAllNvlink) || work->recvNetReg; int chunkSize = useLargeChunk ? NCCL_MAX_NET_SIZE : u32fp8Decode(work->recvChunkSize_u32fp8); int stepSize = useLargeChunk ? NCCL_MAX_NET_SIZE : ncclShmem.comm.p2pChunkSize; Primitives, 1, Proto, 1> prims(tid, tn, &work->recvRank, nullptr, nullptr, work->recvAddr, /*redOpArg(ignored)=*/0, group, 1, 1, nullptr, work, stepSize); size_t cursor = 0; do { int n = min(size_t(chunkSize), bytes-cursor); prims.directRecv(cursor, n); cursor += n; } while (cursor < bytes); } __device__ __forceinline__ void run() { const int tid = threadIdx.x; const int tn = blockDim.x; const int wid = tid/WARP_SIZE; const int nWarps = tn/WARP_SIZE; const int lane = tid%WARP_SIZE; struct Shared { uint32_t workSendMask; // bitmasks of which work indices have send/recv uint32_t workRecvMask; }; Shared* shared = (Shared*)ncclScratchForWarp(0); struct ncclDevWorkP2p* works = (ncclDevWorkP2p*)ncclShmem.workStorage; int nWorks = ncclShmem.nWorks; if (wid == 0) { // Modify the memory range of each work[] to reflect this channel's // partition of the work. Since integer divides are very heavy it's // best to do them all in one warp. int workIx = lane%16; int isSend = lane < 16 ? 0 : 1; bool hasWork = false; if (workIx < nWorks) { struct ncclDevWorkP2p* work = &works[workIx]; size_t bytes = isSend ? work->sendBytes : work->recvBytes; int nParts = isSend ? work->nSendChannels : work->nRecvChannels; int part = ncclP2pChannelToPart(work->nP2pChannels, work->channelBase, ncclShmem.channelId); hasWork = (part < nParts); if (nParts != 0) { size_t partBeg, partEnd; ncclP2pPartBounds(nParts, part, bytes, &partBeg, &partEnd); (isSend ? work->sendAddr : work->recvAddr) = (char*)(isSend ? work->sendAddr : work->recvAddr) + partBeg; (isSend ? work->sendBytes : work->recvBytes) = partEnd - partBeg; } } // Coverity reports a possible thread divergence due to not all threads participating in the collective. // However, the code ensures that the participation is on a per-warp basis. // coverity[device_thread_diverged:FALSE] uint32_t mask = __ballot_sync(~0u, hasWork); if (lane == 0) { shared->workSendMask = mask>>16; shared->workRecvMask = mask & 0xffff; } } // The fastest way to compute a warp uniform division x/y in [0,32) is to // use each lane to guess a solution and count the ones that don't exceed // the numerator: // __popc(__ballot_sync(~0u, y*(lane+1) <= x)) // That takes 1/3 the time of standard division and about 3/4 the time of // approximate floating point division: // __float2int_rd(__fdividef(float(x),float(y))). // nWarpPerWork = nWarps/nWorks int nWarpPerWork = __popc(__ballot_sync(~0u, nWorks*(lane+1) <= nWarps)); int nRecvWarpPerWork = nWarpPerWork<=4 ? nWarpPerWork/2 : (nWarpPerWork-1)/2; int nSendWarpPerWork = nWarpPerWork<=4 ? nRecvWarpPerWork : nRecvWarpPerWork+1; // This might reduce nWarpPerWork which is probably desirable. It is better // to have a balanced number of reading and writing threads even if that // leaves warps unused. nWarpPerWork = nSendWarpPerWork + nRecvWarpPerWork; // The work index this warp belongs to: workIx = wid/nWarpPerWork int workIx = __popc(__ballot_sync(~0u, (lane+1)*nWarpPerWork <= wid)); __syncthreads(); // Wait for works[] and shared->* to be updated by warp=0 uint32_t workSendMask = shared->workSendMask; uint32_t workRecvMask = shared->workRecvMask; __syncthreads(); // release scratch space used by shared->* if (nWorks <= workIx) return; // Thread range for whole work (send & recv combined) int subtid = tid - workIx*nWarpPerWork*WARP_SIZE; int subtn = nWarpPerWork*WARP_SIZE; // A send primtive of sufficient size requires 2 cuda barrier ids. constexpr int nSendWarpsForExtraGroup = NCCL_SIMPLE_EXTRA_GROUP_IF_NTHREADS_GE/WARP_SIZE; // Count up all group ids used below this workIx: int group, extra; // Each recv gets one group id: group = __popc(workRecvMask & ((1<= nSendWarpsForExtraGroup) ? 1 : 0; group += __popc((workSendMask & workRecvMask) & ((1<= nSendWarpsForExtraGroup) ? 1 : 0; group += __popc((workSendMask & ~workRecvMask) & ((1<>workIx); bool hasRecv = 1 & (workRecvMask>>workIx); bool isCopy = work->sendRank == ncclShmem.comm.rank; bool isSend = !hasRecv || (hasSend && subtid < nSendWarpPerWork*WARP_SIZE); if (!isCopy && hasSend && hasRecv) { // Translate thread ids to reflect just this send or recv as opposed to whole work. if (isSend) { subtn = nSendWarpPerWork*WARP_SIZE; } else { subtid -= nSendWarpPerWork*WARP_SIZE; subtn = nRecvWarpPerWork*WARP_SIZE; group += 1 + (nSendWarpPerWork >= nSendWarpsForExtraGroup ? 1 : 0); } } if (isCopy) { reduceCopy (subtid, subtn, 0, false, 1, &work->sendAddr, 1, &work->recvAddr, (ssize_t)work->sendBytes); } else if (isSend) { if (work->sendProtoLL) { runSend(subtid, subtn, group, work); } else { runSend>(subtid, subtn, group, work); } } else { if (work->recvProtoLL) { runRecv(subtid, subtn, group, work); } else { runRecv>(subtid, subtn, group, work); } } } }; nccl-2.29.7-1/src/device/symmetric/000077500000000000000000000000001515037102200167265ustar00rootroot00000000000000nccl-2.29.7-1/src/device/symmetric/all_gather.cuh000066400000000000000000000272731515037102200215440ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "sym_kernels.h" #include "kernel.cuh" #include "primitives.cuh" template static __device__ void bcastDeep( ncclSymkArgsHandler const& handler, int tn, int t, bool waitNeeded, ncclLsaBarrierSession& bar, ncclSymPtr input, ncclSymPtr output, bool inPlace, int nIters ) { using Pack = BytePack; int wn = tn/WARP_SIZE; int w = t/WARP_SIZE; int lane = t%WARP_SIZE; int const& rank = handler.comm.rank; int const& nRanks = handler.comm.nRanks; Pack* inpPacks = (Pack*)input.localPtr() + intptr_t(w)*UnrollPacks*WARP_SIZE + lane; ncclSymPtr outPacks = (ncclSymPtr)output + intptr_t(w)*UnrollPacks*WARP_SIZE + lane; Pack tmp[UnrollPacks]; nIters -= w; if (0 < nIters) { #pragma unroll for (int u=0; u < UnrollPacks; u++) { tmp[u] = inpPacks[u*WARP_SIZE]; } } if (waitNeeded) bar.wait(ncclCoopCta(), cuda::memory_order_relaxed); if (0 < nIters) { while (true) { int dr = inPlace ? 1 : 0; int r = rank + dr; if (r == nRanks) r = 0; #pragma unroll 2 for (int partial=0; partial <= 1; partial++) { #pragma unroll 1 for (int i = 0; partial ? i < 1 : (dr + UnrollPeers <= nRanks); partial ? i++ : (dr += UnrollPeers)) { #pragma unroll for (int ur=0; ur < UnrollPeers-partial; ur++) { if (partial && dr == nRanks) break; #pragma unroll UnrollPacks for (int u=0; u < UnrollPacks; u++) { outPacks.lsaPtr(r)[u*WARP_SIZE] = tmp[u]; } if (++r == nRanks) r = 0; } } } inpPacks += intptr_t(wn)*UnrollPacks*WARP_SIZE; outPacks += intptr_t(wn)*UnrollPacks*WARP_SIZE; nIters -= wn; if (nIters <= 0) break; // Load data for next iteration. #pragma unroll for (int u=0; u < UnrollPacks; u++) { tmp[u] = inpPacks[u*WARP_SIZE]; } } } } template static __device__ void bcastEnds( ncclSymkArgsHandler const& handler, int tn, int t, ncclSymPtr input, ncclSymPtr output, bool inPlace, size_t nElts, uint32_t nPreElts, size_t nSufElts ) { int const& rank = handler.comm.rank; int const& nRanks = handler.comm.nRanks; BytePack* inpPacks = (BytePack*)input.localPtr(); ncclSymPtr> outPacks = (ncclSymPtr>)output; #pragma unroll 1 for (size_t i = t; i < nPreElts+nSufElts; i += tn) { size_t elt = i < nPreElts ? i : nElts-nPreElts-nSufElts+i; BytePack tmp = inpPacks[elt]; int dr = inPlace ? 1 : 0; int r = rank + dr; if (r == nRanks) r = 0; #pragma unroll 1 for (; dr + UnrollPeers <= nRanks; dr += UnrollPeers) { #pragma unroll UnrollPeers for (int u=0; u < UnrollPeers; u++) { outPacks.lsaPtr(r)[elt] = tmp; if (++r == nRanks) r = 0; } } #pragma unroll UnrollPeers for (int u=0; u < UnrollPeers; u++) { if (dr+u == nRanks) break; outPacks.lsaPtr(r)[elt] = tmp; if (++r == nRanks) r = 0; } } } template static __device__ void bcast( ncclSymkArgsHandler const& handler, int tn, int t, int nBlocks, bool waitNeeded, ncclLsaBarrierSession& bar, ncclSymPtr input, ncclSymPtr output, size_t nElts ) { bool inPlace = (input == output); size_t nBytes = nElts*sizeof(T); uint32_t nBlocks_rcp32 = nccl::utility::idivRcp32_upto64(nBlocks); uint32_t nPreBytes = (16 - input.offset)%16; nPreBytes = min((size_t)nPreBytes, nBytes); uintptr_t cursor = nPreBytes; constexpr int MinWarpPerBlock = 4; if ((input.offset - output.offset)%16 == 0) { constexpr int BytePerPack = 16, UnrollPacks = 4, UnrollPeers = 2; constexpr int BytePerChunk = MinWarpPerBlock*UnrollPacks*WARP_SIZE*BytePerPack; uint32_t chunks = (nBytes-cursor)/BytePerChunk; chunks -= imodFast32(chunks, nBlocks, nBlocks_rcp32); if (chunks != 0) { uintptr_t cursorAfter = cursor + uintptr_t(chunks)*BytePerChunk; bcastDeep( handler, tn, t, waitNeeded, bar, (ncclSymPtr)input + cursor, (ncclSymPtr)output + cursor, inPlace, chunks*MinWarpPerBlock ); cursor = cursorAfter; waitNeeded = false; } } if (sizeof(T) == 4 || (sizeof(T) < 4 && (input.offset - output.offset)%4 == 0)) { constexpr int BytePerPack = 4, UnrollPacks = 4, UnrollPeers = 4; constexpr int BytePerChunk = MinWarpPerBlock*UnrollPacks*WARP_SIZE*BytePerPack; uint32_t chunks = (nBytes-cursor)/BytePerChunk; chunks -= imodFast32(chunks, nBlocks, nBlocks_rcp32); if (chunks != 0) { uintptr_t cursorAfter = cursor + uintptr_t(chunks)*BytePerChunk; bcastDeep<(sizeof(T) <= BytePerPack ? BytePerPack : 0), UnrollPacks, UnrollPeers>( handler, tn, t, waitNeeded, bar, (ncclSymPtr)input + cursor, (ncclSymPtr)output + cursor, inPlace, chunks*MinWarpPerBlock ); cursor = cursorAfter; waitNeeded = false; } } if (waitNeeded) bar.wait(ncclCoopCta(), cuda::memory_order_relaxed); constexpr int UnrollPeers = 8; size_t nSufElts = (nBytes-cursor)/sizeof(T); bcastEnds(handler, tn, t, input, output, inPlace, nElts, nPreBytes/sizeof(T), nSufElts); } __device__ __forceinline__ void ncclSymkRun_AllGather_ST(ncclSymkDevWorkArgs const* args) { ncclSymkArgsHandler handler{args}; ncclLsaBarrierSession bar{ ncclCoopCta(), handler.comm, ncclTeamTagLsa(), blockIdx.x }; int const& rank = handler.comm.rank; bar.arrive(ncclCoopCta(), cuda::memory_order_relaxed); bool waitNeeded = true; handler.forEachWork( [&]__device__(int block, int nBlocks, size_t nElts, size_t nAllElts, ncclSymPtr input, ncclSymPtr output) { // Threads numbered over rank. int bt = flattenIx(threadIdx.x%WARP_SIZE, WARP_SIZE, block, nBlocks, threadIdx.x/WARP_SIZE, blockDim.x/WARP_SIZE); int btn = nBlocks*blockDim.x; bcast(handler, btn, bt, nBlocks, waitNeeded, bar, input, output + rank*nAllElts, nElts); waitNeeded = false; } ); bar.sync(ncclCoopCta(), cuda::memory_order_release); } __device__ __forceinline__ void ncclSymkRun_AllGather_STMC(ncclSymkDevWorkArgs const* args) { ncclSymkArgsHandler handler{args}; ncclLsaBarrierSession bar( ncclCoopCta(), handler.comm, ncclTeamTagLsa(), blockIdx.x, /*multimem=*/true ); int const& rank = handler.comm.rank; bar.sync(ncclCoopCta(), cuda::memory_order_relaxed); handler.forEachWork( [&]__device__(int block, int nBlocks, size_t nElts, size_t nAllElts, ncclSymPtr input, ncclSymPtr output) { // Round robin memory to blocks. int t = flattenIx(threadIdx.x%WARP_SIZE, WARP_SIZE, block, nBlocks, threadIdx.x/WARP_SIZE, blockDim.x/WARP_SIZE); int tn = nBlocks*blockDim.x; bcastMultimem(handler, tn, t, input, output + rank*nAllElts, nElts); } ); bar.sync(ncclCoopCta(), cuda::memory_order_release); } template static __device__ void allgather_LL_body( ncclSymkArgsHandler& handler, ncclLLA2ASession& lla2a, EltType* input, EltType* output, int nElts, int nPacks, int nStrideElts ) { using Pack = BytePack<8>; constexpr int EltPerPack = 8/sizeof(EltType); int const& rank = handler.comm.rank; int const& nRanks = handler.comm.nRanks; int t = threadIdx.x; constexpr int tn = ncclSymkMaxThreads; #pragma unroll 1 while (0 < nElts) { int nIterPacks = min(nPacks, tn); if (t < nIterPacks) { Pack x = loadPack(input, t*EltPerPack, nElts); lla2a.bcast(/*slot=*/nIterPacks*rank + t, x); } int tn_div_nPacks = tn/nIterPacks; int tn_mod_nPacks = tn%nIterPacks; int peer = t/nIterPacks; int pack = t%nIterPacks; #if 1 // NOTE: Unrolling speedup on eos nranks=8 size=64K: 5.7us vs 6.7us constexpr int Unroll = 4; #pragma unroll 1 for (int i = t; i < (nRanks*nIterPacks & -(Unroll*tn)); i += Unroll*tn) { Pack got[Unroll]; lla2a.template recvUnrolled(i, Unroll, tn, /*&*/got); #pragma unroll for (int u=0; u < Unroll; u++) { storePack(output + peer*nStrideElts, pack*EltPerPack, nElts, got[u]); peer += tn_div_nPacks; pack += tn_mod_nPacks; if (nIterPacks <= pack) { peer += 1; pack -= nIterPacks; } } } int i = (nRanks*nIterPacks & -(Unroll*tn)) + t; int n = (nRanks*nIterPacks)/tn % Unroll; if (i + n*tn < nRanks*nIterPacks) n += 1; if (n != 0) { Pack got[Unroll]; lla2a.template recvUnrolled<1, Unroll>(i, n, tn, /*&*/got); #pragma unroll for (int u=0; u < Unroll; u++) { if (u != 0 && u == n) break; storePack(output + peer*nStrideElts, pack*EltPerPack, nElts, got[u]); peer += tn_div_nPacks; pack += tn_mod_nPacks; if (nIterPacks <= pack) { peer += 1; pack -= nIterPacks; } } } #else // The non-unrolled but "obviously correct" implementation for reference. #pragma unroll 1 for (int i = t; i < nRanks*nIterPacks; i += tn) { Pack got = lla2a.template recv(i); storePack(output + peer*nStrideElts, pack*EltPerPack, nElts, got); peer += tn_div_nPacks; pack += tn_mod_nPacks; if (nIterPacks <= pack) { peer += 1; pack -= nIterPacks; } } #endif lla2a.endEpoch(ncclCoopCta()); input += tn*EltPerPack; output += tn*EltPerPack; nElts -= tn*EltPerPack; nPacks -= tn; } } static __device__ void ncclSymkRun_AllGather_LL_impl(ncclSymkDevWorkArgs const* args, bool multimem) { ncclSymkArgsHandler handler{args}; ncclLLA2ASession lla2a( ncclCoopCta(), handler.comm, ncclTeamLsa(handler.comm), handler.lsaLLA2A, blockIdx.x, /*maxElts=*/ncclSymkMaxThreads, multimem, handler.comm.lsaMultimem ); using Pack = BytePack<8>; constexpr int BytePerPack = 8; handler.singleWork( [&]__device__(int nElts, int nAllElts, ncclSymPtr input, ncclSymPtr output) { int nPacks = divUp(nElts, BytePerPack); char* blockInput = input.localPtr(); char* blockOutput = output.localPtr(); uint32_t lowBits = nAllElts; lowBits |= (uintptr_t)blockInput; lowBits |= (uintptr_t)blockOutput; if (__builtin_expect(lowBits%8 == 0, true)) { // NOTE: Specializing for 8-byte alignment in one case help at size=65K: 8.9us vs 5.6us allgather_LL_body(handler, lla2a, (BytePack<8>*)blockInput, (BytePack<8>*)blockOutput, nElts/8, nPacks, nAllElts/8); } else { allgather_LL_body(handler, lla2a, blockInput, blockOutput, nElts, nPacks, nAllElts); } } ); } __device__ __forceinline__ void ncclSymkRun_AllGather_LL(ncclSymkDevWorkArgs const* args) { ncclSymkRun_AllGather_LL_impl(args, /*multimem=*/false); } __device__ __forceinline__ void ncclSymkRun_AllGather_LLMC(ncclSymkDevWorkArgs const* args) { ncclSymkRun_AllGather_LL_impl(args, /*multimem=*/true); } nccl-2.29.7-1/src/device/symmetric/all_gather_gin.cuh000066400000000000000000000111741515037102200223720ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "sym_kernels.h" #include "kernel.cuh" #include "primitives.cuh" #include "gin_scratch__types.h" __device__ __forceinline__ void ncclSymkRun_AllGather_RailRing_LsaSTMC(struct ncclSymkDevWorkArgs const* args) { ncclCoopCta cta; ncclSymkArgsHandler handler(args); ncclTeam rail = ncclTeamRail(handler.comm); ncclGin gin(handler.comm, blockIdx.x % handler.comm.ginContextCount); constexpr int chunkSize = ncclSymkAllGather_RailRing_ChunkSize; ncclGinSignal_t railSignals = handler.ginSyncHandle.railSignals + blockIdx.x * rail.nRanks; ncclBarrierSession bar(cta, ncclTeamTagWorld(), gin, blockIdx.x, /*multimem=*/true); int nextPeer = (rail.rank + 1) % rail.nRanks; int prevPeer = (rail.rank + rail.nRanks - 1) % rail.nRanks; uint64_t* localSignalPtr = gin.getSignalShadowPtr(railSignals + prevPeer); uint64_t localSignalValue = *localSignalPtr; const int ringThreads = WARP_SIZE; bar.sync(cta, cuda::memory_order_relaxed, ncclGinFenceLevel::Relaxed); handler.template forEachWorkNoFusion( [&]__device__(size_t nElts, size_t nAllElts, ncclSymPtr input, ncclSymPtr output) { if (threadIdx.x < ringThreads) { ncclCoopWarpSpan warps(0, 1, 0); for (int step = 0; step < rail.nRanks - 1; step++) { int dataPeer = (rail.rank - step + rail.nRanks) % rail.nRanks; int dgrank = ncclTeamRankToWorld(handler.comm, rail, dataPeer); size_t remainingElts = nElts; size_t offset = 0; if (dataPeer == rail.rank) { while (remainingElts) { size_t chunkElts = min(remainingElts, size_t(chunkSize)); // Send data chunk to next peer in ring gin.put(rail, nextPeer, output + dgrank * nAllElts + offset, input + offset, chunkElts, ncclGin_SignalInc{ railSignals + rail.rank }, ncclGin_None{}, warps); offset += chunkElts; remainingElts -= chunkElts; } } else { while (remainingElts) { size_t chunkElts = min(remainingElts, size_t(chunkSize)); // Wait for ready signal from next peer before sending gin.waitSignal(warps, railSignals + prevPeer, localSignalValue + 1, 32); // Send data chunk to next peer in ring gin.put(rail, nextPeer, output + dgrank * nAllElts + offset, output + dgrank * nAllElts + offset, chunkElts, ncclGin_SignalInc{ railSignals + rail.rank }, ncclGin_None{}, warps); offset += chunkElts; remainingElts -= chunkElts; localSignalValue++; } } } gin.flush(warps); } else { ncclCoopWarpSpan warps(1, blockDim.x / WARP_SIZE - 1, 1); // Loop through rail ranks starting from itself for (int step = 0; step < rail.nRanks; step++) { int dataPeer = (rail.rank - step + rail.nRanks) % rail.nRanks; int dgrank = ncclTeamRankToWorld(handler.comm, rail, dataPeer); size_t remainingElts = nElts; size_t offset = 0; if (dataPeer == rail.rank) { while (remainingElts) { size_t chunkElts = min(remainingElts, size_t(chunkSize)); // Put self rank's data bcastMultimem(handler, warps.num_threads(), warps.thread_rank(), input + offset, output + dgrank * nAllElts + offset, chunkElts); offset += chunkElts; remainingElts -= chunkElts; } } else { while (remainingElts) { size_t chunkElts = min(remainingElts, size_t(chunkSize)); // Wait for signal from other peers before putting their data gin.waitSignal(warps, railSignals + prevPeer, localSignalValue + 1, 32); bcastMultimem(handler, warps.num_threads(), warps.thread_rank(), output + dgrank * nAllElts + offset, output + dgrank * nAllElts + offset, chunkElts); offset += chunkElts; remainingElts -= chunkElts; localSignalValue++; } } } } } ); // update the shadow signal value if (threadIdx.x == ringThreads) { *localSignalPtr = localSignalValue; } bar.sync(cta, cuda::memory_order_release, ncclGinFenceLevel::Relaxed); } nccl-2.29.7-1/src/device/symmetric/all_reduce.cuh000066400000000000000000000417751515037102200215440ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "sym_kernels.h" #include "nccl_device.h" #include "kernel.cuh" #include "primitives.cuh" template static __device__ __forceinline__ void allreduceDeep( ncclSymkArgsHandler const& handler, int tn, int t, bool waitNeeded, ncclLsaBarrierSession& bar, Red red, ncclSymPtr input, ncclSymPtr output, int32_t nIters ) { using Pack = BytePack; using Acc = typename Red::EltType; using AccPack = BytePack; ncclTeam world = ncclTeamWorld(handler.comm); int wn = tn/WARP_SIZE; int w = t/WARP_SIZE; int lane = t%WARP_SIZE; int const& rank = handler.comm.rank; int const& nRanks = handler.comm.nRanks; ncclSymPtr inpPacks = (ncclSymPtr)input + intptr_t(w)*UnrollPacks*WARP_SIZE + lane; ncclSymPtr outPacks = (ncclSymPtr)output + intptr_t(w)*UnrollPacks*WARP_SIZE + lane; Pack acc0[UnrollPacks]; nIters -= w; if (0 < nIters) { #pragma unroll for (int u=0; u < UnrollPacks; u++) { acc0[u] = inpPacks.peerPtr(world, rank)[u*WARP_SIZE]; } } if (waitNeeded) bar.wait(ncclCoopCta(), cuda::memory_order_relaxed); if (0 < nIters) { while (true) { AccPack acc1[UnrollPacks]; int r = rank; if (++r == nRanks) r = 0; { Pack tmp1[UnrollPacks]; #pragma unroll for (int u=0; u < UnrollPacks; u++) { tmp1[u] = inpPacks.peerPtr(world, r)[u*WARP_SIZE]; } #pragma unroll for (int u=0; u < UnrollPacks; u++) { acc1[u] = applyReduce(red, applyCast(acc0[u]), applyCast(tmp1[u])); } } if (++r == nRanks) r = 0; int dr = 2; #pragma unroll 2 for (int partial=0; partial <= 1; partial++) { #pragma unroll 1 for (int i = 0; partial ? i < 1 : (dr + UnrollPeers <= nRanks); partial ? i++ : (dr += UnrollPeers)) { if (partial && dr == nRanks) break; Pack tmp1[UnrollPeers][UnrollPacks]; #pragma unroll for (int ur=0; ur < UnrollPeers-partial; ur++) { if (partial && ur!=0 && dr+ur == nRanks) break; #pragma unroll UnrollPacks for (int u=0; u < UnrollPacks; u++) { tmp1[ur][u] = inpPacks.peerPtr(world, r)[u*WARP_SIZE]; } if (++r == nRanks) r = 0; } #pragma unroll for (int ur=0; ur < UnrollPeers-partial; ur++) { if (partial && ur!=0 && dr+ur == nRanks) break; #pragma unroll UnrollPacks for (int u=0; u < UnrollPacks; u++) { acc1[u] = applyReduce(red, acc1[u], applyCast(tmp1[ur][u])); } } } } #pragma unroll for (int u=0; u < UnrollPacks; u++) acc0[u] = applyCast(acc1[u]); dr = 0; r = rank; #pragma unroll 2 for (int partial=0; partial <= 1; partial++) { #pragma unroll 1 for (int i = 0; partial ? i < 1 : (dr + UnrollPeers <= nRanks); partial ? i++ : (dr += UnrollPeers)) { #pragma unroll for (int ur=0; ur < UnrollPeers-partial; ur++) { if (partial && dr == nRanks) break; #pragma unroll UnrollPacks for (int u=0; u < UnrollPacks; u++) { outPacks.peerPtr(world, r)[u*WARP_SIZE] = acc0[u]; } if (++r == nRanks) r = 0; } } } inpPacks += intptr_t(wn)*UnrollPacks*WARP_SIZE; outPacks += intptr_t(wn)*UnrollPacks*WARP_SIZE; nIters -= wn; if (nIters <= 0) break; // Load data for next iteration. #pragma unroll for (int u=0; u < UnrollPacks; u++) { acc0[u] = inpPacks.peerPtr(world, rank)[u*WARP_SIZE]; } } } } template static __device__ __forceinline__ void allreduceEnds( ncclSymkArgsHandler const& handler, int tn, int t, Red red, ncclSymPtr input, ncclSymPtr output, size_t nElts, uint32_t nPreElts, size_t nSufElts ) { using Acc = typename Red::EltType; ncclTeam world = ncclTeamWorld(handler.comm); int const& rank = handler.comm.rank; int const& nRanks = handler.comm.nRanks; ncclSymPtr> inpPacks = (ncclSymPtr>)input; ncclSymPtr> outPacks = (ncclSymPtr>)output; #pragma unroll 1 for (size_t i = t; i < nPreElts+nSufElts; i += tn) { size_t elt = i < nPreElts ? i : nElts-nSufElts-nPreElts+i; BytePack acc0 = inpPacks.peerPtr(world, rank)[elt]; BytePack acc1; BytePack tmp[UnrollPeers]; int dr = 1; int r = rank+1; if (nRanks == r) r = 0; bool first = true; #pragma unroll 2 for (int partial=0; partial <= 1; partial++) { #pragma unroll 1 for (int j = 0; partial ? j < 1 : (dr + UnrollPeers <= nRanks); partial ? j++ : (dr += UnrollPeers)) { if (partial && dr == nRanks) break; #pragma unroll for (int u=0; u < UnrollPeers-partial; u++) { if (partial && u!=0 && dr+u == nRanks) break; tmp[u] = inpPacks.peerPtr(world, r)[elt]; r += 1; if (r == nRanks) r = 0; } if (first) { first = false; acc1 = applyCast(acc0); } #pragma unroll for (int u=0; u < UnrollPeers-partial; u++) { if (partial && u!=0 && dr+u == nRanks) break; acc1 = applyReduce(red, acc1, applyCast(tmp[u])); } } } acc0 = applyCast(acc1); dr = 0; r = rank; #pragma unroll 2 for (int partial=0; partial <= 1; partial++) { #pragma unroll 1 for (int j=0; partial ? j < 1 : (dr + UnrollPeers <= nRanks); partial ? j++ : (dr += UnrollPeers)) { #pragma unroll for (int u=0; u < UnrollPeers-partial; u++) { if (partial && dr+u == nRanks) break; outPacks.peerPtr(world, r)[elt] = acc0; r += 1; if (r == nRanks) r = 0; } } } } } template static __device__ void allreduce( ncclSymkArgsHandler const& handler, int tn, int t, int nBlocks, bool waitNeeded, ncclLsaBarrierSession& bar, Red red, ncclSymPtr input, ncclSymPtr output, size_t nElts ) { int const& nRanks = handler.comm.nRanks; int const& nRanks_rcp32 = handler.nRanks_rcp32; size_t nBytes = nElts*sizeof(T); uint32_t nBlocks_rcp32 = nccl::utility::idivRcp32_upto64(nBlocks); uint32_t nRanks_nBlocks_rcp32 = nccl::utility::imulRcp32(nRanks, nRanks_rcp32, nBlocks, nBlocks_rcp32); uint32_t nPreBytes = (16u - input.offset)%16u; nPreBytes = min((size_t)nPreBytes, nBytes); uintptr_t cursor = nPreBytes; constexpr int MinWarpPerBlock = 4; if ((input.offset - output.offset)%16 == 0) { constexpr int BytePerPack = 16, UnrollPacks = 4, UnrollPeers = 2; constexpr int BytePerChunk = MinWarpPerBlock*UnrollPacks*WARP_SIZE*BytePerPack; uint32_t chunks = (nBytes-cursor)/BytePerChunk; chunks -= imodFast32(chunks, nRanks*nBlocks, nRanks_nBlocks_rcp32); if (chunks != 0) { uintptr_t cursorAfter = cursor + uintptr_t(chunks)*BytePerChunk; allreduceDeep( handler, tn, t, waitNeeded, bar, red, (ncclSymPtr)input + cursor, (ncclSymPtr)output + cursor, chunks*MinWarpPerBlock ); cursor = cursorAfter; waitNeeded = false; } } if (sizeof(T) == 4 || (sizeof(T) < 4 && (input.offset - output.offset)%4 == 0)) { constexpr int BytePerPack = 4, UnrollPacks = 4, UnrollPeers = 4; constexpr int BytePerChunk = MinWarpPerBlock*UnrollPacks*WARP_SIZE*BytePerPack; uint32_t chunks = (nBytes-cursor)/BytePerChunk; chunks -= imodFast32(chunks, nRanks*nBlocks, nRanks_nBlocks_rcp32); if (chunks != 0) { uintptr_t cursorAfter = cursor + uintptr_t(chunks)*BytePerChunk; allreduceDeep<(sizeof(T) <= BytePerPack ? BytePerPack : 0), UnrollPacks, UnrollPeers, T>( handler, tn, t, waitNeeded, bar, red, (ncclSymPtr)input + cursor, (ncclSymPtr)output + cursor, chunks*MinWarpPerBlock ); cursor = cursorAfter; waitNeeded = false; } } if (waitNeeded) bar.wait(ncclCoopCta(), cuda::memory_order_relaxed); constexpr int UnrollPeers = 8; size_t nSufElts = (nBytes-cursor)/sizeof(T); allreduceEnds(handler, tn, t, red, input, output, nElts, nPreBytes/sizeof(T), nSufElts); } template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_AllReduce_RSxLD_AGxST(ncclSymkDevWorkArgs const* args) { ncclSymkArgsHandler handler{args}; ncclLsaBarrierSession bar{ ncclCoopCta(), handler.comm, ncclTeamTagLsa(), blockIdx.x }; Red::Type> red(handler.devWork->redOpArg); int const& rank = handler.comm.rank; int const& nRanks = handler.comm.nRanks; bar.arrive(ncclCoopCta(), cuda::memory_order_relaxed); bool waitNeeded = true; handler.forEachWork( [&]__device__(int block, int nBlocks, size_t nElts, size_t nAllElts, ncclSymPtr input, ncclSymPtr output) { // Threads numbered globally such that we round robin warps by rank then block. int gt = flattenIx(threadIdx.x%WARP_SIZE, WARP_SIZE, rank, nRanks, block, nBlocks, threadIdx.x/WARP_SIZE, blockDim.x/WARP_SIZE); int gtn = nRanks*nBlocks*blockDim.x; allreduce(handler, gtn, gt, nBlocks, waitNeeded, bar, red, input, output, nElts); waitNeeded = false; } ); bar.sync(ncclCoopCta(), cuda::memory_order_release); } template static __device__ void allreduceMultimem( int tn, int t, Red red, T* input, T* output, size_t nElts ) { uintptr_t inputUptr = reinterpret_cast(input); uintptr_t outputUptr = reinterpret_cast(output); size_t nBytes = nElts*sizeof(T); constexpr int BytePerPack = LoadMultimem_BigPackSize::BigPackSize; uint32_t nPreBytes = (BytePerPack - inputUptr)%BytePerPack; nPreBytes = min((size_t)nPreBytes, nBytes); uintptr_t nSufBytes; if (alignof(T) == BytePerPack || (inputUptr-outputUptr)%BytePerPack == 0) { constexpr int UnrollPacks = 16*8/BytePerPack; constexpr int BytePerChunk = UnrollPacks*WARP_SIZE*BytePerPack; uintptr_t cursor = nPreBytes; int nChunks = (nBytes-cursor)/BytePerChunk; uintptr_t cursorAfter = cursor + uintptr_t(nChunks)*BytePerChunk; nSufBytes = nBytes - cursorAfter; cursor += (t/WARP_SIZE)*UnrollPacks*WARP_SIZE*BytePerPack; cursor += (t%WARP_SIZE)*BytePerPack; int nIters = nChunks - t/WARP_SIZE; #pragma unroll 1 while (0 < nIters) { BytePack tmp[UnrollPacks]; #pragma unroll for (int u=0; u < UnrollPacks; u++) { tmp[u] = applyLoadMultimem(red, inputUptr + cursor + u*WARP_SIZE*BytePerPack); } #pragma unroll for (int u=0; u < UnrollPacks; u++) { multimem_st_global(outputUptr + cursor + u*WARP_SIZE*BytePerPack, tmp[u]); } cursor += tn*UnrollPacks*BytePerPack; nIters -= tn/WARP_SIZE; } } else { nPreBytes = 0; nSufBytes = nBytes; } // Get the prefix+suffix element one at a time. #pragma unroll 4 for (uintptr_t i = t*sizeof(T); i < nPreBytes + nSufBytes; i += tn*sizeof(T)) { uintptr_t cursor = i < nPreBytes ? i : nBytes-nSufBytes+(i-nPreBytes); BytePack val = applyLoadMultimem(red, inputUptr + cursor); multimem_st_global(outputUptr + cursor, val); } } template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_AllReduce_RSxLDMC_AGxSTMC(ncclSymkDevWorkArgs const* args) { ncclSymkArgsHandler handler{args}; ncclLsaBarrierSession bar{ ncclCoopCta(), handler.comm, ncclTeamTagLsa(), blockIdx.x, /*multimem=*/true }; Red::Type> red(handler.devWork->redOpArg); int const& rank = handler.comm.rank; int const& nRanks = handler.comm.nRanks; auto const& multimem = handler.comm.lsaMultimem; bar.sync(ncclCoopCta(), cuda::memory_order_relaxed); handler.forEachWork( [&]__device__(int block, int nBlocks, size_t nElts, size_t nAllElts, ncclSymPtr input, ncclSymPtr output) { // Threads numbered globally such that we round robin warps by rank then block. int gt = flattenIx(threadIdx.x%WARP_SIZE, WARP_SIZE, rank, nRanks, block, nBlocks, threadIdx.x/WARP_SIZE, blockDim.x/WARP_SIZE); int gtn = nRanks*nBlocks*blockDim.x; allreduceMultimem(gtn, gt, red, input.multimemPtr(multimem), output.multimemPtr(multimem), nElts); } ); bar.sync(ncclCoopCta(), cuda::memory_order_release); } template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_AllReduce_AGxLL_R_impl(ncclSymkDevWorkArgs const* args, bool multimem) { ncclSymkArgsHandler handler{args}; ncclLLA2ASession lla2a( ncclCoopCta(), handler.comm, ncclTeamLsa(handler.comm), handler.lsaLLA2A, blockIdx.x, ncclSymkMaxThreads, multimem, handler.comm.lsaMultimem ); int const& rank = handler.comm.rank; int const& nRanks = handler.comm.nRanks; using Acc = typename ncclSymkAccumType::Type; Red red(handler.devWork->redOpArg); using Pack = BytePack<8>; using AccPack = BytePack<8*sizeof(Acc)/sizeof(T)>; constexpr int EltPerPack = 8/sizeof(T); handler.singleWork( [&]__device__(int nElts, int nAllElts, ncclSymPtr inputPtr, ncclSymPtr outputPtr) { int nPacks = divUp(nElts, EltPerPack); T* input = (T*)inputPtr.localPtr(); T* output = (T*)outputPtr.localPtr(); bool packAligned = 8 <= alignof(T) || (nElts*sizeof(T) | (uintptr_t)input | (uintptr_t)output)%8 == 0; ncclCoopCta cta; int t = threadIdx.x; int tn = ncclSymkMaxThreads; if (__builtin_expect(packAligned, true)) { #pragma unroll 1 while (0 < nPacks) { if (t < nPacks) { int nIterPacks = min(nPacks, tn); Pack inp = loadPack((Pack*)input, t, nPacks); lla2a.bcast(/*slot=*/nIterPacks*rank + t, inp); AccPack out = lla2a.template recvReduce( /*slotStart=*/t, /*slotCount=*/nRanks, /*slotStride=*/nIterPacks, /*eltToAcc=*/[&] __device__ (Pack x)->AccPack { return applyCast(x); }, /*reduce=*/[&] __device__ (AccPack a, AccPack b)->AccPack { return applyReduce(red, a, b); } ); storePack((Pack*)output, t, nPacks, applyCast(out)); } lla2a.endEpoch(cta); input += tn*EltPerPack; output += tn*EltPerPack; nPacks -= tn; } } else { #pragma unroll 1 while (0 < nElts) { if (t*EltPerPack < nElts) { int nIterPacks = min(nPacks, tn); Pack inp = loadPack(input, t*EltPerPack, nElts); lla2a.bcast(/*slot=*/nIterPacks*rank + t, inp); AccPack out = lla2a.template recvReduce( /*slotStart=*/t, /*slotCount=*/nRanks, /*slotStride=*/nIterPacks, /*eltToAcc=*/[&] __device__ (Pack x)->AccPack { return applyCast(x); }, /*reduce=*/[&] __device__ (AccPack a, AccPack b)->AccPack { return applyReduce(red, a, b); } ); storePack(output, t*EltPerPack, nElts, applyCast(out)); } lla2a.endEpoch(cta); input += tn*EltPerPack; output += tn*EltPerPack; nElts -= tn*EltPerPack; nPacks -= tn; } } } ); } template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_AllReduce_AGxLL_R(ncclSymkDevWorkArgs const* args) { ncclSymkRun_AllReduce_AGxLL_R_impl(args, /*multimem=*/false); } template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_AllReduce_AGxLLMC_R(ncclSymkDevWorkArgs const* args) { ncclSymkRun_AllReduce_AGxLL_R_impl(args, /*multimem=*/true); } nccl-2.29.7-1/src/device/symmetric/data_ops.cuh000066400000000000000000000561101515037102200212240ustar00rootroot00000000000000#include "primitives.cuh" struct SMemTag {}; // Shared memory struct GMemTag {}; // Global memory struct GenMemTag {}; // Generic memory (either global or shared) // Like CUDA's __ldcs() except works for all types T and handles smem so long as // it is tagged accurately. template static __device__ __forceinline__ T ldcs(GenMemTag, T *p) { return *p; } template static __device__ __forceinline__ T ldcs(SMemTag, T *p) { __builtin_assume(__isShared(p)); return *p; } template static __device__ __forceinline__ T ldcs(GMemTag, T *p) { union { T x; uint8_t u8[sizeof(T)]; uint16_t u16[(sizeof(T)+2-1)/2]; uint32_t u32[(sizeof(T)+4-1)/4]; uint64_t u64[(sizeof(T)+8-1)/8]; uint4 u32v4[(sizeof(T)+16-1)/16]; }; switch (alignof(T)) { case 1: for (int i=0; i < sizeof(T)/1; i++) u8[i] = __ldcs((uint8_t*)p + i); break; case 2: for (int i=0; i < sizeof(T)/2; i++) u16[i] = __ldcs((uint16_t*)p + i); break; case 4: for (int i=0; i < sizeof(T)/4; i++) u32[i] = __ldcs((uint32_t*)p + i); break; case 8: for (int i=0; i < sizeof(T)/8; i++) u64[i] = __ldcs((uint64_t*)p + i); break; case 16: for (int i=0; i < sizeof(T)/16; i++) u32v4[i] = __ldcs((uint4*)p + i); break; default: __builtin_unreachable(); } return x; } // Like CUDA's __stcs() except works for all types T and handles smem so long as // it is tagged accurately. template static __device__ __forceinline__ void stcs(GenMemTag, T *p, T val) { *p = val; } template static __device__ __forceinline__ void stcs(SMemTag, T *p, T val) { __builtin_assume(__isShared(p)); *p = val; } template static __device__ __forceinline__ void stcs(GMemTag, T *p, T val) { union { T x; uint8_t u8[sizeof(T)]; uint16_t u16[(sizeof(T)+2-1)/2]; uint32_t u32[(sizeof(T)+4-1)/4]; uint64_t u64[(sizeof(T)+8-1)/8]; uint4 u32v4[(sizeof(T)+16-1)/16]; }; x = val; switch (alignof(T)) { case 1: for (int i=0; i < sizeof(T)/1; i++) __stcs((uint8_t*)p + i, u8[i]); break; case 2: for (int i=0; i < sizeof(T)/2; i++) __stcs((uint16_t*)p + i, u16[i]); break; case 4: for (int i=0; i < sizeof(T)/4; i++) __stcs((uint32_t*)p + i, u32[i]); break; case 8: for (int i=0; i < sizeof(T)/8; i++) __stcs((uint64_t*)p + i, u64[i]); break; case 16: for (int i=0; i < sizeof(T)/16; i++) __stcs((uint4*)p + i, u32v4[i]); break; default: __builtin_unreachable(); } } // Load packs from element buffer. Pack index=0 is loaded from the buffer rounded // down to pack alignment so it begins with unused padding elements equal in // number to how misaligned the buffer is. The last pack may alias out of bounds // elements. If it is completely out of bounds it will not be loaded. template static __device__ __forceinline__ void loadPacks( Pack(&packs)[nPacks], MemTag mem, unsigned eltsAlignMin, // (Compile time) Minimum byte alignment of elts. Zero = infinitely aligned. Elt* elts, // The element buffer. Need not be aligned. int nElts, // Total number of elements so we can tell what is legal to load. int padElts, // Number of misaligned elements in first pack. int packIx, // Index of our first pack (typically coop.thread_rank()). int stride, // Index stride of packs (typically coop.size()). bool lastEmpty // Is our last pack completely out of bounds. ) { constexpr int nEltPerPack = sizeof(Pack)/sizeof(Elt); bool eltsAligned = sizeof(Pack)-1 <= eltsAlignMin-1 || reinterpret_cast(elts)%sizeof(Pack) == 0; if (__builtin_expect(padElts == 0 && eltsAligned, true)) { #pragma unroll nPacks for (int p=0; p < nPacks; p++) { if (p < nPacks-1 || !lastEmpty) { packs[p] = ldcs(mem, (Pack*)elts + packIx + p*stride); } } } else { Pack stage[nPacks]; #pragma unroll 16/nEltPerPack for (int p=0; p < nPacks; p++) { union { Pack tmp; Elt elt[nEltPerPack]; }; tmp = {}; #pragma unroll 8 for (int pe=0; pe < nEltPerPack; pe++) { int e = -padElts + (packIx + p*stride)*nEltPerPack + pe; // if (0 <= e && e < nElts) if (unsigned(e) < unsigned(nElts)) elt[pe] = ldcs(mem, elts + e); } stage[p] = tmp; } #pragma unroll for (int p=0; p < nPacks; p++) packs[p] = stage[p]; } } // Reverse of loadPacks except the destination buffer alignemnt has no assumed // relationship with the number of padding elements. template static __device__ __forceinline__ void storePacks( Pack(&packs)[nPacks], MemTag mem, unsigned eltsAlignMin, Elt* elts, int nElts, int padElts, int packIx, int stride, bool lastEmpty ) { constexpr int nEltPerPack = sizeof(Pack)/sizeof(Elt); bool eltsAligned = sizeof(Pack)-1 <= eltsAlignMin-1 || reinterpret_cast(elts)%sizeof(Pack) == 0; if (__builtin_expect(padElts == 0 && eltsAligned, true)) { #pragma unroll nPacks for (int p=0; p < nPacks; p++) { if (p < nPacks-1 || !lastEmpty) { stcs(mem, (Pack*)elts + packIx + p*stride, packs[p]); } } } else { Pack stage[nPacks]; #pragma unroll for (int p=0; p < nPacks; p++) stage[p] = packs[p]; #pragma unroll 16/nEltPerPack for (int p=0; p < nPacks; p++) { union { Pack tmp; Elt elt[nEltPerPack]; }; tmp = stage[p]; #pragma unroll 8 for (int pe=0; pe < nEltPerPack; pe++) { int e = -padElts + (packIx + p*stride)*nEltPerPack + pe; // if (0 <= e && e < nElts) if (unsigned(e) < unsigned(nElts)) stcs(mem, elts + e, elt[pe]); } } } } template typename Red, typename Acc, typename AccPack, int UnrollData, typename GetSrcFn> static __device__ __forceinline__ void accumulateLoads( Red red, bool inPlace, AccPack(&accs)[UnrollData], int stride, bool lastPackEmpty, int nSrcs, GetSrcFn getSrc ) { using EltPack = ncclDecayType_t; static_assert(sizeof(Acc)/sizeof(Elt) == sizeof(AccPack)/sizeof(EltPack), "Required"); constexpr int UnrollSrcs = 4*16 < sizeof(AccPack)*UnrollData ? 2 : 1*16 <= sizeof(AccPack)*UnrollData ? 4 : 8; int srcIx = 0; bool accValid = inPlace; #pragma unroll 1 while (srcIx + UnrollSrcs <= nSrcs) { EltPack tmp[UnrollSrcs][UnrollData] = {}; #pragma unroll UnrollSrcs for (int su=0; su < UnrollSrcs; su++) { EltPack* srcPtr = getSrc(srcIx + su); #pragma unroll UnrollData for (int du=0; du < UnrollData; du++) { if (du != UnrollData-1 || !lastPackEmpty) { tmp[su][du] = ldcs(GMemTag(), srcPtr + du*stride); } } } #pragma unroll UnrollSrcs for (int su=0; su < UnrollSrcs; su++) { #pragma unroll UnrollData for (int du=0; du < UnrollData; du++) { if (du != UnrollData-1 || !lastPackEmpty) { AccPack a = fromPack(applyCast(tmp[su][du])); accs[du] = su == 0 && !accValid ? a : applyReduce(red, accs[du], a); } } } accValid = true; srcIx += UnrollSrcs; } if (srcIx < nSrcs) { EltPack tmp[UnrollSrcs-1 ? UnrollSrcs-1 : 1][UnrollData] = {}; #pragma unroll (UnrollSrcs-1 ? UnrollSrcs-1 : 1) for (int su=0; su < UnrollSrcs-1; su++) { EltPack* srcPtr = getSrc(srcIx + su); if (!(su != 0 && nSrcs <= srcIx + su)) { #pragma unroll UnrollData for (int du=0; du < UnrollData; du++) { if (du != UnrollData-1 || !lastPackEmpty) { tmp[su][du] = ldcs(GMemTag(), srcPtr + du*stride); } } } } #pragma unroll (UnrollSrcs-1 ? UnrollSrcs-1 : 1) for (int su=0; su < UnrollSrcs-1; su++) { if (su != 0 && nSrcs <= srcIx + su) break; #pragma unroll UnrollData for (int du=0; du < UnrollData; du++) { if (du != UnrollData-1 || !lastPackEmpty) { AccPack a = fromPack(applyCast(tmp[su][du])); accs[du] = su == 0 && !accValid ? a : applyReduce(red, accs[du], a); } } accValid = true; } } } static __device__ inline unsigned getWorstPackCount(unsigned nElts, unsigned nEltPerPack) { // Count packs rounding up for worst possible alignment. // nElts=1 --> nPacks=1 // nElts=2 --> nPacks=2 // nElts=nEltPerPack+1 --> nPacks=2 // nElts=nEltPerPack+2 --> nPacks=3 return (nElts + 2*(nEltPerPack-1))/nEltPerPack; } template typename Red, typename Acc, typename GetSrc, typename GetSrcPtrMasked> __device__ void reduceBatch( Coop coop, bool inPlace, int nBatch, int nElts, DstSpace dstMem, unsigned dstAlignMin, /*(int i)->DstT* */GetDst getDst, Red red, int nSrcs, /*(int i, int srcIx)->SrcT* */GetSrc getSrc, // srcPtrCommonMask: All srcs must have matching values of srcPtr & srcPtrCommonMask unsigned srcPtrCommonMask, // getSrcPtrMasked: The common srcPtr & srcPtrCommonMask value shared by all srcs in group /*(int i)->unsigned*/GetSrcPtrMasked getSrcPtrMasked ) { using DstT = ncclDecayType_t; using SrcT = ncclDecayType_t; static_assert(sizeof(SrcT) <= sizeof(DstT), "Required"); int tn = coop.size(); int t = coop.thread_rank(); using SrcPack = BytePack<16>; constexpr unsigned nEltPerPack = 16/sizeof(SrcT); using DstPack = BytePack; using AccPack = BytePack; // Handle source elements as packs if: // 1. All sources share a sufficient common alignment to support pack access. // 2. There is enough for every thread to have one. if (!(NCCL_CUDA_ARCH == 900 && sizeof(SrcT) == 1) && ( sizeof(SrcT) == sizeof(SrcPack) || (/*1*/sizeof(SrcPack)-1 <= srcPtrCommonMask && /*2*/tn*(int)sizeof(SrcPack) <= nBatch*nElts*(int)sizeof(SrcT)) )) { int nPacks = getWorstPackCount(nElts, nEltPerPack); constexpr int UnrollData = 4; constexpr int nPackPerBlob = UnrollData*32; // A blob is a whole warp's worth of unrolled packs int nBlobs = unsigned(nPacks)/nPackPerBlob; #pragma unroll 1 for (int row = unsigned(t)/32; row < nBatch*nBlobs; row += unsigned(tn)/32) { int batchIx = unsigned(row)/unsigned(nBlobs); int blobIx = unsigned(row)%unsigned(nBlobs); int padElts = (getSrcPtrMasked(batchIx) % sizeof(SrcPack))/sizeof(SrcT); DstT* dstPtr = getDst(batchIx); int lane = unsigned(t)%32; int packIx = blobIx*nPackPerBlob + lane; bool lastPackEmpty = padElts + nElts <= (packIx + (UnrollData-1)*32)*nEltPerPack; AccPack acc[UnrollData] = {}; if (inPlace) { DstPack dstVal[UnrollData]; loadPacks(dstVal, dstMem, dstAlignMin, dstPtr, nElts, padElts, packIx, 32, lastPackEmpty); #pragma unroll for (int u=0; u < UnrollData; u++) acc[u] = applyCast(dstVal[u]); } accumulateLoads( red, inPlace, acc, /*stride=*/32, lastPackEmpty, nSrcs, [&]__device__(int srcIx)->SrcPack* { return (SrcPack*)(getSrc(batchIx, srcIx) - padElts) + packIx; } ); DstPack dstVal[UnrollData]; #pragma unroll for (int u=0; u < UnrollData; u++) dstVal[u] = applyCast(acc[u]); storePacks(dstVal, dstMem, dstAlignMin, dstPtr, nElts, padElts, packIx, 32, lastPackEmpty); } int nRemPacks = nPacks - nBlobs*nPackPerBlob; #pragma unroll 1 for (int row = t; row < nBatch*nRemPacks; row += tn) { int batchIx = unsigned(row)/unsigned(nRemPacks); int packIx = nBlobs*nPackPerBlob + unsigned(row)%unsigned(nRemPacks); int padElts = (getSrcPtrMasked(batchIx) % sizeof(SrcPack))/sizeof(SrcT); DstT* dstPtr = getDst(batchIx); bool packEmpty = padElts + nElts <= packIx*nEltPerPack; packEmpty = __builtin_expect(packEmpty, false); if (!packEmpty) { AccPack acc[1]; if (inPlace) { DstPack dstVal[1]; loadPacks(dstVal, dstMem, dstAlignMin, dstPtr, nElts, padElts, packIx, tn, false); acc[0] = applyCast(dstVal[0]); } accumulateLoads( red, inPlace, acc, /*stride(no data unroll)=*/0, /*lastPackEmpty=*/false, nSrcs, [&]__device__(int srcIx)->SrcPack* { return (SrcPack*)(getSrc(batchIx, srcIx) - padElts) + packIx; } ); DstPack dstVal[1] = {applyCast(acc[0])}; storePacks(dstVal, dstMem, dstAlignMin, dstPtr, nElts, padElts, packIx, tn, false); } } } else { #pragma unroll 1 for (int row = t; row < nBatch*nElts; row += tn) { int batchIx = unsigned(row)/unsigned(nElts); int eltIx = unsigned(row)%unsigned(nElts); DstT* dstPtr = getDst(batchIx); Acc acc[1]; if (inPlace) acc[0] = (Acc)ldcs(dstMem, dstPtr + eltIx); accumulateLoads( red, inPlace, acc, /*stride(no data unroll)=*/0, /*lastPackEmpty=*/false, nSrcs, [&]__device__(int srcIx)->SrcT* { return getSrc(batchIx, srcIx) + eltIx; } ); stcs(dstMem, dstPtr + eltIx, (DstT)acc[0]); } } } template typename Red, typename Acc, typename DstSpace, typename DstT, typename GetSrc> static __device__ void reduce( Coop coop, Red red, bool inPlace, int nElts, DstSpace dstMem, unsigned dstAlignMin, DstT* dst, int nSrcs, unsigned srcPtrCommonMask, unsigned srcPtrMasked, /*(int srcIx)->SrcT* */ GetSrc getSrc ) { reduceBatch(coop, inPlace, /*nBatch=*/1, nElts, dstMem, dstAlignMin, [=]__device__(int d)->DstT* { return dst; }, red, nSrcs, [&]__device__(int d, int s) { return getSrc(s); }, srcPtrCommonMask, /*getSrcPtrMasked=*/[=]__device__(int d)->unsigned { return srcPtrMasked; } ); } template typename Red, typename SrcT, typename GetSrc> __device__ void reduceMultimemBatch( Coop coop, int nBatch, int nElts, DstSpace dstMem, unsigned dstAlignMin, /*(int i)->DstT* */ GetDst getDst, Red srcRed, /*(int i)->SrcT* */ GetSrc getSrc ) { using DstT = ncclDecayType_t; using SrcT_1 = ncclDecayType_t; static_assert(sizeof(SrcT_1) == sizeof(SrcT), "Required"); static_assert(sizeof(SrcT) <= sizeof(DstT), "Required"); int tn = coop.size(); int t = coop.thread_rank(); using SrcPack = BytePack>::BigPackSize>; constexpr int nEltPerPack = sizeof(SrcPack)/sizeof(SrcT); using DstPack = BytePack; // Handle source elements as packs if there is enough for every thread to have one. if (sizeof(SrcT) == sizeof(SrcPack) || tn*(int)sizeof(SrcPack) <= nBatch*nElts*(int)sizeof(SrcT)) { int nPacks = getWorstPackCount(nElts, nEltPerPack); constexpr int UnrollData = 4; constexpr int nPackPerBlob = UnrollData*32; // A blob is a whole warp's worth of unrolled packs int nBlobs = unsigned(nPacks)/nPackPerBlob; #pragma unroll 1 for (int row = unsigned(t)/32; row < nBatch*nBlobs; row += unsigned(tn)/32) { int batchIx = unsigned(row)/unsigned(nBlobs); int blobIx = unsigned(row)%unsigned(nBlobs); DstT* dstPtr = getDst(batchIx); SrcT* srcPtr = getSrc(batchIx); int padElts = (reinterpret_cast(srcPtr) % sizeof(SrcPack))/sizeof(SrcT); int lane = unsigned(t)%32; int packIx = blobIx*nPackPerBlob + lane; bool lastPackEmpty = padElts + nElts <= (packIx + (UnrollData-1)*32)*nEltPerPack; SrcPack* srcPackPtr = (SrcPack*)(srcPtr - padElts) + packIx; SrcPack srcVal[UnrollData] = {}; #pragma unroll for (int u=0; u < UnrollData; u++) { if (u < UnrollData-1 || !lastPackEmpty) { srcVal[u] = applyLoadMultimem, sizeof(SrcPack)>( srcRed, reinterpret_cast(srcPackPtr + u*32) ); } } DstPack dstVal[UnrollData]; #pragma unroll for (int u=0; u < UnrollData; u++) dstVal[u] = applyCast(srcVal[u]); storePacks(dstVal, dstMem, dstAlignMin, dstPtr, nElts, padElts, packIx, 32, lastPackEmpty); } int nRemPacks = nPacks - nBlobs*nPackPerBlob; #pragma unroll 1 for (int row = t; row < nBatch*nRemPacks; row += tn) { int batchIx = unsigned(row)/unsigned(nRemPacks); int packIx = nBlobs*nPackPerBlob + unsigned(row)%unsigned(nRemPacks); DstT* dstPtr = getDst(batchIx); SrcT* srcPtr = getSrc(batchIx); int padElts = (reinterpret_cast(srcPtr) % sizeof(SrcPack))/sizeof(SrcT); bool packEmpty = padElts + nElts <= packIx*nEltPerPack; packEmpty = __builtin_expect(packEmpty, false); if (!packEmpty) { SrcPack* srcPackPtr = (SrcPack*)(srcPtr - padElts) + packIx; SrcPack srcVal = applyLoadMultimem, sizeof(SrcPack)>( srcRed, reinterpret_cast(srcPackPtr) ); DstPack dstVal[1] = {applyCast(srcVal)}; storePacks(dstVal, dstMem, dstAlignMin, dstPtr, nElts, padElts, packIx, tn, false); } } } else { #pragma unroll 1 for (int row = t; row < nBatch*nElts; ) { int batchIx = unsigned(row)/unsigned(nElts); int eltIx = unsigned(row)%unsigned(nElts); DstT* dstPtr = getDst(batchIx) + eltIx; SrcT* srcPtr = getSrc(batchIx) + eltIx; constexpr int UnrollData = 4; // Test if the whole warp can be unrolled within the same run of elts // by testing if both lane 0 and 31 are in our run. int lane = unsigned(t)%32; if ((0 <= eltIx-lane) && (eltIx-lane + 31 + 1 + (UnrollData-1)*tn <= nElts)) { SrcT srcVal[UnrollData] = {}; #pragma unroll for (int u=0; u < UnrollData; u++) { srcVal[u] = fromPack(applyLoadMultimem, sizeof(SrcT)>( srcRed, reinterpret_cast(srcPtr + u*tn) )); } #pragma unroll for (int u=0; u < UnrollData; u++) { stcs(dstMem, dstPtr + u*tn, (DstT)srcVal[u]); } row += UnrollData*tn; } else { SrcT srcVal = fromPack(applyLoadMultimem, sizeof(SrcT)>( srcRed, reinterpret_cast(srcPtr) )); stcs(dstMem, dstPtr, (DstT)srcVal); row += tn; } } } } // reduceLsa helpers sum symmetric elements from whole LSA team. template typename Red, typename Acc, typename SrcT, typename GetSrcOffset> static __device__ void reduceLsaBatch( Coop coop, int nBatch, int nElts, DstSpace dstMem, unsigned dstAlignMin, /*(int i)->DstT* */GetDst getDst, Red srcRedUc, Red srcRedMc, ncclSymPtr srcBase, /*(int i)->size_t*/GetSrcOffset getSrcOffset, ncclDevComm const& comm, BoolTag multimemTrue ) { SrcT* srcPtr = srcBase.lsaMultimemPtr(comm); reduceMultimemBatch(coop, nBatch, nElts, dstMem, dstAlignMin, getDst, srcRedMc, /*getSrc=*/[&]__device__(int i) { return srcPtr + getSrcOffset(i); } ); } template typename Red, typename Acc, typename SrcT, typename GetSrcOffset> static __device__ void reduceLsaBatch( Coop coop, int nBatch, int nElts, DstSpace dstMem, unsigned dstAlignMin, /*(int i)->DstT* */GetDst getDst, Red srcRedUc, Red srcRedMc, ncclSymPtr srcBase, /*(int i)->size_t*/GetSrcOffset getSrcOffset, ncclDevComm const& comm, BoolTag multimemFalse ) { ncclTeam lsa = ncclTeamLsa(comm); ncclLsaPointerGetter getLsaPtr{srcBase}; reduceBatch(coop, /*inPlace=*/false, nBatch, nElts, dstMem, dstAlignMin, getDst, srcRedUc, lsa.nRanks, /*getSrcOffset=*/[&]__device__(int i, int s)->SrcT* { int r = lsa.rank + s; if (lsa.nRanks <= r) r -= lsa.nRanks; // This could be `srcBase.lsaPtr(r) + getSrcOffset(i)` but by using the // `getLsaPtr` functor we have hoisted the window meta data loads outside // of the reduction loop this lambda is embedded in. The compiler isn't // being smart about the `__ldg()'s` in `.lsaPtr()` despite it having all // the static knowledge necessary to do so. return getLsaPtr(r) + getSrcOffset(i); }, /*srcPtrCommonMask=*/-1u, // LSA is 4GB common /*getSrcPtrMasked=*/[&]__device__(int i)->unsigned { __builtin_assume(srcBase.offset % alignof(SrcT) == 0); return unsigned(srcBase.offset + getSrcOffset(i)*sizeof(SrcT)); } ); } template typename Red, typename Acc, typename SrcT, bool multimem> static __device__ void reduceLsa( Coop coop, int nElts, DstSpace dstMem, unsigned dstAlignMin, DstT* dstPtr, Red srcRedUc, Red srcRedMc, ncclSymPtr srcPtr, ncclDevComm const& comm, BoolTag multimemTag ) { reduceLsaBatch(coop, /*nBatch=*/1, nElts, dstMem, dstAlignMin, [=]__device__(int)->DstT* { return dstPtr; }, srcRedUc, srcRedMc, srcPtr, /*getSrcOffset=*/[=]__device__(int)->int { return 0; }, comm, multimemTag); } template static __device__ void copy( Coop coop, int nElts, GMemTag dstMem, DstT* dst, SMemTag srcMem, SrcT* src ) { static_assert(sizeof(DstT) <= sizeof(SrcT), "Required"); __builtin_assume(__isShared(src)); int tn = coop.size(); int t = coop.thread_rank(); constexpr int nEltPerPack = 16/sizeof(DstT); using DstPack = BytePack<16>; using SrcPack = BytePack<16*sizeof(SrcT)/sizeof(DstT)>; int nPreElts = (16 - reinterpret_cast(dst))%16/sizeof(DstT); int nSufElts = reinterpret_cast(dst + nElts)%16/sizeof(DstT); int nPacks = (nElts-nPreElts-nSufElts)/nEltPerPack; unsigned srcAlign16 = reinterpret_cast(src + nPreElts)%16; if (srcAlign16 == 0) { // TODO: When SrcT==DstT we can replace loop with single `cp.async.bulk` #pragma unroll 1 for (int p = t; p < nPacks; p += tn) { SrcPack srcPack = ((SrcPack*)(src + nPreElts))[p]; stcs(GMemTag(), (DstPack*)(dst + nPreElts) + p, applyCast(srcPack)); } } else { unsigned srcAlign4 = srcAlign16%4; uint32_t* srcWords = reinterpret_cast(reinterpret_cast(src + nPreElts) & -uintptr_t(4)); #pragma unroll 1 for (int p = t; p < nPacks; p += tn) { constexpr int nWordPerPack = sizeof(SrcPack)/4; union { SrcPack srcPack; uint32_t w[nWordPerPack + 1]; }; int i; #pragma unroll for (i = 0; i < nWordPerPack; i++) w[i] = (srcWords + p*nWordPerPack)[i]; if (srcAlign4 != 0) { w[i] = srcWords[i]; #pragma unroll for (i = 0; i < nWordPerPack; i++) w[i] = __funnelshift_r(w[i], w[i+1], 8*srcAlign4); } stcs(GMemTag(), (DstPack*)(dst + nPreElts) + p, applyCast(srcPack)); } } #pragma unroll 1 for (int i = t; i < nPreElts + nSufElts; i += tn) { int e = i < nPreElts ? i : nElts - nSufElts + (i - nPreElts); stcs(GMemTag(), dst + e, (DstT)src[e]); } } nccl-2.29.7-1/src/device/symmetric/generate.py000077500000000000000000000240601515037102200210770ustar00rootroot00000000000000#!/usr/bin/env python3 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # See LICENSE.txt for more license information import os import sys import shutil ################################################################################ # The first command line argument is the path to the directory to generate and # populate. gensrc = sys.argv[1] if os.path.exists(gensrc): for name in os.listdir(gensrc): path = os.path.join(gensrc, name) if os.path.isfile(path): os.remove(path) elif os.path.isdir(path): shutil.rmtree(path) else: os.mkdir(gensrc) def paste(sep, *args): return sep.join(args) indents = 0 def emitln(f, lines): global indents for ln in ((lines,) if isinstance(lines, str) else lines): f.write(' '*indents + ln + '\n') def indent(s): endl = '\n' if s.endswith('\n') else '' return '\n'.join(' '+l for l in s.splitlines()) + endl class Rec(object): def __init__(me, **kw): me.__dict__.update(kw) def __eq__(x, y): return x.__dict__ == y.__dict__ def __hash__(me): h = 0 for k in me.__dict__: h += hash((k, me.__dict__[k])) return h ################################################################################ # Edit this region for introducing new algos etc reductions = ["AllReduce","ReduceScatter"] all_reds = ["sum"] all_tys = ["f32","f16","bf16","f8e4m3","f8e5m2"] gin_algos = ["RailA2A_LsaLD", "RailA2A_LsaLDMC", "RailRing_LsaSTMC"] nvls_algos_by_coll = { "AllReduce": ["AGxLLMC_R","RSxLDMC_AGxSTMC"], "ReduceScatter": ["LDMC","RailA2A_LsaLDMC"] } ldmc_algos = ["RSxLDMC_AGxSTMC", "LDMC", "RailA2A_LsaLDMC"] coll_to_lower = { "AllGather": "all_gather", "AllReduce": "all_reduce", "ReduceScatter": "reduce_scatter" } red_to_ncclDevRedOp = { "sum": "ncclDevSum" } red_to_Func = { "sum": "FuncSum" } ty_to_ncclDataType = { "f32": "ncclFloat32", "f16": "ncclFloat16", "bf16": "ncclBfloat16", "f8e4m3": "ncclFloat8e4m3", "f8e5m2": "ncclFloat8e5m2" } ty_to_cxxtype = { "f32": "float", "f16": "half", "bf16": "__nv_bfloat16", "f8e4m3": "__nv_fp8_e4m3", "f8e5m2": "__nv_fp8_e5m2" } def enumerate_kernels(): for algo in ["LL","LLMC","ST","STMC","RailRing_LsaSTMC"]: yield Rec(coll="AllGather", algo=algo) for red in all_reds: for ty in all_tys: for algo in ["AGxLL_R","AGxLLMC_R","RSxLD_AGxST","RSxLDMC_AGxSTMC"]: yield Rec(coll="AllReduce", algo=algo, red=red, ty=ty) for algo in ["LL","LD","LDMC","RailA2A_LsaLD","RailA2A_LsaLDMC"]: yield Rec(coll="ReduceScatter", algo=algo, red=red, ty=ty) def required_cuda(k): cudart, arch, specific_sms = 0, 600, None is_nvls = k.algo in nvls_algos_by_coll.get(k.coll, []) if is_nvls: cudart = max(cudart, 12010) arch = 900 if k.coll in reductions: if k.ty == "bf16": cudart = max(cudart, 11000) if k.ty.startswith("f8"): cudart = max(cudart, 11080) arch = 900 if k.algo in ldmc_algos: cudart = 12070 arch = None specific_sms = ["100a", "101a", "100f", "101f", "120a", "121a"] return (cudart, arch, specific_sms) ################################################################################ def kernel_fbase(k): return coll_to_lower[k.coll] + ("_gin" if k.algo in gin_algos else "") def kernel_fname(k): parts = [coll_to_lower[k.coll]] if k.algo in gin_algos: parts += ['gin'] if k.coll in reductions: if k.algo in ldmc_algos and k.ty.startswith('f8'): parts += [k.red, k.ty, k.algo] else: parts += [k.red, k.ty] return paste('_', *parts) + '.cu' def kernel_gencode(k): if k.coll in reductions and k.algo in ldmc_algos and k.ty.startswith('f8'): return "$(NVCC_GENCODE_LDMC_FP8)" else: return "$(NVCC_GENCODE)" def kernel_cname(k): if k.coll in reductions: return paste("_", "ncclSymkDevKernel", k.coll, k.algo, k.red, k.ty) else: return paste("_", "ncclSymkDevKernel", k.coll, k.algo) def kernel_conds(k): cudart, arch, specific_sms = required_cuda(k) if cudart == 0 and arch == 0: return (None, None) cudart_cond = "CUDART_VERSION >= %d"%cudart if not specific_sms: arch_cond = "__CUDA_ARCH__ >= %d"%arch else: arch_cond = " || ".join(["0"] + ["NCCL_CUDA_ARCH_%sSPECIFIC==%d"%("FAMILY_" if sm[-1] == "f" else "", 10*int(sm.replace('a', '').replace('f', ''))) for sm in specific_sms]) return cudart_cond, arch_cond def instantiate(k): cudart_cond, arch_cond = kernel_conds(k) if (cudart_cond, arch_cond) == (None, None): form_red_ty = ( "__global__ void {cname}(ncclSymkDevWorkArgs4K NCCL_GRID_CONSTANT const args4K) {{\n" " ncclSymkRun_{id}<{red}, {ty}>(&args4K.args);\n" "}}" ) form = ( "__global__ void {cname}(ncclSymkDevWorkArgs4K NCCL_GRID_CONSTANT const args4K) {{\n" " ncclSymkRun_{id}(&args4K.args);\n" "}}" ) else: form_red_ty = ( "#if {cudart_cond}\n" " __global__ void {cname}(ncclSymkDevWorkArgs4K NCCL_GRID_CONSTANT const args4K) {{\n" " #if {arch_cond}\n" " ncclSymkRun_{id}<{red}, {ty}>(&args4K.args);\n" " #endif\n" " }}\n" "#endif" ) form = ( "#if {cudart_cond}\n" " __global__ void {cname}(ncclSymkDevWorkArgs4K NCCL_GRID_CONSTANT const args4K) {{\n" " #if {arch_cond}\n" " ncclSymkRun_{id}(&args4K.args);\n" " #endif\n" " }}\n" "#endif" ) id = k.coll+'_'+k.algo cname = kernel_cname(k) if k.coll in reductions: inst = form_red_ty.format(cname=cname, id=id, red=red_to_Func[k.red], ty=ty_to_cxxtype[k.ty], cudart_cond=cudart_cond, arch_cond=arch_cond) else: inst = form.format(cname=cname, id=id, cudart_cond=cudart_cond, arch_cond=arch_cond) return inst def prototype(k): cudart_cond, arch_cond = kernel_conds(k) if cudart_cond is None: form = "__global__ void {cname}(ncclSymkDevWorkArgs4K const);" else: form = ( "#if {cudart_cond}\n" " __global__ void {cname}(ncclSymkDevWorkArgs4K const);\n" "#else\n" " constexpr void* {cname} = nullptr;\n" "#endif" ) return form.format(cname=kernel_cname(k), cudart_cond=cudart_cond) ################################################################################ def partition(vals, keyfn): ans = {} for x in vals: k = keyfn(x) if k not in ans: ans[k] = [] ans[k].append(x) return ans kernels_by_file = partition(enumerate_kernels(), lambda k: (kernel_fname(k), kernel_fbase(k))) # Add dependency only files (e.g. allreduce.cu) for fbase in set(kernel_fbase(k) for k in enumerate_kernels()): fname = fbase + '.cu' if (fname, fbase) not in kernels_by_file: kernels_by_file[fname, fbase] = [] files_to_print = "" # Generate each kernel instantiation file for (fname, fbase), ks in kernels_by_file.items(): files_to_print += fname + ";" with open(os.path.join(gensrc, fname), "w") as f: emitln(f, '#include "sym_kernels.h"') emitln(f, '#include "symmetric/kernel.cuh"') emitln(f, '#include "symmetric/{fbase}.cuh"'.format(fbase=fbase)) for k in ks: emitln(f, instantiate(k)) # Generate /sym_kernels_host.cc with open(os.path.join(gensrc, "sym_kernels_host.cc"), "w") as f: emitln(f, '#include "sym_kernels.h"') emitln(f, '#include "device.h"') emitln(f, '') kernel_list = list(enumerate_kernels()) for k in kernel_list: emitln(f, prototype(k)) emitln(f, '') emitln(f, 'extern int const ncclSymkKernelCount = %d;' % len(kernel_list)) emitln(f, 'void* ncclSymkKernelList[] = {') for k in kernel_list: emitln(f, '(void*){cname},'.format(cname=kernel_cname(k))) emitln(f, 'nullptr};') emitln(f, '') emitln(f, 'int ncclSymkKernelRequirements[] = {') for index,k in enumerate(kernel_list): cudart, _, _ = required_cuda(k) sym = kernel_cname(k) emitln(f, ' %7d, /*%4d %s*/' % (cudart or 0, index, sym)); emitln(f, '};') emitln(f, '') emitln(f, 'int ncclSymkKernelMaxDynamicSmem[%d];' % len(kernel_list)) emitln(f, '') emitln(f, 'int ncclSymkGetKernelIndex(ncclSymkKernelId id, int red, ncclDataType_t ty) {') indents += 1 emitln(f, 'switch (id) {') emitln(f, 'default: return -1;') for (coll, algo), coll_algo_ks in partition(kernel_list, lambda k: (k.coll, k.algo)).items(): emitln(f, 'case ncclSymkKernelId_'+coll+'_'+algo+':') indents += 1 if len(coll_algo_ks) == 1: emitln(f, 'return %d;' % kernel_list.index(coll_algo_ks[0])) else: emitln(f, 'switch ((ncclDevRedOp_t)red) {') emitln(f, 'default: return -1;') for red, coll_algo_red_ks in partition(coll_algo_ks, lambda k: k.red).items(): emitln(f, 'case '+red_to_ncclDevRedOp[red]+':') indents += 1 emitln(f, 'switch (ty) {') emitln(f, 'default: return -1;') for k in coll_algo_red_ks: emitln(f, 'case %s: return %d;' % (ty_to_ncclDataType[k.ty], kernel_list.index(k))) emitln(f, '}') indents -= 1 emitln(f, '}') indents -= 1 emitln(f, '}') indents -= 1 emitln(f, '}') # Output file list for CMake (excludes rules.mk since it's not generated for CMake) files_to_print += "sym_kernels_host.cc;" if os.environ.get("NCCL_USE_CMAKE", "0") == "1": print(files_to_print) # Generate /rules.mk (only needed for Makefile builds, not CMake) if os.environ.get("NCCL_USE_CMAKE", "0") != "1": with open(os.path.join(gensrc, "rules.mk"), "w") as f: inst_names = sorted(set(kernel_fname(k) for k in enumerate_kernels())) names = inst_names + ["sym_kernels_host.cc"] f.write("LIB_OBJS_SYM_GEN = $(patsubst %,$(OBJDIR)/genobj/symmetric/%.o,{names})\n" .format(names=" ".join(names))) f.write("\n") inst_names = sorted(set((kernel_fname(k), kernel_fbase(k), kernel_gencode(k)) for k in enumerate_kernels())) for fname, fbase, gencode in inst_names: f.write( "$(OBJDIR)/genobj/symmetric/{fname}.o: $(OBJDIR)/gensrc/symmetric $(OBJDIR)/genobj/symmetric/{fbase}.cu.d\n" "\t" "$(call COMPILE_SYM,$@,$(OBJDIR)/gensrc/symmetric/{fname},{gencode})\n" "\n" .format(fname=fname, fbase=fbase, gencode=gencode) ) nccl-2.29.7-1/src/device/symmetric/gin_scratch.h000066400000000000000000000114231515037102200213640ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_DEVICE_GIN_SCRATCH_H_ #define _NCCL_DEVICE_GIN_SCRATCH_H_ #if 1 // When this file is not in "nccl_device/" #include "nccl_device.h" #else // When this file is public in "nccl_device/" #include "core.h" #endif struct ncclGinOutboxHandle; struct ncclGinInboxA2AHandle; constexpr int ncclGinScratchMaxBufs_log2 = /*log2(512)=*/9; constexpr int ncclGinScratchMaxBufsPerPeer_log2 = /*log2(4)=*/2; NCCL_EXTERN_C __host__ ncclResult_t ncclGinOutboxCreateRequirement( int nBlocks, int size_log2, ncclGinOutboxHandle* outHandle, ncclDevResourceRequirements* outReq ); NCCL_EXTERN_C __host__ ncclResult_t ncclGinInboxA2ACreateRequirement( ncclTeam peers, int nBlocks, int size_log2, ncclGinInboxA2AHandle* outHandle, ncclDevResourceRequirements* outReq ); #if NCCL_CHECK_CUDACC template struct ncclGinOutboxSession_internal; struct ncclGinScratch_GetBufPtr; template struct ncclGinOutboxSession: ncclGinOutboxSession_internal { NCCL_DEVICE_INLINE ncclGinOutboxSession(Coop, ncclGin_BackendMask const&, ncclGinOutboxHandle handle, uint32_t index); NCCL_DEVICE_INLINE ~ncclGinOutboxSession(); ncclGinOutboxSession(ncclGinOutboxSession const&) = delete; // non-copyable // Subdivide the capacity into (1< NCCL_DEVICE_INLINE void apportion(Coop, SubCoop, bool subcoopIsNonTrivial, int nBufs_log2, bool deferSync=false); template NCCL_DEVICE_INLINE void waitBufs(SubCoop, int i0, int n); NCCL_DEVICE_INLINE ncclSymPtr getBuf(int i) const; NCCL_DEVICE_INLINE ncclGinScratch_GetBufPtr make_getBufPtr(int i0) const; NCCL_DEVICE_INLINE ncclGinCounter_t getCounter(int i) const; NCCL_DEVICE_INLINE void advance(Coop, int n); }; #endif #if NCCL_CHECK_CUDACC template struct ncclGinInboxA2ASession_internal; struct ncclGinInboxA2A_GetBufPtr; template struct ncclGinInboxA2ASession: ncclGinInboxA2ASession_internal { NCCL_DEVICE_INLINE ncclGinInboxA2ASession(Coop, ncclGin_BackendMask const&, ncclTeam team, ncclGinInboxA2AHandle, uint32_t index); NCCL_DEVICE_INLINE ~ncclGinInboxA2ASession(); ncclGinInboxA2ASession(ncclGinInboxA2ASession const&) = delete; // non-copyable // Subdivide the available space into individual buffers. Cooperative over all threads // in Coop but they can be partitioned into multiple subcoop's of which exactly // one must have subcoopIsNonTrivial=true. Before entry the nontrivial coop // must already be synced with threads doing waitRecvs/finishRecvs from the previous // round. // Required: nBufs_log2 <= ncclGinScratchMaxBufs_log2 template NCCL_DEVICE_INLINE void apportion(Coop, SubCoop, bool subcoopIsNonTrivial, int nBufs_log2); // When `stepLtPeers=true` we require `step < team.nRanks-1` NCCL_DEVICE_INLINE int getSendPeer(int step, bool stepLtPeers=false) const; NCCL_DEVICE_INLINE int getRecvPeer(int step, bool stepLtPeers=false) const; NCCL_DEVICE_INLINE ncclSymPtr getBuf(int step) const; NCCL_DEVICE_INLINE ncclGinScratch_GetBufPtr make_getBufPtr(int step0) const; // Post sends for steps [step0, step0+nSteps). The lambdas take index in [0, nSteps). template NCCL_DEVICE_INLINE void postSends( SubCoop, int step0, int nSteps, /*(int index, int peer)->ncclSymPtr*/GetPtr getPtr, /*(int index, int peer)->int*/GetEltCount getEltCount, /*(int index, int peer)->ncclGin_???*/GetCompletion getCompletion ); // Wait for recvs for steps [step0, step0+nSteps). template NCCL_DEVICE_INLINE void waitRecvs(SubCoop, int step0, int nSteps); // Finish recvs for steps [step0, step0+nSteps). template NCCL_DEVICE_INLINE void finishRecvs(SubCoop, int step0, int nSteps); // Move to next round of steps. NCCL_DEVICE_INLINE void endRound(Coop); }; #endif #endif // _NCCL_DEVICE_GIN_SCRATCH_H_ // Remove if we move to public "nccl_device/" #include "gin_scratch__funcs.h" nccl-2.29.7-1/src/device/symmetric/gin_scratch__funcs.h000066400000000000000000000247621515037102200227330ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_DEVICE_GIN_SCRATCH__FUNCS_H_ #define _NCCL_DEVICE_GIN_SCRATCH__FUNCS_H_ #include "gin_scratch__types.h" //////////////////////////////////////////////////////////////////////////////// // ncclGinOutboxSession: #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE ncclGinOutboxSession::ncclGinOutboxSession( Coop coop, ncclGin_BackendMask const& gin, ncclGinOutboxHandle handle, uint32_t index ): ncclGinOutboxSession_internal{coop, gin.comm, gin, handle, (int)index} { this->state = this->getStatePtr()->unpadded; } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE ncclGinOutboxSession::~ncclGinOutboxSession() { if (this->coop.thread_rank() == 0) { this->getStatePtr()->unpadded = this->state; } this->coop.sync(); } #endif #if NCCL_CHECK_CUDACC template template NCCL_DEVICE_INLINE void ncclGinOutboxSession::apportion( Coop, SubCoop subcoop, bool subcoopIsNonTrivial, int nBufs_log2_next, bool deferSync ) { int nBufs_log2_cur = this->state.nBufs_log2; if (nBufs_log2_cur != nBufs_log2_next) { if (subcoopIsNonTrivial) { #pragma unroll 1 for (int i = subcoop.thread_rank(); i < (1<state.cursor + i; ncclGinCounter_t ctr = this->handle.counter0 + (this->block << ncclGinScratchMaxBufs_log2); ctr += id & (1<> nBufs_log2_cur; this->gin.waitCounter(ncclCoopThread(), ctr, val, ncclGinOutboxState::CursorBits); this->gin.resetCounter(ctr); } } if (!deferSync) this->coop.sync(); this->state.nBufs_log2 = nBufs_log2_next; this->state.cursor = 0; } } #endif #if NCCL_CHECK_CUDACC template template NCCL_DEVICE_INLINE void ncclGinOutboxSession::waitBufs(SubCoop subcoop, int i0, int n) { #pragma unroll 1 for (int i=subcoop.thread_rank(); i < n; i += subcoop.size()) { uint32_t id = this->state.cursor + i0 + i; ncclGinCounter_t ctr = this->handle.counter0 + (this->block << ncclGinScratchMaxBufs_log2); ctr += id & (1<state.nBufs_log2)-1; uint32_t val = id >> this->state.nBufs_log2; this->gin.waitCounter(ncclCoopThread(), ctr, val, ncclGinOutboxState::CursorBits); } } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE ncclSymPtr ncclGinOutboxSession::getBuf(int i) const { uint32_t id = this->state.cursor + i; int nBufs_log2 = this->state.nBufs_log2; ncclSymPtr bufs = (ncclSymPtr)(this->getStateSymPtr() + 1); return bufs + ((id & (1<handle.size_log2 - nBufs_log2)); } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE ncclGinScratch_GetBufPtr ncclGinOutboxSession::make_getBufPtr(int i0) const { ncclGinScratch_GetBufPtr ret; ret.bufs = (char*)(this->getStatePtr() + 1); ret.nBufs_minus_1 = (1<state.nBufs_log2)-1; ret.bufSize_log2 = this->handle.size_log2 - this->state.nBufs_log2; ret.cursor = this->state.cursor + i0; return ret; } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE ncclGinCounter_t ncclGinOutboxSession::getCounter(int i) const { uint32_t id = this->state.cursor + i; ncclGinCounter_t ctr = this->handle.counter0 + (this->block << ncclGinScratchMaxBufs_log2); return ctr + (id & (1<state.nBufs_log2)-1); } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE void ncclGinOutboxSession::advance(Coop, int n) { this->state.cursor += n; } #endif //////////////////////////////////////////////////////////////////////////////// // ncclGinInboxA2ASession: #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE ncclGinInboxA2ASession::ncclGinInboxA2ASession( Coop coop, ncclGin_BackendMask const& gin, ncclTeam team, ncclGinInboxA2AHandle handle, uint32_t index ): ncclGinInboxA2ASession_internal {coop, gin.comm, gin, team, handle, (int)index} { this->nPeers = team.nRanks - 1; this->state = this->getStatePtr()->unpadded; // The `index` to `context` relationship must be 1:1. assert(this->state.ginContextId_plus_1 == 0 || this->state.ginContextId_plus_1 == gin.contextId + 1); this->state.ginContextId_plus_1 = gin.contextId + 1; } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE ncclGinInboxA2ASession::~ncclGinInboxA2ASession() { if (this->coop.thread_rank() == 0) { this->getStatePtr()->unpadded = this->state; } this->coop.sync(); } #endif #if NCCL_CHECK_CUDACC template template NCCL_DEVICE_INLINE void ncclGinInboxA2ASession::apportion( Coop coop, SubCoop subcoop, bool subcoopIsNonTrivial, int nBufs_log2_next ) { int nBufs_log2_cur = this->state.nBufs_log2_plus_1 - 1; if (nBufs_log2_cur != nBufs_log2_next) { if (subcoopIsNonTrivial) { // Send initial C2S's for all bufs for next (+1) phase. int nPeers = this->nPeers; int nBufs = 1<handle.nPeers_rcp32); #pragma unroll 1 for (int step = subcoop.thread_rank(); step < min(nPeers, nBufs); step += subcoop.size()) { int credits = nBufs_div_nPeers + (step < (int)nBufs_mod_nPeers ? 1 : 0); this->sendC2S(/*phaseDelta=*/+1, step, /*step_lt_nPeers=*/true, credits); } // Reset all signals of previous (-1) phase. The current phase can have // inbound C2S still in flight but the previous cannot because of signal's // release semantics combined with fact that we've communicated with all peers. this->resetSignals(subcoop, /*phaseDelta=*/-1); } // Move to next phase this->state.nBufs_log2_plus_1 = nBufs_log2_next + 1; this->state.phase += 1; // implicitly modulo 4 this->state.monoRound = 0; // round resets with phase change. this->state.monoStep = 0; } } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE int ncclGinInboxA2ASession::getSendPeer(int step, bool step_lt_nPeers) const { return this->getPeer(/*sendNotRecv=*/true, step, step_lt_nPeers); } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE int ncclGinInboxA2ASession::getRecvPeer(int step, bool step_lt_nPeers) const { return this->getPeer(/*sendNotRecv=*/false, step, step_lt_nPeers); } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE ncclSymPtr ncclGinInboxA2ASession::getBuf(int step) const { return this->getBufSymPtr(this->state.monoStep + step); } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE ncclGinScratch_GetBufPtr ncclGinInboxA2ASession::make_getBufPtr(int step0) const { int nBufs_log2 = this->state.nBufs_log2_plus_1 - 1; ncclGinScratch_GetBufPtr ret; ret.bufs = this->getBufsPtr(); ret.nBufs_minus_1 = (1<handle.size_log2 - nBufs_log2; ret.cursor = this->state.monoStep + step0; return ret; } #endif #if NCCL_CHECK_CUDACC template template NCCL_DEVICE_INLINE void ncclGinInboxA2ASession::postSends( SubCoop subcoop, int step0, int nSteps, GetPtr getPtr, GetEltCount getEltCount, GetCompletion getCompletion ) { #pragma unroll 1 for (int i=subcoop.thread_rank(); i < nSteps; i += subcoop.size()) { int step = step0 + i; uint32_t monoStep = this->state.monoStep + step; this->waitC2S(step); int peer = this->getSendPeer(step, /*step_lt_nPeers=*/true); auto srcPtr = getPtr(i, peer); int nElts = getEltCount(i, peer); this->gin.put(this->team, peer, /*dst=*/this->getBufSymPtr(monoStep), /*src=*/(ncclSymPtr)srcPtr, /*size=*/nElts*(int)sizeof(decltype(srcPtr)::ElementType), ncclGin_SignalInc{this->getR2RSignal(monoStep)}, getCompletion(i, peer)); } } #endif #if NCCL_CHECK_CUDACC template template NCCL_DEVICE_INLINE void ncclGinInboxA2ASession::waitRecvs( SubCoop subcoop, int step0, int nSteps ) { #pragma unroll 1 for (int i=subcoop.thread_rank(); i < nSteps; i += subcoop.size()) { this->waitR2R(this->state.monoStep + step0 + i); } } #endif #if NCCL_CHECK_CUDACC template template NCCL_DEVICE_INLINE void ncclGinInboxA2ASession::finishRecvs( SubCoop subcoop, int step0, int nSteps ) { int nBufs_log2 = this->state.nBufs_log2_plus_1 - 1; int nBufs_mod_nPeers = imodFast32(1<nPeers, this->handle.nPeers_rcp32); #pragma unroll 1 for (int i=subcoop.thread_rank(); i < nSteps; i += subcoop.size()) { // Determine next step that will alias the buffer of this step. int step = step0 + i; // guaranteed: step < nPeers int nextStep = step + nBufs_mod_nPeers; if (this->nPeers <= nextStep) nextStep -= this->nPeers; // modulo for + nBufs_mod_nPeers this->sendC2S(/*phaseDelta=*/0, nextStep, /*step_lt_nPeers=*/true, /*credits=*/1); } } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE void ncclGinInboxA2ASession::endRound(Coop coop) { this->state.monoRound += 1; this->state.monoStep += this->nPeers; } #endif #endif // _NCCL_DEVICE_GIN_SCRATCH__FUNCS_H_ nccl-2.29.7-1/src/device/symmetric/gin_scratch__types.h000066400000000000000000000153211515037102200227500ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_DEVICE_GIN__SCRATCH_A2A__TYPES_H_ #define _NCCL_DEVICE_GIN__SCRATCH_A2A__TYPES_H_ #if 1 // When this file is not in "nccl_device/impl/" #include "gin_scratch.h" #else // When this file is public in "nccl_device/impl/" #include "../gin_scratch.h" #include "core__types.h" #include "ptr__types.h" #include "../utility.h" #endif struct ncclGinOutboxHandle { ncclDevResourceHandle bufHandle; ncclGinCounter_t counter0; uint32_t size_log2; }; #if __cplusplus struct alignas(128) ncclGinOutboxState { static constexpr int CursorBits = 32-5; struct Unpadded { uint32_t nBufs_log2:5, cursor:CursorBits; } unpadded; }; #endif struct ncclGinInboxA2AHandle { ncclDevResourceHandle bufHandle; ncclGinSignal_t signals; uint32_t size_log2; uint32_t nPeers_rcp32; }; #if __cplusplus struct alignas(128) ncclGinInboxA2AState { static constexpr int RoundBits = 16; //static constexpr int RoundBits = ncclGinScratchMaxBufsPerPeer_log2 + 1; static_assert(ncclGinScratchMaxBufsPerPeer_log2 + 1 <= RoundBits, "Required"); struct Unpadded { // Memory to ensure the same buffers aren't controlled with different contexts. uint32_t ginContextId_plus_1:9; // Num of bufs we are divided into. +1 so the zero default is invalid (-1). uint32_t nBufs_log2_plus_1:5; // Every time num bufs changes we move to next phase. uint32_t phase:2; // Number of completed alltoalls for this phase. uint32_t monoRound:RoundBits; // Step counter that does not reset. uint32_t monoStep; } unpadded; }; #endif #if NCCL_CHECK_CUDACC struct ncclGinScratch_GetBufPtr { char* bufs; uint32_t nBufs_minus_1, bufSize_log2; uint32_t cursor; NCCL_DEVICE_INLINE void* operator()(int i) const { return bufs + (((cursor + i) & nBufs_minus_1) << bufSize_log2); } }; #endif #if NCCL_CHECK_CUDACC template struct ncclGinOutboxSession_internal { Coop coop; ncclDevComm const& comm; ncclGin_BackendMask gin; ncclGinOutboxHandle handle; int block; ncclGinOutboxState::Unpadded state; NCCL_DEVICE_INLINE ncclGinOutboxState* getStatePtr() const { char* p = (char*)ncclGetResourceBufferLocalPointer(comm, handle.bufHandle); p += block*size_t((int)sizeof(ncclGinOutboxState) + alignUp(1< getStateSymPtr() const { ncclSymPtr p = ncclGetResourceBuffer(comm, handle.bufHandle); p += block*size_t((int)sizeof(ncclGinOutboxState) + alignUp(1<)p; } }; #endif #if NCCL_CHECK_CUDACC template struct ncclGinInboxA2ASession_internal { Coop coop; ncclDevComm const& comm; ncclGin_BackendMask gin; ncclTeam team; ncclGinInboxA2AHandle handle; int block; int nPeers; ncclGinInboxA2AState::Unpadded state; NCCL_DEVICE_INLINE ncclGinInboxA2AState* getStatePtr() const { char* p = (char*)ncclGetResourceBufferLocalPointer(comm, handle.bufHandle); p += block*size_t((int)sizeof(ncclGinInboxA2AState) + alignUp(1< getBufSymPtr(uint32_t monoStep) const { ncclSymPtr p = ncclGetResourceBuffer(comm, handle.bufHandle); p += block*size_t((int)sizeof(ncclGinInboxA2AState) + alignUp(1< NCCL_DEVICE_INLINE void resetSignals(SubCoop subcoop, int phaseDelta) { int nSigs = nPeers + (1<gin.resetSignal(sig0 + i); } } NCCL_DEVICE_INLINE ncclGinSignal_t getC2SSignal(int phaseDelta, uint32_t step) const { return getSignal0(phaseDelta) + step; } NCCL_DEVICE_INLINE ncclGinSignal_t getR2RSignal(uint32_t monoStep) const { int nBufs = 1 << (state.nBufs_log2_plus_1 - 1); return getSignal0(/*phaseDelta=*/0) + nPeers + (monoStep & (nBufs-1)); } NCCL_DEVICE_INLINE void waitC2S(uint32_t step) const { ncclGinSignal_t sig = getC2SSignal(/*phaseDelta=*/0, step); uint32_t desired = state.monoRound + 1; gin.waitSignal(ncclCoopThread(), sig, desired, /*bits=*/ncclGinInboxA2AState::RoundBits); } NCCL_DEVICE_INLINE void waitR2R(uint32_t monoStep) const { int nBufs_log2 = state.nBufs_log2_plus_1 - 1; uint32_t desired = 1 + (monoStep >> nBufs_log2); gin.waitSignal(ncclCoopThread(), getR2RSignal(monoStep), desired, /*bits=*/32-ncclGinScratchMaxBufs_log2); } NCCL_DEVICE_INLINE int getPeer(bool sendNotRecv, int step, bool step_lt_nPeers) const { if (!step_lt_nPeers) step = imodFast32(step, nPeers, handle.nPeers_rcp32); int sign = sendNotRecv ? 1 : -1; int peer = team.rank + sign*(1 + step); if (unsigned(team.nRanks) <= unsigned(peer)) peer += -sign*team.nRanks; return peer; } NCCL_DEVICE_INLINE void sendC2S(int phaseDelta, int step, bool step_lt_nPeers, int credits) { int peer = getPeer(/*sendNotRecv=*/false, step, step_lt_nPeers); ncclGinSignal_t sig = getC2SSignal(phaseDelta, step); gin.signal(team, peer, ncclGin_SignalAdd{sig, (uint64_t)credits}); } }; #endif struct ncclGinSyncHandle { // signals to sync with remote peers ncclGinSignal_t railSignals; }; #endif // _NCCL_DEVICE_GIN__SCRATCH_A2A__TYPES_H_ nccl-2.29.7-1/src/device/symmetric/kernel.cuh000066400000000000000000000045661515037102200207220ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_DEVICE_SYMMETRIC_KERNEL_H_ #define NCCL_DEVICE_SYMMETRIC_KERNEL_H_ #include "sym_kernels.h" template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_AllReduce_AGxLL_R(struct ncclSymkDevWorkArgs const* args); template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_AllReduce_AGxLLMC_R(struct ncclSymkDevWorkArgs const* args); template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_AllReduce_RSxLD_AGxST(struct ncclSymkDevWorkArgs const* args); template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_AllReduce_RSxLDMC_AGxSTMC(struct ncclSymkDevWorkArgs const* args); __device__ __forceinline__ void ncclSymkRun_AllGather_LL(struct ncclSymkDevWorkArgs const* args); __device__ __forceinline__ void ncclSymkRun_AllGather_LLMC(struct ncclSymkDevWorkArgs const* args); __device__ __forceinline__ void ncclSymkRun_AllGather_ST(struct ncclSymkDevWorkArgs const* args); __device__ __forceinline__ void ncclSymkRun_AllGather_STMC(struct ncclSymkDevWorkArgs const* args); template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_ReduceScatter_LL(struct ncclSymkDevWorkArgs const* args); template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_ReduceScatter_LD(struct ncclSymkDevWorkArgs const* args); template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_ReduceScatter_LDMC(struct ncclSymkDevWorkArgs const* args); template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_ReduceScatter_RailA2A_LsaLD(struct ncclSymkDevWorkArgs const* args); template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_ReduceScatter_RailA2A_LsaLDMC(struct ncclSymkDevWorkArgs const* args); __device__ __forceinline__ void ncclSymkRun_AllGather_RailRing_LsaSTMC(struct ncclSymkDevWorkArgs const* args); #endif nccl-2.29.7-1/src/device/symmetric/primitives.cuh000066400000000000000000000277251515037102200216370ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_DEVICE_SYMMETRIC_PRIMITIVES_H_ #define NCCL_DEVICE_SYMMETRIC_PRIMITIVES_H_ #include "sym_kernels.h" #include "bitops.h" #include "collectives.h" #include "../op128.h" #include "../reduce_kernel.h" #include "gin_scratch.h" #if __CUDA_ARCH__ >= 700 // __grid_constant__ appears to break cuda-gdb #define NCCL_GRID_CONSTANT __grid_constant__ #else #define NCCL_GRID_CONSTANT #endif template struct BoolTag { static constexpr bool value = val; }; // A cheap approximation of std::decay template struct ncclDecayType { using Type = T; }; template struct ncclDecayType { using Type = T; }; template struct ncclDecayType { using Type = T; }; template struct ncclDecayType { using Type = T; }; template struct ncclDecayType { using Type = T; }; template using ncclDecayType_t = typename ncclDecayType::Type; // flattenIx(pos0, dim0, pos1, dim1, pos2, dim2, ...) // Given a position vector `pos` in a rectangular index space with lengths in the `dim` // vector, flatten that down to a linear index. The fastest moving dimension is given first. __device__ __forceinline__ int flattenIx() { return 0; } template static __device__ Int0 flattenIx(Int0 pos, Int1 size, Ints ...more) { return pos + size*flattenIx(more...); } template static __device__ void partitionElts( unsigned nParts, unsigned part, size_t* nElts, ncclSymPtr* inPtr, ncclSymPtr* outPtr ) { constexpr int eltPerB16 = 16/sizeof(T); size_t nB16 = (*nElts + eltPerB16-1)/eltPerB16; size_t beginB16 = part*(nB16/nParts) + min(part, uint32_t(nB16%nParts)); *inPtr += beginB16*eltPerB16; *outPtr += beginB16*eltPerB16; if (part < nParts-1) { nB16 = nB16/nParts + (part < nB16%nParts ? 1 : 0); *nElts = nB16*eltPerB16; } else { *nElts = *nElts - beginB16*eltPerB16; } } namespace { struct ncclSymkArgsHandler { ncclDevComm const& comm; ncclLLA2AHandle const& lsaLLA2A; ncclGinOutboxHandle const& ginOutbox; ncclGinInboxA2AHandle const& ginInboxRail; ncclGinCounter_t ginCounterPerBlock; ncclGinSyncHandle const& ginSyncHandle; struct ncclSymkChannelWorkRange* channelWorkRange; struct ncclSymkDevWork* devWork; uint32_t nRanks_rcp32; __device__ ncclSymkArgsHandler(ncclSymkDevWorkArgs const* args): comm(args->kcomm.devComm), lsaLLA2A(args->kcomm.lsaLLA2A), ginOutbox(args->kcomm.ginOutbox), ginInboxRail(args->kcomm.ginInboxRail), ginCounterPerBlock(args->kcomm.ginCounterPerBlock), ginSyncHandle(args->kcomm.ginSyncHandle) { channelWorkRange = args->getWorkRange(); devWork = args->getWorks(args->nMaxChannels); nRanks_rcp32 = comm.nRanks_rcp32; } template __device__ void getWorkRange(int block, uint16_t& workLo, size_t& indexLo, uint16_t& workHi, size_t& indexHi) { constexpr int EltPerCell = NCCL_SYM_KERNEL_CELL_SIZE / sizeof(T); uint32_t fracLo, fracHi; // Where the work begins workLo = (block==0) ? 0 : channelWorkRange[block-1].workHi; // start where predecessor ends fracLo = (block==0) ? 0 : channelWorkRange[block-1].fracHi + 1; // If the predecessor ended on the work boundary, then we step to the beginning of the next work. // This ensures we never have empty parts. if (fracLo == 0x10000) { workLo++; fracLo = 0; } struct ncclSymkDevWork const& dwLo = devWork[workLo]; indexLo = ((fracLo * divUp(dwLo.nElts, EltPerCell)) >> 16) * EltPerCell; // Where the work ends workHi = channelWorkRange[block].workHi; fracHi = channelWorkRange[block].fracHi + 1; struct ncclSymkDevWork const& dwHi = devWork[workHi]; indexHi = min(((fracHi * divUp(dwHi.nElts, EltPerCell)) >> 16) * EltPerCell, dwHi.nElts); } template __device__ void getWorkRangeFused(int blockIdx, int w, int& block, int& nBlocks, size_t& indexLo, size_t& indexHi) { constexpr int EltPerCell = NCCL_SYM_KERNEL_CELL_SIZE / sizeof(T); struct ncclSymkDevWork const& dw = devWork[w]; uint32_t fracLo, fracHi; int lastBlock; block = blockIdx - dw.sChannelId; nBlocks = dw.nChannels; lastBlock = dw.sChannelId+dw.nChannels-1; // Where the work begins fracLo = (dw.sChannelId>0 && channelWorkRange[dw.sChannelId-1].workHi == w) ? ((channelWorkRange[dw.sChannelId-1].fracHi + 1) & 0xFFFF) : 0; indexLo = ((fracLo * divUp(dw.nElts, EltPerCell)) >> 16) * EltPerCell; fracHi = (channelWorkRange[lastBlock].workHi == w) ? channelWorkRange[lastBlock].fracHi + 1 : 0x10000; indexHi = min(((fracHi * divUp(dw.nElts, EltPerCell)) >> 16) * EltPerCell, dw.nElts); } template __device__ void forEachWork(Fn const& fn) { uint16_t workLo, workHi; size_t indexLo, indexHi; getWorkRange(blockIdx.x, workLo, indexLo, workHi, indexHi); #pragma unroll 1 for (int w = workLo; w <= workHi; w++) { struct ncclSymkDevWork const& dw = devWork[w]; size_t const& nAllElts = dw.nElts; size_t currentIndexLo, currentIndexHi; int block, nBlocks; if (blockIdx.x >= dw.sChannelId && blockIdx.x < dw.sChannelId + dw.nChannels) { getWorkRangeFused(blockIdx.x, w, block, nBlocks, currentIndexLo, currentIndexHi); } else { currentIndexLo = (w > workLo) ? 0 : indexLo; currentIndexHi = (w < workHi) ? nAllElts : indexHi; block = 0; nBlocks = 1; } fn(block, nBlocks, currentIndexHi - currentIndexLo, nAllElts, ncclSymPtr(dw.inputWin, dw.inputOff) + currentIndexLo, ncclSymPtr(dw.outputWin, dw.outputOff) + currentIndexLo); currentIndexLo = 0; } } template __device__ void singleWork(Fn const& fn) { uint16_t w; size_t indexLo, indexHi; getWorkRange(blockIdx.x, w, indexLo, w, indexHi); struct ncclSymkDevWork const& dw = devWork[w]; fn(indexHi - indexLo, dw.nElts, ncclSymPtr(dw.inputWin, dw.inputOff) + indexLo, ncclSymPtr(dw.outputWin, dw.outputOff) + indexLo); } template __device__ void forEachWorkNoFusion(Fn const& fn) { uint16_t workLo, workHi; size_t indexLo, indexHi; getWorkRange(blockIdx.x, workLo, indexLo, workHi, indexHi); #pragma unroll 1 for (int w = workLo; w <= workHi; w++) { struct ncclSymkDevWork const& dw = devWork[w]; size_t const& nAllElts = dw.nElts; size_t currentIndexLo, currentIndexHi; currentIndexLo = (w > workLo) ? 0 : indexLo; currentIndexHi = (w < workHi) ? nAllElts : indexHi; fn(currentIndexHi - currentIndexLo, nAllElts, ncclSymPtr(dw.inputWin, dw.inputOff) + currentIndexLo, ncclSymPtr(dw.outputWin, dw.outputOff) + currentIndexLo); } } }; } template typename Red, typename T, bool nvls> struct ncclSymkAccumType { using Type = T; }; // Only Red's whose opArg is invariant w.r.t. the datatype can have a different // accumulator type. At the moment this excludes integer min/max, sumpostdiv, // and premulsum. template<> struct ncclSymkAccumType { using Type = float; }; #if defined(__CUDA_BF16_TYPES_EXIST__) template<> struct ncclSymkAccumType { using Type = float; }; #endif #if defined(__CUDA_FP8_TYPES_EXIST__) template<> struct ncclSymkAccumType { using Type = float; }; template<> struct ncclSymkAccumType { using Type = float; }; #endif // Accumulator type held in smem for GIN algos. template typename Red, typename T> struct ncclSymkGinAccumType { using Type = T; }; template<> struct ncclSymkGinAccumType { using Type = float; }; #if defined(__CUDA_BF16_TYPES_EXIST__) template<> struct ncclSymkGinAccumType { using Type = float; }; #endif #if defined(__CUDA_FP8_TYPES_EXIST__) // fp8 types accumulate in fp16. Multimem algo sends fp8 on wire because it's // impossible to get fp16 accumulator from switch. Non-multimem sends fp16 to // give users a higher precision alternative. template<> struct ncclSymkGinAccumType { using Type = __half; }; template<> struct ncclSymkGinAccumType { using Type = __half; }; #endif // TODO: move this into data_ops.cuh template static __device__ void bcastMultimem( ncclSymkArgsHandler& handler, int tn, int t, ncclSymPtr input, ncclSymPtr output, size_t nElts ) { size_t nBytes = nElts*sizeof(T); uintptr_t inputUptr = reinterpret_cast(input.localPtr()); uintptr_t outputUptr = reinterpret_cast(output.multimemPtr(handler.comm.lsaMultimem)); uint32_t nPreBytes = (16 - input.offset)%16; nPreBytes = min((size_t)nPreBytes, nBytes); uintptr_t nSufBytes; if ((inputUptr-outputUptr)%16 == 0) { constexpr int BytePerPack = 16, UnrollPacks = 8; constexpr int BytePerChunk = UnrollPacks*WARP_SIZE*BytePerPack; uintptr_t cursor = nPreBytes; uint32_t nChunks = (nBytes-cursor)/BytePerChunk; uintptr_t cursorAfter = cursor + uintptr_t(nChunks)*BytePerChunk; nSufBytes = nBytes - cursorAfter; cursor += (t/WARP_SIZE)*UnrollPacks*WARP_SIZE*BytePerPack; cursor += (t%WARP_SIZE)*BytePerPack; int nIters = nChunks - t/WARP_SIZE; #pragma unroll 1 while (0 < nIters) { BytePack tmp[UnrollPacks]; #pragma unroll for (int u=0; u < UnrollPacks; u++) { tmp[u] = *reinterpret_cast*>(inputUptr + cursor + u*WARP_SIZE*BytePerPack); } #pragma unroll for (int u=0; u < UnrollPacks; u++) { multimem_st_global(outputUptr + cursor + u*WARP_SIZE*BytePerPack, tmp[u]); } cursor += tn*UnrollPacks*BytePerPack; nIters -= tn/WARP_SIZE; } } else { nPreBytes = 0; nSufBytes = nBytes; } // Get the prefix+suffix element one at a time. #pragma unroll 4 for (uintptr_t i = t*sizeof(T); i < nPreBytes + nSufBytes; i += tn*sizeof(T)) { uintptr_t cursor = i < nPreBytes ? i : nBytes-nSufBytes+(i-nPreBytes); BytePack val = *reinterpret_cast*>(inputUptr + cursor); multimem_st_global(outputUptr + cursor, val); } } extern __shared__ ulong2 ncclSymkSmem[]; static __device__ void ncclSymkSmemPartition_help(int bumper) {} template static __device__ void ncclSymkSmemPartition_help(int bumper, T** ptr, int size, More ...more) { T *ans = reinterpret_cast(ncclSymkSmem + bumper); __builtin_assume(__isShared(ans)); // Let compiler know this is shared memory (reinterpret_cast obscured as much). __builtin_assume_aligned(ans, sizeof(ulong2)); *ptr = ans; bumper += (size*sizeof(T) + sizeof(ulong2)-1)/sizeof(ulong2); ncclSymkSmemPartition_help(bumper, more...); } template static __device__ void ncclSymkSmemPartition(Arg ...args) { ncclSymkSmemPartition_help(/*bumper=*/0, args...); } //////////////////////////////////////////////////////////////////////////////// // Extensions to nccl_device.h needed to help compiler make good SASS: //////////////////////////////////////////////////////////////////////////////// template struct ncclLsaPointerGetter { void* base; uint32_t stride4G; __device__ ncclLsaPointerGetter(ncclSymPtr ptr) { base = (char*)nccl::utility::loadConst(&ptr.window->lsaFlatBase); base = (char*)base + ptr.offset; stride4G = nccl::utility::loadConst(&ptr.window->stride4G); } __device__ T* operator()(int lsaPeer) const { return (T*)nccl::utility::add4G(base, lsaPeer*stride4G); } }; #endif // NCCL_DEVICE_SYMMETRIC_PRIMITIVES_H_ nccl-2.29.7-1/src/device/symmetric/reduce_scatter.cuh000066400000000000000000000363461515037102200224370ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "sym_kernels.h" #include "kernel.cuh" #include "primitives.cuh" template static __device__ void reduceDeep( ncclSymkArgsHandler const& handler, int tn, int t, bool waitNeeded, ncclLsaBarrierSession& bar, Red red, ncclSymPtr input, ncclSymPtr output, int32_t nIters ) { using Pack = BytePack; using Acc = typename Red::EltType; using AccPack = BytePack; ncclTeam world = ncclTeamWorld(handler.comm); int wn = tn/WARP_SIZE; int w = t/WARP_SIZE; int lane = t%WARP_SIZE; int const& rank = handler.comm.rank; int const& nRanks = handler.comm.nRanks; ncclSymPtr inpPacks = (ncclSymPtr)input + intptr_t(w)*UnrollPacks*WARP_SIZE + lane; ncclSymPtr outPacks = (ncclSymPtr)output + intptr_t(w)*UnrollPacks*WARP_SIZE + lane; Pack acc0[UnrollPacks]; nIters -= w; if (0 < nIters) { #pragma unroll for (int u=0; u < UnrollPacks; u++) { acc0[u] = inpPacks.peerPtr(world, rank)[u*WARP_SIZE]; } } if (waitNeeded) bar.wait(ncclCoopCta(), cuda::memory_order_relaxed); if (0 < nIters) { while (true) { AccPack acc1[UnrollPacks]; int r = rank+1; if (r == nRanks) r = 0; { Pack tmp1[UnrollPacks]; #pragma unroll for (int u=0; u < UnrollPacks; u++) { tmp1[u] = inpPacks.peerPtr(world, r)[u*WARP_SIZE]; } #pragma unroll for (int u=0; u < UnrollPacks; u++) { acc1[u] = applyReduce(red, applyCast(acc0[u]), applyCast(tmp1[u])); } } r += 1; if (r == nRanks) r = 0; int dr = 2; #pragma unroll 2 for (int partial=0; partial <= 1; partial++) { #pragma unroll 1 for (int i = 0; partial ? i < 1 : (dr + UnrollPeers <= nRanks); partial ? i++ : (dr += UnrollPeers)) { if (partial && dr == nRanks) break; Pack tmp1[UnrollPeers][UnrollPacks]; #pragma unroll for (int ur=0; ur < UnrollPeers-partial; ur++) { if (partial && ur!=0 && dr+ur == nRanks) break; #pragma unroll UnrollPacks for (int u=0; u < UnrollPacks; u++) { tmp1[ur][u] = inpPacks.peerPtr(world, r)[u*WARP_SIZE]; } r += 1; if (r == nRanks) r = 0; } #pragma unroll for (int ur=0; ur < UnrollPeers-partial; ur++) { if (partial && ur!=0 && dr+ur == nRanks) break; #pragma unroll UnrollPacks for (int u=0; u < UnrollPacks; u++) { acc1[u] = applyReduce(red, acc1[u], applyCast(tmp1[ur][u])); } } } } #pragma unroll for (int u=0; u < UnrollPacks; u++) acc0[u] = applyCast(acc1[u]); #pragma unroll UnrollPacks for (int u=0; u < UnrollPacks; u++) outPacks.localPtr()[u*WARP_SIZE] = acc0[u]; inpPacks += intptr_t(wn)*UnrollPacks*WARP_SIZE; outPacks += intptr_t(wn)*UnrollPacks*WARP_SIZE; nIters -= wn; if (nIters <= 0) break; // Load data for next iteration. #pragma unroll for (int u=0; u < UnrollPacks; u++) { acc0[u] = inpPacks.peerPtr(world, rank)[u*WARP_SIZE]; } } } } template static __device__ void reduceEnds( ncclSymkArgsHandler const& handler, int tn, int t, Red red, ncclSymPtr input, ncclSymPtr output, size_t nElts, uint32_t nPreElts, size_t nSufElts ) { using Acc = typename Red::EltType; ncclTeam world = ncclTeamWorld(handler.comm); int const& rank = handler.comm.rank; int const& nRanks = handler.comm.nRanks; ncclSymPtr> inpPacks = (ncclSymPtr>)input; ncclSymPtr> outPacks = (ncclSymPtr>)output; #pragma unroll 1 for (size_t i = t; i < nPreElts+nSufElts; i += tn) { size_t elt = i < nPreElts ? i : nElts-nSufElts-nPreElts+i; BytePack acc0 = inpPacks.peerPtr(world, rank)[elt]; BytePack acc1; BytePack tmp[UnrollPeers]; int dr = 1; int r = rank+1; if (nRanks == r) r = 0; bool first = true; #pragma unroll 2 for (int partial=0; partial <= 1; partial++) { #pragma unroll 1 for (int j = 0; partial ? j < 1 : (dr + UnrollPeers <= nRanks); partial ? j++ : (dr += UnrollPeers)) { if (partial && dr == nRanks) break; #pragma unroll for (int u=0; u < UnrollPeers-partial; u++) { if (partial && u!=0 && dr+u == nRanks) break; tmp[u] = inpPacks.peerPtr(world, r)[elt]; r += 1; if (r == nRanks) r = 0; } if (first) { first = false; acc1 = applyCast(acc0); } #pragma unroll for (int u=0; u < UnrollPeers-partial; u++) { if (partial && u!=0 && dr+u == nRanks) break; acc1 = applyReduce(red, acc1, applyCast(tmp[u])); } } } acc0 = applyCast(acc1); outPacks.localPtr()[elt] = acc0; } } template static __device__ void reduce( ncclSymkArgsHandler const& handler, int tn, int t, int nBlocks, bool waitNeeded, ncclLsaBarrierSession& bar, Red red, ncclSymPtr input, ncclSymPtr output, size_t nElts ) { int const& nRanks = handler.comm.nRanks; int const& nRanks_rcp32 = handler.nRanks_rcp32; uint32_t nBlocks_rcp32 = nccl::utility::idivRcp32_upto64(nBlocks); uint32_t nRanks_nBlocks_rcp32 = nccl::utility::imulRcp32(nRanks, nRanks_rcp32, nBlocks, nBlocks_rcp32); uint32_t alignment = uint32_t(input.offset - output.offset); size_t nBytes = nElts*sizeof(T); uint32_t nPreBytes = (16u - input.offset)%16u; nPreBytes = min((size_t)nPreBytes, nBytes); uintptr_t cursor = nPreBytes; constexpr int MinWarpPerBlock = 4; if (alignment%16 == 0) { constexpr int BytePerPack = 16, UnrollPacks = 4, UnrollPeers = 2; constexpr int BytePerChunk = MinWarpPerBlock*UnrollPacks*WARP_SIZE*BytePerPack; uint32_t chunks = (nBytes-cursor)/BytePerChunk; chunks -= imodFast32(chunks, nRanks*nBlocks, nRanks_nBlocks_rcp32); if (chunks != 0) { uintptr_t cursorAfter = cursor + uintptr_t(chunks)*BytePerChunk; reduceDeep( handler, tn, t, waitNeeded, bar, red, (ncclSymPtr)input + cursor, (ncclSymPtr)output + cursor, chunks*MinWarpPerBlock ); cursor = cursorAfter; waitNeeded = false; } } if (sizeof(T) == 4 || (sizeof(T) < 4 && alignment%4 == 0)) { constexpr int BytePerPack = 4, UnrollPacks = 4, UnrollPeers = 4; constexpr int BytePerChunk = MinWarpPerBlock*UnrollPacks*WARP_SIZE*BytePerPack; uint32_t chunks = (nBytes-cursor)/BytePerChunk; chunks -= imodFast32(chunks, nRanks*nBlocks, nRanks_nBlocks_rcp32); if (chunks != 0) { uintptr_t cursorAfter = cursor + uintptr_t(chunks)*BytePerChunk; reduceDeep<(sizeof(T) <= BytePerPack ? BytePerPack : 0), UnrollPacks, UnrollPeers, T>( handler, tn, t, waitNeeded, bar, red, (ncclSymPtr)input + cursor, (ncclSymPtr)output + cursor, chunks*MinWarpPerBlock ); cursor = cursorAfter; waitNeeded = false; } } if (waitNeeded) bar.wait(ncclCoopCta(), cuda::memory_order_relaxed); constexpr int UnrollPeers = 8; size_t nSufElts = (nBytes-cursor)/sizeof(T); reduceEnds(handler, tn, t, red, input, output, nElts, nPreBytes/sizeof(T), nSufElts); } template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_ReduceScatter_LD(ncclSymkDevWorkArgs const* args) { ncclSymkArgsHandler handler{args}; ncclLsaBarrierSession bar{ ncclCoopCta(), handler.comm, ncclTeamTagLsa(), blockIdx.x }; Red::Type> red(handler.devWork->redOpArg); int const& rank = handler.comm.rank; bar.arrive(ncclCoopCta(), cuda::memory_order_relaxed); bool waitNeeded = true; handler.forEachWork( [&]__device__(int block, int nBlocks, size_t nElts, size_t nAllElts, ncclSymPtr input, ncclSymPtr output) { // Round robin warps over blocks. int t = flattenIx(threadIdx.x%WARP_SIZE, WARP_SIZE, block, nBlocks, threadIdx.x/WARP_SIZE, blockDim.x/WARP_SIZE); int tn = nBlocks*blockDim.x; reduce(handler, tn, t, nBlocks, waitNeeded, bar, red, input + rank*nAllElts, output, nElts); waitNeeded = false; } ); bar.sync(ncclCoopCta(), cuda::memory_order_relaxed); } template static __device__ void reduceMultimem( int tn, int t, Red red, T* input, T* output, size_t nElts ) { uintptr_t inputUptr = reinterpret_cast(input); uintptr_t outputUptr = reinterpret_cast(output); size_t nBytes = nElts*sizeof(T); constexpr int BytePerPack = LoadMultimem_BigPackSize::BigPackSize; uint32_t nPreBytes = (BytePerPack - inputUptr)%BytePerPack; nPreBytes = min((size_t)nPreBytes, nBytes); uintptr_t nSufBytes; if (sizeof(T) == BytePerPack || (inputUptr-outputUptr)%BytePerPack == 0) { constexpr int UnrollPacks = 8*(16/BytePerPack); constexpr int BytePerChunk = UnrollPacks*WARP_SIZE*BytePerPack; uintptr_t cursor = nPreBytes; uint32_t nChunks = (nBytes-cursor)/BytePerChunk; uintptr_t cursorAfter = cursor + uintptr_t(nChunks)*BytePerChunk; nSufBytes = nBytes - cursorAfter; cursor += (t/WARP_SIZE)*UnrollPacks*WARP_SIZE*BytePerPack; cursor += (t%WARP_SIZE)*BytePerPack; int nIters = nChunks - t/WARP_SIZE; #pragma unroll 1 while (0 < nIters) { BytePack tmp[UnrollPacks]; #pragma unroll for (int u=0; u < UnrollPacks; u++) { tmp[u] = applyLoadMultimem(red, inputUptr + cursor + u*WARP_SIZE*BytePerPack); } #pragma unroll for (int u=0; u < UnrollPacks; u++) { *reinterpret_cast*>(outputUptr + cursor + u*WARP_SIZE*BytePerPack) = tmp[u]; } cursor += tn*UnrollPacks*BytePerPack; nIters -= tn/WARP_SIZE; } } else { nPreBytes = 0; nSufBytes = nBytes; } // Get the prefix+suffix element one at a time. #pragma unroll 4 for (uintptr_t i = t*sizeof(T); i < nPreBytes + nSufBytes; i += tn*sizeof(T)) { uintptr_t cursor = i < nPreBytes ? i : nBytes-nSufBytes+(i-nPreBytes); BytePack val = applyLoadMultimem(red, inputUptr + cursor); *reinterpret_cast*>(outputUptr + cursor) = val; } } template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_ReduceScatter_LDMC(ncclSymkDevWorkArgs const* args) { ncclSymkArgsHandler handler{args}; ncclLsaBarrierSession bar{ ncclCoopCta(), handler.comm, ncclTeamTagLsa(), blockIdx.x, /*multimem=*/true }; Red::Type> red(handler.devWork->redOpArg); int const& rank = handler.comm.rank; auto const& multimem = handler.comm.lsaMultimem; bar.sync(ncclCoopCta(), cuda::memory_order_relaxed); handler.forEachWork( [&]__device__(int block, int nBlocks, size_t nElts, size_t nAllElts, ncclSymPtr input, ncclSymPtr output) { // Round robin warps over blocks. int t = flattenIx(threadIdx.x%WARP_SIZE, WARP_SIZE, block, nBlocks, threadIdx.x/WARP_SIZE, blockDim.x/WARP_SIZE); int tn = nBlocks*blockDim.x; reduceMultimem(tn, t, red, input.multimemPtr(multimem) + rank*nAllElts, output.localPtr(), nElts); } ); bar.sync(ncclCoopCta(), cuda::memory_order_relaxed); } // T is user type, EltType is the most aligned type template __device__ __forceinline__ void ncclSymkRun_ReduceScatter_LL_body( ncclSymkArgsHandler& handler, ncclLLA2ASession& lla2a, Red red, EltType* input, EltType* output, int nElts, int nPacks, int nStrideElts) { using Pack = BytePack<8>; using Acc = typename Red::EltType; using AccPack = BytePack<8*sizeof(Acc)/sizeof(T)>; constexpr int EltPerPack = 8/sizeof(EltType); int const& nRanks = handler.comm.nRanks; int const& rank = handler.comm.rank; int t = threadIdx.x; constexpr int tn = ncclSymkMaxThreads; ncclCoopCta cta; #pragma unroll 1 while (0 < nElts) { int nIterPacks = min(nPacks, tn); int tn_div_nPacks = tn/nIterPacks; int tn_mod_nPacks = tn%nIterPacks; int peer = t/nIterPacks; int pack = t%nIterPacks; #pragma unroll 1 for (int i = t; i < nRanks*nIterPacks; i += tn) { Pack got = loadPack(input + peer*nStrideElts, pack*EltPerPack, nElts); lla2a.send(peer, rank*nIterPacks + pack, got); peer += tn_div_nPacks; pack += tn_mod_nPacks; if (nIterPacks <= pack) { peer += 1; pack -= nIterPacks; } } if (t < nIterPacks) { AccPack got = lla2a.template recvReduce( /*slotStart=*/t, /*slotCount=*/nRanks, /*slotStride=*/nIterPacks, /*eltToAcc=*/[&] __device__ (Pack x)->AccPack { return applyCast(x); }, /*reduce=*/[&] __device__ (AccPack a, AccPack b)->AccPack { return applyReduce(red, a, b); } ); storePack(output, t*EltPerPack, nElts, applyCast(got)); } lla2a.endEpoch(cta); input += tn*EltPerPack; output += tn*EltPerPack; nElts -= tn*EltPerPack; nPacks -= tn; } } template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_ReduceScatter_LL(ncclSymkDevWorkArgs const* args) { ncclSymkArgsHandler handler{args}; ncclLLA2ASession lla2a( ncclCoopCta(), handler.comm, ncclTeamLsa(handler.comm), handler.lsaLLA2A, blockIdx.x, ncclSymkMaxThreads ); Red::Type> red(handler.devWork->redOpArg); using Pack = BytePack<8>; constexpr int EltPerPack = 8/sizeof(T); handler.singleWork( [&]__device__(int nElts, int nAllElts, ncclSymPtr inputPtr, ncclSymPtr outputPtr) { int nPacks = divUp(nElts, EltPerPack); T* input = (T*)inputPtr.localPtr(); T* output = (T*)outputPtr.localPtr(); uint32_t lowBits = nAllElts*sizeof(T); lowBits |= (uintptr_t)input; lowBits |= (uintptr_t)output; if (__builtin_expect(lowBits%8 == 0, true)) { ncclSymkRun_ReduceScatter_LL_body(handler, lla2a, red, (Pack*)input, (Pack*)output, nPacks, nPacks, divUp(nAllElts, EltPerPack)); } else { ncclSymkRun_ReduceScatter_LL_body(handler, lla2a, red, input, output, nElts, nPacks, nAllElts); } } ); } nccl-2.29.7-1/src/device/symmetric/reduce_scatter_gin.cuh000066400000000000000000000313341515037102200232640ustar00rootroot00000000000000#include "sym_kernels.h" #include "kernel.cuh" #include "primitives.cuh" #include "data_ops.cuh" template typename Red, typename T, bool multimem> static __device__ void rsAlgoHier(ncclSymkDevWorkArgs const* args, BoolTag multimemTag) { ncclCoopCta cta; ncclSymkArgsHandler handler{args}; ncclTeam world = ncclTeamWorld(handler.comm); ncclTeam rail = ncclTeamRail(handler.comm); ncclTeam lsa = ncclTeamLsa(handler.comm); ncclGin gin{handler.comm, int(blockIdx.x % handler.comm.ginContextCount)}; using AccT = typename ncclSymkGinAccumType::Type; Red red(handler.devWork->redOpArg); Red mmRed(handler.devWork->redOpArg); int nWorkWarps = blockDim.x/32 - 2; int stage0_nWorkWarps; if (lsa.nRanks == 1) { stage0_nWorkWarps = 0; // Stage 0 just posts sends so no workers. } else { // Only count reads because they dominate writes. int stage0_work = lsa.nRanks == 1 ? 0 : (multimem ? 1 : lsa.nRanks)*(rail.nRanks-1); int stage1_work = (multimem ? 1 : lsa.nRanks) + rail.nRanks-1; stage0_nWorkWarps = __float2int_rn(__fdividef(nWorkWarps*stage0_work, stage0_work + stage1_work)); stage0_nWorkWarps = min(stage0_nWorkWarps, nWorkWarps-1); // Stage 1 requires at least 1 worker. } // 2 stage pipeline, one coop per stage. int stage = threadIdx.x/32 < 1 + stage0_nWorkWarps ? 0 : 1; ncclCoopWarpSpan coopStage{ /*warp0=*/stage == 0 ? 0 : 1 + stage0_nWorkWarps, /*nWarps=*/1 + (stage == 0 ? stage0_nWorkWarps : nWorkWarps-stage0_nWorkWarps), /*id=*/stage }; // Within each stage we have 2 roles: GIN warp, worker warps. bool roleIsWorker = 32 <= coopStage.thread_rank(); ncclCoopWarpSpan coopRole{ /*warp0=*/(stage == 0 ? 0 : 1 + stage0_nWorkWarps) + (roleIsWorker ? 1 : 0), /*nWarps=*/!roleIsWorker ? 1 : (stage == 0 ? stage0_nWorkWarps : nWorkWarps - stage0_nWorkWarps), /*id=*/2 + stage }; // Construct outbox only for stage=0 when lsa peers exist. alignas(ncclGinOutboxSession) char outbox_storage[sizeof(ncclGinOutboxSession)]; ncclGinOutboxSession& outbox = stage == 0 && lsa.nRanks != 1 ? *::new(&outbox_storage) ncclGinOutboxSession {coopStage, gin, handler.ginOutbox, blockIdx.x} : reinterpret_cast&>(outbox_storage); __shared__ int totalSends; if (stage == 0 && !roleIsWorker && lsa.nRanks == 1) { if (coopRole.thread_rank() == 0) { totalSends = 0; // When pure rail we use a counter to track sends. We could leave them untracked // and end with a flush but by using a counter we can reuse same postSends code // for the rail-only and hybrid (lsa!=1) cases. gin.resetCounter(handler.ginCounterPerBlock + blockIdx.x); } coopRole.sync(); } ncclGinInboxA2ASession inbox {cta, gin, rail, handler.ginInboxRail, blockIdx.x}; ncclLsaBarrierSession lsaBar {cta, handler.comm, ncclTeamTagLsa(), blockIdx.x, multimem}; lsaBar.sync(cta, cuda::memory_order_relaxed); int maxChunkElts = args->maxDynamicSmem/sizeof(AccT); handler.template forEachWorkNoFusion( [&]__device__(size_t nElts, size_t nAllElts, ncclSymPtr input, ncclSymPtr output) { AccT* accum; ncclSymkSmemPartition(&accum, maxChunkElts); int chunkBytes_log2 = log2Up(nElts) + log2Up(sizeof(T)); // Chunk size should not be larger than what host dictates and 1/2 of // total capacity so we enjoy some pipeline overlap. This is a soft constraint // because chunks which are too big can always be partially used. int maxChunkBytes_log2 = min( log2Up(maxChunkElts) + log2Up(sizeof(T)), handler.ginInboxRail.size_log2-1); chunkBytes_log2 = min(chunkBytes_log2, maxChunkBytes_log2); // Chunk size must not be so small that the chunk count exceeds either the // per peer number or total number. This is a hard constraint imposed // by inbox credit logic so we enforce after the max chunk size. int minChunkBytes_log2 = handler.ginInboxRail.size_log2 - min( log2Down(rail.nRanks-1) + ncclGinScratchMaxBufsPerPeer_log2, ncclGinScratchMaxBufs_log2); chunkBytes_log2 = max(chunkBytes_log2, minChunkBytes_log2); int nBufs_log2 = handler.ginInboxRail.size_log2 - chunkBytes_log2; maxChunkElts = min(maxChunkElts, (1u< loopInput = input; ncclSymPtr loopOutput = output; #pragma unroll 1 while (loopElts != 0) { int nChunkElts = min(loopElts, (size_t)maxChunkElts); int nSteps = min(rail.nRanks-1, 1<<(nBufs_log2-1)); int step = 0; initFn(nChunkElts, loopInput); do { nSteps = min(nSteps, rail.nRanks-1 - step); stepFn(step, nSteps, nChunkElts, loopInput); step += nSteps; } while (step != rail.nRanks-1); finishFn(nChunkElts, loopOutput); inbox.endRound(cta); loopInput += nChunkElts; loopOutput += nChunkElts; loopElts -= nChunkElts; } }; if (stage == 0) { // !!! Pipeline Stage 0 !!! inbox.apportion(cta, /*subcoop=*/coopStage, /*subcoopIsNonTrivial=*/false, nBufs_log2); if (lsa.nRanks != 1) { outbox.apportion(coopStage, /*subcoop=*/coopRole, /*subcoopIsNonTrivial=*/roleIsWorker, nBufs_log2, /*deferSync=*/true); } // Generalize worker and non-worker logic with compile time BoolTag to differentiate. auto stage0_impl = [&](/*BoolTag*/auto roleIsWorker_tag) { // Shadow runtime value with compile time value. constexpr bool roleIsWorker = roleIsWorker_tag.value; skeleton( /*initFn*/[&]__device__(int nChunkElts, ncclSymPtr inPtr)->void { if (!roleIsWorker) { if (coopRole.thread_rank() == 0) { // totalSends += rail.nRanks-1; #if __CUDA_ARCH__ >= 700 asm volatile("red.relaxed.shared.add.s32 [%0],%1;" :: "r"((uint32_t)__cvta_generic_to_shared(&totalSends)), "r"(rail.nRanks-1) : "memory"); #else __trap(); #endif } } else { // outbox.apportion() was told to defer sync so that we don't sync // the whole stage. We sync just this warp. coopRole.sync(); } }, /*stepFn=*/[&]__device__(int step0, int nSteps, int nChunkElts, ncclSymPtr inPtr)->void { auto getInputOffset = [&]__device__(int step)->size_t { int peer = inbox.getSendPeer(step, /*step_lt_nPeers=*/true); int dstWorld = ncclTeamRankToTeam(world, rail, peer); return dstWorld*nAllElts; }; if (lsa.nRanks != 1) { // No need to process data when we can send from input buf. if (roleIsWorker) { // Wait for outbox bufs to free up. Since outbox advances with each call of this // function we always index starting at 0. outbox.waitBufs(coopRole, 0, nSteps); coopRole.sync(); // Make `outbox.getBufPtr()` as cheap as possible within reduction loop. auto outbox_getBufPtr = outbox.make_getBufPtr(0); reduceLsaBatch(coopRole, /*nBatch=*/nSteps, nChunkElts, /*dstMem=*/GMemTag(), /*dstAlignMin=*/16, /*getDst=*/[&]__device__(int i)->T* { return (T*)outbox_getBufPtr(i); }, /*srcRedUc=*/red, /*srcRedMc=*/mmRed, /*srcBase=*/inPtr, /*getSrcOffset=*/[&]__device__(int i)->size_t { return getInputOffset(step0 + i); }, handler.comm, multimemTag ); } coopStage.sync(); } if (!roleIsWorker) { inbox.postSends(coopRole, step0, nSteps, /*getPtr*/[&]__device__(int i, int peer) { return lsa.nRanks == 1 ? inPtr + getInputOffset(step0 + i) : (ncclSymPtr)outbox.getBuf(i); }, /*getEltCount*/[&]__device__(int i, int peer) { return nChunkElts; }, /*getCompletion*/[&]__device__(int i, int peer) { return ncclGin_CounterInc{ lsa.nRanks == 1 ? handler.ginCounterPerBlock + blockIdx.x : outbox.getCounter(i) }; } ); } if (lsa.nRanks != 1) { // We advance outbox with every iteration because it is only guaranteed // to have `nSteps` buffers. If we advanced it after the step loop it // would need rail.nRanks-1 buffers. outbox.advance(coopStage, nSteps); } }, /*finishFn=*/nop ); }; // Instantiate stage0_impl specialized to each case of roleIsWorker. if (!roleIsWorker) stage0_impl(BoolTag()); else stage0_impl(BoolTag()); } else { // !!! Pipeline Stage 1 !!! // Generalize worker and non-worker logic with compile time BoolTag to differentiate. auto stage1_impl = [&]__device__(/*BoolTag*/auto roleIsWorker_tag)->void { constexpr bool roleIsWorker = roleIsWorker_tag.value; inbox.apportion(cta, /*subcoop=*/coopRole, /*subcoopIsNonTrivial=*/!roleIsWorker, nBufs_log2); coopStage.sync(); skeleton( /*initFn*/[&]__device__(int nChunkElts, ncclSymPtr inPtr)->void { if (roleIsWorker) { reduceLsa(coopRole, nChunkElts, /*dstMem=*/SMemTag(), /*dstAlignMin=*/16, /*dstPtr=*/accum, /*srcRedUc=*/red, /*srcRedMc=*/mmRed, /*srcPtr=*/inPtr + world.rank*nAllElts, handler.comm, multimemTag ); } }, /*stepFn*/[&]__device__(int step0, int nSteps, int nChunkElts, ncclSymPtr inPtr)->void { if (roleIsWorker) { inbox.waitRecvs(coopRole, step0, nSteps); coopRole.sync(); // Make `inbox.getBufPtr()` as cheap as possible within reduction loop. auto inbox_getBufPtr = inbox.make_getBufPtr(step0); reduce(coopRole, red, /*inPlace=*/true, nChunkElts, /*dstMem=*/SMemTag(), /*dstAlignMin=*/16, /*dst=*/accum, /*nSrcs=*/nSteps, /*srcPtrCommonMask=*/16-1, /*srcPtrMasked=*/0, /*getSrc=*/[&]__device__(int s)->T* { return (T*)inbox_getBufPtr(s); } ); } coopStage.sync(); if (!roleIsWorker) { inbox.finishRecvs(coopRole, step0, nSteps); } }, /*finishFn*/[&]__device__(int nChunkElts, ncclSymPtr outPtr)->void { if (roleIsWorker) { copy(coopRole, nChunkElts, /*dstMem=*/GMemTag(), /*dst=*/outPtr.localPtr(), /*srcMem=*/SMemTag(), /*src=*/accum); coopRole.sync(); // prevent initFn from trampling accum } } ); }; // Instantiate stage1_impl specialized to each case of roleIsWorker. if (!roleIsWorker) stage1_impl(BoolTag()); else stage1_impl(BoolTag()); } } ); if (stage == 0) { if (lsa.nRanks == 1) { if (!roleIsWorker && coopRole.thread_rank() == 0) { gin.waitCounter(ncclCoopThread(), handler.ginCounterPerBlock + blockIdx.x, totalSends, 32); } } else { outbox.template ~ncclGinOutboxSession(); } } lsaBar.sync(cta, cuda::memory_order_relaxed); } template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_ReduceScatter_RailA2A_LsaLD(ncclSymkDevWorkArgs const* args) { rsAlgoHier(args, /*multimem=*/BoolTag{}); } template typename Red, typename T> __device__ __forceinline__ void ncclSymkRun_ReduceScatter_RailA2A_LsaLDMC(ncclSymkDevWorkArgs const* args) { rsAlgoHier(args, /*multimem=*/BoolTag{}); } nccl-2.29.7-1/src/enhcompat.cc000066400000000000000000000027371515037102200157510ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ /* Define weak symbols used to allow libnccl_static.a to work with older libcudart_static.a */ enum cudaError_t { cudaErrorStubLibrary = 34 }; extern "C" { cudaError_t cudaStreamGetCaptureInfo_v2(...) __attribute__((visibility("hidden"))) __attribute((weak)); cudaError_t cudaStreamGetCaptureInfo_v2(...) { return cudaErrorStubLibrary; } cudaError_t cudaUserObjectCreate(...) __attribute__((visibility("hidden"))) __attribute((weak)); cudaError_t cudaUserObjectCreate(...) { return cudaErrorStubLibrary; } cudaError_t cudaGraphRetainUserObject(...) __attribute__((visibility("hidden"))) __attribute((weak)); cudaError_t cudaGraphRetainUserObject(...) { return cudaErrorStubLibrary; } cudaError_t cudaStreamUpdateCaptureDependencies(...) __attribute__((visibility("hidden"))) __attribute((weak)); cudaError_t cudaStreamUpdateCaptureDependencies(...) { return cudaErrorStubLibrary; } cudaError_t cudaGetDriverEntryPoint(...) __attribute__((visibility("hidden"))) __attribute((weak)); cudaError_t cudaGetDriverEntryPoint(...) { return cudaErrorStubLibrary; } } nccl-2.29.7-1/src/enqueue.cc000066400000000000000000004140411515037102200154350ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "enqueue.h" #include "argcheck.h" #include "coll_net.h" #include "gdrwrap.h" #include "bootstrap.h" #include "channel.h" #include "cudawrap.h" #include "profiler.h" #include "transport.h" #include "register_inline.h" #include "ce_coll.h" #include "nvtx.h" #include "scheduler.h" #include "compiler.h" #include "rma/rma.h" #include // std::memcpy #include // PRIx64 #include #include // FLT_MAX NCCL_PARAM(L1SharedMemoryCarveout, "L1_SHARED_MEMORY_CARVEOUT", 0); NCCL_PARAM(AllgathervEnable, "ALLGATHERV_ENABLE", 1); NCCL_PARAM(SymCeThreshold, "SYM_CE_THRESHOLD", 8*1024*1024); // Returns maximum kernel stack size of all CUDA kernels ncclResult_t ncclInitKernelsForDevice(int cudaArch, int maxSharedMem, size_t* maxStackSize) { ncclResult_t result = ncclSuccess; if (maxStackSize) *maxStackSize = 0; int carveout = ncclParamL1SharedMemoryCarveout(); int maxDynamicSmem = 1<<30; int driverVersion; NCCLCHECK(ncclCudaDriverVersion(&driverVersion)); for (int sym=0; sym <= 1; sym++) { int kcount = sym==0 ? ncclDevKernelCount : ncclSymkKernelCount; void** kptrs = sym==0 ? ncclDevKernelList : ncclSymkKernelList; int* krequires = sym==0 ? ncclDevKernelRequirements : ncclSymkKernelRequirements; for (int k=0; k < kcount; k++) { if (kptrs[k] != nullptr && driverVersion < krequires[k]) { INFO(NCCL_INIT, "Skipping %skernel %d which requires driver %d", sym ? "symmetric " : "", k, krequires[k]); kptrs[k] = nullptr; } void* fn = kptrs[k]; cudaFuncAttributes attr = {0}; if (fn == nullptr) continue; if (!CUDASUCCESS(cudaFuncGetAttributes(&attr, fn))) continue; // Silently ignore failures if (maxStackSize) { if (attr.localSizeBytes > *maxStackSize) *maxStackSize = attr.localSizeBytes; } if (carveout) { CUDACHECKGOTO(cudaFuncSetAttribute(fn, cudaFuncAttributePreferredSharedMemoryCarveout, carveout), result, ignore1); ignore1:; } { int dynSmem = maxSharedMem - attr.sharedSizeBytes; if (sym) { ncclSymkKernelMaxDynamicSmem[k] = dynSmem; } else { maxDynamicSmem = std::min(maxDynamicSmem, dynSmem); } CUDACHECKGOTO(cudaFuncSetAttribute(fn, cudaFuncAttributeMaxDynamicSharedMemorySize, dynSmem), result, next_kernel); } next_kernel:; } } if (ncclShmemDynamicSize(cudaArch) > maxDynamicSmem) { WARN("cudaArch %d dynamic smem %d exceeds device/fn maxSharedMem %d", cudaArch, ncclShmemDynamicSize(cudaArch), maxDynamicSmem); return ncclSystemError; } return result; } //////////////////////////////////////////////////////////////////////////////// // Data movement metrics. static inline int ncclFuncTrafficPerByte(ncclFunc_t func, int nRanks) { switch (func) { case ncclFuncAllReduce: return 2; case ncclFuncAllGather: return nRanks; case ncclFuncReduceScatter: return nRanks; default: return 1; } } /*****************************************************************************/ /* Launch system : synchronization and CUDA kernel launch */ /*****************************************************************************/ ncclResult_t ncclAddProxyOpIfNeeded(struct ncclComm* comm, struct ncclKernelPlan* plan, struct ncclProxyOp* op) { bool needed = true; NCCLCHECK(ncclProxySaveOp(comm, op, &needed)); if (needed) { struct ncclProxyOp* q = ncclMemoryPoolAlloc(&comm->memPool_ncclProxyOp, &comm->memPermanent); *q = *op; // C++ struct assignment ncclIntruQueueEnqueue(&comm->planner.wipPlan.channels[op->channelId].proxyOpQueue, q); } return ncclSuccess; } NCCL_PARAM(P2pEpochEnable, "P2P_EPOCH_ENABLE", 1); void ncclAddWorkBatchToPlan( struct ncclComm* comm, struct ncclKernelPlan* plan, int channelId, enum ncclDevWorkType workType, int devFuncId, uint32_t workOffset, int p2pEpoch, int p2pRound, bool newBatch ) { size_t workSize = ncclDevWorkSize(workType); ncclKernelPlanner::WipPlan::Channel* chan = &comm->planner.wipPlan.channels[channelId]; // Conditions causing us to create a new blank batch. newBatch = (chan->workBatchQueue.tail == nullptr); struct ncclDevWorkBatch* batch = nullptr; if (!newBatch) { batch = &chan->workBatchQueue.tail->batch; // All of the conditions that prevent us from appending to current batch. newBatch |= batch->workType != (uint8_t)workType; newBatch |= batch->funcId != devFuncId; // The following ensure the device can handle a batch this large. They have to // account for all extension batches being fused together which is why // wipBatch.workBytes and wipBatch.nP2ps aren't reset to 0 for a new extension // batch further down. if (workType == ncclDevWorkTypeP2p) { if (ncclParamP2pEpochEnable()) newBatch |= chan->wipBatch.p2pEpoch != p2pEpoch; // We only allow NCCL_MAX_DEV_WORK_P2P_PER_BATCH ops per batch. newBatch |= chan->wipBatch.nP2ps == NCCL_MAX_DEV_WORK_P2P_PER_BATCH; for (int i = 0; i < chan->wipBatch.nP2ps; i++) { // Do not allow the same round twice in the same batch, it would use the same connection. newBatch |= p2pRound == chan->wipBatch.p2pRounds[i]; // Make sure we only aggregate p2p operations within the same p2p group (one group is NCCL_MAX_DEV_WORK_P2P_PER_BATCH ops). // This enforces uniform batching accross ranks in the communicator and prevents hangs. newBatch |= (p2pRound / NCCL_MAX_DEV_WORK_P2P_PER_BATCH) != (chan->wipBatch.p2pRounds[i] / NCCL_MAX_DEV_WORK_P2P_PER_BATCH); } } if (workType == ncclDevWorkTypeBcast) { int maxitem = ncclMaxDevWorkBatchBytes(comm->cudaArch) / sizeof(ncclDevWorkBcast); newBatch |= chan->wipBatch.nBcasts == maxitem; } else { newBatch |= NCCL_MAX_DEV_WORK_BATCH_BYTES < chan->wipBatch.workBytes + workSize; } } // Conditions causing us to create an extension batch (prev->nextExtends=1) uint32_t offset = newBatch ? 0 : (workOffset - batch->offsetBase); bool extendBatch = 63*workSize < offset; extendBatch |= 0 != offset%workSize; if (newBatch || extendBatch) { if (!newBatch) batch->nextExtends = extendBatch; // Extending the previous batch. struct ncclWorkBatchList* batchNode = ncclMemoryStackAlloc(&comm->memScoped); // Coverity thinks that ncclIntruQueueEnqueue will access chan->workBatchQueue->tail, which might // be NULL. But that code is guarded by chan->workBatchQueue->head not being NULL, in which // case tail won't be NULL either. // coverity[var_deref_model:FALSE] ncclIntruQueueEnqueue(&chan->workBatchQueue, batchNode); batch = &batchNode->batch; batch->nextExtends = 0; batch->workType = (uint32_t)workType; batch->funcId = devFuncId; batch->offsetBase = workOffset; batch->offsetBitset = 0; offset = 0; if (newBatch) { // Since extension batches are fused together on the device, and these values // account for constraints on the fused batch, we only reset the values on // a new batch chan->wipBatch.workBytes = 0; chan->wipBatch.nP2ps = 0; chan->wipBatch.nBcasts = 0; // We don't count extension batches since this is used to derive a proxyOpCount, // and we wan't all ops which are fused together to have the same value. chan->nWorkBatchesP2p += (workType == ncclDevWorkTypeP2p ? 1 : 0); chan->nWorkBatchesBcast += (workType == ncclDevWorkTypeBcast ? 1 : 0); } plan->nWorkBatches += 1; } batch->offsetBitset |= 1ull<<(offset/workSize); chan->wipBatch.workBytes += workSize; if (workType == ncclDevWorkTypeP2p) { chan->wipBatch.p2pEpoch = p2pEpoch; chan->wipBatch.p2pRounds[chan->wipBatch.nP2ps++] = p2pRound; } if (workType == ncclDevWorkTypeBcast) { chan->wipBatch.nBcasts += 1; } } static void finishPlan(struct ncclComm* comm, struct ncclKernelPlan* plan) { ncclKernelPlanner::WipPlan::Channel* wipChannels = comm->planner.wipPlan.channels; size_t workBytes = plan->workBytes; size_t batchBytes = plan->nWorkBatches*sizeof(struct ncclDevWorkBatch); if (plan->isSymColl) return; plan->threadPerBlock = std::max(plan->threadPerBlock, NCCL_MIN_NTHREADS); // If we can fit everything into the kernel args we do so. if (sizeof(ncclDevKernelArgs) + batchBytes + workBytes <= comm->workArgsBytes) { plan->workStorageType = ncclDevWorkStorageTypeArgs; } plan->kernelArgsSize = sizeof(struct ncclDevKernelArgs) + batchBytes; plan->kernelArgsSize += (plan->workStorageType == ncclDevWorkStorageTypeArgs) ? workBytes : 0; plan->kernelArgsSize = alignUp(plan->kernelArgsSize, 16); plan->kernelArgs = (struct ncclDevKernelArgs*)ncclMemoryStackAlloc(&comm->memScoped, plan->kernelArgsSize, /*align=*/16); plan->kernelArgs->comm = comm->devComm; plan->kernelArgs->channelMask = plan->channelMask; plan->kernelArgs->workStorageType = plan->workStorageType; // Put batches into the kernel arguments. The first batch for each channel // must be located at batchZero[blockIdx.x]. To achieve this we round robin // over the channels in ascending order until they're exhausted. uint64_t hasBatchMask = plan->channelMask; struct ncclDevWorkBatch* batchPrev[MAXCHANNELS] = {}; // {0...} struct ncclDevWorkBatch* batchZero = (struct ncclDevWorkBatch*)(plan->kernelArgs+1); int batchIx = 0; while (hasBatchMask != 0) { uint64_t tmpMask = hasBatchMask; // channels with a batch for this round. do { int c = popFirstOneBit(&tmpMask); if (!ncclIntruQueueEmpty(&wipChannels[c].workBatchQueue)) { struct ncclWorkBatchList* batchNode = ncclIntruQueueDequeue(&wipChannels[c].workBatchQueue); if (batchPrev[c] != nullptr) { batchPrev[c]->nextJump = int(&batchZero[batchIx] - batchPrev[c]); } batchPrev[c] = &batchZero[batchIx]; batchZero[batchIx++] = batchNode->batch; } if (ncclIntruQueueEmpty(&wipChannels[c].workBatchQueue)) { hasBatchMask ^= 1ull<proxyOpQueue // Phase 1: scan first op of each channel, store opCount in headIds[c]. uint64_t headIds[MAXCHANNELS]; int nHeads = 0; int channelUbound = 0; for (int c=0; c < MAXCHANNELS; c++) { struct ncclProxyOp* op = ncclIntruQueueHead(&wipChannels[c].proxyOpQueue); headIds[c] = op ? op->opCount : uint64_t(-1); if (op) nHeads += 1; if (op) plan->hasProxyOps = true; if (op) channelUbound = c+1; } // Phase 2: Dequeue from planner->channels[c], enqueue in merged order to plan while (nHeads != 0) { int c = -1; uint64_t minId = uint64_t(-1); // Find channel with least proxy-op id. We store the heads[c]->opCount in // headIds[c] to remove indirect loads from this loop. for (int c1=0; c1 < channelUbound; c1++) { uint64_t id = headIds[c1]; id = (id>>1 | id<<63); // Move tag bit to order collectives before p2p's if (id < minId) { c = c1; minId = id; } } struct ncclProxyOp* op = ncclIntruQueueDequeue(&wipChannels[c].proxyOpQueue); struct ncclProxyOp* opNext = ncclIntruQueueHead(&wipChannels[c].proxyOpQueue); headIds[c] = opNext ? opNext->opCount : uint64_t(-1); nHeads -= opNext ? 0 : 1; ncclIntruQueueEnqueue(&plan->proxyOpQueue, op); } } NCCL_PARAM(GraphRegister, "GRAPH_REGISTER", 1); static ncclResult_t calcCollChunking( struct ncclComm* comm, struct ncclTaskColl* task, int nChannels, size_t nBytes, /*outputs*/uint32_t* outChunkSize, uint32_t* outDirectFlags, struct ncclProxyOp* proxyOp ); struct ncclKernelPlanBudget { ssize_t inArgsBytes; // Space available within kernel args struct ssize_t outArgsBytes; // Space available outside of args struct (fifo or persistent buf) }; bool ncclTestBudget( struct ncclKernelPlanBudget* budget, int nWorkBatches, ssize_t workBytes ) { ssize_t batchBytes = nWorkBatches*sizeof(struct ncclDevWorkBatch); bool ok = false; ok |= (batchBytes + workBytes <= budget->inArgsBytes); ok |= (batchBytes <= budget->inArgsBytes) && (workBytes <= budget->outArgsBytes); return ok; } ncclResult_t ncclTasksRegAndEnqueue(struct ncclComm* comm) { struct ncclKernelPlanner* planner = &comm->planner; struct ncclTaskColl *task; task = ncclIntruQueueHead(&planner->collTaskQueue); while (task != nullptr) { // Build a ncclDevWorkColl[Reg?] struct for each task. void* regBufSend[NCCL_MAX_LOCAL_RANKS]; void* regBufRecv[NCCL_MAX_LOCAL_RANKS]; bool regNeedConnect = true; struct ncclWorkList* workNode = NULL; struct ncclDevWorkColl devWork = {}; if (task->algorithm == NCCL_ALGO_NVLS_TREE || task->algorithm == NCCL_ALGO_NVLS) { workNode = ncclIntruQueueDequeue(&planner->tmpCollWorkQueue); goto next; } ncclRegisterCollBuffers(comm, task, regBufSend, regBufRecv, &planner->collCleanupQueue, ®NeedConnect); devWork.sendbuff = (void*)task->sendbuff; devWork.recvbuff = (void*)task->recvbuff; devWork.sendbuffOffset = task->sendbuffOffset; devWork.recvbuffOffset = task->recvbuffOffset; devWork.sendbuffRmtAddrs = task->sendbuffRmtAddrs; devWork.recvbuffRmtAddrs = task->recvbuffRmtAddrs; devWork.root = task->root; devWork.nWarps = task->nWarps; devWork.redOpArg = task->opDev.scalarArg; devWork.redOpArgIsPtr = task->opDev.scalarArgIsPtr; devWork.oneNode = (comm->nNodes == 1); devWork.isOneRPN = comm->isOneRPN; devWork.netRegUsed = devWork.regUsed = 0; devWork.profilerEnabled = ncclProfilerPluginLoaded() && (task->eActivationMask & ncclProfileKernelCh); if (task->regBufType & NCCL_NET_REG_BUFFER) devWork.netRegUsed = 1; if (task->regBufType & (NCCL_IPC_REG_BUFFER | NCCL_NVLS_REG_BUFFER)) devWork.regUsed = 1; if (task->regBufType & NCCL_NVLS_REG_BUFFER) { struct ncclDevWorkCollReg workReg = {}; workReg.coll = devWork; // C++ struct assignment /* NVLS only has one send and recv buffer registered */ workReg.dnInputs[0] = regBufSend[0]; workReg.dnOutputs[0] = regBufRecv[0]; workNode = ncclMemoryStackAllocInlineArray(&comm->memScoped, 1); workNode->workType = ncclDevWorkTypeCollReg; workNode->size = sizeof(struct ncclDevWorkCollReg); memcpy((void*)(workNode+1), (void*)&workReg, workNode->size); } else { workNode = ncclMemoryStackAllocInlineArray(&comm->memScoped, 1); workNode->workType = ncclDevWorkTypeColl; workNode->size = sizeof(struct ncclDevWorkColl); memcpy((void*)(workNode+1), (void*)&devWork, workNode->size); } next: ncclIntruQueueEnqueue(&planner->collWorkQueue, workNode); task = task->next; } assert(ncclIntruQueueEmpty(&planner->tmpCollWorkQueue)); return ncclSuccess; } // Called once per ncclGroup to organize the user submitted tasks in // comm->planner so that they can be peeled off into plans. ncclResult_t ncclPrepareTasks(struct ncclComm* comm, bool* algoNeedConnect, bool* needConnect, ncclSimInfo_t* simInfo) { struct ncclKernelPlanner* planner = &comm->planner; planner->persistent = ncclCudaGraphValid(planner->capturingGraph); // Put bcast tasks into collSorter if there's only one bcast peer if (planner->bcast_info.BcastPeers == 1) { while (!ncclIntruQueueEmpty(&planner->peers[planner->bcast_info.minBcastPeer].bcastQueue)) { struct ncclTaskBcast* bcastTask = ncclIntruQueueDequeue(&planner->peers[planner->bcast_info.minBcastPeer].bcastQueue); struct ncclTaskColl *t = ncclMemoryPoolAlloc(&comm->memPool_ncclTaskColl, &comm->memPermanent); t->func = ncclFuncBroadcast; t->sendbuff = bcastTask->sendbuff; t->recvbuff = bcastTask->recvbuff; t->count = bcastTask->count; t->root = bcastTask->root; t->datatype = bcastTask->datatype; t->trafficBytes = t->count*ncclFuncTrafficPerByte(t->func, comm->nRanks); t->chunkSteps = BROADCAST_CHUNKSTEPS; t->sliceSteps = BROADCAST_SLICESTEPS; ncclTaskCollSorterInsert(&planner->collSorter, t, t->trafficBytes); planner->nTasksColl += 1; ncclMemoryPoolFree(&comm->memPool_ncclTaskBcast, bcastTask); } // reset bcast info planner->nTasksBcast = 0; planner->bcast_info.BcastPeers = 0; } // Tasks from the sorter come out ordered size descending. struct ncclTaskColl* task = ncclTaskCollSorterDequeueAll(&planner->collSorter); // Tasks are assembled by (fn,op,ty) size ascending. struct ncclTaskColl* tasksByFnOpTy[ncclNumFuncs*ncclNumDevRedOps*ncclNumTypes]; memset(tasksByFnOpTy, 0, sizeof(tasksByFnOpTy)); int fnOpTyIndices[ncclNumFuncs*ncclNumDevRedOps*ncclNumTypes]; int fnOpTyCount = 0; if (comm->symmetricSupport) { NCCLCHECK(ncclMakeSymmetricTaskList(comm, task, &planner->collSymTaskQueue, &task)); } // Walk the size sorted tasks, binning them by (fn,op,ty). while (task != nullptr) { struct ncclTaskColl* next = task->next; int index = ((int)task->func*ncclNumDevRedOps + (int)task->opDev.op)*ncclNumTypes + (int)task->datatype; // Add to set of (fn,op,ty) indices on first occurrence if (tasksByFnOpTy[index] == nullptr) fnOpTyIndices[fnOpTyCount++] = index; // Add to LIFO for this (fn,op,ty) task->next = tasksByFnOpTy[index]; tasksByFnOpTy[index] = task; // Next task task = next; } // Walk (fn,op,ty) bins, compute algo and proto etc. Then bin them by their // scheduling constraints (collnet x nvls). struct ncclIntruQueue collBins[2][2] = {}; for (int cursor=0; cursor < fnOpTyCount; cursor++) { struct ncclTaskColl* aggBeg = tasksByFnOpTy[fnOpTyIndices[cursor]]; int collNetSupport = 0; NCCLCHECK(ncclGetCollNetSupport(comm, aggBeg, &collNetSupport)); int nvlsSupport = comm->nvlsSupport && (ncclNvlsSupported(aggBeg->opDev.op, aggBeg->datatype) || aggBeg->func == ncclFuncAllGather); // Crudely estimate number of tasks per channel. This is using the wrong number // of channels for NVLS algos, but knowing the algo requires having this value, // so either be crude our iterate until fixed point, we chose the former. int nTasksPerChannel = divUp(comm->planner.nTasksColl, comm->nChannels); do { struct ncclTaskColl* aggEnd = aggBeg->next; struct ncclTaskColl agg = *aggBeg; // We aggregate operations that are within 4X size of each other. while (aggEnd != nullptr && aggEnd->trafficBytes < 4*aggBeg->trafficBytes) { agg.count += aggEnd->count; agg.trafficBytes += aggEnd->trafficBytes; aggEnd = aggEnd->next; } NCCLCHECK(ncclGetAlgoInfo(comm, &agg, collNetSupport, nvlsSupport, nTasksPerChannel, simInfo)); agg.devFuncId = ncclDevFuncId(agg.func, agg.opDev.op, agg.datatype, agg.algorithm, agg.protocol); int isCollnet=0, isNvls=0; switch (agg.algorithm) { case NCCL_ALGO_NVLS: case NCCL_ALGO_NVLS_TREE: isNvls = 1; isCollnet = agg.algorithm == NCCL_ALGO_NVLS && comm->nNodes > 1; break; case NCCL_ALGO_COLLNET_CHAIN: case NCCL_ALGO_COLLNET_DIRECT: isCollnet = 1; break; } // Update the aggregated tasks with the computed values. do { struct ncclTaskColl* next = aggBeg->next; aggBeg->algorithm = agg.algorithm; aggBeg->protocol = agg.protocol; if (aggBeg->protocol == NCCL_PROTO_LL) aggBeg->trafficBytes *= 4; aggBeg->nMaxChannels = agg.nMaxChannels; aggBeg->nWarps = agg.nWarps; aggBeg->devFuncId = agg.devFuncId; aggBeg->isCollnet = isCollnet; aggBeg->isNvls = isNvls; ncclIntruQueueEnqueue(&collBins[isCollnet][isNvls], aggBeg); aggBeg = next; } while (aggBeg != aggEnd); } while (aggBeg != nullptr); } // Concatenate `collBins[*][*]` together into final list `planner->collTaskQueue`. // Collnet is the outer dimension since that affects how we divide over the // channels. for (int isCollnet=0; isCollnet <= 1; isCollnet++) { for (int isNvls=0; isNvls <= 1; isNvls++) { ncclIntruQueueTransfer(&planner->collTaskQueue, &collBins[isCollnet][isNvls]); } } // Walk tasks again to: // 1. Possibly register buffers. // 2. Build ncclDevWorkColl structs. // 3. Bin the work structs according to the number of valid channels they // may be assigned to {collnet, nvls, standard} task = ncclIntruQueueHead(&planner->collTaskQueue); while (task != nullptr) { // Build a ncclDevWorkColl[Reg?] struct for each task. void* regBufSend[NCCL_MAX_LOCAL_RANKS]; void* regBufRecv[NCCL_MAX_LOCAL_RANKS]; bool regNeedConnect = true; ncclRegisterCollNvlsBuffers(comm, task, regBufSend, regBufRecv, &planner->collCleanupQueue, ®NeedConnect); if (comm->runtimeConn && comm->initAlgoChannels[task->algorithm] == false) { if (task->algorithm == NCCL_ALGO_NVLS_TREE && comm->initAlgoChannels[NCCL_ALGO_NVLS] == false && regNeedConnect == true) { comm->initAlgoChannels[NCCL_ALGO_NVLS] = true; algoNeedConnect[NCCL_ALGO_NVLS] = true; } if (task->algorithm != NCCL_ALGO_NVLS || regNeedConnect == true) { comm->initAlgoChannels[task->algorithm] = true; algoNeedConnect[task->algorithm] = true; *needConnect = true; } } if (task->algorithm == NCCL_ALGO_NVLS_TREE || task->algorithm == NCCL_ALGO_NVLS) { struct ncclDevWorkColl devWork = {}; devWork.sendbuff = (void*)task->sendbuff; devWork.recvbuff = (void*)task->recvbuff; devWork.sendbuffOffset = task->sendbuffOffset; devWork.recvbuffOffset = task->recvbuffOffset; devWork.sendbuffRmtAddrs = task->sendbuffRmtAddrs; devWork.recvbuffRmtAddrs = task->recvbuffRmtAddrs; devWork.root = task->root; devWork.nWarps = task->nWarps; devWork.redOpArg = task->opDev.scalarArg; devWork.redOpArgIsPtr = task->opDev.scalarArgIsPtr; devWork.oneNode = (comm->nNodes == 1); devWork.netRegUsed = devWork.regUsed = 0; devWork.profilerEnabled = ncclProfilerPluginLoaded() && (task->eActivationMask & ncclProfileKernelCh); if (task->regBufType & NCCL_NET_REG_BUFFER) devWork.netRegUsed = 1; if (task->regBufType & (NCCL_IPC_REG_BUFFER | NCCL_NVLS_REG_BUFFER)) devWork.regUsed = 1; struct ncclWorkList* workNode; if (task->regBufType & NCCL_NVLS_REG_BUFFER) { struct ncclDevWorkCollReg workReg = {}; workReg.coll = devWork; // C++ struct assignment /* NVLS only has one send and recv buffer registered */ workReg.dnInputs[0] = regBufSend[0]; workReg.dnOutputs[0] = regBufRecv[0]; workNode = ncclMemoryStackAllocInlineArray(&comm->memScoped, 1); workNode->workType = ncclDevWorkTypeCollReg; workNode->size = sizeof(struct ncclDevWorkCollReg); memcpy((void*)(workNode + 1), (void*)&workReg, workNode->size); } else { workNode = ncclMemoryStackAllocInlineArray(&comm->memScoped, 1); workNode->workType = ncclDevWorkTypeColl; workNode->size = sizeof(struct ncclDevWorkColl); memcpy((void*)(workNode + 1), (void*)&devWork, workNode->size); } ncclIntruQueueEnqueue(&planner->tmpCollWorkQueue, workNode); } task = task->next; } // Process broadcast tasks for runtimeConn if (comm->runtimeConn && planner->nTasksBcast > 0) { for (int peer = planner->bcast_info.minBcastPeer; peer <= planner->bcast_info.maxBcastPeer; peer++) { struct ncclTaskBcast* bcastTask = ncclIntruQueueHead(&planner->peers[peer].bcastQueue); while (bcastTask != nullptr) { if (comm->initAlgoChannels[NCCL_ALGO_RING] == false) { comm->initAlgoChannels[NCCL_ALGO_RING] = true; algoNeedConnect[NCCL_ALGO_RING] = true; *needConnect = true; } bcastTask = bcastTask->next; } } } return ncclSuccess; } static ncclResult_t addProfilerProxyOpIfNeeded(struct ncclComm* comm, struct ncclKernelPlan* plan, struct ncclProxyOp* op) { int tmp = op->pattern; op->pattern = ncclPatternProfiler; ncclResult_t ret = ncclAddProxyOpIfNeeded(comm, plan, op); op->pattern = tmp; return ret; } static ncclResult_t scheduleCollTasksToPlan( struct ncclComm* comm, struct ncclKernelPlan* plan, struct ncclKernelPlanBudget* budget ) { struct ncclKernelPlanner* planner = &comm->planner; // Estimate number of tasks that will fit in this plan. int nPlanColls = 0; size_t trafficBytes[2*2] = {0, 0, 0, 0}; // [collnet][nvls] int nChannels[2*2] = {0, 0, 0, 0}; // [collnet][nvls] int const nMaxChannels[2*2] = {comm->nChannels, comm->nvlsChannels, // [collnet][nvls] comm->nChannels, std::min(comm->nChannels, comm->nvlsChannels)}; constexpr size_t MinTrafficPerChannel = 32 << 10; // 32K traffic as minimal do { size_t workBytes = 0; struct ncclTaskColl* task = ncclIntruQueueHead(&planner->collTaskQueue); struct ncclWorkList* workNode = ncclIntruQueueHead(&planner->collWorkQueue); while (task != nullptr) { int nBatches = divUp(nPlanColls, 4); // Rough guess: 4 colls per batch. if (!ncclTestBudget(budget, nBatches, workBytes + workNode->size)) goto plan_full; nPlanColls += 1; workBytes += workNode->size; int kind = 2*task->isCollnet + task->isNvls; trafficBytes[kind] += std::max(MinTrafficPerChannel, task->trafficBytes); nChannels[kind] += task->nMaxChannels; nChannels[kind] = std::min(nChannels[kind], nMaxChannels[kind]); task = task->next; workNode = workNode->next; } plan_full:; } while (0); int kindPrev = -1; size_t trafficPerChannel = 0; int channelId = 0; size_t currentTraffic = 0; while (nPlanColls!=0 && !ncclIntruQueueEmpty(&planner->collTaskQueue)) { struct ncclTaskColl* task = ncclIntruQueueHead(&planner->collTaskQueue); struct ncclWorkList* workNode = ncclIntruQueueHead(&planner->collWorkQueue); struct ncclDevWorkColl* devWork = (struct ncclDevWorkColl*)(workNode+1); size_t elementSize = ncclTypeSize(task->datatype); int kind = 2*task->isCollnet + task->isNvls; if (kind != kindPrev) { trafficPerChannel = divUp(trafficBytes[kind] / nChannels[kind], 16) * 16; kindPrev = kind; channelId = 0; currentTraffic = 0; } if (task->isCollnet) { int nChannels = task->nMaxChannels; // Ensure room for worst case of one new batch per channel if (!ncclTestBudget(budget, plan->nWorkBatches + nChannels, plan->workBytes + workNode->size)) { return ncclSuccess; } size_t globalBytesPerElement = elementSize*ncclFuncMaxSendRecvCount(task->func, comm->nRanks, 1); struct ncclProxyOp proxyOp; uint32_t chunkSize, directFlags=0; NCCLCHECK(calcCollChunking(comm, task, nChannels, globalBytesPerElement*task->count, &chunkSize, &directFlags, &proxyOp)); devWork->channelLo = 0; devWork->channelHi = nChannels-1; devWork->collnet.count = task->count; devWork->collnet.chunkCount = chunkSize/ncclTypeSize(task->datatype); devWork->direct = directFlags; uint64_t proxyOpId = uint64_t(plan->collOpCount++)<<1 | 0; for (int c=devWork->channelLo; c <= (int)devWork->channelHi; c++) { proxyOp.channelId = c; proxyOp.opCount = proxyOpId; proxyOp.task.coll = task; proxyOp.rank = comm->rank; proxyOp.eActivationMask = task->eActivationMask; proxyOp.incWorkCounter = true; ncclAddWorkBatchToPlan(comm, plan, c, workNode->workType, task->devFuncId, plan->workBytes); // Set pattern to profiler to add a proxy profiler for kernel events NCCLCHECK(ncclAddProxyOpIfNeeded(comm, plan, &proxyOp)); NCCLCHECK(addProfilerProxyOpIfNeeded(comm, plan, &proxyOp)); } } else { // not task->isCollnet int trafficPerByte = ncclFuncTrafficPerByte(task->func, comm->nRanks); if (task->protocol == NCCL_PROTO_LL) trafficPerByte *= 4; size_t cellSize = divUp(divUp(MinTrafficPerChannel, (size_t)trafficPerByte), 16) * 16; int elementsPerCell = cellSize/elementSize; size_t cells = divUp(task->count*elementSize, cellSize); size_t trafficPerElement = elementSize*trafficPerByte; size_t trafficPerCell = cellSize*trafficPerByte; size_t cellsPerChannel = std::min(cells, divUp(trafficPerChannel, trafficPerCell)); size_t cellsLo; if (channelId+1 == nMaxChannels[kind]) { // On last channel everything goes to "lo" cellsLo = cells; } else { cellsLo = std::min(cells, divUp((trafficPerChannel-currentTraffic),trafficPerCell)); } int nMidChannels = (cells-cellsLo)/cellsPerChannel; size_t cellsHi = (cells-cellsLo)%cellsPerChannel; int nChannels = (cellsLo!=0 ? 1 : 0) + nMidChannels + (cellsHi!=0 ? 1 : 0); if (nMaxChannels[kind] < channelId + nChannels) { // Overflowed available channels nMidChannels = nMaxChannels[kind] - channelId - 2; cellsPerChannel = (cells-cellsLo)/(nMidChannels+1); cellsHi = cellsPerChannel + (cells-cellsLo)%(nMidChannels+1); } if (cellsHi == 0 && nMidChannels != 0) { cellsHi = cellsPerChannel; nMidChannels -= 1; } if (cellsLo == 0) { // Least channel skipped. Make the next channel the new least. channelId += 1; if (nMidChannels == 0) { cellsLo = cellsHi; cellsHi = 0; } else { cellsLo = cellsPerChannel; nMidChannels -= 1; } } size_t countMid = nMidChannels!=0 ? cellsPerChannel*elementsPerCell : 0; size_t countLo = cellsLo*elementsPerCell; size_t countHi = cellsHi*elementsPerCell; (countHi != 0 ? countHi : countLo) -= cells*elementsPerCell - task->count; nChannels = (countLo!=0 ? 1 : 0) + nMidChannels + (cellsHi!=0 ? 1 : 0); // Update number of channels propagated to the profiler task->nChannels = (uint8_t)nChannels; // Ensure room for worst case of one new batch per channel if (!ncclTestBudget(budget, plan->nWorkBatches + nChannels, plan->workBytes + workNode->size)) { return ncclSuccess; } devWork->channelLo = channelId; devWork->channelHi = channelId + nChannels-1; devWork->cbd.countLo = countLo; devWork->cbd.countMid = countMid; devWork->cbd.countHi = countHi; // calcCollChunking() uses global bytes instead of traffic which differs // in that allreduce isn't multiplied by 2. size_t globalBytesPerElement = elementSize*ncclFuncMaxSendRecvCount(task->func, comm->nRanks, 1); struct ncclProxyOp proxyOpLo, proxyOpMid, proxyOpHi; uint32_t chunkSize, directFlags=0; size_t grainSize = ncclProtoGrainSize(task->protocol); if (countLo != 0) { NCCLCHECK(calcCollChunking(comm, task, /*nChannels=*/1, globalBytesPerElement*countLo, &chunkSize, &directFlags, &proxyOpLo)); devWork->cbd.chunkGrainsLo = chunkSize/grainSize; } if (countHi != 0) { NCCLCHECK(calcCollChunking(comm, task, /*nChannels=*/1, globalBytesPerElement*countHi, &chunkSize, &directFlags, &proxyOpHi)); devWork->cbd.chunkGrainsHi = chunkSize/grainSize; } if (nMidChannels != 0) { NCCLCHECK(calcCollChunking(comm, task, /*nChannels=*/1, globalBytesPerElement*countMid, &chunkSize, &directFlags, &proxyOpMid)); devWork->cbd.chunkGrainsMid = chunkSize/grainSize; } devWork->direct = directFlags; // Update the current channel and vacant traffic budget. if (countHi != 0) { channelId += nChannels-1; currentTraffic = cellsHi*elementsPerCell*trafficPerElement; } else if (nMidChannels != 0) { channelId += nChannels; currentTraffic = 0; } else { currentTraffic += cellsLo*elementsPerCell*trafficPerElement; } if (currentTraffic >= trafficPerChannel && channelId+1 != nMaxChannels[kind]) { channelId += 1; currentTraffic = 0; } uint64_t proxyOpId = uint64_t(plan->collOpCount++)<<1 | 0; for (int c=devWork->channelLo; c <= (int)devWork->channelHi; c++) { struct ncclProxyOp* proxyOp; if (c == (int)devWork->channelLo) { proxyOp = &proxyOpLo; proxyOp->loopOffset = 0; proxyOp->channelSize = countLo * elementSize; } else if (c == (int)devWork->channelHi) { proxyOp = &proxyOpHi; proxyOp->loopOffset = (countLo + nMidChannels * countMid) * elementSize; proxyOp->channelSize = countHi * elementSize; } else { proxyOp = &proxyOpMid; proxyOp->loopOffset = (countLo + (c - devWork->channelLo - 1) * countMid) * elementSize; proxyOp->channelSize = countMid * elementSize; } proxyOp->channelId = c; proxyOp->opCount = proxyOpId; proxyOp->task.coll = task; proxyOp->rank = comm->rank; proxyOp->ringAlgo = NULL; if (proxyOp->reg && task->algorithm == NCCL_ALGO_RING && (task->recvNetHandles[c] || task->sendNetHandles[c])) { if (task->func == ncclFuncAllGather) { proxyOp->ringAlgo = new RingAGAlgorithm(task->sendbuff, task->recvbuff, comm->nRanks, comm->channels[c].ring.userRanks, proxyOp->chunkSteps, proxyOp->sliceSteps, proxyOp->chunkSize, proxyOp->sliceSize, proxyOp->loopOffset, proxyOp->channelSize, elementSize, task->count * elementSize, task->sendNetHandles[c], task->recvNetHandles[c], task->srecvNetHandles[c]); } else if (task->func == ncclFuncAllReduce) { proxyOp->ringAlgo = new RingARAlgorithm(task->sendbuff, task->recvbuff, comm->nRanks, comm->channels[c].ring.index, proxyOp->chunkSteps, proxyOp->sliceSteps, proxyOp->chunkSize, proxyOp->sliceSize, proxyOp->loopOffset, proxyOp->channelSize, elementSize, task->sendNetHandles[c], task->recvNetHandles[c], task->srecvNetHandles[c]); } else if (task->func == ncclFuncBroadcast) { proxyOp->ringAlgo = new RingBCAlgorithm(task->sendbuff, task->recvbuff, comm->rank, task->root, comm->nRanks, comm->channels[c].ring.userRanks, proxyOp->chunkSteps, proxyOp->sliceSteps, proxyOp->chunkSize, proxyOp->sliceSize, proxyOp->loopOffset, proxyOp->channelSize, task->sendNetHandles[c], task->recvNetHandles[c], task->srecvNetHandles[c]); } proxyOp->ringAlgo->incRefCount(); } proxyOp->eActivationMask = task->eActivationMask; proxyOp->incWorkCounter = true; proxyOp->nChannels = nChannels; ncclAddWorkBatchToPlan(comm, plan, c, workNode->workType, task->devFuncId, plan->workBytes); // Coverity reports "proxyOp->connection" as being possibly uninitialized. It's hard to // determine if that's actually true but it's also not clear if that would be an issue. // coverity[uninit_use_in_call:FALSE] NCCLCHECK(ncclAddProxyOpIfNeeded(comm, plan, proxyOp)); NCCLCHECK(addProfilerProxyOpIfNeeded(comm, plan, proxyOp)); } } plan->channelMask |= (2ull<channelHi) - (1ull<channelLo); plan->threadPerBlock = std::max(plan->threadPerBlock, task->nWarps*WARP_SIZE); if (!plan->kernelSpecialized) { plan->kernelFn = ncclDevKernelForFunc[task->devFuncId]; plan->kernelSpecialized = ncclDevKernelForFuncIsSpecialized[task->devFuncId]; } // Profiler plan->groupApiEventHandle = task->groupApiEventHandle; if (comm->rank == 0) { INFO(NCCL_TUNING, "%s: %ld Bytes -> Algo %s proto %s channel{Lo..Hi}={%d..%d}", ncclFuncToString(task->func), task->count * ncclTypeSize(task->datatype), ncclAlgoToString(task->algorithm), ncclProtoToString(task->protocol), devWork->channelLo, devWork->channelHi); if (task->isCollnet) { TRACE(NCCL_COLL, "Collective %s(%s, %s, %s, %s) count=%ld devFuncId=%d channel{Lo..Hi}={%d..%d} count=%ld chunkCount=%d", ncclFuncToString(task->func), ncclDevRedOpToString(task->opDev.op), ncclDatatypeToString(task->datatype), ncclAlgoToString(task->algorithm), ncclProtoToString(task->protocol), (long)task->count, task->devFuncId, devWork->channelLo, devWork->channelHi, (long)devWork->collnet.count, devWork->collnet.chunkCount); } else { TRACE(NCCL_COLL, "Collective %s(%s, %s, %s, %s) count=%ld devFuncId=%d channel{Lo..Hi}={%d..%d} count{Lo,Mid,Hi}={%ld,%ld,%ld} chunkBytes{Lo,Mid,Hi}={%d,%d,%d}", ncclFuncToString(task->func), ncclDevRedOpToString(task->opDev.op), ncclDatatypeToString(task->datatype), ncclAlgoToString(task->algorithm), ncclProtoToString(task->protocol), (long)task->count, task->devFuncId, devWork->channelLo, devWork->channelHi, (long)devWork->cbd.countLo, (long)devWork->cbd.countMid, (long)devWork->cbd.countHi, int(devWork->cbd.chunkGrainsLo*ncclProtoGrainSize(task->protocol)), int(devWork->cbd.chunkGrainsMid*ncclProtoGrainSize(task->protocol)), int(devWork->cbd.chunkGrainsHi*ncclProtoGrainSize(task->protocol))); } } for (int i=0; i < task->nCleanupQueueElts; i++) { ncclIntruQueueEnqueue(&plan->cleanupQueue, ncclIntruQueueDequeue(&planner->collCleanupQueue)); } ncclIntruQueueDequeue(&planner->collTaskQueue); ncclIntruQueueDequeue(&planner->collWorkQueue); nPlanColls -= 1; planner->nTasksColl -= 1; ncclIntruQueueEnqueue(&plan->collTaskQueue, task); ncclIntruQueueEnqueue(&plan->workQueue, workNode); plan->workBytes += workNode->size; } return ncclSuccess; } NCCL_PARAM(P2pLLThreshold, "P2P_LL_THRESHOLD", 16384); NCCL_PARAM(ChunkSize, "CHUNK_SIZE", 0); // Put p2p op in plan assuming there is sizeof(ncclDevWorkBatch) in batch budget // and sizeof(ncclDevWorkP2p) in work budget. "sendRank" and "recvRank" must // match the corresponding values for this round of the p2p schedule (no -1's). // No-op's are encoded with a -1 size. static ncclResult_t addP2pToPlan( struct ncclComm* comm, struct ncclKernelPlan* plan, int nChannelsMin, int nChannelsMax, int p2pEpoch, int p2pRound, int sendRank, void* sendAddr, ssize_t sendBytes, int recvRank, void* recvAddr, ssize_t recvBytes, const int planTotalTasks[], struct ncclTaskP2p** p2pTasks ) { ncclResult_t ret = ncclSuccess; constexpr int connIndex = 1; bool selfSend = (sendRank == comm->rank); // recv: dir=0, send: dir=1 void* addrs[2] = {recvAddr, sendAddr}; ssize_t bytes[2] = {recvBytes, sendBytes}; bool protoLL[2] = {!selfSend, !selfSend}; bool network[2] = {false, false}; bool proxySameProcess[2] = {true, true}; void** handles[2] = {NULL, NULL}; uint8_t base = ncclP2pChannelBaseForRound(comm, p2pRound); struct ncclProxyOp proxyOps[2] = {}; int nProxyOps = selfSend ? 0 : 2; if (!selfSend) { for (int part=0; part < nChannelsMax; part++) { int channelId = ncclP2pChannelForPart(comm->p2pnChannels, base, part); struct ncclChannelPeer** channelPeers = comm->channels[channelId].peers; for (int dir=0; dir <= 1; dir++) { int peerRank = dir ? sendRank : recvRank; struct ncclConnector* conn = dir ? &channelPeers[peerRank]->send[connIndex] : &channelPeers[peerRank]->recv[connIndex]; protoLL[dir] &= conn->conn.buffs[NCCL_PROTO_LL] != nullptr; network[dir] |= conn->transportComm == (dir ? &netTransport.send : &netTransport.recv); proxySameProcess[dir] &= conn->proxyConn.sameProcess; } } } ssize_t paramChunkSize = ncclParamChunkSize(); // Arrays indexed by dir where recv=0, send=1: int nChannels[2]; int protocol[2]; int stepSize[2]; int chunkSize[2]; int chunkDataSize[2]; int chunkDataSize_u32fp8[2]; bool netRegistered[2] = {false, false}; bool ipcRegistered[2] = {false, false}; for (int dir=0; dir < 2; dir++) { // 0=recv, 1=send // Assume SIMPLE protocol to start with to determine number of channels stepSize[dir] = comm->p2pChunkSize; if (bytes[dir] == -1) nChannels[dir] = 0; else if (bytes[dir] == 0) nChannels[dir] = 1; else { ssize_t minPartSize = comm->nNodes > 1 ? stepSize[dir]/2 : stepSize[dir]/8; ssize_t maxPartSize = comm->nNodes > 1 ? stepSize[dir] : stepSize[dir]*32; nChannels[dir] = std::min(nChannelsMin, divUp(bytes[dir], minPartSize)); size_t partSize = std::max(minPartSize, divUp(bytes[dir], nChannels[dir])); while (partSize > maxPartSize && nChannels[dir] <= nChannelsMax/2) { nChannels[dir] *= 2; partSize = divUp(bytes[dir], nChannels[dir]); } } // Update number of channels propagated to the profiler if (p2pTasks[dir]) p2pTasks[dir]->nChannels = nChannels[dir]; // Select protocol (LL vs SIMPLE) used based on payload per channel if (bytes[dir] != -1) protoLL[dir] &= bytes[dir] <= nChannels[dir] * ncclParamP2pLLThreshold(); protocol[dir] = protoLL[dir] ? NCCL_PROTO_LL : NCCL_PROTO_SIMPLE; stepSize[dir] = comm->buffSizes[protocol[dir]]/NCCL_STEPS; if (protocol[dir] == NCCL_PROTO_SIMPLE) stepSize[dir] = comm->p2pChunkSize; chunkSize[dir] = stepSize[dir]; if (paramChunkSize != 0) { chunkSize[dir] = paramChunkSize; } else if (network[dir]) { // Tune chunk size for the network if (protocol[dir] == NCCL_PROTO_SIMPLE && bytes[dir] < stepSize[dir]) chunkSize[dir] /= 4; else if (bytes[dir] < 8*stepSize[dir]) chunkSize[dir] /= 2; } chunkDataSize[dir] = chunkSize[dir]; if (protocol[dir] == NCCL_PROTO_LL) chunkDataSize[dir] /= 2; chunkDataSize_u32fp8[dir] = u32fp8Encode(chunkDataSize[dir]); chunkDataSize[dir] = u32fp8Decode(chunkDataSize_u32fp8[dir]); chunkSize[dir] = chunkDataSize[dir]; if (protocol[dir] == NCCL_PROTO_LL) chunkSize[dir] *= 2; if (p2pTasks[dir] && p2pTasks[dir]->allowUB) { if (network[dir]) { bool pxnUsed = !ncclPxnDisable(comm) && comm->isAllNvlink && comm->maxLocalRanks > 1; if (bytes[dir] > 0 && proxySameProcess[dir] && protocol[dir] == NCCL_PROTO_SIMPLE && (!pxnUsed)) { int regFlag = 0; NCCLCHECKGOTO(ncclCalloc(&handles[dir], nChannelsMax), ret, cleanup); for (int part = 0; part < nChannelsMax; part++) { int channelId = ncclP2pChannelForPart(comm->p2pnChannels, base, part); struct ncclChannelPeer** channelPeers = comm->channels[channelId].peers; int peerRank = dir ? sendRank : recvRank; struct ncclConnector* conn = dir ? &channelPeers[peerRank]->send[connIndex] : &channelPeers[peerRank]->recv[connIndex]; if (conn->conn.flags & NCCL_DIRECT_NIC) ncclRegisterP2pNetBuffer(comm, addrs[dir], bytes[dir], conn, ®Flag, &handles[dir][part], &plan->cleanupQueue); if (!regFlag) break; } netRegistered[dir] = regFlag ? true : false; } } else if (bytes[dir] > 0 && addrs[dir] && protocol[dir] == NCCL_PROTO_SIMPLE && !selfSend) { int peerRank = dir ? sendRank : recvRank; int regFlag = 0; int channelId = ncclP2pChannelForPart(comm->p2pnChannels, base, 0); struct ncclChannelPeer** channelPeers = comm->channels[channelId].peers; struct ncclConnector* conn = dir ? &channelPeers[peerRank]->send[connIndex] : &channelPeers[peerRank]->recv[connIndex]; void* regAddr = NULL; if (conn->conn.flags & (NCCL_P2P_WRITE | NCCL_P2P_READ)) { // We require users registering buffers on both sides NCCLCHECKGOTO(ncclRegisterP2pIpcBuffer(comm, addrs[dir], bytes[dir], peerRank, ®Flag, ®Addr, &plan->cleanupQueue), ret, cleanup); if (regFlag) { if (dir == 0 && (conn->conn.flags & NCCL_P2P_WRITE)) recvAddr = regAddr; else if (dir == 1 && (conn->conn.flags & NCCL_P2P_READ)) sendAddr = regAddr; } } ipcRegistered[dir] = regFlag ? true : false; } } } struct ncclWorkList* workNode; workNode = ncclMemoryStackAllocInlineArray(&comm->memScoped, 1); workNode->workType = ncclDevWorkTypeP2p; workNode->size = sizeof(struct ncclDevWorkP2p); ncclIntruQueueEnqueue(&plan->workQueue, workNode); uint32_t workOffset; workOffset = plan->workBytes; plan->workBytes += sizeof(struct ncclDevWorkP2p); struct ncclDevWorkP2p* work; work = (struct ncclDevWorkP2p*)(workNode+1); work->nP2pChannels = comm->p2pnChannels; work->channelBase = base; work->nSendChannels = nChannels[1]; work->sendProtoLL = protoLL[1]; work->sendNetReg = netRegistered[1]; work->sendIpcReg = ipcRegistered[1]; work->sendChunkSize_u32fp8 = chunkDataSize_u32fp8[1]; work->sendRank = sendRank; work->sendAddr = sendAddr; work->sendBytes = sendBytes==-1 ? 0 : sendBytes; work->nRecvChannels = nChannels[0]; work->recvProtoLL = protoLL[0]; work->recvNetReg = netRegistered[0]; work->recvIpcReg = ipcRegistered[0]; work->recvChunkSize_u32fp8 = chunkDataSize_u32fp8[0]; work->recvRank = recvRank; work->recvAddr = recvAddr; work->recvBytes = recvBytes==-1 ? 0 : recvBytes; work->profilerEnabled = ncclProfilerPluginLoaded() && ((p2pTasks[0] ? p2pTasks[0] : p2pTasks[1])->eActivationMask & ncclProfileKernelCh); for (int dir=0; dir < nProxyOps; dir++) { struct ncclProxyOp* op = &proxyOps[dir]; op->root = dir ? sendRank : recvRank; op->sliceSteps = 1; op->chunkSteps = 1; op->dtype = ncclInt8; op->redOp = ncclSum; op->protocol = protocol[dir]; op->pattern = dir ? ncclPatternSend : ncclPatternRecv; op->chunkSize = chunkSize[dir]; op->reg = netRegistered[dir]; op->coll = p2pTasks[dir] ? p2pTasks[dir]->func : 0; op->collAPI = p2pTasks[dir] ? p2pTasks[dir]->collAPI : 0; op->task.p2p = p2pTasks[dir]; op->rank = comm->rank; op->eActivationMask = p2pTasks[dir] ? p2pTasks[dir]->eActivationMask : 0; // The following are modified per channel part in addWorkToChannels(): // op->buffer, op->nbytes, op->nsteps = ...; } nChannelsMax = std::max(nChannels[0], nChannels[1]); // Determine how many peers this plan will target concurrently. Make a // simplifying assumption that each task targets a different peer. // Each task is striped across 'nChannelsMax' of 'p2pnChannels' channels. // Each channel runs up to NCCL_MAX_DEV_WORK_P2P_PER_BATCH tasks concurrently. int maxConcurrent; int concurrentTasks[2]; maxConcurrent = comm->p2pnChannels / nChannelsMax * NCCL_MAX_DEV_WORK_P2P_PER_BATCH; concurrentTasks[0] = std::min(planTotalTasks[0], maxConcurrent); concurrentTasks[1] = std::min(planTotalTasks[1], maxConcurrent); for (int part=0; part < nChannelsMax; part++) { int incWorkCounter = -1; int channelId = ncclP2pChannelForPart(comm->p2pnChannels, base, part); plan->channelMask |= uint64_t(1)<nSendChannels : work->nRecvChannels; void* addr = dir ? work->sendAddr : work->recvAddr; size_t bytes = dir ? work->sendBytes : work->recvBytes; proxyOps[dir].recvbuff = nullptr; if (nParts <= part) { proxyOps[dir].nsteps = 0; } else if (bytes == 0) { proxyOps[dir].nsteps = 1; proxyOps[dir].nbytes = 0; } else { size_t chunkDataSize = u32fp8Decode(dir ? work->sendChunkSize_u32fp8 : work->recvChunkSize_u32fp8); size_t partBeg, partEnd; ncclP2pPartBounds(nParts, part, bytes, &partBeg, &partEnd); if (proxyOps[dir].reg) { (dir ? proxyOps[dir].sendbuff : proxyOps[dir].recvbuff) = (uint8_t*)addr + partBeg; (dir ? proxyOps[dir].sendMhandle : proxyOps[dir].recvMhandle) = handles[dir][part]; proxyOps[dir].nbytes = partEnd - partBeg; proxyOps[dir].nsteps = DIVUP(proxyOps[dir].nbytes, NCCL_MAX_NET_SIZE); } else { proxyOps[dir].nsteps = divUp(partEnd-partBeg, chunkDataSize); proxyOps[dir].nbytes = std::min(partEnd-partBeg, chunkDataSize); } if (proxyOps[dir].protocol == NCCL_PROTO_LL) { proxyOps[dir].nbytes *= 2; proxyOps[dir].nbytes = roundUp(proxyOps[dir].nbytes, sizeof(union ncclLLFifoLine)); } } // Increment work counter for pair rather than individual p2p if (proxyOps[dir].nsteps && incWorkCounter < 0) { proxyOps[dir].incWorkCounter = true; incWorkCounter = dir; } if (proxyOps[dir].nsteps != 0) { // Calculate the opCount after adding batch since then the batch count will // equal one plus the batch index this p2p settled in. proxyOps[dir].channelId = channelId; proxyOps[dir].opCount = uint64_t(comm->planner.wipPlan.channels[channelId].nWorkBatchesP2p)<<1 | 1; proxyOps[dir].nChannels = nChannels[dir]; proxyOps[dir].nPeers = concurrentTasks[dir]; NCCLCHECKGOTO(ncclAddProxyOpIfNeeded(comm, plan, &proxyOps[dir]), ret, cleanup); NCCLCHECKGOTO(addProfilerProxyOpIfNeeded(comm, plan, &proxyOps[dir]), ret, cleanup); } } } cleanup: free(handles[0]); free(handles[1]); return ret; } static int calcP2pChannelCount(size_t totalSize, int minChannels, int maxChannels, size_t minSize, size_t maxSize) { size_t size = std::max(minSize, divUp(totalSize, minChannels)); int nChannels = minChannels; while (size > maxSize && nChannels <= maxChannels/2) { nChannels *= 2; size = divUp(totalSize, nChannels); } return nChannels; } static ncclResult_t scheduleP2pTasksToPlan(struct ncclComm* comm, int* p2pEpoch, int* p2pRound, struct ncclKernelPlan* plan, struct ncclKernelPlanBudget* budget) { int nRanks = comm->nRanks; struct ncclKernelPlanner::Peer* peers = comm->planner.peers; plan->threadPerBlock = std::max(plan->threadPerBlock, NCCL_MAX_NTHREADS); if (!plan->kernelSpecialized) { plan->kernelFn = ncclDevKernelForFunc[ncclDevFuncId_P2p()]; plan->kernelSpecialized = ncclDevKernelForFuncIsSpecialized[ncclDevFuncId_P2p()]; } // Compute how much to split operations // Try to use all channels int nChannelsMax = comm->p2pnChannelsPerPeer; int nChannelsMin = nChannelsMax; // Try to use all channels, but one channel per operation. while (nChannelsMin*nRanks > comm->p2pnChannels && nChannelsMin > 1) nChannelsMin /= 2; // Save the total count of send/recv tasks in the plan int planTotalTasks[2] = {comm->planner.nTasksP2pRecv, comm->planner.nTasksP2pSend}; while (comm->planner.nTasksP2p != 0) { for (; *p2pRound < nRanks; (*p2pRound)++) { int sendRank = comm->p2pSchedule[*p2pRound].sendRank; int recvRank = comm->p2pSchedule[*p2pRound].recvRank; struct ncclTaskP2p* send = ncclIntruQueueHead(&peers[sendRank].sendQueue); struct ncclTaskP2p* recv = ncclIntruQueueHead(&peers[recvRank].recvQueue); if (send == nullptr && recv == nullptr) continue; if (sendRank == comm->rank) { if (send != nullptr && recv == nullptr) { WARN("Trying to send to self without a matching recv"); return ncclInvalidUsage; } if (send == nullptr && recv != nullptr) { WARN("Trying to recv to self without a matching send"); return ncclInvalidUsage; } } ssize_t sendBytes = send ? send->bytes : -1; ssize_t recvBytes = recv ? recv->bytes : -1; void* sendBuff = send ? send->buff : nullptr; void* recvBuff = recv ? recv->buff : nullptr; if (sendRank == comm->rank && send->buff == recv->buff) { // Skip send to self in-place (we don't need to support this). ncclIntruQueueDequeue(&peers[sendRank].sendQueue); ncclIntruQueueDequeue(&peers[recvRank].recvQueue); ncclMemoryPoolFree(&comm->memPool_ncclTaskP2p, send); ncclMemoryPoolFree(&comm->memPool_ncclTaskP2p, recv); comm->planner.nTasksP2p -= 2; comm->planner.nTasksP2pSend -= 1; comm->planner.nTasksP2pRecv -= 1; } else { // Ensure room for worst case of one new batch per channel. if (!ncclTestBudget(budget, plan->nWorkBatches+nChannelsMax, plan->workBytes + sizeof(struct ncclDevWorkP2p))) { return ncclSuccess; } struct ncclTaskP2p* p2pTasks[2] = { recv, send }; NCCLCHECK(addP2pToPlan(comm, plan, nChannelsMin, nChannelsMax, *p2pEpoch, *p2pRound, sendRank, sendBuff, sendBytes, recvRank, recvBuff, recvBytes, planTotalTasks, p2pTasks)); if (send != nullptr) { ncclIntruQueueDequeue(&peers[sendRank].sendQueue); // Profiler - We can overwrite groupAPI event handles here since all operations here belong to the same group plan->groupApiEventHandle = send->groupApiEventHandle; ncclIntruQueueEnqueue(&plan->p2pTaskQueue, send); comm->planner.nTasksP2p -= 1; comm->planner.nTasksP2pSend -= 1; } if (recv != nullptr) { ncclIntruQueueDequeue(&peers[recvRank].recvQueue); // Profiler - We can overwrite groupAPI event handles here since all operations here belong to the same group plan->groupApiEventHandle = recv->groupApiEventHandle; ncclIntruQueueEnqueue(&plan->p2pTaskQueue, recv); comm->planner.nTasksP2p -= 1; comm->planner.nTasksP2pRecv -= 1; } } } *p2pRound=0; (*p2pEpoch)++; } return ncclSuccess; } // Spin until its safe to increase comm->workFifoProduced to desiredProduced. static ncclResult_t waitWorkFifoAvailable(struct ncclComm* comm, uint32_t desiredProduced) { bool hasRoom = (desiredProduced - comm->workFifoConsumed) <= comm->workFifoBytes; if (!hasRoom) { while (true) { // Check abort flag to break deadlock when abort is signaled if (__atomic_load_n(comm->abortFlag, __ATOMIC_ACQUIRE)) { return ncclInternalError; } NCCLCHECK(ncclCommPollEventCallbacks(comm, /*waitSome=*/true)); hasRoom = (desiredProduced - comm->workFifoConsumed) <= comm->workFifoBytes; if (hasRoom) break; std::this_thread::yield(); } } return ncclSuccess; } namespace { struct uploadWork_cleanup_t { struct ncclCommEventCallback base; void *hostBuf; }; ncclResult_t uploadWork_cleanup_fn( struct ncclComm* comm, struct ncclCommEventCallback* cb ) { struct uploadWork_cleanup_t* me = (struct uploadWork_cleanup_t*)cb; ncclOsAlignedFree(me->hostBuf); CUDACHECK(cudaEventDestroy(me->base.event)); free(me); return ncclSuccess; } } static ncclResult_t uploadWork(struct ncclComm* comm, struct ncclKernelPlan* plan) { if (plan->isSymColl || plan->isCeColl || plan->isRma) return ncclSuccess; size_t workBytes = plan->workBytes; size_t batchBytes = plan->nWorkBatches*sizeof(struct ncclDevWorkBatch); void* fifoBufHost; uint32_t fifoCursor, fifoMask; switch (plan->workStorageType) { case ncclDevWorkStorageTypeArgs: plan->kernelArgs->workBuf = nullptr; fifoBufHost = (void*)plan->kernelArgs; fifoCursor = sizeof(ncclDevKernelArgs) + batchBytes; fifoMask = ~0u; break; case ncclDevWorkStorageTypeFifo: fifoBufHost = comm->workFifoBuf; fifoCursor = comm->workFifoProduced; fifoMask = comm->workFifoBytes-1; NCCLCHECK(waitWorkFifoAvailable(comm, fifoCursor + workBytes)); plan->kernelArgs->workBuf = comm->workFifoBufDev; break; case ncclDevWorkStorageTypePersistent: // We rely on 16-byte alignment #if __cplusplus >= 201103L fifoBufHost = ncclOsAlignedAlloc(16, ROUNDUP(workBytes, 16)); #else static_assert(16 <= alignof(max_align_t), "We rely on 16-byte alignment."); fifoBufHost = malloc(workBytes); #endif fifoCursor = 0; fifoMask = ~0u; break; default: return ncclInternalError; } plan->kernelArgs->workMask = fifoMask; // Batches were placed after kernelArgs by finishPlan(). Only thing left to // do is translate the work offset from zero based (in plan) to: // ncclDevWorkStorageTypeArgs: offset from beginning of kernel args // ncclDevWorkStorageTypeFifo: offset from base of fifo // ncclDevWorkStorageTypePersistent: no translation since our dedicated buffer will also begin at zero. struct ncclDevWorkBatch* batchZero = (struct ncclDevWorkBatch*)(plan->kernelArgs+1); for (int b=0; b < plan->nWorkBatches; b++) { batchZero[b].offsetBase += fifoCursor; } // Write the channel-shared work structs. struct ncclWorkList* workNode = ncclIntruQueueHead(&plan->workQueue); while (workNode != nullptr) { char* dst = (char*)fifoBufHost; char* src = (char*)(workNode+1); for (int n = workNode->size; n != 0; n -= 16) { memcpy( COMPILER_ASSUME_ALIGNED(dst + (fifoCursor & fifoMask), 16), COMPILER_ASSUME_ALIGNED(src, 16), 16 ); fifoCursor += 16; src += 16; } workNode = workNode->next; } switch (plan->workStorageType) { case ncclDevWorkStorageTypeFifo: comm->workFifoProduced = fifoCursor; if (comm->workFifoBufGdrHandle != nullptr) wc_store_fence(); break; case ncclDevWorkStorageTypePersistent: { ncclResult_t result = ncclSuccess; struct uploadWork_cleanup_t* cleanup = nullptr; cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; void* fifoBufDev = nullptr; cudaStream_t deviceStream; CUDACHECKGOTO(cudaThreadExchangeStreamCaptureMode(&mode), result, fail); // Acquire deviceStream. Since the user's graph will be launched later and it also // acquires the deviceStream, it will observe this upload. NCCLCHECKGOTO(ncclStrongStreamAcquire(ncclCudaGraphNone(comm->config.graphUsageMode), &comm->sharedRes->deviceStream, /*concurrent=*/false, &deviceStream), result, fail); CUDACHECKGOTO(cudaMallocAsync(&fifoBufDev, workBytes, comm->memPool, deviceStream), result, fail); plan->workBufPersistent = fifoBufDev; plan->kernelArgs->workBuf = fifoBufDev; // coverity[uninit_use_in_call:FALSE] => fifoBufHost is never NULL CUDACHECKGOTO(cudaMemcpyAsync(fifoBufDev, fifoBufHost, workBytes, cudaMemcpyDefault, deviceStream), result, fail); cudaEvent_t memcpyDone; CUDACHECKGOTO(cudaEventCreateWithFlags(&memcpyDone, cudaEventDisableTiming), result, fail); CUDACHECKGOTO(cudaEventRecord(memcpyDone, deviceStream), result, fail); NCCLCHECKGOTO(ncclCalloc(&cleanup, 1), result, fail); cleanup->base.fn = uploadWork_cleanup_fn; cleanup->base.event = memcpyDone; cleanup->hostBuf = fifoBufHost; ncclIntruQueueEnqueue(&comm->eventCallbackQueue, (struct ncclCommEventCallback *)cleanup); NCCLCHECKGOTO(ncclStrongStreamRelease(ncclCudaGraphNone(comm->config.graphUsageMode), &comm->sharedRes->deviceStream, /*concurrent=*/false), result, fail); NCCLCHECKGOTO(ncclCommPollEventCallbacks(comm, /*waitSome=*/false), result, fail); finish_scope: if (mode != cudaStreamCaptureModeRelaxed) (void)cudaThreadExchangeStreamCaptureMode(&mode); return result; fail: if (!cleanup) ncclOsAlignedFree(fifoBufHost); goto finish_scope; } break; default: break; } return ncclSuccess; } static int geteActivationMask(struct ncclProxyOp * op) { if (ncclFuncSendRecv <= op->coll && op->coll <= ncclFuncRecv) { return op->task.p2p->eActivationMask; } if (op->coll == ncclFuncAllGatherV) { return 0; } return op->task.coll->eActivationMask; } static void *gettaskEventHandle(struct ncclProxyOp * op) { if (ncclFuncSendRecv <= op->coll && op->coll <= ncclFuncRecv) { return op->task.p2p->eventHandle; } if (op->coll == ncclFuncAllGatherV) { return nullptr; } return op->task.coll->eventHandle; } static ncclResult_t uploadProxyOps(struct ncclComm* comm, struct ncclKernelPlan* plan) { uint64_t collOpCount = comm->sharedRes->collOpCount; uint64_t p2pOpBump[MAXCHANNELS] = {/*0...*/}; // Advance comm's collOpCount by number of colls in this plan. int hasp2p = 0; comm->sharedRes->collOpCount += plan->collOpCount; comm->collOpCount += plan->collOpCount; struct ncclProxyOp* op = ncclIntruQueueHead(&plan->proxyOpQueue); while (op != nullptr) { op->profilerContext = comm->profilerContext; op->eActivationMask = geteActivationMask(op); op->taskEventHandle = gettaskEventHandle(op); ncclProfilerAddPidToProxyOp(op); uint64_t oldId = op->opCount; // Ignoring the bottom tag bit, opCount's are zero-based within plan so // translate them to the tip of the comm's history. if (oldId & 1) { // p2p // opCount is monotonic increasing within a plan's channel so just // remember last value to compute max. p2pOpBump[op->channelId] = (oldId>>1) + 1; // +1 to ensure next plan doesn't collide op->opCount = (comm->sharedRes->p2pOpCount[op->channelId]<<1) + oldId; hasp2p = 1; } else { // coll op->opCount = (collOpCount<<1) + oldId; } NCCLCHECK(ncclProxySaveOp(comm, op, nullptr)); op->opCount = oldId; // Restore for next uploadProxyOps() op = op->enqNext; } if (hasp2p) { for (int c=0; c < MAXCHANNELS; c++) { // Advance channel's p2pOpCount by number of p2p's in this plan channel. comm->sharedRes->p2pOpCount[c] += p2pOpBump[c]; } } return ncclSuccess; } static ncclResult_t hostStreamPlanTask(struct ncclComm* comm, struct ncclKernelPlan* plan) { NCCLCHECK(ncclProfilerStartGroupEvent(plan)); NCCLCHECK(ncclProfilerStartTaskEvents(plan)); if (ncclIntruQueueHead(&plan->proxyOpQueue)) { NCCLCHECK(uploadProxyOps(comm, plan)); NCCLCHECK(ncclProxyStart(comm)); } NCCLCHECK(ncclProfilerStopTaskEvents(plan)); NCCLCHECK(ncclProfilerStopGroupEvent(plan)); if (!plan->persistent) { // Notify main thread of our reclaiming. This will reclaim plan concurrently. ncclIntruQueueMpscEnqueue(&comm->callbackQueue, &plan->reclaimer); } return ncclSuccess; } static void CUDART_CB hostStreamPlanCallback(void *plan_) { NCCL_NVTX3_FUNC_RANGE; struct ncclKernelPlan* plan = (struct ncclKernelPlan*)plan_; ncclResult_t result = hostStreamPlanTask(plan->comm, plan); if (result != ncclSuccess) { WARN("hostStreamPlanCallback() failed : %s", ncclGetErrorString(result)); } return; } static ncclResult_t reclaimPlan(struct ncclComm* comm, struct ncclCommCallback* me) { struct ncclKernelPlan* plan = (struct ncclKernelPlan*)me; // cast from first member `reclaim` if (plan->persistent) { comm->sharedRes->persistentRefs -= 1; comm->localPersistentRefs -= 1; if (plan->workStorageType == ncclDevWorkStorageTypePersistent) { cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); CUDACHECK(cudaFree(plan->workBufPersistent)); CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); } } if (plan->isSymColl) { free(plan->kernelSymArgs); } // Free coll tasks struct ncclTaskColl* ct = ncclIntruQueueHead(&plan->collTaskQueue); while (ct != nullptr) { struct ncclTaskColl* ct1 = ct->next; free(ct->sendNetHandles); free(ct->recvNetHandles); free(ct->srecvNetHandles); ncclMemoryPoolFree(&comm->memPool_ncclTaskColl, ct); ct = ct1; } // Free p2p tasks struct ncclTaskP2p* pt = ncclIntruQueueHead(&plan->p2pTaskQueue); while (pt != nullptr) { struct ncclTaskP2p* pt1 = pt->next; ncclMemoryPoolFree(&comm->memPool_ncclTaskP2p, pt); pt = pt1; } // Free broadcast tasks struct ncclTaskBcast* bt = ncclIntruQueueHead(&plan->bcastTaskQueue); while (bt != nullptr) { struct ncclTaskBcast* bt1 = bt->next; ncclMemoryPoolFree(&comm->memPool_ncclTaskBcast, bt); bt = bt1; } // Free proxy ops struct ncclProxyOp* q = ncclIntruQueueHead(&plan->proxyOpQueue); while (q != nullptr) { struct ncclProxyOp* q1 = q->enqNext; if (q->ringAlgo && q->ringAlgo->decRefCount() == 0) delete q->ringAlgo; ncclMemoryPoolFree(&comm->memPool_ncclProxyOp, q); q = q1; } // Run other free callbacks ncclResult_t result = ncclSuccess; while (!ncclIntruQueueEmpty(&plan->cleanupQueue)) { struct ncclCommCallback* cb = ncclIntruQueueDequeue(&plan->cleanupQueue); ncclResult_t res1 = cb->fn(comm, cb); // Expect to reclaim memory of cb if (res1 != ncclSuccess) result = res1; } NCCLCHECK(result); // Free plan struct ncclMemoryPoolFree(&comm->memPool_ncclKernelPlan, plan); return ncclSuccess; } static void persistentDestructor(void* plans_) { struct ncclKernelPlan* plan = (struct ncclKernelPlan*)plans_; struct ncclComm* comm = plan->comm; while (plan != nullptr) { struct ncclKernelPlan* next = plan->next; ncclIntruQueueMpscEnqueue(&comm->callbackQueue, &plan->reclaimer); plan = next; } } NCCL_PARAM(LaunchOrderImplicit, "LAUNCH_ORDER_IMPLICIT", 0); namespace { enum ncclImplicitOrder { ncclImplicitOrderNone, ncclImplicitOrderSerial, ncclImplicitOrderLaunch }; } static ncclResult_t getImplicitOrder(enum ncclImplicitOrder *mode, bool capturing, int driver=-1) { if (ncclParamLaunchOrderImplicit()) { if (driver < 0) { NCCLCHECK(ncclCudaDriverVersion(&driver)); } if (capturing && driver < 12090) { *mode = ncclImplicitOrderSerial; return ncclSuccess; } *mode = 12030 <= std::min(CUDART_VERSION, driver) ? ncclImplicitOrderLaunch : ncclImplicitOrderSerial; return ncclSuccess; } *mode = ncclImplicitOrderNone; return ncclSuccess; } ncclResult_t ncclLaunchPrepare(struct ncclComm* comm) { ncclResult_t result = ncclSuccess; struct ncclKernelPlanner* planner = &comm->planner; bool persistent = ncclCudaGraphValid(planner->capturingGraph); planner->persistent = persistent; // Operations from different plans will not be batched together. A new batch will be created for each new plan that is used to schedule the ops (see ncclAddWorkBatchToPlan). // For p2p ops, we further guarantee that ops from different epochs will not be batched together (to avoid hangs). // The p2pEpoch value is incremented in scheduleP2pTasksToPlan and its value is carried over from one plan to another (even if not strictly required) int nPlans = 0, p2pEpoch=0, p2pRound=0; if (planner->nTasksColl + planner->nTasksP2p + planner->nTasksBcast != 0 || !ncclIntruQueueEmpty(&planner->collSymTaskQueue) || !ncclIntruQueueEmpty(&planner->collCeTaskQueue) || planner->nTasksRma != 0) { do { memset(&planner->wipPlan, 0, sizeof(planner->wipPlan)); struct ncclKernelPlan* plan = ncclMemoryPoolAlloc(&comm->memPool_ncclKernelPlan, &comm->memPermanent); plan->comm = comm; plan->reclaimer.fn = reclaimPlan; plan->persistent = persistent; // finishPlan() promotes ncclDevWorkStorageType[Fifo|Persistent]->Args if the work can fit. plan->workStorageType = persistent ? ncclDevWorkStorageTypePersistent : ncclDevWorkStorageTypeFifo; if (planner->nTasksRma != 0) { NCCLCHECKGOTO(scheduleRmaTasksToPlan(comm, plan), result, failure); if (plan->isRma && plan->rmaArgs != NULL && plan->rmaArgs->nRmaTasks > 0) { ncclIntruQueueEnqueue(&planner->planQueue, plan); nPlans += 1; } } else if (!ncclIntruQueueEmpty(&planner->collCeTaskQueue)) { struct ncclTaskColl* task = ncclIntruQueueHead(&planner->collCeTaskQueue); plan->isCeColl = true; plan->ceCollArgs = ncclMemoryStackAlloc(&comm->memScoped); plan->ceCollArgs->rootRank = task->root; plan->ceCollArgs->datatype = task->datatype; plan->ceCollArgs->nElts = task->count; plan->ceCollArgs->eltSize = ncclTypeSize(task->datatype); plan->ceCollArgs->sendBuff = (uint8_t*)task->sendbuff; plan->ceCollArgs->recvBuff = (uint8_t*)task->recvbuff; plan->ceCollArgs->func = task->func; plan->ceCollArgs->sendWin = task->sendWin; plan->ceCollArgs->recvWin = task->recvWin; plan->ceCollArgs->collApiEventHandle = task->collApiEventHandle; if (comm->rank == 0) { const char* nvlsSync = comm->nvlsSupport ? "; CE synchronization with NVLS" : ""; INFO(NCCL_TUNING, "%s [Copy Engine]: %ld Bytes -> cudaMemcpy%s", ncclFuncToString(task->func), task->count * ncclTypeSize(task->datatype), nvlsSync); } ncclIntruQueueEnqueue(&planner->planQueue, plan); ncclIntruQueueDequeue(&planner->collCeTaskQueue); ncclMemoryPoolFree(&comm->memPool_ncclTaskColl, task); nPlans += 1; } else { if (!ncclIntruQueueEmpty(&planner->collSymTaskQueue)) { NCCLCHECKGOTO(ncclSymmetricTaskScheduler(comm, &planner->collSymTaskQueue, plan), result, failure); } else { struct ncclKernelPlanBudget budget; budget.inArgsBytes = comm->workArgsBytes - sizeof(struct ncclDevKernelArgs); // Non-persistent kernels fill up at most half of our fifo per kernel. budget.outArgsBytes = plan->persistent ? (1<<30) : comm->workFifoBytes/2; // Drain coll tasks first. This is essential since we partition tasks based // on the work budget and p2p work isn't collective. If we were to drain p2p // first, the place where we cut the kernel could vary by rank which would // cause the "shortest channel first" channel picker to have divergent results. if (planner->nTasksColl != 0) { NCCLCHECKGOTO(scheduleCollTasksToPlan(comm, plan, &budget), result, failure); } if (planner->nTasksColl == 0 && planner->nTasksBcast != 0) { NCCLCHECKGOTO(ncclScheduleBcastTasksToPlan(comm, plan, &budget), result, failure); } // And only drain p2p tasks once colls are depleted. if (planner->nTasksColl == 0 && planner->nTasksBcast == 0 && planner->nTasksP2p != 0) { NCCLCHECKGOTO(scheduleP2pTasksToPlan(comm, &p2pEpoch, &p2pRound, plan, &budget), result, failure); } } finishPlan(comm, plan); if (plan->workBytes != 0) { ncclIntruQueueEnqueue(&planner->planQueue, plan); nPlans += 1; } } } while (planner->nTasksColl + planner->nTasksP2p + planner->nTasksBcast != 0 || !ncclIntruQueueEmpty(&planner->collSymTaskQueue) || !ncclIntruQueueEmpty(&planner->collCeTaskQueue) || planner->nTasksRma != 0); struct ncclKernelPlan* planHead = ncclIntruQueueHead(&planner->planQueue); planner->unlaunchedPlansHead = planHead; if (nPlans == 0) return ncclSuccess; cudaStream_t launchStream = planner->streams->stream; cudaStream_t deviceStream, launchOrder; NCCLCHECKGOTO(ncclStrongStreamAcquire(planner->capturingGraph, &comm->sharedRes->deviceStream, /*concurrent=*/false, &deviceStream), result, failure); // userStream[0] waits on each userStream[i]... for (struct ncclCudaStreamList* l=planner->streams->next; l != nullptr; l = l->next) { CUDACHECKGOTO(cudaEventRecord(comm->sharedRes->scratchEvent, l->stream), result, failure); CUDACHECKGOTO(cudaStreamWaitEvent(launchStream, comm->sharedRes->scratchEvent, 0), result, failure); } // userStream[0] waits on deviceStream NCCLCHECKGOTO(ncclStreamWaitStream(launchStream, deviceStream, comm->sharedRes->scratchEvent), result, failure); bool capturing = ncclCudaGraphValid(planner->capturingGraph); enum ncclImplicitOrder implicitOrder; cudaError_t status = cudaSuccess; NCCLCHECKGOTO(getImplicitOrder(&implicitOrder, capturing), result, failure); if (implicitOrder != ncclImplicitOrderNone) { // userStream[0] waits on per-device (context) launchOrder. Concurrent strong stream access is // required if this is a graph capture, non-captured cannot be concurrent because that would violate // deterministic program order of launches. bool concurrent = capturing; NCCLCHECKGOTO(ncclStrongStreamAcquire(planner->capturingGraph, &comm->context->launchOrder, concurrent, &launchOrder), result, failure); NCCLCHECKGOTO(ncclStreamWaitStream(launchStream, launchOrder, comm->sharedRes->scratchEvent), result, failure); } if (!persistent && comm->sharedRes->persistentRefs) status = CUDACLEARERROR(cudaEventQuery(comm->sharedRes->hostStream.serialEvent)); if (persistent || ncclCudaLaunchBlocking || status == cudaErrorNotReady) { // We have to launch host tasks to push proxy args. We are careful to only // do this if necessary since host tasks impose a high performance cost in CUDA. bool acquired = false; cudaStream_t hostStream; for (struct ncclKernelPlan* plan=planHead; plan != nullptr; plan = plan->next) { if (plan->hasProxyOps) { if (!acquired) { acquired = true; NCCLCHECKGOTO(ncclStrongStreamAcquire(planner->capturingGraph, &comm->sharedRes->hostStream, /*concurrent=*/false, &hostStream), result, failure); } plan->isHostCbEnq = true; CUDACHECKGOTO(cudaLaunchHostFunc(hostStream, hostStreamPlanCallback, plan), result, failure); } } if (acquired) { // Make to-be-launched kernels dependent on just-launched host stream tasks. NCCLCHECKGOTO(ncclStreamWaitStream(launchStream, hostStream, comm->sharedRes->scratchEvent), result, failure); NCCLCHECKGOTO(ncclStrongStreamRelease(planner->capturingGraph, &comm->sharedRes->hostStream, /*concurrent=*/false), result, failure); } } if (persistent) { comm->sharedRes->persistentRefs += nPlans; comm->localPersistentRefs += nPlans; NCCLCHECKGOTO(ncclCudaGraphAddDestructor(planner->capturingGraph, persistentDestructor, (void*)planHead), result, failure); } } failure: return result; } ncclResult_t ncclLaunchKernelBefore_NoUncapturedCuda(struct ncclComm* comm, struct ncclKernelPlan* plan) { // This code is called after we've checked in to the intra-process barrier // but before launching the kernel. We are not allowed to call CUDA unless the // kernel launch is captured. NCCLCHECK(uploadWork(comm, plan)); return ncclSuccess; } #if CUDART_VERSION >= 12000 // NCCL uses the "Remote" Mem Sync domain by default NCCL_PARAM(MemSyncDomain, "MEM_SYNC_DOMAIN", cudaLaunchMemSyncDomainRemote); #endif ncclResult_t ncclLaunchKernel(struct ncclComm* comm, struct ncclKernelPlan* plan) { ncclResult_t ret = ncclSuccess; struct ncclKernelPlanner* planner = &comm->planner; int nChannels = countOneBits(plan->channelMask); void* sym = plan->kernelFn; dim3 grid = {(unsigned)nChannels, 1, 1}; dim3 block = {(unsigned)plan->threadPerBlock, 1, 1}; int smem = plan->isSymColl ? plan->kernelDynSmem : ncclShmemDynamicSize(comm->cudaArch); cudaStream_t launchStream = planner->streams->stream; NCCLCHECK(ncclProfilerStartKernelLaunchEvent(plan, launchStream)); void* extra[] = { CU_LAUNCH_PARAM_BUFFER_POINTER, plan->kernelArgs, CU_LAUNCH_PARAM_BUFFER_SIZE, &plan->kernelArgsSize, CU_LAUNCH_PARAM_END }; int driverVersion; NCCLCHECKGOTO(ncclCudaDriverVersion(&driverVersion), ret, do_return); CUfunction fn; CUDACHECKGOTO(cudaGetFuncBySymbol(&fn, sym), ret, do_return); if (CUDART_VERSION >= 11080 && driverVersion >= 11080) { #if CUDART_VERSION >= 11080 int compCap = comm->compCap; unsigned int clusterSize = (compCap >= 90) ? comm->config.cgaClusterSize : 0; CUlaunchConfig launchConfig = {0}; CUlaunchAttribute launchAttrs[6] = {}; int attrs = 0; /* Cooperative Group Array (CGA) * On sm90 and later we have an extra level of hierarchy where we * can group together several blocks within the Grid, called * Thread Block Clusters. * Clusters enable multiple thread blocks running concurrently * across multiple SMs to synchronize and collaboratively fetch * and exchange data. A cluster of blocks are guaranteed to be * concurrently scheduled onto a group of SMs. * The maximum value is 8 and it must be divisible into the grid dimensions */ if (clusterSize) { // Grid dimension must be divisible by clusterSize if (grid.x % clusterSize) clusterSize = 1; launchAttrs[attrs].id = CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION; launchAttrs[attrs++].value.clusterDim = {clusterSize, 1, 1}; launchAttrs[attrs].id = CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE; launchAttrs[attrs++].value.clusterSchedulingPolicyPreference = CU_CLUSTER_SCHEDULING_POLICY_SPREAD; } #if CUDART_VERSION >= 12000 if (compCap >= 90 && driverVersion >= 12000) { // Set the NCCL Mem Sync domain on CUDA 12.0 and later (sm90) launchAttrs[attrs].id = CU_LAUNCH_ATTRIBUTE_MEM_SYNC_DOMAIN; launchAttrs[attrs++].value.memSyncDomain = (CUlaunchMemSyncDomain) ncclParamMemSyncDomain(); } #endif #if CUDART_VERSION >= 12030 enum ncclImplicitOrder implicitOrder; NCCLCHECKGOTO(getImplicitOrder(&implicitOrder, plan->persistent, driverVersion), ret, do_return); if (implicitOrder == ncclImplicitOrderLaunch) { launchAttrs[attrs].id = CU_LAUNCH_ATTRIBUTE_LAUNCH_COMPLETION_EVENT; launchAttrs[attrs].value.launchCompletionEvent.event = comm->sharedRes->launchEvent; launchAttrs[attrs].value.launchCompletionEvent.flags = 0; attrs++; } if (plan->isSymColl && compCap >= 90 && driverVersion >= 12030) { launchAttrs[attrs].id = CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION; launchAttrs[attrs].value.programmaticStreamSerializationAllowed = 1; attrs++; } #endif #if CUDART_VERSION >= 13000 if (compCap >= 100 && driverVersion >= 13000) { launchAttrs[attrs].id = CU_LAUNCH_ATTRIBUTE_NVLINK_UTIL_CENTRIC_SCHEDULING; launchAttrs[attrs].value.nvlinkUtilCentricScheduling = comm->config.nvlinkCentricSched; attrs++; } #endif launchConfig.gridDimX = grid.x; launchConfig.gridDimY = grid.y; launchConfig.gridDimZ = grid.z; launchConfig.blockDimX = block.x; launchConfig.blockDimY = block.y; launchConfig.blockDimZ = block.z; launchConfig.sharedMemBytes = smem; launchConfig.attrs = launchAttrs; launchConfig.numAttrs = attrs; launchConfig.hStream = launchStream; CUCHECKGOTO(cuLaunchKernelEx(&launchConfig, fn, nullptr, extra), ret, do_return); #endif } else { // Standard kernel launch CUCHECKGOTO(cuLaunchKernel(fn, grid.x, grid.y, grid.z, block.x, block.y, block.z, smem, launchStream, nullptr, extra), ret, do_return); } do_return: NCCLCHECK(ncclProfilerStopKernelLaunchEvent(plan)); return ret; } ncclResult_t ncclLaunchKernelAfter_NoCuda(struct ncclComm* comm, struct ncclKernelPlan* plan) { if (!plan->isHostCbEnq) { // we are not using the host stream for proxy ops and reclaimation submission, call // hostStreamPlanTask directly NCCLCHECK(hostStreamPlanTask(comm, plan)); } return ncclSuccess; } namespace { struct KernelFinishCallback { struct ncclCommEventCallback base; uint32_t workFifoConsumed; }; ncclResult_t KernelFinishCallback_fn( struct ncclComm* comm, struct ncclCommEventCallback* cb ) { struct KernelFinishCallback* me = (struct KernelFinishCallback*)cb; comm->workFifoConsumed = me->workFifoConsumed; CUDACHECK(cudaEventDestroy(me->base.event)); free(me); return ncclSuccess; } } ncclResult_t ncclLaunchFinish(struct ncclComm* comm) { struct ncclKernelPlanner* planner = &comm->planner; if (!ncclIntruQueueEmpty(&planner->planQueue)) { // Reset queue to empty without destroying plans since those will be sent // back to us for reclaiming via callbackQueue. ncclIntruQueueConstruct(&planner->planQueue); cudaStream_t launchStream = planner->streams->stream; // First user stream gets launch cudaStream_t deviceStream, launchOrder; cudaEvent_t finishedEvent = comm->sharedRes->scratchEvent; CUDACHECK(cudaEventRecord(finishedEvent, launchStream)); if (comm->workFifoProduced - comm->workFifoProducedLastRecorded > comm->workFifoBytes/8) { comm->workFifoProducedLastRecorded = comm->workFifoProduced; struct KernelFinishCallback* cb; NCCLCHECK(ncclCalloc(&cb, 1)); cb->base.event = finishedEvent; cb->base.fn = KernelFinishCallback_fn; cb->workFifoConsumed = comm->workFifoProduced; ncclIntruQueueEnqueue(&comm->eventCallbackQueue, &cb->base); // We just stole scratchEvent so must create a new one. CUDACHECK(cudaEventCreateWithFlags(&comm->sharedRes->scratchEvent, cudaEventDisableTiming)); } // deviceStream waits on userStream[0] NCCLCHECK(ncclStrongStreamAcquiredWorkStream(planner->capturingGraph, &comm->sharedRes->deviceStream, /*concurrent=*/false, &deviceStream)); // We know that deviceStream is strictly behind the launchStream because launchStream // synced with it before kernel launch. This allows us to to see deviceStream waiting // on launchStream as a fast-forward. When building CUDA graphs fast forwards should // be handled specially so as not to create graphs with a blowup in the number of edges. // So we could do this: // CUDACHECK(cudaStreamWaitEvent(deviceStream, finishedEvent, 0)); // But instead we do: NCCLCHECK(ncclStreamAdvanceToEvent(planner->capturingGraph, deviceStream, finishedEvent)); // Each userStream[i] waits on userStream[0] for (struct ncclCudaStreamList* l=planner->streams->next; l != nullptr; l = l->next) { CUDACHECK(cudaStreamWaitEvent(l->stream, finishedEvent, 0)); } bool capturing = ncclCudaGraphValid(planner->capturingGraph); enum ncclImplicitOrder implicitOrder; NCCLCHECK(getImplicitOrder(&implicitOrder, capturing)); if (implicitOrder != ncclImplicitOrderNone) { // As in ncclLaunchPrepare, strong stream can be non-concurrent when non-captured. bool concurrent = capturing; // Incorporate launch event into per-device (context) launch order. NCCLCHECK(ncclStrongStreamAcquiredWorkStream(planner->capturingGraph, &comm->context->launchOrder, concurrent, &launchOrder)); // If we don't have launch events (requires CUDA 12.3) then just use completion event (serialize execution). CUDACHECK(cudaStreamWaitEvent(launchOrder, implicitOrder == ncclImplicitOrderLaunch ? comm->sharedRes->launchEvent : finishedEvent)); // Release launchOrder as acquired in ncclLaunchPrepare() NCCLCHECK(ncclStrongStreamRelease(planner->capturingGraph, &comm->context->launchOrder, concurrent)); } // Release deviceStream as acquired in ncclLaunchPrepare() NCCLCHECK(ncclStrongStreamRelease(planner->capturingGraph, &comm->sharedRes->deviceStream, /*concurrent=*/false)); } return ncclSuccess; } /*****************************************************************************/ /* Enqueueing system : computation of kernel and proxy operations parameters */ /*****************************************************************************/ ncclResult_t ncclGetCollNetSupport( struct ncclComm* comm, struct ncclTaskColl* info, int* collNetSupport ) { // Translate ncclAvg and PreMulSum ncclRedOp_t netOp = info->opHost; if (info->opDev.op == ncclDevPreMulSum || info->opDev.op == ncclDevSumPostDiv) { netOp = ncclSum; } *collNetSupport = comm->config.collnetEnable; switch (info->func) { case ncclFuncAllReduce: case ncclFuncReduce: case ncclFuncReduceScatter: *collNetSupport &= comm->collNetSupportMatrix[netOp][info->datatype]; break; default: break; } return ncclSuccess; } static void initCollCostTable(float** collCostTable) { float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable; for (int a = 0; a < NCCL_NUM_ALGORITHMS; a++) { for (int p = 0; p < NCCL_NUM_PROTOCOLS; p++) { table[a][p] = NCCL_ALGO_PROTO_IGNORE; } } } // numPipeOps: number of pipelined ops. Can be greater than 1 in aggregation mode. Used to adjust latency. static ncclResult_t updateCollCostTable( struct ncclComm* comm, struct ncclTaskColl* info, size_t nBytes, int collNetSupport, int nvlsSupport, int numPipeOps, float** collCostTable) { float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable; if (comm->nRanks == 1) { table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] = 0.0; return ncclSuccess; } for (int a=0; amaxLocalRanks > NCCL_MAX_DIRECT_ARITY+1) continue; // Disable CollNet Chain for more than 8 local GPUs if (a == NCCL_ALGO_COLLNET_CHAIN && comm->maxLocalRanks > NCCL_MAX_DIRECT_ARITY+1) continue; if ((a == NCCL_ALGO_NVLS || a == NCCL_ALGO_NVLS_TREE) && (!nvlsSupport || (info->func != ncclFuncAllReduce && comm->localRanks > NCCL_MAX_NVLS_ARITY))) continue; if (a == NCCL_ALGO_NVLS && collNetSupport != 1 && comm->nNodes > 1) continue; /* Tree reduceScatter doesn't support scaling yet */ if (a == NCCL_ALGO_PAT && info->func == ncclFuncReduceScatter && (info->opDev.op == ncclDevPreMulSum || info->opDev.op == ncclDevSumPostDiv)) continue; for (int p=0; pfunc, a, p, nBytes, numPipeOps, &table[a][p])); // Relegate fp8 reduction trees of sufficient depth that they incur precision loss // to be least preferred. if (info->datatype == ncclFloat8e4m3 || info->datatype == ncclFloat8e5m2) { if (a == NCCL_ALGO_RING && comm->nRanks > 8) { table[a][p] *= 1024.0; // Any factor large enough to act as a partition between lossy and non-lossy algos. } } } } return ncclSuccess; } static ncclResult_t topoGetAlgoInfo( struct ncclComm* comm, struct ncclTaskColl* info, size_t nBytes, float** collCostTable, ncclSimInfo_t* simInfo ) { float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable; float minTime = FLT_MAX; int algorithm = info->algorithm = NCCL_ALGO_UNDEF; int protocol = info->protocol = NCCL_PROTO_UNDEF; for (int a=0; a= 0.0 && table[a][p] < minTime) { algorithm = a; protocol = p; minTime = table[a][p]; } } } info->algorithm = algorithm; info->protocol = protocol; float time = minTime; // Yes, we are first assigning and then testing if protocol is sane, but that's OK in this case. // coverity[check_after_sink] if (info->algorithm == NCCL_ALGO_UNDEF || info->protocol == NCCL_PROTO_UNDEF) { char ncclAlgoEnvStr[1024] = ""; char ncclProtoEnvStr[1024] = ""; const char* algoEnv = ncclGetEnv("NCCL_ALGO"); if (algoEnv) { snprintf(ncclAlgoEnvStr, 1023, " NCCL_ALGO was set to %s.", algoEnv); } const char* protoEnv = ncclGetEnv("NCCL_PROTO"); if (protoEnv) { snprintf(ncclProtoEnvStr, 1023, " NCCL_PROTO was set to %s.", protoEnv); } WARN("Error : no algorithm/protocol available for function %s with datatype %s.%s%s", ncclFuncToString(info->func), ncclDatatypeToString(info->datatype), ncclAlgoEnvStr, ncclProtoEnvStr); return (algoEnv || protoEnv) ? ncclInvalidUsage : ncclInternalError; } if (simInfo) simInfo->estimatedTime = time; TRACE(NCCL_COLL, "%ld Bytes -> Algo %d proto %d time %f", nBytes, info->algorithm, info->protocol, time); int nc = comm->nChannels; int nt = comm->maxThreads[info->algorithm][info->protocol]; int threadThreshold = comm->threadThresholds[info->algorithm][info->protocol]; if (info->algorithm == NCCL_ALGO_COLLNET_DIRECT) { // CollNet channel tuning int ncSwitch = 16; bool flag = true; while (ncSwitch >= 1 && flag) { while ((flag = nBytes < nc*nt*comm->channels[0].collnetDirect.nHeads*threadThreshold) && nc > ncSwitch) { if (nc == ncSwitch+ncSwitch/2) threadThreshold /= 2; nc--; } ncSwitch /= 2; } } else if (info->algorithm == NCCL_ALGO_NVLS || info->algorithm == NCCL_ALGO_NVLS_TREE) { // NVLS should not need more than 16 channels to get peak BW. if (comm->nNodes > 1 && info->algorithm == NCCL_ALGO_NVLS) { nc = std::min(comm->nvlsChannels, comm->nChannels); } else { nc = comm->nvlsChannels; } } else { // Ring/Tree channel tuning while (nBytes < nc * nt * threadThreshold) { if (nc >= 2) nc--; else break; } } if (info->algorithm != NCCL_ALGO_NVLS && info->algorithm != NCCL_ALGO_NVLS_TREE && info->algorithm != NCCL_ALGO_COLLNET_DIRECT) { while (nBytes < nc * nt * threadThreshold) { if (nt % 128 == 0) nt /= 2; else break; } } if (info->protocol == NCCL_PROTO_SIMPLE) { if (info->algorithm == NCCL_ALGO_RING) nt += WARP_SIZE; // Extra warp for sync // More threads or sync warps needed due to split thread model if (info->algorithm == NCCL_ALGO_TREE) nt += 4*WARP_SIZE; } nt = nt/WARP_SIZE < 3 ? 3*WARP_SIZE : nt; if (info->algorithm == NCCL_ALGO_TREE) nt = NCCL_MAX_NTHREADS; // Tree now uses all threads always. if (info->algorithm == NCCL_ALGO_PAT) nt = NCCL_MAX_NTHREADS; info->nMaxChannels = nc; info->nWarps = nt/WARP_SIZE; return ncclSuccess; } // Use the default topo-based tuner if tuner plugin is not successful. // Call the plugin first. Let it set algo+proto, and/or nChannels. // Then, topoGetAlgoInfo will set algo/proto if not set, then nChannels and nThreads based on algo/proto. // Finally, nChannels will be overriden by the plugin setting. ncclResult_t ncclGetAlgoInfo( struct ncclComm* comm, struct ncclTaskColl* info, int collNetSupport, int nvlsSupport, int numPipeOps, ncclSimInfo_t* simInfo/* = NULL*/ ) { size_t elementSize = ncclTypeSize(info->datatype); size_t nBytes = elementSize * ncclFuncMaxSendRecvCount(info->func, comm->nRanks, info->count); struct ncclReg* regSendBuf = NULL; struct ncclReg* regRecvBuf = NULL; int regBuff; bool isSendValid, isRecvValid; size_t sendbuffSize = elementSize * ncclFuncSendCount(info->func, comm->nRanks, info->count); size_t recvbuffSize = elementSize * ncclFuncRecvCount(info->func, comm->nRanks, info->count); info->algorithm = NCCL_ALGO_UNDEF; info->protocol = NCCL_PROTO_UNDEF; int nMaxChannels = 0; float collCostTable[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; initCollCostTable((float **)collCostTable); NCCLCHECK(updateCollCostTable(comm, info, nBytes, collNetSupport, nvlsSupport, numPipeOps, (float **)collCostTable)); if (comm->tuner != NULL) { NCCLCHECK(ncclRegFind(comm, info->sendbuff, sendbuffSize, ®SendBuf)); NCCLCHECK(ncclRegFind(comm, info->recvbuff, recvbuffSize, ®RecvBuf)); NCCLCHECK(ncclRegLocalIsValid(regSendBuf, &isSendValid)); NCCLCHECK(ncclRegLocalIsValid(regRecvBuf, &isRecvValid)); regBuff = (regSendBuf && regRecvBuf && isSendValid && isRecvValid) || (ncclCudaGraphValid(comm->planner.capturingGraph) && ncclParamGraphRegister()); NCCLCHECK(comm->tuner->getCollInfo( comm->tunerContext, info->func, nBytes, numPipeOps, (float **)collCostTable, NCCL_NUM_ALGORITHMS, NCCL_NUM_PROTOCOLS, regBuff, &nMaxChannels)); NCCLCHECK(topoGetAlgoInfo(comm, info, nBytes, (float **)collCostTable, simInfo)); } else { NCCLCHECK(topoGetAlgoInfo(comm, info, nBytes, (float **)collCostTable, simInfo)); // NCCL_CTA_POLICY_EFFICIENCY requires user (non-symmetric) buffer registration (currently unsupported with MNNVL) if ((comm->config.CTAPolicy & NCCL_CTA_POLICY_EFFICIENCY) && ncclGetEnv("NCCL_ALGO") == NULL && ncclGetEnv("NCCL_PROTO") == NULL && !comm->MNNVL) { // make algorithm selection based on buffer registration // there can be other specialized policies for algorithms and protocols pickup in the future NCCLCHECK(ncclRegFind(comm, info->sendbuff, sendbuffSize, ®SendBuf)); NCCLCHECK(ncclRegFind(comm, info->recvbuff, recvbuffSize, ®RecvBuf)); NCCLCHECK(ncclRegLocalIsValid(regSendBuf, &isSendValid)); NCCLCHECK(ncclRegLocalIsValid(regRecvBuf, &isRecvValid)); regBuff = (regSendBuf && regRecvBuf && isSendValid && isRecvValid) || (ncclCudaGraphValid(comm->planner.capturingGraph) && ncclParamGraphRegister()); if (regBuff && (info->func == ncclFuncAllGather || info->func == ncclFuncReduceScatter)) { if ((comm->nNodes > 1 && collNetSupport && nvlsSupport) || (comm->nNodes == 1 && nvlsSupport)) { int recChannels; NCCLCHECK(ncclNvlsRegResourcesQuery(comm, info, &recChannels)); if (recChannels <= info->nMaxChannels) { info->algorithm = NCCL_ALGO_NVLS; info->protocol = NCCL_PROTO_SIMPLE; info->nMaxChannels = recChannels; info->nWarps = comm->maxThreads[info->algorithm][info->protocol] / WARP_SIZE; } } } } } info->nMaxChannels = nMaxChannels == 0 ? info->nMaxChannels : nMaxChannels; return ncclSuccess; } NCCL_PARAM(NvlsTreeMaxChunkSize, "NVLSTREE_MAX_CHUNKSIZE", -2); static ncclResult_t calcCollChunking( struct ncclComm* comm, struct ncclTaskColl* info, int nChannels, size_t nBytes, /*outputs*/uint32_t* outChunkSize, uint32_t* outDirectFlags, struct ncclProxyOp* proxyOp ) { ncclPattern_t pattern; size_t grainSize = ncclProtoGrainSize(info->protocol); switch (info->func) { case ncclFuncBroadcast: pattern = info->algorithm == NCCL_ALGO_TREE ? ncclPatternTreeDown : ncclPatternPipelineFrom; break; case ncclFuncReduce: pattern = info->algorithm == NCCL_ALGO_TREE ? ncclPatternTreeUp : ncclPatternPipelineTo; break; case ncclFuncReduceScatter: pattern = info->algorithm == NCCL_ALGO_PAT ? ncclPatternPatUp : info->algorithm == NCCL_ALGO_NVLS ? ncclPatternNvls : info->algorithm == NCCL_ALGO_COLLNET_DIRECT ? ncclPatternCollnetDirect : ncclPatternRing; break; case ncclFuncAllGather: pattern = info->algorithm == NCCL_ALGO_PAT ? ncclPatternPatDown : info->algorithm == NCCL_ALGO_NVLS ? ncclPatternNvls : info->algorithm == NCCL_ALGO_COLLNET_DIRECT ? ncclPatternCollnetDirect : ncclPatternRing; break; case ncclFuncAllReduce: pattern = info->algorithm == NCCL_ALGO_NVLS ? ncclPatternNvls : info->algorithm == NCCL_ALGO_NVLS_TREE ? ncclPatternNvlsTree : info->algorithm == NCCL_ALGO_COLLNET_DIRECT ? ncclPatternCollnetDirect : info->algorithm == NCCL_ALGO_COLLNET_CHAIN ? ncclPatternCollnetChain : info->algorithm == NCCL_ALGO_TREE ? ncclPatternTreeUpDown : ncclPatternRingTwice; break; default: WARN("Unknown pattern for collective %d algorithm %d", info->func, info->algorithm); return ncclInternalError; } int nstepsPerLoop, nchunksPerLoop; size_t loopOffset = 0; int stepSize = comm->buffSizes[info->protocol]/NCCL_STEPS; int chunkSteps = (info->protocol == NCCL_PROTO_SIMPLE && info->algorithm == NCCL_ALGO_RING) ? info->chunkSteps : 1; int sliceSteps = (info->protocol == NCCL_PROTO_SIMPLE && info->algorithm == NCCL_ALGO_RING) ? info->sliceSteps : 1; int chunkSize = stepSize*chunkSteps; if (info->protocol == NCCL_PROTO_LL) chunkSize /= 2; if (info->protocol == NCCL_PROTO_LL128) chunkSize = (chunkSize / NCCL_LL128_LINEELEMS) * NCCL_LL128_DATAELEMS; if (info->algorithm == NCCL_ALGO_COLLNET_DIRECT) { // Optimize chunkSize / nSteps while (nBytes / (nChannels * comm->channels[0].collnetDirect.nHeads * chunkSize) < comm->channels[0].collnetDirect.depth * 64 && chunkSize > 131072) chunkSize /= 2; while (nBytes / (nChannels * comm->channels[0].collnetDirect.nHeads * chunkSize) < comm->channels[0].collnetDirect.depth * 8 && chunkSize > 65536) chunkSize /= 2; while (nBytes / (nChannels * comm->channels[0].collnetDirect.nHeads * chunkSize) < comm->channels[0].collnetDirect.depth * 8 && chunkSize > 32768) chunkSize /= 2; } else if (info->algorithm == NCCL_ALGO_COLLNET_CHAIN) { stepSize = comm->buffSizes[NCCL_PROTO_SIMPLE] / NCCL_STEPS; chunkSize = std::min(256 * 1024, stepSize * chunkSteps); while (nBytes / (nChannels * chunkSize) < comm->channels[0].collnetChain.depth * 64 && chunkSize > 131072) chunkSize /= 2; while (nBytes / (nChannels * chunkSize) < comm->channels[0].collnetChain.depth * 8 && chunkSize > 65536) chunkSize /= 2; while (nBytes / (nChannels * chunkSize) < comm->channels[0].collnetChain.depth && chunkSize > 32768) chunkSize /= 2; } else if (info->algorithm == NCCL_ALGO_NVLS) { if ((info->regBufType & NCCL_NVLS_REG_BUFFER) && (info->func == ncclFuncAllGather || info->func == ncclFuncReduceScatter)) { chunkSize = comm->buffSizes[NCCL_PROTO_SIMPLE] / NCCL_STEPS; } else { int maxChunkSize = comm->nvlsChunkSize; if (comm->nNodes > 1 && comm->bandwidths[ncclFuncAllReduce][NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] < 150) maxChunkSize = 32768; if (chunkSize > maxChunkSize) chunkSize = maxChunkSize; // Use uint64_t so that concurrentOps*chunkSize*X does not overflow. // However, nChannels * comm->channels[0].nvls.nHeads should easily fit in 32 bits. // coverity[overflow_before_widen] uint64_t concurrentOps = nChannels * comm->channels[0].nvls.nHeads; if ((nBytes < (64 * (concurrentOps * chunkSize))) && (chunkSize > 65536)) chunkSize = 65536; if ((nBytes < (8 * (concurrentOps * chunkSize))) && (chunkSize > 32768)) chunkSize = 32768; if ((nBytes < (2 * (concurrentOps * chunkSize))) && (chunkSize > 16384)) chunkSize = 16384; } } else if (info->algorithm == NCCL_ALGO_NVLS_TREE) { // Use uint64_t so that concurrentOps*chunkSize*X does not overflow. // However, nChannels * comm->channels[0].nvls.nHeads should easily fit in 32 bits. // coverity[overflow_before_widen] uint64_t concurrentOps = nChannels * comm->channels[0].nvls.nHeads; chunkSize = comm->nvlsChunkSize; int maxChunkSize = (int)ncclParamNvlsTreeMaxChunkSize(); if (maxChunkSize == -2) maxChunkSize = comm->nNodes >= 4 ? 65536 : chunkSize; chunkSize = std::min(chunkSize, maxChunkSize); if ((nBytes < (32 * (concurrentOps * chunkSize))) && (chunkSize > 262144)) chunkSize = 262144; if ((nBytes < (16 * (concurrentOps * chunkSize))) && (chunkSize > 131072)) chunkSize = 131072; if ((nBytes < (4 * (concurrentOps * chunkSize))) && (chunkSize > 65536)) chunkSize = 65536; if ((nBytes < (1 * (concurrentOps * chunkSize))) && (chunkSize > 32768)) chunkSize = 32768; } else if (info->algorithm == NCCL_ALGO_TREE && info->protocol == NCCL_PROTO_LL128) { int nNodes = comm->nNodes; float ppn = comm->nRanks / (float)nNodes; float nstepsLL128 = 1+log2i(nNodes) + 0.1*ppn; // Yes, we are OK with the division on the left side of the < operand being integer. // coverity[integer_division] while (nBytes / (nChannels*chunkSize) < nstepsLL128*64/ppn && chunkSize > 131072) chunkSize /= 2; // coverity[integer_division] while (nBytes / (nChannels*chunkSize) < nstepsLL128*16/ppn && chunkSize > 32768) chunkSize /= 2; } else if (info->func == ncclFuncAllGather && info->algorithm == NCCL_ALGO_PAT) { while (chunkSize*nChannels*32 > nBytes && chunkSize > 65536) chunkSize /= 2; } else if (info->func == ncclFuncReduceScatter && info->algorithm == NCCL_ALGO_PAT) { while (chunkSize*nChannels*16 > nBytes && chunkSize > 65536) chunkSize /= 2; } // Compute directFlags of work struct. if (info->algorithm == NCCL_ALGO_COLLNET_DIRECT) { *outDirectFlags = NCCL_P2P_WRITE; } else { *outDirectFlags = 0; } // Compute nSteps for proxies chunkSize = chunkSize / grainSize * grainSize; // align chunkSize to multiple grainSize switch (pattern) { case ncclPatternTreeUp: case ncclPatternTreeDown: case ncclPatternTreeUpDown: case ncclPatternPatUp: case ncclPatternPatDown: case ncclPatternPipelineFrom: case ncclPatternPipelineTo: case ncclPatternCollnetChain: nstepsPerLoop = nchunksPerLoop = 1; break; case ncclPatternNvls: nstepsPerLoop = 1; nchunksPerLoop = comm->channels[0].nvls.nHeads; loopOffset = nChannels * chunkSize * comm->channels[0].nvls.headRank; break; case ncclPatternCollnetDirect: nstepsPerLoop = 1; nchunksPerLoop = comm->channels[0].collnetDirect.nHeads; loopOffset = nChannels * chunkSize * comm->channels[0].collnetDirect.headRank; break; case ncclPatternRing: nstepsPerLoop = comm->nRanks-1; nchunksPerLoop = comm->nRanks; break; case ncclPatternRingTwice: nstepsPerLoop = 2*(comm->nRanks-1); nchunksPerLoop = comm->nRanks; break; case ncclPatternNvlsTree: nstepsPerLoop = 1; nchunksPerLoop = comm->channels[0].nvls.nHeads; break; default: WARN("Unknown pattern %d", pattern); return ncclInternalError; } // Compute nSteps for proxies size_t loopSize = size_t(nChannels)*nchunksPerLoop*chunkSize; int nLoops = (int)DIVUP(nBytes, loopSize); memset(proxyOp, 0, sizeof(*proxyOp)); proxyOp->nsteps = nstepsPerLoop * nLoops * chunkSteps; proxyOp->sliceSteps = sliceSteps; proxyOp->chunkSteps = chunkSteps; proxyOp->chunkSize = chunkSize; proxyOp->sliceSize = chunkSize / chunkSteps * sliceSteps; proxyOp->loopSize = loopSize; proxyOp->loopOffset = loopOffset; proxyOp->protocol = info->protocol; proxyOp->dtype = info->datatype; proxyOp->algorithm = info->algorithm; if (info->opDev.op == ncclDevPreMulSum || info->opDev.op == ncclDevSumPostDiv) { proxyOp->redOp = ncclSum; // Network sees avg as sum } else { proxyOp->redOp = info->opHost; } proxyOp->pattern = pattern; proxyOp->coll = info->func; proxyOp->collAPI = info->func; proxyOp->root = info->root; proxyOp->isOneRPN = comm->isOneRPN; // This is used by P2P to reduce the receive buffer size. We don't use it in collectives // because some protocols need to transmit more than the total size, plus they sometimes // round up proxyOp->nbytes = stepSize*sliceSteps; if (info->regBufType & NCCL_NET_REG_BUFFER) { proxyOp->reg = 1; if (info->algorithm == NCCL_ALGO_COLLNET_DIRECT || info->algorithm == NCCL_ALGO_NVLS || info->algorithm == NCCL_ALGO_COLLNET_CHAIN) { if (proxyOp->isOneRPN) { proxyOp->nsteps = 1; proxyOp->loopOffset = 0; proxyOp->sendbuff = (uint8_t*)info->sendbuff; proxyOp->sendMhandle = info->sendMhandle; } else { if (info->func == ncclFuncAllGather || info->func == ncclFuncReduceScatter) { proxyOp->nbytes = nBytes / nchunksPerLoop; proxyOp->loopSize = proxyOp->loopSize / nchunksPerLoop; proxyOp->loopOffset = 0; if (info->func == ncclFuncAllGather) { proxyOp->sendbuff = (uint8_t*)info->sendbuff; proxyOp->sendMhandle = info->sendMhandle; } } else { proxyOp->sendbuff = (uint8_t*)info->recvbuff; proxyOp->sendMhandle = info->recvMhandle; } } } else if (info->algorithm == NCCL_ALGO_RING) { if (proxyOp->isOneRPN && info->func == ncclFuncAllGather) { proxyOp->chunkSize = NCCL_MAX_NET_SIZE; proxyOp->sliceSize = NCCL_MAX_NET_SIZE; proxyOp->chunkSteps = 1; proxyOp->sliceSteps = 1; proxyOp->loopSize = size_t(nChannels) * nchunksPerLoop * proxyOp->chunkSize; proxyOp->nsteps = DIVUP(nBytes, proxyOp->loopSize) * nstepsPerLoop; proxyOp->loopOffset = 0; } } else { WARN("Net registration invalid algorithm %s", ncclAlgoToString(info->algorithm)); return ncclInternalError; } proxyOp->recvMhandle = info->recvMhandle; proxyOp->recvbuff = (uint8_t*)info->recvbuff; proxyOp->nbytes = nBytes; } else { proxyOp->reg = 0; } if (pattern == ncclPatternCollnetDirect || pattern == ncclPatternNvls) { proxyOp->specifics.collnetDirect.nNodes = comm->nNodes; proxyOp->specifics.collnetDirect.node = comm->node; if (info->func == ncclFuncAllGather || info->func == ncclFuncReduceScatter) { proxyOp->specifics.collnetDirect.sizePerRank = info->count*ncclTypeSize(info->datatype); } } if (pattern == ncclPatternPatUp || pattern == ncclPatternPatDown) { proxyOp->nbytes = DIVUP(nBytes, nChannels); } // Set peer count hints used by network plugin switch (proxyOp->pattern) { case ncclPatternRing: case ncclPatternRingTwice: case ncclPatternPipelineFrom: case ncclPatternPipelineTo: case ncclPatternPatUp: case ncclPatternPatDown: proxyOp->nPeers = 1; break; case ncclPatternTreeUp: case ncclPatternTreeDown: case ncclPatternTreeUpDown: case ncclPatternNvlsTree: proxyOp->nPeers = (NCCL_MAX_TREE_ARITY - 1) * 2; break; case ncclPatternCollnetChain: case ncclPatternCollnetDirect: case ncclPatternNvls: case ncclPatternProfiler: // Peer count hints unused break; case ncclPatternSend: case ncclPatternRecv: default: WARN("Unknown pattern %d", pattern); return ncclInternalError; } *outChunkSize = proxyOp->chunkSize; return ncclSuccess; } static ncclResult_t hostToDevRedOp( ncclDevRedOpFull *opFull, ncclRedOp_t op, ncclDataType_t datatype, ncclComm *comm ) { union { int8_t i8; uint8_t u8; int32_t i32; uint32_t u32; int64_t i64; uint64_t u64; __half f16; float f32; double f64; #if defined(__CUDA_BF16_TYPES_EXIST__) __nv_bfloat16 bf16; #endif #if defined(__CUDA_FP8_TYPES_EXIST__) __nv_fp8_storage_t f8; #endif void *ptr; }; u64 = 0; opFull->scalarArgIsPtr = false; opFull->proxyOp = op; int nbits = 8*ncclTypeSize(datatype); if (nbits <= 0) return ncclInvalidArgument; uint64_t allBits = uint64_t(-1)>>(64-nbits); uint64_t signBit = allBits^(allBits>>1); bool datatype_signed = false; switch (int(op)) { case ncclSum: opFull->op = ncclDevSum; break; case ncclProd: opFull->op = ncclDevProd; break; case ncclMin: case ncclMax: opFull->op = ncclDevMinMax; opFull->scalarArg = 0; // The xormask used by ncclFuncMinMax<[u]int> is the XOR of the sign bit // for signed (opposed to unsigned) types and all the bits for max (opposed to min). if (datatype==ncclInt8 || datatype==ncclInt32 || datatype==ncclInt64) { opFull->scalarArg ^= signBit; } opFull->scalarArg ^= (op == ncclMax) ? allBits : 0; break; case ncclAvg: switch ((int)datatype) { case ncclInt8: case ncclInt32: case ncclInt64: datatype_signed = true; // no break, we want to fall through... case ncclUint8: case ncclUint32: case ncclUint64: opFull->op = ncclDevSumPostDiv; u64 = comm->nRanks<<1 | datatype_signed; break; #if defined(__CUDA_FP8_TYPES_EXIST__) case ncclFloat8e4m3: opFull->op = ncclDevPreMulSum; f8 = __nv_cvt_float_to_fp8(float(1.0/comm->nRanks), __NV_SATFINITE, __NV_E4M3); break; case ncclFloat8e5m2: opFull->op = ncclDevPreMulSum; f8 = __nv_cvt_float_to_fp8(float(1.0/comm->nRanks), __NV_SATFINITE, __NV_E5M2); break; #endif case ncclFloat16: opFull->op = ncclDevPreMulSum; f16 = __float2half(float(1.0/comm->nRanks)); // __double2half not supported pre CUDA 11.x break; #if defined(__CUDA_BF16_TYPES_EXIST__) case ncclBfloat16: opFull->op = ncclDevPreMulSum; bf16 = __float2bfloat16(float(1.0/comm->nRanks)); break; #endif case ncclFloat32: opFull->op = ncclDevPreMulSum; f32 = float(1.0/comm->nRanks); break; case ncclFloat64: opFull->op = ncclDevPreMulSum; f64 = 1.0/comm->nRanks; break; } opFull->scalarArgIsPtr = false; opFull->scalarArg = u64; break; default: // user created int ix = int(ncclUserRedOpMangle(comm, op)) - int(ncclNumOps); ncclUserRedOp *user = &comm->userRedOps[ix]; if (datatype != user->datatype) { WARN("Data type supplied to user-created ncclRedOp_t does not match type " "given to reduction operation"); return ncclInvalidArgument; } *opFull = user->opFull; break; } return ncclSuccess; } static ncclResult_t ncclPlannerSetCapturingGraph(struct ncclComm* comm, struct ncclInfo* info) { struct ncclKernelPlanner *planner = &comm->planner; if (info->stream != planner->streamRecent || planner->streams == nullptr) { planner->streamRecent = info->stream; struct ncclCudaStreamList* l = planner->streams; while (true) { if (l == nullptr) { // Got to the end, this must be a new stream. struct ncclCudaGraph graph; NCCLCHECK(ncclCudaGetCapturingGraph(&graph, info->stream, comm->config.graphUsageMode)); if (planner->streams != nullptr && !ncclCudaGraphSame(planner->capturingGraph, graph)) { WARN("Streams given to a communicator within a NCCL group must either be all uncaptured or all captured by the same graph."); return ncclInvalidUsage; } planner->capturingGraph = graph; // C++ struct assignment // Add stream to list l = ncclMemoryStackAlloc(&comm->memScoped); l->stream = info->stream; l->next = planner->streams; planner->streams = l; break; } if (l->stream == info->stream) break; // Already seen stream. l = l->next; } } return ncclSuccess; } static ncclResult_t p2pTaskAppend( struct ncclComm* comm, struct ncclInfo* info, ncclFunc_t coll, ncclFunc_t collAPI, void* buff, size_t count, ncclDataType_t datatype, int peer, bool allowUB) { struct ncclKernelPlanner *planner = &comm->planner; // Determine peer and basic parameters. ssize_t nBytes = count*ncclTypeSize(datatype); bool isSendNotRecv = coll == ncclFuncSend; // Must be in thread local group before tasks can be alloc'd in `comm->memScoped`. ncclGroupCommJoin(comm, ncclGroupTaskTypeCollective); info->coll = coll; // Set capturing graph. Called here so that profiler can emit a group API event with this information NCCLCHECK(ncclPlannerSetCapturingGraph(comm, info)); bool isGraphCaptured = ncclCudaGraphValid(planner->capturingGraph); NCCLCHECK(ncclProfilerStartGroupApiEvent(info, isGraphCaptured)); NCCLCHECK(ncclProfilerRecordGroupApiEventState(ncclProfilerGroupStartApiStop)); NCCLCHECK(ncclProfilerStartP2pApiEvent(info, isGraphCaptured)); struct ncclTaskP2p* p2p = ncclMemoryPoolAlloc(&comm->memPool_ncclTaskP2p, &comm->memPermanent); p2p->func = coll; p2p->collAPI = collAPI; p2p->buff = buff; p2p->count = count; p2p->datatype = datatype; p2p->root = peer; p2p->bytes = nBytes; p2p->allowUB = allowUB; p2p->eActivationMask = ncclProfilerApiState.eActivationMask; p2p->groupApiEventHandle = ncclProfilerApiState.groupApiEventHandle; p2p->p2pApiEventHandle = ncclProfilerApiState.p2pApiEventHandle; ncclIntruQueueEnqueue( isSendNotRecv ? &planner->peers[peer].sendQueue : &planner->peers[peer].recvQueue, p2p); planner->nTasksP2p += 1; if (isSendNotRecv) planner->nTasksP2pSend += 1; else planner->nTasksP2pRecv += 1; // Mark channels that need pre-connect if (comm->rank != peer) { if (!(isSendNotRecv ? planner->peers[peer].sendSeen : planner->peers[peer].recvSeen)) { // planner->peers[peer].send/recvSeen is private to each comm, so we need to set it anyway. (isSendNotRecv ? planner->peers[peer].sendSeen : planner->peers[peer].recvSeen) = true; int round = 0; while (peer != (isSendNotRecv ? comm->p2pSchedule[round].sendRank : comm->p2pSchedule[round].recvRank)) { round += 1; } uint8_t base = ncclP2pChannelBaseForRound(comm, round); for (int c=0; c < comm->p2pnChannelsPerPeer; c++) { int channelId = ncclP2pChannelForPart(comm->p2pnChannels, base, c); if (isSendNotRecv) { if (comm->channels[channelId].peers[peer]->send[1].hasSeen == 0) { // P2P uses only 1 connector // the send/recv connector is shared among split shared comms. We need to set hasSeen to // 1 in order to avoid duplicate connection setup if user group sendrecv ops with split // shared comms together. comm->channels[channelId].peers[peer]->send[1].hasSeen = 1; comm->channels[channelId].peers[peer]->send[1].p2pOnly = 1; comm->connectSend[peer] |= (1ULL<channels[channelId].peers[peer]->recv[1].hasSeen == 0) { // P2P uses only 1 connector comm->channels[channelId].peers[peer]->recv[1].hasSeen = 1; comm->channels[channelId].peers[peer]->recv[1].p2pOnly = 1; comm->connectRecv[peer] |= (1ULL<planner; // Must be in thread local group before tasks can be alloc'd in `comm->memScoped`. ncclGroupCommJoin(info->comm, ncclGroupTaskTypeCollective); // Set capturing graph. Called here so that profiler can emit a group API event with this information NCCLCHECK(ncclPlannerSetCapturingGraph(comm, info)); bool isGraphCaptured = ncclCudaGraphValid(planner->capturingGraph); NCCLCHECK(ncclProfilerStartGroupApiEvent(info, isGraphCaptured)); NCCLCHECK(ncclProfilerRecordGroupApiEventState(ncclProfilerGroupStartApiStop)); NCCLCHECK(ncclProfilerStartCollApiEvent(info, isGraphCaptured)); if (info->coll == ncclFuncBroadcast && ncclParamAllgathervEnable() && !comm->ccEnable) { // Must be in thread local group before tasks can be alloc'd in `comm->memScoped`. struct ncclTaskBcast* t = ncclMemoryPoolAlloc(&comm->memPool_ncclTaskBcast, &comm->memPermanent); t->func = ncclFuncAllGatherV; t->sendbuff = info->sendbuff; t->recvbuff = info->recvbuff; t->count = info->count * ncclTypeSize(info->datatype); t->datatype = ncclInt8; t->root = info->root; // update bcast min/max peer planner->bcast_info.minBcastPeer = std::min(planner->bcast_info.minBcastPeer, info->root); planner->bcast_info.maxBcastPeer = std::max(planner->bcast_info.maxBcastPeer, info->root); if (ncclIntruQueueEmpty(&planner->peers[info->root].bcastQueue)) { planner->bcast_info.BcastPeers += 1; } // enqueue to peer's bcast queue instead of collSorter ncclIntruQueueEnqueue(&planner->peers[info->root].bcastQueue, t); planner->nTasksBcast += 1; } else { struct ncclTaskColl* t = ncclMemoryPoolAlloc(&comm->memPool_ncclTaskColl, &comm->memPermanent); t->func = info->coll; t->sendbuff = info->sendbuff; t->recvbuff = info->recvbuff; t->count = info->count; t->root = info->root; t->datatype = info->datatype; size_t elementSize = ncclTypeSize(t->datatype); if (t->func == ncclFuncAllGather || t->func == ncclFuncBroadcast) { t->count *= elementSize; t->datatype = ncclInt8; elementSize = 1; } t->trafficBytes = t->count*elementSize*ncclFuncTrafficPerByte(t->func, comm->nRanks); t->opHost = info->op; t->opDev = opDev; // C++ struct assignment t->chunkSteps = info->chunkSteps; t->sliceSteps = info->sliceSteps; t->eActivationMask = ncclProfilerApiState.eActivationMask; t->groupApiEventHandle = ncclProfilerApiState.groupApiEventHandle; t->collApiEventHandle = ncclProfilerApiState.collApiEventHandle; planner->nTasksColl += 1; ncclTaskCollSorterInsert(&planner->collSorter, t, t->trafficBytes); } ncclProfilerStopCollApiEvent(); return ncclSuccess; } static ncclResult_t ceCollTaskAppend( struct ncclComm* comm, struct ncclInfo* info, struct ncclDevrWindow* sendWin, struct ncclDevrWindow* recvWin, struct ncclDevRedOpFull opDev) { struct ncclKernelPlanner *planner = &comm->planner; // Check if CE needs initialization if (comm->ceColl.baseUCSymReadyPtr == NULL && ncclIntruQueueEmpty(&comm->ceInitTaskQueue)) { struct ncclCeInitTask* ceTask; NCCLCHECK(ncclCalloc(&ceTask, 1)); ceTask->comm = comm; ncclIntruQueueEnqueue(&comm->ceInitTaskQueue, ceTask); ncclGroupCommJoin(comm, ncclGroupTaskTypeSymRegister); } // Must be in thread local group before tasks can be alloc'd in `comm->memScoped`. ncclGroupCommJoin(info->comm, ncclGroupTaskTypeCollective); // Set capturing graph. Called here so that profiler can emit a group API event with this information NCCLCHECK(ncclPlannerSetCapturingGraph(comm, info)); bool isGraphCaptured = ncclCudaGraphValid(planner->capturingGraph); NCCLCHECK(ncclProfilerStartGroupApiEvent(info, isGraphCaptured)); NCCLCHECK(ncclProfilerRecordGroupApiEventState(ncclProfilerGroupStartApiStop)); NCCLCHECK(ncclProfilerStartCollApiEvent(info, isGraphCaptured)); struct ncclTaskColl* t = ncclMemoryPoolAlloc(&comm->memPool_ncclTaskColl, &comm->memPermanent); t->func = info->coll; t->sendbuff = info->sendbuff; t->recvbuff = info->recvbuff; t->count = info->count; t->root = info->root; t->datatype = info->datatype; size_t elementSize = ncclTypeSize(t->datatype); if (t->func == ncclFuncAllGather || t->func == ncclFuncBroadcast) { t->count *= elementSize; t->datatype = ncclInt8; elementSize = 1; } t->trafficBytes = t->count*elementSize*ncclFuncTrafficPerByte(t->func, comm->nRanks); t->opHost = info->op; t->opDev = opDev; // C++ struct assignment t->chunkSteps = info->chunkSteps; t->sliceSteps = info->sliceSteps; t->eActivationMask = COMPILER_ATOMIC_LOAD(&ncclProfilerEventMask, std::memory_order_relaxed); t->groupApiEventHandle = ncclProfilerApiState.groupApiEventHandle; t->collApiEventHandle = ncclProfilerApiState.collApiEventHandle; t->sendWin = sendWin; t->recvWin = recvWin; ncclIntruQueueEnqueue(&planner->collCeTaskQueue, t); ncclProfilerStopCollApiEvent(); return ncclSuccess; } static ncclResult_t rmaTaskAppend( struct ncclComm* comm, struct ncclInfo* info) { struct ncclKernelPlanner *planner = &comm->planner; void const* srcBuff = info->sendbuff; if (!comm->hostRmaSupport) { WARN("One sided RMA: host RMA is not supported in this communicator."); return ncclInvalidArgument; } // Check if context is valid (must be 0 for now) if (info->ctx != 0) { WARN("Context %d is invalid (must be 0)", info->ctx); return ncclInvalidArgument; } // Check if signal index is valid (must be 0 for now) if (info->sigIdx != 0) { WARN("Signal index %d is invalid (must be 0)", info->sigIdx); return ncclInvalidArgument; } // Check if flags is valid if (info->flags != 0) { WARN("Flags %u is invalid (must be 0)", info->flags); return ncclInvalidArgument; } // Initialize window pointers - only needed for Put and Signal struct ncclDevrWindow* peerWinHost = NULL; struct ncclDevrWindow* srcWinHost = NULL; size_t srcWinOffset = 0; if (info->coll == ncclFuncPutSignal) { // Validate peer window with detailed debugging if (info->peerWin == NULL) { WARN("ncclPutSignal: peerWin is NULL"); return ncclInvalidArgument; } struct ncclWindow_vidmem* peerWinDevHost = NULL; NCCLCHECK(ncclShadowPoolToHost(&comm->devrState.shadows, info->peerWin, &peerWinDevHost)); peerWinHost = (struct ncclDevrWindow*)peerWinDevHost->winHost; // Validate source buffer and window if (srcBuff == NULL) { WARN("ncclPutSignal: srcBuff is NULL"); return ncclInvalidArgument; } NCCLCHECK(ncclDevrFindWindow(comm, srcBuff, &srcWinHost)); if (srcWinHost == NULL || !(srcWinHost->winFlags & NCCL_WIN_COLL_SYMMETRIC)) { WARN("ncclPutSignal: srcWinHost is not in a valid symmetric window"); return ncclInvalidArgument; } srcWinOffset = (char*)srcBuff - (char*)srcWinHost->userPtr; } else if (info->coll == ncclFuncSignal) { // Check if count is valid if (info->count != 0) { WARN("ncclSignal: count must be 0"); return ncclInvalidArgument; } } else if (info->coll == ncclFuncWaitSignal) { // Check if signalDescs is valid if (info->signalDescs == NULL || info->nDesc == 0) { WARN("ncclWaitSignal: invalid arguments"); return ncclInvalidArgument; } // Validate each descriptor for (int i = 0; i < info->nDesc; i++) { if (info->signalDescs[i].opCnt <= 0) { WARN("ncclWaitSignal: descriptor %d has invalid opCnt %d", i, info->signalDescs[i].opCnt); return ncclInvalidArgument; } if (info->signalDescs[i].sigIdx != 0) { WARN("ncclWaitSignal: descriptor %d has invalid sigIdx %d (must be 0)", i, info->signalDescs[i].sigIdx); return ncclInvalidArgument; } if (info->signalDescs[i].ctx != 0) { WARN("ncclWaitSignal: descriptor %d has invalid context %d (must be 0)", i, info->signalDescs[i].ctx); return ncclInvalidArgument; } } } // Check if RMA CE needs initialization if (!comm->rmaState.rmaCeState.initialized && ncclIntruQueueEmpty(&comm->rmaCeInitTaskQueue)) { struct ncclRmaCeInitTask* ceTask; NCCLCHECK(ncclCalloc(&ceTask, 1)); ceTask->comm = comm; ncclIntruQueueEnqueue(&comm->rmaCeInitTaskQueue, ceTask); ncclGroupCommJoin(comm, ncclGroupTaskTypeSymRegister); } // Must be in thread local group before tasks can be alloc'd in `comm->memScoped`. ncclGroupCommJoin(info->comm, ncclGroupTaskTypeCollective); NCCLCHECK(ncclPlannerSetCapturingGraph(comm, info)); // Handle WaitSignal separately if (info->coll == ncclFuncWaitSignal) { struct ncclTaskRma* t = ncclMemoryPoolAlloc(&comm->memPool_ncclTaskRma, &comm->memPermanent); t->func = ncclFuncWaitSignal; t->ctx = 0; t->count = 0; t->bytes = 0; t->srcBuff = NULL; t->srcWinOffset = 0; t->srcWinHost = NULL; t->peer = 0; t->peerWinOffset = 0; t->peerWinHost = NULL; t->signalMode = NCCL_SIGNAL; // Convert descriptors to peers and nsignals arrays t->npeers = info->nDesc; t->peers = ncclMemoryStackAlloc(&comm->memScoped, info->nDesc); t->nsignals = ncclMemoryStackAlloc(&comm->memScoped, info->nDesc); for (int i = 0; i < info->nDesc; i++) { t->peers[i] = info->signalDescs[i].peer; t->nsignals[i] = info->signalDescs[i].opCnt; } t->eActivationMask = __atomic_load_n(&ncclProfilerEventMask, __ATOMIC_RELAXED); planner->nTasksRma++; ncclIntruQueueEnqueue(&planner->rmaTaskQueues[t->ctx], t); } else if (info->coll == ncclFuncPutSignal || info->coll == ncclFuncSignal) { // Calculate total bytes for the operation size_t totalBytes = info->count * ncclTypeSize(info->datatype); // Define 1GB chunk size for splitting large put operations const size_t chunkSize = 1ULL << 30; // 1GB = 1073741824 bytes // Determine if we need to split the operation int numChunks = 1; if (info->coll == ncclFuncPutSignal && totalBytes > chunkSize) { numChunks = (totalBytes + chunkSize - 1) / chunkSize; } // Create tasks for each chunk for (int chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) { struct ncclTaskRma* t = ncclMemoryPoolAlloc(&comm->memPool_ncclTaskRma, &comm->memPermanent); // Calculate chunk-specific size and offsets size_t chunkBytes = (chunkIdx == numChunks - 1) ? (totalBytes - chunkIdx * chunkSize) : chunkSize; size_t chunkOffset = chunkIdx * chunkSize; t->func = info->coll; t->srcBuff = (const char*)srcBuff + chunkOffset; t->srcWinOffset = srcWinOffset + chunkOffset; t->srcWinHost = srcWinHost; t->count = chunkBytes / ncclTypeSize(info->datatype); t->datatype = info->datatype; t->bytes = chunkBytes; t->ctx = info->ctx; t->peer = info->root; t->peerWinOffset = info->peerWinOffset + chunkOffset; t->peerWinHost = peerWinHost; // Signal handling: only the last chunk gets the signal bool isLastChunk = (chunkIdx == numChunks - 1); if (isLastChunk) { t->signalMode = NCCL_SIGNAL; } else { // Earlier chunks: no signal t->signalMode = NCCL_SIGNAL_NONE; } t->peers = NULL; t->nsignals = NULL; t->npeers = 0; t->eActivationMask = __atomic_load_n(&ncclProfilerEventMask, __ATOMIC_RELAXED); planner->nTasksRma++; // Enqueue the task into the appropriate context queue ncclIntruQueueEnqueue(&planner->rmaTaskQueues[t->ctx], t); } } return ncclSuccess; } // Converts `info` to a task and adds it to `comm->planner`. The exception is with // single rank communicators, collectives are issued as `ncclMemcpyAsync`s and // thus don't need a task. static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) { ncclFunc_t collAPI = info->coll; if (info->coll == ncclFuncSend || info->coll == ncclFuncRecv) { NCCLCHECK(p2pTaskAppend(comm, info, info->coll, collAPI, (void*)info->recvbuff, info->count, info->datatype, info->root, true)); } else if (info->coll == ncclFuncPutSignal || info->coll == ncclFuncSignal || info->coll == ncclFuncWaitSignal) { NCCLCHECK(rmaTaskAppend(comm, info)); } else { // Empty collectives can be discarded. if (info->count == 0) return ncclSuccess; if (info->datatype == ncclFloat8e4m3 || info->datatype == ncclFloat8e5m2) { if (comm->minCompCap < 90 && info->coll != ncclFuncAllGather && info->coll != ncclFuncBroadcast && info->coll != ncclFuncAlltoAll && info->coll != ncclFuncScatter && info->coll != ncclFuncGather) { WARN("FP8 reduction support begins with sm90 capable devices."); return ncclInvalidArgument; } } // Copy reduction op state from op handle into info struct here since the // op handle may be destroyed before ncclGroupEnd(). struct ncclDevRedOpFull opDev; NCCLCHECK(hostToDevRedOp(&opDev, info->op, info->datatype, comm)); if (comm->nRanks == 1) { NCCLCHECK(ncclLaunchOneRank(info->recvbuff, info->sendbuff, info->count, opDev, info->datatype, info->stream)); return ncclSuccess; } else { struct ncclDevrWindow* sendWin; struct ncclDevrWindow* recvWin; ncclDevrFindWindow(comm, info->sendbuff, &sendWin); ncclDevrFindWindow(comm, info->recvbuff, &recvWin); // Append CE collective task if CE is supported and requested by user ncclSymRegType_t winRegType; NCCLCHECK(ncclGetSymRegType(sendWin, recvWin, &winRegType)); bool ceAvailable = ncclCeAvailable(comm, info->coll, info->op, info->datatype, winRegType); if ((comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO) && ceAvailable) { NCCLCHECK(ceCollTaskAppend(comm, info, sendWin, recvWin, opDev)); } // Append kernel-based collective else { // currently legacy sendrecv needs src and dst buffers to be registered // we cannot allow UB if alltoall/scatter/gather fallback to legacy sendrecv // when src or dst buffers are not registered struct ncclReg* sendReg = NULL; struct ncclReg* recvReg = NULL; bool allowUB = false; bool captured = false; struct ncclCudaGraph graph; // For cuda graph checking NCCLCHECK(ncclCudaGetCapturingGraph(&graph, info->stream, comm->config.graphUsageMode)); captured = ncclCudaGraphValid(graph); if (info->coll == ncclFuncAlltoAll) { NCCLCHECK(ncclRegFind(comm, info->sendbuff, comm->nRanks * info->count * ncclTypeSize(info->datatype), &sendReg)); NCCLCHECK(ncclRegFind(comm, info->recvbuff, comm->nRanks * info->count * ncclTypeSize(info->datatype), &recvReg)); allowUB = captured || (sendReg != NULL && recvReg != NULL); for (int r=0; rnRanks; r++) { NCCLCHECK(p2pTaskAppend(comm, info, ncclFuncSend, collAPI, (void*)((char*)info->sendbuff+r*info->count*ncclTypeSize(info->datatype)), info->count, info->datatype, r, allowUB)); NCCLCHECK(p2pTaskAppend(comm, info, ncclFuncRecv, collAPI, (void*)((char*)info->recvbuff+r*info->count*ncclTypeSize(info->datatype)), info->count, info->datatype, r, allowUB)); } } else if (info->coll == ncclFuncGather){ size_t offset = 0; allowUB = captured; NCCLCHECK(p2pTaskAppend(comm, info, ncclFuncSend, collAPI, (void*)info->sendbuff, info->count, info->datatype, info->root, allowUB)); if (comm->rank == info->root) { for (int r=0; rnRanks; r++) { void* buff = (void*)((char*)info->recvbuff + offset); NCCLCHECK(p2pTaskAppend(comm, info, ncclFuncRecv, collAPI, buff, info->count, info->datatype, r, allowUB)); offset += info->count * ncclTypeSize(info->datatype); } } } else if (info->coll == ncclFuncScatter) { size_t offset = 0; allowUB = captured; if (comm->rank == info->root) { for (int r = 0; r < comm->nRanks; r++) { void* buff = (void*)((char*)info->sendbuff + offset); NCCLCHECK(p2pTaskAppend(comm, info, ncclFuncSend, collAPI, buff, info->count, info->datatype, r, allowUB)); offset += info->count * ncclTypeSize(info->datatype); } } NCCLCHECK(p2pTaskAppend(comm, info, ncclFuncRecv, collAPI, (void*)info->recvbuff, info->count, info->datatype, info->root, allowUB)); } else if (ceAvailable && comm->symmetricSupport && info->coll == ncclFuncAllGather && info->count > ncclParamSymCeThreshold() && comm->minCompCap >= 100 && comm->isAllDirectNvlink) { // Use CE for Allgather on Blackwell with size > 8MB NCCLCHECK(ceCollTaskAppend(comm, info, sendWin, recvWin, opDev)); } else { NCCLCHECK(collTaskAppend(comm, info, opDev)); } } } } return ncclSuccess; } ncclResult_t ncclEnqueueCheck(struct ncclInfo* info) { // Early-out on invalid or revoked communicator ncclResult_t ret = CommCheck(info->comm, info->opName, "comm"); if (ret != ncclSuccess) return ncclGroupErrCheck(ret); if (info->comm->revokedFlag) { WARN("%s: communicator was revoked", info->opName); return ncclGroupErrCheck(ncclInvalidUsage); } // Profiler - If a group API event has already started, update the profilerGroupDepth so that the depth // updates correctly for implicit ncclGroupStartInternal and ncclGroupEndInternal calls if (ncclProfilerApiState.profilerGroupDepth > 0) { ncclProfilerApiState.profilerGroupDepth++; } NCCLCHECK(ncclGroupStartInternal()); ret = ncclSuccess; int devOld = -1; // Check whether communicator is ready to communicate NCCLCHECKGOTO(ncclCommEnsureReady(info->comm), ret, fail); if (info->comm->checkMode != ncclCheckModeDefault) { CUDACHECKGOTO(cudaGetDevice(&devOld), ret, fail); CUDACHECKGOTO(cudaSetDevice(info->comm->cudaDev), ret, fail); } // If info->comm->checkMode == ncclCheckModeDebugGlobal, ArgsCheck will enqueue info // for collectives and the pairs of peers for sendrecv for global check later NCCLCHECKGOTO(ArgsCheck(info), ret, fail); INFO(NCCL_COLL,"%s: opCount %lx sendbuff %p recvbuff %p count %zu datatype %d op %d root %d comm %p [nranks=%d] stream %p", info->opName, info->comm->opCount, info->sendbuff, info->recvbuff, info->count, info->datatype, info->op, info->root, info->comm, info->comm->nRanks, info->stream); TRACE_CALL("nccl%s(%" PRIx64 ",%" PRIx64 ",%zu,%d,%d,%d,%p,%p)", info->opName, reinterpret_cast(info->sendbuff), reinterpret_cast(info->recvbuff), info->count, info->datatype, info->op, info->root, info->comm, info->stream); NCCLCHECKGOTO(taskAppend(info->comm, info), ret, fail); exit: if (devOld != -1) CUDACHECK(cudaSetDevice(devOld)); ncclGroupErrCheck(ret); NCCLCHECK(ncclGroupEndInternal()); /* if depth is 1, ncclGroupEndInternal() will trigger group ops. The state can change * so we have to check state here. */ if (info->comm && !info->comm->config.blocking) { NCCLCHECK(ncclCommGetAsyncError(info->comm, &ret)); } return ret; fail: if (info->comm && !info->comm->config.blocking) (void) ncclCommSetAsyncError(info->comm, ret); goto exit; } NCCL_API(ncclResult_t, ncclRedOpCreatePreMulSum, ncclRedOp_t *op, void *scalar, ncclDataType_t datatype, ncclScalarResidence_t residence, ncclComm_t comm); ncclResult_t ncclRedOpCreatePreMulSum(ncclRedOp_t *op, void *scalar, ncclDataType_t datatype, ncclScalarResidence_t residence, ncclComm_t comm) { NCCLCHECK(CommCheck(comm, "ncclRedOpCreatePreMulSum", "comm")); /* join init thread before creating PreMulSum op. */ NCCLCHECK(ncclCommEnsureReady(comm)); if (comm->userRedOpFreeHead == comm->userRedOpCapacity) { // double capacity and resize int cap = 2*comm->userRedOpCapacity; if (cap < 4) cap = 4; ncclUserRedOp *ops = new ncclUserRedOp[cap]; if (comm->userRedOpCapacity > 0) std::memcpy(ops, comm->userRedOps, comm->userRedOpCapacity*sizeof(ncclUserRedOp)); for(int ix=comm->userRedOpCapacity; ix < cap; ix++) ops[ix].freeNext = ix + 1; delete[] comm->userRedOps; comm->userRedOps = ops; comm->userRedOpCapacity = cap; } // pop from free list int ix = comm->userRedOpFreeHead; ncclUserRedOp *user = &comm->userRedOps[ix]; comm->userRedOpFreeHead = user->freeNext; user->freeNext = -1; // allocated user->datatype = datatype; user->opFull.op = ncclDevPreMulSum; if (residence == ncclScalarHostImmediate) { int size = ncclTypeSize(datatype); if (size < 1) return ncclInternalError; user->opFull.scalarArgIsPtr = false; std::memcpy(&user->opFull.scalarArg, scalar, size); } else { user->opFull.scalarArgIsPtr = true; user->opFull.scalarArg = reinterpret_cast(scalar); } *op = ncclRedOp_t(int(ncclNumOps) + ix); *op = ncclUserRedOpMangle(comm, *op); TRACE_CALL("ncclRedOpCreatePreMulSum(%d,%p,%d,%d,%p)", *op, scalar, datatype, residence, comm); return ncclSuccess; } NCCL_API(ncclResult_t, ncclRedOpDestroy, ncclRedOp_t op, ncclComm_t comm); ncclResult_t ncclRedOpDestroy(ncclRedOp_t op, ncclComm_t comm) { if (0 <= int(op) && int(op) < int(ncclNumOps)) { WARN("ncclRedOpDestroy : operator is a NCCL builtin."); return ncclInvalidArgument; } // int(ncclMaxRedOp) < int(op) will always be false due to the sizes of // the datatypes involved, and that's by design. We keep the check though // just as a reminder. // coverity[result_independent_of_operands] if (int(op) < 0 || int(ncclMaxRedOp) < int(op)) { WARN("ncclRedOpDestroy : operator is garbage."); return ncclInvalidArgument; } if (comm == NULL) { WARN("ncclRedOpDestroy : invalid communicator passed."); return ncclInvalidArgument; } int ix = int(ncclUserRedOpMangle(comm, op)) - int(ncclNumOps); if (comm->userRedOpCapacity <= ix || comm->userRedOps[ix].freeNext != -1) { WARN("ncclRedOpDestroy : operator unknown to this communicator."); return ncclInvalidArgument; } // push to free list comm->userRedOps[ix].freeNext = comm->userRedOpFreeHead; comm->userRedOpFreeHead = ix; TRACE_CALL("ncclRedOpDestroy(%d,%p)", op, comm); return ncclSuccess; } nccl-2.29.7-1/src/gin/000077500000000000000000000000001515037102200142305ustar00rootroot00000000000000nccl-2.29.7-1/src/gin/CMakeLists.txt000066400000000000000000000003161515037102200167700ustar00rootroot00000000000000# Gin sources set(GIN_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/gin_host.cc ${CMAKE_CURRENT_SOURCE_DIR}/gin_host_proxy.cc ) # Add gin sources to parent scope set(GIN_SOURCES ${GIN_SOURCES} PARENT_SCOPE) nccl-2.29.7-1/src/gin/gin_host.cc000066400000000000000000000352561515037102200163640ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "comm.h" #include "gin.h" #include "param.h" #include "graph.h" #include "transport.h" #include "register_inline.h" #include "gin/gin_host.h" #include "gin/gin_host_proxy.h" #include "compiler.h" #include NCCL_PARAM(GinEnable, "GIN_ENABLE", 1); NCCL_PARAM(GinSignalPoolSize, "GIN_SIGNAL_POOL_SIZE", 512 << 10); NCCL_PARAM(GinCounterPoolSize, "GIN_COUNTER_POOL_SIZE", 512 << 10); ncclResult_t getGlobalGinType(struct ncclComm* comm, ncclGinType_t* ginType) { if (comm == nullptr || ginType == nullptr) { return ncclInternalError; } if (comm->globalGinSupport != NCCL_GIN_CONNECTION_FULL) { *ginType = NCCL_GIN_TYPE_NONE; return ncclSuccess; } *ginType = comm->sharedRes->ginState.ginType; return ncclSuccess; } ncclResult_t getGlobalRailedGinType(struct ncclComm* comm, ncclGinType_t* ginType) { if (comm == nullptr || ginType == nullptr) { return ncclInternalError; } if (comm->globalGinSupport == NCCL_GIN_CONNECTION_NONE) { *ginType = NCCL_GIN_TYPE_NONE; return ncclSuccess; } *ginType = comm->sharedRes->ginState.ginType; return ncclSuccess; } ncclResult_t setLocalGinType(struct ncclComm* comm) { if (comm == nullptr || comm->sharedRes->ginState.ncclGin == nullptr) { return ncclInternalError; } ncclGinState& ginState = comm->sharedRes->ginState; ginState.ginType = NCCL_GIN_TYPE_NONE; if (!ncclParamGinEnable()) { return ncclSuccess; } ncclNetProperties_t props; NCCLCHECK(ginState.ncclGin->getProperties(0, &props)); if (props.netDeviceType == NCCL_NET_DEVICE_GIN_PROXY || props.netDeviceType == NCCL_NET_DEVICE_GIN_GDAKI) { // NOTE: The following cast is valid because ncclGinType_t variant values // should match NCCL_NET_DEVICE_GIN_* values from `enum ncclNetDeviceType`. ginState.ginType = static_cast(props.netDeviceType); return ncclSuccess; } WARN("Cannot get gin type: ncclGin is not null but net device type (%d) is not a gin type", props.netDeviceType); return ncclInternalError; } void* ncclGinProgress(struct ncclGinState* ginState_) { struct ncclGinState* ginState = (struct ncclGinState*)ginState_; while (1) { std::unique_lock lock(ginState->mutex); if (ginState->ginProgress == 1) { lock.unlock(); for (int n=0; nginCommCount; n++) { ncclResult_t ret; if (ginState->ginType == NCCL_GIN_TYPE_PROXY) { ret = ncclGinProxyProgress(ginState->ncclGin, ginState->ginCtx[n]); } else { ret = ginState->ncclGin->ginProgress(ginState->ginComms[n]); } if (ret != ncclSuccess) { COMPILER_ATOMIC_STORE(&ginState->asyncResult, ret, std::memory_order_release); INFO(NCCL_ALL,"%s:%d -> %d [GIN Progress Thread]", __FILE__, __LINE__, ret); ginState->ginProgress = -2; return NULL; } } std::this_thread::yield(); } else if (ginState->ginProgress == -1) { return NULL; } else if (ginState->ginProgress == 0) { ginState->cond.wait(lock); } else { INFO(NCCL_ALL,"%s:%d -> [GIN Progress Thread] state unknown %d", __FILE__, __LINE__, ginState->ginProgress); ginState->ginProgress = -2; return NULL; } } } NCCL_PARAM(GinNconnections, "GIN_NCONNECTIONS", -2); NCCL_PARAM(GinNcontexts, "GIN_NCONTEXTS", -1); ncclResult_t ncclGinConnectOnce(struct ncclComm* comm, ncclGinConnectionType_t requestedConnectionType, int reqGinContextCount, int reqGinQueueDepth) { struct ncclGinState* ginState = &comm->sharedRes->ginState; if (ginState->connected) return ncclSuccess; ncclResult_t ret = ncclSuccess; if (ncclParamGinEnable() == 0) { WARN("GIN is disabled."); return ncclInternalError; } // Load plugin if (ginState->ncclGin == NULL) { WARN("GIN not supported."); return ncclInvalidUsage; } ginState->ginConnectionType = requestedConnectionType; ginState->ginInstance = comm->ginContext; int ndev = 0; NCCLCHECK(ginState->ncclGin->devices(&ndev)); if (ndev <= 0) { WARN("No GIN-capable devices found."); return ncclInternalError; } if (!comm->symmetricSupport) { WARN("Communicator does not support symmetric memory!"); return ncclInternalError; } int nLocalGinDevs; int localGinDevs[NCCL_TOPO_MAX_NODES]; NCCLCHECK(ncclTopoGetLocalGinDevs(comm, localGinDevs, &nLocalGinDevs)); void** handles = NULL; char* allHandles = NULL; int* ginCommCountHandles = NULL; int nContextsTotal; int nContextsPerComm; if (reqGinQueueDepth == 0) reqGinQueueDepth = ginState->ginQueueDepth; ginState->ginQueueDepth = reqGinQueueDepth; NCCLCHECKGOTO(ncclCalloc(&ginCommCountHandles, comm->nRanks), ret, fail); ginState->ginCommCount = nLocalGinDevs; if (ginState->ginVersion == 11) { ginState->ginCommCount = reqGinContextCount; if (ncclParamGinNcontexts() > 0) ginState->ginCommCount = ncclParamGinNcontexts(); } if (ncclParamGinNconnections() != -2) ginState->ginCommCount = ncclParamGinNconnections(); ginState->ginCommCount = std::min(NCCL_GIN_MAX_CONNECTIONS, ginState->ginCommCount); ginCommCountHandles[comm->rank] = ginState->ginCommCount; NCCLCHECKGOTO(bootstrapAllGather(comm->bootstrap, ginCommCountHandles, sizeof(int)), ret, fail); for (int r = 0; r < comm->nRanks; r++) { ginState->ginCommCount = std::min(ginState->ginCommCount, ginCommCountHandles[r]); } nContextsTotal = ncclParamGinNcontexts(); if (nContextsTotal <= 0) { nContextsTotal = std::max(reqGinContextCount, NCCL_GIN_MAX_CONNECTIONS); } nContextsTotal = ROUNDUP(nContextsTotal, ginState->ginCommCount); if (ginState->ginVersion == 11) { nContextsTotal = ginState->ginCommCount; } nContextsPerComm = nContextsTotal / ginState->ginCommCount; ginState->ginContextCount = nContextsTotal; ginState->ctxFirstAvailable = 0; ginState->ctxLastExclusive = nContextsTotal; INFO(NCCL_INIT, "devCommCreate: %d Local NET, creating %d GIN connections with %d contexts each (%d contexts total requested)", nLocalGinDevs, ginState->ginCommCount, nContextsPerComm, reqGinContextCount); NCCLCHECKGOTO(ncclCalloc(&allHandles, (size_t)comm->nRanks * NCCL_NET_HANDLE_MAXSIZE), ret, fail); NCCLCHECKGOTO(ncclCalloc(&handles, comm->nRanks), ret, fail); int nGinRanks; int myGinRank; if (requestedConnectionType == NCCL_GIN_CONNECTION_FULL) { nGinRanks = comm->nRanks; myGinRank = comm->rank; for (int r = 0; r < nGinRanks; r++) { handles[r] = allHandles + r * NCCL_NET_HANDLE_MAXSIZE; } } else { ncclTeam_t railTeam = ncclTeamRail(comm); nGinRanks = railTeam.nRanks; myGinRank = railTeam.rank; for (int r = 0; r < nGinRanks; r++) { int worldRank = ncclTeamRankToWorld(comm, railTeam, r); handles[r] = allHandles + worldRank * NCCL_NET_HANDLE_MAXSIZE; } } // The upper limits below are connected to the sizes of signalId and counterId // in ncclGinProxyQword_t in gin_proxy_device_host_common.h. We need to keep // them in sync. ginState->signalSpaceSize = ncclParamGinSignalPoolSize(); if (ginState->signalSpaceSize < 0 || (1 << 24) <= ginState->signalSpaceSize) { INFO(NCCL_INIT|NCCL_ENV, "NCCL_GIN_SIGNAL_POOL_SIZE has an invalid value"); ginState->signalSpaceSize = 512 << 10; } ginState->counterSpaceSize = ncclParamGinCounterPoolSize(); if (ginState->counterSpaceSize < 0 || (1 << 23) <= ginState->counterSpaceSize) { INFO(NCCL_INIT|NCCL_ENV, "NCCL_GIN_COUNTER_POOL_SIZE has an invalid value"); ginState->counterSpaceSize = 512 << 10; } for (int n = 0; n < ginState->ginCommCount; n++) { void* listenComm; NCCLCHECKGOTO( ginState->ncclGin->listen(ginState->ginInstance, localGinDevs[n%nLocalGinDevs], allHandles + NCCL_NET_HANDLE_MAXSIZE * comm->rank, &listenComm), ret, fail); NCCLCHECKGOTO(bootstrapAllGather(comm->bootstrap, allHandles, NCCL_NET_HANDLE_MAXSIZE), ret, fail); if (ginState->ginType == NCCL_GIN_TYPE_PROXY) { NCCLCHECKGOTO(ginState->ncclGin->connect(comm->ginContext, handles, nGinRanks, myGinRank, nContextsPerComm, ginState->ginQueueDepth, listenComm, ginState->ginComms + n), ret, fail); NCCLCHECKGOTO(ncclGinProxyCreateContext(comm, ginState->ginComms[n], localGinDevs[n % nLocalGinDevs], ginState->signalSpaceSize, ginState->counterSpaceSize, nContextsPerComm, &ginState->ginCtx[n], &ginState->ginDevHandles[n]), ret, fail); } else { NCCLCHECKGOTO(ginState->ncclGin->connect( comm->ginContext, handles, nGinRanks, myGinRank, 1, ginState->ginQueueDepth, listenComm, ginState->ginComms + n), ret, fail); NCCLCHECKGOTO(ginState->ncclGin->createContext( ginState->ginComms[n], ginState->signalSpaceSize, ginState->counterSpaceSize, nContextsPerComm, &ginState->ginCtx[n], &ginState->ginDevHandles[n]), ret, fail); } NCCLCHECKGOTO(ginState->ncclGin->closeListen(listenComm), ret, fail); } free(handles); handles = NULL; free(allHandles); allHandles = NULL; free(ginCommCountHandles); ginCommCountHandles = NULL; // Check whether we need proxy progress and if so, start / wake up the progress thread. ginState->needsProxyProgress = 0; for (int n = 0; n < ginState->ginCommCount; n++) { if (ginState->ginDevHandles[n]->needsProxyProgress) ginState->needsProxyProgress = 1; } if (ginState->needsProxyProgress) { ginState->ginProgress = 1; ginState->thread = std::thread(ncclGinProgress, ginState); ncclSetThreadName(ginState->thread, "NCCL GIN Progress%2d", comm->cudaDev); } ncclSpaceConstruct(&ginState->counterSpace); ncclSpaceConstruct(&ginState->signalSpace); exit: if (ret == ncclSuccess) ginState->connected = true; return ret; fail: if (allHandles) free(allHandles); if (handles) free(handles); if (ginCommCountHandles) free(ginCommCountHandles); goto exit; } ncclResult_t ncclGinHostFinalize(struct ncclComm* comm) { struct ncclGinState* ginState = &comm->sharedRes->ginState; if (!ginState->connected) return ncclSuccess; if (ginState->needsProxyProgress) { { std::lock_guard lock(ginState->mutex); comm->sharedRes->ginState.ginProgress = -1; ginState->cond.notify_one(); } ginState->thread.join(); } if (ginState->ginType == NCCL_GIN_TYPE_PROXY) { for (int n = 0; n < ginState->ginCommCount; n++) { if (ginState->ginCtx[n] != NULL) { NCCLCHECK(ncclGinProxyDestroyContext(ginState->ncclGin, ginState->ginCtx[n])); ginState->ginCtx[n] = NULL; } } } for (int n = 0; n < ginState->ginCommCount; n++) { if (ginState->ginCtx[n] != NULL) { NCCLCHECK(ginState->ncclGin->destroyContext(ginState->ginCtx[n])); ginState->ginCtx[n] = NULL; } if (ginState->ginComms[n] != NULL) { NCCLCHECK(ginState->ncclGin->closeColl(ginState->ginComms[n])); ginState->ginComms[n] = NULL; } } memset((void*)ginState, 0, sizeof(*ginState)); return ncclSuccess; } ncclResult_t ncclGinRegister(struct ncclComm* comm, void* address, size_t size, void* ginHostWins[NCCL_GIN_MAX_CONNECTIONS], ncclGinWindow_t ginDevWins[NCCL_GIN_MAX_CONNECTIONS], int winFlags) { struct ncclGinState* ginState = &comm->sharedRes->ginState; int mrFlags = (winFlags & NCCL_WIN_STRICT_ORDERING) ? NCCL_NET_MR_FLAG_FORCE_SO : 0; for (int n = 0; n < ginState->ginCommCount; n++) { if (ginState->ginType == NCCL_GIN_TYPE_PROXY) { NCCLCHECK(ncclGinProxyRegister(ginState->ncclGin, ginState->ginCtx[n], address, size, NCCL_PTR_CUDA, mrFlags, &ginHostWins[n], &ginDevWins[n])); } else { NCCLCHECK(ginState->ncclGin->regMrSym(ginState->ginComms[n], address, size, NCCL_PTR_CUDA, mrFlags, &ginHostWins[n], &ginDevWins[n])); } if (ginHostWins[n] == NULL) { WARN("rank %d - GIN Symmetric register failed: buff %p, size %ld", comm->rank, address, size); return ncclSystemError; } } return ncclSuccess; } ncclResult_t ncclGinDeregister(struct ncclComm* comm, void* ginHostWins[NCCL_GIN_MAX_CONNECTIONS]) { struct ncclGinState* ginState = &comm->sharedRes->ginState; for (int n = 0; n < ginState->ginCommCount; n++) { if (ginState->ginType == NCCL_GIN_TYPE_PROXY) { NCCLCHECK(ncclGinProxyDeregister(ginState->ncclGin, ginState->ginCtx[n], ginHostWins[n])); } else { NCCLCHECK(ginState->ncclGin->deregMrSym(ginState->ginComms[n], ginHostWins[n])); } } return ncclSuccess; } ncclResult_t ncclGinAllocSignalsCounters(struct ncclComm* comm, int nSignals, uint32_t* outSignal0, int nCounters, uint32_t* outCounter0) { ncclResult_t ret = ncclSuccess; struct ncclGinState* ginState = &comm->sharedRes->ginState; int64_t start; if (nSignals != 0) { NCCLCHECKGOTO( ncclSpaceAlloc(&ginState->signalSpace, ginState->signalSpaceSize, nSignals, 1, &start), ret, fail); *outSignal0 = (uint32_t)start; } if (nCounters != 0) { NCCLCHECKGOTO( ncclSpaceAlloc(&ginState->counterSpace, ginState->counterSpaceSize, nCounters, 1, &start), ret, fail_signals); *outCounter0 = (uint32_t)start; } return ncclSuccess; fail_signals: if (nSignals != 0) ncclSpaceFree(&ginState->signalSpace, *outSignal0, nSignals); fail: return ret; } ncclResult_t ncclGinFreeSignalsCounters(struct ncclComm* comm, uint32_t signal0, int nSignals, uint32_t counter0, int nCounters) { struct ncclGinState* ginState = &comm->sharedRes->ginState; if (nSignals != 0) ncclSpaceFree(&ginState->signalSpace, signal0, nSignals); if (nCounters != 0) ncclSpaceFree(&ginState->counterSpace, counter0, nCounters); return ncclSuccess; } ncclResult_t ncclGinQueryLastError(struct ncclGinState* ginState, bool* hasError) { bool hasError_ = false; for (int n = 0; n < ginState->ginCommCount; n++) { if (ginState->ginType == NCCL_GIN_TYPE_PROXY) NCCLCHECK(ncclGinProxyQueryLastError(ginState->ncclGin, ginState->ginCtx[n], &hasError_)); else NCCLCHECK(ginState->ncclGin->queryLastError(ginState->ginCtx[n], &hasError_)); if (hasError_) break; } *hasError = hasError_; return ncclSuccess; } nccl-2.29.7-1/src/gin/gin_host_proxy.cc000066400000000000000000000532151515037102200176200ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include #include "nccl.h" #include "comm.h" #include "gin/gin_host.h" #include "alloc.h" #include "checks.h" #include "gdrwrap.h" #include "nccl_device/gin/proxy/gin_proxy_device_host_common.h" #include "compiler.h" NCCL_PARAM(GinProxyQueueSize, "GIN_PROXY_QUEUE_SIZE", -1); extern int64_t ncclParamIbDataDirect(); extern int64_t ncclParamDmaBufEnable(); struct ginProxyGfdState { ncclGinProxyOp_t op; uint16_t counterId; int done; void *request; }; // a member might be on the GPU, if it has a *GdrHandle counterpart struct ginProxyHostGpuCtx { int contextId; size_t queueSize; // size = nRanks * queueSize ncclGinProxyGfd_t *queues; void *cisGdrHandle; // Consumed Indices, one per rank uint32_t *cis; // to decrease the number of reads/writes to cis which might be on the GPU uint32_t *cisShadow; // Seen Indices one per rank uint32_t *sis; // same size as queues struct ginProxyGfdState *states; // same size as queues uint64_t *inlines; // inlines is registered as a memory region with the GIN plugin void *inlinesMhandle; void *inlinesGinHandle; }; struct ginProxyCtx { struct ncclComm *comm; void *collComm; ncclNetDeviceHandle_t *devHandle; ncclNetProperties_t props; // GPU queues, if GDR on the GPU, else on the CPU // Queue size, must be a power of 2 struct ginProxyHostGpuCtx *hostGpuCtx; void *countersGdrHandle; uint64_t *counters; uint64_t *countersDev; CUmemGenericAllocationHandle signalsCumemhandle; void *signalsMhandle; void *signalsGinHandle; uint64_t *signalsDev; int hasError; int nContexts; int nCountersPerContext; int nSignalsPerContext; }; static ncclResult_t getDmaBufFd(void *addr, size_t length, int *fd, bool forceNonDataDirect = false) { if (ncclParamDmaBufEnable() == 0) return ncclInvalidUsage; #if CUDA_VERSION >= 11070 static size_t hostPageSize = ncclOsGetPageSize(); size_t alignedSize = length; ALIGN_SIZE(alignedSize, hostPageSize); #if CUDA_VERSION >= 12080 if (ncclParamIbDataDirect() && !forceNonDataDirect) { CUresult status = pfn_cuMemGetHandleForAddressRange( (void *)fd, (CUdeviceptr)addr, alignedSize, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, CU_MEM_RANGE_FLAG_DMA_BUF_MAPPING_TYPE_PCIE); if (status == CUDA_SUCCESS) return ncclSuccess; } #endif CUresult status = pfn_cuMemGetHandleForAddressRange((void *)fd, (CUdeviceptr)addr, alignedSize, CU_MEM_RANGE_HANDLE_TYPE_DMA_BUF_FD, 0); if (status == CUDA_SUCCESS) return ncclSuccess; #endif return ncclInvalidUsage; } static ncclResult_t proxyGinPollCompletions(ncclGin_t *ginComm, void *collComm, struct ginProxyCtx *ctx, struct ginProxyHostGpuCtx *hostGpuCtx) { for (int targetRank = 0; targetRank < ctx->comm->nRanks; targetRank++) { // loop on all seen but unconsumed GFDs for (uint32_t i = hostGpuCtx->cisShadow[targetRank]; i < hostGpuCtx->sis[targetRank]; i++) { uint32_t idx = i & (hostGpuCtx->queueSize - 1); struct ginProxyGfdState *state = &hostGpuCtx->states[targetRank * hostGpuCtx->queueSize + idx]; // no need to poll if already done if (!state->done) { ginComm->test(collComm, state->request, &state->done); if (state->done) { TRACE(NCCL_NET, "GFD completed - contextId: %d, stateIdx: %lu, request: %p", hostGpuCtx->contextId, state - hostGpuCtx->states, state->request); // update the counter specified in the GFD if (state->op & ncclGinProxyOpWithCounter) { int contextId = hostGpuCtx->contextId; uint64_t* counterPtr = &ctx->counters[contextId * ctx->nCountersPerContext + state->counterId]; COMPILER_ATOMIC_STORE(counterPtr, *counterPtr + 1, std::memory_order_relaxed); TRACE(NCCL_NET, "Updated counter %d to %ld for context %d", state->counterId, *counterPtr, contextId); } } } // allow holes in the CI space to get resolved if (state->done && i == hostGpuCtx->cisShadow[targetRank]) { // tell the GPU that we have consumed the GFD COMPILER_ATOMIC_STORE(&hostGpuCtx->cis[targetRank], ++hostGpuCtx->cisShadow[targetRank], std::memory_order_relaxed); TRACE(NCCL_NET, "Updated cis[%u] to %u for context %d", targetRank, hostGpuCtx->cisShadow[targetRank], hostGpuCtx->contextId); } } } return ncclSuccess; } static int proxyGinPollGfd(struct ginProxyCtx *ctx, ginProxyHostGpuCtx *hostGpuCtx, int targetRank, ncclGinProxyGfd_t *gfd, struct ginProxyGfdState **state) { ncclGinProxyGfd_t *q = hostGpuCtx->queues + targetRank * hostGpuCtx->queueSize; uint32_t idx = hostGpuCtx->sis[targetRank] & (hostGpuCtx->queueSize - 1); ncclGinProxyQword_t qword; COMPILER_ATOMIC_LOAD_DEST(&q[idx].qword[ncclGinProxyGfdHeader].raw, &qword.raw, std::memory_order_relaxed); if (qword.flag.v == 0) { return 0; } // We know for sure that the first qword is there, copy it. gfd->qword[ncclGinProxyGfdHeader] = q[idx].qword[ncclGinProxyGfdHeader]; // Wait for and copy the other qwords. for (int k = 1; k < ncclGinProxyGfdQwords; k++) { do { COMPILER_ATOMIC_LOAD_DEST(&q[idx].qword[k].raw, &qword.raw, std::memory_order_relaxed); } while (qword.flag.v == 0); gfd->qword[k] = qword; } // Now we have the full GFD in the local struct. // Reset the GFD in the queue. This ensures that the proxy doesn't try to process the GFD again. for (int k = 0; k < ncclGinProxyGfdQwords; k++) { COMPILER_ATOMIC_STORE(&q[idx].qword[k].raw, 0, std::memory_order_relaxed); } // set the counter_id into the state uint32_t stateIdx = targetRank * hostGpuCtx->queueSize + idx; *state = &hostGpuCtx->states[stateIdx]; (*state)->op = (ncclGinProxyOp_t)(gfd->qword[ncclGinProxyGfdHeader].header.op); (*state)->counterId = gfd->qword[ncclGinProxyGfdCompletion].completion.counterId; (*state)->done = 0; (*state)->request = NULL; TRACE(NCCL_NET, "GFD on context %d to target PE %d raw idx: %u, idx: %u - op: %#lx, size: %lu, srcOff: %lu, dstOff: %lu, " "srcHandle: %lu, dstHandle: %lu, counterId: %u, signalId: %u, stateIdx: %u", hostGpuCtx->contextId, targetRank, hostGpuCtx->sis[targetRank], idx, gfd->qword[ncclGinProxyGfdHeader].header.op, gfd->qword[ncclGinProxyGfdHeader].header.size, gfd->qword[ncclGinProxyGfdSrcOff].srcOff.srcOff, gfd->qword[ncclGinProxyGfdDstOff].dstOff.dstOff, gfd->qword[ncclGinProxyGfdSrcHandle].srcHandle.srcHandle, gfd->qword[ncclGinProxyGfdDstHandle].dstHandle.dstHandle, gfd->qword[ncclGinProxyGfdCompletion].completion.counterId, gfd->qword[ncclGinProxyGfdCompletion].completion.signalId, stateIdx); hostGpuCtx->sis[targetRank]++; return 1; } static int mapGfdOpToSignalOp(ncclGinProxyGfd_t *gfd) { uint8_t op = gfd->qword[ncclGinProxyGfdHeader].header.op; uint8_t signalOp = op & (ncclGinProxyOpWithSignalInc | ncclGinProxyOpWithSignalAdd); switch (signalOp) { case ncclGinProxyOpWithSignalInc: return NCCL_NET_SIGNAL_OP_INC; case ncclGinProxyOpWithSignalAdd: return NCCL_NET_SIGNAL_OP_ADD; default: return -1; } } static inline uint64_t extractSignalVal(ncclGinProxyGfd_t *gfd) { uint64_t signalVal = gfd->qword[ncclGinProxyGfdCompletion].completion.signalValLow; signalVal |= (uint64_t)gfd->qword[ncclGinProxyGfdSignalVal].signalVal.signalValLow2 << 16; signalVal |= (uint64_t)gfd->qword[ncclGinProxyGfdSignalVal].signalVal.signalValHigh << 32; return signalVal; } static ncclResult_t proxyGinProcessGfd(ncclGin_t *ginComm, void *collComm, struct ginProxyCtx *ctx, struct ginProxyHostGpuCtx *hostGpuCtx, int targetRank, ncclGinProxyGfd_t *gfd, struct ginProxyGfdState *state) { int signalOp; uint64_t signalVal; // Handle VA Signal operations (signal-only, no PUT) if (gfd->qword[ncclGinProxyGfdHeader].header.op & ncclGinProxyOpVASignal) { uint64_t signalOff = gfd->qword[ncclGinProxyGfdVASignalOff].vaSignalOff.vaSignalOff; void *signalHandle = (void *)(uint64_t)gfd->qword[ncclGinProxyGfdVASignalHandle].vaSignalHandle.vaSignalHandle; signalVal = extractSignalVal(gfd); signalOp = mapGfdOpToSignalOp(gfd); NCCLCHECK(ginComm->iputSignal(collComm, 0, nullptr, 0, 0, nullptr, targetRank, signalOff, signalHandle, signalVal, signalOp, hostGpuCtx->contextId, &state->request)); return ncclSuccess; } uint64_t size = gfd->qword[ncclGinProxyGfdHeader].header.size; uint64_t srcOff; void *srcHandle; if (gfd->qword[ncclGinProxyGfdHeader].header.op & ncclGinProxyOpWithInline) { uint64_t *inlineVal = &hostGpuCtx->inlines[state - hostGpuCtx->states]; srcOff = (uint64_t)&inlineVal[0] - (uint64_t)hostGpuCtx->inlines; // reconstruct the inline value from the two qwords *inlineVal = gfd->qword[ncclGinProxyGfdInlineLow].inlineLow.inlineValLow; if (size > 4) *inlineVal |= (uint64_t)gfd->qword[ncclGinProxyGfdInlineLow].inlineLow.inlineValLow2 << 32; if (size > 6) *inlineVal |= (uint64_t)gfd->qword[ncclGinProxyGfdInlineHigh].inlineHigh.inlineValHigh << 48; srcHandle = hostGpuCtx->inlinesMhandle; } else { srcOff = gfd->qword[ncclGinProxyGfdSrcOff].srcOff.srcOff; srcHandle = (void *)(uint64_t)gfd->qword[ncclGinProxyGfdSrcHandle].srcHandle.srcHandle; } uint64_t dstOff = gfd->qword[ncclGinProxyGfdDstOff].dstOff.dstOff; void *dstHandle = (void *)(uint64_t)gfd->qword[ncclGinProxyGfdDstHandle].dstHandle.dstHandle; switch (gfd->qword[ncclGinProxyGfdHeader].header.op & ncclGinProxyOpBaseMask) { case ncclGinProxyOpPut: signalOp = mapGfdOpToSignalOp(gfd); if (signalOp == -1) { // First cast from 63 bits to 64 bits and then to void * to avoid warnings NCCLCHECK(ginComm->iput(collComm, srcOff, srcHandle, size, dstOff, dstHandle, targetRank, hostGpuCtx->contextId, &state->request)); } else { // Reconstruct the signal value signalVal = extractSignalVal(gfd); uint64_t signalOff = (gfd->qword[ncclGinProxyGfdCompletion].completion.signalId + hostGpuCtx->contextId * ctx->nSignalsPerContext) * sizeof(uint64_t); NCCLCHECK(ginComm->iputSignal(collComm, srcOff, srcHandle, size, dstOff, dstHandle, targetRank, signalOff, ctx->signalsGinHandle, signalVal, signalOp, hostGpuCtx->contextId, &state->request)); } break; default: // this error should already have been checked in pollGfd assert(0); } TRACE(NCCL_NET, "GFD submitted into GIN plugin - contextId: %d, stateIdx: %lu, request: %p", hostGpuCtx->contextId, state - hostGpuCtx->states, state->request); return ncclSuccess; } static uint64_t isPowerOfTwo(uint64_t n) { return (n > 0) && ((n & (n - 1)) == 0); } // Check if the GIN plugin supports DMA-BUF, if so we can try to get the DMA-BUF handle from CUDA, // if that fails we fallback to non-DMA-BUF static ncclResult_t ncclGinProxyRegMrSym(ncclGin_t *ginComm, struct ginProxyCtx *ctx, void *addr, size_t size, int type, int mr_flags, void **mhandle, void **ginHandle) { if (type == NCCL_PTR_HOST) { NCCLCHECK(ginComm->regMrSym(ctx->collComm, addr, size, type, mr_flags, mhandle, ginHandle)); } else if (type == NCCL_PTR_CUDA) { ncclResult_t dmabufResult = ncclInvalidUsage; if (ncclParamDmaBufEnable() && (ctx->props.ptrSupport & NCCL_PTR_DMABUF)) { ncclResult_t registrationResult = ncclSuccess; int dmabufFd = -1; dmabufResult = getDmaBufFd(addr, size, &dmabufFd); if (dmabufResult == ncclSuccess) { registrationResult = ginComm->regMrSymDmaBuf(ctx->collComm, addr, size, type, 0, dmabufFd, mr_flags, mhandle, ginHandle); close(dmabufFd); } if (registrationResult != ncclSuccess) { dmabufFd = -1; dmabufResult = getDmaBufFd(addr, size, &dmabufFd, true); if (dmabufResult == ncclSuccess) { NCCLCHECK(ginComm->regMrSymDmaBuf(ctx->collComm, addr, size, type, 0, dmabufFd, mr_flags, mhandle, ginHandle)); close(dmabufFd); } } } // Fallback to non-DMA-BUF if the DMA-BUF handle is not supported if (dmabufResult != ncclSuccess) { NCCLCHECK(ginComm->regMrSym(ctx->collComm, addr, size, type, mr_flags, mhandle, ginHandle)); } } else { return ncclInvalidUsage; } return ncclSuccess; } ncclResult_t ncclGinProxyCreateContext(struct ncclComm *comm, void *collComm, int devId, int nSignals, int nCounters, int nContexts, void **outGinCtx, ncclNetDeviceHandle_t **outDevHandle) { ncclGin_t *ginComm = (ncclGin_t *)comm->sharedRes->ginState.ncclGin; ncclGinProxyGpuCtx_t *devGpuCtxArray_h = nullptr; if (!ncclGdrCopy) INFO(NCCL_NET, "GIN Proxy will not be using GDRCopy"); struct ginProxyCtx *proxyCtx = NULL; NCCLCHECK(ncclCalloc(&proxyCtx, 1)); proxyCtx->comm = comm; proxyCtx->collComm = collComm; proxyCtx->nContexts = nContexts; // Sanitize the queue size NCCLCHECK(ginComm->getProperties(devId, &proxyCtx->props)); uint64_t queueSize = ncclParamGinProxyQueueSize(); uint32_t maxRequests = NCCL_NET_MAX_REQUESTS * proxyCtx->props.maxRecvs; if (queueSize == -1) { queueSize = maxRequests; } if (queueSize > maxRequests) { INFO(NCCL_NET, "NCCL_GIN_PROXY_QUEUE_SIZE is greater than the maximum outstanding requests in the GIN " "plugin (%d), using the default/maximum value instead", maxRequests); queueSize = maxRequests; } if (queueSize < 1) { INFO(NCCL_NET, "NCCL_GIN_PROXY_QUEUE_SIZE is less than 1, using the default/maximum value instead"); queueSize = maxRequests; } if (!isPowerOfTwo(queueSize)) { INFO( NCCL_NET, "NCCL_GIN_PROXY_QUEUE_SIZE is not a power of two, using the default/maximum value instead"); queueSize = maxRequests; } // Allocate the counters on the GPU or CPU depending on GDR NCCLCHECK(allocMemCPUAccessible(&proxyCtx->counters, &proxyCtx->countersDev, nCounters * nContexts, CU_MEMHOSTALLOC_WRITECOMBINED, &proxyCtx->countersGdrHandle, comm->memManager)); proxyCtx->nCountersPerContext = nCounters; // Allocate the signals on the GPU and then register the memory region with the GIN plugin. // Enforcing strong ordering on the signals mr is vital to ensure ordering between puts and // signals. size_t signalsBufSize = nSignals * nContexts * sizeof(uint64_t); NCCLCHECK(ncclCuMemAlloc((void **)&proxyCtx->signalsDev, &proxyCtx->signalsCumemhandle, CU_MEM_HANDLE_TYPE_NONE, signalsBufSize, comm->memManager)); CUDACHECK(cudaMemset(proxyCtx->signalsDev, 0, signalsBufSize)); NCCLCHECK(ncclGinProxyRegMrSym(ginComm, proxyCtx, proxyCtx->signalsDev, signalsBufSize, NCCL_PTR_CUDA, NCCL_NET_MR_FLAG_FORCE_SO, &proxyCtx->signalsMhandle, &proxyCtx->signalsGinHandle)); proxyCtx->nSignalsPerContext = nSignals; NCCLCHECK(ncclCalloc(&proxyCtx->hostGpuCtx, nContexts)); NCCLCHECK(ncclCalloc(&devGpuCtxArray_h, nContexts)); for (int contextId = 0; contextId < nContexts; contextId++) { struct ginProxyHostGpuCtx *hostGpuCtx = proxyCtx->hostGpuCtx + contextId; hostGpuCtx->contextId = contextId; hostGpuCtx->queueSize = queueSize; size_t queuesLength = hostGpuCtx->queueSize * comm->nRanks; NCCLCHECK(ncclCalloc(&hostGpuCtx->states, queuesLength)); NCCLCHECK(ncclCalloc(&hostGpuCtx->cisShadow, comm->nRanks)); NCCLCHECK(ncclCalloc(&hostGpuCtx->sis, comm->nRanks)); NCCLCHECK(ncclCalloc(&hostGpuCtx->inlines, queuesLength)); NCCLCHECK(ncclGinProxyRegMrSym(ginComm, proxyCtx, hostGpuCtx->inlines, queuesLength * sizeof(uint64_t), NCCL_PTR_HOST, 0, &hostGpuCtx->inlinesMhandle, &hostGpuCtx->inlinesGinHandle)); ncclGinProxyGpuCtx_t *devGpuCtx_h = devGpuCtxArray_h + contextId; devGpuCtx_h->nranks = comm->nRanks; devGpuCtx_h->queueSize = hostGpuCtx->queueSize; devGpuCtx_h->counters = proxyCtx->countersDev + contextId * nCounters; devGpuCtx_h->signals = proxyCtx->signalsDev + contextId * nSignals; NCCLCHECK(ncclCudaCalloc(&devGpuCtx_h->pis, comm->nRanks, comm->memManager)); // Allocate the GFD queues, CIs, counters, signals and test/wait variables on the either the CPU // or GPU. NCCLCHECK(allocMemCPUAccessible(&hostGpuCtx->queues, &devGpuCtx_h->queues, queuesLength, 0, NULL, comm->memManager, true /*forceHost*/)); NCCLCHECK(allocMemCPUAccessible(&hostGpuCtx->cis, &devGpuCtx_h->cis, comm->nRanks, CU_MEMHOSTALLOC_WRITECOMBINED, &hostGpuCtx->cisGdrHandle, comm->memManager)); } ncclGinProxyGpuCtx_t *devGpuCtx_d = NULL; NCCLCHECK(ncclCudaCalloc(&devGpuCtx_d, nContexts, comm->memManager)); // Copy the proxy's devGpuCtx to the GPU NCCLCHECK(ncclCudaMemcpy(devGpuCtx_d, devGpuCtxArray_h, nContexts)); ncclNetDeviceHandle_t *devHandle = NULL; NCCLCHECK(ncclCalloc(&devHandle, 1)); devHandle->netDeviceType = NCCL_NET_DEVICE_GIN_PROXY; devHandle->netDeviceVersion = NCCL_GIN_PROXY_VERSION; devHandle->handle = (void *)devGpuCtx_d; devHandle->size = 0; devHandle->needsProxyProgress = 1; proxyCtx->devHandle = devHandle; *outDevHandle = devHandle; *outGinCtx = proxyCtx; free(devGpuCtxArray_h); return ncclSuccess; } ncclResult_t ncclGinProxyRegister(ncclGin_t *ginComm, void *ginCtx, void *addr, size_t size, int type, int mr_flags, void **mhandle, void **ginHandle) { struct ginProxyCtx *ctx = (struct ginProxyCtx *)ginCtx; // Register the memory region with the GIN plugin NCCLCHECK(ncclGinProxyRegMrSym(ginComm, ctx, addr, size, type, mr_flags, mhandle, ginHandle)); return ncclSuccess; } ncclResult_t ncclGinProxyDeregister(ncclGin_t *ginComm, void *ginCtx, void *mhandle) { struct ginProxyCtx *ctx = (struct ginProxyCtx *)ginCtx; // Deregister the memory region with the GIN plugin NCCLCHECK(ginComm->deregMrSym(ctx->collComm, mhandle)); return ncclSuccess; } ncclResult_t ncclGinProxyDestroyContext(ncclGin_t *ginComm, void *ginCtx) { if (!ginCtx) return ncclSuccess; struct ginProxyCtx *ctx = (struct ginProxyCtx *)ginCtx; // Free counters if (ctx) { if (ctx->counters || ctx->countersGdrHandle) freeMemCPUAccessible(ctx->counters, ctx->countersGdrHandle, ctx->comm->memManager); // Free signals if (ginComm && ctx->collComm && ctx->signalsMhandle) ginComm->deregMrSym(ctx->collComm, ctx->signalsMhandle); if (ctx->signalsDev) ncclCudaFree(ctx->signalsDev, ctx->comm->memManager); // Free hostGpuCtx and its allocations if (ctx->hostGpuCtx) { for (int contextId = 0; contextId < ctx->nContexts; contextId++) { struct ginProxyHostGpuCtx *hostGpuCtx = ctx->hostGpuCtx + contextId; if (hostGpuCtx->cisShadow) free(hostGpuCtx->cisShadow); if (hostGpuCtx->sis) free(hostGpuCtx->sis); if (hostGpuCtx->states) free(hostGpuCtx->states); if (hostGpuCtx->inlines) free(hostGpuCtx->inlines); if (ginComm && ctx->collComm && hostGpuCtx->inlinesMhandle) ginComm->deregMrSym(ctx->collComm, hostGpuCtx->inlinesMhandle); if (hostGpuCtx->queues) freeMemCPUAccessible(hostGpuCtx->queues, NULL, ctx->comm->memManager); if (hostGpuCtx->cis || hostGpuCtx->cisGdrHandle) freeMemCPUAccessible(hostGpuCtx->cis, hostGpuCtx->cisGdrHandle, ctx->comm->memManager); } free(ctx->hostGpuCtx); } ncclNetDeviceHandle_t *devHandle = (ncclNetDeviceHandle_t *)ctx->devHandle; if (devHandle) { if (devHandle->handle) ncclCudaFree((void *)devHandle->handle, ctx->comm->memManager); free(devHandle); } free(ctx); } return ncclSuccess; } ncclResult_t ncclGinProxyProgress(ncclGin_t *ginComm, void *ginCtx) { struct ginProxyCtx *ctx = (struct ginProxyCtx *)ginCtx; for (int contextId = 0; contextId < ctx->nContexts; contextId++) { struct ginProxyHostGpuCtx *hostGpuCtx = ctx->hostGpuCtx + contextId; NCCLCHECK(proxyGinPollCompletions(ginComm, ctx->collComm, ctx, hostGpuCtx)); for (int targetRank = 0; targetRank < ctx->comm->nRanks; targetRank++) { // Poll on the GFD queue ncclGinProxyGfd_t gfd; struct ginProxyGfdState *state = NULL; if (proxyGinPollGfd(ctx, hostGpuCtx, targetRank, &gfd, &state)) { ncclResult_t ret = proxyGinProcessGfd(ginComm, ctx->collComm, ctx, hostGpuCtx, targetRank, &gfd, state); if (ret) ctx->hasError = ret; NCCLCHECK(ret); } if (ginComm->ginProgress) ginComm->ginProgress(ctx->collComm); } } return ncclSuccess; } ncclResult_t ncclGinProxyQueryLastError(ncclGin_t *ginComm, void *ginCtx, bool *hasError) { struct ginProxyCtx *ctx = (struct ginProxyCtx *)ginCtx; *hasError = ctx->hasError; return ncclSuccess; } nccl-2.29.7-1/src/graph/000077500000000000000000000000001515037102200145545ustar00rootroot00000000000000nccl-2.29.7-1/src/graph/CMakeLists.txt000066400000000000000000000007031515037102200173140ustar00rootroot00000000000000# Graph sources set(GRAPH_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/topo.cc ${CMAKE_CURRENT_SOURCE_DIR}/tuning.cc ${CMAKE_CURRENT_SOURCE_DIR}/xml.cc ${CMAKE_CURRENT_SOURCE_DIR}/search.cc ${CMAKE_CURRENT_SOURCE_DIR}/paths.cc ${CMAKE_CURRENT_SOURCE_DIR}/connect.cc ${CMAKE_CURRENT_SOURCE_DIR}/rings.cc ${CMAKE_CURRENT_SOURCE_DIR}/trees.cc ) # Add graph sources to parent scope set(GRAPH_SOURCES ${GRAPH_SOURCES} PARENT_SCOPE) nccl-2.29.7-1/src/graph/connect.cc000066400000000000000000000534501515037102200165230ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "comm.h" #include "device.h" #include "graph.h" #include "transport.h" #include "trees.h" #include "rings.h" #include "topo.h" /******************************************************************/ /********************* Internode connection ***********************/ /******************************************************************/ ncclResult_t ncclTopoPreset(struct ncclComm* comm, struct ncclTopoGraph** graphs, struct ncclTopoRanks* topoRanks) { int rank = comm->rank; int localRanks = comm->topo->nodes[GPU].count; int nChannels = comm->nChannels; topoRanks->crossNicRing = graphs[NCCL_ALGO_RING]->crossNic; topoRanks->nvlsHeadNum = 0; for (int c=0; cchannels+c; channel->ring.prev = channel->ring.next = -1; channel->tree.up = -1; channel->collnetChain.up = -1; for (int i=0; itree.down[i] = -1; for (int i=0; icollnetChain.down[i] = -1; channel->collnetDirect.out = -1; channel->collnetDirect.headRank = -1; channel->collnetDirect.nHeads = 0; channel->collnetDirect.shift = 0; for (int i=0; icollnetDirect.heads[i] = -1; for (int i=0; icollnetDirect.up[i] = -1; for (int i=0; icollnetDirect.down[i] = -1; int* ringIntra = graphs[NCCL_ALGO_RING]->intra+c*localRanks; int* treeIntra = graphs[NCCL_ALGO_TREE]->intra+c*localRanks; int* collNetIntra = graphs[NCCL_ALGO_COLLNET_CHAIN]->intra+c*localRanks; for (int i=0; iringRecv[c] = ringIntra[0]; topoRanks->ringSend[c] = ringIntra[localRanks-1]; topoRanks->ringPrev[c] = (i == 0) ? -1 : ringIntra[i-1]; topoRanks->ringNext[c] = (i == localRanks-1) ? -1 : ringIntra[i+1]; } if (treeIntra[i] == rank) { int parentIndex = 0; int child0Index = graphs[NCCL_ALGO_TREE]->pattern == NCCL_TOPO_PATTERN_TREE ? 0 : 1; int child1Index = graphs[NCCL_ALGO_TREE]->pattern == NCCL_TOPO_PATTERN_SPLIT_TREE ? 1 : 0; topoRanks->treeToParent[c] = treeIntra[parentIndex]; topoRanks->treeToChild0[c] = treeIntra[child0Index]; topoRanks->treeToChild1[c] = treeIntra[child1Index]; channel->tree.up = i == 0 ? -1 : treeIntra[i-1]; channel->tree.down[0] = i == localRanks-1 ? -1 : treeIntra[i+1]; } if (collNetIntra[i] == rank) { channel->collnetChain.up = i == 0 ? comm->nRanks : collNetIntra[i-1]; channel->collnetChain.down[0] = i == localRanks-1 ? -1 : collNetIntra[i+1]; } } } // Duplicate channels trees struct ncclChannel* channel0 = comm->channels; struct ncclChannel* channel1 = channel0+nChannels; memcpy(channel1, channel0, nChannels*sizeof(struct ncclChannel)); // Get nvls heads and the number of heads. Duplicate head is not allowed. for (int c = 0; c < graphs[NCCL_ALGO_NVLS]->nChannels; ++c) { bool addHead = true; int* nvlsIntra = graphs[NCCL_ALGO_NVLS]->intra + c * localRanks; for (int dup = 0; dup < topoRanks->nvlsHeadNum; dup++) { if (topoRanks->nvlsHeads[dup] == nvlsIntra[0]) { addHead = false; break; } } if (addHead) { topoRanks->nvlsHeads[topoRanks->nvlsHeadNum++] = nvlsIntra[0]; } } memcpy(comm->nvlsHeads, topoRanks->nvlsHeads, sizeof(int) * topoRanks->nvlsHeadNum); return ncclSuccess; } static ncclResult_t connectRings(struct ncclComm* comm, int* ringRecv, int* ringSend, int* ringPrev, int* ringNext) { int nChannels = comm->nChannels; int nNodes = comm->nNodes; for (int c=0; cnNodes; int* send = ringSend+c*comm->nNodes; int* prev = ringPrev+c*comm->nRanks; int* next = ringNext+c*comm->nRanks; for (int n=0; nup = indexes[u]; return ncclSuccess; } static ncclResult_t setTreeDown(struct ncclTree* tree, int* indexes, int d) { if (d == -1) return ncclSuccess; int x = 0; while (x < NCCL_MAX_TREE_ARITY && tree->down[x] >= 0) x++; if (x == NCCL_MAX_TREE_ARITY) { WARN("Internal error : tree already has %d children (%d %d %d)", x, tree->down[0], tree->down[1], tree->down[2]); return ncclInternalError; } tree->down[x] = indexes[d]; return ncclSuccess; } static ncclResult_t connectTrees(struct ncclComm* comm, int* treeToParent, int* treeToChild0, int* treeToChild1, int* treePatterns) { const int nChannels = comm->nChannels, nNodes = comm->nNodes, node = comm->node; // Compute tree depth. Not an exact value but a good approximation in most // cases int depth = comm->nRanks/nNodes - 1 + log2i(nNodes); int t0u, t0d0, t0d1, t0ChildType = 0, t1u, t1d0, t1d1, t1ChildType = 0; int* ttp, *ttc0, *ttc1; NCCLCHECK(ncclGetDtree(nNodes, node, &t0u, &t0d0, &t0d1, &t0ChildType, &t1u, &t1d0, &t1d1, &t1ChildType)); for (int c=0; cchannels+c; struct ncclChannel* channel1 = channel0+nChannels; ttp = treeToParent+c*comm->nNodes; ttc0 = treeToChild0+c*comm->nNodes; ttc1 = treeToChild1+c*comm->nNodes; if (comm->rank == ttp[node]) { NCCLCHECK(setTreeUp(&channel0->tree, t0ChildType == 0 ? ttc0 : ttc1, t0u)); NCCLCHECK(setTreeUp(&channel1->tree, t1ChildType == 0 ? ttc0 : ttc1, t1u)); } if (comm->rank == ttc0[node]) { NCCLCHECK(setTreeDown(&channel0->tree, ttp, t0d0)); NCCLCHECK(setTreeDown(&channel1->tree, ttp, t1d0)); } if (comm->rank == ttc1[node]) { NCCLCHECK(setTreeDown(&channel0->tree, ttp, t0d1)); NCCLCHECK(setTreeDown(&channel1->tree, ttp, t1d1)); } if (comm->rank == ttp[node] || comm->rank == ttc0[node] || comm->rank == ttc1[node]) { INFO(NCCL_GRAPH, "Tree %d : %d -> %d -> %d/%d/%d", c, channel0->tree.up, comm->rank, channel0->tree.down[0], channel0->tree.down[1], channel0->tree.down[2]); INFO(NCCL_GRAPH, "Tree %d : %d -> %d -> %d/%d/%d", c+nChannels, channel1->tree.up, comm->rank, channel1->tree.down[0], channel1->tree.down[1], channel1->tree.down[2]); } channel0->tree.depth = channel1->tree.depth = depth; } return ncclSuccess; } static ncclResult_t connectCollNet(struct ncclComm* comm, struct ncclTopoGraph* collNetGraph) { int rank = comm->rank; int localRanks = comm->localRanks; int nHeads = 0; int *heads; NCCLCHECK(ncclCalloc(&heads, localRanks)); // Find all head ranks // Head index is always 0 for (int c=0; cnChannels; c++) { int* collNetIntra = collNetGraph->intra+c*localRanks; int head = collNetIntra[0]; for (int h=0; hnChannels; c++) { struct ncclChannel* channel = comm->channels+c; char line[1024]; sprintf(line, "CollNetDirect channel %d rank %d ", c, rank); int nDown = 0; for (int i=0; icollnetDirect.headRank = i; // Mark the index for deciding offset in the CUDA kernel channel->collnetDirect.out = comm->nRanks; // Set root of collnetDirect to id nranks int* collNetIntra = collNetGraph->intra+i*localRanks; sprintf(line+strlen(line), "down "); for (int r=0; rcollnetDirect.down[nDown++] = collNetIntra[r]; // connect to all peers sprintf(line+strlen(line), " %d ", collNetIntra[r]); } sprintf(line+strlen(line), "nDown %d ", nDown); break; } } // Connect to all heads int nUp = 0; sprintf(line+strlen(line), "up "); for (int h=0; hcollnetDirect.up[nUp++] = heads[h]; sprintf(line+strlen(line), " %d ", heads[h]); } sprintf(line+strlen(line), "heads "); { // heads[] is the list of heads ordered in head order startubg with self int h0 = (channel->collnetDirect.headRank == -1) ? 0 : channel->collnetDirect.headRank; for (int h1=0; h1 < nHeads; h1++) { int h = (h0+h1)%nHeads; channel->collnetDirect.heads[h1] = heads[h]; sprintf(line+strlen(line), " %d ", heads[h]); } } channel->collnetDirect.nHeads = nHeads; // nHeads should always be greater than 0. // coverity[divide_by_zero] channel->collnetDirect.shift = (rank%localRanks)%nHeads; // Shift by intraRank so that leaves don't send to same head simultaneously channel->collnetDirect.depth = (nUp == 0 && nDown == 0) ? 1 : 2; sprintf(line+strlen(line), "nUp %d nHeads %d ", nUp, nHeads); sprintf(line+strlen(line), "headRank %d out %d shift %d", channel->collnetDirect.headRank, channel->collnetDirect.out, channel->collnetDirect.shift); INFO(NCCL_GRAPH, "%s", line); } free(heads); return ncclSuccess; } static ncclResult_t connectNvls(struct ncclComm* comm, int* nvlsHeads, int nHeads) { int headRank = -1; if (nHeads == 0) { comm->nvlsChannels = 0; return ncclSuccess; } for (int h = 0; h < nHeads; h++) { if (nvlsHeads[h * comm->nNodes + comm->node] == comm->rank) headRank = h; } for (int c=0; cnvlsChannels; c++) { struct ncclChannel* channel = comm->channels+c; channel->nvls.nHeads = nHeads; for (int h=0; hnvls.up[h] = comm->nRanks+1+h; for (int h=nHeads; hnvls.up[h] = -1; channel->nvls.down = comm->nRanks+1+headRank; channel->nvls.out = -1; // NVLS+SHARP not yet implemented. channel->nvls.headRank = headRank; channel->nvls.treeUp = channel->nvls.treeDown[0] = channel->nvls.treeDown[1] = channel->nvls.treeDown[2] = -1; if (comm->config.collnetEnable && channel->nvls.headRank != -1) channel->nvls.out = comm->nRanks; } if (comm->nNodes == 1) return ncclSuccess; // Connect Trees int tree0Parent, tree0Child0, tree0Child1, tree1Parent, tree1Child0, tree1Child1; int pc0, pc1; // ignored NCCLCHECK(ncclGetDtree(comm->nNodes, comm->node, &tree0Parent, &tree0Child0, &tree0Child1, &pc0, &tree1Parent, &tree1Child0, &tree1Child1, &pc1)); int* heads = NULL; int treeUp[2] = { -1, -1 }; int treeDown0[2] = { -1, -1 }; int treeDown1[2] = { -1, -1 }; if (comm->node == 0) { for (int h=0; hnNodes; for (int n=0; nnNodes && n<20; n++) { sprintf(line+strlen(line), " %2d", heads[n]); } INFO(NCCL_INIT, "%s", line); } } // Find the heads where I'm the head rank and retain tree up/down for (int h=0; hnNodes; if (heads[comm->node] == comm->rank) { treeUp[0] = tree0Parent == -1 ? -1: heads[tree0Parent]; treeDown0[0] = tree0Child0 == -1 ? -1 : heads[tree0Child0]; treeDown1[0] = tree0Child1 == -1 ? -1 : heads[tree0Child1]; treeUp[1] = tree1Parent == -1 ? -1 : heads[tree1Parent]; treeDown0[1] = tree1Child0 == -1 ? -1 : heads[tree1Child0]; treeDown1[1] = tree1Child1 == -1 ? -1 : heads[tree1Child1]; break; } } // Set prev/next in all channels (NVLS compute channels work // orthogonally to NVLS search channels). for (int c=0; cnvlsChannels; c++) { struct ncclChannel* channel = comm->channels+c; channel->nvls.treeUp = treeUp[c%2]; channel->nvls.treeDown[0] = channel->nvls.down; int ix = 1; if (treeDown0[c%2] != -1) channel->nvls.treeDown[ix++] = treeDown0[c%2]; if (treeDown1[c%2] != -1) channel->nvls.treeDown[ix] = treeDown1[c%2]; } struct ncclNvls* nvls0 = &comm->channels[0].nvls; struct ncclNvls* nvls1 = &comm->channels[1].nvls; INFO(NCCL_GRAPH, "NVLS Trees : %d/%d/%d->%d->%d %d/%d/%d->%d->%d", nvls0->treeDown[0], nvls0->treeDown[1], nvls0->treeDown[2], comm->rank, nvls0->treeUp, nvls1->treeDown[0], nvls1->treeDown[1], nvls1->treeDown[2], comm->rank, nvls1->treeUp); return ncclSuccess; } // Legacy naming NCCL_PARAM(MinNrings, "MIN_NRINGS", -2); NCCL_PARAM(MaxNrings, "MAX_NRINGS", -2); // New naming NCCL_PARAM(MinNchannels, "MIN_NCHANNELS", -2); NCCL_PARAM(MaxNchannels, "MAX_NCHANNELS", -2); int ncclMinNchannels() { int minNchannels = 0; if (ncclParamMinNrings() != -2) minNchannels = ncclParamMinNrings(); if (ncclParamMinNchannels() != -2) minNchannels = ncclParamMinNchannels(); if (minNchannels > MAXCHANNELS) { INFO(NCCL_GRAPH|NCCL_ENV, "User asked for a minimum of %d channels, limiting to %d", minNchannels, MAXCHANNELS); minNchannels = MAXCHANNELS; } if (minNchannels < 0) minNchannels = 0; return minNchannels; } extern int64_t ncclParamWorkArgsBytes(); int ncclMaxNchannels() { int maxNchannels = MAXCHANNELS; if (ncclParamMaxNrings() != -2) maxNchannels = ncclParamMaxNrings(); if (ncclParamMaxNchannels() != -2) maxNchannels = ncclParamMaxNchannels(); maxNchannels = std::min(maxNchannels, ncclDevMaxChannelsForArgsBytes(ncclParamWorkArgsBytes())); if (maxNchannels > MAXCHANNELS) maxNchannels = MAXCHANNELS; if (maxNchannels < 1) { INFO(NCCL_GRAPH|NCCL_ENV, "User asked for a maximum of %d channels, setting it to 1", maxNchannels); maxNchannels = 1; } return maxNchannels; } static int copyChannels(struct ncclComm* comm, int start, int end, int* ringPrev, int* ringNext) { int nranks = comm->nRanks; int c; for (c=start; cchannels+c, comm->channels+c-start, sizeof(struct ncclChannel)); } return c; } void exchangeValues(int* v0, int* v1) { int tmp = *v1; *v1 = *v0; *v0 = tmp; } NCCL_PARAM(UnpackDoubleNChannels, "UNPACK_DOUBLE_NCHANNELS", 1); ncclResult_t ncclTopoPostset(struct ncclComm* comm, int* firstRanks, int* treePatterns, struct ncclTopoRanks** allTopoRanks, int* rings, struct ncclTopoGraph** graphs, struct ncclComm* parent) { // Gather data from all ranks ncclResult_t ret = ncclSuccess; int *ringRecv = NULL, *ringSend = NULL, *ringPrev = NULL, *ringNext = NULL, *treeToParent = NULL, *treeToChild0 = NULL, *treeToChild1 = NULL, *nvlsHeads = NULL; int nranks = comm->nRanks; int nNodes = comm->nNodes; int nChannels = comm->nChannels; int minHeadNum = INT_MAX; int shared = parent && parent->nvlsSupport && parent->shareResources; NCCLCHECK(ncclCalloc(&ringRecv, nNodes*MAXCHANNELS)); NCCLCHECKGOTO(ncclCalloc(&ringSend, nNodes*MAXCHANNELS), ret, fail); NCCLCHECKGOTO(ncclCalloc(&ringPrev, nranks*MAXCHANNELS), ret, fail); NCCLCHECKGOTO(ncclCalloc(&ringNext, nranks*MAXCHANNELS), ret, fail); NCCLCHECKGOTO(ncclCalloc(&treeToParent, nNodes*MAXCHANNELS), ret, fail); NCCLCHECKGOTO(ncclCalloc(&treeToChild0, nNodes*MAXCHANNELS), ret, fail); NCCLCHECKGOTO(ncclCalloc(&treeToChild1, nNodes*MAXCHANNELS), ret, fail); NCCLCHECKGOTO(ncclCalloc(&nvlsHeads, nNodes*MAXCHANNELS), ret, fail); // Alternate rings to avoid crossing rails. // CrossNic values could be not the same on all nodes as it depends on the number of net devs and the NVLink bandwidth. // Therefore, it's only done if the rank obtained a solution with crossNic=2. for (int r = 0; r < comm->nRanks; r++) { if (allTopoRanks[r]->crossNicRing == 2 && (nChannels % 2) == 0 && (comm->rankToNode[r] % 2) == 1) { // Exchange rings for (int c=0; cringRecv+c, allTopoRanks[r]->ringRecv+(c^1)); exchangeValues(allTopoRanks[r]->ringSend+c, allTopoRanks[r]->ringSend+(c^1)); exchangeValues(allTopoRanks[r]->ringPrev+c, allTopoRanks[r]->ringPrev+(c^1)); exchangeValues(allTopoRanks[r]->ringNext+c, allTopoRanks[r]->ringNext+(c^1)); } } } for (int c=0; cringRecv[c]; ringSend[c*nNodes+n] = allTopoRanks[r]->ringSend[c]; treeToParent[c*nNodes+n] = allTopoRanks[r]->treeToParent[c]; treeToChild0[c*nNodes+n] = allTopoRanks[r]->treeToChild0[c]; treeToChild1[c*nNodes+n] = allTopoRanks[r]->treeToChild1[c]; } for (int r=0; rringPrev[c]; ringNext[c*nranks+r] = allTopoRanks[r]->ringNext[c]; } } for (int n = 0; n < nNodes; n++) { int r = firstRanks[n]; if (minHeadNum > allTopoRanks[r]->nvlsHeadNum) minHeadNum = allTopoRanks[r]->nvlsHeadNum; } for (int c = 0; c < minHeadNum; c++) { for (int n = 0; n < nNodes; n++) { int r = firstRanks[n]; nvlsHeads[c * nNodes + n] = allTopoRanks[r]->nvlsHeads[c]; } } // Connect rings and trees. This should also duplicate the channels. NCCLCHECKGOTO(connectRings(comm, ringRecv, ringSend, ringPrev, ringNext), ret, fail); NCCLCHECKGOTO(connectTrees(comm, treeToParent, treeToChild0, treeToChild1, treePatterns), ret, fail); // Duplicate ringPrev/ringNext for ncclBuildRing memcpy(ringPrev+nChannels*nranks, ringPrev, nChannels*nranks*sizeof(int)); memcpy(ringNext+nChannels*nranks, ringNext, nChannels*nranks*sizeof(int)); // Set ring prev/next for my rank for (int c=0; cchannels+c; struct ncclChannel* channel1 = channel0+nChannels; channel0->ring.prev = channel1->ring.prev = ringPrev[c*nranks+comm->rank]; channel0->ring.next = channel1->ring.next = ringNext[c*nranks+comm->rank]; } // Duplication should be complete now nChannels = comm->nChannels = std::min(MAXCHANNELS,nChannels*2); // Setup CollNet if (comm->config.collnetEnable) { struct ncclTopoGraph* collNetChainGraph = graphs[NCCL_ALGO_COLLNET_CHAIN]; // Add more channels to saturate intra-node bandwidth, except the 1 PPN case if (collNetChainGraph->bwIntra > collNetChainGraph->bwInter && comm->nRanks > comm->nNodes) { int collNetNchannels = std::min(MAXCHANNELS, nChannels+nChannels/2); nChannels = comm->nChannels = copyChannels(comm, nChannels, collNetNchannels, ringPrev, ringNext); } for (int c = 0; c < comm->nChannels; c++) { comm->channels[c].collnetChain.depth = comm->nRanks/comm->nNodes; } if (comm->maxLocalRanks <= NCCL_MAX_DIRECT_ARITY+1) { NCCLCHECKGOTO(connectCollNet(comm, graphs[NCCL_ALGO_COLLNET_DIRECT]), ret, fail); } } // Use 4 compute channels per search channel to reach peak BW on <8 PPN if (comm->minCompCap >= 90 && comm->nNodes > 1 && graphs[NCCL_ALGO_RING]->bwIntra > 45.0 && nChannels < 16) { nChannels = comm->nChannels = copyChannels(comm, nChannels, 2*nChannels, ringPrev, ringNext); } // Double the number of channels when using unpack networking (greater than 1 node) // We won't automatically double past 16 channels, users can specify 32 if they want if (comm->netDeviceType == NCCL_NET_DEVICE_UNPACK && comm->nNodes > 1 && nChannels < 16 && ncclParamUnpackDoubleNChannels()) { nChannels = comm->nChannels = copyChannels(comm, nChannels, 2*nChannels, ringPrev, ringNext); } // Honor NCCL_MIN_NRINGS/NCCL_MAX_NRINGS. // We permit combining max, then min, to only use the first channels, then duplicate them. if (comm->sharedRes->owner != comm) { /* child comm #channels cannot exceed top parent #channels. */ nChannels = comm->nChannels = std::min(std::min(std::min(ncclMaxNchannels(), nChannels), comm->config.maxCTAs), comm->sharedRes->tpNChannels); nChannels = comm->nChannels = copyChannels(comm, nChannels, std::min(std::max(ncclMinNchannels(), comm->config.minCTAs), comm->sharedRes->tpNChannels), ringPrev, ringNext); } else { nChannels = comm->nChannels = std::min(std::min(ncclMaxNchannels(), nChannels), comm->config.maxCTAs); nChannels = comm->nChannels = copyChannels(comm, nChannels, std::max(ncclMinNchannels(), comm->config.minCTAs), ringPrev, ringNext); } comm->collChannels = comm->nChannels; #if CUDART_VERSION >= 12010 // Support maximal channel usage for aggregation if (shared && comm->nvlsChannels > parent->nvlsResources->nChannels) { comm->nvlsChannels = parent->nvlsResources->nChannels; } NCCLCHECKGOTO(connectNvls(comm, nvlsHeads, minHeadNum), ret, fail); #endif if (shared && comm->nChannels > parent->sharedRes->tpNChannels) { nChannels = comm->nChannels = parent->sharedRes->tpNChannels; comm->collChannels = std::min(comm->collChannels, comm->nChannels); } // Create rings array and check all is fine NCCLCHECKGOTO(ncclBuildRings(nChannels, rings, comm->rank, comm->nRanks, ringPrev, ringNext), ret, fail); exit: if (ringRecv) free(ringRecv); if (ringSend) free(ringSend); if (ringPrev) free(ringPrev); if (ringNext) free(ringNext); if (treeToParent) free(treeToParent); if (treeToChild0) free(treeToChild0); if (treeToChild1) free(treeToChild1); if (nvlsHeads) free(nvlsHeads); return ret; fail: goto exit; } nccl-2.29.7-1/src/graph/paths.cc000066400000000000000000001172531515037102200162130ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2018-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "core.h" #include "graph.h" #include "topo.h" #include "comm.h" #include "net.h" #include "channel.h" #include "transport.h" #include "device.h" // Pre-compute GPU->NIC, GPU->GPU and NIC->GPU paths struct ncclTopoNodeList { struct ncclTopoNode* list[NCCL_TOPO_MAX_NODES]; int count; }; static ncclResult_t getPath(struct ncclTopoSystem* system, struct ncclTopoNode* node, int t, int64_t id, struct ncclTopoLinkList** path) { for (int i=0; inodes[t].count; i++) { if (system->nodes[t].nodes[i].id == id) { *path = node->paths[t]+i; return ncclSuccess; } } WARN("Could not find node of type %d id %lx", t, id); return ncclInternalError; } NCCL_PARAM(NvbDisable, "NVB_DISABLE", 0); static ncclResult_t ncclTopoSetPaths(struct ncclTopoNode* baseNode, struct ncclTopoSystem* system) { if (baseNode->paths[baseNode->type] == NULL) { NCCLCHECK(ncclCalloc(baseNode->paths+baseNode->type, system->nodes[baseNode->type].count)); for (int i=0; inodes[baseNode->type].count; i++) baseNode->paths[baseNode->type][i].type = PATH_DIS; } // breadth-first search to set all paths to that node in the system struct ncclTopoNodeList nodeList; struct ncclTopoNodeList nextNodeList = { { 0 }, 0 }; nodeList.count = 1; nodeList.list[0] = baseNode; struct ncclTopoLinkList* basePath; NCCLCHECK(getPath(system, baseNode, baseNode->type, baseNode->id, &basePath)); basePath->count = 0; basePath->bw = LOC_BW; basePath->type = PATH_LOC; while (nodeList.count) { nextNodeList.count = 0; for (int n=0; ntype, baseNode->id, &path)); for (int l=0; lnlinks; l++) { struct ncclTopoLink* link = node->links+l; struct ncclTopoNode* remNode = link->remNode; if (remNode->paths[baseNode->type] == NULL) { NCCLCHECK(ncclCalloc(remNode->paths+baseNode->type, system->nodes[baseNode->type].count)); for (int i=0; inodes[baseNode->type].count; i++) remNode->paths[baseNode->type][i].type = PATH_DIS; } struct ncclTopoLinkList* remPath; NCCLCHECK(getPath(system, remNode, baseNode->type, baseNode->id, &remPath)); float bw = std::min(path->bw, link->bw); // allow routing through a GPU only as 1 hop if (node != baseNode && node->type == GPU && (ncclParamNvbDisable() || link->type != LINK_NVL || remNode->type != GPU || path->count > 1)) continue; if ((remPath->bw == 0 || remPath->count > path->count) && remPath->bw < bw) { // Find reverse link for (int l=0; lnlinks; l++) { if (remNode->links[l].remNode == node && remNode->links[l].type == link->type) { remPath->list[0] = remNode->links+l; break; } } if (remPath->list[0] == NULL) { WARN("Failed to find reverse path from remNode %d/%lx nlinks %d to node %d/%lx", remNode->type, remNode->id, remNode->nlinks, node->type, node->id); return ncclInternalError; } // Copy the rest of the path for (int i=0; icount; i++) remPath->list[i+1] = path->list[i]; remPath->count = path->count + 1; remPath->bw = bw; // Start with path type = link type. PATH and LINK types are supposed to match. // Don't consider LINK_NET as we only care about the NIC->GPU path. int type = link->type == LINK_NET ? LINK_LOC : link->type; // Differentiate between one and multiple PCI switches if (node->type == PCI && remNode->type == PCI) type = PATH_PXB; // Consider a path going through the CPU as PATH_PHB if (link->type == LINK_PCI && (node->type == CPU || link->remNode->type == CPU)) type = PATH_PHB; // Set 1 hop NVLink as NVB if (node->type == GPU && path->type == PATH_NVL && type == PATH_NVL && remPath->count > 1) type = PATH_NVB; remPath->type = std::max(path->type, type); // Add to the list for the next iteration if not already in the list int i; for (i=0; itype], NCCL_TOPO_ID_SYSTEM_ID(node->id), NCCL_TOPO_ID_LOCAL_ID(node->id)); #else snprintf(line, linesize, "%s/%lx-%lx :", topoNodeTypeStr[node->type], NCCL_TOPO_ID_SYSTEM_ID(node->id), NCCL_TOPO_ID_LOCAL_ID(node->id)); int offset = strlen(line); #endif for (int t=0; tpaths[t] == NULL) continue; for (int n = 0; nnodes[t].count; n++) { #ifdef ENABLE_TRACE line[0] = 0; int offset = 0; for (int i=0; ipaths[t][n].count; i++) { struct ncclTopoLink* link = node->paths[t][n].list[i]; struct ncclTopoNode* remNode = link->remNode; snprintf(line+offset, linesize-offset, "--%s(%g)->%s/%lx-%lx", topoLinkTypeStr[link->type], link->bw, topoNodeTypeStr[remNode->type], NCCL_TOPO_ID_SYSTEM_ID(remNode->id), NCCL_TOPO_ID_LOCAL_ID(remNode->id)); offset = strlen(line); } INFO(NCCL_GRAPH, "%s (%f)", line, node->paths[t][n].bw); #else snprintf(line+offset, linesize-offset, "%s/%lx-%lx (%d/%.1f/%s) ", topoNodeTypeStr[t], NCCL_TOPO_ID_SYSTEM_ID(system->nodes[t].nodes[n].id), NCCL_TOPO_ID_LOCAL_ID(system->nodes[t].nodes[n].id), node->paths[t][n].count, node->paths[t][n].bw, topoPathTypeStr[node->paths[t][n].type]); offset = strlen(line); #endif } } #ifndef ENABLE_TRACE INFO(NCCL_GRAPH, "%s", line); #endif } ncclResult_t ncclTopoPrintPaths(struct ncclTopoSystem* system) { for (int i=0; inodes[GPU].count; i++) { printNodePaths(system, system->nodes[GPU].nodes+i); } for (int i=0; inodes[NET].count; i++) { printNodePaths(system, system->nodes[NET].nodes+i); } for (int i=0; inodes[GIN].count; i++) { printNodePaths(system, system->nodes[GIN].nodes+i); } return ncclSuccess; } ncclResult_t ncclGetLocalCpu(struct ncclTopoSystem* system, int gpu, int* retCpu) { // Find the closest CPU to a GPU int minHops = 0; int localCpu = -1; struct ncclTopoLinkList* paths = system->nodes[GPU].nodes[gpu].paths[CPU]; for (int c=0; cnodes[CPU].count; c++) { int hops = paths[c].count; if (hops > 0 && (minHops == 0 || hops < minHops)) { localCpu = c; minHops = hops; } } if (localCpu == -1) { WARN("Error : could not find CPU close to GPU %d", gpu); return ncclInternalError; } *retCpu = localCpu; return ncclSuccess; } static int mergePathType(int type0, int type1){ int max = std::max(type0,type1); int min = std::min(type0,type1); if(max == PATH_PHB && min == PATH_C2C) return PATH_P2C; else return max; } static ncclResult_t addInterStep(struct ncclTopoSystem* system, int tx, int ix, int t1, int i1, int t2, int i2) { struct ncclTopoNode* cpuNode = system->nodes[tx].nodes+ix; struct ncclTopoNode* srcNode = system->nodes[t1].nodes+i1; int l=0; // Node 1 -> CPU for (int i=0; ipaths[tx][ix].count; i++) srcNode->paths[t2][i2].list[l++] = srcNode->paths[tx][ix].list[i]; // CPU -> Node 2 for (int i=0; ipaths[t2][i2].count; i++) srcNode->paths[t2][i2].list[l++] = cpuNode->paths[t2][i2].list[i]; // Update path characteristics srcNode->paths[t2][i2].count = l; srcNode->paths[t2][i2].type = mergePathType(srcNode->paths[tx][ix].type, cpuNode->paths[t2][i2].type); if (tx == GPU) srcNode->paths[t2][i2].type = PATH_PXN; srcNode->paths[t2][i2].bw = std::min(srcNode->paths[tx][ix].bw, cpuNode->paths[t2][i2].bw); return ncclSuccess; } // Remove/free all paths static void ncclTopoRemovePaths(struct ncclTopoSystem* system) { for (int t1=0; t1nodes[t1].count; n++) { struct ncclTopoNode* node = system->nodes[t1].nodes+n; for (int t2=0; t2paths[t2]) free(node->paths[t2]); node->paths[t2] = NULL; } } } } static const int levelsOldToNew[] = { PATH_LOC, PATH_PIX, PATH_PXB, PATH_PHB, PATH_SYS, PATH_SYS }; ncclResult_t ncclGetLevel(int* level, const char* disableEnv, const char* levelEnv) { if (*level == -1) { int l = -1; if (disableEnv) { const char* str = ncclGetEnv(disableEnv); if (str) { int disable = strtol(str, NULL, 0); if (disable == 1) l = PATH_LOC; if (l >= 0) INFO(NCCL_ALL, "%s set by environment to %d", disableEnv, disable); } } if (l == -1) { const char* str = ncclGetEnv(levelEnv); if (str) { for (int i=0; i<=PATH_SYS; i++) { if (strcmp(str, topoPathTypeStr[i]) == 0) { l = i; break; } } // Old style numbering // levelsOldToNew to is an array with each index corresponding to the // "old level" int, and each value mapping to the correct value defined in topo.h // maxOldLevel is a quick check to handle out of bounds (based on the length of levelsOldToNew) if (l == -1 && str[0] >= '0' && str[0] <= '9') { int oldLevel = strtol(str, NULL, 0); const int maxOldLevel = sizeof(levelsOldToNew)/sizeof(int) - 1; if (oldLevel > maxOldLevel) oldLevel = maxOldLevel; l = levelsOldToNew[oldLevel]; } if (l >= 0) INFO(NCCL_ALL, "%s set by environment to %s", levelEnv, topoPathTypeStr[l]); } } *level = l >= 0 ? l : -2; } return ncclSuccess; } NCCL_PARAM(IgnoreDisabledP2p, "IGNORE_DISABLED_P2P", 0); static int ncclTopoUserP2pLevel = -1; // Initially "uninitialized". When initialized but unset, changes to -2. // Gets the user-provided value of NCCL_P2P_LEVEL/NCCL_P2P_DISABLE. If the user did not provide any, the value // of the "level" argument is left unchanged. ncclResult_t ncclGetUserP2pLevel(int* level) { if (ncclTopoUserP2pLevel == -1) NCCLCHECK(ncclGetLevel(&ncclTopoUserP2pLevel, "NCCL_P2P_DISABLE", "NCCL_P2P_LEVEL")); if (ncclTopoUserP2pLevel != -2) *level = ncclTopoUserP2pLevel; return ncclSuccess; } // Tests two ranks for CUDA P2P connectivity. // *cudaP2p returns 1 if CUDA P2P between the ranks is supported. // *p2p returns 1 only if the distance between the ranks is no greater than NCCL_P2P_LEVEL. The connection may go through an intermediate rank. ncclResult_t ncclTopoCheckP2p(struct ncclComm* comm, struct ncclTopoSystem* system, int rank1, int rank2, int* p2p, int *read, int* intermediateRank, int* cudaP2p) { int mnnvl = 0; struct ncclPeerInfo* info1 = NULL; struct ncclPeerInfo* info2 = NULL; *p2p = 0; if (read) *read = 0; if (intermediateRank) *intermediateRank = -1; if (cudaP2p) *cudaP2p = 0; // Rule out different nodes / isolated containers if (comm) { info1 = comm->peerInfo+rank1; info2 = comm->peerInfo+rank2; if (info1->hostHash != info2->hostHash) { if (comm->MNNVL) { NCCLCHECK(ncclTopoCheckMNNVL(comm->topo, info1, info2, &mnnvl)); if (!mnnvl) return ncclSuccess; } else { return ncclSuccess; } } else if (info1->shmDev != info2->shmDev) { return ncclSuccess; } } // Get GPUs from topology int g1, g2; NCCLCHECK(ncclTopoRankToIndex(system, rank1, &g1, /*showWarn=*/true)); struct ncclTopoNode* gpu1 = system->nodes[GPU].nodes+g1; if (ncclTopoRankToIndex(system, rank2, &g2, /*showWarn=*/false) == ncclInternalError) { // GPU not found, we can't use p2p. return ncclSuccess; } int intermediateIndex = -1; // Set intermediate GPU rank, if routing through an intermediate GPU. struct ncclTopoLinkList* path = gpu1->paths[GPU]+g2; if (path->count == 2) { struct ncclTopoNode* intermediateNode = path->list[0]->remNode; if (intermediateNode->type == GPU) { intermediateIndex = intermediateNode - system->nodes[GPU].nodes; if (intermediateRank) *intermediateRank = intermediateNode->gpu.rank; } } // By default don't use P2P across CPU Host Bridges and further apart int p2pLevel = PATH_PXB; int arch, vendor, model; NCCLCHECK(ncclTopoCpuType(system, &arch, &vendor, &model)); // Allow P2P between pairs of GPUs on AMD systems if ((arch == NCCL_TOPO_CPU_ARCH_X86 && vendor == NCCL_TOPO_CPU_VENDOR_AMD) && system->nodes[GPU].count <= 2) p2pLevel = PATH_SYS; // User override NCCLCHECK(ncclGetUserP2pLevel(&p2pLevel)); // Compute the PCI distance and compare with the p2pLevel. if (path->type <= p2pLevel) *p2p = 1; // NCCL_IGNORE_DISABLED_P2P=2 is used by unit tests that don't want to // validate against NVML at all since they are pretending to be on other hw. bool checkNvml = (ncclParamIgnoreDisabledP2p() != 2 && g1 != g2 && (comm == NULL || (info1->hostHash == comm->peerInfo[comm->rank].hostHash && info1->hostHash == info2->hostHash))); if (*p2p == 1) { if (checkNvml) { int indexes[3] = {-1,-1,-1}; int verticeN = 0; NCCLCHECK(ncclNvmlEnsureInitialized()); indexes[verticeN++] = system->nodes[GPU].nodes[g1].gpu.dev; if (intermediateIndex != -1) indexes[verticeN++] = system->nodes[GPU].nodes[intermediateIndex].gpu.dev; indexes[verticeN++] = system->nodes[GPU].nodes[g2].gpu.dev; for (int i=1; i < verticeN; i++) { nvmlGpuP2PStatus_t status; status = ncclNvmlDevicePairs[indexes[i-1]][indexes[i-0]].p2pStatusRead; bool good = status == NVML_P2P_STATUS_OK; status = ncclNvmlDevicePairs[indexes[i-1]][indexes[i-0]].p2pStatusWrite; good &= status == NVML_P2P_STATUS_OK; if (!good) { if (!ncclParamIgnoreDisabledP2p()) { if (path->type <= PATH_NVB) { WARN("P2P is disabled between NVLINK connected GPUs %d and %d. This should not be the case given their connectivity, and is probably due to a hardware issue. If you still want to proceed, you can set NCCL_IGNORE_DISABLED_P2P=1.", indexes[i-1], indexes[i-0]); return ncclUnhandledCudaError; } else if (path->type < PATH_SYS) { INFO(NCCL_INIT, "P2P is disabled between connected GPUs %d and %d. You can repress this message with NCCL_IGNORE_DISABLED_P2P=1.", indexes[i-1], indexes[i-0]); } } *p2p = 0; } } } } if (path->type == PATH_NVL) { struct ncclTopoNode* gpu2 = system->nodes[GPU].nodes+g2; // Enable P2P Read for Ampere/NVLink only if (read && (gpu1->gpu.cudaCompCap == gpu2->gpu.cudaCompCap) && (gpu1->gpu.cudaCompCap == 80)) *read = 1; } if (cudaP2p) { if (checkNvml) { int n1, n2; n1 = system->nodes[GPU].nodes[g1].gpu.dev; n2 = system->nodes[GPU].nodes[g2].gpu.dev; *cudaP2p = (ncclNvmlDevicePairs[n1][n2].p2pStatusRead == NVML_P2P_STATUS_OK && ncclNvmlDevicePairs[n1][n2].p2pStatusWrite == NVML_P2P_STATUS_OK); } else { // We assume P2P connectivity in case the ranks are connected using MNNVL or are on the same host. *cudaP2p = (mnnvl || comm == NULL || info1->hostHash == info2->hostHash); } } return ncclSuccess; } // MNNVL: Check whether peers are in the same fabric cluster and clique ncclResult_t ncclTopoCheckMNNVL(struct ncclTopoSystem* system, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2, int* ret) { *ret = 0; nvmlGpuFabricInfoV_t *fabricInfo1 = &info1->fabricInfo; nvmlGpuFabricInfoV_t *fabricInfo2 = &info2->fabricInfo; // A zero UUID means we don't have MNNVL fabric info unsigned long uuid0 = 0; unsigned long uuid1 = 0; memcpy(&uuid0, fabricInfo2->clusterUuid, sizeof(uuid0)); memcpy(&uuid1, fabricInfo2->clusterUuid + sizeof(uuid0), sizeof(uuid1)); if ((uuid0 | uuid1) == 0) return ncclSuccess; if ((memcmp(fabricInfo1->clusterUuid, fabricInfo2->clusterUuid, NVML_GPU_FABRIC_UUID_LEN) == 0) && (fabricInfo1->cliqueId == fabricInfo2->cliqueId)) { TRACE(NCCL_NET, "MNNVL matching peer 0x%lx UUID %lx.%lx cliqueId 0x%x", info2->busId, uuid0, uuid1, fabricInfo2->cliqueId); *ret = 1; } return ncclSuccess; } NCCL_PARAM(NetGdrRead, "NET_GDR_READ", -2); int ncclTopoUserGdrLevel = -1; const char* ncclTopoGdrModeStr[ncclTopoGdrModeNum] = { "Disabled", "Default", "PCI" }; // On C2C platforms use GDRDMA on NICs which are connected to the CPUs NCCL_PARAM(NetGdrC2c, "NET_GDR_C2C", 1); ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* system, int rank, int64_t netId, int read, enum ncclTopoGdrMode* gdrMode) { *gdrMode = ncclTopoGdrModeDisable; // Get GPU and NET int n, g; NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, &n)); struct ncclTopoNode* net = system->nodes[NET].nodes+n; NCCLCHECK(ncclTopoRankToIndex(system, rank, &g, /*showWarn=*/true)); struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g; char gpuNetMsg[1024] = ""; snprintf(gpuNetMsg, sizeof(gpuNetMsg), "GPU/%ld-%ld (rank %d) - NET/%ld-%ld (", NCCL_TOPO_ID_SYSTEM_ID(gpu->id), NCCL_TOPO_ID_LOCAL_ID(gpu->id), rank, NCCL_TOPO_ID_SYSTEM_ID(net->id), NCCL_TOPO_ID_LOCAL_ID(net->id)); // Check that both the NIC and GPUs support it if (net->net.gdrSupport == 0) return ncclSuccess; if (gpu->gpu.gdrSupport == 0) return ncclSuccess; if (read) { // For reads (sends) only enable under certain conditions int gdrReadParam = ncclParamNetGdrRead(); if (gdrReadParam == 0) return ncclSuccess; // Disable GDR Reads pre-Ampere when we have other PCI flows if (gdrReadParam < 0 && gpu->gpu.cudaCompCap < 80) { int nvlink = 0; // Since we don't know whether there are other communicators, // it's better to keep things local if we have a single GPU. if (system->nodes[GPU].count == 1) nvlink = 1; for (int i=0; inodes[GPU].count; i++) { if (i == g) continue; if (gpu->paths[GPU][i].type == PATH_NVL) { nvlink = 1; break; } } if (!nvlink) return ncclSuccess; } } // Check if we are close enough that it makes sense to enable GDR int netGdrLevel = ncclParamNetGdrC2c() ? PATH_P2C : PATH_PXB; NCCLCHECK(ncclGetLevel(&ncclTopoUserGdrLevel, NULL, "NCCL_NET_GDR_LEVEL")); if (ncclTopoUserGdrLevel != -2) netGdrLevel = ncclTopoUserGdrLevel; int distance = gpu->paths[NET][n].type; if (distance == PATH_PXN) { // In case of PXN, use the intermediate GPU distance instead int proxyRank; NCCLCHECK(ncclTopoGetIntermediateRank(system, gpu->gpu.rank, netId, &proxyRank)); NCCLCHECK(ncclTopoRankToIndex(system, proxyRank, &g, /*showWarn=*/true)); gpu = system->nodes[GPU].nodes+g; distance = gpu->paths[NET][n].type; snprintf(gpuNetMsg+strlen(gpuNetMsg), sizeof(gpuNetMsg)-strlen(gpuNetMsg), " using PXN via GPU/%ld-%ld, ", NCCL_TOPO_ID_SYSTEM_ID(gpu->id), NCCL_TOPO_ID_LOCAL_ID(gpu->id)); } if (distance > netGdrLevel) { snprintf(gpuNetMsg + strlen(gpuNetMsg), sizeof(gpuNetMsg) - strlen(gpuNetMsg), "distance %d > %d)", distance, netGdrLevel); INFO(NCCL_GRAPH | NCCL_NET, "GPU Direct RDMA Disabled for %s", gpuNetMsg); return ncclSuccess; } // Force PCIe mapping if path goes through PCI on a C2C system int c; NCCLCHECK(ncclGetLocalCpu(system, g, &c)); if (gpu->paths[CPU][c].type == PATH_C2C && distance != PATH_P2C) *gdrMode = ncclTopoGdrModePci; else *gdrMode = ncclTopoGdrModeDefault; snprintf(gpuNetMsg + strlen(gpuNetMsg), sizeof(gpuNetMsg) - strlen(gpuNetMsg), "distance %d <= %d, read %d, mode %s)", distance, netGdrLevel,read, ncclTopoGdrModeStr[*gdrMode]); INFO(NCCL_GRAPH | NCCL_NET, "GPU Direct RDMA Enabled for %s", gpuNetMsg); return ncclSuccess; } ncclResult_t ncclTopoIsGdrAvail(struct ncclTopoSystem* system, int rank, bool *avail) { int netNum = system->nodes[NET].count; enum ncclTopoGdrMode useGdr = ncclTopoGdrModeDisable; *avail = false; for (int n = 0; n < netNum; n++) { int64_t netId = system->nodes[NET].nodes[n].id; NCCLCHECK(ncclTopoCheckGdr(system, rank, netId, 1, &useGdr)); if (useGdr) { *avail = true; break; } NCCLCHECK(ncclTopoCheckGdr(system, rank, netId, 0, &useGdr)); if (useGdr) { *avail = true; break; } } return ncclSuccess; } // Set to 0 to disable the flush on Hopper when using GDR NCCL_PARAM(NetForceFlush, "NET_FORCE_FLUSH", 0); // Determine whether we need to flush the GDR recv buffers ncclResult_t ncclTopoNeedFlush(struct ncclComm* comm, int64_t netId, int netDev, int rank, int* flush) { *flush = 1; ncclNetProperties_t props; NCCLCHECK(comm->ncclNet->getProperties(netDev, &props)); if (props.forceFlush == 1 || ncclParamNetForceFlush()) return ncclSuccess; int g; struct ncclTopoSystem* system = comm->topo; NCCLCHECK(ncclTopoRankToIndex(system, rank, &g, /*showWarn=*/true)); struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g; // Flush is required on Ampere and earlier if (gpu->gpu.cudaCompCap >= 90) *flush = 0; // On C2C platforms, data could go through a PCI switch while completions and // flags would go through C2C. In that case, force a flush. int c, n; NCCLCHECK(ncclGetLocalCpu(system, g, &c)); NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, &n)); if (gpu->paths[NET][n].type <= PATH_PXB && gpu->paths[CPU][c].type == PATH_C2C) { *flush = 1; } return ncclSuccess; } NCCL_PARAM(NetDisableIntra, "NET_DISABLE_INTRA", 0); // Check whether going through the network would be faster than going through P2P/SHM. ncclResult_t ncclTopoCheckNet(struct ncclTopoSystem* system, int rank1, int rank2, int* net) { if (ncclParamNetDisableIntra() == 1) { *net = 0; return ncclSuccess; } *net = 1; // First check the current GPU-to-GPU speed. int g1, g2; if (ncclTopoRankToIndex(system, rank1, &g1, /*showWarn=*/false) != ncclSuccess || ncclTopoRankToIndex(system, rank2, &g2, /*showWarn=*/false) != ncclSuccess) { return ncclSuccess; } struct ncclTopoNode* gpu1 = system->nodes[GPU].nodes+g1; struct ncclTopoNode* gpu2 = system->nodes[GPU].nodes+g2; float speed = gpu1->paths[GPU][g2].bw; // Now check the speed each GPU can access the network through PXB or better float netSpeed1 = 0, netSpeed2 = 0; for (int n=0; nnodes[NET].count; n++) { struct ncclTopoLinkList* path = gpu1->paths[NET]+n; if (path->type <= PATH_PXB && path->bw > netSpeed1) netSpeed1 = path->bw; path = gpu2->paths[NET]+n; if (path->type <= PATH_PXB && path->bw > netSpeed2) netSpeed2 = path->bw; } if (netSpeed1 > speed && netSpeed2 > speed) return ncclSuccess; *net = 0; return ncclSuccess; } ncclResult_t ncclTopoGetIntermediateRank(struct ncclTopoSystem* system, int rank, int64_t netId, int* intermediateRank) { // Get GPU and NET int n, g; NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, &n)); NCCLCHECK(ncclTopoRankToIndex(system, rank, &g, /*showWarn=*/true)); struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g; struct ncclTopoLinkList* path = gpu->paths[NET]+n; if (path->type == PATH_PXN) { struct ncclTopoNode* node; int type = NVS; for (int i=0; icount && type == NVS; i++) { node = path->list[i]->remNode; type = node->type; } if (type != GPU) { WARN("Could not find intermediate GPU between GPU rank %d and NIC %lx", rank, netId); return ncclInternalError; } *intermediateRank = node->gpu.rank; } else { *intermediateRank = rank; } return ncclSuccess; } NCCL_PARAM(PxnDisable, "PXN_DISABLE", 0); // Net v4 plugins don't have non-blocking connect/accept. We can't therefore use // remote proxies without risking deadlocks int ncclPxnDisable(struct ncclComm* comm) { #if defined(NCCL_OS_LINUX) static int pxnDisable = -1; if (pxnDisable == -1) { if (comm && comm->ncclNetVer == 4) { INFO(NCCL_INIT, "PXN Disabled as plugin is v4"); pxnDisable = 1; } else { pxnDisable = ncclParamPxnDisable(); } } return pxnDisable; #else return 1; #endif } ncclResult_t ncclTopoGetPxnRanks(struct ncclComm* comm, int** intermediateRanks, int* nranks) { struct ncclTopoSystem* system = comm->topo; *nranks = 0; *intermediateRanks = NULL; if (system->inter == 0) return ncclSuccess; int nr = 0; int* ranks = NULL; for (int rank=0; ranknRanks; rank++) { int64_t netId; int proxyRank; NCCLCHECK(ncclTopoGetNetDev(comm, comm->rank, NULL, 0, rank, &netId, NULL, &proxyRank)); if (proxyRank == comm->rank) continue; enum ncclTopoGdrMode useGdr; NCCLCHECK(ncclTopoCheckGdr(comm->topo, comm->rank, netId, 1, &useGdr)); if (useGdr == ncclTopoGdrModeDisable) continue; int found = 0; for (int r=0; rnodes[CPU].count; c++) { NCCLCHECK(ncclTopoSetPaths(system->nodes[CPU].nodes+c, system)); } // Set direct paths to GPUs. for (int g=0; gnodes[GPU].count; g++) { NCCLCHECK(ncclTopoSetPaths(system->nodes[GPU].nodes+g, system)); } // Set direct paths to NICs. for (int n=0; nnodes[NET].count; n++) { NCCLCHECK(ncclTopoSetPaths(system->nodes[NET].nodes+n, system)); } // Set direct paths to GIN devices. for (int n=0; nnodes[GIN].count; n++) { NCCLCHECK(ncclTopoSetPaths(system->nodes[GIN].nodes+n, system)); } // Set direct paths to NVSwitches. for (int n=0; nnodes[NVS].count; n++) { NCCLCHECK(ncclTopoSetPaths(system->nodes[NVS].nodes+n, system)); } // Update path for GPUs when we don't want to / can't use GPU Direct P2P for (int g=0; gnodes[GPU].count; g++) { for (int p=0; pnodes[GPU].count; p++) { int p2p; NCCLCHECK(ncclTopoCheckP2p(comm, system, system->nodes[GPU].nodes[p].gpu.rank, system->nodes[GPU].nodes[g].gpu.rank, &p2p, NULL, NULL, NULL)); if (p2p == 0) { // Divert all traffic through the CPU int cpu; NCCLCHECK(ncclGetLocalCpu(system, g, &cpu)); NCCLCHECK(addInterStep(system, CPU, cpu, GPU, p, GPU, g)); } } if (comm == NULL) continue; // Remove GPUs we can't (or don't want to) communicate with through P2P or SHM struct ncclPeerInfo* dstInfo = comm->peerInfo+system->nodes[GPU].nodes[g].gpu.rank; for (int p=0; pnodes[GPU].count; p++) { if (p == g) continue; struct ncclPeerInfo* srcInfo = comm->peerInfo+system->nodes[GPU].nodes[p].gpu.rank; int p2p; NCCLCHECK(ncclTransports[TRANSPORT_P2P]->canConnect(&p2p, comm, NULL, srcInfo, dstInfo)); if (p2p == 0) { int shm; NCCLCHECK(ncclTransports[TRANSPORT_SHM]->canConnect(&shm, comm, NULL, srcInfo, dstInfo)); if (shm == 0) { // Mark this peer as inaccessible. We'll trim it later. system->nodes[GPU].nodes[p].paths[GPU][g].type = PATH_NET; } } } } // update the GPU -> NIC path in the case of C2C + PHB // P2C is only set when the NET is the closest to the GPU. Otherwise PXN connections should be preferred for (int g = 0; g < system->nodes[GPU].count; g++) { struct ncclTopoNode* gpuNode = system->nodes[GPU].nodes + g; int c = 1, localNetCount = 0, localNet[NCCL_TOPO_MAX_NODES]; NCCLCHECK(ncclGetLocalCpu(system, g, &c)); if (c == -1) continue; NCCLCHECK(ncclTopoGetLocal(system, GPU, g, NET, localNet, &localNetCount, /*pathType=*/NULL)); for (int l = 0; l < localNetCount; l++) { int n = localNet[l]; struct ncclTopoNode* netNode = system->nodes[NET].nodes + n; if (mergePathType(gpuNode->paths[CPU][c].type, netNode->paths[CPU][c].type) == PATH_P2C) { gpuNode->paths[NET][n].type = std::min(PATH_P2C, gpuNode->paths[NET][n].type); netNode->paths[GPU][g].type = std::min(PATH_P2C, netNode->paths[GPU][g].type); } } } // Update paths for NICs (no GPU Direct, PXN, ...) for (int n=0; nnodes[NET].count; n++) { struct ncclTopoNode* netNode = system->nodes[NET].nodes+n; for (int g=0; gnodes[GPU].count; g++) { // Check whether we can access the NIC through another NVLink-connected GPU (PXN) struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g; if (ncclPxnDisable(comm) != 1) { int localGpuIndex; NCCLCHECK(ncclTopoGetLocalGpu(system, netNode->id, &localGpuIndex)); if (localGpuIndex != g && localGpuIndex != -1) { // PXN = PCI + NVLink. struct ncclTopoNode* peerNode = system->nodes[GPU].nodes+localGpuIndex; // Only use PXN for NIC n if remote GPU p ... int pxnType = ncclParamPxnC2c() ? PATH_P2C : PATH_PXB; if (/* (1) is connected to the NIC with PxN type*/ peerNode->paths[NET][n].type <= pxnType && /* and (2) is connected to us through NVLink */ peerNode->paths[GPU][g].type <= PATH_NVL && /* and (3) is on the same node as us */ NCCL_TOPO_ID_SYSTEM_ID(peerNode->id) == NCCL_TOPO_ID_SYSTEM_ID(gpu->id) && /* and (4) has either higher bw to that NIC or avoid going through the CPU (path.type is > PATH_PXN)*/ (peerNode->paths[NET][n].bw > gpu->paths[NET][n].bw || gpu->paths[NET][n].type > PATH_PXN)) // We can use that GPU as relay to communicate with that NIC. // Only enabling it in the GPU->NIC direction for now to favor // receiving locally and sending remotely (consistent with net.cc) NCCLCHECK(addInterStep(system, GPU, localGpuIndex, GPU, g, NET, n)); } } if (gpu->paths[NET][n].type < PATH_PHB) { // Update path when we dont want to / can't use GPU Direct RDMA. enum ncclTopoGdrMode gdr; NCCLCHECK(ncclTopoCheckGdr(system, system->nodes[GPU].nodes[g].gpu.rank, netNode->id, 0, &gdr)); if (gdr == 0) { // We cannot use GPU Direct RDMA, divert all traffic through the CPU local to the GPU int localCpu; NCCLCHECK(ncclGetLocalCpu(system, g, &localCpu)); NCCLCHECK(addInterStep(system, CPU, localCpu, NET, n, GPU, g)); NCCLCHECK(addInterStep(system, CPU, localCpu, GPU, g, NET, n)); } } } } // Pre-compute NET local gpus to accelerate search for (int n=0; nnodes[NET].count; n++) { struct ncclTopoNode* net = system->nodes[NET].nodes+n; NCCLCHECK(ncclTopoGetLocalGpu(system, net->id, &net->net.localGpu)); } return ncclSuccess; } ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm* comm) { ncclResult_t ret = ncclSuccess; int *domains; int64_t *ids = NULL; int myDomain = 0; int ngpus = system->nodes[GPU].count; NCCLCHECK(ncclCalloc(&domains, system->nodes[GPU].count)); NCCLCHECKGOTO(ncclCalloc(&ids, system->nodes[GPU].count), ret, fail); for (int g=0; gnodes[GPU].count; g++) { struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g; domains[g] = g; ids[g] = gpu->id; for (int p=0; ppaths[GPU][p].type < PATH_NET) { domains[g] = std::min(domains[g], domains[p]); } } if (gpu->gpu.rank == comm->rank) myDomain = domains[g]; } for (int i=0; inodes[GPU].count /* This one varies over the loops */; g++) { gpu = system->nodes[GPU].nodes+g; if (gpu->id == ids[i]) break; else gpu=NULL; } if (gpu == NULL) { WARN("Could not find id %lx", ids[i]); ret = ncclInternalError; goto fail; } NCCLCHECKGOTO(ncclTopoRemoveNode(system, GPU, g), ret, fail); } system->inter = system->nodes[GPU].count == comm->nRanks ? 0 : 1; exit: free(domains); if (ids) free(ids); return ret; fail: goto exit; } void ncclTopoFree(struct ncclTopoSystem* system) { ncclTopoRemovePaths(system); free(system); } NCCL_PARAM(P2pPerChannelNetBw, "P2P_PER_CHANNEL_NET_BW", /*GB/s*/14); static ncclResult_t ncclTopoGetNchannels(struct ncclComm* comm, int g /*local gpu index*/, int peerRank, int* nChannels) { int peer; struct ncclTopoSystem* system = comm->topo; struct ncclTopoLinkList* path = NULL; if (ncclTopoRankToIndex(system, peerRank, &peer, /*showWarn=*/false) == ncclSuccess) { // Same rank if (g == peer) { *nChannels = -1; return ncclSuccess; } // Local rank path = system->nodes[GPU].nodes[peer].paths[GPU]+g; if (path->type == PATH_NVL) { float nvlBw = ncclTopoNVLinkBw(system->nodes[GPU].nodes[g].gpu.cudaCompCap); *nChannels = 2*std::max(1, (int)(path->bw / nvlBw)); } else { *nChannels = 2; } } else { // Remote rank, use network int nNetChannels = comm->config.nChannelsPerNetPeer; if (nNetChannels == NCCL_CONFIG_UNDEF_INT) { float netBw = 0.0; int netCount = 0; NCCLCHECK(getLocalNetCountByBw(system, g, &netCount, &netBw)); // We use at least 1 channel per NIC, and more if needed to meet the bw requirement. nNetChannels = 2; if (netCount > 0) nNetChannels = std::max(netCount, divUp((int)netBw, (int)ncclParamP2pPerChannelNetBw())); } *nChannels = nNetChannels; } return ncclSuccess; } NCCL_PARAM(MinP2pNChannels, "MIN_P2P_NCHANNELS", 1); NCCL_PARAM(MaxP2pNChannels, "MAX_P2P_NCHANNELS", MAXCHANNELS); extern int64_t ncclParamWorkArgsBytes(); ncclResult_t ncclTopoComputeP2pChannelsPerPeer(struct ncclComm* comm) { int g = 0; while (comm->topo->nodes[GPU].nodes[g].gpu.rank != comm->rank) g++; if (g == comm->topo->nodes[GPU].count) return ncclInternalError; int minChannels = MAXCHANNELS; for (int r = 0; r < comm->nRanks; r++) { int nChannels; NCCLCHECK(ncclTopoGetNchannels(comm, g, r, &nChannels)); if (nChannels >= 0) minChannels = std::min(minChannels, nChannels); } comm->p2pnChannelsPerPeer = minChannels; return ncclSuccess; } ncclResult_t ncclTopoComputeP2pChannels(struct ncclComm* comm) { /* here we already honor comm->max/minCTAs for p2pnChannels. */ if (comm->sharedRes->owner != comm) { comm->p2pnChannels = std::min(comm->nChannels, (int)ncclParamMaxP2pNChannels()); comm->p2pnChannels = std::min(std::max(comm->p2pnChannels, (int)ncclParamMinP2pNChannels()), comm->sharedRes->tpP2pNChannels); } else { comm->p2pnChannels = std::min(comm->nChannels, (int)ncclParamMaxP2pNChannels()); comm->p2pnChannels = std::max(comm->p2pnChannels, (int)ncclParamMinP2pNChannels()); } // Make nChannelsPerPeer and nChannels powers of 2. This is relied on when mapping p2p peers to channels. comm->p2pnChannelsPerPeer = pow2Up(comm->p2pnChannelsPerPeer); comm->p2pnChannels = pow2Up(comm->p2pnChannels); comm->p2pnChannels = std::min(comm->p2pnChannels, pow2Down(ncclDevMaxChannelsForArgsBytes(ncclParamWorkArgsBytes()))); if (comm->nNodes > 1 && comm->config.nChannelsPerNetPeer == NCCL_CONFIG_UNDEF_INT) { // In the case of >1 NVLD (and the user didn't set nChannelsPerNetPeer), the network is the botteneck. // Reduce the number of channels per host to avoid going above p2pnChannels to fit all the peers within a single round. while (comm->p2pnChannelsPerPeer * divUp(comm->nRanks, NCCL_MAX_DEV_WORK_P2P_PER_BATCH) > comm->p2pnChannels && comm->p2pnChannelsPerPeer > 1) comm->p2pnChannelsPerPeer /= 2; } else { comm->p2pnChannelsPerPeer = std::min(comm->p2pnChannels, comm->p2pnChannelsPerPeer); } // Init channels that weren't used so far for (int c=comm->nChannels; cp2pnChannels; c++) NCCLCHECK(initChannel(comm, c)); return ncclSuccess; } ncclResult_t ncclTopoGetNvbGpus(struct ncclTopoSystem* system, int rank, int* nranks, int** ranks) { int ngpus = system->nodes[GPU].count; NCCLCHECK(ncclCalloc(ranks, ngpus)); int nvbGpus = 0; for (int g=0; gnodes[GPU].nodes+g; if (gpu->gpu.rank != rank) continue; for (int p=0; ppaths[GPU][p].type == PATH_NVB) { (*ranks)[nvbGpus++] = system->nodes[GPU].nodes[p].gpu.rank; } } } *nranks = nvbGpus; return ncclSuccess; } ncclResult_t ncclTopoGetGpuMinPath(struct ncclTopoSystem* system, int type, int* min) { int minPath = PATH_SYS; for (int i=0; inodes[GPU].count; i++) { struct ncclTopoLinkList* paths = system->nodes[GPU].nodes[i].paths[type]; if (paths == NULL) continue; for (int j=0; jnodes[type].count; j++) { if (type == GPU && i == j) continue; minPath = std::min(minPath, paths[j].type); } } *min = minPath; return ncclSuccess; } ncclResult_t ncclTopoGetGpuMaxPath(struct ncclTopoSystem* system, int type, int* max) { int maxPath = PATH_LOC; for (int i=0; inodes[GPU].count; i++) { struct ncclTopoLinkList* paths = system->nodes[GPU].nodes[i].paths[type]; if (paths == NULL) continue; for (int j=0; jnodes[type].count; j++) { if (type == GPU && i == j) continue; maxPath = std::max(maxPath, paths[j].type); } } *max = maxPath; return ncclSuccess; } // Check whether the system is all GPUs directly or indirectly connected to each other // through NVLink and C2C. ncclResult_t ncclTopoPathAllNVLink(struct ncclTopoSystem* system, int* allNvLink) { int maxPath; NCCLCHECK(ncclTopoGetGpuMaxPath(system, GPU, &maxPath)); *allNvLink = maxPath >= PATH_PIX ? 0 : 1; return ncclSuccess; } // Check whether the system is all GPUs connected directly to each other through NVLink/NVSwitch. ncclResult_t ncclTopoPathAllDirectNVLink(struct ncclTopoSystem* system, bool* directNvlink) { int maxPath; NCCLCHECK(ncclTopoGetGpuMaxPath(system, GPU, &maxPath)); *directNvlink = maxPath == PATH_NVL; return ncclSuccess; } // Check whether we are in a split NVLink situation, with two NVLink domains, not // connected through NVLink (e.g. QPI). ncclResult_t ncclTopoSplitNvLink(struct ncclTopoSystem* system, int* splitNvLink) { ncclResult_t res = ncclSuccess; int nvlDomains = 0; int *nvlDomain = NULL, *nvlDomainCount = NULL; // Compute NVLink domains NCCLCHECKGOTO(ncclCalloc(&nvlDomain, system->nodes[GPU].count), res, exit); for (int g=0; gnodes[GPU].count; g++) nvlDomain[g] = g; for (int g=0; gnodes[GPU].count; g++) { struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g; int domain = nvlDomain[g]; for (int p=g+1; pnodes[GPU].count; p++) { if (gpu->paths[GPU][p].type == PATH_NVL) { nvlDomain[p] = domain; } } } // Compute number of GPUs per NVLink domain. NCCLCHECKGOTO(ncclCalloc(&nvlDomainCount, system->nodes[GPU].count), res, exit); for (int g=0; gnodes[GPU].count; g++) { nvlDomainCount[nvlDomain[g]]++; } // Count the number of NVLink domains for (int g=0; gnodes[GPU].count; g++) { if (nvlDomainCount[g] > 1) nvlDomains++; } *splitNvLink = nvlDomains == 2 ? 1 : 0; exit: if(nvlDomain) free(nvlDomain); if(nvlDomainCount) free(nvlDomainCount); return res; } nccl-2.29.7-1/src/graph/rings.cc000066400000000000000000000051601515037102200162070ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "core.h" void dumpLine(int* values, int nranks, const char* prefix) { constexpr int line_length = 128; char line[line_length]; int num_width = snprintf(nullptr, 0, "%d", nranks-1); // safe as per "man snprintf" int n = snprintf(line, line_length, "%s", prefix); for (int i = 0; i < nranks && n < line_length-1; i++) { n += snprintf(line + n, line_length - n, " %*d", num_width, values[i]); // At this point n may be more than line_length-1, so don't use it // for indexing into "line". } if (n >= line_length) { // Sprintf wanted to write more than would fit in the buffer. Assume // line_length is at least 4 and replace the end with "..." to // indicate that it was truncated. snprintf(line+line_length-4, 4, "..."); } INFO(NCCL_INIT, "%s", line); } ncclResult_t ncclBuildRings(int nrings, int* rings, int rank, int nranks, int* prev, int* next) { ncclResult_t ret = ncclSuccess; uint64_t* rankFound; int rankFoundSize = DIVUP(nranks, 64); NCCLCHECK(ncclCalloc(&rankFound, rankFoundSize)); for (int r=0; r NCCL_PARAM(CrossNic, "CROSS_NIC", 2); // Initialize system->maxBw. This is the per-channel (i.e. per-SM) // max bw. static float getMaxBw(struct ncclTopoSystem* system, struct ncclTopoNode* gpu, int type) { float maxBw = 0.0; for (int i=0; inodes[type].count; i++) { struct ncclTopoLinkList* path = gpu->paths[type]+i; float bw = path->bw; if (path->count == 0) continue; maxBw = std::max(maxBw, bw); } return maxBw; } static float getTotalBw(struct ncclTopoSystem* system, struct ncclTopoNode* gpu) { float nvlinkBw = 0.0, pciBw = 0.0; for (int l=0; lnlinks; l++) { struct ncclTopoLink* link = gpu->links+l; if (link->type == LINK_NVL) nvlinkBw += link->bw; if (link->type == LINK_PCI) pciBw = link->bw; } return std::max(pciBw, nvlinkBw); } ncclResult_t ncclTopoSearchInit(struct ncclTopoSystem* system) { system->maxBw = 0.0; system->totalBw = 0.0; int inter = system->inter; if (inter == 0 && system->nodes[GPU].count == 1) { system->maxBw = LOC_BW; system->totalBw = LOC_BW; return ncclSuccess; } for (int g=0; gnodes[GPU].count; g++) { struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g; system->maxBw = std::max(system->maxBw, getMaxBw(system, gpu, inter ? NET : GPU)); system->totalBw = std::max(system->totalBw, getTotalBw(system, gpu)); } return ncclSuccess; } ncclResult_t ncclTopoComputeCommCPU(struct ncclComm* comm) { // We assume there is at least one CPU and that the CPUs have the same // architecture and vendor. const struct ncclTopoNodeSet* cpus = &comm->topo->nodes[CPU]; comm->cpuArch = cpus->nodes[0].cpu.arch; comm->cpuVendor = cpus->nodes[0].cpu.vendor; return ncclSuccess; } static ncclResult_t findRevLink(struct ncclTopoNode* node1, struct ncclTopoNode* node2, int type, struct ncclTopoLink** revLink) { for (int l=0; lnlinks; l++) { struct ncclTopoLink* link = node2->links+l; if (link->remNode == node1 && link->type == type) { *revLink = link; return ncclSuccess; } } WARN("Could not find rev link for %d/%ld -> %d/%ld", node1->type, node1->id, node2->type, node2->id); return ncclInternalError; } // This is unfortunately needed since manipulating floats often results in rounding errors. #define SUB_ROUND(a, b) (a = roundf((a-b)*1000)/1000) static ncclResult_t followPath(struct ncclTopoLinkList* path, struct ncclTopoNode* start, int maxSteps, float bw, int* steps) { float pciBw = bw; for (int step=0; stepcount; step++) { struct ncclTopoNode* node = path->list[step]->remNode; if (node->type == CPU) { // Account for P2P inefficiency through Intel CPU RC if (path->type == PATH_PHB && start->type == GPU && node->cpu.arch == NCCL_TOPO_CPU_ARCH_X86 && node->cpu.vendor == NCCL_TOPO_CPU_VENDOR_INTEL) { pciBw = INTEL_P2P_OVERHEAD(bw); } } } struct ncclTopoNode* node = start; for (int step=0; steplist[step]; struct ncclTopoLink* revLink = NULL; float fwBw = link->type == LINK_PCI ? pciBw : bw; float revBw = 0; if (link->remNode->type == GPU && link->remNode->gpu.cudaCompCap < 80 && start->type != GPU) { if (revLink == NULL) NCCLCHECK(findRevLink(node, link->remNode, link->type, &revLink)); revBw += fwBw/8; } if (link->remNode->type == CPU && link->remNode->cpu.arch == NCCL_TOPO_CPU_ARCH_POWER && link->type == LINK_NVL) { if (revLink == NULL) NCCLCHECK(findRevLink(node, link->remNode, link->type, &revLink)); revBw += fwBw; } // Coverity thinks that revLink could be NULL below. However, we access it only if revBw is non-0, and the // logic of the code is that revBw can become non-0 only if revLink is non-NULL (see the "if" statement right above). // coverity[var_deref_op] if (link->bw < fwBw || (revBw && revLink->bw < revBw)) { *steps = step; return ncclSuccess; } SUB_ROUND(link->bw, fwBw); if (revBw) SUB_ROUND(revLink->bw, revBw); node = link->remNode; } *steps = maxSteps; return ncclSuccess; } // Try to go from node type1/index1 to no type2/index2. mult indicates whether we are counting the bandwidth (1) or undoing (-1). static ncclResult_t ncclTopoFollowPath(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, int type1, int index1, int type2, int index2, float mult, struct ncclTopoNode** node) { // First handle easy cases *node = system->nodes[type2].nodes+index2; if (type1 == -1) return ncclSuccess; struct ncclTopoNode* node1 = system->nodes[type1].nodes+index1; struct ncclTopoLinkList* path = node1->paths[type2]+index2; struct ncclTopoNode* node2 = system->nodes[type2].nodes+index2; struct ncclTopoLinkList* revPath = node2->paths[type1]+index1; if (path == NULL) { WARN("No path computed to go from %s/%d to %s/%d", topoNodeTypeStr[type1], index1, topoNodeTypeStr[type2], index2); return ncclInternalError; } // Now check link type *node = NULL; int intra = (type1 == GPU || type1 == NVS) && (type2 == GPU || type2 == NVS); float bw = intra ? graph->bwIntra : graph->bwInter; int type = intra ? graph->typeIntra : graph->typeInter; if (path->type >= PATH_DIS) return ncclSuccess; if (mult == 1 && (path->type > type)) return ncclSuccess; if (mult == 1 && (graph->pattern == NCCL_TOPO_PATTERN_BALANCED_TREE || graph->pattern == NCCL_TOPO_PATTERN_TREE || graph->pattern == NCCL_TOPO_PATTERN_SPLIT_TREE) && (revPath->type > type)) return ncclSuccess; bw *= mult; // Check there is enough bandwidth on paths. int step = 0; NCCLCHECK(followPath(path, node1, path->count, bw, &step)); if (step < path->count) goto rewind; // Enough bandwidth : return destination node. graph->nHops += mult*path->count; *node = system->nodes[type2].nodes+index2; return ncclSuccess; rewind: // Not enough bandwidth : rewind and exit. NCCLCHECK(followPath(path, node1, step, -bw, &step)); return ncclSuccess; } static int gpuPciBw(struct ncclTopoNode* gpu) { for (int l=0; lnlinks; l++) { struct ncclTopoLink* gpuLink = gpu->links+l; if (gpuLink->type != LINK_PCI) continue; struct ncclTopoNode* pci = gpuLink->remNode; for (int l=0; lnlinks; l++) { struct ncclTopoLink* pciLink = pci->links+l; if (pciLink->remNode != gpu) continue; return std::min(gpuLink->bw, pciLink->bw); } } return -1; } /* Choose the order in which we try next GPUs. This is critical for the search to quickly converge to the best solution even if it eventually times out. */ struct ncclGpuScore { int g; // Retain the index int startIndex; // Least important int intraNhops; int intraBw; int interNhops; int interPciBw; int interBw; // Most important }; static int cmpScore(const void * g1, const void * g2) { struct ncclGpuScore *s1 = (struct ncclGpuScore*)g1; struct ncclGpuScore *s2 = (struct ncclGpuScore*)g2; int d; if ((d = (s2->interBw - s1->interBw))) return d; if ((d = (s2->interPciBw - s1->interPciBw))) return d; if ((d = (s1->interNhops - s2->interNhops))) return d; if ((d = (s2->intraBw - s1->intraBw))) return d; if ((d = (s1->intraNhops - s2->intraNhops))) return d; return s1->startIndex - s2->startIndex; } static int cmpIntraScores(struct ncclGpuScore* scores, int count) { int intraBw = scores[0].intraBw; int intraNhops = scores[0].intraNhops; for (int i=1; inodes[GPU].count; g++) { if (system->nodes[GPU].nodes[g].gpu.rank == rank) { *index = g; return ncclSuccess; } } WARN("Could not find gpu rank %d", rank); return ncclInternalError; } static ncclResult_t getNetIndex(struct ncclTopoSystem* system, int64_t id, int* index) { for (int n=0; nnodes[NET].count; n++) { if (system->nodes[NET].nodes[n].id == id) { *index = n; return ncclSuccess; } } WARN("Could not find net id %lx", id); return ncclInternalError; } static ncclResult_t getNetPaths(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoLinkList** netPaths) { int64_t netId = graph->inter[graph->nChannels*2]; int n; NCCLCHECK(getNetIndex(system, netId, &n)); *netPaths=system->nodes[NET].nodes[n].paths[GPU]; return ncclSuccess; } ncclResult_t ncclTopoSearchNextGpuSort(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoNode* gpu, int* next, int* countPtr, int sortNet) { const uint64_t flag = 1ULL<<(graph->nChannels); int ngpus = system->nodes[GPU].count; struct ncclTopoLinkList* paths = gpu->paths[GPU]; struct ncclTopoLinkList* netPaths = NULL; if (sortNet) NCCLCHECK(getNetPaths(system, graph, &netPaths)); struct ncclGpuScore scores[NCCL_TOPO_MAX_NODES]; memset(scores, 0, ngpus*sizeof(struct ncclGpuScore)); int start = gpu-system->nodes[GPU].nodes; int count = 0; for (int i=1; inodes[GPU].nodes[g].used & flag) continue; scores[count].g = g; scores[count].startIndex = i; scores[count].intraNhops = paths[g].count; scores[count].intraBw = paths[g].bw; if (netPaths) { scores[count].interNhops = netPaths[g].count; scores[count].interPciBw = gpuPciBw(system->nodes[GPU].nodes+g); scores[count].interBw = netPaths[g].bw; } count++; } // Sort GPUs qsort(scores, count, sizeof(struct ncclGpuScore), cmpScore); // Check if all have the same intra-node score in which case we go reverse for sortNet = -1 if (sortNet == -1 && cmpIntraScores(scores, count) == 0) { for (int i=0; inodes[NVS].count) { // NVSwitches prefer when we talk to a limited set of peers. Try to use neighbors first. int index = gpu-system->nodes[GPU].nodes; int i; int prevGpu = (index-1+ngpus)%ngpus; int nextGpu = (index+1)%ngpus; int firstGpus[2]; int firstGpuCount = 0; if (graph->pattern == NCCL_TOPO_PATTERN_RING) { firstGpus[0] = nextGpu; firstGpus[1] = prevGpu; firstGpuCount = 2; } else if (graph->pattern == NCCL_TOPO_PATTERN_SPLIT_TREE || graph->pattern == NCCL_TOPO_PATTERN_BALANCED_TREE) { firstGpus[0] = prevGpu; firstGpus[1] = nextGpu; firstGpuCount = 2; } else { firstGpus[0] = nextGpu; firstGpuCount = 1; } if (nextGpu == prevGpu && firstGpuCount == 2) firstGpuCount = 1; int firstGpuRealCount = 0; for (int g=0; g0; i--) next[i] = next[i-1]; next[0] = firstGpus[g]; firstGpuRealCount++; } } *countPtr = firstGpuRealCount; } return ncclSuccess; } ncclResult_t ncclTopoSearchRec(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int* time); // Try to keep all searchs within one second #define NCCL_SEARCH_GLOBAL_TIMEOUT (1ULL<<19) #define NCCL_SEARCH_TIMEOUT (1<<14) #define NCCL_SEARCH_TIMEOUT_TREE (1<<14) #define NCCL_SEARCH_TIMEOUT_SAMECHANNELS (1<<8) #define FORCED_ORDER_PCI 1 #define FORCED_ORDER_REPLAY 2 ncclResult_t ncclTopoReplayGetGpu(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, int step, int* g) { *g = -1; if (graph->nChannels == 0) return ncclInternalError; int ngpus = system->nodes[GPU].count; int nextRank = graph->intra[(graph->nChannels-1)*ngpus+step+1]; for (int i=0; inodes[GPU].nodes[i].gpu.rank == nextRank) { *g = i; return ncclSuccess; } return ncclInternalError; } ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, struct ncclTopoNode* gpu, int step, int backToNet, int backToFirstRank, int forcedOrder, int *time); ncclResult_t ncclTopoSearchTryGpu(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int step, int backToNet, int backToFirstRank, int forcedOrder, int *time, int type, int index, int g) { const uint64_t flag = 1ULL<<(graph->nChannels); struct ncclTopoNode* gpu; NCCLCHECK(ncclTopoFollowPath(system, graph, type, index, GPU, g, 1, &gpu)); if (gpu) { gpu->used ^= flag; NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, gpu, step, backToNet, backToFirstRank, forcedOrder, time)); gpu->used ^= flag; NCCLCHECK(ncclTopoFollowPath(system, graph, type, index, GPU, g, -1, &gpu)); } return ncclSuccess; } ncclResult_t ncclTopoSearchTryCollnetDirect(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int g, int ngpus, int *time) { int fwdg = 0; int bwdg = 0; struct ncclTopoNode* gpu = NULL; float mul = 1.0 / (float)(system->nodes[GPU].count - 1); do { NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, g, GPU, fwdg, mul, &gpu)); } while (gpu && ++fwdg < system->nodes[GPU].count); if (gpu != NULL) { do { NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, bwdg, GPU, g, mul, &gpu)); } while (gpu && ++bwdg < system->nodes[GPU].count); if (gpu != NULL) { // Both directions worked. Now we already have head, so pop the all other intra ranks. int step = 1; for (int index = 0; index < ngpus; ++index) { if (index != g) { graph->intra[graph->nChannels * ngpus + step] = system->nodes[GPU].nodes[index].gpu.rank; step++; } } NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, NULL, ngpus, -1, -1, 0, time)); } while (bwdg) { bwdg--; NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, bwdg, GPU, g, -mul, &gpu)); } } while (fwdg) { fwdg--; NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, g, GPU, fwdg, -mul, &gpu)); } return ncclSuccess; } ncclResult_t ncclTopoSearchTryNvls(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int g, int ngpus, int *time) { struct ncclTopoNode* nvs; struct ncclTopoNode* gpu; int d0=0; // See if there is enough bandwidth for NVS->GPU traffic do { NCCLCHECK(ncclTopoFollowPath(system, graph, NVS, 0, GPU, d0, d0 == g ? 2 : 1, &gpu)); d0++; } while (gpu && d0 < system->nodes[GPU].count); if (gpu == NULL) { d0--; } else { int d1=0; // See if there is enough bandwidth for GPU->NVS traffic do { NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, d1, NVS, 0, d1 == g ? 2 : 1, &nvs)); d1++; } while (nvs && d1 < system->nodes[GPU].count); if (nvs == NULL) { d1--; } else { // Both directions worked. Move on to the next path. NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, NULL, ngpus, -1, -1, 0, time)); } while (d1) { d1--; NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, d1, NVS, 0, d1 == g ? -2 : -1, &nvs)); } } while (d0) { d0--; NCCLCHECK(ncclTopoFollowPath(system, graph, NVS, 0, GPU, d0, d0 == g ? -2 : -1, &gpu)); } return ncclSuccess; } ncclResult_t ncclTopoCompareGraphs(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* refGraph, int* copy) { // 1. Try to get the same nChannels between Rings and Trees if (graph->nChannels < graph->minChannels) return ncclSuccess; if (graph->pattern == NCCL_TOPO_PATTERN_NVLS) { // NVLS channels correspond to GPUs pulling from NVLS. So the more the better. if (graph->nChannels > refGraph->nChannels && graph->nChannels <= system->nodes[GPU].count) *copy = 1; if (graph->nChannels*graph->bwInter > refGraph->nChannels*refGraph->bwInter) *copy = 1; return ncclSuccess; } // 2. Try to get better bandwidth if (graph->nChannels*graph->bwIntra > refGraph->nChannels*refGraph->bwIntra) { *copy = 1; return ncclSuccess; } if (graph->nChannels*graph->bwIntra < refGraph->nChannels*refGraph->bwIntra) return ncclSuccess; // 3. Less hops if (graph->pattern == refGraph->pattern && graph->crossNic == refGraph->crossNic && graph->nHops < refGraph->nHops) *copy = 1; return ncclSuccess; } // Add the preferred NICs ordered by GPU first static ncclResult_t ncclTopoPrefNetsGpuFirst(struct ncclTopoSystem* system, int gpu, int nets[NCCL_TOPO_MAX_NODES], int* netCount) { const int nGpus = (gpu == -1) ? system->nodes[GPU].count : 1; int gpuCount = nGpus; int gpuIds[NCCL_TOPO_MAX_NODES] = {gpu}; int firstNets[NCCL_TOPO_MAX_NODES]; if (gpu == -1) for (int g = 0; g < nGpus; g++) gpuIds[g] = g; for (int c = 0; c < MAXCHANNELS; c++) { for (int g = 0; g < nGpus; g++) { if (gpuIds[g] == -1) continue; int localNet; int64_t netId; struct ncclTopoNode* gpu = system->nodes[GPU].nodes + gpuIds[g]; NCCLCHECK(ncclTopoGetLocalNet(system, gpu->gpu.rank, c, &netId, NULL)); NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, &localNet)); // store the first net found for each GPU in case of duplicates if(c == 0) firstNets[g] = localNet; // if the NET has already been returned for channel 0, that GPU is done if (c > 0 && firstNets[g] == localNet) { gpuIds[g] = -1; gpuCount--; continue; } // only add it to the list if it doesn't already exist int found = 0; while (found < (*netCount) && nets[found] != localNet) found++; if (found == (*netCount)) nets[(*netCount)++] = localNet; } if (gpuCount == 0) break; } return ncclSuccess; } // Add the preferred NICs ordered by channels first static ncclResult_t ncclTopoPrefNetsChannelFirst(struct ncclTopoSystem* system, int gpu, int nets[NCCL_TOPO_MAX_NODES], int* netCount) { for (int g = 0; g < system->nodes[GPU].count; g++) { if (gpu != -1 && gpu != g) continue; int localNetCount = 0, localNets[MAXCHANNELS]; struct ncclTopoNode* gpu = system->nodes[GPU].nodes + g; for (int c = 0; c < MAXCHANNELS; c++) { int64_t netId; NCCLCHECK(ncclTopoGetLocalNet(system, gpu->gpu.rank, c, &netId, NULL)); NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, localNets + localNetCount)); if (localNetCount > 0 && localNets[localNetCount] == localNets[0]) break; localNetCount++; } // Append NICs to list for (int i = 0; i < localNetCount; i++) { int n = localNets[i]; int found = 0; while (found < (*netCount) && nets[found] != n) found++; if (found == (*netCount)) nets[(*netCount)++] = n; } } return ncclSuccess; } // Build a sorted list of the NETs to try, the list will follow the NETDEVS_POLICY set by the user. // // The value of "gpu" can be set to -1 to build a list suitable for all GPUs (for example for the search start). // The value of "gpu" can be set to the desired index when trying to get back to the NIC. // // The list is built the following way: // 1. First gather the preferred NETs for each of the GPU(s), based on the NETDEVS_POLICY and the connection. // 2. If the NETDEV_policy allows it, add all the other NETs satisfying typeInter but not already in the list of preferred NETs. NCCL_PARAM(ScatterEnable, "MNNVL_SCATTER_NETS_ENABLE", 1); ncclResult_t ncclTopoSelectNets(struct ncclTopoSystem* system, int typeInter, int gpu, int nets[NCCL_TOPO_MAX_NODES], int* netCountRet) { ncclResult_t ret = ncclSuccess; int netCount = 0; // First add the preferred NETs. if (system->nHosts > 1 && ncclParamScatterEnable()) { // For MNNVL systems, we sort the devices by GPU first, then by channel NCCLCHECK(ncclTopoPrefNetsGpuFirst(system, gpu, nets, &netCount)); } else { // For other systems, we sort the devices by channel first, then by GPU NCCLCHECK(ncclTopoPrefNetsChannelFirst(system, gpu, nets, &netCount)); } // Get the maximum of network devices allowed, depending on the policy. // If the policy is not MAX, then allow all devices. int maxDevCount = 0; enum netDevsPolicy netDevsPolicy; NCCLCHECK(ncclTopoGetNetDevsPolicy(&netDevsPolicy, &maxDevCount)); if (gpu == -1) maxDevCount *= system->nodes[GPU].count; if (netDevsPolicy != NETDEVS_POLICY_MAX) maxDevCount = NCCL_TOPO_MAX_NODES; if (netCount >= maxDevCount) goto exit; // Then add others satisfying typeInter for (int t=0; t <= typeInter; t++) { for (int g = 0; g < system->nodes[GPU].count; g++) { // do not consider this GPU is it's not the GPU we asked for if (gpu != -1 && gpu != g) continue; int localNetCount = 0, localNets[MAXCHANNELS]; struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g; struct ncclTopoLinkList* paths = gpu->paths[NET]; for (int n=0; nnodes[NET].count && n= maxDevCount) goto exit; } } } exit: *netCountRet = netCount; return ret; } NCCL_PARAM(MnnvlRailPerHost, "MNNVL_RAIL_PER_HOST", 0); static bool ncclTopoSearchCheckNet(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoNode* startNet, int n, int step) { struct ncclTopoNode* net = system->nodes[NET].nodes+n; if (graph->pattern == NCCL_TOPO_PATTERN_TREE && net->id != startNet->id) return false; // Trees are symmetric if (graph->pattern == NCCL_TOPO_PATTERN_RING && graph->crossNic == 2) { if (graph->nChannels & 1 && net->id != graph->inter[(graph->nChannels - 1) * 2]) return false; } else if (graph->crossNic == 0) { if (ncclParamMnnvlRailPerHost() && NCCL_TOPO_ID_SYSTEM_ID(net->id) != NCCL_TOPO_ID_SYSTEM_ID(startNet->id)) { // Different hosts in an MNNVL system: rail are per host and identified with the PCI id. if (net->net.pciId != startNet->net.pciId || net->net.port != startNet->net.port) return false; } else { if (net->net.asic != startNet->net.asic || net->net.port != startNet->net.port) return false; } } if (graph->pattern == NCCL_TOPO_PATTERN_BALANCED_TREE && step != 0 && net->id != graph->inter[graph->nChannels*2+1]) return false; return true; } ncclResult_t ncclTopoSearchRecGpu(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, struct ncclTopoNode* gpu, int step, int backToNet, int backToFirstRank, int forcedOrder, int *time) { if ((*time) <= 0) return ncclSuccess; (*time)--; int ngpus = system->nodes[GPU].count; if (step == ngpus) { // Determine whether we found a better solution or not int copy = 0; graph->nChannels++; NCCLCHECK(ncclTopoCompareGraphs(system, graph, saveGraph, ©)); if (copy) { memcpy(saveGraph, graph, sizeof(struct ncclTopoGraph)); if (graph->nChannels == graph->maxChannels) *time = -1; } if (graph->nChannels < graph->maxChannels) { NCCLCHECK(ncclTopoSearchRec(system, graph, saveGraph, time)); } graph->nChannels--; return ncclSuccess; } graph->intra[graph->nChannels*ngpus+step] = gpu->gpu.rank; int g = gpu - system->nodes[GPU].nodes; int nets[NCCL_TOPO_MAX_NODES]; if (step == backToNet) { // first get back to NIC if (system->inter) { int startNetIndex; NCCLCHECK(getNetIndex(system, graph->inter[graph->nChannels*2], &startNetIndex)); struct ncclTopoNode* startNet = system->nodes[NET].nodes+startNetIndex; int netCount; NCCLCHECK(ncclTopoSelectNets(system, graph->typeInter, g, nets, &netCount)); for (int i=0; ibwInter; if (graph->pattern == NCCL_TOPO_PATTERN_BALANCED_TREE) { // Count half of the bandwidth on each of the first two GPUs if (step == 0) nextBackToNet = 1; graph->bwInter /= 2; } struct ncclTopoNode* net; NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, g, NET, n, 1, &net)); graph->bwInter = bwInterSave; if (net) { graph->inter[graph->nChannels*2+1] = net->id; NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, gpu, step, nextBackToNet, backToFirstRank, forcedOrder, time)); if (graph->pattern == NCCL_TOPO_PATTERN_BALANCED_TREE) graph->bwInter /= 2; NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, g, NET, n, -1, &net)); graph->bwInter = bwInterSave; } } } } else if (graph->pattern == NCCL_TOPO_PATTERN_NVLS) { NCCLCHECK(ncclTopoSearchTryNvls(system, graph, saveGraph, g, ngpus, time)); } else if (graph->pattern == NCCL_TOPO_PATTERN_COLLNET_DIRECT) { NCCLCHECK(ncclTopoSearchTryCollnetDirect(system, graph, saveGraph, g, ngpus, time)); } else if (step < system->nodes[GPU].count-1) { // Go to next GPU int next[NCCL_TOPO_MAX_NODES]; int count; if (forcedOrder == FORCED_ORDER_PCI) { // Try the PCI order next[0] = step+1; count = 1; } else if (forcedOrder == FORCED_ORDER_REPLAY) { // Try last channel order NCCLCHECK(ncclTopoReplayGetGpu(system, graph, step, next)); count = 1; } else { // Normal search NCCLCHECK(ncclTopoSearchNextGpuSort(system, graph, gpu, next, &count, backToNet == -1 ? 0 : backToNet == step+1 ? 1 : -1 )); } for (int i=0; iintra[graph->nChannels*ngpus], &p)); struct ncclTopoNode* firstGpu; NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, g, GPU, p, 1, &firstGpu)); if (firstGpu) { NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, firstGpu, step+1, backToNet, -1, forcedOrder, time)); NCCLCHECK(ncclTopoFollowPath(system, graph, GPU, g, GPU, p, -1, &firstGpu)); } } else { // Next path NCCLCHECK(ncclTopoSearchRecGpu(system, graph, saveGraph, gpu, ngpus, -1, -1, forcedOrder, time)); } return ncclSuccess; } ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int backToNet, int backToFirstRank, int* time) { const int bw = graph->bwInter; int nets[NCCL_TOPO_MAX_NODES]; int netCount; int graphFound = 0; NCCLCHECK(ncclTopoSelectNets(system, graph->typeInter, -1, nets, &netCount)); for (int i=0; ipattern == NCCL_TOPO_PATTERN_NVLS || graph->pattern == NCCL_TOPO_PATTERN_COLLNET_DIRECT) && graphFound) break; int n = nets[(graph->nChannels+i)%netCount]; struct ncclTopoNode* net = system->nodes[NET].nodes+n; if (graph->collNet && net->net.collSupport == 0) continue; if (net->net.bw < bw) continue; if (graph->pattern == NCCL_TOPO_PATTERN_RING && graph->crossNic == 2 && (graph->nChannels & 1) && net->id != graph->inter[(graph->nChannels-1)*2+1]) continue; graph->inter[graph->nChannels*2] = net->id; graph->latencyInter = net->net.latency; for (int i=0; inodes[NET].count; i++) { if ((system->nodes[NET].nodes[i].net.asic == net->net.asic) && (system->nodes[NET].nodes[i].net.port == net->net.port)) { system->nodes[NET].nodes[i].net.bw -= bw; } } if (graph->pattern == NCCL_TOPO_PATTERN_NVLS || graph->pattern == NCCL_TOPO_PATTERN_COLLNET_DIRECT) { // NVLS search only tries to find NIC:GPU combinations to compute the heads. if (graph->nChannels < netCount) { int gpu = net->net.localGpu; if (gpu != -1) { int duplicate = 0; // check whether there is duplicate head when one GPU connects with multiple NICs for (int gc = 0; gc < graph->nChannels; gc++) { if (graph->intra[gc * system->nodes[GPU].count] == system->nodes[GPU].nodes[gpu].gpu.rank) { duplicate = 1; break; } } if (!duplicate) { NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, NET, n, gpu)); graphFound = 1; } } } } else { if (graph->nChannels > 0 && graph->sameChannels == 1) { // Try to replay the last channel int g; NCCLCHECK(ncclTopoReplayGetGpu(system, graph, -1, &g)); NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_REPLAY, time, NET, n, g)); } else { if (graph->nChannels == 0 && system->nodes[NVS].count == 0) { // Always try the PCI order first to set a reference, but don't count in the timeout nor let it run for long int t = 1 << 10; NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_PCI, &t, NET, n, 0)); if (t == -1) *time = -1; } // Then try the most local GPUs int localGpu = net->net.localGpu; if (localGpu != -1) { NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, NET, n, localGpu)); } int localGpus[NCCL_TOPO_MAX_NODES], localGpuCount, pathType; NCCLCHECK(ncclTopoGetLocal(system, NET, n, GPU, localGpus, &localGpuCount, &pathType)); // if no GPUs are connected, skip this net if (pathType == PATH_DIS) continue; for (int g = 0; g < localGpuCount; ++g) { if (localGpus[g] == localGpu) continue; // We already tried this one NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, NET, n, localGpus[g])); } } } for (int i=0; inodes[NET].count; i++) { if ((system->nodes[NET].nodes[i].net.asic == net->net.asic) && (system->nodes[NET].nodes[i].net.port == net->net.port)) { system->nodes[NET].nodes[i].net.bw += bw; } } } return ncclSuccess; } /* Search Patterns * * Intra-node * Ring : GPU a -> GPU b -> .. -> GPU x -> GPU a * (=Split Tree Loop) * Tree : GPU a -> GPU b -> .. -> GPU x * (=Split Tree) * * Inter-node * Ring : NET n -> GPU a -> GPU b -> .. -> GPU x -> NET n (or m if crossNic) * Tree : NET n -> GPU a -> GPU b -> .. -> GPU x * `--> NET n (or m if crossNic) * Split Tree : NET n -> GPU a -> GPU b -> .. -> GPU x * `--> NET n (or m if crossNic) * Split Tree Loop : NET n -> GPU a -> GPU b -> .. -> GPU x -> GPU a * `--> NET n (or m if crossNic) */ ncclResult_t ncclTopoSearchParams(struct ncclTopoSystem* system, int pattern, int* backToNet, int* backToFirstRank) { if (system->inter) { if (pattern == NCCL_TOPO_PATTERN_RING) *backToNet = system->nodes[GPU].count-1; else if (pattern == NCCL_TOPO_PATTERN_SPLIT_TREE) *backToNet = 1; else *backToNet = 0; *backToFirstRank = -1; } else { *backToNet = -1; if (pattern == NCCL_TOPO_PATTERN_RING) *backToFirstRank = system->nodes[GPU].count-1; else *backToFirstRank = -1; } return ncclSuccess; } ncclResult_t ncclTopoSearchRec(struct ncclTopoSystem* system, struct ncclTopoGraph* graph, struct ncclTopoGraph* saveGraph, int* time) { int backToNet, backToFirstRank; NCCLCHECK(ncclTopoSearchParams(system, graph->pattern, &backToNet, &backToFirstRank)); if (system->inter) { // Start from NET ncclTopoSearchRecNet(system, graph, saveGraph, backToNet, backToFirstRank, time); } else { // Intra-node only. if (graph->pattern == NCCL_TOPO_PATTERN_NVLS) { NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, -1, -1, graph->nChannels)); return ncclSuccess; } else if (graph->nChannels == 0) { // Try PCI order first NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_PCI, time, -1, -1, 0)); } else { // Also try to replay previous channel int g; NCCLCHECK(ncclTopoReplayGetGpu(system, graph, -1, &g)); NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_REPLAY, time, -1, -1, g)); } if (graph->sameChannels == 0 || graph->nChannels == 0) { // Finally, try all other possibilities unless we are forced to use the same channels for (int g=0; gnodes[GPU].count; g++) { NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, 0, time, -1, -1, g)); } } } return ncclSuccess; } /************************************/ /* User defined graph from XML file */ /************************************/ struct kvDict kvDictLinkType[] = { { "LOC", PATH_LOC }, { "NVL", PATH_NVL }, { "NVB", PATH_NVB }, { "PIX", PATH_PIX }, { "PXB", PATH_PXB }, { "P2C", PATH_P2C }, { "PXN", PATH_PXN }, { "PHB", PATH_PHB }, { "SYS", PATH_SYS }, { NULL, 0 } }; ncclResult_t ncclTopoGetChannelFromXml(struct ncclXmlNode *xmlChannel, int c, struct ncclTopoSystem* system, struct ncclTopoGraph* graph) { int ngpus = system->nodes[GPU].count; int64_t* inter = graph->inter+2*c; int* intra = graph->intra+ngpus*c; int n=0, g=0; for (int s=0; snSubs; s++) { struct ncclXmlNode* sub = xmlChannel->subs[s]; int64_t dev; const char* str; NCCLCHECK(xmlGetAttrStr(sub, "dev", &str)); dev = strtol(str, NULL, 16); if (strcmp(sub->name, "net") == 0) { inter[n++] = dev; } else if (strcmp(sub->name, "gpu") == 0) { int rank = -1; for (int g=0; gnodes[GPU].nodes[g].id); if (NCCL_TOPO_ID(systemId, system->nodes[GPU].nodes[g].gpu.dev) == dev) rank = system->nodes[GPU].nodes[g].gpu.rank; } if (rank == -1) { WARN("XML Import Channel : dev %ld not found.", dev); return ncclSystemError; } intra[g++] = rank; } } return ncclSuccess; } ncclResult_t ncclTopoGetGraphFromXmlSub(struct ncclXmlNode *xmlGraph, struct ncclTopoSystem* system, struct ncclTopoGraph* graph, int* nChannels) { int id; NCCLCHECK(xmlGetAttrInt(xmlGraph, "id", &id)); if (graph->id != id) return ncclSuccess; int crossNic; NCCLCHECK(xmlGetAttrInt(xmlGraph, "crossnic", &crossNic)); if (ncclParamCrossNic() == 0 && crossNic == 1) return ncclSuccess; graph->crossNic = crossNic; NCCLCHECK(xmlGetAttrInt(xmlGraph, "pattern", &graph->pattern)); NCCLCHECK(xmlGetAttrInt(xmlGraph, "nchannels", &graph->nChannels)); NCCLCHECK(xmlGetAttrFloat(xmlGraph, "speedintra", &graph->bwIntra)); NCCLCHECK(xmlGetAttrFloat(xmlGraph, "speedinter", &graph->bwInter)); const char* str; NCCLCHECK(xmlGetAttr(xmlGraph, "latencyinter", &str)); if (!str) INFO(NCCL_GRAPH, "latencyinter not found in graph, using 0.0"); graph->latencyInter = str ? strtof(str, NULL) : 0.0; NCCLCHECK(xmlGetAttr(xmlGraph, "typeintra", &str)); NCCLCHECK(kvConvertToInt(str, &graph->typeIntra, kvDictLinkType)); NCCLCHECK(xmlGetAttr(xmlGraph, "typeinter", &str)); NCCLCHECK(kvConvertToInt(str, &graph->typeInter, kvDictLinkType)); NCCLCHECK(xmlGetAttrInt(xmlGraph, "samechannels", &graph->sameChannels)); for (int s=0; snSubs; s++) { NCCLCHECK(ncclTopoGetChannelFromXml(xmlGraph->subs[s], s, system, graph)); } *nChannels = xmlGraph->nSubs; return ncclSuccess; } ncclResult_t ncclTopoGetGraphFromXml(struct ncclXmlNode *xmlGraphs, struct ncclTopoSystem* system, struct ncclTopoGraph* graph, int* nChannels) { for (int s=0; snSubs; s++) { NCCLCHECK(ncclTopoGetGraphFromXmlSub(xmlGraphs->subs[s], system, graph, nChannels)); } return ncclSuccess; } /* And the reverse : graph->xml */ ncclResult_t ncclTopoGetXmlFromChannel(struct ncclTopoGraph* graph, int c, struct ncclTopoSystem* system, struct ncclXml *xml, struct ncclXmlNode* parent) { struct ncclXmlNode* xmlChannel; int ngpus = system->nodes[GPU].count; int64_t* inter = graph->inter+2*c; int* intra = graph->intra+ngpus*c; NCCLCHECK(xmlAddNode(xml, parent, "channel", &xmlChannel)); struct ncclXmlNode* node; if (system->inter) { NCCLCHECK(xmlAddNode(xml, xmlChannel, "net", &node)); NCCLCHECK(xmlSetAttrLong(node, "dev", inter[0])); } for (int g=0; gnodes[GPU].nodes[i].gpu.rank == intra[g]) { int systemId = NCCL_TOPO_ID_SYSTEM_ID(system->nodes[GPU].nodes[i].id); dev = NCCL_TOPO_ID(systemId, system->nodes[GPU].nodes[i].gpu.dev); } } if (dev == -1) { WARN("XML Export Channel : rank %d not found.", intra[g]); return ncclInternalError; } NCCLCHECK(xmlSetAttrLong(node, "dev", dev)); if (graph->id == 3) break; // NVLS graphs only use the first GPU } if (system->inter) { NCCLCHECK(xmlAddNode(xml, xmlChannel, "net", &node)); NCCLCHECK(xmlSetAttrLong(node, "dev", inter[1])); } return ncclSuccess; } ncclResult_t ncclTopoGetXmlFromGraph(struct ncclTopoGraph* graph, struct ncclTopoSystem* system, struct ncclXml *xml, struct ncclXmlNode* parent) { struct ncclXmlNode* xmlGraph; NCCLCHECK(xmlAddNode(xml, parent, "graph", &xmlGraph)); NCCLCHECK(xmlSetAttrInt(xmlGraph, "id", graph->id)); NCCLCHECK(xmlSetAttrInt(xmlGraph, "pattern", graph->pattern)); NCCLCHECK(xmlSetAttrInt(xmlGraph, "crossnic", graph->crossNic)); NCCLCHECK(xmlSetAttrInt(xmlGraph, "nchannels", graph->nChannels)); NCCLCHECK(xmlSetAttrFloat(xmlGraph, "speedintra", graph->bwIntra)); NCCLCHECK(xmlSetAttrFloat(xmlGraph, "speedinter", graph->bwInter)); NCCLCHECK(xmlSetAttrFloat(xmlGraph, "latencyinter", graph->latencyInter)); const char* str; NCCLCHECK(kvConvertToStr(graph->typeIntra, &str, kvDictLinkType)); NCCLCHECK(xmlSetAttr(xmlGraph, "typeintra", str)); NCCLCHECK(kvConvertToStr(graph->typeInter, &str, kvDictLinkType)); NCCLCHECK(xmlSetAttr(xmlGraph, "typeinter", str)); NCCLCHECK(xmlSetAttrInt(xmlGraph, "samechannels", graph->sameChannels)); for (int c=0; cnChannels; c++) { NCCLCHECK(ncclTopoGetXmlFromChannel(graph, c, system, xml, xmlGraph)); } return ncclSuccess; } ncclResult_t ncclTopoGetXmlFromGraphs(int ngraphs, struct ncclTopoGraph** graphs, struct ncclTopoSystem* system, struct ncclXml *xml) { xml->maxIndex = 0; struct ncclXmlNode* xmlGraphs; NCCLCHECK(xmlAddNode(xml, NULL, "graphs", &xmlGraphs)); NCCLCHECK(xmlSetAttrInt(xmlGraphs, "version", NCCL_GRAPH_XML_VERSION)); for (int g=0; gnChannels == 0) return ncclSuccess; if (graph->pattern == NCCL_TOPO_PATTERN_NVLS) return ncclSuccess; if (graph->bwIntra < 25.0) return ncclSuccess; if (ccMin > 80 && graph->bwIntra < 50.0 && graph->nChannels > 4) return ncclSuccess; int dupChannels = std::min(graph->nChannels*2, graph->maxChannels); memcpy(graph->intra+graph->nChannels*ngpus, graph->intra, (dupChannels-graph->nChannels)*ngpus*sizeof(int)); memcpy(graph->inter+graph->nChannels*2,graph->inter, (dupChannels-graph->nChannels)*2*sizeof(int64_t)); graph->bwIntra /= DIVUP(dupChannels, graph->nChannels); graph->bwInter /= DIVUP(dupChannels, graph->nChannels); graph->nChannels = dupChannels; return ncclSuccess; } float speedArrayIntra[] = { 40.0, 30.0, 20.0, 18.0, 15.0, 12.0, 10.0, 9.0, 7.0, 6.0, 5.0, 4.0, 3.0 }; float speedArrayInter[] = { 48.0, 30.0, 28.0, 24.0, 20.0, 18.0, 15.0, 12.0, 10.0, 9.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.4, 1.2, 0.24, 0.12 }; #define NSPEEDSINTRA (sizeof(speedArrayIntra)/sizeof(float)) #define NSPEEDSINTER (sizeof(speedArrayInter)/sizeof(float)) float sm90SpeedArrayIntra[] = { 60.0, 50.0, 40.0, 30.0, 24.0, 20.0, 15.0, 12.0, 11.0, 6.0, 3.0 }; float sm90SpeedArrayInter[] = { 48.0, 45.0, 42.0, 40.0, 30.0, 24.0, 22.0, 20.0, 17.5, 15.0, 12.0, 6.0, 3.0, 2.4, 1.2, 0.24, 0.12 }; #define NSPEEDSINTRA_SM90 (sizeof(sm90SpeedArrayIntra)/sizeof(float)) #define NSPEEDSINTER_SM90 (sizeof(sm90SpeedArrayInter)/sizeof(float)) float sm100SpeedArrayIntra[] = { 90.0, 80.0, 70.0, 60.0, 50.0, 45.0, 40.0, 30.0, 24.0, 20.0, 19.0, 18.0 }; float sm100SpeedArrayInter[] = { 96.0, 80.0, 48.0, 45.1, 42.0, 40.0, 30.0, 24.0, 22.0, 20.0, 17.5, 15.0, 12.0, 6.0, 3.0, 2.4, 1.2, 0.24, 0.12 }; #define NSPEEDSINTRA_SM100 (sizeof(sm100SpeedArrayIntra)/sizeof(float)) #define NSPEEDSINTER_SM100 (sizeof(sm100SpeedArrayInter)/sizeof(float)) ncclResult_t ncclTopoCheckCrossNicSupport(bool* supported) { *supported = (ncclParamCrossNic() != 0); return ncclSuccess; } ncclResult_t ncclTopoCheckNicFused(struct ncclComm* comm, bool* fused) { int nDev = 0; int devId = 0; ncclNetProperties_t props; *fused = false; NCCLCHECK(comm->ncclNet->devices(&nDev)); while (devId < nDev) { NCCLCHECK(comm->ncclNet->getProperties(devId, &props)); if (props.vProps.ndevs > 1) { *fused = true; goto exit; } devId++; } exit: return ncclSuccess; } ncclResult_t ncclTopoCompute(ncclTopoSystem* system, struct ncclTopoGraph* graph) { int ccMin; NCCLCHECK(ncclTopoGetCompCap(system, &ccMin, NULL)); int ngpus = system->nodes[GPU].count; int crossNic = (system->nodes[NET].count > 1) && (graph->pattern == NCCL_TOPO_PATTERN_RING || graph->pattern == NCCL_TOPO_PATTERN_BALANCED_TREE || graph->pattern == NCCL_TOPO_PATTERN_SPLIT_TREE) ? ncclParamCrossNic() : 0; graph->crossNic = crossNic == 1 ? 1 : 0; graph->bwIntra = graph->bwInter = 0; graph->latencyInter = 0; int minTypeIntra = PATH_LOC, minTypeInter = PATH_PIX; int maxTypeIntra = PATH_SYS, maxTypeInter = PATH_SYS; if (ngpus > 1) { NCCLCHECK(ncclTopoGetGpuMinPath(system, GPU, &minTypeIntra)); NCCLCHECK(ncclTopoGetGpuMaxPath(system, GPU, &maxTypeIntra)); } if (system->inter) { NCCLCHECK(ncclTopoGetGpuMinPath(system, NET, &minTypeInter)); NCCLCHECK(ncclTopoGetGpuMaxPath(system, NET, &maxTypeInter)); maxTypeIntra = maxTypeInter; } // Ampere relies on BALANCED_TREE which sometimes needs to come back through SYS or PHB. if (ccMin < 90) maxTypeInter = PATH_SYS; graph->typeIntra = minTypeIntra; graph->typeInter = minTypeInter; graph->nChannels = 0; int trySameChannels = graph->pattern == NCCL_TOPO_PATTERN_NVLS ? 0 : 1; graph->sameChannels = trySameChannels; int cpuArch, cpuVendor, cpuModel; NCCLCHECK(ncclTopoCpuType(system, &cpuArch, &cpuVendor, &cpuModel)); const char* str = ncclGetEnv("NCCL_GRAPH_FILE"); if (str) { INFO(NCCL_ENV, "NCCL_GRAPH_FILE set by environment to %s", str); struct ncclXml* xml; NCCLCHECK(xmlAlloc(&xml, NCCL_GRAPH_XML_MAX_NODES)); NCCLCHECK(ncclTopoGetXmlGraphFromFile(str, xml)); int nChannels; NCCLCHECK(ncclTopoGetGraphFromXml(xml->nodes, system, graph, &nChannels)); INFO(NCCL_GRAPH, "Search %d : %d channels loaded from XML graph", graph->id, nChannels); free(xml); if (graph->nChannels > 0) return ncclSuccess; } if (graph->pattern == NCCL_TOPO_PATTERN_NVLS && (system->nodes[NVS].count == 0 || ccMin < 90)) return ncclSuccess; // NVLS and COLLNET_DIRECT search must have ngpus heads at most. if (graph->pattern == NCCL_TOPO_PATTERN_NVLS) graph->maxChannels = std::min(NCCL_MAX_NVLS_ARITY, system->nodes[GPU].count); if (graph->pattern == NCCL_TOPO_PATTERN_COLLNET_DIRECT) graph->maxChannels = std::min(NCCL_MAX_DIRECT_ARITY+1, system->nodes[GPU].count); if (ngpus == 1) if (graph->pattern != NCCL_TOPO_PATTERN_RING) graph->pattern = NCCL_TOPO_PATTERN_TREE; if (system->inter == 0 && graph->pattern == NCCL_TOPO_PATTERN_NVLS) { // Force intra-node NVLS algorithm to pull evenly from all GPUs. graph->minChannels = graph->maxChannels; } int splitNvLink; NCCLCHECK(ncclTopoSplitNvLink(system, &splitNvLink)); if (graph->pattern == NCCL_TOPO_PATTERN_RING && splitNvLink) { // We have two sockets with NVLink and a slower link in between (typically QPI). // Tree is likely going to work better but it needs at least 2 channels. // Since Tree needs to have the same number of channels as Ring, also force Ring to use 2 channels. if (graph->maxChannels >= 2 && graph->minChannels == 1) graph->minChannels = 2; } struct ncclTopoGraph tmpGraph; memcpy(&tmpGraph, graph, sizeof(struct ncclTopoGraph)); // First try crossnic, then decrease bw and finally increase bwIntra. int nspeeds = 0; float* speedArray = NULL; if (system->inter == 0) { nspeeds = ccMin >= 100 ? NSPEEDSINTRA_SM100 : (ccMin >= 90 ? NSPEEDSINTRA_SM90 : NSPEEDSINTRA); speedArray = ccMin >= 100 ? sm100SpeedArrayIntra : (ccMin >= 90 ? sm90SpeedArrayIntra : speedArrayIntra); } else { nspeeds = ccMin >= 100 ? NSPEEDSINTER_SM100 : (ccMin >= 90 ? NSPEEDSINTER_SM90 : NSPEEDSINTER); speedArray = ccMin >= 100 ? sm100SpeedArrayInter : (ccMin >= 90 ? sm90SpeedArrayInter : speedArrayInter); } int pass = 1; int speedIndex = 0; float maxBw = system->maxBw; float totalBw = system->totalBw; if (ngpus > 1 && graph->pattern != NCCL_TOPO_PATTERN_RING) totalBw *= ngpus*1.0/(ngpus-1); while ((speedArray[speedIndex] > maxBw || speedArray[speedIndex]*graph->minChannels > totalBw) && speedIndex < nspeeds-1) speedIndex++; tmpGraph.bwIntra = tmpGraph.bwInter = speedArray[speedIndex]; int64_t globalTimeout = NCCL_SEARCH_GLOBAL_TIMEOUT; search: int time = tmpGraph.sameChannels ? NCCL_SEARCH_TIMEOUT_SAMECHANNELS : tmpGraph.pattern == NCCL_TOPO_PATTERN_TREE ? NCCL_SEARCH_TIMEOUT_TREE : NCCL_SEARCH_TIMEOUT; tmpGraph.nChannels = 0; globalTimeout -= time; NCCLCHECK(ncclTopoSearchRec(system, &tmpGraph, graph, &time)); #if 0 printf("Id %d Pattern %d, crossNic %d, Bw %g/%g, type %d/%d, channels %d-%d sameChannels %d -> nChannels %dx%g/%g %s\n", tmpGraph.id, tmpGraph.pattern, tmpGraph.crossNic, tmpGraph.bwInter, tmpGraph.bwIntra, tmpGraph.typeInter, tmpGraph.typeIntra, tmpGraph.minChannels, tmpGraph.maxChannels, tmpGraph.sameChannels, graph->nChannels, graph->bwInter, graph->bwIntra, time == 0 ? "TIMEOUT" : time == -1 ? "PERFECT" : ""); for (int c=0; cnChannels; c++) { printf("%2d : ", c); for (int g=0; gintra[c*ngpus+g]); } printf("[%lx %lx]", graph->inter[c*2+0], graph->inter[c*2+1]); printf("\n"); } #endif // Optimal solution, stop here if (time == -1) goto done; if (graph->nChannels*graph->bwInter >= system->totalBw) goto done; if (pass == 1) { // First pass, we don't have a solution yet ; try other options // Try having different channels (except when going through AMD CPUs) if (tmpGraph.sameChannels == 1 && !(cpuArch == NCCL_TOPO_CPU_ARCH_X86 && cpuVendor == NCCL_TOPO_CPU_VENDOR_AMD && tmpGraph.typeIntra == PATH_SYS)) { tmpGraph.sameChannels = 0; goto search; } tmpGraph.sameChannels = trySameChannels; if (time != -1) globalTimeout += time; else globalTimeout = NCCL_SEARCH_GLOBAL_TIMEOUT; if (globalTimeout < 0 && graph->nChannels) goto done; // Try a simpler tree if (ccMin >= 90 && tmpGraph.pattern == NCCL_TOPO_PATTERN_BALANCED_TREE) { tmpGraph.pattern = NCCL_TOPO_PATTERN_TREE; goto search; } tmpGraph.pattern = graph->pattern; int maxIntra = system->inter ? tmpGraph.typeInter : maxTypeIntra; if (tmpGraph.typeIntra < maxIntra && (graph->nChannels == 0 || tmpGraph.typeIntra < graph->typeIntra)) { tmpGraph.typeIntra += 1; if (tmpGraph.typeIntra < PATH_DIS) goto search; } tmpGraph.typeIntra = minTypeIntra; if (system->inter && tmpGraph.typeInter < maxTypeInter && (graph->nChannels == 0 || tmpGraph.typeInter < graph->typeInter || tmpGraph.typeInter < PATH_PXN)) { tmpGraph.typeInter += 1; if (tmpGraph.typeInter < PATH_DIS) goto search; } tmpGraph.typeInter = minTypeInter; if (crossNic == 2 && tmpGraph.crossNic == 0 && (graph->pattern == NCCL_TOPO_PATTERN_RING || graph->pattern == NCCL_TOPO_PATTERN_BALANCED_TREE)) { // Try again with crossNic if permitted tmpGraph.crossNic = 2; goto search; } tmpGraph.crossNic = crossNic == 1 ? 1 : 0; // Decrease bw until we find a solution if ((speedIndex < nspeeds-1) && (graph->nChannels == 0 || (speedArray[speedIndex+1]/graph->bwInter > .49))) { tmpGraph.bwInter = tmpGraph.bwIntra = speedArray[++speedIndex]; goto search; } speedIndex = 0; while (speedArray[speedIndex] > maxBw && speedIndex < nspeeds-1) speedIndex++; tmpGraph.bwIntra = tmpGraph.bwInter = speedArray[speedIndex]; } done: // We have a solution. Start from that solution and move to pass 2. if (pass == 1) { time = -1; NCCLCHECK(ncclTopoDupChannels(graph, ccMin, ngpus)); memcpy(&tmpGraph, graph, sizeof(tmpGraph)); speedIndex = 0; while (speedArray[speedIndex] > graph->bwInter && speedIndex < nspeeds-1) speedIndex++; tmpGraph.bwIntra = tmpGraph.bwInter = speedArray[speedIndex]; tmpGraph.minChannels = graph->nChannels; pass = 2; } if (pass == 2) { // See if we can increase bw if (time != 0 && speedIndex > 0) { if (graph->pattern == NCCL_TOPO_PATTERN_RING) { // increase bw for Ring tmpGraph.bwIntra = tmpGraph.bwInter = speedArray[--speedIndex]; goto search; } else if (graph->pattern == NCCL_TOPO_PATTERN_NVLS && tmpGraph.bwInter == graph->bwInter && tmpGraph.bwInter < tmpGraph.bwIntra*2) { tmpGraph.minChannels = tmpGraph.maxChannels = graph->nChannels; tmpGraph.bwInter = speedArray[--speedIndex]; goto search; } else if (tmpGraph.bwIntra == graph->bwIntra && tmpGraph.bwIntra < tmpGraph.bwInter*2) { // increase bwIntra for trees (2 nodes or collnet) tmpGraph.bwIntra = speedArray[--speedIndex]; goto search; } } time = -1; memcpy(&tmpGraph, graph, sizeof(tmpGraph)); } if (graph->nChannels == 0 && graph->collNet == 0 && graph->pattern != NCCL_TOPO_PATTERN_NVLS) { int nets[NCCL_TOPO_MAX_NODES]; int netCount; INFO(NCCL_GRAPH, "Could not find a path for pattern %d, falling back to simple order", graph->pattern); for (int i=0; iintra[i] = system->nodes[GPU].nodes[i].gpu.rank; graph->bwIntra = 0.1; graph->typeIntra = PATH_SYS; NCCLCHECK(ncclTopoSelectNets(system, /*typeInter =*/-1, /*gpu =*/0, nets, &netCount)); graph->inter[0] = (netCount > 0 ? system->nodes[NET].nodes[nets[0]].id : -1); NCCLCHECK(ncclTopoSelectNets(system, /*typeInter =*/-1, /*gpu =*/ngpus-1, nets, &netCount)); graph->inter[1] = (netCount > 0 ? system->nodes[NET].nodes[nets[0]].id : -1); if (graph->inter[0] != -1 && graph->inter[1] != -1) { graph->bwInter = 0.1; graph->typeInter = PATH_SYS; } else { graph->inter[0] = graph->inter[1] = -1; graph->bwInter = 0; graph->typeInter = PATH_DIS; } graph->nChannels = 1; } return ncclSuccess; } // Max chars per GPU entry: " GPU/xxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxx" (~40 chars) #define CHARS_PER_GPU_ENTRY 48 ncclResult_t ncclTopoPrintGraph(struct ncclTopoSystem* system, struct ncclTopoGraph* graph) { INFO(NCCL_GRAPH, "Pattern %d, crossNic %d, nChannels %d, bw %f/%f, type %s/%s, sameChannels %d", graph->pattern, graph->crossNic, graph->nChannels, graph->bwIntra, graph->bwInter, topoPathTypeStr[graph->typeIntra], topoPathTypeStr[graph->typeInter], graph->sameChannels); int ngpus = system->nodes[GPU].count; char* line = (char*)malloc(ngpus * CHARS_PER_GPU_ENTRY); for (int c=0; cnChannels; c++) { sprintf(line, "%2d :", c); int offset = strlen(line); if (system->inter) { sprintf(line+offset, " %s/%lx-%lx", topoNodeTypeStr[NET], NCCL_TOPO_ID_SYSTEM_ID(graph->inter[2*c]), NCCL_TOPO_ID_LOCAL_ID(graph->inter[2*c])); offset = strlen(line); } for (int i=0; iintra[ngpus * c + i], &g, true); int64_t topoId = system->nodes[GPU].nodes[g].id; sprintf(line + offset, " %s/%lx-%lx", topoNodeTypeStr[GPU], NCCL_TOPO_ID_SYSTEM_ID(topoId), NCCL_TOPO_ID_LOCAL_ID(topoId)); offset = strlen(line); if (graph->id == 3) break; // NVLS graphs only use the first GPU } if (system->inter) { sprintf(line+offset, " %s/%lx-%lx", topoNodeTypeStr[NET], NCCL_TOPO_ID_SYSTEM_ID(graph->inter[2*c+1]), NCCL_TOPO_ID_LOCAL_ID(graph->inter[2*c+1])); offset = strlen(line); } INFO(NCCL_GRAPH, "%s", line); } free(line); return ncclSuccess; } ncclResult_t ncclTopoDumpGraphs(struct ncclTopoSystem* system, int ngraphs, struct ncclTopoGraph** graphs) { ncclResult_t ret = ncclSuccess; const char* str = ncclGetEnv("NCCL_GRAPH_DUMP_FILE"); struct ncclXml* xml = NULL; if (str) { INFO(NCCL_ENV, "NCCL_GRAPH_DUMP_FILE set by environment to %s", str); NCCLCHECK(xmlAlloc(&xml, NCCL_GRAPH_XML_MAX_NODES)); NCCLCHECKGOTO(ncclTopoGetXmlFromGraphs(ngraphs, graphs, system, xml), ret, fail); NCCLCHECKGOTO(ncclTopoDumpXmlToFile(str, xml), ret, fail); } exit: if (xml) free(xml); return ret; fail: goto exit; } #include "comm.h" // NVLS channels aren't compute channels. Find which NIC corresponds to our rank being the head ncclResult_t getNvlsNetDev(struct ncclComm* comm, struct ncclTopoGraph* graph, int channelId, int64_t* netId) { ncclResult_t ret = ncclSuccess; int localRanks = comm->topo->nodes[GPU].count; int netNum = 0; int64_t net[MAXCHANNELS]; for (int c = 0; c < graph->nChannels; c++) { if (graph->intra[c * localRanks] == comm->rank) { net[netNum++] = graph->inter[c * 2]; } } if (netNum) { *netId = net[channelId % netNum]; } else { ret = ncclInternalError; goto fail; } exit: return ret; fail: WARN("Could not find NIC for rank %d in NVLS graph", comm->rank); goto exit; } // 0: don't use PXN for P2P, 1: use PXN if needed, 2: use PXN as much as possible to maximize aggregation NCCL_PARAM(P2pPxnLevel, "P2P_PXN_LEVEL", 2); ncclResult_t ncclTopoGetNetDev(struct ncclComm* comm, int rank, struct ncclTopoGraph* graph, int channelId, int peerRank, int64_t* id, int* dev, int* proxyRank) { int64_t netId = -1; int netDev = -1; if (graph) { // Honor the net device in the graph int channel = channelId%graph->nChannels; int ngpus = comm->topo->nodes[GPU].count; int index = graph->intra[channel*ngpus] == rank ? 0 : 1; if (graph->pattern != NCCL_TOPO_PATTERN_NVLS) { netId = graph->inter[channel*2+index]; } else { NCCLCHECK(getNvlsNetDev(comm, graph, channelId, &netId)); } NCCLCHECK(ncclTopoIdToNetDev(comm->topo, netId, &netDev)); if (dev) *dev = netDev; if (id) *id = netId; NCCLCHECK(ncclTopoGetIntermediateRank(comm->topo, rank, netId, proxyRank)); } else if (peerRank == -1) { return ncclInternalError; } else { // Start with our local NIC and local Rank NCCLCHECK(ncclTopoGetLocalNet(comm->topo, rank, channelId, &netId, &netDev)); if (dev) *dev = netDev; if (id) *id = netId; *proxyRank = rank; int pxnLevel = ncclPxnDisable(comm) == 1 ? 0 : ncclParamP2pPxnLevel(); // See whether we can use the remote rank preferred device. if (ncclParamCrossNic() == 0 || (pxnLevel != 0)) { // Find local NIC number close to local nvmlDev int nvmlDev = comm->peerInfo[peerRank].nvmlDev; int localRank; if (ncclTopoDevToRank(comm->topo, nvmlDev, &localRank) != ncclSuccess) return ncclSuccess; NCCLCHECK(ncclTopoGetLocalNet(comm->topo, localRank, channelId, &netId, &netDev)); // Check that device exists on our node if (ncclParamCrossNic() == 0) { if (dev) *dev = netDev; if (id) *id = netId; } if (pxnLevel == 1) { int g, n; NCCLCHECK(ncclTopoRankToIndex(comm->topo, rank, &g, /*showWarn=*/true)); NCCLCHECK(ncclTopoIdToIndex(comm->topo, NET, netId, &n)); struct ncclTopoNode* gpu = comm->topo->nodes[GPU].nodes+g; if (gpu->paths[NET][n].type <= PATH_PXN) { if (dev) *dev = netDev; if (id) *id = netId; NCCLCHECK(ncclTopoGetIntermediateRank(comm->topo, rank, *dev, proxyRank)); } } else if (pxnLevel == 2) { // Check which local GPU corresponds to that NIC and see if we can use PXN. int n, g1, g2; NCCLCHECK(ncclTopoIdToIndex(comm->topo, NET, netId, &n)); NCCLCHECK(ncclTopoRankToIndex(comm->topo, rank, &g1, /*showWarn=*/true)); NCCLCHECK(ncclTopoGetLocalGpu(comm->topo, netId, &g2)); if (g2 != -1) { struct ncclTopoNode* peerGpu = comm->topo->nodes[GPU].nodes+g2; int pxnType = ncclParamPxnC2c() ? PATH_P2C : PATH_PXB; if (peerGpu->paths[GPU][g1].type <= PATH_NVL && peerGpu->paths[NET][n].type <= pxnType) { *proxyRank = peerGpu->gpu.rank; if (dev) *dev = netDev; if (id) *id = netId; return ncclSuccess; } } } } } return ncclSuccess; } nccl-2.29.7-1/src/graph/topo.cc000066400000000000000000002171141515037102200160520ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "core.h" #include "graph.h" #include "topo.h" #include "comm.h" #include "nccl.h" #include "nvmlwrap.h" #include "coll_net.h" #include "gin.h" #include "transport.h" #include #include #include "cpuset.h" #include "bootstrap.h" #include #include #define BUSID_SIZE (sizeof("0000:00:00.0")) #define BUSID_REDUCED_SIZE (sizeof("0000:00")) const char* topoNodeTypeStr[] = { "GPU", "PCI", "NVS", "CPU", "NIC", "NET", "GIN" }; const char* topoLinkTypeStr[] = { "LOC", "NVL", "", "C2C", "PCI", "", "", "", "", "SYS", "NET" }; const char* topoPathTypeStr[] = { "LOC", "NVL", "NVB", "C2C", "PIX", "PXB", "P2C", "PXN", "PHB", "SYS", "NET", "DIS" }; /******************************************************************/ /******************* Graph Creation Functions *********************/ /******************************************************************/ static ncclResult_t findLocalCpu(struct ncclTopoNode* node, struct ncclTopoNode** cpu, struct ncclTopoNode* from) { *cpu = NULL; if (node->type == CPU) { *cpu = node; return ncclSuccess; } for (int l=0; lnlinks; l++) { // Go up the PCI tree to find the CPU. Follow only PCI switches. if (node->links[l].type == LINK_PCI && node->links[l].remNode != from && (node->links[l].remNode->type == PCI || node->links[l].remNode->type == CPU)) { NCCLCHECK(findLocalCpu(node->links[l].remNode, cpu, node)); } if (*cpu != NULL) return ncclSuccess; } return ncclSuccess; } int interCpuBw = 0; int cpuPciBw = 0; static ncclResult_t ncclTopoGetInterCpuBw(struct ncclTopoNode* cpu, float* bw) { *bw = LOC_BW; if (cpu->cpu.arch == NCCL_TOPO_CPU_ARCH_POWER) { *bw = P9_BW; return ncclSuccess; } if (cpu->cpu.arch == NCCL_TOPO_CPU_ARCH_ARM) { *bw = ARM_BW; return ncclSuccess; } if (cpu->cpu.arch == NCCL_TOPO_CPU_ARCH_X86 && cpu->cpu.vendor == NCCL_TOPO_CPU_VENDOR_INTEL) { *bw = cpu->cpu.model == NCCL_TOPO_CPU_MODEL_INTEL_ERP ? ERP_QPI_BW : cpu->cpu.model == NCCL_TOPO_CPU_MODEL_INTEL_SRP ? SRP_QPI_BW : cpu->cpu.model == NCCL_TOPO_CPU_MODEL_INTEL_SKL ? SKL_QPI_BW : BDW_QPI_BW; } if (cpu->cpu.arch == NCCL_TOPO_CPU_ARCH_X86 && cpu->cpu.vendor == NCCL_TOPO_CPU_VENDOR_AMD) { *bw = AMD_BW; } if (cpu->cpu.arch == NCCL_TOPO_CPU_ARCH_X86 && cpu->cpu.vendor == NCCL_TOPO_CPU_VENDOR_ZHAOXIN) { *bw = cpu->cpu.model == NCCL_TOPO_CPU_MODEL_YONGFENG ? YONGFENG_ZPI_BW : ZPI_BW; } return ncclSuccess; } enum ncclNvLinkDeviceType { ncclNvLinkDeviceUnknown, ncclNvLinkDeviceGpu, ncclNvLinkDeviceSwitch, ncclNvLinkDeviceBridge, // IBM/Power NVLink bridge (Device 04ea) }; ncclResult_t ncclTopoGetNode(struct ncclTopoSystem* system, struct ncclTopoNode** node, int type, uint64_t id) { for (int i=0; inodes[type].count; i++) { if (system->nodes[type].nodes[i].id == id) { *node = system->nodes[type].nodes+i; return ncclSuccess; } } return ncclSuccess; } ncclResult_t ncclTopoCreateNode(struct ncclTopoSystem* system, struct ncclTopoNode** node, int type, uint64_t id) { if (system->nodes[type].count == NCCL_TOPO_MAX_NODES) { WARN("Error : tried to create too many nodes of type %d", type); return ncclInternalError; } struct ncclTopoNode* n = system->nodes[type].nodes+system->nodes[type].count; system->nodes[type].count++; n->type = type; n->id = id; if (type == GPU) { n->gpu.dev = NCCL_TOPO_UNDEF; n->gpu.rank = NCCL_TOPO_UNDEF; n->gpu.cudaCompCap = NCCL_TOPO_UNDEF; } else if (type == CPU) { n->cpu.arch = NCCL_TOPO_UNDEF; n->cpu.vendor = NCCL_TOPO_UNDEF; n->cpu.model = NCCL_TOPO_UNDEF; } else if (type == NET) { n->net.asic = 0ULL; n->net.port = NCCL_TOPO_UNDEF; n->net.bw = 0.0; n->net.latency = 0.0; } *node = n; return ncclSuccess; } ncclResult_t ncclTopoRemoveNode(struct ncclTopoSystem* system, int type, int index) { struct ncclTopoNode* delNode = system->nodes[type].nodes+index; for (int t=0; tpaths[t]); for (int n=0; nnodes[t].count; n++) { struct ncclTopoNode* node = system->nodes[t].nodes+n; if (node == delNode) continue; for (int l=0; lnlinks; l++) { while (lnlinks && node->links[l].remNode == delNode) { memmove(node->links+l, node->links+l+1, (node->nlinks-l-1)*sizeof(struct ncclTopoLink)); node->nlinks--; } if (lnlinks && node->links[l].remNode->type == type && node->links[l].remNode >= delNode) { node->links[l].remNode--; } } } } memmove(delNode, delNode+1, (system->nodes[type].count-index-1)*sizeof(struct ncclTopoNode)); system->nodes[type].count--; return ncclSuccess; } ncclResult_t ncclTopoConnectNodes(struct ncclTopoNode* node, struct ncclTopoNode* remNode, int type, float bw) { // Aggregate links into higher bw for NVLink struct ncclTopoLink* link; for (link = node->links; link - node->links != NCCL_TOPO_MAX_LINKS && link->remNode; link++) { if (link->remNode == remNode && link->type == type) break; } if (link - node->links == NCCL_TOPO_MAX_LINKS) { WARN("Error : too many Topo links (max %d)", NCCL_TOPO_MAX_LINKS); return ncclInternalError; } if (link->remNode == NULL) node->nlinks++; link->type = type; link->remNode = remNode; link->bw += bw; // Sort links in BW descending order struct ncclTopoLink linkSave; memcpy(&linkSave, link, sizeof(struct ncclTopoLink)); while (link != node->links) { if ((link-1)->bw >= linkSave.bw) break; memcpy(link, link-1, sizeof(struct ncclTopoLink)); link--; } memcpy(link, &linkSave, sizeof(struct ncclTopoLink)); return ncclSuccess; } // BCM Gen4 Switches present themselves as a two-level hierarchical switch // even though they're supposed to sustain full BW across all ports. // Flatten the switch as this extra level can break the search and make // NCCL take wrong topology decisions. int getBcmGen(uint64_t id, int level) { if ((id & 0xfffffffffffff000) == 0x1000c0101000a000) return 4; if ((id & 0xfffffffffffff000) == (0x1000c03010000000 | level*0x1000)) return 5; return 0; } ncclResult_t ncclTopoFlattenBcmSwitches(struct ncclTopoSystem* system) { ncclResult_t ret = ncclSuccess; for (int s=0; snodes[PCI].count; s++) { struct ncclTopoNode* pciSwitch = system->nodes[PCI].nodes+s; int gen = getBcmGen(pciSwitch->pci.device, 0); // Flatten Gen4 PEX switches in base mode if (gen) { // Find sub switches with the same device ID. int64_t* subSwIds; NCCLCHECK(ncclCalloc(&subSwIds, pciSwitch->nlinks)); int subs = 0; for (int l=0; lnlinks; l++) { struct ncclTopoNode* sub = pciSwitch->links[l].remNode; // Only fuse sub switches with the same device ID. if (sub->type != PCI || getBcmGen(sub->pci.device, 1) != gen) continue; // Save sub switch for later subSwIds[subs++] = sub->id; // Remove link to that sub switch memmove(pciSwitch->links+l, pciSwitch->links+l+1, (pciSwitch->nlinks-l-1)*(sizeof(struct ncclTopoLink))); pciSwitch->nlinks--; // Don't increase l for the next iteration as we just shifted all links by one. l--; } for (int s=0; snodes[PCI].nodes is changing every time we remove a node) int index; NCCLCHECKGOTO(ncclTopoIdToIndex(system, PCI, subSwIds[s], &index), ret, fail); struct ncclTopoNode* sub = system->nodes[PCI].nodes+index; // Connect all sub PCI devices to the parent switch for (int l=0; lnlinks; l++) { struct ncclTopoNode* remNode = sub->links[l].remNode; if (remNode == pciSwitch) continue; // Add link from parent PCI switch -> PCI device if (pciSwitch->nlinks == NCCL_TOPO_MAX_LINKS) { WARN("Error : too many Topo links (max %d)", NCCL_TOPO_MAX_LINKS); ret = ncclInternalError; goto fail; } memcpy(pciSwitch->links+pciSwitch->nlinks, sub->links+l, sizeof(struct ncclTopoLink)); pciSwitch->nlinks++; // Update link from PCI device -> parent PCI switch for (int rl=0; rlnlinks; rl++) { if (remNode->links[rl].remNode == sub) { remNode->links[rl].remNode = pciSwitch; break; } } } NCCLCHECKGOTO(ncclTopoRemoveNode(system, PCI, index), ret, fail); } // Set subdevice to 0xffff to make sure we don't merge this switch again. pciSwitch->pci.device |= 0xffff; free(subSwIds); // Restart, as system->nodes[PCI].nodes has changed. s = -1; // Will be incremented to 0 in the next loop iteration continue; fail: free(subSwIds); return ret; } } return ret; } ncclResult_t ncclTopoConnectCpus(struct ncclTopoSystem* system) { // And connect all CPU nodes together for (int n=0; nnodes[CPU].count; n++) { struct ncclTopoNode* cpu1 = system->nodes[CPU].nodes+n; for (int p=0; pnodes[CPU].count; p++) { struct ncclTopoNode* cpu2 = system->nodes[CPU].nodes+p; if (n == p || (NCCL_TOPO_ID_SYSTEM_ID(cpu1->id) != NCCL_TOPO_ID_SYSTEM_ID(cpu2->id))) continue; float bw; NCCLCHECK(ncclTopoGetInterCpuBw(cpu1, &bw)); NCCLCHECK(ncclTopoConnectNodes(cpu1, cpu2, LINK_SYS, bw)); } } return ncclSuccess; } static ncclResult_t ncclTopoPrintRec(struct ncclTopoNode* node, struct ncclTopoNode* prevNode, char* line, int offset) { if (node->type == GPU) { sprintf(line+offset, "%s/%lx-%lx (%d)", topoNodeTypeStr[node->type], NCCL_TOPO_ID_SYSTEM_ID(node->id), NCCL_TOPO_ID_LOCAL_ID(node->id), node->gpu.rank); } else if (node->type == CPU) { sprintf(line+offset, "%s/%lx-%lx (%d/%d/%d)", topoNodeTypeStr[node->type], NCCL_TOPO_ID_SYSTEM_ID(node->id), NCCL_TOPO_ID_LOCAL_ID(node->id), node->cpu.arch, node->cpu.vendor, node->cpu.model); } else if (node->type == PCI) { sprintf(line+offset, "%s/%lx-%lx (%lx)", topoNodeTypeStr[node->type], NCCL_TOPO_ID_SYSTEM_ID(node->id), NCCL_TOPO_ID_LOCAL_ID(node->id), node->pci.device); } else { sprintf(line+offset, "%s/%lx-%lx", topoNodeTypeStr[node->type], NCCL_TOPO_ID_SYSTEM_ID(node->id), NCCL_TOPO_ID_LOCAL_ID(node->id)); } INFO(NCCL_GRAPH, "%s", line); for (int i=0; inlinks; l++) { struct ncclTopoLink* link = node->links+l; if (link->type == LINK_LOC) { sprintf(line+offset, "+ %s[%2.1f] - %s/%lx-%lx", topoLinkTypeStr[link->type], link->bw, topoNodeTypeStr[link->remNode->type], NCCL_TOPO_ID_SYSTEM_ID(link->remNode->id), NCCL_TOPO_ID_LOCAL_ID(link->remNode->id)); INFO(NCCL_GRAPH, "%s", line); } else if (link->type != LINK_PCI || link->remNode != prevNode) { sprintf(line+offset, "+ %s[%2.1f] - ", topoLinkTypeStr[link->type], link->bw); int nextOffset = strlen(line); if (link->type == LINK_PCI) { NCCLCHECK(ncclTopoPrintRec(link->remNode, node, line, nextOffset)); } else { if (link->remNode->type == NET) { sprintf(line+nextOffset, "%s/%lx-%lx (%d/%lx/%d/%f)", topoNodeTypeStr[link->remNode->type], NCCL_TOPO_ID_SYSTEM_ID(link->remNode->id), NCCL_TOPO_ID_LOCAL_ID(link->remNode->id), link->remNode->net.collSupport, link->remNode->net.asic, link->remNode->net.port, link->remNode->net.bw); } else { sprintf(line+nextOffset, "%s/%lx-%lx", topoNodeTypeStr[link->remNode->type], NCCL_TOPO_ID_SYSTEM_ID(link->remNode->id), NCCL_TOPO_ID_LOCAL_ID(link->remNode->id)); } INFO(NCCL_GRAPH, "%s", line); } } } return ncclSuccess; } ncclResult_t ncclTopoPrint(struct ncclTopoSystem* s) { INFO(NCCL_GRAPH, "=== System : maxBw %2.1f totalBw %2.1f ===", s->maxBw, s->totalBw); char line[1024]; for (int n=0; nnodes[CPU].count; n++) NCCLCHECK(ncclTopoPrintRec(s->nodes[CPU].nodes+n, NULL, line, 0)); INFO(NCCL_GRAPH, "=========================================="); NCCLCHECK(ncclTopoPrintPaths(s)); return ncclSuccess; } static ncclResult_t ncclTopoSort(struct ncclTopoNode* node, struct ncclTopoNode* upNode) { // Shift all links to have upLink as last link if (upNode) { int l=0; while (node->links[l].remNode != upNode) l++; struct ncclTopoLink upLink; memcpy(&upLink, node->links+l, sizeof(struct ncclTopoLink)); while (node->links[l+1].remNode) { memcpy(node->links+l, node->links+l+1, sizeof(struct ncclTopoLink)); l++; } memcpy(node->links+l, &upLink, sizeof(struct ncclTopoLink)); } // Recursively sort the PCI tree for (int l=0; lnlinks; l++) { struct ncclTopoLink* link = node->links+l; if (link->type == LINK_PCI && link->remNode != upNode) NCCLCHECK(ncclTopoSort(link->remNode, node)); } return ncclSuccess; } // We want the graph to be organized to ease/accelerate traversal : // 1. NVLinks (already the case) // 2. PCI down // 3. PCI up // 4. SYS (already the case) ncclResult_t ncclTopoSortSystem(struct ncclTopoSystem* system) { for (int n=0; nnodes[CPU].count; n++) NCCLCHECK(ncclTopoSort(system->nodes[CPU].nodes+n, NULL)); return ncclSuccess; } ncclResult_t ncclTopoGetMinNetBw(struct ncclTopoSystem* system, float* bw) { float minBw = FLT_MAX; for (int n = 0; n < system->nodes[NET].count; n++) { struct ncclTopoNode* net = system->nodes[NET].nodes + n; if (net->net.bw < minBw) minBw = net->net.bw; } *bw = minBw; return ncclSuccess; } ncclResult_t ncclTopoAddNet(struct ncclXmlNode* xmlNet, struct ncclTopoSystem* system, struct ncclTopoNode* nic, int systemId) { int dev; NCCLCHECK(xmlGetAttrInt(xmlNet, "dev", &dev)); int64_t netId = NCCL_TOPO_ID(systemId, dev); struct ncclTopoNode* net; NCCLCHECK(ncclTopoCreateNode(system, &net, NET, netId)); net->net.dev = dev; const char* str; // if not guid is present use the net->id unique id instead, which will be unique within the node/NVLD NCCLCHECK(xmlGetAttr(xmlNet, "guid", &str)); net->net.asic = (str) ? strtoull(str, NULL, 16) : netId; int mbps; NCCLCHECKNOWARN(xmlGetAttrIntDefault(xmlNet, "speed", &mbps, 0), NCCL_GRAPH); if (mbps <= 0) mbps = 10000; // Some NICs define speed = -1 net->net.bw = mbps / 8000.0; ncclResult_t ret; NOWARN(ret = xmlGetAttrFloat(xmlNet, "latency", &net->net.latency), NCCL_GRAPH); if (ret != ncclSuccess) net->net.latency = 0; NCCLCHECKNOWARN(xmlGetAttrIntDefault(xmlNet, "port", &net->net.port, 0), NCCL_GRAPH); NCCLCHECKNOWARN(xmlGetAttrIntDefault(xmlNet, "gdr", &net->net.gdrSupport, 0), NCCL_GRAPH); NCCLCHECKNOWARN(xmlGetAttrIntDefault(xmlNet, "maxconn", &net->net.maxChannels, MAXCHANNELS), NCCL_GRAPH); NCCLCHECKNOWARN(xmlGetAttrIntDefault(xmlNet, "coll", &net->net.collSupport, 0), NCCL_GRAPH); // build the PCI id using the parent PCI link uint64_t hacc[2] = {1, 1}; const char* busId = NULL; struct ncclXmlNode* parent = xmlNet->parent; while (parent != NULL && strcmp(parent->name, "pci") != 0) parent = parent->parent; if (parent) NCCLCHECK(xmlGetAttr(parent, "busid", &busId)); // If we fail to find the PCIe path, we use the GUID instead. if (busId) eatHash(hacc, busId, strlen(busId)); else eatHash(hacc, &net->net.asic); net->net.pciId = digestHash(hacc); NCCLCHECK(ncclTopoConnectNodes(nic, net, LINK_NET, net->net.bw)); NCCLCHECK(ncclTopoConnectNodes(net, nic, LINK_NET, net->net.bw)); return ncclSuccess; } ncclResult_t ncclTopoAddGin(struct ncclXmlNode* xmlNet, struct ncclTopoSystem* system, struct ncclTopoNode* nic, int systemId) { int dev; NCCLCHECK(xmlGetAttrInt(xmlNet, "dev", &dev)); int64_t netId = NCCL_TOPO_ID(systemId, dev); struct ncclTopoNode* net; NCCLCHECK(ncclTopoCreateNode(system, &net, GIN, netId)); net->net.dev = dev; int mbps; NCCLCHECKNOWARN(xmlGetAttrIntDefault(xmlNet, "speed", &mbps, 0), NCCL_GRAPH); if (mbps <= 0) mbps = 10000; // Some NICs define speed = -1 net->net.bw = mbps / 8000.0; NCCLCHECK(ncclTopoConnectNodes(nic, net, LINK_NET, net->net.bw)); NCCLCHECK(ncclTopoConnectNodes(net, nic, LINK_NET, net->net.bw)); return ncclSuccess; } ncclResult_t ncclTopoAddNic(struct ncclXmlNode* xmlNic, struct ncclTopoSystem* system, struct ncclTopoNode* nic, int systemId) { for (int s=0; snSubs; s++) { struct ncclXmlNode* xmlNet = xmlNic->subs[s]; if (strcmp(xmlNet->name, "net") != 0) continue; int index; NCCLCHECK(xmlGetAttrIndex(xmlNet, "dev", &index)); // This means that the "dev" attribute wasn't set on this net xml node. That means it should not be added to the system topology graph if (index == -1) continue; // Backward compatibility: net withouh "net" attr is a net dev, net without a "gin" is not a gin dev int net = 0, gin = 0; NCCLCHECK(xmlGetAttrIntDefault(xmlNet, "net", &net, 1)); NCCLCHECK(xmlGetAttrIntDefault(xmlNet, "gin", &gin, 0)); if (net) NCCLCHECK(ncclTopoAddNet(xmlNet, system, nic, systemId)); if (gin) NCCLCHECK(ncclTopoAddGin(xmlNet, system, nic, systemId)); } return ncclSuccess; } ncclResult_t ncclTopoAddGpu(struct ncclXmlNode* xmlGpu, struct ncclTopoSystem* system, struct ncclTopoNode* gpu) { NCCLCHECK(xmlGetAttrInt(xmlGpu, "sm", &gpu->gpu.cudaCompCap)); NCCLCHECK(xmlGetAttrInt(xmlGpu, "rank", &gpu->gpu.rank)); NCCLCHECK(xmlGetAttrInt(xmlGpu, "dev", &gpu->gpu.dev)); NCCLCHECK(xmlGetAttrInt(xmlGpu, "gdr", &gpu->gpu.gdrSupport)); // Do not go any further, nvlinks will be added in a second pass return ncclSuccess; } #define PCI_BRIDGE_DEVICE_CLASS "0x060400" struct kvDict kvDictPciClass[] = { { PCI_BRIDGE_DEVICE_CLASS, PCI }, {"0x080100", /*CX8 data direct*/PCI}, { "0x068000", NVS }, { "0x068001", CPU }, { "0x03", GPU }, { "0x02", NIC }, { NULL, PCI /* Default fallback value */ } }; struct kvDict kvDictPciGen[] = { { "2.5 GT/s", 15 }, { "5 GT/s", 30 }, { "8 GT/s", 60 }, { "16 GT/s", 120 }, { "32 GT/s", 240 }, /* Kernel 5.6 and earlier */ { "2.5 GT/s PCIe", 15 }, { "5.0 GT/s PCIe", 30 }, { "8.0 GT/s PCIe", 60 }, { "16.0 GT/s PCIe", 120 }, { "32.0 GT/s PCIe", 240 }, { "64.0 GT/s PCIe", 480 }, { NULL, 60 /* Default fallback */ } }; // x100 Mbps per lane ncclResult_t ncclTopoAddPci(struct ncclXmlNode* xmlPci, struct ncclTopoSystem* system, struct ncclTopoNode* parent, int systemId, int numaId) { const char* str; int type; NCCLCHECK(xmlGetAttrStr(xmlPci, "class", &str)); NCCLCHECK(kvConvertToInt(str, &type, kvDictPciClass)); int64_t busId; NCCLCHECK(xmlGetAttrStr(xmlPci, "busid", &str)); NCCLCHECK(busIdToInt64(str, &busId)); struct ncclTopoNode* node = NULL; struct ncclXmlNode* xmlGpu = NULL; NCCLCHECK(xmlGetSub(xmlPci, "gpu", &xmlGpu)); if (xmlGpu != NULL) { type = GPU; int index; NCCLCHECK(xmlGetAttrIndex(xmlGpu, "rank", &index)); if (index == -1) return ncclSuccess; NCCLCHECK(ncclTopoCreateNode(system, &node, type, NCCL_TOPO_ID(systemId, busId))); NCCLCHECK(ncclTopoAddGpu(xmlGpu, system, node)); } struct ncclXmlNode* xmlNic = NULL; NCCLCHECK(xmlGetSub(xmlPci, "nic", &xmlNic)); if (xmlNic != NULL) { type = NIC; // Ignore sub device ID and merge multi-port NICs into one PCI device. struct ncclTopoNode* nicNode = NULL; int64_t localNicId = NCCL_TOPO_LOCAL_NIC_ID(numaId, busId); int64_t id = NCCL_TOPO_ID(systemId, localNicId); NCCLCHECK(ncclTopoGetNode(system, &nicNode, type, id)); if (nicNode == NULL) { NCCLCHECK(ncclTopoCreateNode(system, &nicNode, type, id)); node = nicNode; // Connect it to parent later on } NCCLCHECK(ncclTopoAddNic(xmlNic, system, nicNode, systemId)); } else if (type == PCI) { NCCLCHECK(ncclTopoCreateNode(system, &node, type, NCCL_TOPO_ID(systemId, busId))); NCCLCHECK(xmlGetAttr(xmlPci, "vendor", &str)); if (str) node->pci.device += strtol(str, NULL, 0) << 48; NCCLCHECK(xmlGetAttr(xmlPci, "device", &str)); if (str) node->pci.device += strtol(str, NULL, 0) << 32; NCCLCHECK(xmlGetAttr(xmlPci, "subsystem_vendor", &str)); if (str) node->pci.device += strtol(str, NULL, 0) << 16; NCCLCHECK(xmlGetAttr(xmlPci, "subsystem_device", &str)); if (str) node->pci.device += strtol(str, NULL, 0); for (int s=0; snSubs; s++) { struct ncclXmlNode* xmlSubPci = xmlPci->subs[s]; if (strcmp(xmlSubPci->name, "pcilink") != 0) { // PCI links will be added later NCCLCHECK(ncclTopoAddPci(xmlSubPci, system, node, systemId, numaId)); } } } if (node) { int width, speed; NCCLCHECK(xmlGetAttrInt(xmlPci, "link_width", &width)); NCCLCHECK(xmlGetAttrStr(xmlPci, "link_speed", &str)); // Manage cases where speed was not indicated in /sys if (width == 0) width = 16; NCCLCHECK(kvConvertToInt(str, &speed, kvDictPciGen)); // Values in 100Mbps, per lane (we want GB/s in the end) NCCLCHECK(ncclTopoConnectNodes(node, parent, LINK_PCI, width*speed/80.0)); NCCLCHECK(ncclTopoConnectNodes(parent, node, LINK_PCI, width*speed/80.0)); } return ncclSuccess; } struct kvDict kvDictCpuArch[] = { { "x86_64", NCCL_TOPO_CPU_ARCH_X86 }, { "arm64", NCCL_TOPO_CPU_ARCH_ARM }, { "ppc64", NCCL_TOPO_CPU_ARCH_POWER }, { NULL, 0 } }; struct kvDict kvDictCpuVendor[] = { { "GenuineIntel", NCCL_TOPO_CPU_VENDOR_INTEL }, { "AuthenticAMD", NCCL_TOPO_CPU_VENDOR_AMD }, { "CentaurHauls", NCCL_TOPO_CPU_VENDOR_ZHAOXIN }, { " Shanghai ", NCCL_TOPO_CPU_VENDOR_ZHAOXIN }, { NULL, 0 } }; ncclResult_t ncclGetSystemId(struct ncclTopoSystem* system, struct ncclXmlNode* xmlCpu, int* systemIdPtr) { const char* hostHashStr; NCCLCHECK(xmlGetAttr(xmlCpu, "host_hash", &hostHashStr)); uint64_t hostHash = hostHashStr ? strtoull(hostHashStr, NULL, 16) : 0; int systemId; for (systemId=0; systemIdnHosts; systemId++) if (system->hostHashes[systemId] == hostHash) break; if (systemId == system->nHosts) system->hostHashes[system->nHosts++] = hostHash; *systemIdPtr = systemId; return ncclSuccess; } ncclResult_t ncclTopoAddCpu(struct ncclXmlNode* xmlCpu, struct ncclTopoSystem* system) { int numaId; NCCLCHECK(xmlGetAttrInt(xmlCpu, "numaid", &numaId)); int systemId; NCCLCHECK(ncclGetSystemId(system, xmlCpu, &systemId)); struct ncclTopoNode* cpu; NCCLCHECK(ncclTopoCreateNode(system, &cpu, CPU, NCCL_TOPO_ID(systemId, numaId))); const char* str; NCCLCHECK(xmlGetAttr(xmlCpu, "affinity", &str)); if (str != NULL) { NCCLCHECK(ncclStrToCpuset(str, &cpu->cpu.affinity)); } NCCLCHECK(xmlGetAttrStr(xmlCpu, "arch", &str)); NCCLCHECK(kvConvertToInt(str, &cpu->cpu.arch, kvDictCpuArch)); if (cpu->cpu.arch == NCCL_TOPO_CPU_ARCH_X86) { NCCLCHECK(xmlGetAttrStr(xmlCpu, "vendor", &str)); NCCLCHECK(kvConvertToInt(str, &cpu->cpu.vendor, kvDictCpuVendor)); if (cpu->cpu.vendor == NCCL_TOPO_CPU_VENDOR_INTEL) { int familyId, modelId; NCCLCHECK(xmlGetAttrInt(xmlCpu, "familyid", &familyId)); NCCLCHECK(xmlGetAttrInt(xmlCpu, "modelid", &modelId)); cpu->cpu.model = (familyId == 6 && modelId >= 0xCF) ? NCCL_TOPO_CPU_MODEL_INTEL_ERP : (familyId == 6 && modelId >= 0x8F) ? NCCL_TOPO_CPU_MODEL_INTEL_SRP : (familyId == 6 && modelId >= 0x55) ? NCCL_TOPO_CPU_MODEL_INTEL_SKL : NCCL_TOPO_CPU_MODEL_INTEL_BDW; } else if (cpu->cpu.vendor == NCCL_TOPO_CPU_VENDOR_ZHAOXIN) { int familyId, modelId; NCCLCHECK(xmlGetAttrInt(xmlCpu, "familyid", &familyId)); NCCLCHECK(xmlGetAttrInt(xmlCpu, "modelid", &modelId)); if (familyId == 7 && modelId == 0x5B) cpu->cpu.model = NCCL_TOPO_CPU_MODEL_YONGFENG; } } for (int s=0; snSubs; s++) { struct ncclXmlNode* node = xmlCpu->subs[s]; if (strcmp(node->name, "pci") == 0) NCCLCHECK(ncclTopoAddPci(node, system, cpu, systemId, numaId)); if (strcmp(node->name, "nic") == 0) { struct ncclTopoNode* nic = NULL; int64_t localNicId = NCCL_TOPO_LOCAL_NIC_ID(numaId, 0); int64_t id = NCCL_TOPO_ID(systemId, localNicId); NCCLCHECK(ncclTopoGetNode(system, &nic, NIC, id)); if (nic == NULL) { NCCLCHECK(ncclTopoCreateNode(system, &nic, NIC, id)); NCCLCHECK(ncclTopoConnectNodes(cpu, nic, LINK_PCI, LOC_BW)); NCCLCHECK(ncclTopoConnectNodes(nic, cpu, LINK_PCI, LOC_BW)); } NCCLCHECK(ncclTopoAddNic(node, system, nic, systemId)); } } return ncclSuccess; } ncclResult_t ncclTopoAddNvLinks(struct ncclXmlNode* node, struct ncclTopoSystem* system, const char* parentBusId, int systemId) { if (strcmp(node->name, "nvlink") == 0) { struct ncclTopoNode* gpu = NULL; int64_t pBusId; NCCLCHECK(busIdToInt64(parentBusId, &pBusId)); pBusId = NCCL_TOPO_ID(systemId, pBusId); NCCLCHECK(ncclTopoGetNode(system, &gpu, GPU, pBusId)); if (gpu == NULL) { WARN("Add NVLink error : could not find GPU %lx", pBusId); return ncclInternalError; } int count; NCCLCHECK(xmlGetAttrInt(node, "count", &count)); const char* targetClass; NCCLCHECK(xmlGetAttrStr(node, "tclass", &targetClass)); int targetType; NCCLCHECK(kvConvertToInt(targetClass, &targetType, kvDictPciClass)); struct ncclTopoNode* remote = NULL; if (targetType == GPU) { // NVL P2P connection to another GPU const char* target; NCCLCHECK(xmlGetAttrStr(node, "target", &target)); int64_t busId; NCCLCHECK(busIdToInt64(target, &busId)); NCCLCHECK(ncclTopoGetNode(system, &remote, GPU, NCCL_TOPO_ID(systemId, busId))); } else if (targetType == CPU) { // NVL connection to the local CPU NCCLCHECK(findLocalCpu(gpu, &remote, NULL)); } else { if (system->nodes[NVS].count == 0) { NCCLCHECK(ncclTopoCreateNode(system, &remote, NVS, 0)); } else { remote = system->nodes[NVS].nodes; } } if (remote) { float nvlBw = ncclTopoNVLinkBw(gpu->gpu.cudaCompCap); NCCLCHECK(ncclTopoConnectNodes(gpu, remote, LINK_NVL, count*nvlBw)); if (remote->type != GPU) { NCCLCHECK(ncclTopoConnectNodes(remote, gpu, LINK_NVL, count*nvlBw)); } } } else { if (strcmp(node->name, "cpu") == 0) { NCCLCHECK(ncclGetSystemId(system, node, &systemId)); } const char* busId; NCCLCHECK(xmlGetAttr(node, "busid", &busId)); for (int s=0; snSubs; s++) { NCCLCHECK(ncclTopoAddNvLinks(node->subs[s], system, busId ? busId : parentBusId, systemId)); } } return ncclSuccess; } ncclResult_t ncclTopoAddPciLinks(struct ncclXmlNode* node, struct ncclTopoSystem* system, const char* parentBusId, int systemId) { if (strcmp(node->name, "pcilink") == 0) { struct ncclTopoNode* pci = NULL; int64_t pBusId; NCCLCHECK(busIdToInt64(parentBusId, &pBusId)); pBusId = NCCL_TOPO_ID(systemId, pBusId); NCCLCHECK(ncclTopoGetNode(system, &pci, PCI, pBusId)); if (pci == NULL) { WARN("Add PCI Link error : could not find PCI SW %lx", pBusId); return ncclInternalError; } struct ncclTopoNode* remote = NULL; const char* target; NCCLCHECK(xmlGetAttrStr(node, "target", &target)); int64_t busId; NCCLCHECK(busIdToInt64(target, &busId)); NCCLCHECK(ncclTopoGetNode(system, &remote, PCI, NCCL_TOPO_ID(systemId, busId))); if (remote) NCCLCHECK(ncclTopoConnectNodes(pci, remote, LINK_LOC, LOC_BW)); } else { if (strcmp(node->name, "cpu") == 0) { NCCLCHECK(ncclGetSystemId(system, node, &systemId)); } const char* busId; NCCLCHECK(xmlGetAttr(node, "busid", &busId)); for (int s=0; snSubs; s++) { NCCLCHECK(ncclTopoAddPciLinks(node->subs[s], system, busId ? busId : parentBusId, systemId)); } } return ncclSuccess; } ncclResult_t ncclTopoAddC2c(struct ncclXmlNode* node, struct ncclTopoSystem* system, const char* parentBusId, int systemId) { if (strcmp(node->name, "c2c") == 0) { struct ncclTopoNode* gpu = NULL; int64_t pBusId; NCCLCHECK(busIdToInt64(parentBusId, &pBusId)); pBusId = NCCL_TOPO_ID(systemId, pBusId); NCCLCHECK(ncclTopoGetNode(system, &gpu, GPU, pBusId)); if (gpu == NULL) { WARN("Add NVLink error : could not find GPU %lx", pBusId); return ncclInternalError; } int count = 0; NCCLCHECK(xmlGetAttrInt(node, "count", &count)); int bw = 0; NCCLCHECK(xmlGetAttrInt(node, "bw", &bw)); double c2cBw = (bw*count)/1000.0; struct ncclTopoNode* cpu = NULL; NCCLCHECK(findLocalCpu(gpu, &cpu, NULL)); if (cpu == NULL) return ncclSuccess; NCCLCHECK(ncclTopoConnectNodes(gpu, cpu, LINK_C2C, c2cBw)); NCCLCHECK(ncclTopoConnectNodes(cpu, gpu, LINK_C2C, c2cBw)); } else { if (strcmp(node->name, "cpu") == 0) { NCCLCHECK(ncclGetSystemId(system, node, &systemId)); } const char* busId; NCCLCHECK(xmlGetAttr(node, "busid", &busId)); for (int s=0; snSubs; s++) { NCCLCHECK(ncclTopoAddC2c(node->subs[s], system, busId ? busId : parentBusId, systemId)); } } return ncclSuccess; } ncclResult_t ncclTopoGetSystemFromXml(struct ncclXml* xml, struct ncclTopoSystem** topoSystem, const uint64_t localHostHash) { NCCLCHECK(ncclCalloc(topoSystem, 1)); struct ncclTopoSystem* system = *topoSystem; struct ncclXmlNode* topNode; NCCLCHECK(xmlFindTag(xml, "system", &topNode)); for (int s=0; snSubs; s++) { struct ncclXmlNode* node = topNode->subs[s]; if (strcmp(node->name, "cpu") == 0) NCCLCHECK(ncclTopoAddCpu(node, *topoSystem)); } int systemId = 0; while (systemId < system->nHosts && system->hostHashes[systemId] != localHostHash) systemId++; system->systemId = systemId; if(systemId == system->nHosts){ WARN("localHostHash = 0x%lx not found in the list of system hostHashes",localHostHash); return ncclInvalidArgument; } NCCLCHECK(ncclTopoAddNvLinks(topNode, *topoSystem, NULL, 0)); NCCLCHECK(ncclTopoAddC2c(topNode, *topoSystem, NULL, 0)); NCCLCHECK(ncclTopoAddPciLinks(topNode, *topoSystem, NULL, 0)); NCCLCHECK(ncclTopoFlattenBcmSwitches(*topoSystem)); NCCLCHECK(ncclTopoConnectCpus(*topoSystem)); NCCLCHECK(ncclTopoSortSystem(*topoSystem)); return ncclSuccess; } NCCL_PARAM(TopoDumpFileRank, "TOPO_DUMP_FILE_RANK", 0); // Only set values if not already set static ncclResult_t xmlInitAttrInt(struct ncclXmlNode* node, const char* attrName, const int value) { int index; NCCLCHECK(xmlGetAttrIndex(node, attrName, &index)); if (index == -1) { index = node->nAttrs++; strncpy(node->attrs[index].key, attrName, MAX_STR_LEN); node->attrs[index].key[MAX_STR_LEN] = '\0'; snprintf(node->attrs[index].value, MAX_STR_LEN, "%d", value); } return ncclSuccess; } static ncclResult_t xmlInitAttrUint64(struct ncclXmlNode* node, const char* attrName, const uint64_t value) { int index; NCCLCHECK(xmlGetAttrIndex(node, attrName, &index)); if (index == -1) { index = node->nAttrs++; strncpy(node->attrs[index].key, attrName, MAX_STR_LEN); node->attrs[index].key[MAX_STR_LEN] = '\0'; snprintf(node->attrs[index].value, MAX_STR_LEN, "0x%lx", value); } return ncclSuccess; } static ncclResult_t xmlInitAttrFloat(struct ncclXmlNode* node, const char* attrName, const float value) { int index; NCCLCHECK(xmlGetAttrIndex(node, attrName, &index)); if (index == -1) { index = node->nAttrs++; strncpy(node->attrs[index].key, attrName, MAX_STR_LEN); node->attrs[index].key[MAX_STR_LEN] = '\0'; snprintf(node->attrs[index].value, MAX_STR_LEN, "%f", value); } return ncclSuccess; } ncclResult_t ncclTopoRefreshBcmP2pLinks(void) { //refresh the switch topology by reading the link below FILE *fp = fopen("/sys/kernel/pci_switch_link/refresh_switch_toplogy", "r"); if (fp != NULL) { int tmp; size_t r = fread(&tmp, sizeof(tmp), 1, fp); if (r != 1) INFO(NCCL_GRAPH, "Failed to read refresh_switch_toplogy"); fclose(fp); } return ncclSuccess; } // This is just checking for direct descendence int ncclTopoCheckPix(ncclXmlNode* common, ncclXmlNode** nodes, int nNodes) { const char* tempBusId; // If the common parent isn't a pci switch, then this isn't PIX NCCLCHECK(xmlGetAttrStr(common, "busid", &tempBusId)); if (tempBusId == NULL) return 0; TRACE(NCCL_GRAPH, "Checking pix for busid=%s", tempBusId); // All the nodes must have a "nic" which is a parent, and then a pci node (busid) which must be a child of the "common" for (int i = 0; i < nNodes; i++) { ncclXmlNode* node = nodes[i]; if (strcmp(node->name, "net") == 0) { node = node->parent; if (node == NULL) return 0; if (strcmp(node->name, "nic") == 0) { node = node->parent; if (node == NULL) return 0; // All nodes must descend from the same first level pci switch if (strcmp(node->name, "pci") == 0) { TRACE(NCCL_GRAPH, "Comparing parent of node=%p to common=%p", node->parent, common); if (node->parent != common) return 0; } } } } return 1; } #define NCCL_TOPO_XML_DEPTH_MAX 256 typedef struct xmlNodeStack { ncclXmlNode* elems[NCCL_TOPO_XML_DEPTH_MAX]; int tail; ncclXmlNode* top() { if (!empty()) { return elems[tail - 1]; } else { return NULL; } } ncclXmlNode* pop() { ncclXmlNode* node = top(); if (node) { tail--; } return node; } void push(ncclXmlNode* node) { if (tail < NCCL_TOPO_XML_DEPTH_MAX) { elems[tail++] = node; } } bool empty() { return tail == 0; } } xmlNodeStack; ncclResult_t ncclFindFirstPciParent(ncclXmlNode** parent) { ncclXmlNode* newParent = *parent; while (strcmp(newParent->name, "pci") != 0) { newParent = newParent->parent; if (newParent == nullptr) return ncclSuccess; if (strcmp(newParent->name, "system") == 0) return ncclSuccess; } *parent = newParent; return ncclSuccess; } // 1. Find the common parent xmlNode between the given set of nodes ncclResult_t ncclTopoGetPath(ncclXmlNode** nodes, int nNodes, int* path, ncclXmlNode** parent) { // Track a stack of parents per-net node being merged xmlNodeStack* parents; NCCLCHECK(ncclCalloc(&parents, nNodes)); // Find the common parent ncclXmlNode* common = NULL; if (nNodes == 1) { common = nodes[0]; *path = PATH_LOC; goto out; } for (int i = 0; i < nNodes; i++) { ncclXmlNode* temp; temp = nodes[i]; while (temp) { parents[i].push(temp); temp = strcmp(temp->name, "system") == 0 ? NULL : temp->parent; } } common = NULL; int c; c = 1; while (c && !parents[0].empty()) { ncclXmlNode* temp = parents[0].top(); for (int i = 1; i < nNodes; i++) { if (!parents[i].empty()) { c &= (temp == parents[i].top()); } else { c = 0; break; } } if (c) { common = temp; if (common == NULL) TRACE(NCCL_GRAPH, "COMMON IS NULL"); for (int i = 0; i < nNodes; i++) { parents[i].pop(); } // Check multi-port while we still have the mismatched parents // For multi-port to be true, all parents (peers) must have the busId attribute with all but the last character matching } else { int multiPort = 1; const char* tempBusId; NCCLCHECK(xmlGetAttr(temp, "busid", &tempBusId)); if (tempBusId) { for (int i = 1; i < nNodes; i++) { if (!parents[i].empty()) { const char* busId; NCCLCHECK(xmlGetAttr(parents[i].top(), "busid", &busId)); if (busId) { if (strlen(busId) != strlen(tempBusId)) { multiPort = 0; break; } if (strncmp(busId, tempBusId, strlen(busId)-1) != 0) { multiPort = 0; break; } } else { multiPort = 0; break; } } } } else { multiPort = 0; } if (multiPort) { *path = PATH_PORT; goto out; } } } if (common == NULL) { *path = PATH_DIS; } else if (strcmp(common->name,"system") == 0) { *path = PATH_SYS; } else if (strcmp(common->name, "cpu") == 0) { *path = PATH_PHB; } else if (strcmp(common->name, "nic") == 0) { *path = PATH_PORT; } else if (strcmp(common->name, "net") == 0) { *path = PATH_PORT; } else if (ncclTopoCheckPix(common, nodes, nNodes)) { *path = PATH_PIX; } else { *path = PATH_PXB; } out: ncclFindFirstPciParent(&common); *parent = common; free(parents); return ncclSuccess; } ncclResult_t ncclTopoMakeUniqueBusId(struct ncclXml* xml, char* busId, struct ncclXmlNode** pciNode, struct ncclXmlNode* parent) { int i = 0; int64_t rBusId; NCCLCHECK(busIdToInt64(busId, &rBusId)); // Try to find an unused busid - NCCL expects leaf busid to be unique while (i < 100) { rBusId++; TRACE(NCCL_GRAPH, "Trying to make new busId %lx", rBusId); int64ToBusId(rBusId, busId); struct ncclXmlNode* temp = NULL; NCCLCHECK(xmlFindTagKv(xml, "pci", &temp, "busid", busId)); if (temp == NULL) { NCCLCHECK(xmlAddNode(xml, parent, "pci", pciNode)); NCCLCHECK(xmlSetAttr(*pciNode, "busid", busId)); TRACE(NCCL_GRAPH, "Made new busId %lx", rBusId); return ncclSuccess; } TRACE(NCCL_GRAPH, "Conflicting busId %lx", rBusId); i++; } WARN("TOPO/NET : Couldn't generate unique busId after %d tries", i); return ncclInternalError; } ncclResult_t ncclTopoMakePciParent(struct ncclXml* xml, struct ncclXmlNode** parent, struct ncclXmlNode* physNetNode) { struct ncclXmlNode* newBusId = NULL; struct ncclXmlNode* pci = physNetNode->parent; if (pci) { pci = pci->parent; if (pci) { if (strcmp(pci->name, "pci") == 0) { char busId[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE]; memset(busId, 0, sizeof(busId)); const char* originalBusId; // Seed busId with the current NIC 0's busId to make discovering a unique hash quicker NCCLCHECK(xmlGetAttrStr(pci, "busid", &originalBusId)); snprintf(busId, sizeof(busId), "%s", originalBusId); NCCLCHECK(ncclTopoMakeUniqueBusId(xml, busId, &newBusId, *parent)); for (int i = 0; i < pci->nAttrs; i++) { NCCLCHECK(xmlSetAttr(newBusId, pci->attrs[i].key, pci->attrs[i].value)); } NCCLCHECK(xmlSetAttr(newBusId, "busid", busId)); *parent = newBusId; } } } if (newBusId == NULL) { const char* name; NCCLCHECK(xmlGetAttr(physNetNode, "name", &name)); WARN("TOPO/NET : Can't find busId of child 0 %s", name); return ncclInternalError; } return ncclSuccess; } ncclResult_t ncclTopoMakeVnic(struct ncclXml* xml, struct ncclTopoNetInfo* netInfo, ncclNetVDeviceProps_t* vProps, struct ncclXmlNode** physNetNodes) { if (vProps->ndevs > NCCL_NET_MAX_DEVS_PER_NIC) { WARN("TOPO/NET : Tried to merge too many NICs. %d > %d", vProps->ndevs, NCCL_NET_MAX_DEVS_PER_NIC); return ncclInternalError; } // Don't make vNics of size 1 if (vProps->ndevs == 1) { TRACE(NCCL_GRAPH, "TOPO/NET : Skipping vNic of size 1"); return ncclSuccess; } // Trigger the merge, then get the new device's properties int vDevIndex = 0; ncclResult_t ret; NOWARN(ret = netInfo->makeVDevice(&vDevIndex, vProps), NCCL_GRAPH|NCCL_INIT|NCCL_NET); if (ret != ncclSuccess) { INFO(NCCL_GRAPH|NCCL_INIT|NCCL_NET, "TOPO/NET : Tried merging multiple devices together and failed. vProps={ndevs=%d, devs=[%d %d %d %d]}. Set NCCL_NET_MERGE_LEVEL=LOC to disable NIC fusion.", vProps->ndevs, vProps->devs[0], vProps->devs[1], vProps->devs[2], vProps->devs[3]); return ret; } // Mark original NICs as keep="0" in the topology for (int i = 0; i < vProps->ndevs; i++) { int dev = vProps->devs[i]; struct ncclXmlNode* netNode = physNetNodes[dev]; NCCLCHECK(xmlSetAttrInt(netNode, "keep", 0)); } INFO(NCCL_GRAPH, "TOPO/NET : Made vNic %d", vDevIndex); return ncclSuccess; } ncclResult_t ncclTopoForceMerge(struct ncclXml* xml, struct ncclTopoNetInfo* netInfo, int* placedDevs, ncclNetProperties_t* propsList, struct ncclXmlNode** physNetNodes, int nPhysDevs) { ncclResult_t ret = ncclSuccess; const char* str = netInfo->forceMerge; INFO(NCCL_ENV | NCCL_NET, "TOPO/NET : Force-fusing NICs using NCCL_NET_FORCE_MERGE=%s", str); char* ncStr; NCCLCHECK(ncclCalloc(&ncStr, strlen(str)+1)); strcpy(ncStr, str); char* semi_token; char* semi = strtok_r(ncStr, ";", &semi_token); while (semi) { TRACE(NCCL_NET, "Fusing %s", semi); struct netIf userIfs[NCCL_NET_MAX_DEVS_PER_NIC]; int nUserIfs = parseStringList(semi, userIfs, NCCL_NET_MAX_DEVS_PER_NIC); if (nUserIfs == 0) { INFO(NCCL_NET, "NET/IB : Invalid NCCL_NET_FORCE_MERGE specified %s. Couldn't parse substring %s. Please provide a semicolon-delimited list of comma-delimited NIC groups.", ncStr, semi); continue; } ncclNetVDeviceProps_t vProps = {0}; for (int d = 0; d < nPhysDevs; d++) { if (matchIfList(propsList[d].name, propsList[d].port, userIfs, nUserIfs, 1)) { vProps.devs[vProps.ndevs++] = d; } } if (vProps.ndevs != nUserIfs) { WARN("TOPO/NET : Only matched %d devices, %d requested from %s", vProps.ndevs, nUserIfs, semi); ret = ncclInvalidUsage; goto fail; } if (vProps.ndevs > NCCL_NET_MAX_DEVS_PER_NIC) { WARN("Specified fused NIC %s which has too many devices (%d). Max %d", semi, vProps.ndevs, NCCL_NET_MAX_DEVS_PER_NIC); ret = ncclInvalidUsage; goto fail; } ret = ncclTopoMakeVnic(xml, netInfo, &vProps, physNetNodes); if (ret == ncclSuccess) { // Only set that a device is "placed" after successfully making a vNic (it's possible to exit before this) for (int i = 0; i < vProps.ndevs; i++) { placedDevs[vProps.devs[i]] = 1; } } else { WARN("TOPO/NET : Could not force merge NICs %s. Please specify a valid NCCL_NET_FORCE_MERGE string.", semi); ret = ncclInvalidUsage; goto fail; } semi = strtok_r(NULL, ";", &semi_token);; } exit: free(ncStr); return ret; fail: goto exit; } ncclResult_t ncclTopoAutoMerge(struct ncclXml* xml, struct ncclTopoNetInfo* netInfo, int* placedDevs, ncclNetProperties_t* propsList, struct ncclXmlNode** physNetNodes, int nPhysDevs) { // Compute the path type between each device int* paths = NULL; ncclResult_t res = ncclSuccess; ncclCalloc(&paths, nPhysDevs*nPhysDevs); TRACE(NCCL_GRAPH, "Allocated %d paths", nPhysDevs*nPhysDevs); for (int i = 0; i < nPhysDevs; i++) { for (int j = 0; j < nPhysDevs; j++) { struct ncclXmlNode* nodes[2]; nodes[0] = physNetNodes[i]; nodes[1] = physNetNodes[j]; struct ncclXmlNode* parent; NCCLCHECKGOTO(ncclTopoGetPath(nodes, 2, &paths[i*nPhysDevs + j], &parent), res, out); } } // Place all remaining physical devices into a virtual device given the mergeLevel criteria for (int i = 0; i < nPhysDevs; i++) { // Select the first unplaced device "i" as the root if (placedDevs[i] == 0) { // Init a new vDevice ncclNetVDeviceProps_t vProps; vProps = {0}; vProps.devs[vProps.ndevs++] = i; placedDevs[i] = 1; TRACE(NCCL_GRAPH, "Placed dev %d", i); // Select each unplaced device "j" which is at most "mergeLevel" distance from "i", but not equal to "i" // (Don't merge the same device with itself) for (int j = 0; j < nPhysDevs; j++) { if (paths[i*nPhysDevs + j] <= netInfo->mergeLevel && placedDevs[j] == 0 && j != i) { vProps.devs[vProps.ndevs++] = j; placedDevs[j] = 1; TRACE(NCCL_GRAPH, "Placed dev %d path=%d", j, paths[i*nPhysDevs + j] ); } if (vProps.ndevs == NCCL_NET_MAX_DEVS_PER_NIC) break; } if (vProps.ndevs > NCCL_NET_MAX_DEVS_PER_NIC) { WARN("TOPO/NET : Tried to merge too many NICs. %d > %d", vProps.ndevs, NCCL_NET_MAX_DEVS_PER_NIC); return ncclInternalError; } ncclResult_t ret = ncclTopoMakeVnic(xml, netInfo, &vProps, physNetNodes); // Merging failed. // Mark all as unplaced and increase their distance to disconnected (PATH_DIS) // Set i to 0 to restart the automatic merging process and ensure all are placed if (ret != ncclSuccess) { INFO(NCCL_GRAPH|NCCL_INIT|NCCL_NET, "Marking physical devices as unplaced, increasing distance and restarting search."); placedDevs[i] = 0; TRACE(NCCL_GRAPH, "Setting dev %d as unplaced, keeping distance -> self as PATH_LOC", i); for (int k = 1; k < vProps.ndevs; k++) { int dev = vProps.devs[k]; placedDevs[dev] = 0; paths[i*nPhysDevs + dev] = PATH_DIS; paths[dev*nPhysDevs + i] = PATH_DIS; TRACE(NCCL_GRAPH, "Setting dev %d as unplaced, setting distance -> %d as PATH_DIS", dev, i); } i = 0; } } } out: free(paths); return res; } struct kvDict nicPathKvList[] = { { "LOC", PATH_LOC }, { "PORT", PATH_PORT }, { "PIX", PATH_PIX }, { "PXB", PATH_PXB }, { "P2C", PATH_P2C }, { "PXN", PATH_PXN }, { "PHB", PATH_PHB }, { "SYS", PATH_SYS }, { NULL, 0 } }; ncclResult_t ncclTopoFindLinkWidthRec(ncclXmlNode* node, ncclXmlNode** physNetNodes, int ndevs, int* foundPhysNet, int* linkWidth) { int myLinkWidth = 0; if (strcmp(node->name, "pci") == 0) { NCCLCHECK(xmlGetAttrInt(node, "link_width", &myLinkWidth)); #ifdef ENABLE_TRACE const char *busidAttr, *linkAttr; NCCLCHECK(xmlGetAttrStr(node, "busid", &busidAttr)); NCCLCHECK(xmlGetAttr(node, "link_width", &linkAttr)); TRACE(NCCL_GRAPH, "Found link_width (%s)=%d for busid=%s", linkAttr, myLinkWidth, busidAttr); #endif } *foundPhysNet = 0; // Detect if a physical child is found. This information will be propagated up the stack. int devId = 0; while (devId < ndevs && !(*foundPhysNet)) *foundPhysNet = (node == physNetNodes[devId++]); int totalChildLinkWidth = 0; for (int i = 0; i < node->nSubs; i++) { ncclXmlNode* child = node->subs[i]; int found = 0; int tempLinkWidth = 0; NCCLCHECK(ncclTopoFindLinkWidthRec(child, physNetNodes, ndevs, &found, &tempLinkWidth)); if (found) { *foundPhysNet = 1; totalChildLinkWidth += tempLinkWidth; } } if (*foundPhysNet == 0) { // No child NICs were found, do not accrue any detected link_width *linkWidth = 0; INFO(NCCL_GRAPH, "Did not find child net device. Returning link_width=%d totalChildLinkWidth=%d", *linkWidth, totalChildLinkWidth); } else if (totalChildLinkWidth == 0) { // If A child NIC was found but no link_width was detected among children, assign the link_width to mine (I am the first pci node right above the physNetNode). *linkWidth = myLinkWidth; INFO(NCCL_GRAPH, "Found child net device for %s. Returning link_width=%d totalChildLinkWidth=%d", node->name, *linkWidth, totalChildLinkWidth); } else { // Standard recursive accrual of link_width. The link_width is either the bottleneck of this PCI node's width or the sum of its children's width. *linkWidth = myLinkWidth > 0 ? std::min(myLinkWidth, totalChildLinkWidth) : totalChildLinkWidth; INFO(NCCL_GRAPH, "Found child net device for %s. Returning link_width=%d totalChildLinkWidth=%d", node->name, *linkWidth, totalChildLinkWidth); } return ncclSuccess; } // DFS over nodes under common parent // Exclude link widths of non-physNetNodes chains ncclResult_t ncclTopoFindLinkWidth(ncclXmlNode* parent, ncclXmlNode** physNetNodes, int ndevs, int* linkWidth) { *linkWidth = 0; for (int i = 0; i < parent->nSubs; i++) { ncclXmlNode* child = parent->subs[i]; int foundPhysNet = 0; int childLinkWidth = 0; NCCLCHECK(ncclTopoFindLinkWidthRec(child, physNetNodes, ndevs, &foundPhysNet, &childLinkWidth)); if (foundPhysNet) { *linkWidth += childLinkWidth; } } return ncclSuccess; } ncclResult_t ncclTopoWidenLinks(ncclXmlNode** physNetNodes, int ndevs, ncclXmlNode* parent) { int sumLinkWidth = 0; NCCLCHECK(ncclTopoFindLinkWidth(parent, physNetNodes, ndevs, &sumLinkWidth)); for (int i = 0; i < ndevs; i++) { ncclXmlNode* temp = physNetNodes[i]; while (temp != parent) { if (strcmp(temp->name, "pci") == 0) { NCCLCHECK(xmlSetAttrInt(temp, "link_width", sumLinkWidth)); TRACE(NCCL_GRAPH, "Set link_width to %d for node %s", sumLinkWidth, temp->name); } temp = temp->parent; } } if (strcmp(parent->name, "pci") == 0) { NCCLCHECK(xmlSetAttrInt(parent, "link_width", sumLinkWidth)); TRACE(NCCL_GRAPH, "Set link_width to %d for node %s", sumLinkWidth, parent->name); } return ncclSuccess; } ncclResult_t ncclTopoGetVNicParent(struct ncclXml* xml, ncclResult_t (*getProperties)(int, ncclNetProperties_t*), ncclNetVDeviceProps_t* vProps, ncclXmlNode** parent) { ncclNetProperties_t props[NCCL_NET_MAX_DEVS_PER_NIC]; ncclXmlNode* physNetNodes[NCCL_NET_MAX_DEVS_PER_NIC]; for (int i = 0; i < vProps->ndevs; i++) { NCCLCHECK(getProperties(vProps->devs[i], props + i)); struct ncclXmlNode* physNetNode; NCCLCHECK(xmlFindTagKv(xml, "net", &physNetNode, "name", props[i].name)); physNetNodes[i] = physNetNode; TRACE(NCCL_GRAPH, "Re-found physical ncclNet node %d %s", i, props[i].name); } int path = PATH_LOC; NCCLCHECK(ncclTopoGetPath(physNetNodes, vProps->ndevs, &path, parent)); if (path == PATH_PHB || path == PATH_PXB || path == PATH_PIX) { INFO(NCCL_GRAPH, "Widening links"); NCCLCHECK(ncclTopoWidenLinks(physNetNodes, vProps->ndevs, *parent)); } if (*parent) { if (strcmp((*parent)->name, "pci") == 0) { // Compare PCI class here to avoid NCCL WARN when the "class" attribute doesn't exist const char* c; NCCLCHECK(xmlGetAttrStr(*parent, "class", &c)); if (c && strcmp(c, PCI_BRIDGE_DEVICE_CLASS) == 0) { // If the common parent is a PCI switch, we must reparent the new NIC under a made up pci device with a unique busid NCCLCHECK(ncclTopoMakePciParent(xml, parent, physNetNodes[0])); } } else if (strcmp((*parent)->name, "cpu") == 0) { // If the common parent is a PCI switch, we must reparent the new NIC under a made up pci device with a unique busid NCCLCHECK(ncclTopoMakePciParent(xml, parent, physNetNodes[0])); } } TRACE(NCCL_GRAPH, "Selected parent %s with path %d", (*parent)->name, path); return ncclSuccess; } ncclResult_t ncclTopoMakeVNics(struct ncclXml* xml, struct ncclTopoNetInfo* netInfo, int physicalDevs) { int* placedDevs = NULL; struct ncclXmlNode** physNetNodes = NULL; ncclNetProperties_t* props = NULL; ncclResult_t res = ncclSuccess; if (physicalDevs == 0) return ncclSuccess; NCCLCHECK(ncclCalloc(&physNetNodes, physicalDevs)); NCCLCHECK(ncclCalloc(&placedDevs, physicalDevs)); NCCLCHECK(ncclCalloc(&props, physicalDevs)); for (int i = 0; i < physicalDevs; i++) { NCCLCHECKGOTO(netInfo->getProperties(i, props + i), res, out); struct ncclXmlNode* physNetNode; NCCLCHECKGOTO(xmlFindTagKv(xml, "net", &physNetNode, "name", props[i].name), res, out); physNetNodes[i] = physNetNode; TRACE(NCCL_GRAPH, "Found physical ncclNet node %d %s", i, props[i].name); } if (netInfo->forceMerge) NCCLCHECKGOTO(ncclTopoForceMerge(xml, netInfo, placedDevs, props, physNetNodes, physicalDevs), res, out); NCCLCHECKGOTO(ncclTopoAutoMerge(xml, netInfo, placedDevs, props, physNetNodes, physicalDevs), res, out); out: free(physNetNodes); free(props); if (placedDevs) free(placedDevs); return res; } static ncclResult_t ncclTopoPopulateNics(ncclXml* xml, int startIndex, int endIndex, struct ncclTopoNetInfo* netInfo, int virtualNics) { for (int n = startIndex; n < endIndex; n++) { ncclNetProperties_t props; NCCLCHECK(netInfo->getProperties(n, &props)); struct ncclXmlNode* netNode = NULL; struct ncclXmlNode* parent = NULL; if (virtualNics) { struct ncclXmlNode* net = NULL; NCCLCHECK(xmlFindTagKv(xml, "net", &net, "name", props.name)); // In the event of multithreaded use case, we need to re-discover the shared parent of the given devices for this vNIC // Only run this if the net doesn't exist locally - this may alter the XML state if (net == NULL) NCCLCHECK(ncclTopoGetVNicParent(xml, netInfo->getProperties, &props.vProps, &parent)); } NCCLCHECK(ncclTopoFillNet(xml, "net", props.pciPath, props.name, &netNode, parent)); const char* colAttr; NCCLCHECK(xmlGetAttr(netNode, "coll", &colAttr)); NCCLCHECK(xmlSetAttrInt(netNode, "keep", 1)); int dev; xmlGetAttrIntDefault(netNode, "dev", &dev, -1); if (dev != -1 && dev != n) INFO(NCCL_GRAPH, "TOPO/NET : Changing %s dev index from %d to %d", netInfo->name, dev, n); NCCLCHECK(xmlSetAttrInt(netNode, "dev", n)); NCCLCHECK(xmlInitAttrInt(netNode, "latency", props.latency)); NCCLCHECK(xmlInitAttrInt(netNode, "speed", props.speed)); NCCLCHECK(xmlInitAttrInt(netNode, "port", props.port)); NCCLCHECK(xmlInitAttrUint64(netNode, "guid", props.guid)); NCCLCHECK(xmlInitAttrInt(netNode, "maxconn", props.maxComms)); bool gdrSupport = (props.ptrSupport & NCCL_PTR_CUDA) || (netInfo->dmaBufSupport && (props.ptrSupport & NCCL_PTR_DMABUF)); INFO(NCCL_NET,"NET/%s : GPU Direct RDMA %s for HCA %d '%s'", netInfo->name, gdrSupport ? "Enabled" : "Disabled", n, props.name); NCCLCHECK(xmlInitAttrInt(netNode, "gdr", gdrSupport)); // Do not overwrite the "net" attribute and guarantees that a dev with net=1 will be unchanged int isNet = 0; const char* netAttr = NULL; NCCLCHECK(xmlGetAttr(netNode, "net", &netAttr)); if (netAttr) isNet = strtol(netAttr, NULL, 0); NCCLCHECK(xmlSetAttrInt(netNode, "net", netInfo->net || isNet)); // Only set coll or gin if it's not 0 if (netInfo->coll) NCCLCHECK(xmlInitAttrInt(netNode, "coll", netInfo->coll)); if (netInfo->gin) NCCLCHECK(xmlInitAttrInt(netNode, "gin", netInfo->gin)); const char *keepAttr, *ginAttr; NCCLCHECK(xmlGetAttr(netNode, "net", &netAttr)); NCCLCHECK(xmlGetAttr(netNode, "gin", &ginAttr)); NCCLCHECK(xmlGetAttr(netNode, "coll", &colAttr)); NCCLCHECK(xmlGetAttr(netNode, "keep", &keepAttr)); INFO(NCCL_GRAPH, "ncclTopoPopulateNics : Filled %s in topo with pciPath=%s net=%s gin=%s keep=%s coll=%s", props.name, props.pciPath, netAttr, ginAttr, keepAttr, colAttr); } return ncclSuccess; } // Calls to network plugin APIs should be protected. This function should be called inside a per-process lock. ncclResult_t ncclTopoProcessNet(ncclXml* xml, const char* dumpXmlFile, struct ncclTopoNetInfo* net) { bool usePhysicalDevices = (dumpXmlFile || net->makeVDevice == NULL); int nPhysicalNics, nVirtualNics; NCCLCHECK(net->getDevCount(net->netPluginIndex, &nPhysicalNics, &nVirtualNics)); // List the physical devices in the topo NCCLCHECK(ncclTopoPopulateNics(xml, 0, nPhysicalNics, net, /*virtual=*/false)); if (!usePhysicalDevices) { // Virtual devices are only created once per network if (nVirtualNics == NCCL_UNDEF_DEV_COUNT) { NCCLCHECK(ncclTopoMakeVNics(xml, net, nPhysicalNics)); // Update the number of virtual devices both locally and in the state tracking the plugin. // Note: 0 is a valid number of virtual devices int nDevs; NCCLCHECK(net->devices(&nDevs)); nVirtualNics = nDevs - nPhysicalNics; NCCLCHECK(net->setVirtDevCount(net->netPluginIndex, nVirtualNics)); } // populate the virtual devices if any if (nVirtualNics > 0) { NCCLCHECK(ncclTopoPopulateNics(xml, nPhysicalNics, nPhysicalNics + nVirtualNics, net, /*virtual=*/true)); } } return ncclSuccess; } ncclResult_t ncclTopoGetFusionEnv(int* mergeLevel, const char** forceMerge) { if (forceMerge) *forceMerge = ncclGetEnv("NCCL_NET_FORCE_MERGE"); const char* mergeLevelEnv = ncclGetEnv("NCCL_NET_MERGE_LEVEL"); if (mergeLevelEnv) { kvConvertToInt(mergeLevelEnv, mergeLevel, nicPathKvList); } else { *mergeLevel = PATH_PORT; } return ncclSuccess; } static std::mutex netMutex; ncclResult_t ncclTopoGetSystem(struct ncclComm* comm, struct ncclTopoSystem** system, const char* dumpXmlFile) { ncclResult_t ret = ncclSuccess; struct ncclXml* xml; char* mem = NULL; int* localRanks = NULL; struct ncclXml* rankXml; int localRank = -1, nLocalRanks = 0; struct ncclTopoNetInfo netInfo = {0}; NCCLCHECK(xmlAlloc(&xml, NCCL_TOPO_XML_MAX_NODES)); const char* xmlTopoFile = ncclGetEnv("NCCL_TOPO_FILE"); if (xmlTopoFile) { INFO(NCCL_ENV, "NCCL_TOPO_FILE set by environment to %s", xmlTopoFile); NCCLCHECKGOTO(ncclTopoGetXmlFromFile(xmlTopoFile, xml, 1), ret, fail); } else { // Try default XML topology location NCCLCHECKGOTO(ncclTopoGetXmlFromFile("/var/run/nvidia-topologyd/virtualTopology.xml", xml, 0), ret, fail); } // Fixup the cpu's host_hashes. struct ncclXmlNode* node; // Update every cpu node's host_hash attribute since those are not // intended to be preserved from the XML files that have been read. NCCLCHECKGOTO(xmlFindTag(xml, "cpu", &node), ret, fail); while (node != nullptr) { NCCLCHECKGOTO(xmlSetAttrLong(node, "host_hash", getHostHash()), ret, fail); NCCLCHECKGOTO(xmlFindNextTag(xml, "cpu", node, &node), ret, fail); } if (xml->maxIndex == 0) { // Create top tag struct ncclXmlNode* top; NCCLCHECKGOTO(xmlAddNode(xml, NULL, "system", &top), ret, fail); NCCLCHECKGOTO(xmlSetAttrInt(top, "version", NCCL_TOPO_XML_VERSION), ret, fail); } NCCLCHECKGOTO(ncclTopoRefreshBcmP2pLinks(), ret, fail); // Detect only the GPU managed by this process. We'll get any others through XML fusion. char busId[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE]; NCCLCHECKGOTO(int64ToBusId(comm->peerInfo[comm->rank].busId, busId), ret, fail); NCCLCHECKGOTO(ncclTopoFillGpu(xml, busId, &node), ret, fail); if (node) { NCCLCHECKGOTO(xmlSetAttrInt(node, "keep", 1), ret, fail); NCCLCHECKGOTO(xmlSetAttrInt(node, "rank", comm->rank), ret, fail); NCCLCHECKGOTO(xmlInitAttrInt(node, "gdr", comm->peerInfo[comm->rank].gdrSupport), ret, fail); } // Auto-detect NICs if needed, net/gin/collnet share the same xml/graph nodes. // Start with gin, then with collnet so that they precedence. { std::lock_guard lock(netMutex); INFO(NCCL_GRAPH, "TOPO/NET : Importing network plugins to topology"); ncclGin_t* gin = comm->sharedRes->ginState.ncclGin; if (gin) { netInfo.net = 0; netInfo.coll = 0; netInfo.gin = 1; netInfo.netPluginIndex = comm->ginPluginIndex; netInfo.dmaBufSupport = comm->dmaBufSupport; netInfo.getDevCount = ncclGinGetDevCount; netInfo.name = gin->name; netInfo.getProperties = gin->getProperties; netInfo.makeVDevice = NULL; netInfo.devices = gin->devices; NCCLCHECKGOTO(ncclTopoProcessNet(xml, dumpXmlFile, &netInfo), ret, fail); } if (collNetSupport(comm)) { netInfo.net = 0; netInfo.coll = 1; netInfo.gin = 0; netInfo.netPluginIndex = comm->netPluginIndex; netInfo.dmaBufSupport = comm->dmaBufSupport; netInfo.getDevCount = ncclCollNetGetDevCount; netInfo.setVirtDevCount = ncclCollNetSetVirtDevCount; netInfo.name = comm->ncclCollNet->name; netInfo.getProperties = comm->ncclCollNet->getProperties; netInfo.makeVDevice = comm->ncclCollNet->makeVDevice; netInfo.devices = comm->ncclCollNet->devices; NCCLCHECK(ncclTopoGetFusionEnv(&netInfo.mergeLevel, &netInfo.forceMerge)); NCCLCHECKGOTO(ncclTopoProcessNet(xml, dumpXmlFile, &netInfo), ret, fail); } netInfo.net = 1; netInfo.coll = 0; netInfo.gin = 0; netInfo.netPluginIndex = comm->netPluginIndex; netInfo.dmaBufSupport = comm->dmaBufSupport; netInfo.getDevCount = ncclNetGetDevCount; netInfo.setVirtDevCount = ncclNetSetVirtDevCount; netInfo.name = comm->ncclNet->name; netInfo.getProperties = comm->ncclNet->getProperties; netInfo.makeVDevice = comm->ncclNet->makeVDevice; netInfo.devices = comm->ncclNet->devices; NCCLCHECK(ncclTopoGetFusionEnv(&netInfo.mergeLevel, &netInfo.forceMerge)); NCCLCHECKGOTO(ncclTopoProcessNet(xml, dumpXmlFile, &netInfo), ret, fail); } // Remove XML branches which don't have a node with keep="1" (typically when importing a topology) NCCLCHECKGOTO(ncclTopoTrimXml(xml), ret, fail); // XML topo fusion. if (comm->MNNVL) { // MNNVL clique support nLocalRanks = comm->clique.size; localRank = comm->cliqueRank; localRanks = comm->clique.ranks; } else { // Intra-node fusion. Much of the comm is not initialized yet at this point so we need to do our own calculations. NCCLCHECKGOTO(ncclCalloc(&localRanks, comm->nRanks), ret, fail); for (int i = 0; i < comm->nRanks; i++) { if (comm->peerInfo[i].hostHash == comm->peerInfo[comm->rank].hostHash) { if (i == comm->rank) localRank = nLocalRanks; localRanks[nLocalRanks++] = i; } } } NCCLCHECKGOTO(ncclCalloc(&mem, nLocalRanks * xmlMemSize(NCCL_TOPO_XML_MAX_NODES)), ret, fail); rankXml = (struct ncclXml*)(mem+xmlMemSize(NCCL_TOPO_XML_MAX_NODES)*localRank); memcpy(rankXml, xml, xmlMemSize(NCCL_TOPO_XML_MAX_NODES)); NCCLCHECKGOTO(ncclTopoConvertXml(rankXml, (uintptr_t)xml->nodes, 1), ret, fail); // nLocalRanks can't actually be 0, or we wouldn't be running at all... // coverity[divide_by_zero] NCCLCHECKGOTO(bootstrapIntraNodeAllGather(comm->bootstrap, localRanks, localRank, nLocalRanks, mem, xmlMemSize(NCCL_TOPO_XML_MAX_NODES)), ret, fail); if (comm->MNNVL) { // Ensure that we have enough room when fusing topos from multiple nodes. free(xml); xml = NULL; NCCLCHECKGOTO(xmlAlloc(&xml, nLocalRanks*NCCL_TOPO_XML_MAX_NODES), ret, fail); } else { // In the intra-node case there's no need to enlarge the topo xml. xml->maxIndex = 0; } for (int i = 0; i < nLocalRanks; i++) { struct ncclXml* peerXml = (struct ncclXml*)(mem+xmlMemSize(NCCL_TOPO_XML_MAX_NODES)*i); NCCLCHECKGOTO(ncclTopoConvertXml(peerXml, (uintptr_t)peerXml->nodes, 0), ret, fail); NCCLCHECKGOTO(ncclTopoFuseXml(xml, peerXml), ret, fail); } if (dumpXmlFile && comm->rank == ncclParamTopoDumpFileRank()) { INFO(NCCL_ENV, "NCCL_TOPO_DUMP_FILE set by environment to %s", dumpXmlFile); NCCLCHECKGOTO(ncclTopoDumpXmlToFile(dumpXmlFile, xml), ret, fail); } // Only update our topo tracking structure if we aren't dumping (separate steps) if (dumpXmlFile == NULL) NCCLCHECKGOTO(ncclTopoGetSystemFromXml(xml, system, getHostHash()), ret, fail); exit: if (!comm->MNNVL && localRanks) free(localRanks); if (mem) free(mem); free(xml); return ret; fail: goto exit; } ncclResult_t ncclTopoGetLocal(struct ncclTopoSystem* system, int type, int index, int resultType, int locals[NCCL_TOPO_MAX_NODES], int* localCount, int* pathType) { int minType = PATH_DIS; float maxBw = 0; int count = 0; struct ncclTopoLinkList* paths = system->nodes[type].nodes[index].paths[resultType]; if (paths == NULL) { *localCount = 0; return ncclSuccess; } for (int i=0; inodes[resultType].count; i++) { if (paths[i].bw > maxBw || (paths[i].bw == maxBw && paths[i].type < minType)) { maxBw = paths[i].bw; minType = paths[i].type; if (pathType) *pathType = minType; count = 0; } if (paths[i].bw == maxBw && paths[i].type == minType) { if (count == NCCL_TOPO_MAX_NODES) { WARN("Error : ran out of room to store found nodes in ncclTopoGetLocal." " Filled %d of type %d, starting from index %d of type %d.", NCCL_TOPO_MAX_NODES, resultType, index, type); return ncclInternalError; } locals[count++] = i; } } *localCount = count; return ncclSuccess; } ncclResult_t getLocalNetCountByBw(struct ncclTopoSystem* system, int gpu, int *count, float* bw) { // Assuming BW to CPU reflects the GPU bandwidth via P2P or C2C. // Caveat, this could be wrong if there is a PCIe switch, and a narrower link to the CPU. int c; NCCLCHECK(ncclGetLocalCpu(system, gpu, &c)); float gpuBw = system->nodes[GPU].nodes[gpu].paths[CPU][c].bw; int rank = system->nodes[GPU].nodes[gpu].gpu.rank; int netCountByBw = 0; float totalNetBw = 0; int64_t firstNetId = 0; for (int c = 0; c < MAXCHANNELS; c++) { int net; int64_t netId; NCCLCHECK(ncclTopoGetLocalNet(system, rank, c, &netId, NULL)); NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, &net)); if(c == 0) firstNetId = netId; else if(firstNetId == netId) break; totalNetBw += system->nodes[GPU].nodes[gpu].paths[NET][net].bw; netCountByBw++; if(totalNetBw >= gpuBw) break; } *count = netCountByBw; *bw = totalNetBw; return ncclSuccess; } static int netDevsPolicyNum = -1; static enum netDevsPolicy netDevsPolicy = NETDEVS_POLICY_UNDEF; static void getNetDevsPolicyOnce() { const char* envStr = ncclGetEnv("NCCL_NETDEVS_POLICY"); if (envStr) { if (strcasecmp(envStr, "AUTO") == 0) { netDevsPolicy = NETDEVS_POLICY_AUTO; } else if (strcasecmp(envStr, "ALL") == 0) { netDevsPolicy = NETDEVS_POLICY_ALL; } else if (strncasecmp(envStr, "MAX:", strlen("MAX:")) == 0) { int envNum = atoi(envStr + strlen("MAX:")); if (envNum > 0) { netDevsPolicy = NETDEVS_POLICY_MAX; netDevsPolicyNum = envNum; } } if (netDevsPolicy == NETDEVS_POLICY_UNDEF) INFO(NCCL_ENV, "Unable to recognize NCCL_NETDEVS_POLICY=%s, using NCCL_NETDEVS_POLICY_AUTO instead.", envStr); else INFO(NCCL_ENV, "NCCL_NETDEVS_POLICY set by environment to %s", envStr); } if (netDevsPolicy == NETDEVS_POLICY_UNDEF) netDevsPolicy = NETDEVS_POLICY_AUTO; } ncclResult_t ncclTopoGetNetDevsPolicy(enum netDevsPolicy* policy, int* policyNum) { static std::once_flag onceFlag; std::call_once(onceFlag, getNetDevsPolicyOnce); if (netDevsPolicy == NETDEVS_POLICY_MAX && netDevsPolicyNum <= 0) { WARN("Invalid number of network devices = %d for policy MAX", netDevsPolicyNum); return ncclInternalError; } if (policy) *policy = netDevsPolicy; if (policyNum && netDevsPolicyNum >= 0) *policyNum = netDevsPolicyNum; return ncclSuccess; } ncclResult_t ncclTopoGetLocalNetType(struct ncclTopoSystem* system, int type, int rank, int channelId, int64_t* id, int* dev) { int gpu; NCCLCHECK(ncclTopoRankToIndex(system, rank, &gpu, /*showWarn=*/true)); int localNets[NCCL_TOPO_MAX_NODES]; int localNetCount; NCCLCHECK(ncclTopoGetLocal(system, GPU, gpu, type, localNets, &localNetCount, NULL)); if (localNetCount==0) { WARN("Could not find any local path from gpu %d to net.", gpu); return ncclInternalError; } int netsPerGpu = 0; int policyCount = 0; enum netDevsPolicy policy; NCCLCHECK(ncclTopoGetNetDevsPolicy(&policy, &policyCount)); if (policy == NETDEVS_POLICY_AUTO) { int localGpus[NCCL_TOPO_MAX_NODES]; int localGpuCount; NCCLCHECK(ncclTopoGetLocal(system, type, localNets[0], GPU, localGpus, &localGpuCount, NULL)); netsPerGpu = DIVUP(localNetCount, localGpuCount); } else if (policy == NETDEVS_POLICY_ALL) { netsPerGpu = localNetCount; } else if (policy == NETDEVS_POLICY_MAX) { netsPerGpu = std::min(policyCount, localNetCount); } else { WARN("Unknown netDevs policy"); return ncclInternalError; } int net = system->nodes[GPU].nodes[gpu].gpu.dev; if (isPow2(localNetCount)) net = mirrorBits(net, localNetCount); net += channelId%(netsPerGpu); if (id) *id = system->nodes[type].nodes[localNets[net%localNetCount]].id; if (dev) *dev = system->nodes[type].nodes[localNets[net%localNetCount]].net.dev; return ncclSuccess; } ncclResult_t ncclTopoGetLocalNet(struct ncclTopoSystem* system, int rank, int channelId, int64_t* id, int* dev) { return ncclTopoGetLocalNetType(system, NET, rank, channelId, id, dev); } ncclResult_t ncclTopoGetLocalGinDev(struct ncclTopoSystem* system, int rank, int channelId, int64_t* id, int* dev) { return ncclTopoGetLocalNetType(system, GIN, rank, channelId, id, dev); } ncclResult_t ncclTopoGetLocalGinDevs(struct ncclComm* comm, int* localGinDevs, int* localGinCount) { for (int c=0; ctopo, comm->rank, c, NULL, localGinDevs+c)); if (c > 0 && localGinDevs[c] == localGinDevs[0]) { *localGinCount = c; break; } } return ncclSuccess; } ncclResult_t ncclTopoGetLocalGpu(struct ncclTopoSystem* system, int64_t netId, int* gpuIndex) { ncclResult_t ret = ncclSuccess; int netIndex; NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, &netIndex)); int localGpus[NCCL_TOPO_MAX_NODES]; int localGpuCount; NCCLCHECK(ncclTopoGetLocal(system, NET, netIndex, GPU, localGpus, &localGpuCount, NULL)); int foundGpu = -1; for (int c=0; cnodes[GPU].nodes+g; int64_t id; NCCLCHECK(ncclTopoGetLocalNet(system, gpu->gpu.rank, c, &id, NULL)); if (netId == id) { foundGpu = g; goto exit; } } } exit: *gpuIndex = foundGpu; return ret; } /****************************/ /* External query functions */ /****************************/ ncclResult_t ncclTopoCpuType(struct ncclTopoSystem* system, int* arch, int* vendor, int* model) { *arch = system->nodes[CPU].nodes[0].cpu.arch; *vendor = system->nodes[CPU].nodes[0].cpu.vendor; *model = system->nodes[CPU].nodes[0].cpu.model; return ncclSuccess; } NCCL_PARAM(IgnoreCpuAffinity, "IGNORE_CPU_AFFINITY", 0); ncclResult_t ncclTopoGetCpuAffinity(struct ncclTopoSystem* system, int rank, ncclAffinity* affinity) { struct ncclTopoNode* cpu = NULL, *gpu = NULL; int gpuIndex, cpuIndex; NCCLCHECK(ncclTopoRankToIndex(system, rank, &gpuIndex, /*showWarn=*/true)); NCCLCHECK(ncclGetLocalCpu(system, gpuIndex, &cpuIndex)); gpu = system->nodes[GPU].nodes+gpuIndex; cpu = system->nodes[CPU].nodes+cpuIndex; // Query the CPU affinity set we were provided ncclAffinity mask; NCCLCHECK(ncclOsGetAffinity(&mask)); // Get the affinity of the CPU close to our GPU. ncclAffinity cpuMask = cpu->cpu.affinity; // Get the final affinity ncclAffinity finalMask; if (ncclParamIgnoreCpuAffinity()) // Ignore the CPU affinity set and use the GPU one instead finalMask = cpuMask; else // Use a subset of the GPU affinity set finalMask = ncclOsCpuAnd(mask, cpuMask); memcpy(affinity, &finalMask, sizeof(ncclAffinity)); // display the final affinity char msg[1024] = ""; snprintf(msg + strlen(msg), sizeof(msg) - strlen(msg), "Affinity for GPU %d is ", gpu->gpu.dev); if (ncclOsCpuCount(finalMask)) { (void)ncclCpusetToRangeStr(&finalMask, msg + strlen(msg), sizeof(msg) - strlen(msg)); } else { snprintf(msg + strlen(msg), sizeof(msg) - strlen(msg), "empty, ignoring"); } snprintf(msg + strlen(msg), sizeof(msg) - strlen(msg), ". (GPU affinity = "); (void)ncclCpusetToRangeStr(&cpuMask, msg + strlen(msg), sizeof(msg) - strlen(msg)); if (!ncclParamIgnoreCpuAffinity()) { snprintf(msg + strlen(msg), sizeof(msg) - strlen(msg), " ; CPU affinity = "); (void)ncclCpusetToRangeStr(&mask, msg + strlen(msg), sizeof(msg) - strlen(msg)); } snprintf(msg + strlen(msg), sizeof(msg) - strlen(msg), ")."); INFO(NCCL_INIT, "%s: %s", __func__, msg); return ncclSuccess; } ncclResult_t ncclTopoGetGpuCount(struct ncclTopoSystem* system, int* count) { *count = system->nodes[GPU].count; return ncclSuccess; } ncclResult_t ncclTopoGetNetCount(struct ncclTopoSystem* system, int* count) { *count = system->nodes[NET].count; return ncclSuccess; } ncclResult_t ncclTopoGetNvsCount(struct ncclTopoSystem* system, int* count) { *count = system->nodes[NVS].count; return ncclSuccess; } ncclResult_t ncclTopoGetCompCap(struct ncclTopoSystem* system, int* ccMin, int* ccMax) { if (system->nodes[GPU].count == 0) return ncclInternalError; int min, max; min = max = system->nodes[GPU].nodes[0].gpu.cudaCompCap; for (int g=1; gnodes[GPU].count; g++) { min = std::min(min, system->nodes[GPU].nodes[g].gpu.cudaCompCap); max = std::max(max, system->nodes[GPU].nodes[g].gpu.cudaCompCap); } if (ccMin) *ccMin = min; if (ccMax) *ccMax = max; return ncclSuccess; } nccl-2.29.7-1/src/graph/topo.h000066400000000000000000000217031515037102200157110ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_TOPO_H_ #define NCCL_TOPO_H_ #include "graph.h" #include "core.h" #include "xml.h" #include "net.h" #include "os.h" #define LOC_BW 5000.0 #define SM60_NVLINK_BW 18.0 #define SM70_NVLINK_BW 20.0 #define SM80_NVLINK_BW 20.0 #define SM90_NVLINK_BW 20.6 #define SM86_NVLINK_BW 12.0 #define SM100_NVLINK_BW 40.1 #define PCI_BW 12.0 // PCI Gen3 x16 #define AMD_BW 16.0 #define BDW_QPI_BW 6.0 #define SKL_QPI_BW 10.0 #define SRP_QPI_BW 22.0 #define ERP_QPI_BW 40.0 #define ZPI_BW 6.0 #define YONGFENG_ZPI_BW 9.0 #define P9_BW 32.0 #define ARM_BW 6.0 #define NET_BW 12.0 // 100Gbit // Intel CPU convert GPU P2P traffic into 64B PCI TLPs, so GPU // to GPU traffic consumes more PCI bandwidth. #define INTEL_P2P_OVERHEAD(bw) (bw*6/5) #define NCCL_TOPO_NODE_TYPES 7 #define GPU 0 #define PCI 1 #define NVS 2 #define CPU 3 // Actually NUMA domains #define NIC 4 #define NET 5 #define GIN 6 extern const char* topoNodeTypeStr[]; // We want link types and path types to match as much as possible #define LINK_LOC 0 #define LINK_NVL 1 // Skipping 2 for PATH_NVB #define LINK_C2C 3 #define LINK_PCI 4 // Skipping 5 for PATH_PXB // Skipping 6 for PATH_PXN // Skipping 7 for PATH_P2C // Skipping 8 for PATH_PHB #define LINK_SYS 9 #define LINK_NET 10 extern const char* topoLinkTypeStr[]; // Local (myself) #define PATH_LOC 0 // Connection traversing NVLink #define PATH_NVL 1 // Connection through NVLink using an intermediate GPU #define PATH_NVB 2 // Connection through C2C #define PATH_C2C 3 // Connection traversing at most a single PCIe bridge #define PATH_PIX 4 // Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge) #define PATH_PXB 5 // Connection between a GPU and a NIC using the C2C connection to the CPU and the PCIe connection to the NIC #define PATH_P2C 6 // Connection between a GPU and a NIC using an intermediate GPU. Used to enable rail-local, aggregated network send/recv operations. #define PATH_PXN 7 // Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU) #define PATH_PHB 8 // Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI) #define PATH_SYS 9 // Connection through the network #define PATH_NET 10 // New type of path which should precede PATH_PIX #define PATH_PORT PATH_NVL // Disconnected #define PATH_DIS 11 extern const char* topoPathTypeStr[]; extern int64_t ncclParamPxnC2c(); struct ncclTopoNode; struct ncclTopoLink { int type; float bw; struct ncclTopoNode* remNode; }; // Allows for up to 32 NICs per node on GB200-NVL72 #define NCCL_TOPO_MAX_LINKS 576 #define NCCL_TOPO_MAX_HOPS (NCCL_TOPO_MAX_NODES*NCCL_TOPO_NODE_TYPES) struct ncclTopoLinkList { struct ncclTopoLink* list[NCCL_TOPO_MAX_HOPS]; int count; float bw; int type; }; #define NCCL_TOPO_UNDEF (-1) #define NCCL_TOPO_ID_LOCAL_ID_MASK 0x00ffffffffffffff #define NCCL_TOPO_ID_SYSTEM_ID(id) (id >> 56) #define NCCL_TOPO_ID_LOCAL_ID(id) (id & NCCL_TOPO_ID_LOCAL_ID_MASK) #define NCCL_TOPO_LOCAL_NIC_ID(numaid, busid) (((int64_t)numaid << 56) + busid) #define NCCL_TOPO_ID(systemid, localid) (((int64_t)systemid << 56) + (localid & NCCL_TOPO_ID_LOCAL_ID_MASK)) struct ncclTopoNode { int type; int64_t id; // Type specific data union { struct { int dev; // NVML dev number int rank; int cudaCompCap; int gdrSupport; }gpu; struct { int dev; // Plugin dev number uint64_t pciId; uint64_t asic; int port; float bw; float latency; int gdrSupport; int collSupport; int maxChannels; int localGpu; }net; struct { int arch; int vendor; int model; ncclAffinity affinity; }cpu; struct { uint64_t device; }pci; }; int nlinks; struct ncclTopoLink links[NCCL_TOPO_MAX_LINKS]; // Pre-computed paths to GPUs and NICs struct ncclTopoLinkList* paths[NCCL_TOPO_NODE_TYPES]; // Used during search uint64_t used; }; struct ncclTopoNodeSet { int count; struct ncclTopoNode nodes[NCCL_TOPO_MAX_NODES]; }; struct ncclTopoSystem { int systemId; uint64_t hostHashes[NCCL_TOPO_MAX_NODES]; int nHosts; struct ncclTopoNodeSet nodes[NCCL_TOPO_NODE_TYPES]; float maxBw; float totalBw; int inter; }; ncclResult_t ncclTopoGetNode(struct ncclTopoSystem* system, struct ncclTopoNode** node, int type, uint64_t id); ncclResult_t ncclTopoCreateNode(struct ncclTopoSystem* system, struct ncclTopoNode** node, int type, uint64_t id); ncclResult_t ncclTopoRemoveNode(struct ncclTopoSystem* system, int type, int id); ncclResult_t ncclTopoConnectNodes(struct ncclTopoNode* node, struct ncclTopoNode* remNode, int type, float bw); ncclResult_t ncclTopoPrintPaths(struct ncclTopoSystem* system); ncclResult_t ncclTopoLoadSystem(const char* xmlTopoFile, struct ncclTopoSystem* system); ncclResult_t ncclTopoGetIntermediateRank(struct ncclTopoSystem* system, int rank, int64_t netId, int* intermediateRank); ncclResult_t ncclTopoGetGpuMinPath(struct ncclTopoSystem* system, int type, int* min); ncclResult_t ncclTopoGetGpuMaxPath(struct ncclTopoSystem* system, int type, int* max); ncclResult_t ncclTopoSplitNvLink(struct ncclTopoSystem* system, int* splitNvLink); struct ncclTopoNetInfo { bool coll; bool gin; bool net; // communicator-specific information int netPluginIndex; bool dmaBufSupport; // NIC fusion int mergeLevel; const char* forceMerge; // dev count tracking functions (not part of ncclNet) ncclResult_t (*getDevCount)(int, int*, int*); ncclResult_t (*setVirtDevCount)(int, int); // ncclNet API functions const char* name; ncclResult_t (*getProperties)(int, ncclNetProperties_t*); ncclResult_t (*makeVDevice)(int*, ncclNetVDeviceProps_t*); ncclResult_t (*devices)(int*); }; ncclResult_t ncclTopoProcessNet(ncclXml* xml, const char* dumpXmlFile, struct ncclTopoNetInfo* net); ncclResult_t ncclTopoGetFusionEnv(int* mergeLevel, const char** forceMerge); #define NCCL_TOPO_XML_MAX_NODES 256 #define NCCL_GRAPH_XML_MAX_NODES 65536 ncclResult_t ncclTopoGetSystemFromXml(struct ncclXml* xml, struct ncclTopoSystem** topoSystem, uint64_t localHostHash); ncclResult_t ncclTopoGetGraphFromXml(struct ncclXmlNode *xmlGraphs, struct ncclTopoSystem* system, struct ncclTopoGraph* graph, int* nChannels); ncclResult_t ncclTopoGetXmlFromGraphs(int ngraphs, struct ncclTopoGraph** graphs, struct ncclTopoSystem* system, struct ncclXml *xml); ncclResult_t ncclTopoGetCompCap(struct ncclTopoSystem* system, int* ccMin, int* ccMax); ncclResult_t ncclTopoGetMinNetBw(struct ncclTopoSystem* system, float* bw); static ncclResult_t ncclTopoIdToIndex(struct ncclTopoSystem* system, int type, int64_t id, int* index) { *index = -1; for (int i=0; inodes[type].count; i++) { if (system->nodes[type].nodes[i].id == id) { *index = i; return ncclSuccess; } } return ncclInternalError; } static ncclResult_t ncclTopoRankToIndex(struct ncclTopoSystem* system, int rank, int* index, bool showWarn) { *index = -1; for (int i=0; inodes[GPU].count; i++) { if (system->nodes[GPU].nodes[i].gpu.rank == rank) { *index = i; return ncclSuccess; } } if (showWarn) WARN("ncclTopoRankToIndex could not find rank %d", rank); return ncclInternalError; } static ncclResult_t ncclTopoDevToRank(struct ncclTopoSystem* system, int dev, int* rank) { *rank = -1; for (int i=0; inodes[GPU].count; i++) { if (NCCL_TOPO_ID_SYSTEM_ID(system->nodes[GPU].nodes[i].id) != system->systemId) continue; // Only consider GPUs on our node if (system->nodes[GPU].nodes[i].gpu.dev == dev) { *rank = system->nodes[GPU].nodes[i].gpu.rank; return ncclSuccess; } } return ncclInternalError; } extern struct kvDict nicPathKvList[]; static ncclResult_t ncclTopoIdToNetDev(struct ncclTopoSystem* system, int64_t id, int* netDev) { *netDev = -1; for (int i=0; inodes[NET].count; i++) { if (system->nodes[NET].nodes[i].id == id) { *netDev = system->nodes[NET].nodes[i].net.dev; return ncclSuccess; } } WARN("Could not find NET with id %lx", id); return ncclInternalError; } // Returns NVLink bw in GB/s static float ncclTopoNVLinkBw(int cudaCompCap) { return cudaCompCap >= 100 ? SM100_NVLINK_BW : cudaCompCap >= 90 ? SM90_NVLINK_BW : cudaCompCap == 86 ? SM86_NVLINK_BW : cudaCompCap >= 80 ? SM80_NVLINK_BW : cudaCompCap >= 70 ? SM70_NVLINK_BW : cudaCompCap >= 60 ? SM60_NVLINK_BW : SM80_NVLINK_BW; } // Mirror bits static bool isPow2(int val) { return (val & (val-1)) == 0; } static int mirrorBits(int val, int pow2) { int mirror = 0; for (int b=1, mb=(pow2>>1); b>=1) if (val & b) mirror |= mb; return mirror; } #endif nccl-2.29.7-1/src/graph/trees.cc000066400000000000000000000075401515037102200162130ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "nccl.h" #define RANK_TO_INDEX(r) (rank > root ? rank-1 : rank) /* Btree which alternates leaves and nodes. * Assumes root is 0, which conveniently builds a tree on powers of two, * (because we have pow2-1 ranks) which lets us manipulate bits. * Find first non-zero bit, then : * Find the parent : * xx01[0] -> xx10[0] (1,5,9 below) or xx00[0] if xx10[0] is out of bounds (13 below) * xx11[0] -> xx10[0] (3,7,11 below) * Find the children : * xx10[0] -> xx01[0] (2,4,6,8,10,12) or -1 (1,3,5,7,9,11,13) * xx10[0] -> xx11[0] (2,4,6,8,10) or xx101[0] (12) or xx1001[0] ... or -1 (1,3,5,7,9,11,13) * * Illustration : * 0---------------8 * ______/ \______ * 4 12 * / \ / \ * 2 6 10 \ * / \ / \ / \ \ * 1 3 5 7 9 11 13 */ ncclResult_t ncclGetBtree(int nranks, int rank, int* u, int* d0, int* d1, int* parentChildType) { int up, down0, down1; int bit; for (bit=1; bit 0 so it has to be our child 1, not 0. *d1 = nranks > 1 ? bit >> 1 : -1; return ncclSuccess; } up = (rank ^ bit) | (bit << 1); // if smaller than the parent, we are his first child, otherwise we're his second if (up >= nranks) up = (rank ^ bit); *parentChildType = (rank < up) ? 0 : 1; *u = up; int lowbit = bit >> 1; // down0 is always within bounds down0 = lowbit == 0 ? -1 : rank-lowbit; down1 = lowbit == 0 ? -1 : rank+lowbit; // Make sure down1 is within bounds while (down1 >= nranks) { down1 = lowbit == 0 ? -1 : rank+lowbit; lowbit >>= 1; } *d0 = down0; *d1 = down1; return ncclSuccess; } /* Build a double binary tree. Take the previous tree for the first tree. * For the second tree, we use a mirror tree (if nranks is even) * * 0---------------8 3----------------11 * ______/ \ / \______ * 4 \ / 7 * / \ \ / / \ * 2 6 10 1 5 9 * / \ / \ / \ / \ / \ / \ * 1 3 5 7 9 11 0 2 4 6 8 10 * * or shift it by one rank (if nranks is odd). * * 0---------------8 1---------------9 * ______/ \______ ______/ \______ * 4 12 5 0 * / \ / / \ / * 2 6 10 3 7 11 * / \ / \ / \ / \ / \ / \ * 1 3 5 7 9 11 2 4 6 8 10 12 */ ncclResult_t ncclGetDtree(int nranks, int rank, int* s0, int* d0_0, int* d0_1, int* parentChildType0, int* s1, int* d1_0, int* d1_1, int* parentChildType1) { // First tree ... use a btree ncclGetBtree(nranks, rank, s0, d0_0, d0_1, parentChildType0); // Second tree ... mirror or shift if (nranks % 2 == 1) { // shift int shiftrank = (rank-1+nranks) % nranks; int u, d0, d1; ncclGetBtree(nranks, shiftrank, &u, &d0, &d1, parentChildType1); *s1 = u == -1 ? -1 : (u+1) % nranks; *d1_0 = d0 == -1 ? -1 : (d0+1) % nranks; *d1_1 = d1 == -1 ? -1 : (d1+1) % nranks; } else { // mirror int u, d0, d1; ncclGetBtree(nranks, nranks-1-rank, &u, &d0, &d1, parentChildType1); *s1 = u == -1 ? -1 : nranks-1-u; *d1_0 = d0 == -1 ? -1 : nranks-1-d0; *d1_1 = d1 == -1 ? -1 : nranks-1-d1; } return ncclSuccess; } nccl-2.29.7-1/src/graph/tuning.cc000066400000000000000000000705321515037102200163760ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "core.h" #include "device.h" #include "comm.h" #include "topo.h" #include "nccl_tuner.h" NCCL_PARAM(Nthreads, "NTHREADS", -2); NCCL_PARAM(Ll128Nthreads, "LL128_NTHREADS", -2); static int getNthreads(const char* name, int env, int min, int max, int def) { int nt = env; if (nt > 0) { if (nt % WARP_SIZE != 0) { INFO(NCCL_GRAPH|NCCL_ENV, "Invalid %s %d (must be a multiple of %d)", name, nt, WARP_SIZE); nt = max; } else if (nt > max) { INFO(NCCL_GRAPH|NCCL_ENV, "Invalid %s %d (maximum %d).", name, nt, max); nt = max; } else if (nt < min) { INFO(NCCL_GRAPH|NCCL_ENV, "Invalid %s %d (minimum %d).", name, nt, min); nt = min; } } else { nt = def; } return nt; } // Parse a map of prefixes to a list of elements. The first prefix is // optional and, if not present, the list of elements will be applied // to all prefixes. Only the first list of elements can lack a // prefix. Prefixes (if present) are followed by a colon. Lists of // elements are comma delimited. Mappings of prefix to the lists of // elements are semi-colon delimited. // // For example: // // NCCL_ALGO="ring,collnetdirect;allreduce:tree,collnetdirect;broadcast:ring" // Enable ring and collnetdirect for all functions, then select tree // and collnetdirect for allreduce and ring for broadcast. // // NCCL_PROTO="LL,Simple;allreduce:^LL" // Enable LL and Simple for all functions, but everything except LL // for allreduce. // // NCCL_PROTO="^LL128;allreduce:LL128" // Enable everything but LL128, but only LL128 for allreduce. ncclResult_t parseList(const char* str, const char* prefixElems[], int nprefixes, const char* elems[], int nelems, int* list) { ncclResult_t ret = ncclSuccess; char* fullStr = strdup(str); char* tmpFullStr; char* fullToken = strtok_r(fullStr, ";", &tmpFullStr); char* subToken = nullptr; char* tokStr = nullptr; while (fullToken) { subToken = strdup(fullToken); char* tmpSubStr; char* prefix = strtok_r(subToken, ":", &tmpSubStr); char* elemList = strtok_r(NULL, ":", &tmpSubStr); if (elemList == NULL) { if (fullToken != fullStr) { // It makes no sense for any entry other than the first to not have a prefix, // because then all the prefixes before the prefix-less entry would be // overwritten. WARN("All entries except the first must have a prefix: \"%s\"", str); ret = ncclInvalidUsage; goto fail; } elemList = prefix; prefix = NULL; } int unset, set; if (elemList[0] == '^') { unset = 1; set = 0; elemList++; } else { unset = 0; set = 1; } bool foundPrefix = false; for (int p=0; pminCompCap < 60) return 0; // Need SM60 or higher for CUDA atomics if (patEnable != 2) return patEnable; if (comm->nNodes != comm->nRanks) return 0; // PAT only supports 1 GPU per node if (comm->netDeviceType != NCCL_NET_DEVICE_HOST) return 0; // PAT doesn't support net device offload return 1; } // Network post overhead in ns (1000 = 1 us) NCCL_PARAM(NetOverhead, "NET_OVERHEAD", -2); static float getNetOverhead(struct ncclComm* comm) { if (ncclParamNetOverhead() != -2) return ncclParamNetOverhead() * .001; if (comm->cpuArch == NCCL_TOPO_CPU_ARCH_X86 && comm->cpuVendor == NCCL_TOPO_CPU_VENDOR_INTEL) return 1.0; if (comm->cpuArch == NCCL_TOPO_CPU_ARCH_X86 && comm->cpuVendor == NCCL_TOPO_CPU_VENDOR_AMD) return 2.0; return 1.0; } NCCL_PARAM(Ll128C2c, "LL128_C2C", 1); ncclResult_t ncclTopoInitTunerConstants(struct ncclComm* comm) { comm->tunerConstants = ncclTunerConstantsDefaults; return ncclSuccess; } ncclResult_t ncclTopoTuneModel(struct ncclComm* comm, int minCompCap, int maxCompCap, struct ncclTopoGraph** graphs) { int simpleDefaultThreads = (graphs[NCCL_ALGO_RING]->bwIntra*graphs[NCCL_ALGO_RING]->nChannels <= PCI_BW) ? 256 : NCCL_SIMPLE_MAX_NTHREADS; comm->maxThreads[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] = getNthreads("NCCL_NTHREADS", ncclParamNthreads(), 2*WARP_SIZE, NCCL_SIMPLE_MAX_NTHREADS, simpleDefaultThreads); comm->maxThreads[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE] = getNthreads("NCCL_NTHREADS", ncclParamNthreads(), 2*WARP_SIZE, NCCL_SIMPLE_MAX_NTHREADS, NCCL_SIMPLE_MAX_NTHREADS); comm->maxThreads[NCCL_ALGO_COLLNET_DIRECT][NCCL_PROTO_SIMPLE] = comm->maxThreads[NCCL_ALGO_COLLNET_CHAIN][NCCL_PROTO_SIMPLE] = comm->maxThreads[NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] = comm->maxThreads[NCCL_ALGO_NVLS_TREE][NCCL_PROTO_SIMPLE] = NCCL_MAX_NTHREADS; comm->maxThreads[NCCL_ALGO_RING][NCCL_PROTO_LL] = comm->maxThreads[NCCL_ALGO_TREE][NCCL_PROTO_LL] = getNthreads("NCCL_NTHREADS", ncclParamNthreads(), 2*WARP_SIZE, NCCL_LL_MAX_NTHREADS, NCCL_LL_MAX_NTHREADS); comm->maxThreads[NCCL_ALGO_RING][NCCL_PROTO_LL128] = comm->maxThreads[NCCL_ALGO_TREE][NCCL_PROTO_LL128] = getNthreads("NCCL_LL128_NTHREADS", ncclParamLl128Nthreads(), NCCL_LL128_MAX_NTHREADS/4, NCCL_LL128_MAX_NTHREADS, NCCL_LL128_MAX_NTHREADS); int nNodes = comm->nNodes; int nRanks = comm->nRanks; if (nRanks <= 1) return ncclSuccess; int compCapIndex = minCompCap >= 100 ? NCCL_BLACKWELL_COMPCAP_IDX : (minCompCap >= 90 ? NCCL_HOPPER_COMPCAP_IDX : minCompCap >= 80 ? NCCL_AMPERE_COMPCAP_IDX : NCCL_VOLTA_COMPCAP_IDX); int index2 = nNodes <= 2 ? nNodes-1 : 2; // LL: for single node, we look at GPU type; for multi-node, we look at CPU type int index1 = nNodes == 1 ? compCapIndex : (comm->cpuVendor == NCCL_TOPO_CPU_VENDOR_AMD || comm->cpuVendor == NCCL_TOPO_CPU_VENDOR_MIXED) ? 1 : 0; double llMaxBw = comm->tunerConstants.llMaxBws[index1][index2]; double perChMaxTreeBw = comm->tunerConstants.perChMaxTreeBws[compCapIndex][index2]; double perChMaxRingLL128Bw = comm->tunerConstants.perChMaxRingLL128Bws[compCapIndex][index2]; double perChMaxTreeLL128Bw = comm->tunerConstants.perChMaxTreeLL128Bws[compCapIndex][index2]; double perChMaxNVLSTreeBw = comm->tunerConstants.perChMaxNVLSTreeBws[compCapIndex][index2]; // De-penalize Tree/Simple latency on Power systems to favor Tree than Ring if (comm->cpuArch == NCCL_TOPO_CPU_ARCH_POWER) comm->tunerConstants.hwLatencies[NCCL_HW_PCI][NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE] = comm->tunerConstants.hwLatencies[NCCL_HW_PCI][NCCL_ALGO_RING][NCCL_PROTO_SIMPLE]; float ppn = (float)nRanks / nNodes; int intraHw[NCCL_NUM_ALGORITHMS], hw[NCCL_NUM_ALGORITHMS]; for (int a=0; atypeIntra == LINK_NVL ? NCCL_HW_NVLINK : NCCL_HW_PCI; for (int a=0; abwIntra : graphs[a]->bwInter; if (a == NCCL_ALGO_NVLS_TREE || a == NCCL_ALGO_NVLS) { // NVLS/NVLStree needs at least 2 channels if (graphs[a]->nChannels < 2 ) continue; // Convert to NVLS busBW/channel float intraBw = graphs[a]->bwIntra * nvlsEfficiency[compCapIndex] * (graphs[a]->nChannels - 1) / graphs[a]->nChannels; // AllReduce pipelines two operations. if (coll == ncclFuncAllReduce) { intraBw *= 2.0f; } else { intraBw *= (ppn - 1) / ppn; } // Handle 2 node case of NVLSTree float interBw = graphs[a]->bwInter * ((nNodes <= 2 && a == NCCL_ALGO_NVLS_TREE) ? 2 : 1); bw = std::min( {intraBw, interBw, a == NCCL_ALGO_NVLS_TREE ? (float)perChMaxNVLSTreeBw : std::numeric_limits::max()} ); }; float busBw = graphs[a]->nChannels * bw; // Various model refinements if (a == NCCL_ALGO_RING && p == NCCL_PROTO_LL) { busBw = std::min(llMaxBw, busBw * .5); } if (a == NCCL_ALGO_RING && p == NCCL_PROTO_LL128) busBw = std::min(busBw * (0.92 /*120.0/128.0*/), graphs[a]->nChannels*perChMaxRingLL128Bw); if (a == NCCL_ALGO_TREE && coll == ncclFuncAllReduce) busBw = std::min(busBw*.92, graphs[a]->nChannels*perChMaxTreeBw); if (a == NCCL_ALGO_TREE && p == NCCL_PROTO_LL) busBw = std::min(busBw*1.0/3.8, llMaxBw); if (a == NCCL_ALGO_TREE && p == NCCL_PROTO_LL128) busBw = std::min(busBw * (nNodes == 1 ? 7.0/9.0 : 120.0/128.0), graphs[a]->nChannels*perChMaxTreeLL128Bw); if (a == NCCL_ALGO_TREE && comm->maxTreePattern == NCCL_TOPO_PATTERN_TREE) busBw *= .85; if (a == NCCL_ALGO_PAT) busBw *= .75; if (a == NCCL_ALGO_COLLNET_DIRECT && p != NCCL_PROTO_SIMPLE) busBw = 0; // Not used if (a == NCCL_ALGO_COLLNET_CHAIN && p != NCCL_PROTO_SIMPLE) busBw = 0; // Not used if (a == NCCL_ALGO_COLLNET_DIRECT && p == NCCL_PROTO_SIMPLE) { if (coll == ncclFuncAllGather || coll == ncclFuncReduceScatter) { busBw = ppn * std::min(graphs[a]->bwIntra, graphs[a]->bwInter * 0.9f); } else { // Collnet+Direct requires all GPUs to have a local NIC to work at full speed float factor = ppn / (1.0*graphs[a]->nChannels); // GPU/NIC ratio factor -= (factor-1)/2; busBw /= factor; if (minCompCap >= 90) busBw *= .85; } } // disable collnet for allgather/reducescatter if #localranks > #heads // AllGather/ReduceScatter requires 1:1 GPU:NIC if ((a == NCCL_ALGO_NVLS || a == NCCL_ALGO_COLLNET_DIRECT) && p == NCCL_PROTO_SIMPLE && (coll == ncclFuncAllGather || coll == ncclFuncReduceScatter) && comm->nNodes > 1) { int nHeads = 0; if (coll == ncclFuncAllGather && comm->nNodes > 1 && (!comm->ncclCollNet || !comm->ncclCollNet->iallgather)) busBw = 0.0f; if (coll == ncclFuncReduceScatter && comm->nNodes > 1 && (!comm->ncclCollNet || !comm->ncclCollNet->ireducescatter)) busBw = 0.0f; if (comm->config.collnetEnable) nHeads = comm->collNetHeadsNum; else busBw = 0.0f; if (busBw > 0.0f) { for (int r = 0; r < comm->nRanks; r++) { int node = comm->rankToNode[r]; if (comm->nodeRanks[node].localRanks > nHeads) { busBw = 0.0f; break; } } } } // Convert bus BW to algorithm BW if (!(a != NCCL_ALGO_RING && (coll == ncclFuncAllGather || coll == ncclFuncReduceScatter))) { float ratio = 1.0f; if (a == NCCL_ALGO_RING || a == NCCL_ALGO_NVLS || a == NCCL_ALGO_NVLS_TREE) ratio *= (1.0 * nRanks) / nsteps; else ratio *= .5; busBw *= ratio; } comm->bandwidths[coll][a][p] = busBw; comm->latencies[coll][a][p] = comm->tunerConstants.baseLatencies[a][p]; float intraLat = comm->tunerConstants.hwLatencies[intraHw[a]][a][p]; // With ppn=1 latencies are fully exposed, use the Tree network latency float interLat = ppn == 1 ? comm->tunerConstants.hwLatencies[NCCL_HW_NET][NCCL_ALGO_TREE][p] : comm->tunerConstants.hwLatencies[NCCL_HW_NET][a][p]; interLat += graphs[a]->latencyInter; // Also add the flush extra latency if (p == NCCL_PROTO_SIMPLE) interLat += graphs[a]->latencyInter; if (a == NCCL_ALGO_RING) { float lat = comm->tunerConstants.hwLatencies[hw[a]][a][p]; if ((coll == ncclFuncReduce || coll == ncclFuncBroadcast)) { if (graphs[a]->sameChannels) { comm->latencies[coll][a][p] += lat; } else { if (p == NCCL_PROTO_SIMPLE) lat = comm->tunerConstants.hwLatencies[hw[a]][NCCL_ALGO_TREE][p]; // Add some chunk latency, waiting for proper chunk modeling comm->latencies[coll][a][p] += nsteps*lat; } } else { // Inter-node rings still have to launch nsteps * net overhead. float netOverhead = 0.0; if (nNodes > 1) { netOverhead = getNetOverhead(comm); if (p == NCCL_PROTO_SIMPLE) netOverhead *= 3; } intraLat = std::max(intraLat, netOverhead); int nInterSteps = nNodes == 1 ? 0 : coll == ncclFuncAllReduce ? 2*(nNodes-1) : nNodes-1; comm->latencies[coll][a][p] += (nsteps-nInterSteps)*intraLat + nInterSteps*interLat; } } else if (a == NCCL_ALGO_TREE) { if (coll == ncclFuncAllReduce) { comm->latencies[coll][a][p] += 2 * ((nRanks/nNodes-1) * intraLat + log2i(nNodes) * interLat); } } else if (a == NCCL_ALGO_COLLNET_DIRECT) { comm->latencies[coll][a][p] += 2 * (std::min(1, (nRanks/nNodes-1)) * intraLat + (nRanks/nNodes-1) * 0.4) + interLat; // Add 0.4 us arity serialization latency } else if (a == NCCL_ALGO_COLLNET_CHAIN) { comm->latencies[coll][a][p] += 2 * (nRanks/nNodes-1) * intraLat + interLat; } else if (a == NCCL_ALGO_NVLS) { comm->latencies[coll][a][p] = intraLat; if (nNodes > 1) comm->latencies[coll][a][p] += interLat; } else if (a == NCCL_ALGO_NVLS_TREE) { comm->latencies[coll][a][p] += intraLat + 2 * log2i(nNodes) * interLat; } else if (a == NCCL_ALGO_PAT) { if (coll == ncclFuncAllGather || coll == ncclFuncReduceScatter) { comm->latencies[coll][a][p] += log2i(nNodes) * (interLat/3.5) // Log latency + nRanks * 2.8; // Still a linear part; hopefully we'll manage to remove it at some point. } } } } } // Protocols/Algorithms enable/disable, and user overrides. // All are enabled except ll128 which is enabled by default only in certain cases. int protoEnable[NCCL_NUM_FUNCTIONS*NCCL_NUM_PROTOCOLS]; int algoEnable[NCCL_NUM_FUNCTIONS*NCCL_NUM_ALGORITHMS]; for (int f=0; frank == 0 && (algoStr||protoStr)) { constexpr int strLength = 1024; char funcAlgoProtoTuningStr[strLength]; int offset = 0; offset += snprintf(funcAlgoProtoTuningStr+offset, std::max(0, strLength-offset), "\n Function | "); for (int p=0; ptopo, &nvsCount)); for (int f=0; fnNodes == 1 && a == NCCL_ALGO_NVLS_TREE) disable = 1; // Disable Collnet+Direct, Collnet+Chain or Collnet+NVLS if collnet is not supported. if (comm->config.collnetEnable == 0 && (a == NCCL_ALGO_COLLNET_DIRECT || a == NCCL_ALGO_COLLNET_CHAIN || (a == NCCL_ALGO_NVLS && comm->nNodes > 1))) disable = 1; // Disable CollNet+Direct if not on an NVSwitch system if (nvsCount == 0 && a == NCCL_ALGO_COLLNET_DIRECT) disable = 1; if (disable) algoEnable[f*NCCL_NUM_ALGORITHMS+a] = 0; } } for (int c=0; c= 90) { // Enable LL128 by default only on Hopper/Blackwell for all connections up to P2C and PXN. pEnable &= (graphs[a]->typeInter <= PATH_PXN); } else { // Enable LL128 only up to PXB. Don't enable LL128 over PxN because PxN can encapsulate PxB or P2C links. pEnable &= (graphs[a]->typeInter <= PATH_PXB); if (!ncclParamLl128C2c() && minCompCap >= 90) INFO(NCCL_GRAPH, "Disabling LL128 over all PxN connections (PXB and C2C). This ensures that no C2C link will be used by LL128."); } pEnable &= (graphs[a]->typeIntra <= PATH_NVB); pEnable &= (minCompCap == maxCompCap); pEnable &= !(minCompCap < 70 || (minCompCap == 90 && CUDART_VERSION == 11080 && c == ncclFuncAllReduce && a == NCCL_ALGO_RING && comm->nRanks == 2)); } if (pEnable == 0) comm->bandwidths[c][a][p] = 0; if (algoEnable[c*NCCL_NUM_ALGORITHMS+a] == 0) comm->bandwidths[c][a][p] = 0; } if (comm->rank == 0) { constexpr int lineLen = 1024; char line[lineLen]; int offset = 0; for (int block=0; block= NCCL_NUM_ALGORITHMS) continue; offset += snprintf(line+offset, std::max(0, lineLen-offset), " %14s %14s %14s |", "", ncclAlgoStr[a], ""); } INFO(NCCL_TUNING, "%s", line); offset = snprintf(line, lineLen, " Protocol |"); for (int ba=0; ba<3; ba++) { for (int p=0; p= NCCL_NUM_ALGORITHMS) continue; for (int p=0; pmaxThreads[a][p]); } } INFO(NCCL_TUNING, "%s", line); for (int c=0; c= NCCL_NUM_ALGORITHMS) continue; for (int p=0; platencies[c][a][p], comm->bandwidths[c][a][p]); } } INFO(NCCL_TUNING, "%s", line); } } } // Set per-thread amount of work before we increase nThreads and nChannels for (int a=0; athreadThresholds[a][NCCL_PROTO_LL] = NCCL_LL_THREAD_THRESHOLD; comm->threadThresholds[a][NCCL_PROTO_LL128] = NCCL_LL128_THREAD_THRESHOLD; comm->threadThresholds[a][NCCL_PROTO_SIMPLE] = NCCL_SIMPLE_THREAD_THRESHOLD; } comm->threadThresholds[NCCL_ALGO_RING][NCCL_PROTO_LL] *= nRanks; comm->threadThresholds[NCCL_ALGO_COLLNET_DIRECT][NCCL_PROTO_SIMPLE] = 512; comm->threadThresholds[NCCL_ALGO_COLLNET_CHAIN][NCCL_PROTO_SIMPLE] = 512; // Override defaults with user env const char* str = ncclGetEnv("NCCL_THREAD_THRESHOLDS"); if (str) { INFO(NCCL_ENV, "NCCL_THREAD_THRESHOLDS set by environment to %s", str); ssize_t t[2][NCCL_NUM_PROTOCOLS] = {{ -2, -2, -2 }, { -2, -2, -2 }}; sscanf(str, "%ld %ld %ld %ld %ld %ld", t[0], t[0]+1, t[0]+2, t[1], t[1]+1, t[1]+2); for (int a=0; a<2; a++) { for (int p=0; p= 0) comm->threadThresholds[a][p] = t[a][p]; } } } NCCLCHECK(ncclTopoGetMinNetBw(comm->topo, &comm->minNetBw)); INFO(NCCL_INIT, "threadThresholds %ld/%ld/%ld | %ld/%ld/%ld | %ld | %ld", comm->threadThresholds[NCCL_ALGO_TREE][NCCL_PROTO_LL], comm->threadThresholds[NCCL_ALGO_TREE][NCCL_PROTO_LL128], comm->threadThresholds[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE], comm->threadThresholds[NCCL_ALGO_RING][NCCL_PROTO_LL], comm->threadThresholds[NCCL_ALGO_RING][NCCL_PROTO_LL128], comm->threadThresholds[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE], comm->threadThresholds[NCCL_ALGO_COLLNET_DIRECT][NCCL_PROTO_SIMPLE], comm->threadThresholds[NCCL_ALGO_COLLNET_CHAIN][NCCL_PROTO_SIMPLE]); return ncclSuccess; } // Trees are not perfectly sticking to the model for medium sizes. Applying a static correction // factor is not ideal but works quite well. Powers of two, 64 B to 256MB. static float treeCorrectionFactor[NCCL_NUM_PROTOCOLS][24] = { { 1.0, 1.0, 1.0, 1.0, .9, .8, .7, .7, .7, .7, .6, .5, .4, .4, .5, .6, .7, .8, .9, 1.0, 1.0, 1.0, 1.0, 1.0 }, { 1.0, 1.0, 1.0, 1.0, 1.0, .9, .8, .8, .8, .7, .6, .6, .6, .6, .6, .6, .8, .9, .9, .9, .9, 1.0, 1.0, 1.0 }, { .9, .9, .9, .9, .9, .9, .9, .8, .7, .6, .6, .5, .5, .5, .5, .6, .7, .8, .7, .7, .8, .9, .9, .9 } }; ncclResult_t ncclTopoGetAlgoTime(struct ncclComm* comm, int coll, int algorithm, int protocol, size_t nBytes, int numPipeOps, float* time) { float bw = comm->bandwidths[coll][algorithm][protocol]; float lat = comm->latencies[coll][algorithm][protocol]; if (bw == 0) { *time = -1.0; return ncclSuccess; } int logSize = log2i(nBytes>>6); if (algorithm == NCCL_ALGO_TREE && coll == ncclFuncAllReduce && logSize >= 0 && logSize < 23) bw *= treeCorrectionFactor[protocol][logSize]; if (algorithm == NCCL_ALGO_NVLS_TREE && coll == ncclFuncAllReduce && logSize >= 0 && logSize < 24 && comm->minCompCap >= 100 && comm->cpuArch == NCCL_TOPO_CPU_ARCH_X86) bw *= treeCorrectionFactor[protocol][logSize]; if (algorithm == NCCL_ALGO_RING && protocol == NCCL_PROTO_SIMPLE && comm->nNodes > 1 && coll == ncclFuncAllReduce && nBytes/(comm->nChannels*comm->nRanks) >= 64) { lat *= comm->minCompCap < 80 ? 1.9 : 1.4; // Plateau effect of ring } // Tree pipelining saves latency in aggregation cases int latCount = algorithm == NCCL_ALGO_RING ? numPipeOps : DIVUP(numPipeOps, NCCL_MAX_DEV_WORK_BATCH_COLLS); *time = lat * latCount + nBytes / (1000 * bw); return ncclSuccess; } nccl-2.29.7-1/src/graph/xml.cc000066400000000000000000001111131515037102200156610ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include #include #include #include #include #include #include "core.h" #include "nvmlwrap.h" #include "xml.h" #if defined(__x86_64__) #include #endif // Arbitrarily large number for constructing virtual topology string #define NCCL_MAX_XML_DEPTH 1024 /*******************/ /* XML File Parser */ /*******************/ ncclResult_t xmlGetChar(FILE* file, char* c) { if (fread(c, 1, 1, file) == 0) { WARN("XML Parse : Unexpected EOF"); return ncclInternalError; } return ncclSuccess; } ncclResult_t xmlGetValue(FILE* file, char* value, char* last) { char c; NCCLCHECK(xmlGetChar(file, &c)); if (c != '"' && c != '\'') { #if INT_OK int o = 0; do { value[o] = c; if (o == MAX_STR_LEN-1) { value[o] = '\0'; WARN("Error : value %s too long (max %d)", value, MAX_STR_LEN); return ncclInternalError; } o++; NCCLCHECK(xmlGetChar(file, &c)); } while (c >= '0' && c <= '9'); value[o] = '\0'; *last = c; return ncclSuccess; #else WARN("XML Parse : Expected (double) quote."); return ncclInternalError; #endif } int o = 0; char quote = c; // Remember which quote type we started with do { NCCLCHECK(xmlGetChar(file, &c)); value[o] = c; if (o == MAX_STR_LEN-1) { value[o] = '\0'; WARN("Error : value %s too long (max %d)", value, MAX_STR_LEN); return ncclInternalError; } o++; } while (c != quote); value[o-1] = '\0'; NCCLCHECK(xmlGetChar(file, last)); return ncclSuccess; } ncclResult_t xmlGetToken(FILE* file, char* name, char* value, char* last) { char c; char* ptr = name; int o = 0; do { NCCLCHECK(xmlGetChar(file, &c)); if (c == '=') { ptr[o] = '\0'; if (value == NULL) { WARN("XML Parse : Unexpected value with name %s", ptr); return ncclInternalError; } return xmlGetValue(file, value, last); } ptr[o] = c; if (o == MAX_STR_LEN-1) { ptr[o] = '\0'; WARN("Error : name %s too long (max %d)", ptr, MAX_STR_LEN); return ncclInternalError; } o++; } while (c != ' ' && c != '>' && c != '/' && c != '\n' && c != '\r'); ptr[o-1] = '\0'; *last = c; return ncclSuccess; } // Shift the 3-chars string by one char and append c at the end #define SHIFT_APPEND(s, c) do { s[0]=s[1]; s[1]=s[2]; s[2]=c; } while(0) ncclResult_t xmlSkipComment(FILE* file, char* start, char next) { // Start from something neutral with \0 at the end. char end[4] = "..."; // Inject all trailing chars from previous reads. We don't need // to check for --> here because there cannot be a > in the name. for (int i=0; i" while (strcmp(end, "-->") != 0) { int c; if (fread(&c, 1, 1, file) != 1) { WARN("XML Parse error : unterminated comment"); return ncclInternalError; } SHIFT_APPEND(end, c); } return ncclSuccess; } ncclResult_t xmlGetNode(FILE* file, struct ncclXmlNode* node) { node->type = NODE_TYPE_NONE; char c = ' '; while (c == ' ' || c == '\n' || c == '\r') { if (fread(&c, 1, 1, file) == 0) return ncclSuccess; } if (c != '<') { WARN("XML Parse error : expecting '<', got '%c'", c); return ncclInternalError; } // Read XML element name NCCLCHECK(xmlGetToken(file, node->name, NULL, &c)); // Check for comments if (strncmp(node->name, "!--", 3) == 0) { NCCLCHECK(xmlSkipComment(file, node->name+3, c)); return xmlGetNode(file, node); } // Check for closing tag if (node->name[0] == '\0' && c == '/') { node->type = NODE_TYPE_CLOSE; // Re-read the name, we got '/' in the first call NCCLCHECK(xmlGetToken(file, node->name, NULL, &c)); if (c != '>') { WARN("XML Parse error : unexpected trailing %c in closing tag %s", c, node->name); return ncclInternalError; } return ncclSuccess; } node->type = NODE_TYPE_OPEN; // Get Attributes int a = 0; while (c == ' ') { NCCLCHECK(xmlGetToken(file, node->attrs[a].key, node->attrs[a].value, &c)); if (a == MAX_ATTR_COUNT) { INFO(NCCL_GRAPH, "XML Parse : Ignoring extra attributes (max %d)", MAX_ATTR_COUNT); // Actually we need to still consume the extra attributes so we have an extra one. } else a++; } node->nAttrs = a; if (c == '/') { node->type = NODE_TYPE_SINGLE; char str[MAX_STR_LEN]; NCCLCHECK(xmlGetToken(file, str, NULL, &c)); } if (c != '>') { WARN("XML Parse : expected >, got '%c'", c); return ncclInternalError; } return ncclSuccess; } typedef ncclResult_t (*xmlHandlerFunc_t)(FILE*, struct ncclXml*, struct ncclXmlNode*); struct xmlHandler { const char * name; xmlHandlerFunc_t func; }; ncclResult_t xmlLoadSub(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head, struct xmlHandler handlers[], int nHandlers) { if (head && head->type == NODE_TYPE_SINGLE) return ncclSuccess; while (1) { if (xml->maxIndex == xml->maxNodes) { WARN("Error : XML parser is limited to %d nodes", xml->maxNodes); return ncclInternalError; } struct ncclXmlNode* node = xml->nodes+xml->maxIndex; memset(node, 0, sizeof(struct ncclXmlNode)); NCCLCHECK(xmlGetNode(file, node)); if (node->type == NODE_TYPE_NONE) { if (head) { WARN("XML Parse : unterminated %s", head->name); return ncclInternalError; } else { // All done return ncclSuccess; } } if (head && node->type == NODE_TYPE_CLOSE) { if (strcmp(node->name, head->name) != 0) { WARN("XML Mismatch : %s / %s", head->name, node->name); return ncclInternalError; } return ncclSuccess; } int found = 0; for (int h=0; hname, handlers[h].name) == 0) { if (head) { if (head->nSubs == MAX_SUBS) { WARN("Error : XML parser is limited to %d subnodes", MAX_SUBS); return ncclInternalError; } head->subs[head->nSubs++] = node; } node->parent = head; node->nSubs = 0; xml->maxIndex++; NCCLCHECK(handlers[h].func(file, xml, node)); found = 1; break; } } if (!found) { if (nHandlers) INFO(NCCL_GRAPH, "Ignoring element %s", node->name); NCCLCHECK(xmlLoadSub(file, xml, node, NULL, 0)); } } } /**************/ /* XML Writer */ /**************/ // exp == 1 -- serialize; exp == 0 -- deserialize ncclResult_t ncclTopoConvertXml(struct ncclXml* xml, uintptr_t base, int exp) { for (int n = 0; n < xml->maxIndex; n++) { struct ncclXmlNode *node = &xml->nodes[n]; // For "parent", we shift the base by 1 so that we can distinguish actual // NULL pointers from pointers pointing to the first node. if (node->parent) node->parent = (struct ncclXmlNode *) (exp ? ((uintptr_t)node->parent - base + 1) : (base - 1 + (uintptr_t)node->parent)); for (int s = 0; s < node->nSubs; s++) { node->subs[s] = (struct ncclXmlNode *) (exp ? ((uintptr_t)node->subs[s] - base) : (base + (uintptr_t)node->subs[s])); } } return ncclSuccess; } ncclResult_t ncclTopoDumpXmlRec(int indent, FILE* file, struct ncclXmlNode* node) { for (int i=0; iname); for (int a=0; anAttrs; a++) { fprintf(file, " %s=\"%s\"", node->attrs[a].key, node->attrs[a].value); } if (node->nSubs == 0) { fprintf(file, "/>\n"); } else { fprintf(file, ">\n"); for (int s=0; snSubs; s++) { NCCLCHECK(ncclTopoDumpXmlRec(indent+2, file, node->subs[s])); } for (int i=0; i\n", node->name); } return ncclSuccess; } ncclResult_t ncclTopoDumpXmlToFile(const char* xmlTopoFile, struct ncclXml* xml) { FILE* file = fopen(xmlTopoFile, "w"); if (file == NULL) { INFO(NCCL_GRAPH|NCCL_ENV, "Unable to open %s, not dumping topology.", xmlTopoFile); return ncclSuccess; } NCCLCHECK(ncclTopoDumpXmlRec(0, file, xml->nodes)); fclose(file); return ncclSuccess; } static ncclResult_t xmlTopoFuseXmlRecursive(struct ncclXml* dst, struct ncclXmlNode* dstParent, struct ncclXmlNode* srcParent) { for (int i = 0; i < srcParent->nSubs; i++) { struct ncclXmlNode* srcNode = srcParent->subs[i]; struct ncclXmlNode* dstNode; NCCLCHECK(xmlFindNode(dstParent, srcNode, &dstNode)); if (dstNode == NULL) { NCCLCHECK(xmlAddTree(dst, dstParent, srcNode)); } else { NCCLCHECK(xmlTopoFuseXmlRecursive(dst, dstNode, srcNode)); } } return ncclSuccess; } ncclResult_t ncclTopoFuseXml(struct ncclXml* dst, struct ncclXml* src) { struct ncclXmlNode* topNodeDst; NCCLCHECK(xmlFindTag(dst, "system", &topNodeDst)); if (topNodeDst == NULL) { xmlAddTree(dst, NULL, src->nodes); return ncclSuccess; } struct ncclXmlNode* topNodeSrc; NCCLCHECK(xmlFindTag(src, "system", &topNodeSrc)); NCCLCHECK(xmlTopoFuseXmlRecursive(dst, topNodeDst, topNodeSrc)); return ncclSuccess; } /****************************************/ /* Parser rules for our specific format */ /****************************************/ ncclResult_t ncclTopoXmlLoadNvlink(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { NCCLCHECK(xmlLoadSub(file, xml, head, NULL, 0)); return ncclSuccess; } ncclResult_t ncclTopoXmlLoadPciLink(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { NCCLCHECK(xmlLoadSub(file, xml, head, NULL, 0)); return ncclSuccess; } ncclResult_t ncclTopoXmlLoadC2c(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { NCCLCHECK(xmlLoadSub(file, xml, head, NULL, 0)); return ncclSuccess; } ncclResult_t ncclTopoXmlLoadGpu(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { struct xmlHandler handlers[] = { { "nvlink", ncclTopoXmlLoadNvlink }, { "c2c", ncclTopoXmlLoadC2c } }; NCCLCHECK(xmlLoadSub(file, xml, head, handlers, 2)); return ncclSuccess; } ncclResult_t ncclTopoXmlLoadNet(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { NCCLCHECK(xmlLoadSub(file, xml, head, NULL, 0)); return ncclSuccess; } ncclResult_t ncclTopoXmlLoadNic(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { struct xmlHandler handlers[] = { { "net", ncclTopoXmlLoadNet } }; NCCLCHECK(xmlLoadSub(file, xml, head, handlers, 1)); return ncclSuccess; } ncclResult_t ncclTopoXmlLoadPci(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { struct xmlHandler handlers[] = { { "pci", ncclTopoXmlLoadPci }, { "gpu", ncclTopoXmlLoadGpu }, { "nic", ncclTopoXmlLoadNic}, { "pcilink", ncclTopoXmlLoadPciLink} }; NCCLCHECK(xmlLoadSub(file, xml, head, handlers, 4)); return ncclSuccess; } ncclResult_t ncclTopoXmlLoadCpu(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { struct xmlHandler handlers[] = { { "pci", ncclTopoXmlLoadPci }, { "nic", ncclTopoXmlLoadNic } }; NCCLCHECK(xmlLoadSub(file, xml, head, handlers, 2)); return ncclSuccess; } ncclResult_t ncclTopoXmlLoadSystem(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { int version; NCCLCHECK(xmlGetAttrInt(head, "version", &version)); if (version != NCCL_TOPO_XML_VERSION) { WARN("XML Topology has wrong version %d, %d needed", version, NCCL_TOPO_XML_VERSION); return ncclInvalidUsage; } const char* name; NCCLCHECK(xmlGetAttr(head, "name", &name)); if (name != NULL) INFO(NCCL_GRAPH, "Loading topology %s", name); else INFO(NCCL_GRAPH, "Loading unnamed topology"); struct xmlHandler handlers[] = { { "cpu", ncclTopoXmlLoadCpu } }; NCCLCHECK(xmlLoadSub(file, xml, head, handlers, 1)); return ncclSuccess; } ncclResult_t ncclTopoGetXmlFromFile(const char* xmlTopoFile, struct ncclXml* xml, int warn) { FILE* file = fopen(xmlTopoFile, "r"); if (file == NULL) { if (warn) { INFO(NCCL_GRAPH|NCCL_ENV, "Could not open XML topology file %s : %s", xmlTopoFile, strerror(errno)); } return ncclSuccess; } INFO(NCCL_GRAPH, "Loading topology file %s", xmlTopoFile); struct xmlHandler handlers[] = { { "system", ncclTopoXmlLoadSystem } }; xml->maxIndex = 0; NCCLCHECK(xmlLoadSub(file, xml, NULL, handlers, 1)); fclose(file); return ncclSuccess; } /**********************/ /* XML creation */ /* from autodetection */ /**********************/ #define BUSID_SIZE (sizeof("0000:00:00.0")) #define BUSID_REDUCED_SIZE (sizeof("0000:00")) static void memcpylower(char* dst, const char* src, const size_t size) { for (int i=0; i static ncclResult_t getBcmLinks(const char* busId, int* nlinks, char** peers) { *nlinks = 0; *peers = NULL; char dirPath[] = "/sys/kernel/pci_switch_link/virtual_switch_links/0000:00:00.0"; memcpylower(dirPath+sizeof("/sys/kernel/pci_switch_link/virtual_switch_links/")-1, busId, BUSID_SIZE-1); DIR *dir = opendir(dirPath); if (dir) { struct dirent* file; while ((file = readdir(dir)) != NULL) { if (strlen(file->d_name) != BUSID_SIZE-1) continue; char* path; if (getPciPath(file->d_name, &path) == ncclSystemError) continue; free(path); NCCLCHECK(ncclRealloc(peers, (*nlinks)*BUSID_SIZE, ((*nlinks)+1)*BUSID_SIZE)); memcpy((*peers)+BUSID_SIZE*(*nlinks)++, file->d_name, BUSID_SIZE); } closedir(dir); } return ncclSuccess; } ncclResult_t ncclTopoGetStrFromSys(const char* path, const char* fileName, char* strValue) { char filePath[PATH_MAX]; snprintf(filePath, sizeof(filePath), "%s/%s", path, fileName); int offset = 0; FILE* file; if ((file = fopen(filePath, "r")) != NULL) { while (feof(file) == 0 && ferror(file) == 0 && offset < MAX_STR_LEN) { int len = fread(strValue+offset, 1, MAX_STR_LEN-offset, file); offset += len; } fclose(file); } if (offset == 0) { strValue[0] = '\0'; INFO(NCCL_GRAPH, "Topology detection : could not read %s, ignoring", filePath); } else { strValue[offset-1] = '\0'; } return ncclSuccess; } ncclResult_t ncclTopoSetAttrFromSys(struct ncclXmlNode* pciNode, const char* path, const char* fileName, const char* attrName) { char strValue[MAX_STR_LEN]; NCCLCHECK(ncclTopoGetStrFromSys(path, fileName, strValue)); if (strValue[0] != '\0') { NCCLCHECK(xmlSetAttr(pciNode, attrName, strValue)); } TRACE(NCCL_GRAPH, "Read from sys %s/%s -> %s=%s", path, fileName, attrName, strValue); return ncclSuccess; } ncclResult_t ncclTopoGetXmlFromCpu(struct ncclXmlNode* cpuNode, struct ncclXml* xml) { int index; NCCLCHECK(xmlGetAttrIndex(cpuNode, "affinity", &index)); if (index == -1) { const char* numaId; NCCLCHECK(xmlGetAttr(cpuNode, "numaid", &numaId)); if (numaId == NULL) { WARN("GetXmlFromCpu : could not find CPU numa ID."); return ncclInternalError; } // Set affinity char cpumaskPath[] = "/sys/devices/system/node/node000000"; snprintf(cpumaskPath, sizeof(cpumaskPath), "/sys/devices/system/node/node%s", numaId); NCCLCHECK(ncclTopoSetAttrFromSys(cpuNode, cpumaskPath, "cpumap", "affinity")); } NCCLCHECK(xmlGetAttrIndex(cpuNode, "arch", &index)); if (index == -1) { // Fill CPU type / vendor / model #if defined(__PPC__) NCCLCHECK(xmlSetAttr(cpuNode, "arch", "ppc64")); #elif defined(__aarch64__) NCCLCHECK(xmlSetAttr(cpuNode, "arch", "arm64")); #elif defined(__x86_64__) NCCLCHECK(xmlSetAttr(cpuNode, "arch", "x86_64")); #endif } #if defined(__x86_64__) NCCLCHECK(xmlGetAttrIndex(cpuNode, "vendor", &index)); if (index == -1) { union { struct { // CPUID 0 String register order uint32_t ebx; uint32_t edx; uint32_t ecx; }; char vendor[12]; } cpuid0; unsigned unused; __cpuid(0, unused, cpuid0.ebx, cpuid0.ecx, cpuid0.edx); char vendor[13]; strncpy(vendor, cpuid0.vendor, 12); vendor[12] = '\0'; NCCLCHECK(xmlSetAttr(cpuNode, "vendor", vendor)); } NCCLCHECK(xmlGetAttrIndex(cpuNode, "familyid", &index)); if (index == -1) { union { struct { unsigned steppingId:4; unsigned modelId:4; unsigned familyId:4; unsigned processorType:2; unsigned resv0:2; unsigned extModelId:4; unsigned extFamilyId:8; unsigned resv1:4; }; uint32_t val; } cpuid1; unsigned unused; __cpuid(1, cpuid1.val, unused, unused, unused); int familyId = cpuid1.familyId + (cpuid1.extFamilyId << 4); int modelId = cpuid1.modelId + (cpuid1.extModelId << 4); NCCLCHECK(xmlSetAttrInt(cpuNode, "familyid", familyId)); NCCLCHECK(xmlSetAttrInt(cpuNode, "modelid", modelId)); } #endif return ncclSuccess; } ncclResult_t ncclTopoGetPciNode(struct ncclXml* xml, const char* busId, struct ncclXmlNode** pciNode) { NCCLCHECK(xmlFindTagKv(xml, "pci", pciNode, "busid", busId)); if (*pciNode == NULL) { NCCLCHECK(xmlAddNode(xml, NULL, "pci", pciNode)); NCCLCHECK(xmlSetAttr(*pciNode, "busid", busId)); } return ncclSuccess; } // Check whether a string is in BDF format or not. // BDF (Bus-Device-Function) is "BBBB:BB:DD.F" where B, D and F are hex digits. // There can be trailing chars. int isHex(char c) { return ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')); } int checkBDFFormat(char* bdf) { if (strlen(bdf) != 12) return 0; if ((bdf[4] != ':') || (bdf[7] != ':') || (bdf[10] != '.')) return 0; if ((isHex(bdf[0]) == 0) || (isHex(bdf[1]) == 0) || (isHex(bdf[2]) == 0) || (isHex(bdf[3]) == 0) || (isHex(bdf[5]) == 0) || (isHex(bdf[6]) == 0) || (isHex(bdf[8]) == 0) || (isHex(bdf[9]) == 0) || (isHex(bdf[11]) == 0)) return 0; return 1; } ncclResult_t ncclTopoGetXmlFromSys(struct ncclXmlNode* pciNode, struct ncclXml* xml) { // Fill info, then parent const char* busId; NCCLCHECK(xmlGetAttr(pciNode, "busid", &busId)); char* path = NULL; NOWARN(getPciPath(busId, &path), NCCL_GRAPH); if (path) { NCCLCHECK(ncclTopoSetAttrFromSys(pciNode, path, "class", "class")); } int index; NCCLCHECKNOWARN(xmlGetAttrIndex(pciNode, "vendor", &index), NCCL_GRAPH); if (index == -1) { if (path) NOWARN(ncclTopoSetAttrFromSys(pciNode, path, "vendor", "vendor"), NCCL_GRAPH); } NCCLCHECKNOWARN(xmlGetAttrIndex(pciNode, "device", &index), NCCL_GRAPH); if (index == -1) { if (path) NOWARN(ncclTopoSetAttrFromSys(pciNode, path, "device", "device"), NCCL_GRAPH); } NCCLCHECKNOWARN(xmlGetAttrIndex(pciNode, "subsystem_vendor", &index), NCCL_GRAPH); if (index == -1) { if (path) NOWARN(ncclTopoSetAttrFromSys(pciNode, path, "subsystem_vendor", "subsystem_vendor"), NCCL_GRAPH); } NCCLCHECKNOWARN(xmlGetAttrIndex(pciNode, "subsystem_device", &index), NCCL_GRAPH); if (index == -1) { if (path) NOWARN(ncclTopoSetAttrFromSys(pciNode, path, "subsystem_device", "subsystem_device"), NCCL_GRAPH); } NCCLCHECK(xmlGetAttrIndex(pciNode, "link_speed", &index)); if (index == -1) { if (path) { char deviceSpeedStr[MAX_STR_LEN]; float deviceSpeed = FLT_MAX; NCCLCHECK(ncclTopoGetStrFromSys(path, "max_link_speed", deviceSpeedStr)); sscanf(deviceSpeedStr, "%f GT/s", &deviceSpeed); char portSpeedStr[MAX_STR_LEN]; float portSpeed = FLT_MAX; NCCLCHECK(ncclTopoGetStrFromSys(path, "../max_link_speed", portSpeedStr)); sscanf(portSpeedStr, "%f GT/s", &portSpeed); NCCLCHECK(xmlSetAttr(pciNode, "link_speed", portSpeed < deviceSpeed ? portSpeedStr : deviceSpeedStr)); } else { NCCLCHECK(xmlSetAttr(pciNode, "link_speed", "")); } } NCCLCHECK(xmlGetAttrIndex(pciNode, "link_width", &index)); if (index == -1) { if (path) { char strValue[MAX_STR_LEN]; NCCLCHECK(ncclTopoGetStrFromSys(path, "max_link_width", strValue)); int deviceWidth = strtol(strValue, NULL, 0); NCCLCHECK(ncclTopoGetStrFromSys(path, "../max_link_width", strValue)); int portWidth = strtol(strValue, NULL, 0); NCCLCHECK(xmlSetAttrInt(pciNode, "link_width", std::min(deviceWidth,portWidth))); } else { NCCLCHECK(xmlSetAttr(pciNode, "link_width", "")); } } const char* vendor; NCCLCHECK(xmlGetAttr(pciNode, "vendor", &vendor)); if (vendor != NULL && strcmp(vendor, "0x1000") == 0) { // BCM switch, look for P2P connections int nlinks; char* peers = NULL; NCCLCHECK(getBcmLinks(busId, &nlinks, &peers)); for (int l=0; lparent; if (parent == NULL) { if (path) { // Save that for later in case next step is a CPU char numaIdStr[MAX_STR_LEN]; NCCLCHECK(ncclTopoGetStrFromSys(path, "numa_node", numaIdStr)); // Go up one level in the PCI tree. Rewind two "/" and follow the upper PCI // switch, or stop if we reach a CPU root complex. int slashCount = 0; int parentOffset; for (parentOffset = strlen(path)-1; parentOffset>0; parentOffset--) { if (path[parentOffset] == '/') { slashCount++; path[parentOffset] = '\0'; int start = parentOffset - 1; while (start>0 && path[start] != '/') start--; // Check whether the parent path looks like "BBBB:BB:DD.F" or not. if (checkBDFFormat(path+start+1) == 0) { // This a CPU root complex. Create a CPU tag and stop there. struct ncclXmlNode* topNode; NCCLCHECK(xmlFindTag(xml, "system", &topNode)); NCCLCHECK(xmlGetSubKv(topNode, "cpu", &parent, "numaid", numaIdStr)); if (parent == NULL) { NCCLCHECK(xmlAddNode(xml, topNode, "cpu", &parent)); NCCLCHECK(xmlSetAttrLong(parent, "host_hash", getHostHash())); NCCLCHECK(xmlSetAttr(parent, "numaid", numaIdStr)); } } else if (slashCount == 2) { // Continue on the upper PCI switch for (int i = strlen(path)-1; i>0; i--) { if (path[i] == '/') { NCCLCHECK(xmlFindTagKv(xml, "pci", &parent, "busid", path+i+1)); if (parent == NULL) { NCCLCHECK(xmlAddNode(xml, NULL, "pci", &parent)); NCCLCHECK(xmlSetAttr(parent, "busid", path+i+1)); } break; } } } } if (parent) break; } } else { // No information on /sys, attach GPU to unknown CPU NCCLCHECK(xmlFindTagKv(xml, "cpu", &parent, "numaid", "-1")); if (parent == NULL) { struct ncclXmlNode* topNode; NCCLCHECK(xmlFindTag(xml, "system", &topNode)); NCCLCHECK(xmlAddNode(xml, topNode, "cpu", &parent)); NCCLCHECK(xmlSetAttrLong(parent, "host_hash", getHostHash())); NCCLCHECK(xmlSetAttr(parent, "numaid", "-1")); NCCLCHECK(ncclTopoGetXmlFromCpu(parent, xml)); } } pciNode->parent = parent; // Keep PCI sub devices ordered by PCI Bus ID (Issue #820) // Coverity complains about dereferenced parent being NULL // but this can never happen. // coverity[var_deref_op] int subIndex = parent->nSubs; const char* newBusId; NCCLCHECK(xmlGetAttrStr(pciNode, "busid", &newBusId)); for (int s=0; snSubs; s++) { const char* busId; NCCLCHECK(xmlGetAttr(parent->subs[s], "busid", &busId)); if (busId != NULL && strcmp(newBusId, busId) < 0) { subIndex = s; break; } } if (parent->nSubs == MAX_SUBS) { WARN("Error : XML parser is limited to %d subnodes", MAX_SUBS); return ncclInternalError; } for (int s = parent->nSubs; s > subIndex; s--) parent->subs[s] = parent->subs[s-1]; parent->subs[subIndex] = pciNode; parent->nSubs++; } if (strcmp(parent->name, "pci") == 0) { NCCLCHECK(ncclTopoGetXmlFromSys(parent, xml)); } else if (strcmp(parent->name, "cpu") == 0) { NCCLCHECK(ncclTopoGetXmlFromCpu(parent, xml)); } free(path); return ncclSuccess; } ncclResult_t ncclTopoGetXmlFromGpu(struct ncclXmlNode* pciNode, nvmlDevice_t nvmlDev, struct ncclXml* xml, struct ncclXmlNode** gpuNodeRet) { struct ncclXmlNode* gpuNode = NULL; NCCLCHECK(xmlGetSub(pciNode, "gpu", &gpuNode)); if (gpuNode == NULL) NCCLCHECK(xmlAddNode(xml, pciNode, "gpu", &gpuNode)); int index = -1; int dev = -1; NCCLCHECK(xmlGetAttrIndex(gpuNode, "dev", &index)); if (index == -1) { NCCLCHECK(ncclNvmlDeviceGetIndex(nvmlDev, (unsigned int*)&dev)); NCCLCHECK(xmlSetAttrInt(gpuNode, "dev", dev)); } NCCLCHECK(xmlGetAttrInt(gpuNode, "dev", &dev)); if (dev == -1) { *gpuNodeRet = NULL; return ncclSuccess; } NCCLCHECK(xmlGetAttrIndex(gpuNode, "sm", &index)); if (index == -1) { int cudaMajor, cudaMinor; if (nvmlDev == NULL) { cudaDeviceProp devProp; CUDACHECK(cudaGetDeviceProperties(&devProp, dev)); cudaMajor = devProp.major; cudaMinor = devProp.minor; } else { NCCLCHECK(ncclNvmlDeviceGetCudaComputeCapability(nvmlDev, &cudaMajor, &cudaMinor)); } NCCLCHECK(xmlSetAttrInt(gpuNode, "sm", cudaMajor*10+cudaMinor)); } int sm; NCCLCHECK(xmlGetAttrInt(gpuNode, "sm", &sm)); struct ncclXmlNode* nvlNode = NULL; NCCLCHECK(xmlGetSub(gpuNode, "nvlink", &nvlNode)); if (nvlNode == NULL) { // NVML NVLink detection int maxNvLinks = (sm < 60) ? 0 : (sm < 70) ? 4 : (sm < 80) ? 6 : (sm < 90) ? 12 : 18; if (maxNvLinks > 0 && nvmlDev == NULL) { INFO(NCCL_GRAPH, "No NVML device handle. Skipping nvlink detection."); maxNvLinks = 0; } for (int l=0; l= 11080 if (sm >= 90) { nvmlFieldValue_t fv; fv.fieldId = NVML_FI_DEV_NVLINK_GET_STATE; fv.scopeId = l; // fv.value will contain NV_FEATURE_ENABLED or NV_FEATURE_DISABLED if ((ncclNvmlDeviceGetFieldValues(nvmlDev, 1, &fv) == ncclSuccess) && (fv.nvmlReturn == NVML_SUCCESS)) isActive = (nvmlEnableState_t) fv.value.uiVal; } else /* FALLTHRU to GetNvLinkState if before SM90 */ #endif { (void) ncclNvmlDeviceGetNvLinkState(nvmlDev, l, &isActive); } if (isActive != NVML_FEATURE_ENABLED) continue; // Try to figure out what's on the other side of the NVLink nvmlPciInfo_t remoteProc; if (ncclNvmlDeviceGetNvLinkRemotePciInfo(nvmlDev, l, &remoteProc) != ncclSuccess) continue; // Make a lower case copy of the bus ID for calling ncclDeviceType // PCI system path is in lower case char* p = remoteProc.busId; char lowerId[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE]; for (int c=0; c= 11080 struct ncclXmlNode* c2cNode = NULL; NCCLCHECK(xmlGetSub(gpuNode, "c2c", &c2cNode)); if (c2cNode == NULL) { if (sm >= 90) { int c2cLinksCount = 0; nvmlFieldValue_t fv; fv.fieldId = NVML_FI_DEV_C2C_LINK_COUNT; if ((ncclNvmlDeviceGetFieldValues(nvmlDev, 1, &fv) == ncclSuccess) && (fv.nvmlReturn == NVML_SUCCESS)) { c2cLinksCount = fv.value.uiVal; int bw = 0; int count = 0; for (int l=0; l 0) { NCCLCHECK(xmlAddNode(xml, gpuNode, "c2c", &c2cNode)); NCCLCHECK(xmlSetAttrInt(c2cNode, "bw", bw)); NCCLCHECK(xmlSetAttrInt(c2cNode, "count", count)); } } } } #endif // Fill target classes for (int s=0; snSubs; s++) { struct ncclXmlNode* sub = gpuNode->subs[s]; if (strcmp(sub->name, "nvlink") != 0) continue; int index; NCCLCHECK(xmlGetAttrIndex(sub, "tclass", &index)); if (index == -1) { const char* busId; NCCLCHECK(xmlGetAttr(sub, "target", &busId)); char* path; NOWARN(getPciPath(busId, &path), NCCL_GRAPH); if (path == NULL || strcmp(busId, "fffffff:ffff:ff") == 0) { // Remote NVLink device is not visible inside this VM. Assume NVSwitch. NCCLCHECK(xmlSetAttr(sub, "tclass", "0x068000")); } else { NCCLCHECK(ncclTopoSetAttrFromSys(sub, path, "class", "tclass")); free(path); } } } *gpuNodeRet = gpuNode; return ncclSuccess; } ncclResult_t ncclTopoFillGpu(struct ncclXml* xml, const char* busId, struct ncclXmlNode** gpuNode) { struct ncclXmlNode* node; NCCLCHECK(ncclTopoGetPciNode(xml, busId, &node)); NCCLCHECK(xmlSetAttrIfUnset(node, "class", "0x03")); NCCLCHECK(ncclTopoGetXmlFromSys(node, xml)); nvmlDevice_t nvmlDev; NCCLCHECK(ncclNvmlDeviceGetHandleByPciBusId(busId, &nvmlDev)); NCCLCHECK(ncclTopoGetXmlFromGpu(node, nvmlDev, xml, gpuNode)); return ncclSuccess; } // Returns the subsystem name of a path, i.e. the end of the path // where sysPath/subsystem points to. ncclResult_t ncclTopoGetSubsystem(const char* sysPath, char* subSys) { char subSysPath[PATH_MAX]; snprintf(subSysPath, sizeof(subSysPath), "%s/subsystem", sysPath); char* path = realpath(subSysPath, NULL); if (path == NULL) { subSys[0] = '\0'; } else { int offset; for (offset = strlen(path); offset > 0 && path[offset] != '/'; offset--); strcpy(subSys, path+offset+1); free(path); } return ncclSuccess; } ncclResult_t ncclTopoFillNet(struct ncclXml* xml, const char* tagName, const char* pciPath, const char* netName, struct ncclXmlNode** netNode, struct ncclXmlNode* forceParent) { NCCLCHECK(xmlFindTagKv(xml, tagName, netNode, "name", netName)); if (*netNode != NULL) return ncclSuccess; struct ncclXmlNode* parent = NULL; if (forceParent) { parent = forceParent; } else { const char* pciSysPath = pciPath; if (pciSysPath) { char subSystem[PATH_MAX]; NCCLCHECK(ncclTopoGetSubsystem(pciSysPath, subSystem)); // This is not a PCI device (virtual, usb, ...). if (strcmp(subSystem, "pci") != 0 && !forceParent) { INFO(NCCL_NET | NCCL_GRAPH, "Topology detection: network path (name = %s) %s is not a PCI device (%s). Attaching to first CPU", netName, pciSysPath, subSystem); pciSysPath = NULL; } } if (pciSysPath) { int offset; for (offset = strlen(pciSysPath) - 1; pciSysPath[offset] != '/'; offset--); char busId[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE]; strcpy(busId, pciSysPath + offset + 1); NCCLCHECK(ncclTopoGetPciNode(xml, busId, &parent)); NCCLCHECK(xmlSetAttrIfUnset(parent, "class", "0x02")); NCCLCHECK(ncclTopoGetXmlFromSys(parent, xml)); } else { // Virtual NIC, no PCI device, attach to first CPU NCCLCHECK(xmlFindTag(xml, "cpu", &parent)); } } struct ncclXmlNode* nicNode = NULL; NCCLCHECK(xmlGetSub(parent, "nic", &nicNode)); if (nicNode == NULL) { NCCLCHECK(xmlAddNode(xml, parent, "nic", &nicNode)); } // We know that this net does not exist yet (we searched for it at the // beginning of this function), so we can add it. NCCLCHECK(xmlAddNode(xml, nicNode, tagName, netNode)); NCCLCHECK(xmlSetAttr(*netNode, "name", netName)); return ncclSuccess; } ncclResult_t ncclTopoTrimXmlRec(struct ncclXmlNode* node, int* keep) { const char* str; NCCLCHECK(xmlGetAttr(node, "keep", &str)); if (str && strcmp(str, "1") == 0) { NCCLCHECK(xmlUnsetAttr(node, "keep")); *keep = 1; } else { // Copy nSubs and subs as they could change as we trim recursively. struct ncclXmlNode** subs = (struct ncclXmlNode**)malloc(MAX_SUBS * sizeof(struct ncclXmlNode*)); int nSubs = node->nSubs; memcpy(subs, node->subs, node->nSubs*sizeof(struct ncclXmlNode*)); *keep = 0; for (int s=0; sname, "pci") == 0 || strcmp(node->name, "cpu") == 0 || strcmp(node->name, "nic") == 0 || strcmp(node->name, "net") == 0)) { #ifdef ENABLE_TRACE const char* name; const char* busid; NCCLCHECK(xmlGetAttr(node, "name", &name)); NCCLCHECK(xmlGetAttr(node, "busid", &busid)); TRACE(NCCL_GRAPH, "Removing node %s %s %s\n", node->name, name, busid); #endif NCCLCHECK(xmlRemoveNode(node)); } } return ncclSuccess; } ncclResult_t ncclTopoTrimXml(struct ncclXml* xml) { int keep = 0; NCCLCHECK(ncclTopoTrimXmlRec(xml->nodes, &keep)); return ncclSuccess; } /**************************************************/ /* Parser rules for the user-defined graph search */ /**************************************************/ ncclResult_t ncclTopoXmlGraphLoadGpu(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { NCCLCHECK(xmlLoadSub(file, xml, head, NULL, 0)); return ncclSuccess; } ncclResult_t ncclTopoXmlGraphLoadNet(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { NCCLCHECK(xmlLoadSub(file, xml, head, NULL, 0)); return ncclSuccess; } ncclResult_t ncclTopoXmlGraphLoadChannel(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { struct xmlHandler handlers[] = { { "net", ncclTopoXmlGraphLoadNet }, { "gpu", ncclTopoXmlGraphLoadGpu } }; NCCLCHECK(xmlLoadSub(file, xml, head, handlers, 2)); return ncclSuccess; } ncclResult_t ncclTopoXmlGraphLoadGraph(FILE* file, struct ncclXml* xml, struct ncclXmlNode* head) { struct xmlHandler handlers[] = { { "channel", ncclTopoXmlGraphLoadChannel } }; NCCLCHECK(xmlLoadSub(file, xml, head, handlers, 1)); return ncclSuccess; } ncclResult_t ncclTopoXmlGraphLoadGraphs(FILE* file, struct ncclXml* xmlGraph, struct ncclXmlNode* head) { int version; NCCLCHECK(xmlGetAttrInt(head, "version", &version)); if (version != NCCL_GRAPH_XML_VERSION) { WARN("XML Graph has wrong version %d, %d needed", version, NCCL_GRAPH_XML_VERSION); return ncclInvalidUsage; } const char* name; NCCLCHECK(xmlGetAttr(head, "name", &name)); if (name != NULL) INFO(NCCL_GRAPH, "Loading graphs for topology %s", name); else INFO(NCCL_GRAPH, "Loading graphs"); struct xmlHandler handlers[] = { { "graph", ncclTopoXmlGraphLoadGraph } }; NCCLCHECK(xmlLoadSub(file, xmlGraph, head, handlers, 1)); return ncclSuccess; } ncclResult_t ncclTopoGetXmlGraphFromFile(const char* xmlGraphFile, struct ncclXml* xml) { FILE* file = fopen(xmlGraphFile, "r"); if (file == NULL) { WARN("Could not open XML graph file %s : %s", xmlGraphFile, strerror(errno)); return ncclSystemError; } struct xmlHandler handlers[] = { { "graphs", ncclTopoXmlGraphLoadGraphs } }; xml->maxIndex = 0; NCCLCHECK(xmlLoadSub(file, xml, NULL, handlers, 1)); fclose(file); return ncclSuccess; } nccl-2.29.7-1/src/graph/xml.h000066400000000000000000000335621515037102200155360ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef XML_H_ #define XML_H_ #include "nccl.h" #include "debug.h" #include "checks.h" #include "alloc.h" #include // A few constraints to make the implementation easy #define MAX_STR_LEN 255 #define MAX_ATTR_COUNT 16 #define MAX_SUBS 640 #define NODE_TYPE_NONE 0 #define NODE_TYPE_OPEN 1 #define NODE_TYPE_CLOSE 2 #define NODE_TYPE_SINGLE 3 struct ncclXmlNode { char name[MAX_STR_LEN+1]; struct { char key[MAX_STR_LEN+1]; char value[MAX_STR_LEN+1]; } attrs[MAX_ATTR_COUNT+1]; // Need an extra one to consume extra params int nAttrs; int type; struct ncclXmlNode* parent; struct ncclXmlNode* subs[MAX_SUBS]; int nSubs; }; struct ncclXml { int maxIndex, maxNodes; struct ncclXmlNode nodes[1]; }; /* File functions */ #define NCCL_TOPO_XML_VERSION 1 ncclResult_t ncclTopoGetXmlFromFile(const char* xmlTopoFile, struct ncclXml* xml, int warn); ncclResult_t ncclTopoDumpXmlToFile(const char* xmlTopoFile, struct ncclXml* xml); #define NCCL_GRAPH_XML_VERSION 1 ncclResult_t ncclTopoGetXmlGraphFromFile(const char* xmlGraphFile, struct ncclXml* xml); /* Auto-detect functions */ ncclResult_t ncclTopoFillGpu(struct ncclXml* xml, const char* busId, struct ncclXmlNode** gpuNode); ncclResult_t ncclTopoFillNet(struct ncclXml* xml, const char* tagName, const char* pciPath, const char* netName, struct ncclXmlNode** netNode, struct ncclXmlNode* forceParent=NULL); /* Remove unneeded parts */ ncclResult_t ncclTopoTrimXml(struct ncclXml* xml); /* Fuse multiple system XMLs into one, skipping duplicate entries */ ncclResult_t ncclTopoFuseXml(struct ncclXml* dst, struct ncclXml* src); /* Relocate pointers in XML to (de-)serialize the structure */ ncclResult_t ncclTopoConvertXml(struct ncclXml* xml, uintptr_t base, int exp); /**************/ /* XML Struct */ /* Functions */ /**************/ static size_t xmlMemSize(int maxNodes) { return offsetof(struct ncclXml, nodes) + sizeof(struct ncclXmlNode)*maxNodes; } static ncclResult_t xmlAlloc(struct ncclXml** xml, int maxNodes) { char* mem; NCCLCHECK(ncclCalloc(&mem, xmlMemSize(maxNodes))); *xml = (struct ncclXml*)mem; (*xml)->maxNodes = maxNodes; return ncclSuccess; } static ncclResult_t xmlGetAttrIndex(struct ncclXmlNode* node, const char* attrName, int* index) { *index = -1; const int nAttrs = node->nAttrs; for (int a=0; aattrs[a].key, attrName, MAX_STR_LEN) == 0) { *index = a; return ncclSuccess; } } return ncclSuccess; } static ncclResult_t xmlGetAttr(struct ncclXmlNode* node, const char* attrName, const char** value) { int index; NCCLCHECK(xmlGetAttrIndex(node, attrName, &index)); *value = index == -1 ? NULL : node->attrs[index].value; return ncclSuccess; } static ncclResult_t xmlGetAttrStr(struct ncclXmlNode* node, const char* attrName, const char** value) { NCCLCHECK(xmlGetAttr(node, attrName, value)); if (*value == NULL) { WARN("Attribute %s of node %s not found", attrName, node->name); return ncclInternalError; } return ncclSuccess; } static ncclResult_t xmlGetAttrInt(struct ncclXmlNode* node, const char* attrName, int* value) { const char* str; NCCLCHECK(xmlGetAttrStr(node, attrName, &str)); *value = strtol(str, NULL, 0); return ncclSuccess; } static ncclResult_t xmlGetAttrIntDefault(struct ncclXmlNode* node, const char* attrName, int* value, int defaultValue) { const char* str; NCCLCHECK(xmlGetAttr(node, attrName, &str)); *value = str ? strtol(str, NULL, 0) : defaultValue; return ncclSuccess; } static ncclResult_t xmlGetAttrUint64(struct ncclXmlNode* node, const char* attrName, uint64_t* value) { const char* str; NCCLCHECK(xmlGetAttrStr(node, attrName, &str)); *value = strtoull(str, NULL, 0); return ncclSuccess; } static ncclResult_t xmlGetAttrUint64Default(struct ncclXmlNode* node, const char* attrName, uint64_t* value, uint64_t defaultValue) { const char* str; NCCLCHECK(xmlGetAttr(node, attrName, &str)); *value = str ? strtoull(str, NULL, 0) : defaultValue; return ncclSuccess; } static ncclResult_t xmlGetAttrLong(struct ncclXmlNode* node, const char* attrName, int64_t* value) { const char* str; NCCLCHECK(xmlGetAttrStr(node, attrName, &str)); *value = strtol(str, NULL, 0); return ncclSuccess; } static ncclResult_t xmlGetAttrFloat(struct ncclXmlNode* node, const char* attrName, float* value) { const char* str; NCCLCHECK(xmlGetAttrStr(node, attrName, &str)); *value = strtof(str, NULL); return ncclSuccess; } static ncclResult_t xmlGetAttrFloatDefault(struct ncclXmlNode* node, const char* attrName, float* value, float defaultValue) { const char* str; NCCLCHECK(xmlGetAttr(node, attrName, &str)); *value = str ? strtof(str, NULL) : defaultValue; return ncclSuccess; } static ncclResult_t xmlFindTag(struct ncclXml* xml, const char* tagName, struct ncclXmlNode** node) { *node = NULL; for (int i=0; imaxIndex; i++) { struct ncclXmlNode* n = xml->nodes+i; if (strcmp(n->name, tagName) == 0) { *node = n; return ncclSuccess; } } return ncclSuccess; } static ncclResult_t xmlFindNextTag(struct ncclXml* xml, const char* tagName, struct ncclXmlNode* prev, struct ncclXmlNode** node) { *node = NULL; for (int i=prev-xml->nodes+1; imaxIndex; i++) { struct ncclXmlNode* n = xml->nodes+i; if (strcmp(n->name, tagName) == 0) { *node = n; return ncclSuccess; } } return ncclSuccess; } static ncclResult_t xmlFindTagKv(struct ncclXml* xml, const char* tagName, struct ncclXmlNode** node, const char* attrName, const char* attrValue) { *node = NULL; for (int i=0; imaxIndex; i++) { struct ncclXmlNode* n = xml->nodes+i; if (strcmp(n->name, tagName) == 0) { const char* value; NCCLCHECK(xmlGetAttr(n, attrName, &value)); if (value && strcmp(value, attrValue) == 0) { *node = n; return ncclSuccess; } } } return ncclSuccess; } static ncclResult_t xmlFindNode(struct ncclXmlNode* parentNode, struct ncclXmlNode* searchNode, struct ncclXmlNode** node) { *node = NULL; // Search for the node at the current level only. for (int i=0; inSubs; i++) { struct ncclXmlNode* n = parentNode->subs[i]; if (strcmp(n->name, searchNode->name) == 0 && n->type == searchNode->type && n->nAttrs == searchNode->nAttrs) { int a; // Ensure that all the attributes are the same. for (a=0; anAttrs; a++) { const char* val; NCCLCHECK(xmlGetAttr(n, searchNode->attrs[a].key, &val)); if (!val || strcmp(val, searchNode->attrs[a].value)) break; } if (a == searchNode->nAttrs) { *node = n; return ncclSuccess; } } } return ncclSuccess; } static ncclResult_t xmlSetAttr(struct ncclXmlNode* node, const char* attrName, const char* value) { int index; NCCLCHECK(xmlGetAttrIndex(node, attrName, &index)); if (index == -1) { index = node->nAttrs++; strncpy(node->attrs[index].key, attrName, MAX_STR_LEN); node->attrs[index].key[MAX_STR_LEN] = '\0'; } strncpy(node->attrs[index].value, value, MAX_STR_LEN); node->attrs[index].value[MAX_STR_LEN] = '\0'; return ncclSuccess; } static ncclResult_t xmlPrintNodeRecursive(struct ncclXmlNode* node, const char* name) { while (node) { char line[1024*8]; int cursor = 0; snprintf(line, sizeof(line), "name); for (int i = 0; i < node->nAttrs; i++) { cursor = strlen(line); snprintf(line + cursor, sizeof(line) - cursor, " %s=%s", node->attrs[i].key, node->attrs[i].value); } cursor = strlen(line); snprintf(line + cursor, sizeof(line) - cursor, ">"); INFO(NCCL_GRAPH, "%s", line); node = node->parent; } return ncclSuccess; } static ncclResult_t xmlSetAttrIfUnset(struct ncclXmlNode* node, const char* attrName, const char* value) { int index; NCCLCHECK(xmlGetAttrIndex(node, attrName, &index)); if (index != -1) return ncclSuccess; index = node->nAttrs++; strncpy(node->attrs[index].key, attrName, MAX_STR_LEN); node->attrs[index].key[MAX_STR_LEN] = '\0'; strncpy(node->attrs[index].value, value, MAX_STR_LEN); node->attrs[index].value[MAX_STR_LEN] = '\0'; return ncclSuccess; } static ncclResult_t xmlSetAttrInt(struct ncclXmlNode* node, const char* attrName, const int value) { int index; NCCLCHECK(xmlGetAttrIndex(node, attrName, &index)); if (index == -1) { index = node->nAttrs++; strncpy(node->attrs[index].key, attrName, MAX_STR_LEN); node->attrs[index].key[MAX_STR_LEN] = '\0'; } snprintf(node->attrs[index].value, MAX_STR_LEN, "%d", value); return ncclSuccess; } static ncclResult_t xmlSetAttrFloat(struct ncclXmlNode* node, const char* attrName, const float value) { int index; NCCLCHECK(xmlGetAttrIndex(node, attrName, &index)); if (index == -1) { index = node->nAttrs++; strncpy(node->attrs[index].key, attrName, MAX_STR_LEN); node->attrs[index].key[MAX_STR_LEN] = '\0'; } snprintf(node->attrs[index].value, MAX_STR_LEN, "%g", value); return ncclSuccess; } static ncclResult_t xmlSetAttrLong(struct ncclXmlNode* node, const char* attrName, const int64_t value) { int index; NCCLCHECK(xmlGetAttrIndex(node, attrName, &index)); if (index == -1) { index = node->nAttrs++; strncpy(node->attrs[index].key, attrName, MAX_STR_LEN); node->attrs[index].key[MAX_STR_LEN] = '\0'; } snprintf(node->attrs[index].value, MAX_STR_LEN, "%#lx", value); return ncclSuccess; } static ncclResult_t xmlUnsetAttr(struct ncclXmlNode* node, const char* attrName) { int index; NCCLCHECK(xmlGetAttrIndex(node, attrName, &index)); if (index == -1) return ncclSuccess; for (int i=index+1; inAttrs; i++) { strcpy(node->attrs[i-1].key, node->attrs[i].key); strcpy(node->attrs[i-1].value, node->attrs[i].value); } node->nAttrs--; return ncclSuccess; } static ncclResult_t xmlGetSub(struct ncclXmlNode* node, const char* subName, struct ncclXmlNode** sub) { *sub = NULL; for (int s=0; snSubs; s++) { if (strcmp(node->subs[s]->name, subName) == 0) { *sub = node->subs[s]; return ncclSuccess; } } return ncclSuccess; } static ncclResult_t xmlGetSubKv(struct ncclXmlNode* node, const char* subName, struct ncclXmlNode** sub, const char* attrName, const char* attrValue) { *sub = NULL; for (int s=0; snSubs; s++) { struct ncclXmlNode* subNode = node->subs[s]; if (strcmp(subNode->name, subName) == 0) { const char* value; NCCLCHECK(xmlGetAttr(subNode, attrName, &value)); if (value && strcmp(value, attrValue) == 0) { *sub = node->subs[s]; return ncclSuccess; } } } return ncclSuccess; } static ncclResult_t xmlGetSubKvInt(struct ncclXmlNode* node, const char* subName, struct ncclXmlNode** sub, const char* attrName, const int attrValue) { char strValue[10]; snprintf(strValue, 10, "%d", attrValue); NCCLCHECK(xmlGetSubKv(node, subName, sub, attrName, strValue)); return ncclSuccess; } static ncclResult_t xmlAddNode(struct ncclXml* xml, struct ncclXmlNode* parent, const char* subName, struct ncclXmlNode** sub) { if (xml->maxIndex == xml->maxNodes) { WARN("Error : too many XML nodes (max %d)", xml->maxNodes); return ncclInternalError; } struct ncclXmlNode* s = xml->nodes+xml->maxIndex++; s->nSubs = 0; s->nAttrs = 0; *sub = s; s->parent = parent; if (parent) { if (parent->nSubs == MAX_SUBS) { WARN("Error : too many XML subnodes (max %d)", MAX_SUBS); return ncclInternalError; } parent->subs[parent->nSubs++] = s; } strncpy(s->name, subName, MAX_STR_LEN); s->name[MAX_STR_LEN] = '\0'; return ncclSuccess; } static ncclResult_t xmlRemoveNode(struct ncclXmlNode* node) { node->type = NODE_TYPE_NONE; struct ncclXmlNode* parent = node->parent; if (parent == NULL) return ncclSuccess; int shift = 0; for (int s=0; snSubs; s++) { if (parent->subs[s] == node) shift = 1; else if (shift) parent->subs[s-1] = parent->subs[s]; } parent->nSubs--; return ncclSuccess; } static ncclResult_t xmlAddTree(struct ncclXml* dst, struct ncclXmlNode* parent, struct ncclXmlNode* srcNode) { if (dst->maxIndex == dst->maxNodes) { WARN("Error : too many XML nodes (max %d)", dst->maxNodes); return ncclInternalError; } struct ncclXmlNode* dstNode = dst->nodes+dst->maxIndex++; *dstNode = *srcNode; dstNode->parent = parent; if (parent) { if (parent->nSubs == MAX_SUBS) { WARN("Error : too many XML subnodes (max %d)", MAX_SUBS); return ncclInternalError; } parent->subs[parent->nSubs++] = dstNode; } dstNode->nSubs = 0; // Recursively copy the subtree(s) for (int i=0; inSubs; i++) NCCLCHECK(xmlAddTree(dst, dstNode, srcNode->subs[i])); return ncclSuccess; } // Dictionary for STR -> INT conversions. No dictionary size information, // there needs to be a last element with str == NULL. struct kvDict { const char* str; int value; }; static ncclResult_t kvConvertToInt(const char* str, int* value, struct kvDict* dict) { struct kvDict* d = dict; while (d->str) { if (strncmp(str, d->str, strlen(d->str)) == 0) { *value = d->value; return ncclSuccess; } d++; } INFO(NCCL_GRAPH, "KV Convert to int : could not find value of '%s' in dictionary, falling back to %d", str, d->value); *value = d->value; return ncclSuccess; } static ncclResult_t kvConvertToStr(int value, const char** str, struct kvDict* dict) { struct kvDict* d = dict; while (d->str) { if (value == d->value) { *str = d->str; return ncclSuccess; } d++; } WARN("KV Convert to str : could not find value %d in dictionary", value); return ncclInternalError; } #endif nccl-2.29.7-1/src/group.cc000066400000000000000000001000721515037102200151160ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #include "group.h" #include "debug.h" #include "enqueue.h" #include "transport.h" #include "channel.h" #include #include "bootstrap.h" #include "ce_coll.h" #include "profiler.h" #include "nvtx.h" #include "compiler.h" #include "rma/rma.h" #include "argcheck.h" #define GROUP_MAX_RECLAIM_STEPS 10 thread_local int ncclGroupDepth = 0; // depth of ncclGroupStart nesting thread_local ncclResult_t ncclGroupError = ncclSuccess; thread_local struct ncclComm* ncclGroupCommHead[ncclGroupTaskTypeNum] = {nullptr}; thread_local struct ncclComm* ncclGroupCommPreconnectHead = nullptr; thread_local struct ncclIntruQueue ncclAsyncJobs; thread_local int ncclGroupBlocking = -1; /* default mode */ void* ncclAsyncJobMain(void* arg); ncclResult_t ncclAsyncLaunch( struct ncclAsyncJob* job, ncclResult_t(*func)(struct ncclAsyncJob*), void(*undo)(struct ncclAsyncJob*), void(*destructor)(void*), ncclComm_t comm ) { ncclResult_t ret = ncclSuccess; job->destroyFlag = comm->destroyFlag; if (ncclGroupDepth == 0) { ret = func(job); if (ret != ncclSuccess && undo) undo(job); if (destructor) destructor(job); } else { job->func = func; job->undo = undo; job->destructor = destructor; job->abortFlag = comm->abortFlag; job->abortFlagDev = comm->abortFlagDev; job->childAbortFlag = comm->childAbortFlag; job->childAbortFlagDev = comm->childAbortFlagDev; job->state = ncclGroupJobRunning; job->comm = comm; /* check if there are blocking and nonblocking comms at the same time in group. */ if (comm->destroyFlag) { ncclGroupBlocking = 1; } else if (ncclGroupBlocking == -1) { /* first met communicator */ ncclGroupBlocking = comm->config.blocking; } else if (ncclGroupBlocking != comm->config.blocking) { WARN("Blocking and nonblocking communicators are not allowed in the same group."); ret = ncclInvalidArgument; } if (ret == ncclSuccess) { ncclIntruQueueEnqueue(&ncclAsyncJobs, job); } else { // no need to undo, the job hasn't run if (destructor) destructor(job); } } return ret; } void* ncclAsyncJobMain(void* arg) { struct ncclAsyncJob* job = (struct ncclAsyncJob*)arg; job->result = job->func(job); if (job->result != ncclSuccess) { INFO(NCCL_INIT,"%s:%d -> %d [Async thread]", __FILE__, __LINE__, job->result); } COMPILER_ATOMIC_STORE(&job->state, ncclGroupJobDone, std::memory_order_release); return arg; } ncclResult_t ncclAsyncJobComplete(struct ncclAsyncJob* job) { ncclResult_t ret; NCCLCHECK(ncclThreadJoin(job->thread)); if (job->result != ncclSuccess) { WARN("ncclAsyncJobComplete: job %p failed, job error %d", job, job->result); } ret = job->result; if (job->destructor) job->destructor((void*)job); return ret; } NCCL_API(ncclResult_t, ncclGroupStart); ncclResult_t ncclGroupStart() { ncclResult_t ret = ncclSuccess; NCCL_NVTX3_FUNC_RANGE; NCCLCHECK(ncclGroupStartInternal()); TRACE_CALL("ncclGroupStart()"); return ret; } NCCL_API(ncclResult_t, ncclGroupEnd); ncclResult_t ncclGroupEnd() { ncclResult_t ret = ncclSuccess; NCCL_NVTX3_FUNC_RANGE; NCCLCHECKGOTO(ncclGroupEndInternal(), ret, exit); TRACE_CALL("ncclGroupEnd()"); exit: return ret; } NCCL_API(ncclResult_t, ncclGroupSimulateEnd, ncclSimInfo_t* simInfo); ncclResult_t ncclGroupSimulateEnd(ncclSimInfo_t* simInfo) { ncclResult_t ret = ncclSuccess; NCCL_NVTX3_FUNC_RANGE; NCCLCHECKGOTO(ncclGroupEndInternal(simInfo), ret, exit); TRACE_CALL("ncclGroupSimulateEnd()"); exit: return ret; } struct ncclPreconnectJob { struct ncclAsyncJob base; struct ncclComm* comm; bool* algoNeedConnect; }; struct ncclPrepareTasksAndCollPreconnectJob { struct ncclAsyncJob base; struct ncclComm* comm; ncclSimInfo_t* simInfo; }; ncclResult_t ncclP2PPreconnectFunc(struct ncclAsyncJob* job_) { struct ncclPreconnectJob* job = (struct ncclPreconnectJob*)job_; struct ncclComm* comm = job->comm; CUDACHECK(cudaSetDevice(comm->cudaDev)); if (!job_->isThreadMain && ncclOsCpuCount(comm->cpuAffinity)) ncclOsSetAffinity(comm->cpuAffinity); NCCLCHECK(ncclTransportP2pSetup(comm, NULL, 1)); return ncclSuccess; } static ncclResult_t ncclCollPreconnect(struct ncclComm* comm, bool* algoNeedConnect) { for (int i = 0; i < NCCL_NUM_ALGORITHMS; ++i) { if (algoNeedConnect[i]) { switch (i) { case NCCL_ALGO_RING: { NCCLCHECK(ncclTransportRingConnect(comm)); break; } case NCCL_ALGO_TREE: { NCCLCHECK(ncclTransportTreeConnect(comm)); break; } case NCCL_ALGO_NVLS: { /* If we are using NVLS_TREE algo, we must mark NVLS algo to set up * NVLS intra-node buffer */ NCCLCHECK(ncclNvlsBufferSetup(comm)); break; } case NCCL_ALGO_NVLS_TREE: { NCCLCHECK(ncclNvlsTreeConnect(comm)); break; } case NCCL_ALGO_COLLNET_CHAIN: { NCCLCHECK(ncclCollNetChainBufferSetup(comm)); break; } case NCCL_ALGO_COLLNET_DIRECT: { NCCLCHECK(ncclCollNetDirectBufferSetup(comm)); break; } case NCCL_ALGO_PAT: { NCCLCHECK(ncclTransportPatConnect(comm)); break; } // Yes, it's a dead code. That's fine... // coverity[dead_error_begin] default: { NCCLCHECK(ncclInternalError); } } } } return ncclSuccess; } ncclResult_t ncclPrepareTasksAndCollPreconnectFunc(struct ncclAsyncJob* job_) { struct ncclPrepareTasksAndCollPreconnectJob* job = (ncclPrepareTasksAndCollPreconnectJob*)job_; struct ncclComm* comm = job->comm; bool needConnect; bool algoNeedConnect[NCCL_NUM_ALGORITHMS]; memset(algoNeedConnect, 0, sizeof(bool)*NCCL_NUM_ALGORITHMS); CUDACHECK(cudaSetDevice(comm->cudaDev)); if (!job_->isThreadMain && ncclOsCpuCount(comm->cpuAffinity)) ncclOsSetAffinity(comm->cpuAffinity); NCCLCHECK(ncclPrepareTasks(comm, algoNeedConnect, &needConnect, job->simInfo)); if (comm->cuMemSupport && needConnect) NCCLCHECK(ncclCollPreconnect(comm, algoNeedConnect)); return ncclSuccess; } ncclResult_t ncclCollPreconnectFunc(struct ncclAsyncJob* job_) { struct ncclPreconnectJob* job = (struct ncclPreconnectJob*)job_; struct ncclComm* comm = job->comm; ncclResult_t ret = ncclSuccess; if (!job_->isThreadMain) CUDACHECK(cudaSetDevice(comm->cudaDev)); if (!job_->isThreadMain && ncclOsCpuCount(comm->cpuAffinity)) ncclOsSetAffinity(comm->cpuAffinity); NCCLCHECKGOTO(ncclCollPreconnect(comm, job->algoNeedConnect), ret, fail); exit: free(job->algoNeedConnect); return ret; fail: goto exit; } struct ncclGroupSymmetricJob { struct ncclAsyncJob base; struct ncclComm* comm; }; struct ncclGroupDebugJob { struct ncclAsyncJob base; struct ncclComm* comm; }; ncclResult_t ncclCommGroupArgsGlobalCheck(struct ncclAsyncJob* job_) { struct ncclGroupDebugJob* job = (struct ncclGroupDebugJob*)job_; struct ncclComm* comm = job->comm; ncclResult_t ret = ncclSuccess; while (!ncclIntruQueueEmpty(&comm->argsInfoQueue)) { struct ncclArgsInfo* argsInfo = ncclIntruQueueDequeue(&comm->argsInfoQueue); NCCLCHECKGOTO(ncclArgsGlobalCheck(argsInfo), ret, fail); free(argsInfo); } exit: return ret; fail: goto exit; } ncclResult_t ncclCommGroupRegisterSymmetric(struct ncclAsyncJob* job_) { struct ncclGroupSymmetricJob* job = (struct ncclGroupSymmetricJob*)job_; struct ncclComm* comm = job->comm; ncclResult_t ret = ncclSuccess; CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), ret, fail); while (!ncclIntruQueueEmpty(&comm->devrState.regTaskQueue)) { struct ncclDevrRegTask* task = ncclIntruQueueDequeue(&comm->devrState.regTaskQueue); NCCLCHECKGOTO(ncclDevrWindowRegisterInGroup( comm, task->userPtr, task->userSize, task->winFlags, task->outWinDev), ret, fail); free(task); } while (!ncclIntruQueueEmpty(&comm->devrState.commCreateTaskQueue)) { struct ncclDevrCommCreateTask* task = ncclIntruQueueDequeue(&comm->devrState.commCreateTaskQueue); NCCLCHECKGOTO(ncclDevrCommCreateInternal( comm, (struct ncclDevCommRequirements const*)task->reqs, task->outDevComm, /*isInternal=*/false, task->outDevCommCopyCB), ret, fail); freeDevCommRequirements(task->reqs); // free additional task memory for reqs free(task); } while (!ncclIntruQueueEmpty(&comm->ceInitTaskQueue)) { struct ncclCeInitTask* task = ncclIntruQueueDequeue(&comm->ceInitTaskQueue); NCCLCHECKGOTO(ncclCeInit(task->comm), ret, fail); free(task); } while (!ncclIntruQueueEmpty(&comm->rmaCeInitTaskQueue)) { struct ncclRmaCeInitTask* task = ncclIntruQueueDequeue(&comm->rmaCeInitTaskQueue); NCCLCHECKGOTO(ncclRmaCeInit(task->comm), ret, fail); free(task); } exit: return ret; fail: goto exit; } static ncclResult_t doLaunches(struct ncclComm* head) { ncclResult_t result = ncclSuccess; struct ncclComm* cliqueHead = head; struct ncclComm* cliqueNextHead; bool useBarrier = ncclParamLaunchMode == ncclLaunchModeGroup; // This outer loop iterates over cliques of comms which are siblings of the // same global entity. We calculate a clique as all comms which have the same // `intraComm0` value. do { struct ncclComm* comm = cliqueHead; bool capturingYes = false, capturingNo = false; do { (ncclCudaGraphValid(comm->planner.capturingGraph) ? capturingYes : capturingNo) = true; CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), result, failure); NCCLCHECKGOTO(ncclLaunchPrepare(comm), result, failure); if (useBarrier) ncclCommIntraBarrierIn(comm, 1); comm = comm->groupNext[ncclGroupTaskTypeCollective]; } while (comm != nullptr && comm->intraComm0 == cliqueHead->intraComm0); cliqueNextHead = comm; if (capturingYes && capturingNo) { // We have entered barriers but are aborting without leaving them. Thus // these comms are permanently trashed. We need a good mechanism for // tracking and reporting that. WARN("Either none or all communicators in a ncclGroup() can be CUDA graph captured."); result = ncclInvalidUsage; goto failure; } while (true) { // Iterate rounds of launches for clique. bool moreRounds = false; comm = cliqueHead; do { // Iterate clique members. struct ncclComm* next = comm->groupNext[ncclGroupTaskTypeCollective]; if (useBarrier) { // Barrier reduction result tells us if this was the final round. moreRounds = 0 != ncclCommIntraBarrierOut(comm); } else { moreRounds |= comm->planner.unlaunchedPlansHead != nullptr; } if (moreRounds) { // Pop next unlaunched kernel struct ncclKernelPlan* plan = comm->planner.unlaunchedPlansHead; if (plan != nullptr) { comm->planner.unlaunchedPlansHead = plan->next; CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), result, failure); NCCLCHECKGOTO(ncclLaunchKernelBefore_NoUncapturedCuda(comm, plan), result, failure); if (plan->isCeColl) { NCCLCHECKGOTO(ncclLaunchCeColl(comm, plan), result, failure); } else if (plan->isRma) { NCCLCHECKGOTO(ncclLaunchRma(comm, plan), result, failure); } else { NCCLCHECKGOTO(ncclLaunchKernel(comm, plan), result, failure); } } // Barrier reduction input indicates if we require further rounds. if (useBarrier) ncclCommIntraBarrierIn(comm, comm->planner.unlaunchedPlansHead != nullptr ? 1 : 0); if (plan != nullptr) { NCCLCHECKGOTO(ncclLaunchKernelAfter_NoCuda(comm, plan), result, failure); } } else { // Final round. CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), result, failure); NCCLCHECKGOTO(ncclLaunchFinish(comm), result, failure); } comm = next; } while (comm != cliqueNextHead); if (!moreRounds) break; } cliqueHead = cliqueNextHead; } while (cliqueHead != nullptr); failure: return result; } static inline void groupLocalResetJobState() { ncclGroupError = ncclSuccess; for (int type = 0; type < ncclGroupTaskTypeNum; ++type) ncclGroupCommHead[type] = NULL; ncclGroupCommPreconnectHead = NULL; ncclGroupBlocking = -1; ncclIntruQueueConstruct(&ncclAsyncJobs); return; } static void groupCleanup(struct ncclComm** groupCommHeadPtr, struct ncclIntruQueue* asyncJobsPtr, ncclResult_t error) { struct ncclComm* comm; for (int type = 0; type < ncclGroupTaskTypeNum; ++type) { comm = groupCommHeadPtr[type]; // reset groupCommHeadPtr[type] groupCommHeadPtr[type] = nullptr; while (comm != nullptr) { struct ncclComm* next = comm->groupNext[type]; (void)ncclGroupCommLeave(comm, type); // overwrites comm->groupNext // We don't know if preconnect succeeded or happened at all, so clear // the flags that let `taskAppend()` skip over checking if preconnect // is needed. if (type == ncclGroupTaskTypeCollective) { comm->preconnectNext = reinterpret_cast(0x1); for (int i = 0; i < comm->nRanks; i++) { comm->connectSend[i] = 0UL; comm->connectRecv[i] = 0UL; } // Reclaim abandoned kernel plan memory. Note ncclWork structs were already // reclaimed by a `ncclMemoryStackPop(&comm->memScoped)` during `ncclGroupCommLeave()`. while (!ncclIntruQueueEmpty(&comm->planner.planQueue)) { struct ncclKernelPlan* plan = ncclIntruQueueDequeue(&comm->planner.planQueue); // Persistent plans will be reclaimed via the callbackQueue when the // graph drops its UserObject reference. if (!plan->persistent) { while (!ncclIntruQueueEmpty(&plan->proxyOpQueue)) { struct ncclProxyOp* pxop = ncclIntruQueueDequeue(&plan->proxyOpQueue); ncclMemoryPoolFree(&comm->memPool_ncclProxyOp, pxop); } ncclMemoryPoolFree(&comm->memPool_ncclKernelPlan, plan); } } { // Reset comm->planner to empty. ncclKernelPlanner::Peer* tmp = comm->planner.peers; ncclIntruQueue* tmpRmaQueues = comm->planner.rmaTaskQueues; int numRmaCtx = comm->config.numRmaCtx; memset(&comm->planner, 0, sizeof(comm->planner)); comm->planner.peers = tmp; if (comm->planner.peers != NULL) memset(comm->planner.peers, 0, comm->nRanks * sizeof(comm->planner.peers[0])); comm->planner.bcast_info.minBcastPeer = INT_MAX; comm->planner.bcast_info.maxBcastPeer = INT_MIN; comm->planner.rmaTaskQueues = tmpRmaQueues; if (comm->planner.rmaTaskQueues != NULL) { for (int i = 0; i < numRmaCtx; i++) { ncclIntruQueueConstruct(&comm->planner.rmaTaskQueues[i]); } } } } if (!comm->config.blocking) (void)ncclCommSetAsyncError(comm, error); comm = next; } } /* reset everything */ while (!ncclIntruQueueEmpty(asyncJobsPtr)) { struct ncclAsyncJob* job = ncclIntruQueueDequeue(asyncJobsPtr); if (!job->destroyFlag && job->comm && !job->comm->config.blocking) (void) ncclCommSetAsyncError(job->comm, error); if (job->undo) job->undo(job); if (job->destructor) job->destructor((void*)job); } return; } static ncclResult_t asyncJobLaunch(struct ncclIntruQueue *asyncJobsMain, volatile bool *groupAbortFlag) { ncclResult_t ret = ncclSuccess; bool jobsDone = false; bool errorJobAbortFlag = false; if (!ncclIntruQueueEmpty(asyncJobsMain)) { struct ncclAsyncJob* job = ncclIntruQueueHead(asyncJobsMain); if (job->next == nullptr) { job->isThreadMain = true; ncclAsyncJobMain(job); job->state = ncclGroupJobJoined; return job->result; } do { STDTHREADCREATE(job->thread, ncclAsyncJobMain, job); job = job->next; } while (job != nullptr); do { jobsDone = true; job = ncclIntruQueueHead(asyncJobsMain); do { ncclGroupJobState_t state = COMPILER_ATOMIC_LOAD(&job->state, std::memory_order_acquire); if (state == ncclGroupJobRunning) { jobsDone = false; } else if (state == ncclGroupJobDone) { int err; if ((err = ncclThreadJoin(job->thread)) != ncclSuccess) { WARN("asyncJobLaunch: failed to join thread for job"); ret = ncclSystemError; } job->state = ncclGroupJobJoined; if (job->result != ncclSuccess && ret == ncclSuccess) { ret = job->result; errorJobAbortFlag = true; } } else { /* safety check */ assert(state == ncclGroupJobJoined); } if (!job->destroyFlag && (COMPILER_ATOMIC_LOAD(groupAbortFlag, std::memory_order_acquire) || errorJobAbortFlag == true)) { COMPILER_ATOMIC_STORE(job->abortFlag, 1, std::memory_order_release); COMPILER_ATOMIC_STORE(job->abortFlagDev, 1, std::memory_order_release); if (job->childAbortFlag) { COMPILER_ATOMIC_STORE(job->childAbortFlag, 1, std::memory_order_release); COMPILER_ATOMIC_STORE(job->childAbortFlagDev, 1, std::memory_order_release); } } job = job->next; } while (job != nullptr); // Let preconnect threads progress. if (jobsDone == false) usleep(1); } while (jobsDone == false); if (ret != ncclSuccess) goto fail; } exit: return ret; fail: goto exit; } NCCL_PARAM(SingleProcMemRegEnable, "SINGLE_PROC_MEM_REG_ENABLE", 0); static void ncclPrepareTasksAndCollPreconnectJobFree(void* _job) { struct ncclPrepareTasksAndCollPreconnectJob* job = (struct ncclPrepareTasksAndCollPreconnectJob*)_job; delete job; } static void ncclPreconnectJobFree(void* _job) { struct ncclPreconnectJob* job = (struct ncclPreconnectJob*)_job; delete job; } static void ncclGroupSymmetricJobFree(void* _job) { struct ncclGroupSymmetricJob* job = (struct ncclGroupSymmetricJob*)_job; delete job; } static ncclResult_t ncclPrepareTasksAndCollPreconnect(struct ncclComm* comm, ncclSimInfo_t* simInfo, struct ncclIntruQueue* asyncCollJobs) { if (ncclParamSingleProcMemRegEnable()) { struct ncclPrepareTasksAndCollPreconnectJob* job; NEW_NOTHROW(job, ncclPrepareTasksAndCollPreconnectJob); job->base.func = ncclPrepareTasksAndCollPreconnectFunc; job->base.undo = nullptr; job->base.destructor = ncclPrepareTasksAndCollPreconnectJobFree; job->base.state = ncclGroupJobRunning; job->base.abortFlag = comm->abortFlag; job->base.abortFlagDev = comm->abortFlagDev; job->comm = comm; job->simInfo = simInfo; ncclIntruQueueEnqueue(asyncCollJobs, &job->base); } else { bool needConnect = false; bool algoNeedConnect[NCCL_NUM_ALGORITHMS]; memset(algoNeedConnect, 0, sizeof(bool) * NCCL_NUM_ALGORITHMS); CUDACHECK(cudaSetDevice(comm->cudaDev)); NCCLCHECK(ncclPrepareTasks(comm, algoNeedConnect, &needConnect, simInfo)); if (comm->cuMemSupport && needConnect) { ncclResult_t ret; struct ncclPreconnectJob* job; NEW_NOTHROW(job, ncclPreconnectJob); job->base.func = ncclCollPreconnectFunc; job->base.undo = nullptr; job->base.destructor = ncclPreconnectJobFree; job->base.state = ncclGroupJobRunning; job->base.abortFlag = comm->abortFlag; job->base.abortFlagDev = comm->abortFlagDev; job->comm = comm; if ((ret = ncclCalloc(&job->algoNeedConnect, NCCL_NUM_ALGORITHMS))) { delete job; NCCLCHECK(ret); } memcpy(job->algoNeedConnect, algoNeedConnect, sizeof(bool) * NCCL_NUM_ALGORITHMS); ncclIntruQueueEnqueue(asyncCollJobs, &job->base); } } return ncclSuccess; } static ncclResult_t groupLaunch(struct ncclAsyncJob *job_, ncclSimInfo_t* simInfo = NULL) { ncclResult_t ret = ncclSuccess; struct ncclGroupJob *gjob = (struct ncclGroupJob*) job_; struct ncclComm **groupCommHeadMain = gjob->groupCommHead; struct ncclComm *groupCommPreconnectHeadMain = gjob->groupCommPreconnectHead; struct ncclIntruQueue *asyncJobsMain = &gjob->asyncJobs; bool *groupAbortFlag = &gjob->abortFlag; if (!simInfo && groupCommPreconnectHeadMain != nullptr) { struct ncclComm* comm = groupCommPreconnectHeadMain; do { struct ncclPreconnectJob* job; NEW_NOTHROW_GOTO(job, ncclPreconnectJob, ret, fail); job->base.func = ncclP2PPreconnectFunc; job->base.undo = nullptr; job->base.destructor = ncclPreconnectJobFree; job->base.state = ncclGroupJobRunning; job->base.abortFlag = comm->abortFlag; job->base.abortFlagDev = comm->abortFlagDev; job->comm = comm; ncclIntruQueueEnqueue(asyncJobsMain, (struct ncclAsyncJob*)job); struct ncclComm* next = comm->preconnectNext; comm->preconnectNext = reinterpret_cast(0x1); comm = next; } while (comm != nullptr); } NCCLCHECKGOTO(asyncJobLaunch(asyncJobsMain, groupAbortFlag), ret, fail); // only loop through sym alloc and register tasks for (int type = ncclGroupTaskTypeSymRegister; type <= ncclGroupTaskTypeSymRegister; ++type) { if (groupCommHeadMain[type]) { struct ncclComm* cliqueHead = groupCommHeadMain[type]; struct ncclComm* comm = NULL; struct ncclIntruQueue asyncSymJobs; ncclIntruQueueConstruct(&asyncSymJobs); do { comm = cliqueHead; do { struct ncclGroupSymmetricJob* job; NEW_NOTHROW_GOTO(job, ncclGroupSymmetricJob, ret, fail); job->base.func = ncclCommGroupRegisterSymmetric; job->base.undo = nullptr; job->base.destructor = ncclGroupSymmetricJobFree; job->base.state = ncclGroupJobRunning; job->base.abortFlag = comm->abortFlag; job->base.abortFlagDev = comm->abortFlagDev; job->comm = comm; ncclIntruQueueEnqueue(&asyncSymJobs, (struct ncclAsyncJob*)job); comm = comm->groupNext[type]; } while (comm != nullptr && comm->intraComm0 == cliqueHead->intraComm0); NCCLCHECKGOTO(asyncJobLaunch(&asyncSymJobs, groupAbortFlag), ret, fail); while (!ncclIntruQueueEmpty(&asyncSymJobs)) { struct ncclAsyncJob* job = ncclIntruQueueDequeue(&asyncSymJobs); if (job->destructor) job->destructor((void*)job); } cliqueHead = comm; } while (cliqueHead != nullptr); } } /* Connect channels at runtime if cumem is supported */ if (groupCommHeadMain[ncclGroupTaskTypeCollective] != nullptr) { struct ncclComm* cliqueHead = groupCommHeadMain[ncclGroupTaskTypeCollective]; struct ncclComm* comm = NULL; struct ncclIntruQueue asyncCollJobs; struct ncclIntruQueue asyncDebugJobs; ncclIntruQueueConstruct(&asyncCollJobs); ncclIntruQueueConstruct(&asyncDebugJobs); do { // We need to preconnect connections for collectives clique by clique to avoid // race condition for split shared comms which can connect the same connections // at the same time. comm = cliqueHead; do { NCCLCHECKGOTO(ncclPrepareTasksAndCollPreconnect(comm, simInfo, &asyncCollJobs), ret, fail); comm = comm->groupNext[ncclGroupTaskTypeCollective]; } while (comm != nullptr && comm->intraComm0 == cliqueHead->intraComm0); // connect NCCLCHECKGOTO(asyncJobLaunch(&asyncCollJobs, groupAbortFlag), ret, fail); while (!ncclIntruQueueEmpty(&asyncCollJobs)) { struct ncclAsyncJob* job = ncclIntruQueueDequeue(&asyncCollJobs); if (job->destructor) job->destructor((void*)job); } cliqueHead = comm; } while (cliqueHead != nullptr); // done with all buffer allocation, start registration and enqueue comm = groupCommHeadMain[ncclGroupTaskTypeCollective]; do { CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), ret, fail); NCCLCHECKGOTO(ncclTasksRegAndEnqueue(comm), ret, fail); comm = comm->groupNext[ncclGroupTaskTypeCollective]; } while (comm); // debug check cliqueHead = groupCommHeadMain[ncclGroupTaskTypeCollective]; do { comm = cliqueHead; do { if (comm->checkMode == ncclCheckModeDebugGlobal) { struct ncclGroupSymmetricJob* job; NCCLCHECK(ncclCalloc(&job, 1)); job->base.func = ncclCommGroupArgsGlobalCheck; job->base.undo = nullptr; job->base.destructor = free; job->base.state = ncclGroupJobRunning; job->base.abortFlag = comm->abortFlag; job->base.abortFlagDev = comm->abortFlagDev; job->comm = comm; ncclIntruQueueEnqueue(&asyncDebugJobs, (struct ncclAsyncJob*)job); } comm = comm->groupNext[ncclGroupTaskTypeCollective]; } while (comm != nullptr && comm->intraComm0 == cliqueHead->intraComm0); NCCLCHECKGOTO(asyncJobLaunch(&asyncDebugJobs, groupAbortFlag), ret, fail); while (!ncclIntruQueueEmpty(&asyncDebugJobs)) { struct ncclAsyncJob* job = ncclIntruQueueDequeue(&asyncDebugJobs); if (job->destructor) job->destructor((void*)job); } cliqueHead = comm; } while (cliqueHead != nullptr); } if ((!simInfo) && (groupCommHeadMain[ncclGroupTaskTypeCollective] != nullptr)) { NCCLCHECKGOTO(doLaunches(groupCommHeadMain[ncclGroupTaskTypeCollective]), ret, fail); } while (!ncclIntruQueueEmpty(asyncJobsMain)) { struct ncclAsyncJob* job = ncclIntruQueueDequeue(asyncJobsMain); if (!job->destroyFlag && job->comm && !job->comm->config.blocking && groupCommHeadMain[ncclGroupTaskTypeCollective] == nullptr) (void) ncclCommSetAsyncError(job->comm, ret); if (job->destructor) job->destructor((void*)job); } for (int type = 0; type < ncclGroupTaskTypeNum; ++type) { while (groupCommHeadMain[type] != nullptr) { struct ncclComm* comm = groupCommHeadMain[type]; struct ncclComm* next = comm->groupNext[type]; // Poll for callbacks sent to us from other threads. Typically these free // resources from to our memory pools and UB if (comm->reclaimSteps == GROUP_MAX_RECLAIM_STEPS) { NCCLCHECKGOTO(ncclCommPollCallbacks(comm, /*waitSome=*/false), ret, fail); comm->reclaimSteps = 0; } else { comm->reclaimSteps++; } (void)ncclGroupCommLeave(comm, type); if (!comm->config.blocking) { (void)ncclCommSetAsyncError(comm, ret); } groupCommHeadMain[type] = next; } } exit: return ret; fail: groupCleanup(gjob->groupCommHead, &gjob->asyncJobs, ret); goto exit; } static ncclResult_t groupLaunchNonBlocking(struct ncclAsyncJob *job_) { return groupLaunch(job_ /* estimatedTime = NULL */); } ncclResult_t ncclGroupEndInternal(ncclSimInfo_t* simInfo) { ncclResult_t ret = ncclSuccess; ncclSimInfo_t internalSimInfo = NCCL_SIM_INFO_INITIALIZER; ncclSimInfo_t* internalSimInfoPtr = NULL; size_t realSize = 0; bool hasCommHead = false; ncclGroupJob* groupJob = NULL; internalSimInfo.magic = 0; if (ncclGroupDepth == 0) { WARN("ncclGroupEnd: not in a group call."); ret = ncclInvalidUsage; goto exit; } if (ncclProfilerApiState.profilerGroupDepth > 0) { ncclProfilerApiState.profilerGroupDepth--; } if (ncclProfilerApiState.profilerGroupDepth == 0) { NCCLCHECK(ncclProfilerRecordGroupApiEventState(ncclProfilerGroupEndApiStart)); } if ((--ncclGroupDepth) > 0) goto exit; if ((ret = ncclGroupError) != ncclSuccess) goto fail; if (simInfo) { memcpy((void*)&realSize, (void*)&simInfo->size, sizeof(size_t)); realSize = realSize > sizeof(ncclSimInfo_t) ? sizeof(ncclSimInfo_t) : realSize; memcpy((void*)&internalSimInfo, (void*)simInfo, realSize); if (internalSimInfo.magic != 0x74685283) { WARN("ncclSimInfo_t argument not initialized via NCCL_SIM_INFO_INITIALIZER"); ret = ncclInvalidArgument; goto fail; } internalSimInfoPtr = &internalSimInfo; } for (int type = 0; type < ncclGroupTaskTypeNum; ++type) { if (ncclGroupCommHead[type]) { hasCommHead = true; break; } } NEW_NOTHROW_GOTO(groupJob, ncclGroupJob, ret, fail); ncclIntruQueueConstruct(&groupJob->asyncJobs); groupJob->groupRefCount = 0; groupJob->nonBlockingInit = false; memcpy(groupJob->groupCommHead, ncclGroupCommHead, sizeof(ncclGroupCommHead)); groupJob->groupCommPreconnectHead = ncclGroupCommPreconnectHead; groupJob->groupError = ncclSuccess; groupJob->abortFlag = false; groupJob->joined = false; ncclIntruQueueTransfer(&groupJob->asyncJobs, &ncclAsyncJobs); if (hasCommHead || !ncclIntruQueueEmpty(&groupJob->asyncJobs) || ncclGroupCommPreconnectHead != nullptr) { /* make sure ncclGroupBlocking has been set. */ assert(ncclGroupBlocking == 0 || ncclGroupBlocking == 1); if (ncclGroupBlocking == 0) { /* nonblocking group */ if (!ncclIntruQueueEmpty(&groupJob->asyncJobs)) { ncclAsyncJob* job = ncclIntruQueueHead(&groupJob->asyncJobs); do { NCCLCHECKGOTO(ncclCommSetAsyncError(job->comm, ncclInProgress), ret, fail); if (job->comm->groupJob == NULL) { job->comm->groupJob = groupJob; groupJob->groupRefCount++; } job = job->next; } while (job); } for (int type = 0; type < ncclGroupTaskTypeNum; ++type) { if (ncclGroupCommHead[type]) { ncclComm_t comm = ncclGroupCommHead[type]; do { NCCLCHECKGOTO(ncclCommSetAsyncError(comm, ncclInProgress), ret, fail); /* link group job to communicators. */ if (comm->groupJob == NULL) { comm->groupJob = groupJob; groupJob->groupRefCount++; } comm = comm->groupNext[type]; } while (comm); } } groupJob->base.func = groupLaunchNonBlocking; STDTHREADCREATE_GOTO(groupJob->base.thread, ncclAsyncJobMain, ret, fail, &groupJob->base); groupJob->nonBlockingInit = true; ret = ncclInProgress; } else { /* blocking group */ int savedDev; CUDACHECKGOTO(cudaGetDevice(&savedDev), ret, fail); NCCLCHECKGOTO(groupLaunch(&groupJob->base, internalSimInfoPtr), ret, fail); CUDACHECKGOTO(cudaSetDevice(savedDev), ret, fail); if (simInfo) memcpy((void*)simInfo, (void*)internalSimInfoPtr, realSize); delete groupJob; } } else { // Free when not needed (single rank case) delete groupJob; } /* Reset the job state for the next group call. */ groupLocalResetJobState(); exit: // Profiler group API start is called inside taskAppend to get graph capture information for the event NCCLCHECK(ncclProfilerStopGroupApiEvent()); return ret; fail: if (groupJob) { groupCleanup(groupJob->groupCommHead, &groupJob->asyncJobs, ret); delete groupJob; } else { groupCleanup(ncclGroupCommHead, &ncclAsyncJobs, ret); } groupLocalResetJobState(); goto exit; } ncclResult_t ncclGroupJobComplete(struct ncclGroupJob* groupJob) { ncclResult_t ret = ncclSuccess; if (groupJob && groupJob->nonBlockingInit) { if (!COMPILER_ATOMIC_EXCHANGE(&groupJob->joined, true, std::memory_order_acq_rel)) { ret = ncclAsyncJobComplete(&groupJob->base); } if (ncclAtomicRefCountDecrement(&groupJob->groupRefCount) == 0) { delete groupJob; } } return ret; } ncclResult_t ncclGroupJobAbort(struct ncclGroupJob* groupJob) { if (groupJob && groupJob->nonBlockingInit) { if (!COMPILER_ATOMIC_EXCHANGE(&groupJob->joined, true, std::memory_order_acq_rel)) { COMPILER_ATOMIC_STORE(&groupJob->abortFlag, true, std::memory_order_relaxed); ncclAsyncJobComplete(&groupJob->base); } if (ncclAtomicRefCountDecrement(&groupJob->groupRefCount) == 0) { delete groupJob; } } return ncclSuccess; } nccl-2.29.7-1/src/include/000077500000000000000000000000001515037102200150765ustar00rootroot00000000000000nccl-2.29.7-1/src/include/alloc.h000066400000000000000000000525341515037102200163520ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_ALLOC_H_ #define NCCL_ALLOC_H_ #include "nccl.h" #include "checks.h" #include "bitops.h" #include "utils.h" #include "p2p.h" #include "mem_manager.h" struct ncclComm; #include "os.h" #include #include #include #include #if CUDART_VERSION >= 11030 #include #include "cudawrap.h" #endif uint64_t clockNano(); // from utils.h with which we have a circular dependency template constexpr size_t ncclSizeOfT() { return sizeof(T); } template<> constexpr size_t ncclSizeOfT() { return 1; } // C++14-compatible wrapper that captures function pointers through template parameters. template struct ncclDeleterWrapper { template constexpr auto operator()(Args &&...args) const { return Function(std::forward(args)...); } }; // struct ncclDeleterWrapper using ncclDeleterFree = ncclDeleterWrapper; template using ncclUniquePtr = std::unique_ptr; template using ncclUniqueArrayPtr = std::unique_ptr; #if CUDART_VERSION >= 12020 static inline ncclResult_t ncclCuMemHostAlloc(void** ptr, CUmemGenericAllocationHandle *handlep, size_t size) { ncclResult_t result = ncclSuccess; size_t granularity = 0; CUdevice currentDev; CUmemAllocationProp prop = {}; CUmemAccessDesc accessDesc = {}; CUmemGenericAllocationHandle handle; int cudaDev; int cpuNumaNodeId = -1; CUmemAllocationHandleType type = ncclCuMemHandleType; CUDACHECK(cudaGetDevice(&cudaDev)); CUCHECK(cuDeviceGet(¤tDev, cudaDev)); CUCHECK(cuDeviceGetAttribute(&cpuNumaNodeId, CU_DEVICE_ATTRIBUTE_HOST_NUMA_ID, currentDev)); if (cpuNumaNodeId < 0) cpuNumaNodeId = 0; prop.location.type = CU_MEM_LOCATION_TYPE_HOST_NUMA; prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; prop.requestedHandleTypes = type; // So it can be exported prop.location.id = cpuNumaNodeId; CUCHECK(cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM)); ALIGN_SIZE(size, granularity); /* Allocate the physical memory on the device */ CUCHECK(cuMemCreate(&handle, size, &prop, 0)); /* Reserve a virtual address range */ CUCHECK(cuMemAddressReserve((CUdeviceptr*)ptr, size, granularity, 0, 0)); /* Map the virtual address range to the physical allocation */ CUCHECK(cuMemMap((CUdeviceptr)*ptr, size, 0, handle, 0)); /* Now allow RW access to the newly mapped memory for local GPU */ accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; accessDesc.location.id = cudaDev; accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; CUCHECK(cuMemSetAccess((CUdeviceptr)*ptr, size, &accessDesc, 1)); /* Now allow RW access to the newly mapped memory from the CPU */ accessDesc.location.type = CU_MEM_LOCATION_TYPE_HOST_NUMA; accessDesc.location.id = cpuNumaNodeId; accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; CUCHECK(cuMemSetAccess((CUdeviceptr)*ptr, size, &accessDesc, 1)); if (handlep) *handlep = handle; INFO(NCCL_ALLOC, "CUMEM Host Alloc Size %zi pointer %p handle %llx numa %d dev %d granularity %ld", size, *ptr, handle, cpuNumaNodeId, cudaDev, granularity); return result; } static inline ncclResult_t ncclCuMemHostFree(void* ptr) { if (ptr == NULL) return ncclSuccess; ncclResult_t result = ncclSuccess; CUmemGenericAllocationHandle handle; size_t size = 0; CUCHECK(cuMemRetainAllocationHandle(&handle, ptr)); CUCHECK(cuMemRelease(handle)); CUCHECK(cuMemGetAddressRange(NULL, &size, (CUdeviceptr)ptr)); TRACE(NCCL_ALLOC, "CUMEM Host Free Size %zi pointer %p handle 0x%llx", size, ptr, handle); CUCHECK(cuMemUnmap((CUdeviceptr)ptr, size)); CUCHECK(cuMemRelease(handle)); CUCHECK(cuMemAddressFree((CUdeviceptr)ptr, size)); return result; } #else /* CUDART_VERSION >= 12020 */ static inline ncclResult_t ncclCuMemHostAlloc(void** ptr, void* handlep, size_t size) { WARN("CUMEM Host is not supported prior to CUDA 12.2"); return ncclInternalError; } static inline ncclResult_t ncclCuMemHostFree(void* ptr) { WARN("CUMEM Host is not supported prior to CUDA 12.2"); return ncclInternalError; } #endif /* CUDART_VERSION >= 12020 */ template ncclResult_t ncclCudaHostCallocDebug(T** ptr, size_t nelem, const char *filefunc, int line) { ncclResult_t result = ncclSuccess; cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; *ptr = nullptr; CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); if (nelem > 0) { CUDACHECKGOTO(cudaHostAlloc(ptr, nelem*ncclSizeOfT(), cudaHostAllocMapped), result, finish); memset(*ptr, 0, nelem*ncclSizeOfT()); } finish: CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); if (*ptr == nullptr && nelem > 0) WARN("Failed to CUDA host alloc %ld bytes", nelem*ncclSizeOfT()); INFO(NCCL_ALLOC, "%s:%d Cuda Host Alloc Size %ld pointer %p", filefunc, line, nelem*ncclSizeOfT(), *ptr); return result; } static inline ncclResult_t ncclCudaHostFree(void* ptr) { CUDACHECK(cudaFreeHost(ptr)); return ncclSuccess; } #define ncclCudaHostCalloc(...) ncclCudaHostCallocDebug(__VA_ARGS__, __FILE__, __LINE__) template ncclResult_t ncclCallocDebug(T** ptr, size_t nelem, const char *filefunc, int line) { if (nelem > 0) { T* p = (T*)malloc(nelem*ncclSizeOfT()); if (p == NULL) { WARN("Failed to malloc %ld bytes", nelem*ncclSizeOfT()); return ncclSystemError; } //INFO(NCCL_ALLOC, "%s:%d malloc Size %ld pointer %p", filefunc, line, nelem*ncclSizeOfT(), p); memset((void*)p, 0, nelem*ncclSizeOfT()); *ptr = p; } else { *ptr = NULL; } return ncclSuccess; } template ncclResult_t ncclCallocDebug(ncclUniquePtr& ptr, size_t nelem, const char *filefunc, int line) { typename ncclUniquePtr::pointer p = nullptr; ncclResult_t result = ncclCallocDebug(&p, nelem, filefunc, line); ptr.reset(p); return result; } template ncclResult_t ncclCallocDebug(ncclUniqueArrayPtr& ptr, size_t nelem, const char *filefunc, int line) { typename ncclUniqueArrayPtr::pointer p = nullptr; ncclResult_t result = ncclCallocDebug(&p, nelem, filefunc, line); ptr.reset(p); return result; } #define ncclCalloc(...) ncclCallocDebug(__VA_ARGS__, __FILE__, __LINE__) template ncclResult_t ncclRealloc(T** ptr, size_t oldNelem, size_t nelem) { T* oldp = *ptr; if (nelem < oldNelem || (oldp == NULL && oldNelem > 0)) return ncclInternalError; if (nelem == oldNelem) return ncclSuccess; T* p = (T*)malloc(nelem*ncclSizeOfT()); if (p == NULL) { WARN("Failed to malloc %ld bytes", nelem*ncclSizeOfT()); return ncclSystemError; } if (oldp && oldNelem) memcpy(p, oldp, oldNelem * ncclSizeOfT()); if (oldp) free(oldp); memset(p+oldNelem, 0, (nelem-oldNelem)*ncclSizeOfT()); *ptr = (T*)p; INFO(NCCL_ALLOC, "Mem Realloc old size %ld, new size %ld pointer %p", oldNelem*ncclSizeOfT(), nelem*ncclSizeOfT(), *ptr); return ncclSuccess; } #if CUDART_VERSION >= 11030 #include #include "cudawrap.h" // Helper function to map memory and set access permissions for a device static inline ncclResult_t ncclCuMemMapAndSetAccess(void *ptr, size_t size, CUmemGenericAllocationHandle handle, int cudaDev) { ncclResult_t result = ncclSuccess; // Map the virtual address range to the physical allocation CUCHECK(cuMemMap((CUdeviceptr)ptr, size, 0, handle, 0)); // Set access permissions for the device CUmemAccessDesc accessDesc = {}; accessDesc.location.type = CU_MEM_LOCATION_TYPE_DEVICE; accessDesc.location.id = cudaDev; accessDesc.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE; CUCHECK(cuMemSetAccess((CUdeviceptr)ptr, size, &accessDesc, 1)); return result; } // ncclCuMemAllocAddr takes memory handle and size and returns the mapped address pointer static inline ncclResult_t ncclCuMemAllocAddr(void **ptr, CUmemGenericAllocationHandle *handleIn, size_t size) { ncclResult_t result = ncclSuccess; size_t granularity = 0; CUmemAllocationProp prop = {}; int cudaDev; CUDACHECK(cudaGetDevice(&cudaDev)); CUCHECK(cuMemGetAllocationPropertiesFromHandle(&prop, *handleIn)); CUCHECK(cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM)); ALIGN_SIZE(size, granularity); /* Reserve a virtual address range */ CUCHECK(cuMemAddressReserve((CUdeviceptr *)ptr, size, granularity, 0, 0)); /* Map the virtual address range to the physical allocation and set access permissions */ NCCLCHECK(ncclCuMemMapAndSetAccess(*ptr, size, *handleIn, cudaDev)); TRACE(NCCL_ALLOC, "CuMem Map Size %zu pointer %p handle %llx", size, *ptr, *handleIn); return result; } static inline ncclResult_t ncclCuMemFreeAddr(void *ptr, int numSegments = 1) { if (ptr == NULL) return ncclSuccess; ncclResult_t result = ncclSuccess; size_t totalSize = 0; for (int segment = 0; segment < numSegments; segment++) { size_t segmentSize = 0; CUCHECK(cuMemGetAddressRange(NULL, &segmentSize, (CUdeviceptr)ptr + totalSize)); CUCHECK(cuMemUnmap((CUdeviceptr)ptr + totalSize, segmentSize)); totalSize += segmentSize; } CUCHECK(cuMemAddressFree((CUdeviceptr)ptr, totalSize)); return result; } static inline ncclResult_t ncclCuMemAlloc(void **ptr, CUmemGenericAllocationHandle *handlep, CUmemAllocationHandleType type, size_t size, struct ncclMemManager* manager, ncclMemType_t memType = ncclMemPersist) { ncclResult_t result = ncclSuccess; size_t granularity = 0; CUdevice currentDev; CUmemAllocationProp prop = {}; CUmemGenericAllocationHandle handle; int cudaDev; int flag = 0; CUDACHECK(cudaGetDevice(&cudaDev)); CUCHECK(cuDeviceGet(¤tDev, cudaDev)); prop.type = CU_MEM_ALLOCATION_TYPE_PINNED; prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE; prop.requestedHandleTypes = type; prop.location.id = currentDev; // Query device to see if RDMA support is available CUCHECK(cuDeviceGetAttribute(&flag, CU_DEVICE_ATTRIBUTE_GPU_DIRECT_RDMA_WITH_CUDA_VMM_SUPPORTED, currentDev)); if (flag) prop.allocFlags.gpuDirectRDMACapable = 1; CUCHECK(cuMemGetAllocationGranularity(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM)); ALIGN_SIZE(size, granularity); /* Allocate the physical memory on the device */ CUCHECK(cuMemCreate(&handle, size, &prop, 0)); /* Reserve a virtual address range */ CUCHECK(cuMemAddressReserve((CUdeviceptr *)ptr, size, granularity, 0, 0)); /* Map the virtual address range to the physical allocation and set access permissions */ NCCLCHECK(ncclCuMemMapAndSetAccess(*ptr, size, handle, currentDev)); if (handlep) *handlep = handle; TRACE(NCCL_ALLOC, "CuMem Alloc Size %zu pointer %p handle %llx", size, *ptr, handle); /* Track allocation in memory manager */ if (manager != nullptr) { ncclMemTrack(manager, *ptr, size, handle, type, memType); } return result; } static inline ncclResult_t ncclCuMemFree(void *ptr, struct ncclMemManager* manager, int numSegments = 1) { if (ptr == NULL) return ncclSuccess; ncclResult_t result = ncclSuccess; size_t totalSize = 0; for (int segment = 0; segment < numSegments; segment++) { CUmemGenericAllocationHandle handle; size_t segmentSize = 0; CUCHECK(cuMemRetainAllocationHandle(&handle, (void*) ((char *) ptr + totalSize))); CUCHECK(cuMemRelease(handle)); CUCHECK(cuMemGetAddressRange(NULL, &segmentSize, (CUdeviceptr)ptr + totalSize)); TRACE(NCCL_ALLOC, "CuMem Free Size %zu pointer %p handle 0x%llx segment %d numSegments %d", segmentSize, ptr, handle, segment, numSegments); CUCHECK(cuMemUnmap((CUdeviceptr)ptr + totalSize, segmentSize)); CUCHECK(cuMemRelease(handle)); totalSize += segmentSize; } // Update tracking with total size after processing all segments if (manager != nullptr) { NCCLCHECK(ncclMemUntrack(manager, ptr, totalSize)); } CUCHECK(cuMemAddressFree((CUdeviceptr)ptr, totalSize)); return result; } // Get the base and size of all segments that span a given user buffer static inline ncclResult_t ncclCuMemGetAddressRange(CUdeviceptr userBuff, size_t userBuffSize, CUdeviceptr* mappedPtrBase, size_t* totalMappedBufferSize, int* numSegments) { *totalMappedBufferSize = 0; *mappedPtrBase = 0; if (numSegments) *numSegments = 0; CUdeviceptr userBuffStart = userBuff; CUdeviceptr userBuffEnd = userBuffStart + userBuffSize; CUdeviceptr mappedPtrEnd = userBuffStart; CUdeviceptr baseSend; size_t baseSendSize; while (mappedPtrEnd < userBuffEnd) { CUCHECK(cuMemGetAddressRange(&baseSend, &baseSendSize, mappedPtrEnd)); if (*totalMappedBufferSize == 0) { *mappedPtrBase = baseSend; } *totalMappedBufferSize += baseSendSize; mappedPtrEnd = baseSend + baseSendSize; if (numSegments) *numSegments = *numSegments + 1; } return ncclSuccess; } #else extern int ncclCuMemEnable(); static inline ncclResult_t ncclCuMemAlloc(void **ptr, void *handlep, int type, size_t size, struct ncclMemManager* manager, ncclMemType_t memType = ncclMemScratch) { WARN("CUMEM not supported prior to CUDA 11.3"); return ncclInternalError; } static inline ncclResult_t ncclCuMemFree(void *ptr, struct ncclMemManager* manager, int numSegments = 1) { WARN("CUMEM not supported prior to CUDA 11.3"); return ncclInternalError; } static inline ncclResult_t ncclCuMemAllocAddr(void **ptr, CUmemGenericAllocationHandle *handleIn, size_t size) { WARN("CUMEM not supported prior to CUDA 11.3"); return ncclInternalError; } static inline ncclResult_t ncclCuMemFreeAddr(void *ptr, int numSegments = 1) { WARN("CUMEM not supported prior to CUDA 11.3"); return ncclInternalError; } static inline ncclResult_t ncclCuMemGetAddressRange(CUdeviceptr userBuff, size_t userBuffSize, CUdeviceptr* mappedPtrBase, size_t* totalMappedBufferSize, int* numSegments) { WARN("CUMEM not supported prior to CUDA 11.3"); return ncclInternalError; } #endif template ncclResult_t ncclCudaMallocDebug(T** ptr, size_t nelem, const char *filefunc, int line, struct ncclMemManager* manager, ncclMemType_t memType = ncclMemPersist) { ncclResult_t result = ncclSuccess; cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; *ptr = nullptr; CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); if (nelem > 0) { if (ncclCuMemEnable()) { NCCLCHECKGOTO(ncclCuMemAlloc((void **)ptr, NULL, ncclCuMemHandleType, nelem*ncclSizeOfT(), manager, memType), result, finish); } else { CUDACHECKGOTO(cudaMalloc(ptr, nelem*ncclSizeOfT()), result, finish); } } finish: CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); if (*ptr == nullptr && nelem > 0) WARN("Failed to CUDA malloc %ld bytes", nelem*ncclSizeOfT()); INFO(NCCL_ALLOC, "%s:%d Cuda Alloc Size %ld pointer %p memType %d", filefunc, line, nelem*ncclSizeOfT(), *ptr, memType); return result; } #define ncclCudaMalloc(ptr, nelem, manager, ...) ncclCudaMallocDebug(ptr, nelem, __FILE__, __LINE__, manager, ##__VA_ARGS__) template ncclResult_t ncclCudaCallocDebug(T** ptr, size_t nelem, const char *filefunc, int line, struct ncclMemManager* manager, ncclMemType_t memType = ncclMemPersist) { ncclResult_t result = ncclSuccess; cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; *ptr = nullptr; CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); if (nelem > 0) { // Need a side stream so as not to interfere with graph capture. cudaStream_t stream; CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking)); if (ncclCuMemEnable()) { NCCLCHECKGOTO(ncclCuMemAlloc((void **)ptr, NULL, ncclCuMemHandleType, nelem*ncclSizeOfT(), manager, memType), result, finish); } else { CUDACHECKGOTO(cudaMalloc(ptr, nelem*ncclSizeOfT()), result, finish); } CUDACHECKGOTO(cudaMemsetAsync(*ptr, 0, nelem*ncclSizeOfT(), stream), result, finish); CUDACHECKGOTO(cudaStreamSynchronize(stream), result, finish); CUDACHECKGOTO(cudaStreamDestroy(stream), result, finish); } finish: CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); if (*ptr == nullptr && nelem > 0) WARN("Failed to CUDA calloc %ld bytes", nelem*ncclSizeOfT()); INFO(NCCL_ALLOC, "%s:%d Cuda Calloc Size %ld pointer %p memType %d", filefunc, line, nelem*ncclSizeOfT(), *ptr, memType); return result; } #define ncclCudaCalloc(ptr, nelem, manager, ...) ncclCudaCallocDebug(ptr, nelem, __FILE__, __LINE__, manager, ##__VA_ARGS__) template ncclResult_t ncclCudaCallocAsyncDebug(T** ptr, size_t nelem, cudaStream_t stream, const char *filefunc, int line, struct ncclMemManager* manager, ncclMemType_t memType = ncclMemPersist) { ncclResult_t result = ncclSuccess; cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; *ptr = nullptr; CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); if (nelem > 0) { if (ncclCuMemEnable()) { NCCLCHECKGOTO(ncclCuMemAlloc((void **)ptr, NULL, ncclCuMemHandleType, nelem*ncclSizeOfT(), manager, memType), result, finish); } else { CUDACHECKGOTO(cudaMalloc(ptr, nelem*ncclSizeOfT()), result, finish); } CUDACHECKGOTO(cudaMemsetAsync(*ptr, 0, nelem*ncclSizeOfT(), stream), result, finish); } finish: CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); if (*ptr == nullptr && nelem > 0) WARN("Failed to CUDA calloc async %ld bytes", nelem*ncclSizeOfT()); INFO(NCCL_ALLOC, "%s:%d Cuda CallocAsync Size %ld pointer %p memType %d", filefunc, line, nelem*ncclSizeOfT(), *ptr, memType); return result; } #define ncclCudaCallocAsync(ptr, nelem, stream, manager, ...) ncclCudaCallocAsyncDebug(ptr, nelem, stream, __FILE__, __LINE__, manager, ##__VA_ARGS__) template ncclResult_t ncclCudaMemcpy(T* dst, T* src, size_t nelem) { ncclResult_t result = ncclSuccess; cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); // Need a side stream so as not to interfere with graph capture. cudaStream_t stream; CUDACHECKGOTO(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), result, finish); NCCLCHECKGOTO(ncclCudaMemcpyAsync(dst, src, nelem, stream), result, finish); CUDACHECKGOTO(cudaStreamSynchronize(stream), result, finish); CUDACHECKGOTO(cudaStreamDestroy(stream), result, finish); finish: CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); return result; } template ncclResult_t ncclCudaMemset(T* dst, int value, size_t nelem) { ncclResult_t result = ncclSuccess; cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); // Need a side stream so as not to interfere with graph capture. cudaStream_t stream; CUDACHECKGOTO(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), result, finish); CUDACHECKGOTO(cudaMemsetAsync((void*)dst, value, nelem * ncclSizeOfT(), stream), result, finish); CUDACHECKGOTO(cudaStreamSynchronize(stream), result, finish); CUDACHECKGOTO(cudaStreamDestroy(stream), result, finish); finish: CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); return result; } template ncclResult_t ncclCudaMemcpyAsync(T* dst, T* src, size_t nelem, cudaStream_t stream) { ncclResult_t result = ncclSuccess; cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); CUDACHECKGOTO(cudaMemcpyAsync(dst, src, nelem*ncclSizeOfT(), cudaMemcpyDefault, stream), result, finish); finish: CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); return result; } template ncclResult_t ncclCudaFree(T* ptr, struct ncclMemManager* manager, int numSegments = 1) { ncclResult_t result = ncclSuccess; cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; TRACE(NCCL_ALLOC, "Cuda Free pointer %p", ptr); CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); if (ncclCuMemEnable()) { NCCLCHECKGOTO(ncclCuMemFree((void *)ptr, manager, numSegments), result, finish); } else{ if (numSegments > 1) { result = ncclUnhandledCudaError; goto finish; } else { CUDACHECKGOTO(cudaFree(ptr), result, finish); } } finish: CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); return result; } // Allocate memory to be potentially ibv_reg_mr'd. This needs to be // allocated on separate pages as those pages will be marked DONTFORK // and if they are shared, that could cause a crash in a child process inline ncclResult_t ncclIbMallocDebug(void** ptr, size_t size, const char *filefunc, int line) { if (size > 0) { long page_size = ncclOsGetPageSize(); if (page_size < 0) return ncclSystemError; void* p; int size_aligned = ROUNDUP(size, page_size); int ret = posix_memalign(&p, page_size, size_aligned); if (ret != 0) return ncclSystemError; memset(p, 0, size); *ptr = p; } else { *ptr = NULL; } INFO(NCCL_ALLOC, "%s:%d Ib Alloc Size %ld pointer %p", filefunc, line, size, *ptr); return ncclSuccess; } #define ncclIbMalloc(...) ncclIbMallocDebug(__VA_ARGS__, __FILE__, __LINE__) #endif nccl-2.29.7-1/src/include/allocator.h000066400000000000000000000047051515037102200172350ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_ALLOCATOR_H_ #define NCCL_ALLOCATOR_H_ #include "nccl.h" #include #include //////////////////////////////////////////////////////////////////////////////// // ncclSpace: Allocates contiguous segments of non-negative integers. Useful // as a memory allocator when we can't put allocator state within the memory // being allocated. struct ncclSpace { int count; int capacity; int64_t* cuts; }; void ncclSpaceConstruct(struct ncclSpace* a); void ncclSpaceDestruct(struct ncclSpace* a); ncclResult_t ncclSpaceAlloc(struct ncclSpace* a, int64_t spaceLimit, int64_t objSize, int objAlign, int64_t* outObjOffset); ncclResult_t ncclSpaceFree(struct ncclSpace* a, int64_t objOffset, int64_t objSize); //////////////////////////////////////////////////////////////////////////////// // ncclShadowPool: Allocates device-side objects, their host-side shadows, and // maintains the device->host object address mapping. struct ncclShadowObject; struct ncclShadowPage; struct ncclShadowPool { int count, hbits; struct ncclShadowObject** table; cudaMemPool_t memPool; struct ncclShadowPage* pages; }; void ncclShadowPoolConstruct(struct ncclShadowPool*); ncclResult_t ncclShadowPoolDestruct(struct ncclShadowPool*); ncclResult_t ncclShadowPoolAlloc(struct ncclShadowPool*, size_t size, void** outDevObj, void** outHostObj, cudaStream_t stream); ncclResult_t ncclShadowPoolFree(struct ncclShadowPool*, void* devObj, cudaStream_t stream); ncclResult_t ncclShadowPoolToHost(struct ncclShadowPool*, void* devObj, void** outHostObj); template static inline ncclResult_t ncclShadowPoolAlloc(struct ncclShadowPool* pool, T** outDevObj, T** outHostObj, cudaStream_t stream) { void* devObj; void* hostObj; ncclResult_t got = ncclShadowPoolAlloc(pool, sizeof(T), &devObj, &hostObj, stream); if (outDevObj) *outDevObj = (T*)devObj; if (outHostObj) *outHostObj = (T*)hostObj; return got; } template static inline ncclResult_t ncclShadowPoolToHost(struct ncclShadowPool* pool, T* devObj, T** hostObj) { return ncclShadowPoolToHost(pool, (void*)devObj, (void**)hostObj); } #endif nccl-2.29.7-1/src/include/argcheck.h000066400000000000000000000016161515037102200170220ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_ARGCHECK_H_ #define NCCL_ARGCHECK_H_ #include "core.h" #include "info.h" struct ncclArgsInfo { struct ncclInfo info; struct ncclArgsInfo* next; }; ncclResult_t PtrCheck(const void* ptr, const char* opname, const char* ptrname); ncclResult_t CommCheck(struct ncclComm* ptr, const char* opname, const char* ptrname); ncclResult_t ArgsCheck(struct ncclInfo* info); ncclResult_t CudaPtrCheck(const void* pointer, struct ncclComm* comm, const char* ptrname, const char* opname); ncclResult_t ncclArgsGlobalCheck(struct ncclArgsInfo* info); #endif nccl-2.29.7-1/src/include/bitops.h000066400000000000000000000353531515037102200165600ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_BITOPS_H_ #define NCCL_BITOPS_H_ #include #include #include "compiler.h" #if !__NVCC__ #ifndef __host__ #define __host__ #endif #ifndef __device__ #define __device__ #endif #endif template constexpr static __host__ __device__ Int minval(Int a) { return a; } template constexpr static __host__ __device__ Int minval(Int a, Int b, More ...more) { #if __CUDA_ARCH__ return minval(min(a, b), more...); #else return minval(a < b ? a : b, more...); #endif } template constexpr static __host__ __device__ Int maxval(Int a) { return a; } template constexpr static __host__ __device__ Int maxval(Int a, Int b, More ...more) { #if __CUDA_ARCH__ return maxval(max(a, b), more...); #else return maxval(a > b ? a : b, more...); #endif } #define BIT(x) (1UL << (x)) #define MASK(x) ((1UL << x) - 1UL) #define DIVUP(x, y) \ (((x)+(y)-1)/(y)) #define ROUNDUP(x, y) \ (DIVUP((x), (y))*(y)) #define ALIGN_POWER(x, y) \ ((x) > (y) ? ROUNDUP(x, y) : ((y)/((y)/(x)))) #define ALIGN_SIZE(size, align) \ size = ((size + (align) - 1) / (align)) * (align); template static __host__ __device__ constexpr Z divUp(X x, Y y) { return (x+y-1)/y; } template static __host__ __device__ constexpr Z roundUp(X x, Y y) { return (x+y-1) - (x+y-1)%y; } template static __host__ __device__ constexpr Z roundDown(X x, Y y) { return x - x%y; } // assumes second argument is a power of 2 template static __host__ __device__ constexpr Z alignUp(X x, Y a) { return (x + a-1) & -Z(a); } template static __host__ __device__ T* alignUp(T* x, size_t a) { static_assert(sizeof(T) == 1, "Only single byte types allowed."); return reinterpret_cast((reinterpret_cast(x) + a-1) & -uintptr_t(a)); } // assumes second argument is a power of 2 template static __host__ __device__ constexpr Z alignDown(X x, Y a) { return x & -Z(a); } template static __host__ __device__ T* alignDown(T* x, size_t a) { static_assert(sizeof(T) == 1, "Only single byte types allowed."); return reinterpret_cast(reinterpret_cast(x) & -uintptr_t(a)); } template constexpr __host__ __device__ bool isPow2(Int x) { return (x & (x-1)) == 0; } template static __host__ __device__ T add4G(T base, int delta4G) { union { T tmp; uint32_t u32[2]; }; tmp = base; u32[1] += delta4G; return tmp; } template static __host__ __device__ T incWrap4G(T ptr, uint32_t delta4G, uint32_t lo4G, uint32_t hi4G) { union { T tmp; uint32_t u32[2]; }; tmp = ptr; u32[1] += delta4G; if (u32[1] >= hi4G) u32[1] -= hi4G-lo4G; return tmp; } template static __host__ __device__ T decWrap4G(T ptr, uint32_t delta4G, uint32_t lo4G, uint32_t hi4G) { union { T tmp; uint32_t u32[2]; }; tmp = ptr; u32[1] -= delta4G; if (u32[1] < lo4G) u32[1] += hi4G-lo4G; return tmp; } // Produce the reciprocal of x for use in idivByRcp constexpr __host__ __device__ uint32_t idivRcp32(uint32_t x) { return uint32_t(uint64_t(0x100000000)/x); } constexpr __host__ __device__ uint64_t idivRcp64(uint64_t x) { return uint64_t(-1)/x + isPow2(x); } static __host__ __device__ uint32_t mul32hi(uint32_t a, uint32_t b) { #if __CUDA_ARCH__ return __umulhi(a, b); #else return uint64_t(a)*b >> 32; #endif } static __host__ __device__ uint64_t mul64hi(uint64_t a, uint64_t b) { #if __CUDA_ARCH__ return __umul64hi(a, b); #else return (uint64_t)(((unsigned __int128)a)*b >> 64); #endif } // Produce the reciprocal of x*y given their respective reciprocals. This incurs // no integer division on device. static __host__ __device__ uint32_t imulRcp32(uint32_t x, uint32_t xrcp, uint32_t y, uint32_t yrcp) { if (xrcp == 0) return yrcp; if (yrcp == 0) return xrcp; uint32_t rcp = mul32hi(xrcp, yrcp); uint32_t rem = -x*y*rcp; if (x*y <= rem) rcp += 1; return rcp; } static __host__ __device__ uint64_t imulRcp64(uint64_t x, uint64_t xrcp, uint64_t y, uint64_t yrcp) { if (xrcp == 0) return yrcp; if (yrcp == 0) return xrcp; uint64_t rcp = mul64hi(xrcp, yrcp); uint64_t rem = -x*y*rcp; if (x*y <= rem) rcp += 1; return rcp; } // Fast integer division where divisor has precomputed reciprocal. // idivFast(x, y, idivRcp(y)) == x/y static __host__ __device__ void idivmodFast32(uint32_t *quo, uint32_t *rem, uint32_t x, uint32_t y, uint32_t yrcp) { uint32_t q = x, r = 0; if (yrcp != 0) { q = mul32hi(x, yrcp); r = x - y*q; if (r >= y) { q += 1; r -= y; } } *quo = q; *rem = r; } static __host__ __device__ void idivmodFast64(uint64_t *quo, uint64_t *rem, uint64_t x, uint64_t y, uint64_t yrcp) { uint64_t q = x, r = 0; if (yrcp != 0) { q = mul64hi(x, yrcp); r = x - y*q; if (r >= y) { q += 1; r -= y; } } *quo = q; *rem = r; } static __host__ __device__ uint32_t idivFast32(uint32_t x, uint32_t y, uint32_t yrcp) { uint32_t q, r; idivmodFast32(&q, &r, x, y, yrcp); return q; } static __host__ __device__ uint32_t idivFast64(uint64_t x, uint64_t y, uint64_t yrcp) { uint64_t q, r; idivmodFast64(&q, &r, x, y, yrcp); return q; } static __host__ __device__ uint32_t imodFast32(uint32_t x, uint32_t y, uint32_t yrcp) { uint32_t q, r; idivmodFast32(&q, &r, x, y, yrcp); return r; } static __host__ __device__ uint32_t imodFast64(uint64_t x, uint64_t y, uint64_t yrcp) { uint64_t q, r; idivmodFast64(&q, &r, x, y, yrcp); return r; } template static __host__ __device__ int countOneBits(Int x) { #if __CUDA_ARCH__ if (sizeof(Int) <= sizeof(unsigned int)) { return __popc((unsigned int)x); } else if (sizeof(Int) <= sizeof(unsigned long long)) { return __popcll((unsigned long long)x); } else { static_assert(sizeof(Int) <= sizeof(unsigned long long), "Unsupported integer size."); return -1; } #else if (sizeof(Int) <= sizeof(unsigned int)) { return COMPILER_POPCOUNT32((unsigned int)x); } else if (sizeof(Int) <= sizeof(unsigned long)) { return COMPILER_POPCOUNT64((unsigned long)x); } else if (sizeof(Int) <= sizeof(unsigned long long)) { return COMPILER_POPCOUNT64((unsigned long long)x); } else { static_assert(sizeof(Int) <= sizeof(unsigned long long), "Unsupported integer size."); return -1; } #endif } // Returns index of first one bit or returns -1 if mask is zero. template static __host__ __device__ int firstOneBit(Int mask) { int i; #if __CUDA_ARCH__ if (sizeof(Int) <= sizeof(int)) { i = __ffs((int)mask); } else if (sizeof(Int) <= sizeof(long long)) { i = __ffsll((long long)mask); } else { static_assert(sizeof(Int) <= sizeof(long long), "Unsupported integer size."); } #else if (sizeof(Int) <= sizeof(int)) { i = COMPILER_FFS((int)mask); } else if (sizeof(Int) <= sizeof(long)) { i = COMPILER_FFSL((long)mask); } else if (sizeof(Int) <= sizeof(long long)) { i = COMPILER_FFSLL((long long)mask); } else { static_assert(sizeof(Int) <= sizeof(long long), "Unsupported integer size."); } #endif return i-1; } template static __host__ __device__ int popFirstOneBit(Int* mask) { Int tmp = *mask; *mask &= *mask-1; return firstOneBit(tmp); } template static __host__ __device__ int log2Down(Int x) { int w, n; #if __CUDA_ARCH__ if (sizeof(Int) <= sizeof(int)) { w = 8*sizeof(int); n = __clz((int)x); } else if (sizeof(Int) <= sizeof(long long)) { w = 8*sizeof(long long); n = __clzll((long long)x); } else { static_assert(sizeof(Int) <= sizeof(long long), "Unsupported integer size."); } #else if (x == 0) { return -1; } else if (sizeof(Int) <= sizeof(unsigned int)) { w = 8*sizeof(unsigned int); n = COMPILER_CLZ((unsigned int)x); } else if (sizeof(Int) <= sizeof(unsigned long)) { w = 8*sizeof(unsigned long); n = COMPILER_CLZL((unsigned long)x); } else if (sizeof(Int) <= sizeof(unsigned long long)) { w = 8*sizeof(unsigned long long); n = COMPILER_CLZLL((unsigned long long)x); } else { static_assert(sizeof(Int) <= sizeof(unsigned long long), "Unsupported integer size."); } #endif return (w-1)-n; } template static __host__ __device__ int log2Up(Int x) { int w, n; if (x != 0) x -= 1; #if __CUDA_ARCH__ if (sizeof(Int) <= sizeof(int)) { w = 8*sizeof(int); n = __clz((int)x); } else if (sizeof(Int) <= sizeof(long long)) { w = 8*sizeof(long long); n = __clzll((long long)x); } else { static_assert(sizeof(Int) <= sizeof(long long), "Unsupported integer size."); } #else if (x == 0) { return 0; } else if (sizeof(Int) <= sizeof(unsigned int)) { w = 8*sizeof(unsigned int); n = COMPILER_CLZ((unsigned int)x); } else if (sizeof(Int) <= sizeof(unsigned long)) { w = 8*sizeof(unsigned long); n = COMPILER_CLZL((unsigned long)x); } else if (sizeof(Int) <= sizeof(unsigned long long)) { w = 8*sizeof(unsigned long long); n = COMPILER_CLZLL((unsigned long long)x); } else { static_assert(sizeof(Int) <= sizeof(unsigned long long), "Unsupported integer size."); } #endif return w-n; } template static __host__ __device__ Int pow2Up(Int x) { return Int(1)< static __host__ __device__ Int pow2Down(Int x) { // True, log2Down can return -1, but we don't normally pass 0 as an argument... // coverity[negative_shift] return Int(1)< static __host__ UInt reverseSubBits(UInt x) { if (nSubBits >= 16 && 8*sizeof(UInt) == nSubBits) { switch (8*sizeof(UInt)) { case 16: x = COMPILER_BSWAP16(x); break; case 32: x = COMPILER_BSWAP32(x); break; case 64: x = COMPILER_BSWAP64(x); break; default: static_assert(8*sizeof(UInt) <= 64, "Unsupported integer type."); } return reverseSubBits(x); } else if (nSubBits <= 1) { return x; } else { UInt m = UInt(-1)/((UInt(1)<<(nSubBits/2))+1); x = (x & m)<<(nSubBits/2) | (x & ~m)>>(nSubBits/2); return reverseSubBits(x); } } template struct ncclToUnsigned; template<> struct ncclToUnsigned { using type = unsigned char; }; template<> struct ncclToUnsigned { using type = unsigned char; }; template<> struct ncclToUnsigned { using type = unsigned char; }; template<> struct ncclToUnsigned { using type = unsigned short; }; template<> struct ncclToUnsigned { using type = unsigned short; }; template<> struct ncclToUnsigned { using type = unsigned int; }; template<> struct ncclToUnsigned { using type = unsigned int; }; template<> struct ncclToUnsigned { using type = unsigned long; }; template<> struct ncclToUnsigned { using type = unsigned long; }; template<> struct ncclToUnsigned { using type = unsigned long long; }; template<> struct ncclToUnsigned { using type = unsigned long long; }; // Reverse the bottom nBits bits of x. The top bits will be overwritten with 0's. template static __host__ __device__ Int reverseBits(Int x, int nBits) { using UInt = typename ncclToUnsigned::type; union { UInt ux; Int sx; }; sx = x; #if __CUDA_ARCH__ if (sizeof(Int) <= sizeof(unsigned int)) { ux = __brev(ux); } else if (sizeof(Int) <= sizeof(unsigned long long)) { ux = __brevll(ux); } else { static_assert(sizeof(Int) <= sizeof(unsigned long long), "Unsupported integer type."); } #else ux = reverseSubBits(ux); #endif ux = nBits==0 ? 0 : ux>>(8*sizeof(UInt)-nBits); return sx; } //////////////////////////////////////////////////////////////////////////////// // Custom 8 bit floating point format for approximating 32 bit uints. This format // has nearly the full range of uint32_t except it only keeps the top 3 bits // beneath the leading 1 bit and thus has a max value of 0xf0000000. static __host__ __device__ uint32_t u32fpEncode(uint32_t x, int bitsPerPow2) { int log2x; #if __CUDA_ARCH__ log2x = 31-__clz(x|1); #else log2x = 31-COMPILER_CLZ(x|1); #endif uint32_t mantissa = x>>(log2x >= bitsPerPow2 ? log2x-bitsPerPow2 : 0) & ((1u<= bitsPerPow2 ? log2x-(bitsPerPow2-1) : 0; return exponent<>bitsPerPow2; uint32_t mantissa = (x & ((1u<> 31; acc[0] *= 0x9de62bbc8cef3ce3; acc[1] ^= acc[1] >> 32; acc[1] *= 0x485cd6311b599e79; // Read in a chunk of input. size_t chunkSize = size < sizeof(uint64_t) ? size : sizeof(uint64_t); uint64_t x = 0; memcpy(&x, ptr, chunkSize); ptr += chunkSize; size -= chunkSize; // Add to accumulator. acc[0] += x; } } template static __host__ __device__ void eatHash(uint64_t acc[2], const T* bytes) { eatHash(acc, (const void*)bytes, sizeof(T)); } static __host__ __device__ uint64_t digestHash(uint64_t const acc[2]) { uint64_t h = acc[0]; h ^= h >> 31; h *= 0xbac3bd562846de6b; h += acc[1]; h ^= h >> 32; h *= 0x995a187a14e7b445; return h; } static __host__ __device__ uint64_t getHash(const void* bytes, size_t size) { uint64_t acc[2] = {1, 1}; eatHash(acc, bytes, size); return digestHash(acc); } template static __host__ __device__ uint64_t getHash(const T* bytes) { return getHash((const void*)bytes, sizeof(T)); } #endif nccl-2.29.7-1/src/include/bootstrap.h000066400000000000000000000041251515037102200172660ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_BOOTSTRAP_H_ #define NCCL_BOOTSTRAP_H_ #include "nccl.h" #include "comm.h" struct ncclBootstrapHandle { uint64_t magic; union ncclSocketAddress addr; int nRanks; // number of existing ranks }; static_assert(sizeof(struct ncclBootstrapHandle) <= sizeof(ncclUniqueId), "Bootstrap handle is too large to fit inside NCCL unique ID"); ncclResult_t bootstrapNetInit(); ncclResult_t bootstrapCreateRoot(struct ncclBootstrapHandle* handle, bool idFromEnv); ncclResult_t bootstrapGetUniqueId(struct ncclBootstrapHandle* handle, struct ncclComm* comm); ncclResult_t bcastGrowHandle(struct ncclBootstrapHandle* handle, struct ncclComm* parent, bool isRoot); ncclResult_t bootstrapInit(int nHandles, void* handle, struct ncclComm* comm, struct ncclComm* parent); ncclResult_t bootstrapSplit(uint64_t magic, struct ncclComm* comm, struct ncclComm* parent, int color, int key, int* parentRanks); ncclResult_t bootstrapAllGather(void* commState, void* allData, int size); ncclResult_t bootstrapSend(void* commState, int peer, int tag, void* data, int size); ncclResult_t bootstrapRecv(void* commState, int peer, int tag, void* data, int size); ncclResult_t bootstrapBarrier(void* commState, int rank, int nranks, int tag); ncclResult_t bootstrapBroadcast(void* commState, int rank, int nranks, int root, void* bcastData, int size); ncclResult_t bootstrapIntraNodeBarrier(void* commState, int *ranks, int rank, int nranks, int tag); ncclResult_t bootstrapIntraNodeAllGather(void* commState, int *ranks, int rank, int nranks, void* allData, int size); ncclResult_t bootstrapIntraNodeBroadcast(void* commState, int *ranks, int rank, int nranks, int root, void* bcastData, int size); ncclResult_t bootstrapClose(void* commState); ncclResult_t bootstrapAbort(void* commState); #endif nccl-2.29.7-1/src/include/ce_coll.h000066400000000000000000000046401515037102200166530ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_CE_COLL_H_ #define NCCL_CE_COLL_H_ #include "nccl.h" #include "nccl_common.h" #include "bitops.h" // Memory operations per rank for different synchronization protocols #define NCCL_CE_SYNC_OPS_PER_RANK_MC 2 #define NCCL_CE_SYNC_OPS_PER_RANK_UC 3 struct ncclCeColl { uint8_t* baseUCSymReadyPtr; uint8_t* baseUCSymComplPtr; size_t baseUCSymReadyOffset; size_t baseUCSymComplOffset; uint32_t ceSeqNum; bool useCompletePtr; uint32_t intraBatchSyncFreq; uint64_t intraBatchSyncMsgThreshold; struct ncclDevrWindow* ceSyncWin; }; struct ncclCeInitTask { struct ncclCeInitTask *next; struct ncclComm* comm; }; struct alignas(16) ncclCeCollArgs { ncclFunc_t func; int rootRank; ncclDataType_t datatype; size_t nElts; size_t eltSize; uint8_t* sendBuff; uint8_t* recvBuff; struct ncclDevrWindow* sendWin; struct ncclDevrWindow* recvWin; void* collApiEventHandle; // Parent API event handle for profiler hierarchy void* ceCollProfHandle; // CE collective profiler event handle }; struct ncclCeBatchOpsParams { void** dsts; void** srcs; size_t* sizes; size_t numOps; bool intraBatchSync; #if CUDART_VERSION >= 12080 cudaMemcpyAttributes* attrs; size_t* attrIdxs; size_t numAttrs; #endif }; bool ncclCeAvailable(struct ncclComm* comm, ncclFunc_t coll, int/*ncclDevRedOp_t*/ red, ncclDataType_t ty, ncclSymRegType_t winRegType); ncclResult_t ncclCeInit(struct ncclComm* comm); ncclResult_t ncclCeFinalize(struct ncclComm* comm); ncclResult_t ncclMemOpSync(struct ncclComm* comm, cudaStream_t stream, void* ceCollHandle); ncclResult_t ncclLaunchCeColl(struct ncclComm* comm, struct ncclKernelPlan* plan); ncclResult_t ncclCeAllGather(struct ncclComm* comm, struct ncclCeCollArgs* args, cudaStream_t stream); ncclResult_t ncclCeScatter(struct ncclComm* comm, struct ncclCeCollArgs* args, cudaStream_t stream); ncclResult_t ncclCeGather(struct ncclComm* comm, struct ncclCeCollArgs* args, cudaStream_t stream); ncclResult_t ncclCeAlltoAll(struct ncclComm* comm, struct ncclCeCollArgs* args, cudaStream_t stream); #endif /* NCCL_CE_COLL_H_ */ nccl-2.29.7-1/src/include/channel.h000066400000000000000000000024731515037102200166650ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_CHANNEL_H_ #define NCCL_CHANNEL_H_ #include "comm.h" #include "utils.h" #include ncclResult_t initChannel(struct ncclComm* comm, int channelid); ncclResult_t initNvlsChannel(struct ncclComm* comm, int channelId, struct ncclComm* parent, bool share); ncclResult_t initCollnetChannel(struct ncclComm* comm, int channelId, struct ncclComm* parent, bool share); ncclResult_t freeChannel(struct ncclChannel* channel, int nRanks, int collnetNRanks, int nvlsNRanks, struct ncclComm* comm); inline uint8_t ncclP2pChannelBaseForRound(struct ncclComm* comm, int p2pRound) { int base; if (comm->nNodes > 1) { int localSize = comm->p2pSchedGroupSize; int groupDelta = p2pRound / localSize; int localDelta = p2pRound % localSize; base = groupDelta*divUp(localSize, NCCL_MAX_DEV_WORK_P2P_PER_BATCH); base += localDelta/NCCL_MAX_DEV_WORK_P2P_PER_BATCH; } else { base = p2pRound; } return reverseBits(base, log2Up(comm->p2pnChannels)); } #endif nccl-2.29.7-1/src/include/checks.h000066400000000000000000000222651515037102200165160ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_CHECKS_H_ #define NCCL_CHECKS_H_ #include "debug.h" // Check CUDA RT calls #define CUDACHECK(cmd) \ do { \ cudaError_t err = cmd; \ if (err != cudaSuccess) { \ WARN("Cuda failure '%s'", cudaGetErrorString(err)); \ (void)cudaGetLastError(); \ return ncclUnhandledCudaError; \ } \ } while (false) #define CUDACHECKGOTO(cmd, RES, label) \ do { \ cudaError_t err = cmd; \ if (err != cudaSuccess) { \ WARN("Cuda failure '%s'", cudaGetErrorString(err)); \ (void)cudaGetLastError(); \ RES = ncclUnhandledCudaError; \ goto label; \ } \ } while (false) // Report failure but clear error and continue #define CUDACHECKIGNORE(cmd) \ do { \ cudaError_t err = cmd; \ if (err != cudaSuccess) { \ INFO(NCCL_ALL, "%s:%d Cuda failure '%s'", __FILE__, __LINE__, \ cudaGetErrorString(err)); \ (void)cudaGetLastError(); \ } \ } while (false) // Use inline function to clear CUDA error inside expressions static inline cudaError_t cuda_clear(cudaError_t err) { if (err != cudaSuccess) (void)cudaGetLastError(); return err; } // Check if cudaSuccess & clear CUDA error #define CUDASUCCESS(cmd) cuda_clear(cmd) == cudaSuccess // Clear CUDA error, return CUDA return code #define CUDACLEARERROR(cmd) cuda_clear(cmd) #include // Check system calls #define SYSCHECK(statement, name) do { \ int retval; \ SYSCHECKSYNC((statement), name, retval); \ if (retval == -1) { \ WARN("Call to " name " failed: %s", strerror(errno)); \ return ncclSystemError; \ } \ } while (false) #define SYSCHECKSYNC(statement, name, retval) do { \ retval = (statement); \ if (retval == -1 && (errno == EINTR || errno == EWOULDBLOCK || errno == EAGAIN)) { \ INFO(NCCL_ALL,"Call to " name " returned %s, retrying", strerror(errno)); \ } else { \ break; \ } \ } while(true) #define SYSCHECKGOTO(statement, name, RES, label) do { \ int retval; \ SYSCHECKSYNC((statement), name, retval); \ if (retval == -1) { \ WARN("Call to " name " failed: %s", strerror(errno)); \ RES = ncclSystemError; \ goto label; \ } \ } while (0) // Pthread calls don't set errno and never return EINTR. #define PTHREADCHECK(statement, name) do { \ int retval = (statement); \ if (retval != 0) { \ WARN("Call to " name " failed: %s", strerror(retval)); \ return ncclSystemError; \ } \ } while (0) #define PTHREADCHECKGOTO(statement, name, RES, label) do { \ int retval = (statement); \ if (retval != 0) { \ WARN("Call to " name " failed: %s", strerror(retval)); \ RES = ncclSystemError; \ goto label; \ } \ } while (0) #define NEQCHECK(statement, value) do { \ if ((statement) != value) { \ /* Print the back trace*/ \ INFO(NCCL_ALL,"%s:%d -> %d (%s)", __FILE__, __LINE__, ncclSystemError, strerror(errno)); \ return ncclSystemError; \ } \ } while (0) #define NEQCHECKGOTO(statement, value, RES, label) do { \ if ((statement) != value) { \ /* Print the back trace*/ \ RES = ncclSystemError; \ INFO(NCCL_ALL,"%s:%d -> %d (%s)", __FILE__, __LINE__, RES, strerror(errno)); \ goto label; \ } \ } while (0) #define EQCHECK(statement, value) do { \ if ((statement) == value) { \ /* Print the back trace*/ \ INFO(NCCL_ALL,"%s:%d -> %d (%s)", __FILE__, __LINE__, ncclSystemError, strerror(errno)); \ return ncclSystemError; \ } \ } while (0) #define EQCHECKGOTO(statement, value, RES, label) do { \ if ((statement) == value) { \ /* Print the back trace*/ \ RES = ncclSystemError; \ INFO(NCCL_ALL,"%s:%d -> %d (%s)", __FILE__, __LINE__, RES, strerror(errno)); \ goto label; \ } \ } while (0) // Propagate errors up #define NCCLCHECK(call) do { \ ncclResult_t RES = call; \ if (RES != ncclSuccess && RES != ncclInProgress) { \ /* Print the back trace*/ \ if (ncclDebugNoWarn == 0) INFO(NCCL_ALL,"%s:%d -> %d", __FILE__, __LINE__, RES); \ return RES; \ } \ } while (0) #define NCCLCHECKGOTO(call, RES, label) do { \ RES = call; \ if (RES != ncclSuccess && RES != ncclInProgress) { \ /* Print the back trace*/ \ if (ncclDebugNoWarn == 0) INFO(NCCL_ALL,"%s:%d -> %d", __FILE__, __LINE__, RES); \ goto label; \ } \ } while (0) // Report failure but continue - useful for cleanup paths where we want to // attempt all cleanup steps. Preserves the first error in RES. #define NCCLCHECKIGNORE(call, RES) do { \ ncclResult_t TMPRES = call; \ if (TMPRES != ncclSuccess && TMPRES != ncclInProgress) { \ if (ncclDebugNoWarn == 0) INFO(NCCL_ALL,"%s:%d -> %d", __FILE__, __LINE__, TMPRES); \ if (RES == ncclSuccess) RES = TMPRES; \ } \ } while (0) #define NCCLCHECKNOWARN(call, FLAGS) do { \ ncclResult_t RES; \ NOWARN(RES = call, FLAGS); \ if (RES != ncclSuccess && RES != ncclInProgress) { \ return RES; \ } \ } while (0) #define NCCLCHECKGOTONOWARN(call, RES, label, FLAGS) do { \ NOWARN(RES = call, FLAGS); \ if (RES != ncclSuccess && RES != ncclInProgress) { \ goto label; \ } \ } while (0) #define NCCLWAIT(call, cond, abortFlagPtr) do { \ uint32_t* tmpAbortFlag = (abortFlagPtr); \ ncclResult_t RES = call; \ if (RES != ncclSuccess && RES != ncclInProgress) { \ if (ncclDebugNoWarn == 0) INFO(NCCL_ALL,"%s:%d -> %d", __FILE__, __LINE__, RES); \ return ncclInternalError; \ } \ if (COMPILER_ATOMIC_LOAD(tmpAbortFlag, std::memory_order_acquire)) NEQCHECK(*tmpAbortFlag, 0); \ } while (!(cond)) #define NCCLWAITGOTO(call, cond, abortFlagPtr, RES, label) do { \ uint32_t* tmpAbortFlag = (abortFlagPtr); \ RES = call; \ if (RES != ncclSuccess && RES != ncclInProgress) { \ if (ncclDebugNoWarn == 0) INFO(NCCL_ALL,"%s:%d -> %d", __FILE__, __LINE__, RES); \ goto label; \ } \ if (COMPILER_ATOMIC_LOAD(tmpAbortFlag, std::memory_order_acquire)) NEQCHECKGOTO(*tmpAbortFlag, 0, RES, label); \ } while (!(cond)) #define NCCLCHECKTHREAD(a, args) do { \ if (((args)->ret = (a)) != ncclSuccess && (args)->ret != ncclInProgress) { \ INFO(NCCL_INIT,"%s:%d -> %d [Async thread]", __FILE__, __LINE__, (args)->ret); \ return args; \ } \ } while(0) #define CUDACHECKTHREAD(a) do { \ if ((a) != cudaSuccess) { \ INFO(NCCL_INIT,"%s:%d -> %d [Async thread]", __FILE__, __LINE__, args->ret); \ args->ret = ncclUnhandledCudaError; \ return args; \ } \ } while(0) // Common thread creation implementation with error handling #define STDTHREADCREATE_IMPL(var, func, error_action, ...) do { \ try { \ (var) = std::thread(func, __VA_ARGS__); \ } catch (const std::exception& e) { \ WARN("Thread creation failed: %s", e.what()); \ error_action; \ } \ } while(0) #define STDTHREADCREATE(var, func, ...) \ STDTHREADCREATE_IMPL(var, func, return ncclSystemError, __VA_ARGS__) #define STDTHREADCREATE_GOTO(var, func, RES, label, ...) \ STDTHREADCREATE_IMPL(var, func, do { RES = ncclSystemError; goto label; } while(0), __VA_ARGS__) #define NEW_NOTHROW(var, x) do { \ (var) = new (std::nothrow) x{}; \ if (!(var)) { \ WARN("Allocation failed at %s:%d", __FILE__, __LINE__); \ return ncclSystemError; \ } \ } while(0) #define NEW_NOTHROW_GOTO(var, x, RES, label) do { \ (var) = new (std::nothrow) x{}; \ if (!(var)) { \ WARN("Allocation failed at %s:%d", __FILE__, __LINE__); \ RES = ncclSystemError; \ goto label; \ } \ } while(0) #endif nccl-2.29.7-1/src/include/coll_net.h000066400000000000000000000070561515037102200170560ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef COLL_NET_H_ #define COLL_NET_H_ #include "comm.h" #include "nccl.h" #include "nccl_net.h" typedef char collNetHandle_t[NCCL_NET_HANDLE_MAXSIZE]; // Translation to external API static const char* collNetName(struct ncclComm* comm) { return comm->ncclCollNet->name; } static ncclResult_t collNetDevices(struct ncclComm* comm, int* ndev) { NCCLCHECK(comm->ncclCollNet->devices(ndev)); return ncclSuccess; } static ncclResult_t collNetGetProperties(struct ncclComm* comm, int dev, ncclNetProperties_t* props) { NCCLCHECK(comm->ncclCollNet->getProperties(dev, props)); return ncclSuccess; } static ncclResult_t collNetListen(struct ncclComm* comm, int dev, void* handle, void** listenComm) { NCCLCHECK(comm->ncclCollNet->listen(comm->collNetContext, dev, handle, listenComm)); return ncclSuccess; } static ncclResult_t collNetConnect(struct ncclComm* comm, void* handles[], int nranks, int rank, void* listenComm, void** collComm) { NCCLCHECK(comm->ncclCollNet->connect(handles, nranks, rank, listenComm, collComm)); return ncclSuccess; } static ncclResult_t collNetReduceSupport(struct ncclComm* comm, ncclDataType_t dataType, ncclRedOp_t redOp, int* supported) { NCCLCHECK(comm->ncclCollNet->reduceSupport(dataType, redOp, supported)); return ncclSuccess; } static ncclResult_t collNetRegMr(struct ncclComm* comm, void* collComm, void* data, size_t size, int type, void** mhandle) { NCCLCHECK(comm->ncclCollNet->regMr(collComm, data, size, type, mhandle)); return ncclSuccess; } /* DMA-BUF support */ static ncclResult_t collNetRegMrDmaBuf(struct ncclComm* comm, void* collComm, void* data, size_t size, int type, uint64_t offset, int fd, void** mhandle) { NCCLCHECK(comm->ncclCollNet->regMrDmaBuf(collComm, data, size, type, offset, fd, mhandle)); return ncclSuccess; } static ncclResult_t collNetDeregMr(struct ncclComm* comm, void* collComm, void* mhandle) { NCCLCHECK(comm->ncclCollNet->deregMr(collComm, mhandle)); return ncclSuccess; } static ncclResult_t collNetIallreduce(struct ncclComm* comm, void* collComm, void* sendData, void* recvData, int count, ncclDataType_t dataType, ncclRedOp_t redOp, void* sendMhandle, void* recvMhandle, void** request) { NCCLCHECK(comm->ncclCollNet->iallreduce(collComm, sendData, recvData, count, dataType, redOp, sendMhandle, recvMhandle, request)); return ncclSuccess; } static ncclResult_t collNetIflush(struct ncclComm* comm, void* collComm, void* data, int size, void* mhandle, void** request) { NCCLCHECK(comm->ncclCollNet->iflush(collComm, data, size, mhandle, request)); return ncclSuccess; } static ncclResult_t collNetTest(struct ncclComm* comm, void* request, int* done, int* size) { NCCLCHECK(comm->ncclCollNet->test(request, done, size)); return ncclSuccess; } static ncclResult_t collNetCloseColl(struct ncclComm* comm, void* collComm) { NCCLCHECK(comm->ncclCollNet->closeColl(collComm)); return ncclSuccess; } static ncclResult_t collNetCloseListen(struct ncclComm* comm, void* listenComm) { NCCLCHECK(comm->ncclCollNet->closeListen(listenComm)); return ncclSuccess; } static ncclResult_t collNetFinalize(struct ncclComm* comm, void* ctx) { NCCLCHECK(comm->ncclCollNet->finalize(ctx)); return ncclSuccess; } static int collNetSupport(struct ncclComm* comm) { return comm->ncclCollNet != nullptr ? 1 : 0; } #endif nccl-2.29.7-1/src/include/collectives.h000066400000000000000000000673631515037102200176020ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_COLLECTIVES_H_ #define NCCL_COLLECTIVES_H_ #include "nccl.h" #include "nccl_tuner.h" #include "device.h" #include "compiler.h" #define NCCL_MAX_NET_SIZE (1024*1024*1024L) // Rather than send INT_MAX which is 2G-1, send a power of two. // CHUNKSIZE must be a multiple of SLICESIZE #define ALLREDUCE_SLICESTEPS (NCCL_STEPS/4) #define ALLREDUCE_CHUNKSTEPS (NCCL_STEPS/2) #define ALLGATHER_SLICESTEPS (NCCL_STEPS/4) #define ALLGATHER_CHUNKSTEPS (NCCL_STEPS/2) #define ALLTOALL_SLICESTEPS 1 #define ALLTOALL_CHUNKSTEPS 1 #define REDUCESCATTER_SLICESTEPS (NCCL_STEPS/4) #define REDUCESCATTER_CHUNKSTEPS (NCCL_STEPS/2) #define BROADCAST_SLICESTEPS 1 #define BROADCAST_CHUNKSTEPS 1 #define GATHER_SLICESTEPS 1 #define GATHER_CHUNKSTEPS 1 #define SCATTER_SLICESTEPS 1 #define SCATTER_CHUNKSTEPS 1 #define REDUCE_SLICESTEPS 1 #define REDUCE_CHUNKSTEPS 1 #define NCCL_MAX_SLICE_PER_CHUNK 2 // max value for CHUNKSTEPS/SLICESTEPS, must accord with above #define NCCL_MAX_NET_SIZE (1024*1024*1024L) // Rather than send INT_MAX which is 2G-1, send a power of two. const char* ncclFuncToString(ncclFunc_t op); const char* ncclDevRedOpToString(ncclDevRedOp_t op); const char* ncclDatatypeToString(ncclDataType_t type); const char* ncclAlgoToString(int algo); const char* ncclProtoToString(int proto); inline int ncclTypeSize(ncclDataType_t type) { switch (type) { case ncclInt8: case ncclUint8: case ncclFloat8e4m3: case ncclFloat8e5m2: return 1; case ncclFloat16: case ncclBfloat16: return 2; case ncclInt32: case ncclUint32: case ncclFloat32: return 4; case ncclInt64: case ncclUint64: case ncclFloat64: return 8; default: return -1; } } #include #define NCCL_MODE_NORMAL 0 #define NCCL_MODE_OFFSET 1 #define NCCL_MODE_PTR 2 struct ncclConnFifo { int mode; int offset; ssize_t size; void* ptr; }; #include class RingAlgorithm { protected: int refCount; int nRanks; int nStepsPerLoop; int chunkSteps; int sliceSteps; ssize_t sliceSize; ssize_t loopSize; ssize_t channelSize; uint8_t *sendbuff; uint8_t *recvbuff; void *sendMhandle; void *recvMhandle; void *srecvMhandle; public: // this ring class is used by proxy thread to retrieve the send and recv buffer, size as well as corresponding // mem handle based on the current step of the proxy args. The derived ring algo class is AR, AG, and BC which // would be allocated during enqueue stage and copied to proxy side through shared memory. For each copy, we will // increase the refCount by incRefCount() since the same ring algo object can be referenced multiple times for send // and recv progress. After all steps are done, we decrease the refCount and only delete the ring object when // refCount == 0. virtual void getNextSendAddr(int curStep, uint8_t **sendbuffOut, size_t *sizeOut, void **mhandleOut) = 0; virtual void getNextRecvAddr(int curStep, uint8_t **recvbuffOut, size_t *sizeOut, void **mhandleOut) = 0; int incRefCount() { return COMPILER_ATOMIC_ADD_FETCH(&refCount, 1, std::memory_order_relaxed); } int decRefCount() { return COMPILER_ATOMIC_SUB_FETCH(&refCount, 1, std::memory_order_release); } RingAlgorithm() { refCount = 0; } virtual ~RingAlgorithm() {}; }; class RingARAlgorithm : public RingAlgorithm { private: int ringIndex; int elemSize; ssize_t chunkSize; int slicePerChunk; public: void getNextSendAddr(int curStep, uint8_t **sendbuffOut, size_t *sizeOut, void **mhandleOut) { int curLoop = curStep / nStepsPerLoop; int curLoopStage = (curStep % nStepsPerLoop) / chunkSteps; int chunkStage = curLoopStage % nRanks; int sliceStage = (curStep % chunkSteps) / sliceSteps; ssize_t elemOffset = curLoop * loopSize; ssize_t remSize = channelSize - elemOffset; ssize_t chunkOffset; ssize_t sliceOffset; ssize_t curSliceSize; ssize_t curChunkSize; ssize_t size; ssize_t nelem; int chunkId; if (remSize < loopSize) { curChunkSize = alignUp(divUp(remSize / elemSize, nRanks), 16 / elemSize) * elemSize; } else { curChunkSize = chunkSize; } chunkId = (ringIndex + nRanks - 1 - chunkStage) % nRanks; chunkOffset = chunkId * curChunkSize; nelem = std::min(remSize - chunkOffset, curChunkSize); curSliceSize = std::max(divUp(nelem / elemSize, 16 * slicePerChunk) * 16, sliceSize / elemSize / 32) * elemSize; sliceOffset = sliceStage * curSliceSize; if (nelem <= sliceOffset) { *sendbuffOut = sendbuff; *mhandleOut = sendMhandle; } else { if (curLoopStage == 0) { *sendbuffOut = sendbuff + elemOffset + chunkOffset + sliceOffset; *mhandleOut = sendMhandle; } else { *sendbuffOut = recvbuff + elemOffset + chunkOffset + sliceOffset; *mhandleOut = srecvMhandle; } } size = std::min(curSliceSize, nelem - sliceOffset); *sizeOut = size < 0 ? 0 : size; return; } void getNextRecvAddr(int curStep, uint8_t **recvbuffOut, size_t *sizeOut, void **mhandleOut) { int curLoop = curStep / nStepsPerLoop; int curLoopStage = ((curStep + chunkSteps) % nStepsPerLoop) / chunkSteps; int chunkStage = curLoopStage % nRanks; int sliceStage = (curStep % chunkSteps) / sliceSteps; ssize_t elemOffset = curLoop * loopSize; ssize_t remSize = channelSize - elemOffset; ssize_t chunkOffset; ssize_t sliceOffset; ssize_t curSliceSize; ssize_t curChunkSize; ssize_t size; ssize_t nelem; int chunkId; if (remSize < loopSize) { curChunkSize = alignUp(divUp(remSize / elemSize, nRanks), 16 / elemSize) * elemSize; } else { curChunkSize = chunkSize; } if (curLoopStage == 0) { chunkId = (ringIndex + 1) % nRanks; } else { chunkId = (ringIndex + nRanks - 1 - chunkStage) % nRanks; } chunkOffset = chunkId * curChunkSize; nelem = std::min(remSize - chunkOffset, curChunkSize); curSliceSize = std::max(divUp(nelem / elemSize, 16 * slicePerChunk) * 16, sliceSize / elemSize / 32) * elemSize; sliceOffset = sliceStage * curSliceSize; if (nelem <= sliceOffset) { *recvbuffOut = recvbuff; } else { *recvbuffOut = recvbuff + elemOffset + chunkOffset + sliceOffset; } if (sizeOut) { size = std::min(curSliceSize, nelem - sliceOffset); *sizeOut = size < 0 ? 0 : size; } *mhandleOut = recvMhandle; return; } RingARAlgorithm(const void *sendbuff, void *recvbuff, int nRanks, int ringIndex, int chunkSteps, int sliceSteps, size_t chunkSize, size_t sliceSize, size_t gridOffset, size_t channelSize, int elemSize, void *sendMhandle, void *recvMhandle, void *srecvMhandle) { this->ringIndex = ringIndex; this->nRanks = nRanks; this->nStepsPerLoop = 2 * (nRanks - 1) * chunkSteps; this->chunkSteps = chunkSteps; this->sliceSteps = sliceSteps; this->chunkSize = chunkSize; this->sliceSize = sliceSize; this->loopSize = nRanks * chunkSize; this->sendbuff = (uint8_t*)sendbuff + gridOffset; this->recvbuff = (uint8_t*)recvbuff + gridOffset; this->channelSize = channelSize; this->elemSize = elemSize; this->sendMhandle = sendMhandle; this->recvMhandle = recvMhandle; this->srecvMhandle = srecvMhandle; this->slicePerChunk = chunkSteps / sliceSteps; } ~RingARAlgorithm() {} }; class RingAGAlgorithm : public RingAlgorithm { private: int *ringRanks; int elemSize; ssize_t sendSize; int slicePerChunk; public: void getNextSendAddr(int curStep, uint8_t **sendbuffOut, size_t *sizeOut, void **mhandleOut) { int curLoop = curStep / nStepsPerLoop; int chunkStage = (curStep % nStepsPerLoop) / chunkSteps; int sliceStage = (curStep % chunkSteps) / sliceSteps; ssize_t sliceOffset; ssize_t curSliceSize; ssize_t offset; ssize_t elemOffset = curLoop * loopSize; ssize_t chunkSize = std::min(loopSize, channelSize - elemOffset); ssize_t size; int rankDest; uint8_t *buff; void *mhandle; curSliceSize = std::max(divUp(chunkSize / elemSize, 16 * slicePerChunk) * 16, sliceSize / elemSize / 32) * elemSize; sliceOffset = sliceStage * curSliceSize; if (chunkStage == 0) { rankDest = ringRanks[0]; offset = elemOffset + sliceOffset; buff = sendbuff + offset; mhandle = sendMhandle; } else { rankDest = ringRanks[nRanks - chunkStage]; offset = elemOffset + rankDest * sendSize + sliceOffset; buff = recvbuff + offset; mhandle = srecvMhandle; } *sendbuffOut = buff; size = std::min(curSliceSize, channelSize - elemOffset - sliceOffset); *sizeOut = size < 0 ? 0 : size; *mhandleOut = mhandle; return; } void getNextRecvAddr(int curStep, uint8_t **recvbuffOut, size_t *sizeOut, void **mhandleOut) { int curLoop = curStep / nStepsPerLoop; int chunkStage = ((curStep + chunkSteps) % nStepsPerLoop) / chunkSteps; int sliceStage = (curStep % chunkSteps) / sliceSteps; ssize_t sliceOffset; ssize_t curSliceSize; ssize_t offset; ssize_t elemOffset = curLoop * loopSize; ssize_t chunkSize = std::min(loopSize, channelSize - elemOffset); ssize_t size; int rankDest; curSliceSize = std::max(divUp(chunkSize / elemSize, 16 * slicePerChunk) * 16, sliceSize / elemSize / 32) * elemSize; sliceOffset = sliceStage * curSliceSize; if (chunkStage == 0) { rankDest = ringRanks[1]; } else { rankDest = ringRanks[nRanks - chunkStage]; } offset = elemOffset + rankDest * sendSize + sliceOffset; *recvbuffOut = recvbuff + offset; if (sizeOut) { size = std::min(sliceSize, channelSize - elemOffset - sliceOffset); *sizeOut = size < 0 ? 0 : size; } *mhandleOut = recvMhandle; } RingAGAlgorithm(const void *sendbuff, void *recvbuff, int nRanks, int *ringRanks, int chunkSteps, int sliceSteps, size_t chunkSize, size_t sliceSize, size_t gridOffset, size_t channelSize, int elemSize, size_t sendSize, void *sendMhandle, void *recvMhandle, void *srecvMhandle) { this->ringRanks = ringRanks; this->nRanks = nRanks; this->nStepsPerLoop = (nRanks - 1) * chunkSteps; this->chunkSteps = chunkSteps; this->sliceSteps = sliceSteps; this->elemSize = elemSize; this->sliceSize = sliceSize; this->loopSize = chunkSize; this->sendSize = sendSize; this->channelSize = channelSize; this->sendbuff = (uint8_t*)sendbuff + gridOffset; this->recvbuff = (uint8_t*)recvbuff + gridOffset; this->sendMhandle = sendMhandle; this->recvMhandle = recvMhandle; this->srecvMhandle = srecvMhandle; this->slicePerChunk = chunkSteps / sliceSteps; } ~RingAGAlgorithm() {} }; class RingBCAlgorithm : public RingAlgorithm { private: int root; int rank; int nextRank; public: void getNextSendAddr(int curStep, uint8_t **sendbuffOut, size_t *sizeOut, void **mhandleOut) { int curLoop = curStep / nStepsPerLoop; int sliceStage = (curStep % chunkSteps) / sliceSteps; ssize_t sliceOffset = sliceStage * sliceSize; ssize_t offset; ssize_t elemOffset = curLoop * loopSize; ssize_t size; uint8_t *buff; void *mhandle; offset = elemOffset + sliceOffset; if (offset >= channelSize) { buff = sendbuff; mhandle = sendMhandle; } else if (rank == root) { buff = sendbuff + offset; mhandle = sendMhandle; } else { buff = recvbuff + offset; mhandle = srecvMhandle; } *sendbuffOut = buff; size = std::min(sliceSize, channelSize - offset); *sizeOut = size < 0 ? 0 : size; *mhandleOut = mhandle; return; } void getNextRecvAddr(int curStep, uint8_t **recvbuffOut, size_t *sizeOut, void **mhandleOut) { int curLoop = curStep / nStepsPerLoop; int sliceStage = (curStep % chunkSteps) / sliceSteps; ssize_t sliceOffset = sliceStage * sliceSize; ssize_t offset; ssize_t elemOffset = curLoop * loopSize; ssize_t size; offset = elemOffset + sliceOffset; if (offset >= channelSize) { *recvbuffOut = recvbuff; } else { *recvbuffOut = recvbuff + offset; } if (sizeOut) { size = std::min(sliceSize, channelSize - offset); *sizeOut = size < 0 ? 0 : size; } *mhandleOut = recvMhandle; return; } RingBCAlgorithm(const void* sendbuff, void* recvbuff, int rank, int root, int nRanks, int *ringRanks, int chunkSteps, int sliceSteps, size_t chunkSize, size_t sliceSize, size_t gridOffset, size_t channelSize, void *sendMhandle, void *recvMhandle, void *srecvMhandle) { this->root = root; this->rank = rank; this->nextRank = ringRanks[1]; this->nStepsPerLoop = chunkSteps; this->chunkSteps = chunkSteps; this->sliceSteps = sliceSteps; this->sliceSize = sliceSize; this->loopSize = chunkSize; this->channelSize = channelSize; this->sendbuff = (uint8_t*)sendbuff + gridOffset; this->recvbuff = (uint8_t*)recvbuff + gridOffset; this->sendMhandle = sendMhandle; this->recvMhandle = recvMhandle; this->srecvMhandle = srecvMhandle; } ~RingBCAlgorithm() {} }; #if !defined (__CUDA_ARCH__) || __CUDA_ARCH__ >= 600 #include #endif // Need a power of two to ensure it divides by parallelFactor (which is also a power of two) #define NCCL_PAT_NWORKERS 512 static constexpr int PatUsed = 0x1, PatSkipped = 0x2; struct ncclPatStep { int recvDim, sendDim, recvOffset, sendOffset, stepOffset, postRecv, postSend, nelem, last, flags; size_t inpIx, outIx; }; struct ncclPatPeer { uint64_t step; struct ncclConnInfo* conn; struct ncclConnFifo* connFifo; void* buff; uint64_t *headPtr; uint64_t *tailPtr; uint64_t stepCache; long long int accSize; int connStepSize; }; #define NCCL_SHMEM_PAT_STEPS 32 struct ncclPatShmem { struct ncclPatStep patSteps[NCCL_SHMEM_PAT_STEPS]; int parallelFactor; long long int localAccSize; struct ncclPatPeer sendDims[32]; // Should cover 2^32 ranks struct ncclPatPeer recvDims[32]; }; template class PatRSAlgorithm{ size_t offset; size_t end; size_t count; int chunkCount; int nelem; int rank; int nranks; int nrPow2; int postFreq; int lastA; int parallelFactor; int aggFactor; int as; // aggregated steps int a; // step inside aggregated step int sendSkipped; // number of skipped steps during aggregation int stepOffset; int aggDelta; int scale; int phase; __device__ __host__ ssize_t min(ssize_t a, ssize_t b) { return (a>=1) { if ((i&mask) == 0) ret += imask; } return ret; } __device__ __host__ int firstBitSet(int i, int max) { int ffs = #ifdef __CUDA_ARCH__ __ffs(i); #else COMPILER_FFS(i); #endif return ffs ? ffs-1 : max; } __device__ __host__ void resetA() { a = 0; sendSkipped = stepOffset = 0; lastA = aggFactor; if (phase >= 2) lastA /= 2*scale; if (phase == 4) lastA = 1; } __device__ __host__ void reset() { nelem = getNelem(); phase = 0; scale = 1; as = aggDelta - 1; resetA(); } __device__ __host__ int nBitsSet(int i) { int nbits = #ifdef __CUDA_ARCH__ __popc(i); #else COMPILER_POPCOUNT32(i); #endif return nbits; } // Return 1 when only upper bits are set. For example, if nrpow2==16 we'll return 1 for 8, 12, 14, 15. // A number being in the form of 1111000 implies that the complementary is 0000111 meaning it's a power of 2 minus 1. __device__ __host__ int newPeer(int i, int pow2) { //printf("New peer %d/%d -> %d\n", i, pow2, nBitsSet((i ^ (pow2-1)) + 1) == 1 ? 1 : 0); return nBitsSet((i ^ (pow2-1)) + 1) == 1 ? 1 : 0; } public: __device__ __host__ PatRSAlgorithm(int stepSize, int stepDepth, int maxParallelFactor, size_t offset, size_t end, size_t count, int chunkCount, int rank, int nranks): offset(offset), end(end), count(count), chunkCount(chunkCount), rank(rank), nranks(nranks) { parallelFactor = maxParallelFactor; aggDelta = nrPow2 = (1<= 2 && aggFactor < nranks/2) { aggFactor *= 2; aggDelta /= 2; } postFreq = aggFactor; if (postFreq < parallelFactor) parallelFactor = postFreq; int d = stepDepth; while (d > 1 && aggFactor < nranks/2) { d /= 2; aggFactor *= 2; aggDelta /= 2; } reset(); } __device__ __host__ int getParallelFactor() { return parallelFactor; } __device__ __host__ void getNextOp(struct ncclPatStep* ps) { ps->last = 0; ps->nelem = nelem; ps->outIx = offset; ps->stepOffset = stepOffset; int skip = 0; if (a >= lastA) { skip = 1; } else if (phase == 0) { int s = mirrorInvert(a, lastA)*aggDelta + as; if (s >= nranks) skip = 1; int sendDataRank = (rank + s) % nranks; ps->inpIx = sendDataRank * count + offset; ps->recvDim = -1; ps->sendDim = 0; ps->outIx = 0; ps->recvOffset = -1; ps->sendOffset = (a%postFreq) * nelem; if (((a%postFreq) + 1 >= postFreq) || (a == lastA-1)) { ps->postSend = 1; } else { ps->postSend = 0; } ps->postRecv = 0; } else if (phase == 1) { int s = mirrorInvert(a, lastA)*aggDelta + as; if (s >= nranks) skip = 1; ps->recvDim = firstBitSet(s, nrPow2); ps->sendOffset = (a%postFreq)*nelem; ps->recvOffset = (a%postFreq)*nelem; ps->postSend = 0; if (ps->recvDim == 0 && (((a%postFreq) + 1 >= postFreq) || (a == lastA-1))) ps->postSend = 1; if (((a%postFreq) + 1 >= postFreq) || (a == lastA-1)) { ps->postRecv = 1; } else { ps->postRecv = 0; } s -= (1<recvDim); int recvDataRank = (rank + nranks + s) % nranks; ps->inpIx = recvDataRank * count + offset; ps->sendDim = s ? firstBitSet(s, nrPow2) : -1; if (ps->sendDim == -1) { ps->sendOffset = -1; } else if (as - (1<recvDim) == 0) { if (newPeer(a, aggFactor)) { sendSkipped = a; ps->stepOffset = stepOffset = 0; } int foffset = a - sendSkipped; ps->sendOffset = (foffset%postFreq)*nelem; } int recvDim = ps->recvDim; if (s < nranks && skip) { ps->recvDim = -1; ps->recvOffset = -1; ps->postRecv = 0; skip = 0; } if (recvDim > 0 && (((a-sendSkipped)%postFreq) + 1 >= postFreq) && skip == 0) stepOffset++; } else if (phase == 2) { int s = (2*mirrorInvert(a, lastA)+1)*scale*aggDelta + 1; ps->postRecv = 0; if (s >= nranks) skip = 1; ps->recvDim = 0; ps->postSend = a == lastA-1 ? 1 : 0; s -= 1; if (s < nranks && skip) { ps->recvDim = -1; ps->recvOffset = -1; skip = 0; } else if (!skip) { int foffset = a + aggFactor - aggFactor/scale; ps->postRecv |= ((foffset+1)%postFreq) == 0 ? 1 : 0; ps->recvOffset = (foffset%postFreq) * nelem; } int recvDataRank = (rank + nranks + s) % nranks; ps->inpIx = recvDataRank * count + offset; ps->sendDim = s ? firstBitSet(s, nrPow2) : -1; int foffset = a; ps->postSend |= ((foffset+1)%postFreq) == 0 ? 1 : 0; ps->sendOffset = (foffset%postFreq) * nelem; } else if (phase == 3) { int s = (2*mirrorInvert(a, lastA)+1)*scale*aggDelta; ps->postRecv = a == lastA-1 ? 1 : 0; if (s >= nranks) skip = 1; ps->recvDim = firstBitSet(s, nrPow2); ps->postSend = 0; s -= (1<recvDim); int foffset = a; ps->postRecv |= (foffset+1)%postFreq == 0 ? 1 : 0; ps->recvOffset = (foffset%postFreq) * nelem; int recvDataRank = (rank + nranks + s) % nranks; ps->inpIx = recvDataRank * count + offset; ps->sendDim = s ? firstBitSet(s, nrPow2) : -1; if (s < nranks && skip) { ps->recvDim = -1; ps->recvOffset = -1; ps->postRecv = 0; skip = 0; } if (newPeer(a, aggFactor/(2*scale))) { sendSkipped = a; ps->stepOffset = stepOffset = 0; } foffset = a - sendSkipped; if ((foffset%postFreq) + 1 >= postFreq && skip == 0) stepOffset++; ps->sendOffset = ps->sendDim >= 0 ? (foffset%postFreq) * nelem : -1; } else if (phase == 4) { ps->recvDim = 0; ps->sendDim = -1; ps->inpIx = rank * count + offset; ps->recvOffset = ((aggFactor-1)%postFreq) * nelem; ps->sendOffset = -1; ps->postRecv = 1; ps->postSend = 0; offset += chunkCount; } a++; if (a >= lastA && a >= parallelFactor) { int p = phase; if (p == 1) as--; if (p == 3) scale *= 2; phase = p == 0 ? as == 1 ? (aggFactor > 1 ? 2 : 4) : 1 : p == 1 ? as % 2 == 1 ? 0 : 1 : p == 2 ? 3 : p == 3 ? scale < aggFactor ? 2 : 4 : 5; if (p == 4) { if (offset >= end) { ps->last = 2; } else { reset(); } } else { resetA(); } } else if (phase == 4 && offset >= end) { ps->last = 1; } int flags = PatUsed | (skip ? PatSkipped : 0); #if __CUDA_ARCH__ >= 600 cuda::atomic_ref a(ps->flags); a.store(flags, cuda::memory_order_release); #else ps->flags = flags; #endif } }; template class PatAGAlgorithm{ size_t offset; size_t end; size_t count; int chunkCount; int nelem; int rank; int nranks; int nrPow2; int postFreq; int lastA; int parallelFactor; int aggFactor; int as; // aggregated steps int a; // step inside aggregated step int aggDelta; int scale; int phase; // AS computation int asDim; int v; int bitCount[32]; int bitZeroStep[32]; __device__ __host__ ssize_t min(ssize_t a, ssize_t b) { return (a>=1) { if ((i&mask)) ret += imask; } return ret; } __device__ __host__ int firstBitSet(int i, int max) { int ffs = #ifdef __CUDA_ARCH__ __ffs(i); #else COMPILER_FFS(i); #endif return ffs ? ffs-1 : max; } __device__ __host__ void resetA() { a = 0; lastA = aggFactor; if (phase >= 2) lastA /= 2*scale; } __device__ __host__ void reset() { nelem = getNelem(); scale = aggFactor/2; phase = scale ? 2 : 1; v = 0; for (int i = 0; i= 2 && aggFactor < nranks/2) { aggFactor *= 2; aggDelta /= 2; } postFreq = aggFactor; if (postFreq < parallelFactor) parallelFactor = postFreq; int d = stepDepth; while (d > 1 && aggFactor < nranks/2) { d /= 2; aggFactor *= 2; aggDelta /= 2; } asDim = log2Up(aggDelta); reset(); } __device__ __host__ int getParallelFactor() { return parallelFactor; } __device__ __host__ void getNextOp(struct ncclPatStep* ps) { ps->last = 0; ps->nelem = nelem; ps->inpIx = offset; int skip = 0; if (a >= lastA) { skip = 1; } else if (phase == 0) { int s = a*aggDelta + as; if (s >= nranks) skip = 1; int recvDataRank = (rank + s) % nranks; ps->outIx = recvDataRank * count + offset; ps->sendDim = -1; ps->recvDim = 0; ps->inpIx = 0; ps->sendOffset = -1; ps->recvOffset = (a % postFreq) * nelem; ps->stepOffset = 0; ps->postRecv = (a % postFreq == postFreq-1) || ((a+1)*aggDelta+as >= nranks) ? 1 : 0; ps->postSend = 0; } else if (phase == 1) { int s = a*aggDelta + as; if (s >= nranks) skip = 1; ps->sendDim = firstBitSet(s, nrPow2); s -= (1<sendDim); int sendDataRank = (rank + nranks + s) % nranks; ps->outIx = sendDataRank * count + offset; ps->recvDim = s ? firstBitSet(s, nrPow2) : -1; ps->sendOffset = ps->recvOffset = (a % postFreq) * nelem; ps->postSend = (a % postFreq == postFreq-1) || ((a+1)*aggDelta+as >= nranks) ? 1 : 0; ps->postRecv = (ps->sendDim == 0) && ((a % postFreq == postFreq-1) || ((a+1)*aggDelta+as-1 >= nranks)) ? 1 : 0; ps->stepOffset = (ps->sendDim == 0) ? 0 : a/postFreq; if (ps->recvDim == -1) { ps->recvOffset = -1; ps->postRecv = 0; } else if (as - (1<sendDim) == 0) { int foffset = (a*aggDelta) >> (ps->recvDim+1); ps->recvOffset = (foffset%postFreq)*nelem; ps->postRecv = (ps->sendDim == 0) && ((foffset % postFreq == postFreq-1) || ((((foffset+1)*2)+1)<recvDim) >= nranks) ? 1 : 0; ps->stepOffset = (ps->sendDim == 0) ? 0 : foffset/postFreq; } if (s < nranks && ps->sendDim == 0 && skip) { // Don't forget to receive at least once even if we don't send afterwards ps->sendDim = -1; ps->sendOffset = -1; ps->postSend = 0; skip = 0; } } else if (phase == 2) { int s = (2*a+1)*scale*aggDelta; ps->postSend = (a % postFreq == postFreq-1) || ((2*(a+1)+1)*scale*aggDelta >= nranks) ? 1 : 0; ps->postRecv = 0; if (s >= nranks) skip = 1; ps->sendDim = firstBitSet(s, nrPow2); s -= (1<sendDim); ps->sendOffset = (a%postFreq) * nelem; ps->stepOffset = a / postFreq; int sendDataRank = (rank + nranks + s) % nranks; ps->outIx = sendDataRank * count + offset; ps->recvDim = s ? firstBitSet(s, nrPow2) : -1; if (ps->recvDim == -1) { ps->recvOffset = -1; } else { s -= (1<recvDim); int foffset = (a*2*scale*aggDelta) >> (ps->recvDim+1); ps->recvOffset = (foffset%postFreq)*nelem; ps->stepOffset = foffset / postFreq; } } a++; if (a >= lastA && a >= parallelFactor) { int p = phase; if (p == 2) scale /= 2; phase = p == 2 ? scale ? 2 : 1 : p == 1 ? as % 2 == 1 ? 0 : 1 : 1; if (p == 0 || (p == 1 && as % 2 == 0)) as = nextAs(); if (p == 0 && as == aggDelta/2) { offset += chunkCount; if (offset >= end) { ps->last = 2; } else { reset(); } } else { resetA(); } } else if (phase == 0 && as == 1 && offset + chunkCount >= end && a-1 >= ((lastA-1) / parallelFactor) * parallelFactor) { ps->last = 1; } int flags = PatUsed | (skip ? PatSkipped : 0); #if __CUDA_ARCH__ >= 600 cuda::atomic_ref a(ps->flags); a.store(flags, cuda::memory_order_release); #else ps->flags = flags; #endif } }; #endif nccl-2.29.7-1/src/include/comm.h000066400000000000000000000713021515037102200162050ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_COMM_H_ #define NCCL_COMM_H_ //#include "transport.h" #include "p2p.h" #include "collectives.h" #include "nccl_tuner.h" #include "proxy.h" #include "strongstream.h" #include "nccl_net.h" #include "register.h" #include "graph.h" #include "profiler.h" #include "allocator.h" #include "dev_runtime.h" #include "sym_kernels.h" #include "ce_coll.h" #include "rma/rma.h" #include "argcheck.h" #include "mem_manager.h" #if CUDART_VERSION < 9000 struct cudaLaunchParams { void *func; dim3 gridDim; dim3 blockDim; void **args; size_t sharedMem; cudaStream_t stream; }; #endif #define CACHE_LINE_SIZE 128 #define MEM_ALIGN 4096 #define CUDA_IPC_MIN 2097152UL // Channels / LL tuning #define NCCL_LL_THREAD_THRESHOLD 8 #define NCCL_LL128_THREAD_THRESHOLD 8 #define NCCL_SIMPLE_THREAD_THRESHOLD 64 struct ncclSendMem { union { struct { uint64_t head; char pad1[CACHE_LINE_SIZE-sizeof(uint64_t)]; void* ptrExchange; uint64_t redOpArgExchange[2]; char pad2[CACHE_LINE_SIZE-sizeof(void*)-2*sizeof(uint64_t)]; int offsFifo[NCCL_STEPS]; }; char pad3[MEM_ALIGN]; }; }; struct ncclRecvMem { union { struct { uint64_t tail; char pad1[CACHE_LINE_SIZE-sizeof(uint64_t)]; struct ncclConnFifo connFifo[NCCL_STEPS]; int flush; // For GDRCopy-based flush }; char pad4[MEM_ALIGN]; }; }; enum helperThreadState {ThreadStart, ThreadStop}; #define NCCL_IPC_POOL_SIZE (2*NCCL_MAX_LOCAL_RANKS*NCCL_MAX_OPS) struct ncclUserRedOp { int freeNext; // -1=allocated, otherwise index of next free entry in array ncclDataType_t datatype; ncclDevRedOpFull opFull; }; struct ncclNodeRanks { int localRanks; int* localRankToRank; }; struct cliqueInfo { int id; int size; int *ranks; }; struct ncclDestructor { struct ncclDestructor* next; void* obj; struct ncclComm* comm; ncclResult_t(*fn)(struct ncclDestructor* me); }; struct ncclCommCallback { struct ncclCommCallback* next; ncclResult_t(*fn)(struct ncclComm* comm, struct ncclCommCallback* cb); }; struct ncclCommEventCallback { struct ncclCommEventCallback* next; cudaEvent_t event; ncclResult_t(*fn)(struct ncclComm* comm, struct ncclCommEventCallback* cb); }; struct ncclSharedResources { int refCount; struct ncclComm* owner; /* comm which creates this shared res. */ struct ncclChannelPeer* peers[MAXCHANNELS]; struct ncclDevChannelPeer* devPeers[MAXCHANNELS]; /* P2P operation counter, one per channel */ uint64_t p2pOpCount[MAXCHANNELS]; /* Collective operation counter */ uint64_t collOpCount; int tpNRanks; int tpNLocalRanks; int tpNChannels; int tpP2pNChannels; int tpP2pChunkSize; uint64_t magic; // top parent rank to localRank translation table int* tpRankToLocalRank; // Internal streams struct ncclStrongStream deviceStream, hostStream; int persistentRefs; cudaEvent_t launchEvent, scratchEvent; /* proxy related shared res */ struct ncclProxyState* proxyState; // GIN state struct ncclGinState ginState; }; struct ncclChannel { struct ncclChannelPeer** peers; struct ncclDevChannelPeer** devPeers; /* devPeer pointer array used for host side access */ struct ncclDevChannelPeer** devPeersHostPtr; struct ncclRing ring; int* devRingUserRanks; struct ncclTree tree; struct ncclTree collnetChain; struct ncclDirect collnetDirect; struct ncclNvls nvls; int id; // index of this channel uint32_t workFifoProduced; // +1 successor of last used work fifo byte /* comm split sharable resources */ struct ncclChannelPeer* collnetPeers; struct ncclDevChannelPeer* collnetDevPeers; struct ncclChannelPeer* nvlsPeers; struct ncclDevChannelPeer* nvlsDevPeers; }; struct ncclWorkBatchList { struct ncclWorkBatchList* next; struct ncclDevWorkBatch batch; }; struct alignas(16) ncclWorkList { struct ncclWorkList* next; enum ncclDevWorkType workType; int size; // Size of struct following this node // ncclDevWorkColl, ncclDevWorkColLReg, ncclDevWorkP2p[]... }; struct ncclCollnetHandleList { struct ncclCollnetHandleList *next; void* collnetHandle; size_t size; const void* buffer; struct ncclProxyConnector* proxyconn; }; struct ncclTaskColl { struct ncclTaskColl* next; ncclFunc_t func; void const* sendbuff; void* recvbuff; size_t count; int root; ncclDataType_t datatype; ncclRedOp_t opHost; struct ncclDevRedOpFull opDev; int chunkSteps, sliceSteps; // Computed later: size_t trafficBytes; int32_t nMaxChannels:8; int32_t nWarps:8; int32_t algorithm:8, protocol:8; uint32_t isCollnet:1, isNvls:1, isSymLast:1; uint32_t devFuncId:29; int regBufType; // number of elements in planner->ipcMemQueue associated with this collective int nCleanupQueueElts; struct ncclDevrWindow* sendWin; struct ncclDevrWindow* recvWin; ncclSymRegType_t winRegType; void* sendMhandle; void* recvMhandle; void** sendNetHandles; void** recvNetHandles; void** srecvNetHandles; // index for IPC record lookup uintptr_t sendbuffOffset; uintptr_t recvbuffOffset; uintptr_t* sendbuffRmtAddrs; uintptr_t* recvbuffRmtAddrs; // Profiler plugin int eActivationMask; void* groupApiEventHandle; void* collApiEventHandle; void* eventHandle; uint8_t nChannels; }; struct ncclTaskBcast { struct ncclTaskBcast* next; ncclFunc_t func; void* recvbuff; const void* sendbuff; size_t count; ncclDataType_t datatype; int root; int ringDepth; // what computes after... int32_t algorithm:8, protocol:8; // Profiler plugin int eActivationMask; void* groupApiEventHandle; void* collApiEventHandle; void* eventHandle; uint8_t nChannels; }; struct ncclTaskP2p { struct ncclTaskP2p* next; ncclFunc_t func; ncclFunc_t collAPI; void* buff; size_t count; ncclDataType_t datatype; int root; size_t bytes; bool allowUB; // Profiler plugin int eActivationMask; void* groupApiEventHandle; void* p2pApiEventHandle; void* eventHandle; uint8_t nChannels; }; struct ncclTaskRma { struct ncclTaskRma* next; ncclFunc_t func; int ctx; size_t count; ncclDataType_t datatype; size_t bytes; void const* srcBuff; size_t srcWinOffset; struct ncclDevrWindow* srcWinHost; int peer; size_t peerWinOffset; struct ncclDevrWindow* peerWinHost; // Signal operations ncclSignalMode_t signalMode; int*peers; int*nsignals; int npeers; // Profiler plugin int eActivationMask; void* groupApiEventHandle; void* rmaApiEventHandle; void* eventHandle; uint8_t nChannels; }; struct ncclKernelPlan { // A kernel plan is also a callback that reclaims itself. Hence this must // be the first member. struct ncclCommCallback reclaimer; struct ncclComm* comm; struct ncclKernelPlan* next; bool persistent; // aka captured in a graph bool isHostCbEnq; bool isSymColl; bool isCeColl; bool isRma; enum ncclDevWorkStorageType workStorageType; bool kernelSpecialized; int kernelDynSmem; // only for symmetric kernels void* kernelFn; union { struct ncclDevKernelArgs* kernelArgs; void* kernelSymArgs; struct ncclCeCollArgs* ceCollArgs; struct ncclRmaArgs* rmaArgs; }; size_t kernelArgsSize; uint64_t channelMask; // bitset of which channels are present bool hasProxyOps; // does any channel have a non-empty proxyOpQueue int threadPerBlock; int collOpCount; // Number of collectives in this plan. int nWorkBatches; // Number of work batches. int nTasksBcast; // Number of bcast tasks in this plan. size_t workBytes; // Sum size of all work (in the fifo) in bytes. struct ncclIntruQueue workQueue; struct ncclIntruQueue cleanupQueue; void* workBufPersistent; struct ncclIntruQueue p2pTaskQueue; struct ncclIntruQueue bcastTaskQueue; struct ncclIntruQueue rmaTaskQueueProxy; struct ncclIntruQueue rmaTaskQueueCe; struct ncclIntruQueue collTaskQueue; struct ncclIntruQueue proxyOpQueue; // Profiler plugin void* groupApiEventHandle; void* kernelLaunchEventHandle; void* groupEventHandle; }; //////////////////////////////////////////////////////////////////////////////// // Roughly sorts ncclTaskColl's by their size descending. This structure is // self-referential, meaning that pointers it contains internally may point // into the structure itself. This means that it is NOT memcpy-moveable: struct ncclTaskCollSorter { static constexpr int UnitLog2 = 10; // 1K static constexpr size_t UnitSize = 1<>UnitLog2, BitsPerPow2); bin = BinCount-1 - bin; // descending bin if (me->bins[bin] == nullptr) { if (me->binEdge <= bin) { me->binEdge = bin+1; me->bins[bin] = me->tail ? &me->tail->next : &me->head; me->tail = x; } else { // Find successor non-empty bin after this one. int succ = bin+1; while (me->bins[succ] == nullptr) succ++; // What was our successor's head's previous is now our head's previous. me->bins[bin] = me->bins[succ]; // The first node we insert is our tail, so that becomes our successor's // head's new previous. me->bins[succ] = &x->next; } } // Push a new head for this bin. x->next = *me->bins[bin]; *me->bins[bin] = x; } inline bool ncclTaskCollSorterEmpty(struct ncclTaskCollSorter* me) { return me->head == nullptr; } // Reset sorter and return sorted linked list of its coll tasks. inline struct ncclTaskColl* ncclTaskCollSorterDequeueAll(struct ncclTaskCollSorter* me) { struct ncclTaskColl* head = me->head; if (head != nullptr) memset(me, 0, sizeof(*me)); return head; } //////////////////////////////////////////////////////////////////////////////// struct ncclCudaStreamList { struct ncclCudaStreamList *next; cudaStream_t stream; }; struct ncclKernelPlanner { ////////////////////////////////////////////////////////////////////////////// // State for accumulating tasks between ncclGroupStart/End() ////////////////////////////////////////////////////////////////////////////// struct Peer { bool sendSeen, recvSeen; struct ncclIntruQueue sendQueue; struct ncclIntruQueue recvQueue; struct ncclIntruQueue bcastQueue; }; struct ncclTaskCollSorter collSorter; struct Peer* peers/*[nRanks]*/; int nTasksColl, nTasksP2p, nTasksBcast, nTasksRma; int nTasksP2pSend, nTasksP2pRecv; struct { int minBcastPeer; /* initialized to INT_MAX */ int maxBcastPeer; /* initialized to INT_MIN */ int BcastPeers; /* initialized to 0 */ } bcast_info; bool persistent; // The list of user streams aggregated over all tasks present. struct ncclCudaStreamList* streams; // The most recent user stream. Ignored if streams==nullptr cudaStream_t streamRecent; // The graph capturing all user streams or invalid if none. Thus we restrict the // user that all streams must be captured in the same graph or not captured // at all. Technically we could probably relax this, but that would mean // collecting a different `ncclTasks` per graph and one for non-graph. struct ncclCudaGraph capturingGraph; ////////////////////////////////////////////////////////////////////////////// // Lists of tasks to be assembled into plans. ////////////////////////////////////////////////////////////////////////////// struct ncclIntruQueue collTaskQueue; struct ncclIntruQueue collCeTaskQueue; struct ncclIntruQueue *rmaTaskQueues; // Per-context queue for RMA tasks struct ncclIntruQueue collSymTaskQueue; struct ncclIntruQueue collWorkQueue; struct ncclIntruQueue tmpCollWorkQueue; struct ncclIntruQueue collCleanupQueue; ////////////////////////////////////////////////////////////////////////////// // State for building current (Work-In-Progress) plan: ////////////////////////////////////////////////////////////////////////////// struct WipPlan { struct Channel { struct { int workBytes; // Sum size of work metadata referenced by this batch. int nP2ps; // Number of p2p works in this batch int nBcasts; // Number of bcast works in this batch int p2pEpoch; int p2pRounds[NCCL_MAX_DEV_WORK_P2P_PER_BATCH]; // which rounds are present in this batch. } wipBatch; // work-in-progress batch which will be next tail of workBatchQueue int nWorkBatchesP2p; // number of p2p batches for this channel. int nWorkBatchesBcast; // number of bcast batches for this channel. struct ncclIntruQueue workBatchQueue; struct ncclIntruQueue proxyOpQueue; } channels[MAXCHANNELS]; } wipPlan; ////////////////////////////////////////////////////////////////////////////// // State for launching built plans: ////////////////////////////////////////////////////////////////////////////// // List of kernel plans built form tasks. struct ncclIntruQueue planQueue; // First of the unlaunched kernels in `planQueue` struct ncclKernelPlan* unlaunchedPlansHead; }; #define NCCL_MAGIC 0x0280028002800280 // Nickel atomic number is 28. typedef enum ncclGroupTaskType { ncclGroupTaskTypeCollective = 0, ncclGroupTaskTypeSymRegister = 1, ncclGroupTaskTypeNum = 2, } ncclGroupTaskType_t; struct ncclCommSymTeams; // NCCL_CHECK_MODE=DEBUG_LOCAL/DEBUG_GLOBAL // ncclCheckModeDebugLocal : check the input args/pointers locally, it replaces ncclParamCheckPointers() // ncclCheckModeDebugGlobal : check the input args globally such as symmetric buffer check, etc. typedef enum ncclCheckMode { ncclCheckModeDefault = 0, ncclCheckModeDebugLocal = 1, ncclCheckModeDebugGlobal = 2, } ncclCheckMode_t; struct ncclComm { uint64_t startMagic; struct ncclMemoryStack memPermanent, memScoped; // List of destructors to run when comm is destructed struct ncclDestructor* destructorHead; struct ncclCudaContext* context; struct ncclSharedResources* sharedRes; /* map to top parent ranks. */ int* topParentRanks; int* topParentLocalRanks; struct ncclChannel channels[MAXCHANNELS]; struct ncclPeerInfo* peerInfo; struct ncclTopoSystem* topo; struct ncclProxyConnector* gproxyConn; struct ncclIntruQueue legacyRegCleanupQueue; bool peerInfoValid; float minNetBw; ncclNet_t* ncclNet; void* netContext; void* ginContext; int netPluginIndex; int ginPluginIndex; int ncclNetVer; ncclNetDeviceType netDeviceType; ncclCollNet_t* ncclCollNet; void* collNetContext; void* bootstrap; bool isGrow; // true if this comm is created via ncclCommGrow // Bitmasks for ncclTransportP2pSetup uint64_t* connectSend; uint64_t* connectRecv; struct ncclTopoGraph graphs[NCCL_NUM_ALGORITHMS]; int maxTreePattern; bool initAlgoChannels[NCCL_NUM_ALGORITHMS]; bool runtimeConn; // if dynamic connection is supported bool directMode; // if any process manages more than one local rank int cuMemSupport; uint64_t magic; // Magic number for all network communication. Not a security key -- only goal is to detect mismatches. uint64_t commHash; int rank; // my rank in the communicator int nRanks; // number of GPUs in communicator int cudaDev; // my cuda device index int nvmlDev; // my nvml device index int compCap; // compute capability of the GPU int minCompCap, maxCompCap; // min/max compute capability in the communicator int64_t busId; // my PCI bus ID in int format ncclAffinity cpuAffinity; // CPU affinity of the GPU int cudaArch; // matches __CUDA_ARCH__ of device int cpuArch; // architecture - As defined in src/include/graph.h, e.g. x86/arm/ppc/mixed int cpuVendor; // vendor - As defined in src/include/graph.h int node; int nNodes; int localRank; int localRanks; int maxLocalRanks; int minLocalRanks; int* rankToNode; int* rankToLocalRank; int* localRankToRank; // localRanks and localRanktoRank for all nodes struct ncclNodeRanks* nodeRanks; // MNNVL: Multi-Node NVLink int MNNVL; // true when MNNVL is available struct cliqueInfo clique; // Our MNNVL clique information int cliqueRank; // Our rank within the MNNVL clique // NVL Domain info ncclNvlDomainInfo_v5_t nvlDomainInfo; ncclCheckMode_t checkMode; bool dmaBufSupport; bool ccEnable; // Counter for tracking CUDA launches (P2P and collectives included) uint64_t opCount; // Collective operation counter uint64_t collOpCount; // Channels for collectives int nChannels; // connection nChannels int collChannels; // enqueue nChannels int nvlsChannels; // enqueue nChannels // all nvls heads stored to check if we can splitShare int nvlsHeads[MAXCHANNELS]; // Channels (per peer) for p2p int p2pnChannels; int p2pnChannelsPerPeer; int p2pSchedGroupSize; // Should this comm allocate LL buffers for network P2P connections? bool allocP2pNetLLBuffers; // Buffer sizes int buffSizes[NCCL_NUM_PROTOCOLS]; int p2pChunkSize; int nvlsChunkSize; // Tuner values ncclTunerConstants_t tunerConstants; ssize_t threadThresholds[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float latencies[NCCL_NUM_FUNCTIONS][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; float bandwidths[NCCL_NUM_FUNCTIONS][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; int maxThreads[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]; /* This attribute can indicate the states of communicators and return code of * asynchronous NCCL operations. */ ncclResult_t asyncResult; // Flag to ask NCCL kernels to abort uint32_t* abortFlag; uint32_t* abortFlagDev; int* abortFlagRefCount; uint32_t* childAbortFlag; uint32_t* childAbortFlagDev; uint32_t destroyFlag; uint32_t revokedFlag; // Device side of the communicator (for cudaFree's) struct ncclKernelComm* devComm; // actually = &ncclKernelCommAndChannels::comm uint32_t workArgsBytes; // max size of kernel args uint32_t workFifoBytes; // size of workFifoBuf, power of 2 void* workFifoBuf; void* workFifoBufDev; void* workFifoBufGdrHandle; // Monotonic number of bytes (mod 1<<32) sent to fifo. uint32_t workFifoProduced; uint32_t workFifoProducedLastRecorded; uint32_t workFifoConsumed; // Intra-process sync struct ncclComm* intraComm0; // leader of intra-process comms (self possible) struct ncclComm* intraNext; // next of intra-process comms, intraComm0 is head int intraRank; int intraRanks; uint32_t intraBarrierPhase; char intraPad1[64 - sizeof(uint64_t)]; uint64_t intraBarrierCounter; // only used if this is intraComm0 char intraPad2[64 - sizeof(uint64_t)]; uint64_t intraBarrierGate; // only used if this is intraComm0 struct ncclProxyState* proxyState; int proxyRefCountOld; /* store proxy post-atomic-sub refcount */ // Whether this communicator uses collNet bool isOneRPN; uint8_t collNetSupportMatrix[4/*sum,prod,max,min*/][ncclNumTypes]; int* collNetHeads; int collNetHeadsNum; int* collNetDenseToUserRank; int* collNetUserToDenseRank; /* sharable collNet proxy progress resource. */ struct ncclCollNetSharedRes* collNetSharedRes; // NVLink SHARP (NVLS) support int nvlsSupport; int nvlsRegSupport; /* sharable NVLS resource. */ struct ncclNvlsSharedRes* nvlsResources; // pools backed by comm->memPermanent struct ncclMemoryPool memPool_ncclTaskBcast; struct ncclMemoryPool memPool_ncclTaskColl; struct ncclMemoryPool memPool_ncclTaskP2p; struct ncclMemoryPool memPool_ncclTaskRma; struct ncclMemoryPool memPool_ncclProxyOp; struct ncclMemoryPool memPool_ncclKernelPlan; struct ncclMemoryPool memPool_ncclRmaProxyDesc; // Next comm in this thread's active ncclGroup[Start|End](). Holds "0x1" when // this comm is not yet in a group. struct ncclComm* groupNext[ncclGroupTaskTypeNum]; // Subset of those in groupNext list. Holds 0x1 if not needing preconnect. struct ncclComm* preconnectNext; int localPersistentRefs; // number of persistent plan-lists capturing this comm struct P2pSchedulePair { int sendRank; int recvRank; } *p2pSchedule; struct ncclKernelPlanner planner; void* ringTasks; // An array of nRanks pointers used in ring sorting rooted collectives (bcast) cudaMemPool_t memPool; // Queue of events and associated callbacks for cleaning up asynchronous work. // Using this is preferable to using CUDA host callbacks because host callbacks // won't allow the work following the callback to run until the callback completes, // which comes at expense to perf. struct ncclIntruQueue eventCallbackQueue; // user-created reduction ops int userRedOpCapacity, userRedOpFreeHead; ncclUserRedOp *userRedOps; // Queue of things for the main thread to do int reclaimSteps; struct ncclIntruQueueMpsc callbackQueue; ncclConfig_t config; // initState is to more conveniently reclaim resources when errors happen. ncclResult_t initState; // flag to indicate if ncclCommFinalize() is called bool finalizeCalled; // shared structures for finalization int finalizeRankCnt; // group job to support multi-thread FT struct ncclGroupJob *groupJob; // Flag indicating if this communicator shares resources with parent or children bool shareResources; // Tuning plugin int tunerPluginLoaded; ncclTuner_t* tuner; void *tunerContext; // Profiler plugin void* profilerContext; uint64_t seqNumber[NCCL_NUM_FUNCTIONS]; struct ncclProfilerProxy profiler; // RMA state struct ncclRmaState rmaState; struct ncclIntruQueue rmaCeInitTaskQueue; // Debug check struct ncclIntruQueue argsInfoQueue; // CE Collective struct ncclCeColl ceColl; struct ncclIntruQueue ceInitTaskQueue; // buffer registration cache struct ncclRegCache regCache; int isAllNvlink; bool isAllDirectP2p; // Subject to NCCL_P2P_LEVEL (for local ranks only). bool isAllCudaP2p; // Raw CUDA capability (for local ranks only). bool isAllDirectNvlink; // All GPUs are directly connected to each other through NVLink. int symmetricSupport; bool useNetPXN; bool useGdr; ncclGinConnectionType_t globalGinSupport; bool globalRmaProxySupport; bool hostRmaSupport; int childCount; struct ncclDevrState devrState; // The symmetric runtime state struct ncclSymkState symkState; // The symmetric kernels state (built on previous) struct ncclMemManager* memManager; // Memory manager uint64_t endMagic; }; static_assert(offsetof(struct ncclComm, startMagic) == 0, "startMagic must be the first field of ncclComm"); static_assert(offsetof(struct ncclComm, endMagic) == sizeof(struct ncclComm) - sizeof(uint64_t), "endMagic must be the last field of ncclComm"); enum ncclLaunchMode { ncclLaunchModeInvalid=0, ncclLaunchModeParallel, ncclLaunchModeGroup }; extern enum ncclLaunchMode ncclParamLaunchMode; void ncclCommPushFree(struct ncclComm* comm, void* buf); void ncclCommPushCudaFree(struct ncclComm* comm, void* buf); void ncclCommPushCudaHostFree(struct ncclComm* comm, void* buf); void ncclCommPushCudaGdrFree(struct ncclComm* comm, void* handle); inline ncclResult_t ncclCommPollCallbacks(struct ncclComm* comm, bool waitSome) { ncclResult_t result = ncclSuccess; struct ncclCommCallback* cb = ncclIntruQueueMpscDequeueAll(&comm->callbackQueue, waitSome); while (cb != nullptr) { struct ncclCommCallback* next = cb->next; ncclResult_t res1 = cb->fn(comm, cb); // may reclaim memory of cb if (res1 != ncclSuccess) result = res1; cb = next; } NCCLCHECK(result); return ncclSuccess; } inline ncclResult_t ncclCommPollEventCallbacks(struct ncclComm *comm, bool waitSome) { ncclResult_t result = ncclSuccess; cudaStreamCaptureMode mode = cudaStreamCaptureModeRelaxed; CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); while (true) { struct ncclCommEventCallback* cb = ncclIntruQueueHead(&comm->eventCallbackQueue); if (cb == nullptr) break; cudaError_t ok; if (waitSome) { ok = cudaEventSynchronize(cb->event); waitSome = false; } else { ok = cudaEventQuery(cb->event); if (ok == cudaErrorNotReady) break; } ncclIntruQueueDequeue(&comm->eventCallbackQueue); if (ok == cudaSuccess) { NCCLCHECKGOTO(cb->fn(comm, cb), result, finish); } else { CUDACHECKGOTO(ok, result, finish); } } finish: CUDACHECK(cudaThreadExchangeStreamCaptureMode(&mode)); return ncclSuccess; } inline void ncclCommIntraBarrierIn(struct ncclComm* comm, uint32_t x) { int phase = comm->intraBarrierPhase; if (comm->intraRanks == 1) { // Release everyone (just me). comm->intraBarrierGate = (uint64_t(x)<<32) | (phase^1); } else { struct ncclComm* comm0 = comm->intraComm0; uint64_t count = COMPILER_ATOMIC_ADD_FETCH(&comm0->intraBarrierCounter, (uint64_t(x)<<32) + 1, std::memory_order_release); if (uint32_t(count) == uint32_t(comm->intraRanks)) { // Reset. COMPILER_ATOMIC_STORE(&comm0->intraBarrierCounter, 0, std::memory_order_relaxed); // Release everyone. COMPILER_ATOMIC_STORE(&comm0->intraBarrierGate, (count>>32<<32) | (phase^1), std::memory_order_release); } } } // returns sum of x values contributed to ncclCommIntraBarrierIn(comm, x) inline uint32_t ncclCommIntraBarrierOut(struct ncclComm* comm) { struct ncclComm* comm0 = comm->intraComm0; comm->intraBarrierPhase ^= 1; uint32_t phase = comm->intraBarrierPhase; uint64_t gate = COMPILER_ATOMIC_LOAD(&comm0->intraBarrierGate, std::memory_order_relaxed); if ((gate & 1) != phase) { uint64_t t0 = clockNano(); do { // Spin vigorously for first 5us. if (clockNano()-t0 >= 5*1000) std::this_thread::yield(); gate = COMPILER_ATOMIC_LOAD(&comm0->intraBarrierGate, std::memory_order_relaxed); } while ((gate & 1) != phase); } if (comm->intraRanks != 1) std::atomic_thread_fence(std::memory_order_acquire); return gate>>32; } // Scrambles the bits of non-builtin values of ncclRedOp_t according to the // communicator memory address. Used to catch bugs so that integer handles // associated with this communicator won't collide with handles of other // communicatrs. This function is its own inverse. static inline ncclRedOp_t ncclUserRedOpMangle(ncclComm *comm, ncclRedOp_t op) { // Preserve the built-in values. if(int(op) < int(ncclNumOps)) return op; uint64_t h = reinterpret_cast(comm); h ^= h >> 32; h *= 0x9e3779b97f4a7c13u; // Knuth's 64-bit magical hash constant h >>= 32; // h is now an excellent 32-bit hash of the comm pointer h &= int(ncclMaxRedOp); // ncclMaxRedOp is a power of 2 minus 1 int op1 = int(h) ^ int(op); // Since builtin values are preserved, we also have to preserve their preimage. return op1 < int(ncclNumOps) ? op : ncclRedOp_t(op1); } ncclResult_t ncclCommEnsureReady(ncclComm_t comm); ncclResult_t ncclCommSetAsyncError(ncclComm_t comm, ncclResult_t nextState); #endif nccl-2.29.7-1/src/include/compiler.h000066400000000000000000000014521515037102200170630ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_PORTABLE_INTRINSICS_H #define NCCL_PORTABLE_INTRINSICS_H #ifdef __cplusplus extern "C++" { #endif #include #ifdef __cplusplus } #endif // Compiler detection macros #if defined(__GNUC__) || defined(__clang__) #define NCCL_COMPILER_GCC 1 #include "compiler/gcc.h" #elif defined(_MSC_VER) #define NCCL_COMPILER_MSVC 1 #include "compiler/msvc.h" #else #error "Unsupported compiler" #endif #endif // NCCL_PORTABLE_INTRINSICS_H nccl-2.29.7-1/src/include/compiler/000077500000000000000000000000001515037102200167105ustar00rootroot00000000000000nccl-2.29.7-1/src/include/compiler/gcc.h000066400000000000000000000054621515037102200176240ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_COMPILER_GCC_H #define NCCL_COMPILER_GCC_H // Helper macros to convert C++ memory ordering to GCC atomic ordering #define NCCL_CONVERT_ORDER(order) \ ((order) == std::memory_order_relaxed ? __ATOMIC_RELAXED : \ (order) == std::memory_order_consume ? __ATOMIC_CONSUME : \ (order) == std::memory_order_acquire ? __ATOMIC_ACQUIRE : \ (order) == std::memory_order_release ? __ATOMIC_RELEASE : \ (order) == std::memory_order_acq_rel ? __ATOMIC_ACQ_REL : \ (order) == std::memory_order_seq_cst ? __ATOMIC_SEQ_CST : \ __ATOMIC_SEQ_CST) #define COMPILER_ATOMIC_LOAD(ptr, order) \ __atomic_load_n((ptr), NCCL_CONVERT_ORDER(order)) #define COMPILER_ATOMIC_LOAD_DEST(ptr, dest, order) do { \ __atomic_load((ptr), (dest), NCCL_CONVERT_ORDER(order)); \ } while(0) #define COMPILER_ATOMIC_STORE(ptr, val, order) \ __atomic_store_n((ptr), (val), NCCL_CONVERT_ORDER(order)) #define COMPILER_ATOMIC_EXCHANGE(ptr, val, order) \ __atomic_exchange_n((ptr), (val), NCCL_CONVERT_ORDER(order)) #define COMPILER_ATOMIC_COMPARE_EXCHANGE(ptr, expected, desired, success_order, failure_order) \ __atomic_compare_exchange_n((ptr), (expected), (desired), true, NCCL_CONVERT_ORDER(success_order), NCCL_CONVERT_ORDER(failure_order)) #define COMPILER_ATOMIC_FETCH_ADD(ptr, val, order) __atomic_fetch_add((ptr), (val), NCCL_CONVERT_ORDER(order)) #define COMPILER_ATOMIC_ADD_FETCH(ptr, val, order) __atomic_add_fetch((ptr), (val), NCCL_CONVERT_ORDER(order)) #define COMPILER_ATOMIC_SUB_FETCH(ptr, val, order) __atomic_sub_fetch((ptr), (val), NCCL_CONVERT_ORDER(order)) #define COMPILER_PREFETCH(addr) __builtin_prefetch((addr)) #define COMPILER_POPCOUNT32(x) __builtin_popcount(x) #define COMPILER_POPCOUNT64(x) __builtin_popcountll(x) #define COMPILER_EXPECT(x, v) __builtin_expect((x), (v)) // Find First Set (FFS) - returns index of first set bit (1-indexed), 0 if no bits set #define COMPILER_FFS(x) __builtin_ffs(x) #define COMPILER_FFSL(x) __builtin_ffsl(x) #define COMPILER_FFSLL(x) __builtin_ffsll(x) // Count Leading Zeros (CLZ) - undefined behavior if x == 0 #define COMPILER_CLZ(x) __builtin_clz(x) #define COMPILER_CLZL(x) __builtin_clzl(x) #define COMPILER_CLZLL(x) __builtin_clzll(x) // Byte Swap #define COMPILER_BSWAP16(x) __builtin_bswap16(x) #define COMPILER_BSWAP32(x) __builtin_bswap32(x) #define COMPILER_BSWAP64(x) __builtin_bswap64(x) // Compiler hints #define COMPILER_ASSUME_ALIGNED(ptr, alignment) __builtin_assume_aligned((ptr), (alignment)) #endif // NCCL_COMPILER_GCC_H nccl-2.29.7-1/src/include/compiler/msvc.h000066400000000000000000000233041515037102200200330ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_COMPILER_MSVC_H #define NCCL_COMPILER_MSVC_H #include #include // TODO: Check the memory orders and the fences static inline long COMPILER_ATOMIC_LOAD_impl(volatile long* ptr, std::memory_order order) { long result; if (order == std::memory_order_relaxed) { result = InterlockedAdd64NoFence(ptr, 0); } else if (order == std::memory_order_acquire) { result = InterlockedAdd64Acquire(ptr, 0); } else if (order == std::memory_order_release) { result = InterlockedAdd64Release(ptr, 0); } else if (order == std::memory_order_acq_rel) { result = InterlockedAdd64AcquireRelease(ptr, 0); } else if (order == std::memory_order_seq_cst) { result = InterlockedAdd64(ptr, 0); } return result; } #define COMPILER_ATOMIC_LOAD(ptr, order) COMPILER_ATOMIC_LOAD_impl((volatile long*)(ptr), (order)) #define COMPILER_ATOMIC_LOAD_DEST(ptr, dest, order) do { \ if (order == std::memory_order_relaxed) { \ *(dest) = InterlockedAdd64NoFence((volatile long*)(ptr), 0); \ } else if (order == std::memory_order_acquire) { \ *(dest) = InterlockedAdd64Acquire((volatile long*)(ptr), 0); \ } else if (order == std::memory_order_release) { \ *(dest) = InterlockedAdd64Release((volatile long*)(ptr), 0); \ } else if (order == std::memory_order_acq_rel) { \ *(dest) = InterlockedAdd64AcquireRelease((volatile long*)(ptr), 0); \ } else if (order == std::memory_order_seq_cst) { \ *(dest) = InterlockedAdd64((volatile long*)(ptr), 0); \ } \ } while(0) #define COMPILER_ATOMIC_STORE(ptr, val, order) do { \ if (order == std::memory_order_relaxed) { \ InterlockedExchange64NoFence((volatile long*)(ptr), (val)); \ } else if (order == std::memory_order_acquire) { \ InterlockedExchange64Acquire((volatile long*)(ptr), (val)); \ } else if (order == std::memory_order_release) { \ InterlockedExchange64Release((volatile long*)(ptr), (val)); \ } else if (order == std::memory_order_acq_rel) { \ InterlockedExchange64AcquireRelease((volatile long*)(ptr), (val)); \ } else if (order == std::memory_order_seq_cst) { \ InterlockedExchange64((volatile long*)(ptr), (val)); \ } \ } while(0) // COMPILER_ATOMIC_EXCHANGE - Atomic exchange, returns old value static inline long COMPILER_ATOMIC_EXCHANGE_impl(volatile long* ptr, long val, std::memory_order order) { long result; if (order == std::memory_order_relaxed) { result = InterlockedExchange64NoFence(ptr, val); } else if (order == std::memory_order_acquire) { result = InterlockedExchange64Acquire(ptr, val); } else if (order == std::memory_order_release) { result = InterlockedExchange64Release(ptr, val); } else if (order == std::memory_order_acq_rel) { result = InterlockedExchange64AcquireRelease(ptr, val); } else if (order == std::memory_order_seq_cst) { result = InterlockedExchange64(ptr, val); } return result; } #define COMPILER_ATOMIC_EXCHANGE(ptr, val, order) COMPILER_ATOMIC_EXCHANGE_impl((volatile long*)(ptr), (val), (order)) // COMPILER_ATOMIC_COMPARE_EXCHANGE - Compare and exchange // Note: To match GCC API, 'desired' is passed by value, 'expected' is a pointer // Returns true if exchange succeeded, updates *expected with old value on failure static inline bool COMPILER_ATOMIC_COMPARE_EXCHANGE_impl( volatile long* ptr, long* expected, long desired, std::memory_order success_order, std::memory_order failure_order) { long old; if (success_order == std::memory_order_relaxed) { old = InterlockedCompareExchange64NoFence(ptr, desired, *expected); } else if (success_order == std::memory_order_acquire) { old = InterlockedCompareExchange64Acquire(ptr, desired, *expected); } else if (success_order == std::memory_order_release) { old = InterlockedCompareExchange64Release(ptr, desired, *expected); } else if (success_order == std::memory_order_acq_rel) { old = InterlockedCompareExchange64AcquireRelease(ptr, desired, *expected); } else if (success_order == std::memory_order_seq_cst) { old = InterlockedCompareExchange64(ptr, desired, *expected); } bool success = (old == *expected); if (!success) { *expected = old; } // Use appropriate fence based on success/failure std::atomic_thread_fence(success ? success_order : failure_order); return success; } #define COMPILER_ATOMIC_COMPARE_EXCHANGE(ptr, expected, desired, success_order, failure_order) \ COMPILER_ATOMIC_COMPARE_EXCHANGE_impl((volatile long*)(ptr), (expected), (desired), (success_order), (failure_order)) // COMPILER_ATOMIC_ADD_FETCH - Add and return new value static inline long COMPILER_ATOMIC_ADD_FETCH_impl(volatile long* ptr, long val, std::memory_order order) { long result; if (order == std::memory_order_relaxed) { result = InterlockedAdd64NoFence(ptr, val); } else if (order == std::memory_order_acquire) { result = InterlockedAdd64Acquire(ptr, val); } else if (order == std::memory_order_release) { result = InterlockedAdd64Release(ptr, val); } else if (order == std::memory_order_acq_rel) { result = InterlockedAdd64AcquireRelease(ptr, val); } else if (order == std::memory_order_seq_cst) { result = InterlockedAdd64(ptr, val); } return result; } // COMPILER_ATOMIC_FETCH_ADD - Fetch old value then add static inline long COMPILER_ATOMIC_FETCH_ADD_impl(volatile long* ptr, long val, std::memory_order order) { long result; if (order == std::memory_order_relaxed) { result = InterlockedExchangeAdd64NoFence(ptr, val); } else if (order == std::memory_order_acquire) { result = InterlockedExchangeAdd64Acquire(ptr, val); } else if (order == std::memory_order_release) { result = InterlockedExchangeAdd64Release(ptr, val); } else if (order == std::memory_order_acq_rel) { result = InterlockedExchangeAdd64AcquireRelease(ptr, val); } else if (order == std::memory_order_seq_cst) { result = InterlockedExchangeAdd64(ptr, val); } return result; } #define COMPILER_ATOMIC_FETCH_ADD(ptr, val, order) COMPILER_ATOMIC_FETCH_ADD_impl((volatile long*)(ptr), (val), (order)) #define COMPILER_ATOMIC_ADD_FETCH(ptr, val, order) COMPILER_ATOMIC_ADD_FETCH_impl((volatile long*)(ptr), (val), (order)) // COMPILER_ATOMIC_SUB_FETCH - Subtract and return new value static inline long COMPILER_ATOMIC_SUB_FETCH_impl(volatile long* ptr, long val, std::memory_order order) { long result; if (order == std::memory_order_relaxed) { result = InterlockedAdd64NoFence(ptr, -val); } else if (order == std::memory_order_acquire) { result = InterlockedAdd64Acquire(ptr, -val); } else if (order == std::memory_order_release) { result = InterlockedAdd64Release(ptr, -val); } else if (order == std::memory_order_acq_rel) { result = InterlockedAdd64AcquireRelease(ptr, -val); } else if (order == std::memory_order_seq_cst) { result = InterlockedAdd64(ptr, -val); } return result; } #define COMPILER_ATOMIC_SUB_FETCH(ptr, val, order) COMPILER_ATOMIC_SUB_FETCH_impl((volatile long*)(ptr), (val), (order)) // Prefetch #define COMPILER_PREFETCH(addr) _mm_prefetch((const char*)(addr), _MM_HINT_T0) // Population count #define COMPILER_POPCOUNT32(x) __popcnt(x) #define COMPILER_POPCOUNT64(x) __popcnt64(x) // Branch prediction hints (MSVC doesn't support these, so no-op) #define COMPILER_EXPECT(x, v) (x) // Find First Set (FFS) - returns index of first set bit (1-indexed), 0 if no bits set // MSVC uses _BitScanForward which gives 0-indexed position inline int NCCL_FFS_impl(int x) { unsigned long index; return _BitScanForward(&index, (unsigned long)x) ? (int)(index + 1) : 0; } inline int NCCL_FFSl_impl(long x) { unsigned long index; return _BitScanForward(&index, (unsigned long)x) ? (int)(index + 1) : 0; } inline int NCCL_FFSll_impl(long long x) { unsigned long index; #ifdef _WIN64 return _BitScanForward64(&index, (unsigned __int64)x) ? (int)(index + 1) : 0; #else // For 32-bit, check low 32 bits first, then high 32 bits if (_BitScanForward(&index, (unsigned long)(x & 0xFFFFFFFF))) return (int)(index + 1); if (_BitScanForward(&index, (unsigned long)(x >> 32))) return (int)(index + 33); return 0; #endif } #define COMPILER_FFS(x) NCCL_FFS_impl(x) #define COMPILER_FFSL(x) NCCL_FFSl_impl(x) #define COMPILER_FFSLL(x) NCCL_FFSll_impl(x) // Count Leading Zeros (CLZ) - undefined behavior if x == 0 (to match GCC behavior) // MSVC uses _BitScanReverse which gives position of highest bit inline int nccl_clz_impl(unsigned int x) { unsigned long index; _BitScanReverse(&index, x); return 31 - (int)index; } inline int nccl_clzl_impl(unsigned long x) { unsigned long index; _BitScanReverse(&index, x); return (8 * sizeof(unsigned long) - 1) - (int)index; } inline int nccl_clzll_impl(unsigned long long x) { unsigned long index; #ifdef _WIN64 _BitScanReverse64(&index, (unsigned __int64)x); return 63 - (int)index; #else // For 32-bit, check high 32 bits first, then low 32 bits if (_BitScanReverse(&index, (unsigned long)(x >> 32))) return 31 - (int)index; _BitScanReverse(&index, (unsigned long)(x & 0xFFFFFFFF)); return 63 - (int)index; #endif } #define COMPILER_CLZ(x) nccl_clz_impl(x) #define COMPILER_CLZL(x) nccl_clzl_impl(x) #define COMPILER_CLZLL(x) nccl_clzll_impl(x) // Byte Swap #define COMPILER_BSWAP16(x) _byteswap_ushort(x) #define COMPILER_BSWAP32(x) _byteswap_ulong(x) #define COMPILER_BSWAP64(x) _byteswap_uint64(x) // Compiler hints // TODO: Check if __declspec(align(alignment)) can be used #define COMPILER_ASSUME_ALIGNED(ptr, alignment) (ptr) #endif // NCCL_COMPILER_MSVC_H nccl-2.29.7-1/src/include/core.h000066400000000000000000000023401515037102200161760ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_CORE_H_ #define NCCL_CORE_H_ #include #include #include // For std::min/std::max #include "nccl.h" #ifdef PROFAPI #define NCCL_API(ret, func, args...) \ extern "C" \ __attribute__ ((visibility("default"))) \ __attribute__ ((alias(#func))) \ ret p##func (args); \ extern "C" \ __attribute__ ((visibility("default"))) \ __attribute__ ((weak)) \ ret func(args) #else #define NCCL_API(ret, func, args...) \ extern "C" \ __attribute__ ((visibility("default"))) \ ret func(args) #endif // end PROFAPI #include "debug.h" #include "checks.h" #include "cudawrap.h" #include "alloc.h" #include "utils.h" #include "param.h" #include "nvtx.h" #endif // end include guard nccl-2.29.7-1/src/include/cpuset.h000066400000000000000000000062231515037102200165550ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2018-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_CPUSET_H_ #define NCCL_CPUSET_H_ #include "nccl.h" #include #include #include #ifdef NCCL_OS_LINUX #include #elif defined(NCCL_OS_WINDOWS) #define CPU_SETSIZE 64 // Windows uses DWORD_PTR for affinity #endif // Convert local_cpus, e.g. 0003ff,f0003fff to ncclAffinity. // The bitmask is divided into chunks of 32 bits, each of them represented by 8 hex number. #define U32_LEN 32 // using uint32_t #define CPU_SET_N_U32 (CPU_SETSIZE / U32_LEN) static ncclResult_t ncclStrToCpuset(const char* maskStr, ncclAffinity* set) { uint32_t cpumasks[CPU_SET_N_U32] = {0}; // transform the string into an array of 32 bit masks, starting with the highest mask int m = CPU_SET_N_U32; char* str = strdup(maskStr); char* token = strtok(str, ","); while (token != NULL && m > 0) { cpumasks[--m] = strtoul(token, NULL, /*base = hex*/ 16); token = strtok(NULL, ","); } free(str); // list all the CPUs as part of the CPU set, starting with the lowest mask (= current value of m) ncclOsCpuZero(*set); for (int a = 0; (a + m) < CPU_SET_N_U32; a++) { // each mask is U32_LEN CPUs, list them all if the bit is on for (int i = 0; i < U32_LEN; ++i) { if (cpumasks[a + m] & (1UL << i)) ncclOsCpuSet(*set, i + a * U32_LEN); } } return ncclSuccess; } static char* ncclCpusetToRangeStr(ncclAffinity* mask, char* str, size_t len) { int c = 0; int start = -1; // Iterate through all possible CPU bits plus one extra position for (int cpu = 0; cpu <= CPU_SETSIZE; cpu++) { int isSet = (cpu == CPU_SETSIZE) ? 0 : ncclOsCpuIsSet(*mask, cpu); // Start of a new range if (isSet && start == -1) { start = cpu; } // End of a range, add comma between ranges if (!isSet && start != -1) { if (cpu-1 == start) { c += snprintf(str+c, len-c, "%s%d", c ? "," : "", start); } else { c += snprintf(str+c, len-c, "%s%d-%d", c ? "," : "", start, cpu-1); } if (c >= len-1) break; start = -1; } } if (c == 0) str[0] = '\0'; return str; } static ncclResult_t ncclStrListToCpuset(const char* userStr, ncclAffinity* mask) { // reset the CPU set ncclOsCpuZero(*mask); const char delim[] = ","; char* str = strdup(userStr); char* token = strtok(str, delim); while (token != NULL) { uint64_t cpu = strtoull(token, NULL, 0); ncclOsCpuSet(*mask, cpu); token = strtok(NULL, delim); } free(str); return ncclSuccess; } static ncclResult_t ncclCpusetToStrList(ncclAffinity* mask, char* str, size_t len) { if (len == 0) return ncclSuccess; str[0] = '\0'; int count = 0; for (uint64_t id = 0; id < CPU_SETSIZE; ++id) { if (ncclOsCpuIsSet(*mask, id)) { snprintf(str + strlen(str), len - strlen(str), "%s%lu", (count++ == 0) ? "" : ",", id); } } return ncclSuccess; } #endif nccl-2.29.7-1/src/include/cudawrap.h000066400000000000000000000133461515037102200170640ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_CUDAWRAP_H_ #define NCCL_CUDAWRAP_H_ #include #include #include "checks.h" #include "compiler.h" // Is cuMem API usage enabled extern int ncclCuMemEnable(); extern int ncclCuMemHostEnable(); #if CUDART_VERSION >= 11030 #include // Handle type used for cuMemCreate() extern CUmemAllocationHandleType ncclCuMemHandleType; #endif #define CUPFN(symbol) pfn_##symbol // Check CUDA PFN driver calls #define CUCHECK(cmd) do { \ CUresult err = pfn_##cmd; \ if( err != CUDA_SUCCESS ) { \ const char *errStr; \ (void) pfn_cuGetErrorString(err, &errStr); \ WARN("Cuda failure %d '%s'", err, errStr); \ return ncclUnhandledCudaError; \ } \ } while(false) #define CUCALL(cmd) do { \ pfn_##cmd; \ } while(false) #define CUCHECKGOTO(cmd, res, label) do { \ CUresult err = pfn_##cmd; \ if( err != CUDA_SUCCESS ) { \ const char *errStr; \ (void) pfn_cuGetErrorString(err, &errStr); \ WARN("Cuda failure %d '%s'", err, errStr); \ res = ncclUnhandledCudaError; \ goto label; \ } \ } while(false) // Report failure but clear error and continue #define CUCHECKIGNORE(cmd) do { \ CUresult err = pfn_##cmd; \ if( err != CUDA_SUCCESS ) { \ const char *errStr; \ (void) pfn_cuGetErrorString(err, &errStr); \ INFO(NCCL_ALL,"%s:%d Cuda failure %d '%s'", __FILE__, __LINE__, err, errStr); \ } \ } while(false) #define CUCHECKTHREAD(cmd, args) do { \ CUresult err = pfn_##cmd; \ if (err != CUDA_SUCCESS) { \ INFO(NCCL_INIT,"%s:%d -> %d [Async thread]", __FILE__, __LINE__, err); \ args->ret = ncclUnhandledCudaError; \ return args; \ } \ } while(0) #define DECLARE_CUDA_PFN_EXTERN(symbol,version) extern PFN_##symbol##_v##version pfn_##symbol #if CUDART_VERSION >= 11030 /* CUDA Driver functions loaded with cuGetProcAddress for versioning */ DECLARE_CUDA_PFN_EXTERN(cuDeviceGet, 2000); DECLARE_CUDA_PFN_EXTERN(cuDeviceGetAttribute, 2000); DECLARE_CUDA_PFN_EXTERN(cuGetErrorString, 6000); DECLARE_CUDA_PFN_EXTERN(cuGetErrorName, 6000); DECLARE_CUDA_PFN_EXTERN(cuMemGetAddressRange, 3020); DECLARE_CUDA_PFN_EXTERN(cuCtxCreate, 11040); DECLARE_CUDA_PFN_EXTERN(cuCtxDestroy, 4000); DECLARE_CUDA_PFN_EXTERN(cuCtxGetCurrent, 4000); DECLARE_CUDA_PFN_EXTERN(cuCtxSetCurrent, 4000); DECLARE_CUDA_PFN_EXTERN(cuCtxGetDevice, 2000); DECLARE_CUDA_PFN_EXTERN(cuPointerGetAttribute, 4000); DECLARE_CUDA_PFN_EXTERN(cuLaunchKernel, 4000); #if CUDART_VERSION >= 11080 DECLARE_CUDA_PFN_EXTERN(cuLaunchKernelEx, 11060); #endif // cuMem API support DECLARE_CUDA_PFN_EXTERN(cuMemAddressReserve, 10020); DECLARE_CUDA_PFN_EXTERN(cuMemAddressFree, 10020); DECLARE_CUDA_PFN_EXTERN(cuMemCreate, 10020); DECLARE_CUDA_PFN_EXTERN(cuMemGetAllocationGranularity, 10020); DECLARE_CUDA_PFN_EXTERN(cuMemExportToShareableHandle, 10020); DECLARE_CUDA_PFN_EXTERN(cuMemImportFromShareableHandle, 10020); DECLARE_CUDA_PFN_EXTERN(cuMemMap, 10020); DECLARE_CUDA_PFN_EXTERN(cuMemRelease, 10020); DECLARE_CUDA_PFN_EXTERN(cuMemRetainAllocationHandle, 11000); DECLARE_CUDA_PFN_EXTERN(cuMemSetAccess, 10020); DECLARE_CUDA_PFN_EXTERN(cuMemUnmap, 10020); DECLARE_CUDA_PFN_EXTERN(cuMemGetAllocationPropertiesFromHandle, 10020); #if CUDA_VERSION >= 11070 DECLARE_CUDA_PFN_EXTERN(cuMemGetHandleForAddressRange, 11070); // DMA-BUF support #endif #if CUDA_VERSION >= 12010 /* NVSwitch Multicast support */ DECLARE_CUDA_PFN_EXTERN(cuMulticastAddDevice, 12010); DECLARE_CUDA_PFN_EXTERN(cuMulticastBindMem, 12010); DECLARE_CUDA_PFN_EXTERN(cuMulticastBindAddr, 12010); DECLARE_CUDA_PFN_EXTERN(cuMulticastCreate, 12010); DECLARE_CUDA_PFN_EXTERN(cuMulticastGetGranularity, 12010); DECLARE_CUDA_PFN_EXTERN(cuMulticastUnbind, 12010); #endif /* Stream-MemOp support */ DECLARE_CUDA_PFN_EXTERN(cuStreamBatchMemOp, 11070); DECLARE_CUDA_PFN_EXTERN(cuStreamWaitValue32, 11070); DECLARE_CUDA_PFN_EXTERN(cuStreamWaitValue64, 11070); DECLARE_CUDA_PFN_EXTERN(cuStreamWriteValue32, 11070); DECLARE_CUDA_PFN_EXTERN(cuStreamWriteValue64, 11070); #endif ncclResult_t ncclCudaLibraryInit(void); extern int ncclCudaDriverVersionCache; extern bool ncclCudaLaunchBlocking; // initialized by ncclCudaLibraryInit() // Checks whether the given stream is the legacy null stream. inline ncclResult_t ncclCudaStreamIsLegacyNull(cudaStream_t stream, bool* isLegacy) { #if CUDART_VERSION >= 12000 unsigned long long nullStreamId, legacyNullStreamId; CUDACHECK(cudaStreamGetId(NULL, &nullStreamId)); CUDACHECK(cudaStreamGetId(cudaStreamLegacy, &legacyNullStreamId)); *isLegacy = (stream == cudaStreamLegacy) || ((stream == NULL) && (nullStreamId == legacyNullStreamId)); #else *isLegacy = (stream == NULL) || (stream == cudaStreamLegacy); #endif return ncclSuccess; } inline ncclResult_t ncclCudaDriverVersion(int* driver) { int version = COMPILER_ATOMIC_LOAD(&ncclCudaDriverVersionCache, std::memory_order_relaxed); if (version == -1) { CUDACHECK(cudaDriverGetVersion(&version)); COMPILER_ATOMIC_STORE(&ncclCudaDriverVersionCache, version, std::memory_order_relaxed); } *driver = version; return ncclSuccess; } ncclResult_t ncclCuStreamBatchMemOp(cudaStream_t stream, unsigned int numOps, CUstreamBatchMemOpParams* batchParams); #endif nccl-2.29.7-1/src/include/debug.h000066400000000000000000000046621515037102200163450ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_INT_DEBUG_H_ #define NCCL_INT_DEBUG_H_ #include "nccl.h" #include "nccl_common.h" #include #include #include "compiler.h" // Conform to pthread and NVTX standard #define NCCL_THREAD_NAMELEN 16 extern int ncclDebugLevel; extern uint64_t ncclDebugMask; extern FILE *ncclDebugFile; void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char *filefunc, int line, const char *fmt, ...) __attribute__ ((format (printf, 5, 6))); // Let code temporarily downgrade WARN into INFO extern thread_local int ncclDebugNoWarn; extern char ncclLastError[]; #define VERSION(...) ncclDebugLog(NCCL_LOG_VERSION, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__) #define WARN(...) ncclDebugLog(NCCL_LOG_WARN, NCCL_ALL, __FILE__, __LINE__, __VA_ARGS__) #define NOWARN(EXPR, FLAGS) \ do { \ int oldNoWarn = ncclDebugNoWarn; \ ncclDebugNoWarn = FLAGS; \ (EXPR); \ ncclDebugNoWarn = oldNoWarn; \ } while(0) #define INFO(FLAGS, ...) \ do{ \ int level = COMPILER_ATOMIC_LOAD(&ncclDebugLevel, std::memory_order_acquire); \ if((level >= NCCL_LOG_INFO && ((unsigned long)(FLAGS) & ncclDebugMask)) || (level < 0)) \ ncclDebugLog(NCCL_LOG_INFO, (unsigned long)(FLAGS), __func__, __LINE__, __VA_ARGS__); \ } while(0) #define TRACE_CALL(...) \ do { \ int level = COMPILER_ATOMIC_LOAD(&ncclDebugLevel, std::memory_order_acquire); \ if((level >= NCCL_LOG_TRACE && (NCCL_CALL & ncclDebugMask)) || (level < 0)) { \ ncclDebugLog(NCCL_LOG_TRACE, NCCL_CALL, __func__, __LINE__, __VA_ARGS__); \ } \ } while (0) #ifdef ENABLE_TRACE #define TRACE(FLAGS, ...) \ do { \ int level = COMPILER_ATOMIC_LOAD(&ncclDebugLevel, std::memory_order_acquire); \ if ((level >= NCCL_LOG_TRACE && ((unsigned long)(FLAGS) & ncclDebugMask)) || (level < 0)) { \ ncclDebugLog(NCCL_LOG_TRACE, (unsigned long)(FLAGS), __func__, __LINE__, __VA_ARGS__); \ } \ } while (0) #else #define TRACE(...) #endif void ncclSetThreadName(std::thread& thread, const char *fmt, ...); void ncclResetDebugInit(); #endif nccl-2.29.7-1/src/include/dev_runtime.h000066400000000000000000000077751515037102200176100ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_DEVICE_RUNTIME_H_ #define NCCL_DEVICE_RUNTIME_H_ #include "nccl.h" #include "nccl_device.h" #include "nccl_common.h" #include "allocator.h" #include "bitops.h" #include "utils.h" //////////////////////////////////////////////////////////////////////////////// // ncclDevr[_]: runtime implements for symmetric API. struct ncclDevrMemory; struct ncclDevrWindow { struct ncclDevrMemory* memory; void* userPtr; size_t size; size_t bigOffset; // Offset in big VA space. int winFlags; void* localRegHandle; struct ncclWindow_vidmem* vidmem; // key for intrusive map struct ncclDevrWindow* next; // next for intrusive map struct ncclComm* comm; // comm for intrusive map window <> comm look up }; struct ncclDevrWindowSorted; struct ncclDevrTeam; struct ncclDevrRegTask { struct ncclDevrRegTask *next; void* userPtr; size_t userSize; int winFlags; ncclWindow_t* outWinDev; }; struct ncclDevrCommCreateTask { struct ncclDevrCommCreateTask *next; struct ncclDevCommRequirements* reqs; struct ncclDevComm* outDevComm; ncclResult_t (*outDevCommCopyCB)(struct ncclDevComm const* tmpDevComm, void* out); }; struct ncclDevrState { // Like localRank/localRanks except "lsa" ranks must be consecutive in the world // and all lsa subsets have the same number of ranks. If any condition is // false then the lsa team is just the singleton of self. int lsaSelf; int lsaSize; int* lsaRankList; int nLsaTeams; size_t granularity; // cuMemGetAllocationGranularity bool ginEnabled; bool rmaProxyEnabled; struct ncclDevrMemory* memHead; struct ncclDevrWindowSorted* winSorted; int winSortedCapacity, winSortedCount; struct ncclDevrTeam* teamHead; size_t bigSize; // size of our big logical space (128GB?) struct ncclSpace bigSpace; // allocates our big VA space. void* lsaFlatBase; // base ptr for all lsa ranks big VA's concatenated together: size = lsaRanks*bigSize struct ncclShadowPool shadows; struct ncclDevCommWindowTable* windowTable; struct ncclIntruQueue regTaskQueue; struct ncclIntruQueue commCreateTaskQueue; }; // Check if GIN resources have been requested as part of `reqs`. bool ncclGinResourcesRequested(struct ncclDevCommRequirements const* reqs); // We assume ncclComm has a `ncclDevrState symState` member. ncclResult_t ncclDevrInitOnce(struct ncclComm* comm); ncclResult_t ncclDevrFinalize(struct ncclComm* comm); // If found *outWinHost will be populated and *outWinId >= 0, otherwise *outWinId == -1 ncclResult_t ncclDevrFindWindow(struct ncclComm* comm, void const* userPtr, struct ncclDevrWindow** outWin); ncclResult_t ncclDevrWindowRegisterInGroup( struct ncclComm* comm, void* ptr, size_t size, int winFlags, ncclWindow_t* outWinDev ); ncclResult_t ncclDevrCommCreateInternal( struct ncclComm* comm, struct ncclDevCommRequirements const* reqs, struct ncclDevComm* outDevComm, bool isInternal = false, ncclResult_t (*outDevCommCopyCB)(struct ncclDevComm const* tmpDevComm, void* out) = nullptr ); void freeDevCommRequirements( struct ncclDevCommRequirements* reqs ); // Get the corresponding pointer in another lsa rank's symmetric memory window ncclResult_t ncclDevrGetLsaRankPtr(struct ncclComm* comm, struct ncclDevrWindow* winHost, size_t offset, int lsaRank, void** outPtr); // Get the RMA device window handle for a specific context ncclGinWindow_t ncclDevrGetRmaDevWin(struct ncclDevrWindow* winHost, int ctx); // Get the multicast address for a given team ncclResult_t ncclDevrGetLsaTeamPtrMC(struct ncclComm* comm, struct ncclDevrWindow* winHost, size_t offset, struct ncclTeam lsaTeam, void** outPtr); #endif nccl-2.29.7-1/src/include/device.h000066400000000000000000000523421515037102200165140ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_DEVICE_H_ #define NCCL_DEVICE_H_ #include "nccl.h" #include "nccl_device/core.h" #include "nccl_tuner.h" #include "bitops.h" #include #include #include extern const char* ncclFuncStr[NCCL_NUM_FUNCTIONS]; extern const char* ncclAlgoStr[NCCL_NUM_ALGORITHMS]; extern const char* ncclProtoStr[NCCL_NUM_PROTOCOLS]; #define NCCL_MAX_OPS 2048 #define NCCL_STEPS 8 #ifdef __CUDA_ARCH__ #define NCCL_CUDA_ARCH __CUDA_ARCH__ #else #define NCCL_CUDA_ARCH 0 #endif #ifdef __CUDA_ARCH_SPECIFIC__ #define NCCL_CUDA_ARCH_SPECIFIC __CUDA_ARCH_SPECIFIC__ #elif defined(__CUDA_ARCH_HAS_FEATURE__) #if __CUDA_ARCH_HAS_FEATURE__(SM90_ALL) #define NCCL_CUDA_ARCH_SPECIFIC 900 #elif __CUDA_ARCH_HAS_FEATURE__(SM100_ALL) #define NCCL_CUDA_ARCH_SPECIFIC 1000 #elif __CUDA_ARCH_HAS_FEATURE__(SM101_ALL) #define NCCL_CUDA_ARCH_SPECIFIC 1010 #elif __CUDA_ARCH_HAS_FEATURE__(SM120_ALL) #define NCCL_CUDA_ARCH_SPECIFIC 1200 #else #define NCCL_CUDA_ARCH_SPECIFIC 0 #endif #else #define NCCL_CUDA_ARCH_SPECIFIC 0 #endif #ifdef __CUDA_ARCH_FAMILY_SPECIFIC__ #define NCCL_CUDA_ARCH_FAMILY_SPECIFIC __CUDA_ARCH_FAMILY_SPECIFIC__ #else #define NCCL_CUDA_ARCH_FAMILY_SPECIFIC 0 #endif #include "nccl_device/net_device.h" enum ncclDevRedOp_t { ncclDevSum, ncclDevProd, ncclDevMinMax, ncclDevPreMulSum, ncclDevSumPostDiv, ncclNumDevRedOps }; struct ncclDevRedOpFull { ncclDevRedOp_t op; ncclRedOp_t proxyOp; bool scalarArgIsPtr; uint64_t scalarArg; }; union ncclLLFifoLine { /* Flags have to be *after* data, because otherwise, an incomplete receive from the network may receive the flag but not the data. Note this is assuming that either we receive contiguous chunks of data (sockets) or data is written with an atomicity of 8 bytes (IB/RDMA). */ struct { uint32_t data1; uint32_t flag1; uint32_t data2; uint32_t flag2; }; uint64_t v[2]; int4 i4; }; #define WARP_SIZE 32 #define MAXCHANNELS 64 #define NCCL_MAX_LOCAL_RANKS 72 #define NCCL_MAX_NTHREADS 640 #define NCCL_MIN_NTHREADS (4*WARP_SIZE) #define NCCL_SIMPLE_MAX_NTHREADS 512 #define NCCL_SIMPLE_EXTRA_GROUP_IF_NTHREADS_GE (3*WARP_SIZE) #define NCCL_LL_MAX_NTHREADS 512 #define NCCL_LL_LINES_PER_THREAD 8 #ifdef TEST_LL_CLEANUP #define NCCL_LL_CLEAN_MASK 0x078 // Set to 0x100 to disable cleanup #define NCCL_LL_FLAG_MAX 0x100 #define NCCL_LL_FLAG(a) ((uint32_t)((a) % NCCL_LL_FLAG_MAX)) #else #define NCCL_LL_CLEAN_MASK 0x7ffffff8 #define NCCL_LL_FLAG(a) ((uint32_t)(a)) #endif // Make sure the clean mask will last for at least NCCL_NSTEPS static_assert(NCCL_LL_CLEAN_MASK % NCCL_STEPS == 0, "Invalid NCCL_LL_CLEAN_MASK value"); #define NCCL_LL128_LINESIZE 128 #define NCCL_LL128_LINEELEMS (NCCL_LL128_LINESIZE/sizeof(uint64_t)) #define NCCL_LL128_DATAELEMS (NCCL_LL128_LINEELEMS-1) #define NCCL_LL128_MAX_NTHREADS 640 #define NCCL_LL128_ELEMS_PER_THREAD 120 #define NCCL_LL128_SHMEM_ELEMS_PER_THREAD 8 #define NCCL_LL128_SHMEM_SIZE (NCCL_LL128_SHMEM_ELEMS_PER_THREAD*NCCL_LL128_MAX_NTHREADS) #define NCCL_P2P_WRITE 0x01 #define NCCL_P2P_READ 0x02 #define NCCL_DIRECT_NIC 0x04 #define NCCL_NVLS_MIN_POLL 0x80 // Number of named barriers supported by CUDA #define NCCL_MAX_GROUPS 16 #define NCCL_REGULAR_BUFFER 0x00 #define NCCL_IPC_REG_BUFFER 0x01 #define NCCL_NVLS_REG_BUFFER 0x02 #define NCCL_NET_REG_BUFFER 0x04 struct ncclConnInfo { // Regular comm mechanism char *buffs[NCCL_NUM_PROTOCOLS]; // Local for recv, remote for send void* mhandles[NCCL_NUM_PROTOCOLS]; uint64_t *tail; // Local for recv, remote for send uint64_t *head; // Local for send, remote for recv int flags; // Direct communication / other flags int shared; // Buffers are shared int stepSize; // Step size for the SIMPLE buffer void **ptrExchange; // Pointer exchange for direct communication uint64_t* redOpArgExchange; // PreOp scaler exchange for direct pull case struct ncclConnFifo* connFifo; // Used for GPU - Proxy communication uint64_t step; // Keep where we are uint64_t llLastCleaning; ncclNetDeviceHandle_t netDeviceHandle; }; struct ncclProxyConnector { bool initialized; int rank; int tpRank; int tpLocalRank; int sameProcess; struct ncclProxyConnection* connection; ncclResult_t (*proxyProgress)(struct ncclProxyState* proxyState, struct ncclProxyArgs*); // Copied from transport if necessary ncclResult_t (*proxyGinProgress)(struct ncclProxyState* proxyState); }; struct ncclConnector { int connected; int hasSeen; int p2pOnly; struct ncclProxyConnector proxyConn; struct ncclTransportComm* transportComm; void* transportResources; struct ncclConnInfo conn; }; struct ncclRing { // Shortcuts for userRanks[1] and userRanks[n-1] int prev; int next; // Maps an internal nccl index to user-specified rank order. This is necessary // since we need to know how the user expects data to be ordered across // devices. Ordered from current device. int* userRanks; // Maps a user rank to an internal ring index. int* rankToIndex; // inverse lookup of userRanks, setup in setupChannel int index; // This rank's index in the ring }; // The root of each tree only has one node down (+1 intra-node). #define NCCL_MAX_TREE_ARITY_TOP 2 // Nodes inside the binary tree can have to two nodes down (+1 intra-node). #define NCCL_MAX_TREE_ARITY 3 struct ncclTree { int depth; int up; int down[NCCL_MAX_TREE_ARITY]; }; #define NCCL_MAX_DIRECT_ARITY 7 struct ncclDirect { int depth; int out; int nHeads; // Number of parallel N<->1<->net operations we'll do in parallel; size of up/down int headRank; // Index in 0..nHeads-1 I am the head rank of. -1 if I'm not a head rank (no local NIC) int shift; // Shuffling of send/recv for scatter/gather operations, basically localRank%nHeads // The heads[...] are guaranteed to be in rotated order start with self: // headRank, (headRank+1)%nHeads, (headRank+2)%nHeads, ... int heads[NCCL_MAX_DIRECT_ARITY+1]; int up[NCCL_MAX_DIRECT_ARITY]; int down[NCCL_MAX_DIRECT_ARITY]; }; #define NCCL_MAX_NVLS_ARITY 32 #define NCCL_MAX_NVLS_TREE_ARITY 3 struct ncclNvls { int out; int nHeads; // Number of parallel N<->1<->net operations we'll do in parallel; size of up/down int headRank; // Index in 0..nHeads-1 I am the head rank of. -1 if I'm not a head rank (no local NIC) int up[NCCL_MAX_NVLS_ARITY]; int down; int treeUp; int treeDown[NCCL_MAX_NVLS_TREE_ARITY]; }; #if __CUDA_ARCH__ >= 900 #define NCCL_MAX_ARITY NCCL_MAX_NVLS_ARITY #else #define NCCL_MAX_ARITY NCCL_MAX_DIRECT_ARITY #endif #define NCCL_MAX_CONNS 2 struct ncclChannelPeer { struct ncclConnector send[NCCL_MAX_CONNS]; struct ncclConnector recv[NCCL_MAX_CONNS]; int refCount; }; struct ncclKernelComm; struct alignas(16) ncclDevWorkP2p { void *sendAddr, *recvAddr; size_t sendBytes, recvBytes; int sendRank, recvRank; // From the part index, nP2pChannels, and channelBase the device code can // calculate which part of the transfer a channel is responsible for. uint8_t nP2pChannels; // Always equal to comm->p2pnChannels uint8_t channelBase; // Channel owning first part. // Zero channels indicates no work in that direction. uint8_t nSendChannels, nRecvChannels; // Chunk size stored in 8 bits via u32fp8Encode/Decode. uint8_t sendChunkSize_u32fp8, recvChunkSize_u32fp8; uint8_t sendProtoLL:1, recvProtoLL:1; uint8_t sendNetReg:1, recvNetReg:1; uint8_t sendIpcReg:1, recvIpcReg:1; uint8_t profilerEnabled:1; }; // Compute the subset of the data transfer corresponding to the given part index. inline __host__ __device__ void ncclP2pPartBounds(int nParts, int part, size_t bytes, size_t* partBeg, size_t* partEnd) { size_t partBytes = alignUp(divUp(bytes, nParts), 4<<10); #if __CUDA_ARCH__ *partBeg = min((part+0)*partBytes, bytes); *partEnd = min((part+1)*partBytes, bytes); #else *partBeg = std::min((part+0)*partBytes, bytes); *partEnd = std::min((part+1)*partBytes, bytes); #endif } // implemented in channel.h inline __host__ uint8_t ncclP2pChannelBaseForRound(struct ncclComm* comm, int p2pRound); // ncclP2pChannelToPart and ncclP2pChannelForPart are inverses. The device code // uses ncclP2pChannelToPart to determine which part "this" channel is responsible for. inline __host__ int ncclP2pChannelForPart(int nP2pChannels, int base, int part) { return (base + part) & (nP2pChannels-1); } inline __device__ int ncclP2pChannelToPart(int nP2pChannels, int base, int channel) { return (channel - base) & (nP2pChannels-1); } struct alignas(16) ncclDevWorkColl { // Running on channels [channelLo..channelHi], hi is inclusive. // nChannels == (channelHi - channelLo) + 1 uint32_t channelLo:8, channelHi:8; uint32_t nWarps:8; uint32_t redOpArgIsPtr:1, regUsed:1, netRegUsed:1, oneNode:1, direct:2, isOneRPN:1; uint32_t profilerEnabled:1; uint32_t root; void* recvbuff; void* sendbuff; uintptr_t sendbuffOffset; uintptr_t recvbuffOffset; uintptr_t* sendbuffRmtAddrs; uintptr_t* recvbuffRmtAddrs; union { // Continuous-byte-distribution scheduling. The lo and hi channels are of // different size than the channels in the middle. struct { size_t countLo, countMid, countHi; // Chunk counts where units are ncclProtoGrainSize(protocol) bytes uint64_t chunkGrainsLo:21, chunkGrainsMid:21, chunkGrainsHi:21; } cbd; // Collnet scheduling. All channels divide work evenly. struct { size_t count; // Total size, not divided per channel. uint32_t chunkCount; } collnet; }; uint64_t redOpArg; }; struct alignas(16) ncclDevWorkBcast { int ringDepth; int chunkSize; void *sendbuff; void *recvbuff; size_t bytes; size_t bytes_done; uint8_t pad[8]; }; __host__ __device__ constexpr int ncclProtoGrainSize(int proto) { return proto == NCCL_PROTO_LL ? 16 : proto == NCCL_PROTO_LL128 ? WARP_SIZE*NCCL_LL128_SHMEM_ELEMS_PER_THREAD/NCCL_LL128_LINEELEMS*NCCL_LL128_DATAELEMS*sizeof(uint64_t) : proto == NCCL_PROTO_SIMPLE ? 512 : -1; } template __host__ __device__ inline void ncclCollCbdPart( struct ncclDevWorkColl* work, uint32_t channelId, int proto, int eltSize, Int* count, Int* partOffset, Int* partCount, Int* chunkCount ) { int eltPerGrain = ncclProtoGrainSize(proto)/eltSize; int nMidChannels = work->channelHi - work->channelLo - 1; // We can assum that nMidChannels<0 implies countMid==0, which let's us assume // that countMid*nMidChannels == 0. if (count != nullptr) { *count = work->cbd.countLo + work->cbd.countMid*nMidChannels + work->cbd.countHi; } if (channelId == work->channelLo) { *partOffset = 0; *partCount = work->cbd.countLo; *chunkCount = work->cbd.chunkGrainsLo*eltPerGrain; } else if (channelId == work->channelHi) { *partOffset = work->cbd.countLo + nMidChannels*work->cbd.countMid; *partCount = work->cbd.countHi; *chunkCount = work->cbd.chunkGrainsHi*eltPerGrain; } else { int mid = channelId - work->channelLo - 1; *partOffset = work->cbd.countLo + mid*work->cbd.countMid; *partCount = work->cbd.countMid; *chunkCount = work->cbd.chunkGrainsMid*eltPerGrain; } } struct alignas(16) ncclDevWorkCollReg { struct ncclDevWorkColl coll; void* dnInputs[NCCL_MAX_DIRECT_ARITY+1]; void* dnOutputs[NCCL_MAX_DIRECT_ARITY+1]; void* upOutputs[NCCL_MAX_DIRECT_ARITY+1]; }; enum ncclDevWorkType: uint8_t { ncclDevWorkTypeP2p, ncclDevWorkTypeColl, ncclDevWorkTypeCollReg, ncclDevWorkTypeBcast, // for batched broadcast }; constexpr size_t ncclDevWorkSize(enum ncclDevWorkType type) { return type == ncclDevWorkTypeP2p ? sizeof(ncclDevWorkP2p) : type == ncclDevWorkTypeColl ? sizeof(ncclDevWorkColl) : type == ncclDevWorkTypeCollReg ? sizeof(ncclDevWorkCollReg) : type == ncclDevWorkTypeBcast ? sizeof(ncclDevWorkBcast): 0; } __host__ __device__ constexpr int ncclMaxDevWorkBatchBytes(int cudaArch = NCCL_CUDA_ARCH) { return cudaArch < 800 ? (1<<10) : cudaArch < 900 ? (8<<10) : (16<<10); } #define NCCL_MAX_DEV_WORK_BATCH_BYTES 1024 #define NCCL_MAX_DEV_WORK_BATCH_COLLS (NCCL_MAX_DEV_WORK_BATCH_BYTES/sizeof(ncclDevWorkColl)) #define NCCL_MAX_DEV_WORK_P2P_PER_BATCH 8 struct alignas(16) ncclDevWorkBatch { union { struct { // nextExtends: should next one be merged into this one. // nextJump=0: end of this channel's batch list // nextJump>0: batches[thisIndex+nextJump] is next batch in this list uint32_t nextJump:14, nextExtends:1; uint32_t workType:2, funcId:15; }; // Unioning bitfields with underlying type hints compiler to emit the best // SASS LD/ST accesses. uint32_t flags; }; // Rolling offset in fifo where this batch's work structs begin uint32_t offsetBase; // Set of relative offsets from offsetBase for this channel's subset of the batch: // For each bit index i in offsetMask, find work at fifo offset: offsetBase + i*sizeof(WorkStructType) uint64_t offsetBitset; }; struct ncclDevChannelPeer { // Stripped version of ncclChannelPeer where we only keep the ncclConnInfo // instead of the full ncclConnector. struct ncclConnInfo send[NCCL_MAX_CONNS]; struct ncclConnInfo recv[NCCL_MAX_CONNS]; }; struct alignas(16) ncclDevChannel { struct ncclDevChannelPeer** peers; struct ncclRing ring; struct ncclTree tree; struct ncclTree collnetChain; struct ncclDirect collnetDirect; struct ncclNvls nvls; uint32_t* workFifoDone; // Location of done counter, device writes index+1 of last work processed uint64_t workCounter; }; #define MAX_PROFILER_EVENTS_PER_CHANNEL 64 struct ncclDevProfiler { struct { uint64_t counter; uint64_t timestamp; } data[MAX_PROFILER_EVENTS_PER_CHANNEL]; }; struct ncclKernelComm { int rank; int nRanks; int node; int nNodes; int buffSizes[NCCL_NUM_PROTOCOLS]; int p2pChunkSize; int isAllNvlink; int* collNetDenseToUserRank; // Flag to ask NCCL kernels to abort volatile uint32_t* abortFlag; // Channels, device side struct ncclDevChannel* channels/*[MAXCHANNELS]*/; int* rankToLocalRank; // Profiler counters struct ncclDevProfiler* workStarted/*[MAXCHANNELS]*/; struct ncclDevProfiler* workCompleted/*[MAXCHANNELS]*/; }; struct alignas(16) ncclKernelCommAndChannels { struct ncclKernelComm comm; struct ncclDevChannel channels[MAXCHANNELS]; }; enum ncclDevWorkStorageType: uint8_t { ncclDevWorkStorageTypeArgs=0, ncclDevWorkStorageTypeFifo=1, ncclDevWorkStorageTypePersistent=2 }; struct alignas(16) ncclDevKernelArgs { struct ncclKernelComm* comm; uint64_t channelMask; enum ncclDevWorkStorageType workStorageType; uint32_t workMask; void* workBuf; // A channel's first batch is at `blockIdx.x`. Use `nextJump` to follow rest of list. // struct ncclDevWorkBatch batches[]; }; __host__ __device__ constexpr int ncclMaxKernelArgsSize(/*int cudaDriver, */int cudaArch=NCCL_CUDA_ARCH) { //return (cudaArch < 700 || cudaDriver < 12010) ? 4<<10 : (32<<10)-4; return 4<<10; } template struct alignas(16) ncclDevKernelArgsStorage { union { struct ncclDevKernelArgs args; ulong2 storage[capacity/sizeof(ulong2)]; }; }; typedef ncclDevKernelArgsStorage<(4<<10)> ncclDevKernelArgs4K; //typedef ncclDevKernelArgsStorage<(32<<10)-4> ncclDevKernelArgs31K; template __host__ __device__ constexpr T min_constexpr(T a) { return a; } template __host__ __device__ constexpr T min_constexpr(T a, T b, Ts ...c) { return min_constexpr((a < b ? a : b), c...); } template __host__ __device__ constexpr T max_constexpr(T a) { return a; } template __host__ __device__ constexpr T max_constexpr(T a, T b, Ts ...c) { return max_constexpr((a > b ? a : b), c...); } constexpr int ncclDevMaxChannelsForArgsBytes(size_t argsBytes) { return min_constexpr(MAXCHANNELS, (argsBytes - sizeof(struct ncclDevKernelArgs))/sizeof(struct ncclDevWorkBatch)); } // Calculate the unroll factor given: // * bytePerPack: number of bytes accessed per instruction // * insns: max permissible unroll value // * bytes: desired number of in-flight bytes per iteration ( = unroll*bytePerPack) __host__ __device__ constexpr int ncclCalcUnroll(int bytePerPack, int insns, int bytes) { return min_constexpr(insns, (bytes + bytePerPack-1)/bytePerPack); } // Note that all unroll value logic should depend on a given cudaArch argument // and not __CUDA_ARCH__ since these need to be host-side executable where the // arch value is strictly runtime only. By defaulting to NCCL_CUDA_ARCH, device // side code can elide passing the arch for brevity. __host__ __device__ constexpr int ncclCollUnroll(int cudaArch = NCCL_CUDA_ARCH) { // Our collective unroll should move to the same bytes&insns model as NVLS. return cudaArch >= 800 ? (cudaArch / 100 == 12 ? 6 : 8) : 4; } __host__ __device__ constexpr int ncclNvlsUnrollBytes(int cudaArch = NCCL_CUDA_ARCH) { return 4*16; } __host__ __device__ constexpr int ncclNvlsUnrollInsns(int cudaArch = NCCL_CUDA_ARCH) { return 16; } __host__ __device__ constexpr int ncclNvlsUnroll(int bytePerPack, int cudaArch = NCCL_CUDA_ARCH) { return ncclCalcUnroll(bytePerPack, ncclNvlsUnrollInsns(cudaArch), ncclNvlsUnrollBytes(cudaArch)); } // The amount of dynamic shmem per warp __host__ __device__ constexpr int ncclShmemScratchWarpSize(int cudaArch = NCCL_CUDA_ARCH) { return (max_constexpr( /*LL */0, /*LL128 */(NCCL_LL128_SHMEM_ELEMS_PER_THREAD*WARP_SIZE)*sizeof(uint64_t), /*SIMPLE*/(ncclCollUnroll(cudaArch)*WARP_SIZE + 1)*16, // NVLS needs an extra 16B to read unaligned data. /*NVLS */WARP_SIZE*(cudaArch >= 900 ? ncclNvlsUnrollBytes(cudaArch) : 0) + 16 ) + 15) & -16; // pad to 16 bytes } // The amount of dynamic shmem per block __host__ __device__ constexpr int ncclShmemDynamicSize(int cudaArch = NCCL_CUDA_ARCH) { return cudaArch < 700 ? 0 : ncclShmemScratchWarpSize(cudaArch)*(NCCL_MAX_NTHREADS/WARP_SIZE); } // Host-side table of kernel function pointers. extern int const ncclDevKernelCount; extern void* ncclDevKernelList[/*ncclDevKernelCount*/]; extern int ncclDevKernelRequirements[/*ncclDevKernelCount*/]; // Table of most specialized kernel function to run given func index. extern int const ncclDevFuncRowToId[]; extern void* const ncclDevKernelForFunc[/*funcIndex*/]; extern bool const ncclDevKernelForFuncIsSpecialized[/*funcIndex*/]; // Launch a one-rank reduction on stream. ncclResult_t ncclLaunchOneRank(void* dst, void const* src, size_t nElts, struct ncclDevRedOpFull redOp, ncclDataType_t type, cudaStream_t stream); // `ncclNvlsSupported()` needs to be in sync with "func_valid" in "src/device/generate.py" inline bool ncclNvlsSupported(int devRedOp, int type) { switch (type) { case ncclInt32: case ncclUint32: case ncclInt64: case ncclUint64: case ncclFloat16: case ncclBfloat16: return devRedOp == ncclDevSum || devRedOp == ncclDevMinMax; case ncclFloat: case ncclDouble: return devRedOp == ncclDevSum; default: return false; } } // `ncclDevFuncIndex()` needs to be in sync with "all_functions()" in "src/device/generate.py" inline int ncclDevFuncId(int coll, int devRedOp, int type, int algo, int proto) { constexpr int NumTypes = ncclNumTypes; int row; do { row = 0; // ncclDevFuncIndex_P2p if (coll == ncclFuncSendRecv) break; row += 1; int nAlgos = 4; if (coll == ncclFuncAllGather) { int algo1 = algo == NCCL_ALGO_RING ? 0 : algo == NCCL_ALGO_COLLNET_DIRECT ? 1 : algo == NCCL_ALGO_NVLS ? 2 : /*algo == NCCL_ALGO_PAT*/ 3; row += algo1*NCCL_NUM_PROTOCOLS + proto; break; } row += nAlgos*NCCL_NUM_PROTOCOLS; nAlgos = 1; if (coll == ncclFuncBroadcast) { row += proto; break; } row += nAlgos*NCCL_NUM_PROTOCOLS; nAlgos = 1; if (coll == ncclFuncAllGatherV) { row += proto; break; } row += nAlgos*NCCL_NUM_PROTOCOLS; nAlgos = 6; // TREE RING COLLNET_DIRECT COLLNET_CHAIN NVLS NVLS_TREE if (coll == ncclFuncAllReduce) { row += ((devRedOp*NumTypes + type)*nAlgos + algo)*NCCL_NUM_PROTOCOLS + proto; break; } row += ncclNumDevRedOps*NumTypes*nAlgos*NCCL_NUM_PROTOCOLS; nAlgos = 1; if (coll == ncclFuncReduce) { row += (devRedOp*NumTypes + type)*NCCL_NUM_PROTOCOLS + proto; break; } row += ncclNumDevRedOps*NumTypes*nAlgos*NCCL_NUM_PROTOCOLS; nAlgos = 4; if (coll == ncclFuncReduceScatter) { int algo1 = algo == NCCL_ALGO_RING ? 0 : algo == NCCL_ALGO_COLLNET_DIRECT ? 1 : algo == NCCL_ALGO_NVLS ? 2 : /*algo == NCCL_ALGO_PAT*/ 3; row += ((devRedOp*NumTypes + type)*nAlgos + algo1)*NCCL_NUM_PROTOCOLS + proto; break; } row += ncclNumDevRedOps*NumTypes*nAlgos*NCCL_NUM_PROTOCOLS; } while (false); return ncclDevFuncRowToId[row]; } inline int ncclDevFuncId_P2p() { return ncclDevFuncRowToId[0]; } ncclResult_t ncclGinResetSignalsAndCounters(struct ncclComm* comm, ncclDevComm_t const* devComm); #endif nccl-2.29.7-1/src/include/enqueue.h000066400000000000000000000052451515037102200167240ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_ENQUEUE_H_ #define NCCL_ENQUEUE_H_ #include "comm.h" #include "group.h" #include "collectives.h" #include "utils.h" #define NCCL_LL_ALIGNMENT_PER_THREAD sizeof(uint64_t) #define NCCL_LL128_ALIGNMENT_PER_WARP 480 #define NCCL_SIMPLE_ALIGNMENT (WARP_SIZE * 8LL * 16LL) #define NCCL_BYTES_ALIGNMENT 16 ncclResult_t ncclInitKernelsForDevice(int cudaArch, int maxSharedMem, size_t* maxStackSize); ncclResult_t ncclEnqueueCheck(struct ncclInfo* info); ncclResult_t ncclLaunchPrepare(struct ncclComm* comm); ncclResult_t ncclLaunchKernelBefore_NoUncapturedCuda(struct ncclComm* comm, struct ncclKernelPlan* plan); ncclResult_t ncclLaunchKernel(struct ncclComm* comm, struct ncclKernelPlan* plan); ncclResult_t ncclLaunchKernelAfter_NoCuda(struct ncclComm* comm, struct ncclKernelPlan* plan); ncclResult_t ncclLaunchFinish(struct ncclComm* comm); ncclResult_t ncclPrepareTasks(struct ncclComm* comm, bool* algoNeedConnect, bool* needConnect, ncclSimInfo_t* simInfo); ncclResult_t ncclTasksRegAndEnqueue(struct ncclComm* comm); static inline size_t ncclFuncSendCount(ncclFunc_t func, int nRanks, size_t count) { return func == ncclFuncReduceScatter ? nRanks*count : count; } static inline size_t ncclFuncRecvCount(ncclFunc_t func, int nRanks, size_t count) { return func == ncclFuncAllGather ? nRanks*count : count; } static inline size_t ncclFuncMaxSendRecvCount(ncclFunc_t func, int nRanks, size_t count) { return func == ncclFuncAllGather || func == ncclFuncReduceScatter ? nRanks*count : count; } ncclResult_t ncclGetCollNetSupport(struct ncclComm* comm, struct ncclTaskColl* task, int* collNetSupport); ncclResult_t ncclGetAlgoInfo( struct ncclComm* comm, struct ncclTaskColl* task, int collNetSupport, int nvlsSupport, int numPipeOps, ncclSimInfo_t* simInfo = NULL ); bool ncclTestBudget(struct ncclKernelPlanBudget* budget, int nWorkBatches, ssize_t nWorkBytes); void ncclAddWorkBatchToPlan(struct ncclComm* comm, struct ncclKernelPlan* plan, int channelId, enum ncclDevWorkType workType, int devFuncId, uint32_t workOffset, int p2pEpoch =-1, int p2pRound = -1, bool newBatch = false); ncclResult_t ncclAddProxyOpIfNeeded(struct ncclComm* comm, struct ncclKernelPlan* plan, struct ncclProxyOp* op); ncclResult_t ncclAddProfilerProxyOpIfNeeded(struct ncclComm* comm, struct ncclKernelPlan* plan, struct ncclProxyOp* op); #endif // End include guard nccl-2.29.7-1/src/include/env.h000066400000000000000000000013401515037102200160350ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_INT_ENV_H_ #define NCCL_INT_ENV_H_ #include "nccl_env.h" // Initialize Env Plugin ncclResult_t ncclEnvPluginInit(void); // Finalize Env Plugin void ncclEnvPluginFinalize(void); // Env plugin get function for NCCL params, called in ncclGetEnv() const char* ncclEnvPluginGetEnv(const char* name); bool ncclEnvPluginInitialized(void); ncclResult_t ncclInitEnv(void); #endif nccl-2.29.7-1/src/include/gdrwrap.h000066400000000000000000000232541515037102200167230ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2021-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_GDRWRAP_H_ #define NCCL_GDRWRAP_H_ #include "nccl.h" #include "alloc.h" #include // for standard [u]intX_t types #include #include #include // These can be used if the GDR library isn't thread safe std::mutex& getGdrMutex(); #define GDRLOCKCALL(cmd, ret) do { \ std::lock_guard lock(getGdrMutex()); \ ret = cmd; \ } while(false) #define GDRCHECK(cmd) do { \ int e; \ /* GDRLOCKCALL(cmd, e); */ \ e = cmd; \ if( e != 0 ) { \ WARN("GDRCOPY failure %d", e); \ return ncclSystemError; \ } \ } while(false) // This is required as the GDR memory is mapped WC #if !defined(__NVCC__) #if defined(__PPC__) static inline void wc_store_fence(void) { asm volatile("sync" : : : "memory"); } #elif defined(__x86_64__) #include static inline void wc_store_fence(void) { _mm_sfence(); } #elif defined(__aarch64__) #ifdef __cplusplus #include static inline void wc_store_fence(void) { std::atomic_thread_fence(std::memory_order_release); } #else #include static inline void wc_store_fence(void) { atomic_thread_fence(memory_order_release); } #endif #endif #endif //#define GDR_DIRECT 1 #ifdef GDR_DIRECT // Call the GDR API library code directly rather than via // dlopen() wrappers #include static ncclResult_t wrap_gdr_symbols(void) { return ncclSuccess; } static gdr_t wrap_gdr_open(void) { gdr_t g = gdr_open(); return g; } static ncclResult_t wrap_gdr_close(gdr_t g) { GDRCHECK(gdr_close(g)); return ncclSuccess; } static ncclResult_t wrap_gdr_pin_buffer(gdr_t g, unsigned long addr, size_t size, uint64_t p2p_token, uint32_t va_space, gdr_mh_t *handle) { GDRCHECK(gdr_pin_buffer(g, addr, size, p2p_token, va_space, handle)); return ncclSuccess; } static ncclResult_t wrap_gdr_unpin_buffer(gdr_t g, gdr_mh_t handle) { GDRCHECK(gdr_unpin_buffer(g, handle)); return ncclSuccess; } static ncclResult_t wrap_gdr_get_info(gdr_t g, gdr_mh_t handle, gdr_info_t *info) { GDRCHECK(gdr_get_info(g, handle, info)); return ncclSuccess; } static ncclResult_t wrap_gdr_map(gdr_t g, gdr_mh_t handle, void **va, size_t size) { GDRCHECK(gdr_map(gdr_t g, gdr_mh_t handle, void **va, size_t size)); return ncclSuccess; } static ncclResult_t wrap_gdr_unmap(gdr_t g, gdr_mh_t handle, void *va, size_t size) { GDRCHECK(gdr_unmap(gdr_t g, gdr_mh_t handle, void **va, size_t size)); return ncclSuccess; } static void wrap_gdr_runtime_get_version(int *major, int *minor) { gdr_runtime_get_version(major, minor); return ncclSuccess; } static void wrap_gdr_driver_get_version(gdr_t g, int *major, int *minor) { gdr_driver_get_version(g, major, minor); return ncclSuccess; } static ncclResult_t wrap_gdr_copy_to_mapping(gdr_mh_t handle, void *map_d_ptr, const void *h_ptr, size_t size) { GDRCHECK(gdr_copy_to_mapping(handle, map_d_ptr, h_ptr, size)); return ncclSuccess; } static ncclResult_t wrap_gdr_copy_from_mapping(gdr_mh_t handle, void *h_ptr, const void *map_d_ptr, size_t size) { GDRCHECK(gdr_copy_from_mapping(handle, h_ptr, map_d_ptr, size)); return ncclSuccess; } #else // Dynamically handle dependency the GDR API library /* Extracted from gdrapi.h (v2.1 Nov 2020) */ #define GPU_PAGE_SHIFT 16 #define GPU_PAGE_SIZE (1UL << GPU_PAGE_SHIFT) #define GPU_PAGE_OFFSET (GPU_PAGE_SIZE-1) #define GPU_PAGE_MASK (~GPU_PAGE_OFFSET) struct gdr; typedef struct gdr *gdr_t; typedef struct gdr_mh_s { unsigned long h; } gdr_mh_t; struct gdr_info { uint64_t va; uint64_t mapped_size; uint32_t page_size; uint64_t tm_cycles; uint32_t cycles_per_ms; unsigned mapped:1; unsigned wc_mapping:1; }; typedef struct gdr_info gdr_info_t; /* End of gdrapi.h */ ncclResult_t wrap_gdr_symbols(void); gdr_t wrap_gdr_open(void); ncclResult_t wrap_gdr_close(gdr_t g); ncclResult_t wrap_gdr_pin_buffer(gdr_t g, unsigned long addr, size_t size, uint64_t p2p_token, uint32_t va_space, gdr_mh_t *handle); ncclResult_t wrap_gdr_unpin_buffer(gdr_t g, gdr_mh_t handle); ncclResult_t wrap_gdr_get_info(gdr_t g, gdr_mh_t handle, gdr_info_t *info); ncclResult_t wrap_gdr_map(gdr_t g, gdr_mh_t handle, void **va, size_t size); ncclResult_t wrap_gdr_unmap(gdr_t g, gdr_mh_t handle, void *va, size_t size); ncclResult_t wrap_gdr_runtime_get_version(int *major, int *minor); ncclResult_t wrap_gdr_driver_get_version(gdr_t g, int *major, int *minor); ncclResult_t wrap_gdr_copy_to_mapping(gdr_mh_t handle, void *map_d_ptr, const void *h_ptr, size_t size); ncclResult_t wrap_gdr_copy_from_mapping(gdr_mh_t handle, void *h_ptr, const void *map_d_ptr, size_t size); #endif // GDR_DIRECT // Global GDR driver handle extern gdr_t ncclGdrCopy; #include "alloc.h" typedef struct gdr_mem_desc { void *gdrDevMem; void *gdrMap; size_t gdrOffset; size_t gdrMapSize; gdr_mh_t gdrMh; } gdr_mem_desc_t; static gdr_t ncclGdrInit() { int libMajor, libMinor, drvMajor, drvMinor; gdr_t handle = NULL; // Dynamically load the GDRAPI library symbols if (wrap_gdr_symbols() == ncclSuccess) { handle = wrap_gdr_open(); if (handle != NULL) { ncclResult_t res; // Query the version of libgdrapi NCCLCHECKGOTO(wrap_gdr_runtime_get_version(&libMajor, &libMinor), res, error); // Query the version of gdrdrv driver NCCLCHECKGOTO(wrap_gdr_driver_get_version(handle, &drvMajor, &drvMinor), res, error); // Only support GDRAPI 2.1 and later if (libMajor < 2 || (libMajor == 2 && libMinor < 1) || drvMajor < 2 || (drvMajor == 2 && drvMinor < 1)) { goto error; } else INFO(NCCL_INIT, "GDRCOPY enabled library %d.%d driver %d.%d", libMajor, libMinor, drvMajor, drvMinor); } } return handle; error: if (handle != NULL) (void) wrap_gdr_close(handle); return NULL; } template static ncclResult_t ncclGdrCudaCalloc(T** ptr, T** devPtr, size_t nelem, void** gdrHandle, struct ncclMemManager* manager) { gdr_info_t info; size_t mapSize; gdr_mh_t mh; char *devMem; void *gdrMap; mapSize = ncclSizeOfT()*nelem; // GDRCOPY Pinned buffer has to be a minimum of a GPU_PAGE_SIZE ALIGN_SIZE(mapSize, GPU_PAGE_SIZE); // GDRCOPY Pinned buffer has to be GPU_PAGE_SIZE aligned too NCCLCHECK(ncclCudaCalloc(&devMem, mapSize+GPU_PAGE_SIZE-1, manager)); uint64_t alignedAddr = (((uint64_t) devMem) + GPU_PAGE_OFFSET) & GPU_PAGE_MASK; size_t align = alignedAddr - (uint64_t)devMem; //TRACE(NCCL_INIT, "GDRCOPY: Pin buffer 0x%lx (%p) align %zu size %zu", alignedAddr, devMem, align, mapSize); NCCLCHECK(wrap_gdr_pin_buffer(ncclGdrCopy, alignedAddr, mapSize, 0, 0, &mh)); NCCLCHECK(wrap_gdr_map(ncclGdrCopy, mh, &gdrMap, mapSize)); //TRACE(NCCL_INIT, "GDRCOPY : mapped %p (0x%lx) at %p", devMem, alignedAddr, gdrMap); NCCLCHECK(wrap_gdr_get_info(ncclGdrCopy, mh, &info)); // Will offset ever be non zero ? ssize_t off = info.va - alignedAddr; gdr_mem_desc_t* md; NCCLCHECK(ncclCalloc(&md, 1)); md->gdrDevMem = devMem; md->gdrMap = gdrMap; md->gdrMapSize = mapSize; md->gdrOffset = off+align; md->gdrMh = mh; *gdrHandle = md; *ptr = (T *)((char *)gdrMap+off); if (devPtr) *devPtr = (T *)(devMem+off+align); TRACE(NCCL_INIT, "GDRCOPY : allocated devMem %p gdrMap %p offset %lx mh %lx mapSize %zu at %p", md->gdrDevMem, md->gdrMap, md->gdrOffset, md->gdrMh.h, md->gdrMapSize, *ptr); return ncclSuccess; } template static ncclResult_t ncclGdrCudaCopy(void *gdrHandle, T* dst, T* src, size_t nelem) { gdr_mem_desc_t *md = (gdr_mem_desc_t*)gdrHandle; NCCLCHECK(wrap_gdr_copy_to_mapping(md->gdrMh, dst, src, nelem*ncclSizeOfT())); return ncclSuccess; } static ncclResult_t ncclGdrCudaFree(void* gdrHandle, struct ncclMemManager* manager) { gdr_mem_desc_t *md = (gdr_mem_desc_t*)gdrHandle; NCCLCHECK(wrap_gdr_unmap(ncclGdrCopy, md->gdrMh, md->gdrMap, md->gdrMapSize)); NCCLCHECK(wrap_gdr_unpin_buffer(ncclGdrCopy, md->gdrMh)); NCCLCHECK(ncclCudaFree(md->gdrDevMem, manager)); free(md); return ncclSuccess; } // Helper: Allocate memory accessible from CPU (either GDR or host memory) template static ncclResult_t allocMemCPUAccessible(T **ptr, T **devPtr, size_t nelem, int host_flags, void **gdrHandle, struct ncclMemManager* manager, bool forceHost = false) { if (ncclGdrCopy && !forceHost) { NCCLCHECK(ncclGdrCudaCalloc(ptr, devPtr, nelem, gdrHandle, manager)); } else { NCCLCHECK(ncclCuMemHostAlloc((void **)ptr, NULL, nelem * sizeof(T))); memset((void *)*ptr, 0, nelem * sizeof(T)); *devPtr = *ptr; if (gdrHandle) *gdrHandle = NULL; // Mark as host allocated by nulling GDR handle } return ncclSuccess; } // Helper: Free memory allocated by allocMemCPUAccessible template static ncclResult_t freeMemCPUAccessible(T *ptr, void *gdrHandle, struct ncclMemManager* manager) { if (gdrHandle != NULL) { // If a GDR handle exists, it was GDR memory NCCLCHECK(ncclGdrCudaFree(gdrHandle, manager)); } else { // Otherwise, it was host memory (or GDR was off) NCCLCHECK(ncclCuMemHostFree(ptr)); } return ncclSuccess; } #endif // End include guard nccl-2.29.7-1/src/include/gin.h000066400000000000000000000014271515037102200160300ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_INT_GIN_H_ #define NCCL_INT_GIN_H_ #include "nccl_gin.h" ncclResult_t ncclGinInit(struct ncclComm* comm); ncclResult_t ncclGinInitFromParent(struct ncclComm* comm, struct ncclComm* parent); ncclResult_t ncclGinGetDevCount(int ginPluginIndex, int* nPhysDev, int* nVirtDev); ncclResult_t ncclGinFinalize(struct ncclComm* comm); extern ncclGin_t ncclGinIb; extern ncclGin_t ncclGinIbGdaki; extern ncclGin_t ncclGinIbProxy; #endif nccl-2.29.7-1/src/include/gin/000077500000000000000000000000001515037102200156535ustar00rootroot00000000000000nccl-2.29.7-1/src/include/gin/gin_host.h000066400000000000000000000061711515037102200176430ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_GIN_HOST_H_ #define _NCCL_GIN_HOST_H_ #include "allocator.h" #include "nccl.h" #include "nccl_gin.h" #include "nccl_device/gin/gin_device_host_common.h" #include #include #include struct ncclGinState { ncclGin_t* ncclGin; void* ginInstance; bool connected; ncclGinType_t ginType; int ginCommCount; int ginContextCount; void* ginComms[NCCL_GIN_MAX_CONNECTIONS]; void* ginCtx[NCCL_GIN_MAX_CONNECTIONS]; ncclNetDeviceHandle_t* ginDevHandles[NCCL_GIN_MAX_CONNECTIONS]; int needsProxyProgress; // Whether we need to progress GIN operations with the proxy int ginProgress; // GIN progress is enabled std::thread thread; std::mutex mutex; std::condition_variable cond; ncclResult_t asyncResult; int ginVersion; int signalSpaceSize; int counterSpaceSize; ncclSpace signalSpace; ncclSpace counterSpace; int ctxFirstAvailable; // We allocate shared contexts starting from index 0. int ctxLastExclusive; // We allocate exclusive contexts starting from the highest index. int ginQueueDepth; ncclGinConnectionType_t ginConnectionType; }; extern int64_t ncclParamGinType(); // Sets the local GIN type for comm. The GIN type that is set for comm is the // GIN type supported by the call process itself, without taking into account // (1) GIN support of other ranks, and (2) additional local constraints like // cross-NIC ncclResult_t setLocalGinType(struct ncclComm* comm); // Get the GIN type from comm. ginType is set to the GIN type that can be used // by the comm to communicate with other nodes. ncclResult_t getGlobalGinType(struct ncclComm* comm, ncclGinType_t* ginType); ncclResult_t getGlobalRailedGinType(struct ncclComm* comm, ncclGinType_t* ginType); // FIXME change to ncclGinState instead of ncclComm, no need to pass comm ncclResult_t ncclGinConnectOnce(struct ncclComm* comm, ncclGinConnectionType_t requestedConnectionType, int reqGinContextCount = 0, int reqGinQueueDepth = 0); ncclResult_t ncclGinHostFinalize(struct ncclComm* comm); ncclResult_t ncclGinRegister(struct ncclComm* comm, void* address, size_t size, void* ginHostWins[NCCL_GIN_MAX_CONNECTIONS], ncclGinWindow_t ginDevWins[NCCL_GIN_MAX_CONNECTIONS], int winFlags); ncclResult_t ncclGinDeregister(struct ncclComm* comm, void* ginHostWins[NCCL_GIN_MAX_CONNECTIONS]); ncclResult_t ncclGinAllocSignalsCounters(struct ncclComm* comm, int nSignals, uint32_t* outSignal0, int nCounters, uint32_t* outCounter0); ncclResult_t ncclGinFreeSignalsCounters(struct ncclComm* comm, uint32_t signal0, int nSignals, uint32_t counter0, int nCounters); ncclResult_t ncclGinQueryLastError(struct ncclGinState* ginState, bool* hasError); #endif nccl-2.29.7-1/src/include/gin/gin_host_proxy.h000066400000000000000000000024771515037102200211110ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef GIN_HOST_PROXY_H_ #define GIN_HOST_PROXY_H_ #include #include #include #include #include "nccl.h" #include "gin/gin_host.h" #include "plugin/nccl_gin.h" ncclResult_t ncclGinProxyCreateContext(struct ncclComm *comm, void *collComm, int devId, int nSignals, int nCounters, int nContexts, void **outGinCtx, ncclNetDeviceHandle_t **outDevHandle); ncclResult_t ncclGinProxyRegister(ncclGin_t *ginComm, void *ginCtx, void *addr, size_t size, int type, int mr_flags, void **mhandle, void **ginHandle); ncclResult_t ncclGinProxyDeregister(ncclGin_t *ginComm, void *ginCtx, void *mhandle); ncclResult_t ncclGinProxyDestroyContext(ncclGin_t *ginComm, void *ginCtx); ncclResult_t ncclGinProxyProgress(ncclGin_t *ginComm, void *ginCtx); ncclResult_t ncclGinProxyQueryLastError(ncclGin_t *ginComm, void *ginCtx, bool *hasError); #endif nccl-2.29.7-1/src/include/graph.h000066400000000000000000000161211515037102200163510ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_GRAPH_H_ #define NCCL_GRAPH_H_ #include "nccl.h" #include "device.h" #include "os.h" #include #include #include #include #include ncclResult_t ncclTopoCudaPath(int cudaDev, char** path); struct ncclTopoSystem; // Build the topology ncclResult_t ncclTopoGetSystem(struct ncclComm* comm, struct ncclTopoSystem** system, const char* dumpXmlFile=NULL); ncclResult_t ncclTopoSortSystem(struct ncclTopoSystem* system); ncclResult_t ncclTopoPrint(struct ncclTopoSystem* system); ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm* comm); ncclResult_t ncclTopoCheckCrossNicSupport(bool* supported); ncclResult_t ncclTopoCheckNicFused(struct ncclComm* comm, bool* fused); void ncclTopoFree(struct ncclTopoSystem* system); ncclResult_t ncclTopoTrimSystem(struct ncclTopoSystem* system, struct ncclComm* comm); ncclResult_t ncclTopoComputeP2pChannels(struct ncclComm* comm); ncclResult_t ncclTopoComputeP2pChannelsPerPeer(struct ncclComm* comm); ncclResult_t ncclTopoGetNvbGpus(struct ncclTopoSystem* system, int rank, int* nranks, int** ranks); ncclResult_t ncclTopoPathAllNVLink(struct ncclTopoSystem* system, int* allNvLink); ncclResult_t ncclTopoPathAllDirectNVLink(struct ncclTopoSystem* system, bool* allNvlinkConnected); ncclResult_t ncclTopoComputeCommCPU(struct ncclComm* comm); // Query topology ncclResult_t ncclTopoGetNetDev(struct ncclComm* comm, int rank, struct ncclTopoGraph* graph, int channelId, int peerRank, int64_t* id, int* dev, int* proxyRank); ncclResult_t ncclTopoCheckP2p(struct ncclComm* comm, struct ncclTopoSystem* system, int rank1, int rank2, int* p2p, int *read, int* intermediateRank, int* cudaP2p); ncclResult_t ncclTopoCheckMNNVL(struct ncclTopoSystem* system, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2, int* ret); enum ncclTopoGdrMode { ncclTopoGdrModeDisable = 0, ncclTopoGdrModeDefault = 1, ncclTopoGdrModePci = 2, ncclTopoGdrModeNum = 3 }; ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* topo, int rank, int64_t netId, int read, enum ncclTopoGdrMode* gdrMode); ncclResult_t ncclTopoNeedFlush(struct ncclComm* comm, int64_t netId, int netDev, int rank, int* flush); ncclResult_t ncclTopoIsGdrAvail(struct ncclTopoSystem* system, int rank, bool *avail); ncclResult_t ncclTopoCheckNet(struct ncclTopoSystem* system, int rank1, int rank2, int* net); int ncclPxnDisable(struct ncclComm* comm); ncclResult_t ncclTopoGetPxnRanks(struct ncclComm* comm, int** intermediateRanks, int* nranks); ncclResult_t ncclGetLocalCpu(struct ncclTopoSystem* system, int gpu, int* retCpu); ncclResult_t ncclGetUserP2pLevel(int* level); // Find CPU affinity ncclResult_t ncclTopoGetCpuAffinity(struct ncclTopoSystem* system, int rank, ncclAffinity* affinity); #define NCCL_TOPO_CPU_ARCH_X86 1 #define NCCL_TOPO_CPU_ARCH_POWER 2 #define NCCL_TOPO_CPU_ARCH_ARM 3 #define NCCL_TOPO_CPU_ARCH_MIXED 4 #define NCCL_TOPO_CPU_VENDOR_INTEL 1 #define NCCL_TOPO_CPU_VENDOR_AMD 2 #define NCCL_TOPO_CPU_VENDOR_ZHAOXIN 3 #define NCCL_TOPO_CPU_VENDOR_MIXED 4 #define NCCL_TOPO_CPU_MODEL_INTEL_BDW 1 #define NCCL_TOPO_CPU_MODEL_INTEL_SKL 2 #define NCCL_TOPO_CPU_MODEL_INTEL_SRP 3 #define NCCL_TOPO_CPU_MODEL_INTEL_ERP 4 #define NCCL_TOPO_CPU_MODEL_YONGFENG 1 ncclResult_t ncclTopoCpuType(struct ncclTopoSystem* system, int* arch, int* vendor, int* model); ncclResult_t ncclTopoGetGpuCount(struct ncclTopoSystem* system, int* count); ncclResult_t ncclTopoGetNetCount(struct ncclTopoSystem* system, int* count); ncclResult_t ncclTopoGetNvsCount(struct ncclTopoSystem* system, int* count); ncclResult_t ncclTopoGetLocalNet(struct ncclTopoSystem* system, int rank, int channelId, int64_t* id, int* dev); ncclResult_t ncclTopoGetLocalGinDevs(struct ncclComm* comm, int* localGinDevs, int* localGinCount); ncclResult_t ncclTopoGetLocalGpu(struct ncclTopoSystem* system, int64_t netId, int* gpuIndex); ncclResult_t getLocalNetCountByBw(struct ncclTopoSystem* system, int gpu, int *count, float* bw); enum netDevsPolicy { NETDEVS_POLICY_AUTO = 0x0, NETDEVS_POLICY_ALL = 0x1, NETDEVS_POLICY_MAX = 0x2, NETDEVS_POLICY_UNDEF = 0xffffffff }; ncclResult_t ncclTopoGetNetDevsPolicy(enum netDevsPolicy* policy, int* policyNum); // Allows for up to 576 GPUs (e.g., NVLD144) with headroom for internal operations #define NCCL_TOPO_MAX_NODES 640 ncclResult_t ncclTopoGetLocal(struct ncclTopoSystem* system, int type, int index, int resultType, int locals[NCCL_TOPO_MAX_NODES], int* localCount, int* pathType); // Init search. Needs to be done before calling ncclTopoCompute ncclResult_t ncclTopoSearchInit(struct ncclTopoSystem* system); #define NCCL_TOPO_PATTERN_BALANCED_TREE 1 // Spread NIC traffic between two GPUs (Tree parent + one child on first GPU, second child on second GPU) #define NCCL_TOPO_PATTERN_SPLIT_TREE 2 // Spread NIC traffic between two GPUs (Tree parent on first GPU, tree children on the second GPU) #define NCCL_TOPO_PATTERN_TREE 3 // All NIC traffic going to/from the same GPU #define NCCL_TOPO_PATTERN_RING 4 // Ring #define NCCL_TOPO_PATTERN_NVLS 5 // NVLS+SHARP and NVLS+Tree #define NCCL_TOPO_PATTERN_COLLNET_DIRECT 6 // Collnet Direct struct ncclTopoGraph { // Input / output int id; // ring : 0, tree : 1, collnet : 2, nvls : 3, collnetDirect : 4 int pattern; int crossNic; int collNet; int minChannels; int maxChannels; // Output int nChannels; float bwIntra; float bwInter; float latencyInter; int typeIntra; int typeInter; int sameChannels; int nHops; int intra[MAXCHANNELS*NCCL_TOPO_MAX_NODES]; int64_t inter[MAXCHANNELS*2]; }; ncclResult_t ncclTopoCompute(struct ncclTopoSystem* system, struct ncclTopoGraph* graph); ncclResult_t ncclTopoPrintGraph(struct ncclTopoSystem* system, struct ncclTopoGraph* graph); ncclResult_t ncclTopoDumpGraphs(struct ncclTopoSystem* system, int ngraphs, struct ncclTopoGraph** graphs); struct ncclTopoRanks { int crossNicRing; int ringRecv[MAXCHANNELS]; int ringSend[MAXCHANNELS]; int ringPrev[MAXCHANNELS]; int ringNext[MAXCHANNELS]; int treeToParent[MAXCHANNELS]; int treeToChild0[MAXCHANNELS]; int treeToChild1[MAXCHANNELS]; int nvlsHeads[MAXCHANNELS]; int nvlsHeadNum; }; ncclResult_t ncclTopoPreset(struct ncclComm* comm, struct ncclTopoGraph** graphs, struct ncclTopoRanks* topoRanks); ncclResult_t ncclTopoPostset(struct ncclComm* comm, int* firstRanks, int* treePatterns, struct ncclTopoRanks** allTopoRanks, int* rings, struct ncclTopoGraph** graphs, struct ncclComm* parent); ncclResult_t ncclTopoInitTunerConstants(struct ncclComm* comm); ncclResult_t ncclTopoTuneModel(struct ncclComm* comm, int minCompCap, int maxCompCap, struct ncclTopoGraph** graphs); ncclResult_t ncclTopoGetAlgoTime(struct ncclComm* comm, int coll, int algorithm, int protocol, size_t nBytes, int numPipeOps, float* time); #endif nccl-2.29.7-1/src/include/group.h000066400000000000000000000127331515037102200164110ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_GROUP_H_ #define NCCL_GROUP_H_ #include "nccl.h" #include "comm.h" #include "allocator.h" #include "register.h" #include "utils.h" #include ncclResult_t ncclGroupErrCheck(ncclResult_t ret); void ncclGroupCommJoin(struct ncclComm* comm, int type); void ncclGroupCommPreconnect(struct ncclComm* comm); ncclResult_t ncclGroupCommLeave(struct ncclComm* comm); ncclResult_t ncclGroupJobAbort(struct ncclGroupJob* groupJob); ncclResult_t ncclGroupJobComplete(struct ncclGroupJob *groupJob); typedef ncclResult_t(*ncclInitFunc_t)(ncclComm_t* newcomm, int ndev, ncclUniqueId commId, int myrank, int cudaDev); ncclResult_t ncclAsyncInit(ncclInitFunc_t func, ncclComm_t* newcomm, int ndev, ncclUniqueId commId, int myrank, int cudaDev); typedef enum ncclGroupJobState { ncclGroupJobRunning = 0, ncclGroupJobDone = 1, ncclGroupJobJoined = 2, } ncclGroupJobState_t; struct ncclAsyncJob { struct ncclAsyncJob* next; std::thread thread; ncclResult_t result; ncclResult_t(*func)(struct ncclAsyncJob*); void(*undo)(struct ncclAsyncJob*); void(*destructor)(void*); ncclGroupJobState_t state; uint32_t* abortFlag; /* point to comm abortFlag */ uint32_t* abortFlagDev; /* point to comm abortFlagDev */ uint32_t* childAbortFlag; /* point to child abortFlag */ uint32_t* childAbortFlagDev; /* point to child abortFlagDev */ ncclComm_t comm; int destroyFlag; bool isThreadMain; ~ncclAsyncJob() { if (thread.joinable()) { (void)ncclThreadJoin(thread); } } }; ncclResult_t ncclAsyncLaunch( struct ncclAsyncJob* job, ncclResult_t(*func)(struct ncclAsyncJob*), void(*undo)(struct ncclAsyncJob*), void(*destructor)(void*), ncclComm_t comm ); struct ncclGroupJob { struct ncclAsyncJob base; int groupRefCount; bool nonBlockingInit; bool joined; struct ncclComm *groupCommHead[ncclGroupTaskTypeNum]; struct ncclComm *groupCommPreconnectHead; ncclResult_t groupError; bool abortFlag; struct ncclIntruQueue asyncJobs; }; ncclResult_t ncclGroupStartInternal(); ncclResult_t ncclGroupEndInternal(ncclSimInfo_t* simInfo = NULL); ncclResult_t ncclAsyncJobComplete(struct ncclAsyncJob* job); //////////////////////////////////////////////////////////////////////////////// extern thread_local int ncclGroupDepth; // depth of ncclGroupStart nesting extern thread_local ncclResult_t ncclGroupError; extern thread_local struct ncclComm* ncclGroupCommHead[ncclGroupTaskTypeNum]; extern thread_local struct ncclComm* ncclGroupCommPreconnectHead; extern thread_local int ncclGroupBlocking; inline ncclResult_t ncclGroupStartInternal() { ncclGroupDepth++; return ncclSuccess; } inline bool ncclGroupEnabled() { return ncclGroupDepth != 0; } inline ncclResult_t ncclGroupErrCheck(ncclResult_t ret) { if (ncclGroupDepth > 0) { if (ret != ncclSuccess && ret != ncclInProgress) ncclGroupError = ret; } return ret; } // Add comm to this thread's group inline void ncclGroupCommJoin(struct ncclComm* comm, int type) { if (comm->groupNext[type] == reinterpret_cast(0x1)) { // Insert comm into ncclGroupCommHead adjacent to sibling comms. This preserves // the users program order yet insures siblings occur consecutively. This // is required by doLaunches() in "group.cc". struct ncclComm** pp = &ncclGroupCommHead[type]; while (*pp != nullptr && comm->intraComm0 != (*pp)->intraComm0) pp = &(*pp)->groupNext[type]; // didn't find its clique, we need to insert it with ascending order based on commHash if (*pp == nullptr) { pp = &ncclGroupCommHead[type]; while (*pp != nullptr && (*pp)->commHash < comm->commHash) pp = &(*pp)->groupNext[type]; } comm->groupNext[type] = *pp; *pp = comm; // Comms gets a new memory stack scope upon joining. Each task batched for // this comm is allocated there. ncclMemoryStackPush(&comm->memScoped); if (type == ncclGroupTaskTypeCollective) { // Initialize planner ncclKernelPlanner::Peer* tmp = comm->planner.peers; ncclIntruQueue* tmpRmaQueues = comm->planner.rmaTaskQueues; int numRmaCtx = comm->config.numRmaCtx; memset(&comm->planner, 0, sizeof(comm->planner)); comm->planner.peers = tmp; comm->planner.bcast_info.minBcastPeer = INT_MAX; comm->planner.bcast_info.maxBcastPeer = INT_MIN; comm->planner.rmaTaskQueues = tmpRmaQueues; if (comm->planner.rmaTaskQueues != NULL) { for (int i = 0; i < numRmaCtx; i++) { ncclIntruQueueConstruct(&comm->planner.rmaTaskQueues[i]); } } } } ncclGroupBlocking = comm->config.blocking; } // Add comm to this thread's group needing preconnect inline void ncclGroupCommPreconnect(struct ncclComm* comm) { if (comm->preconnectNext == reinterpret_cast(0x1)) { comm->preconnectNext = ncclGroupCommPreconnectHead; ncclGroupCommPreconnectHead = comm; } } // Comm has left group inline ncclResult_t ncclGroupCommLeave(struct ncclComm* comm, int type) { comm->groupNext[type] = reinterpret_cast(0x1); ncclMemoryStackPop(&comm->memScoped); return ncclSuccess; } #endif nccl-2.29.7-1/src/include/ibvcore.h000066400000000000000000000673441515037102200167160ustar00rootroot00000000000000/* * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * Copyright (c) 2004, 2011-2012 Intel Corporation. All rights reserved. * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2005 PathScale, Inc. All rights reserved. * * This software is available to you under the OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 and BSD-3 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_IBV_CORE_H_ #define NCCL_IBV_CORE_H_ /* Basic IB verbs structs. Needed to dynamically load IB verbs functions without * explicit including of IB verbs header. */ #include #include #include #include #include #if __GNUC__ >= 3 # define __attribute_const __attribute__((const)) #else # define __attribute_const #endif union ibv_gid { uint8_t raw[16]; struct { uint64_t subnet_prefix; uint64_t interface_id; } global; }; #ifndef container_of /** * container_of - cast a member of a structure out to the containing structure * @ptr: the pointer to the member. * @type: the type of the container struct this is embedded in. * @member: the name of the member within the struct. * */ #define container_of(ptr, type, member) \ ((type *) ((uint8_t *)(ptr) - offsetof(type, member))) #endif #define vext_field_avail(type, fld, sz) (offsetof(type, fld) < (sz)) /*XXX:__VERBS_ABI_IS_EXTENDED produces warning "integer operation result is out of range" with g++ 4.8.2*/ static void *__VERBS_ABI_IS_EXTENDED = ((uint8_t *)NULL) - 1; enum ibv_node_type { IBV_NODE_UNKNOWN = -1, IBV_NODE_CA = 1, IBV_NODE_SWITCH, IBV_NODE_ROUTER, IBV_NODE_RNIC, /* Leave a gap for future node types before starting with * experimental node types. */ IBV_EXP_NODE_TYPE_START = 32, IBV_EXP_NODE_MIC = IBV_EXP_NODE_TYPE_START }; enum ibv_transport_type { IBV_TRANSPORT_UNKNOWN = -1, IBV_TRANSPORT_IB = 0, IBV_TRANSPORT_IWARP, /* Leave a gap for future transport types before starting with * experimental transport types. */ IBV_EXP_TRANSPORT_TYPE_START = 32, IBV_EXP_TRANSPORT_SCIF = IBV_EXP_TRANSPORT_TYPE_START }; enum ibv_device_cap_flags { IBV_DEVICE_RESIZE_MAX_WR = 1, IBV_DEVICE_BAD_PKEY_CNTR = 1 << 1, IBV_DEVICE_BAD_QKEY_CNTR = 1 << 2, IBV_DEVICE_RAW_MULTI = 1 << 3, IBV_DEVICE_AUTO_PATH_MIG = 1 << 4, IBV_DEVICE_CHANGE_PHY_PORT = 1 << 5, IBV_DEVICE_UD_AV_PORT_ENFORCE = 1 << 6, IBV_DEVICE_CURR_QP_STATE_MOD = 1 << 7, IBV_DEVICE_SHUTDOWN_PORT = 1 << 8, IBV_DEVICE_INIT_TYPE = 1 << 9, IBV_DEVICE_PORT_ACTIVE_EVENT = 1 << 10, IBV_DEVICE_SYS_IMAGE_GUID = 1 << 11, IBV_DEVICE_RC_RNR_NAK_GEN = 1 << 12, IBV_DEVICE_SRQ_RESIZE = 1 << 13, IBV_DEVICE_N_NOTIFY_CQ = 1 << 14, IBV_DEVICE_XRC = 1 << 20, IBV_DEVICE_MANAGED_FLOW_STEERING = 1 << 29 }; enum ibv_atomic_cap { IBV_ATOMIC_NONE, IBV_ATOMIC_HCA, IBV_ATOMIC_GLOB }; struct ibv_device_attr { char fw_ver[64]; uint64_t node_guid; uint64_t sys_image_guid; uint64_t max_mr_size; uint64_t page_size_cap; uint32_t vendor_id; uint32_t vendor_part_id; uint32_t hw_ver; int max_qp; int max_qp_wr; int device_cap_flags; int max_sge; int max_sge_rd; int max_cq; int max_cqe; int max_mr; int max_pd; int max_qp_rd_atom; int max_ee_rd_atom; int max_res_rd_atom; int max_qp_init_rd_atom; int max_ee_init_rd_atom; enum ibv_atomic_cap atomic_cap; int max_ee; int max_rdd; int max_mw; int max_raw_ipv6_qp; int max_raw_ethy_qp; int max_mcast_grp; int max_mcast_qp_attach; int max_total_mcast_qp_attach; int max_ah; int max_fmr; int max_map_per_fmr; int max_srq; int max_srq_wr; int max_srq_sge; uint16_t max_pkeys; uint8_t local_ca_ack_delay; uint8_t phys_port_cnt; }; enum ibv_mtu { IBV_MTU_256 = 1, IBV_MTU_512 = 2, IBV_MTU_1024 = 3, IBV_MTU_2048 = 4, IBV_MTU_4096 = 5 }; enum ibv_port_state { IBV_PORT_NOP = 0, IBV_PORT_DOWN = 1, IBV_PORT_INIT = 2, IBV_PORT_ARMED = 3, IBV_PORT_ACTIVE = 4, IBV_PORT_ACTIVE_DEFER = 5 }; enum { IBV_LINK_LAYER_UNSPECIFIED, IBV_LINK_LAYER_INFINIBAND, IBV_LINK_LAYER_ETHERNET, /* Leave a gap for future link layer types before starting with * experimental link layer. */ IBV_EXP_LINK_LAYER_START = 32, IBV_EXP_LINK_LAYER_SCIF = IBV_EXP_LINK_LAYER_START }; enum ibv_port_cap_flags { IBV_PORT_SM = 1 << 1, IBV_PORT_NOTICE_SUP = 1 << 2, IBV_PORT_TRAP_SUP = 1 << 3, IBV_PORT_OPT_IPD_SUP = 1 << 4, IBV_PORT_AUTO_MIGR_SUP = 1 << 5, IBV_PORT_SL_MAP_SUP = 1 << 6, IBV_PORT_MKEY_NVRAM = 1 << 7, IBV_PORT_PKEY_NVRAM = 1 << 8, IBV_PORT_LED_INFO_SUP = 1 << 9, IBV_PORT_SYS_IMAGE_GUID_SUP = 1 << 11, IBV_PORT_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12, IBV_PORT_EXTENDED_SPEEDS_SUP = 1 << 14, IBV_PORT_CM_SUP = 1 << 16, IBV_PORT_SNMP_TUNNEL_SUP = 1 << 17, IBV_PORT_REINIT_SUP = 1 << 18, IBV_PORT_DEVICE_MGMT_SUP = 1 << 19, IBV_PORT_VENDOR_CLASS = 1 << 24, IBV_PORT_CLIENT_REG_SUP = 1 << 25, IBV_PORT_IP_BASED_GIDS = 1 << 26, }; struct ibv_port_attr { enum ibv_port_state state; enum ibv_mtu max_mtu; enum ibv_mtu active_mtu; int gid_tbl_len; uint32_t port_cap_flags; uint32_t max_msg_sz; uint32_t bad_pkey_cntr; uint32_t qkey_viol_cntr; uint16_t pkey_tbl_len; uint16_t lid; uint16_t sm_lid; uint8_t lmc; uint8_t max_vl_num; uint8_t sm_sl; uint8_t subnet_timeout; uint8_t init_type_reply; uint8_t active_width; uint8_t active_speed; uint8_t phys_state; uint8_t link_layer; uint8_t flags; uint16_t port_cap_flags2; uint32_t active_speed_ex; }; enum ibv_event_type { IBV_EVENT_CQ_ERR, IBV_EVENT_QP_FATAL, IBV_EVENT_QP_REQ_ERR, IBV_EVENT_QP_ACCESS_ERR, IBV_EVENT_COMM_EST, IBV_EVENT_SQ_DRAINED, IBV_EVENT_PATH_MIG, IBV_EVENT_PATH_MIG_ERR, IBV_EVENT_DEVICE_FATAL, IBV_EVENT_PORT_ACTIVE, IBV_EVENT_PORT_ERR, IBV_EVENT_LID_CHANGE, IBV_EVENT_PKEY_CHANGE, IBV_EVENT_SM_CHANGE, IBV_EVENT_SRQ_ERR, IBV_EVENT_SRQ_LIMIT_REACHED, IBV_EVENT_QP_LAST_WQE_REACHED, IBV_EVENT_CLIENT_REREGISTER, IBV_EVENT_GID_CHANGE, /* new experimental events start here leaving enough * room for 14 events which should be enough */ IBV_EXP_EVENT_DCT_KEY_VIOLATION = 32, IBV_EXP_EVENT_DCT_ACCESS_ERR, IBV_EXP_EVENT_DCT_REQ_ERR, }; struct ibv_async_event { union { struct ibv_cq *cq; struct ibv_qp *qp; struct ibv_srq *srq; struct ibv_exp_dct *dct; int port_num; /* For source compatible with Legacy API */ uint32_t xrc_qp_num; } element; enum ibv_event_type event_type; }; enum ibv_wc_status { IBV_WC_SUCCESS, IBV_WC_LOC_LEN_ERR, IBV_WC_LOC_QP_OP_ERR, IBV_WC_LOC_EEC_OP_ERR, IBV_WC_LOC_PROT_ERR, IBV_WC_WR_FLUSH_ERR, IBV_WC_MW_BIND_ERR, IBV_WC_BAD_RESP_ERR, IBV_WC_LOC_ACCESS_ERR, IBV_WC_REM_INV_REQ_ERR, IBV_WC_REM_ACCESS_ERR, IBV_WC_REM_OP_ERR, IBV_WC_RETRY_EXC_ERR, IBV_WC_RNR_RETRY_EXC_ERR, IBV_WC_LOC_RDD_VIOL_ERR, IBV_WC_REM_INV_RD_REQ_ERR, IBV_WC_REM_ABORT_ERR, IBV_WC_INV_EECN_ERR, IBV_WC_INV_EEC_STATE_ERR, IBV_WC_FATAL_ERR, IBV_WC_RESP_TIMEOUT_ERR, IBV_WC_GENERAL_ERR }; const char *ibv_wc_status_str(enum ibv_wc_status status); enum ibv_wc_opcode { IBV_WC_SEND, IBV_WC_RDMA_WRITE, IBV_WC_RDMA_READ, IBV_WC_COMP_SWAP, IBV_WC_FETCH_ADD, IBV_WC_BIND_MW, /* * Set value of IBV_WC_RECV so consumers can test if a completion is a * receive by testing (opcode & IBV_WC_RECV). */ IBV_WC_RECV = 1 << 7, IBV_WC_RECV_RDMA_WITH_IMM }; enum ibv_wc_flags { IBV_WC_GRH = 1 << 0, IBV_WC_WITH_IMM = 1 << 1 }; struct ibv_wc { uint64_t wr_id; enum ibv_wc_status status; enum ibv_wc_opcode opcode; uint32_t vendor_err; uint32_t byte_len; uint32_t imm_data; /* in network byte order */ uint32_t qp_num; uint32_t src_qp; int wc_flags; uint16_t pkey_index; uint16_t slid; uint8_t sl; uint8_t dlid_path_bits; }; enum ibv_access_flags { IBV_ACCESS_LOCAL_WRITE = 1, IBV_ACCESS_REMOTE_WRITE = (1<<1), IBV_ACCESS_REMOTE_READ = (1<<2), IBV_ACCESS_REMOTE_ATOMIC = (1<<3), IBV_ACCESS_MW_BIND = (1<<4), IBV_ACCESS_RELAXED_ORDERING = (1<<20), }; struct ibv_pd { struct ibv_context *context; uint32_t handle; }; enum ibv_xrcd_init_attr_mask { IBV_XRCD_INIT_ATTR_FD = 1 << 0, IBV_XRCD_INIT_ATTR_OFLAGS = 1 << 1, IBV_XRCD_INIT_ATTR_RESERVED = 1 << 2 }; struct ibv_xrcd_init_attr { uint32_t comp_mask; int fd; int oflags; }; struct ibv_xrcd { struct ibv_context *context; }; enum ibv_rereg_mr_flags { IBV_REREG_MR_CHANGE_TRANSLATION = (1 << 0), IBV_REREG_MR_CHANGE_PD = (1 << 1), IBV_REREG_MR_CHANGE_ACCESS = (1 << 2), IBV_REREG_MR_KEEP_VALID = (1 << 3) }; struct ibv_mr { struct ibv_context *context; struct ibv_pd *pd; void *addr; size_t length; uint32_t handle; uint32_t lkey; uint32_t rkey; }; enum ibv_mw_type { IBV_MW_TYPE_1 = 1, IBV_MW_TYPE_2 = 2 }; struct ibv_mw { struct ibv_context *context; struct ibv_pd *pd; uint32_t rkey; }; struct ibv_global_route { union ibv_gid dgid; uint32_t flow_label; uint8_t sgid_index; uint8_t hop_limit; uint8_t traffic_class; }; struct ibv_grh { uint32_t version_tclass_flow; uint16_t paylen; uint8_t next_hdr; uint8_t hop_limit; union ibv_gid sgid; union ibv_gid dgid; }; enum ibv_rate { IBV_RATE_MAX = 0, IBV_RATE_2_5_GBPS = 2, IBV_RATE_5_GBPS = 5, IBV_RATE_10_GBPS = 3, IBV_RATE_20_GBPS = 6, IBV_RATE_30_GBPS = 4, IBV_RATE_40_GBPS = 7, IBV_RATE_60_GBPS = 8, IBV_RATE_80_GBPS = 9, IBV_RATE_120_GBPS = 10, IBV_RATE_14_GBPS = 11, IBV_RATE_56_GBPS = 12, IBV_RATE_112_GBPS = 13, IBV_RATE_168_GBPS = 14, IBV_RATE_25_GBPS = 15, IBV_RATE_100_GBPS = 16, IBV_RATE_200_GBPS = 17, IBV_RATE_300_GBPS = 18 }; /** * ibv_rate_to_mult - Convert the IB rate enum to a multiple of the * base rate of 2.5 Gbit/sec. For example, IBV_RATE_5_GBPS will be * converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec. * @rate: rate to convert. */ int ibv_rate_to_mult(enum ibv_rate rate) __attribute_const; /** * mult_to_ibv_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate enum. * @mult: multiple to convert. */ enum ibv_rate mult_to_ibv_rate(int mult) __attribute_const; /** * ibv_rate_to_mbps - Convert the IB rate enum to Mbit/sec. * For example, IBV_RATE_5_GBPS will return the value 5000. * @rate: rate to convert. */ int ibv_rate_to_mbps(enum ibv_rate rate) __attribute_const; /** * mbps_to_ibv_rate - Convert a Mbit/sec value to an IB rate enum. * @mbps: value to convert. */ enum ibv_rate mbps_to_ibv_rate(int mbps) __attribute_const; struct ibv_ah_attr { struct ibv_global_route grh; uint16_t dlid; uint8_t sl; uint8_t src_path_bits; uint8_t static_rate; uint8_t is_global; uint8_t port_num; }; enum ibv_srq_attr_mask { IBV_SRQ_MAX_WR = 1 << 0, IBV_SRQ_LIMIT = 1 << 1 }; struct ibv_srq_attr { uint32_t max_wr; uint32_t max_sge; uint32_t srq_limit; }; struct ibv_srq_init_attr { void *srq_context; struct ibv_srq_attr attr; }; enum ibv_srq_type { IBV_SRQT_BASIC, IBV_SRQT_XRC }; enum ibv_srq_init_attr_mask { IBV_SRQ_INIT_ATTR_TYPE = 1 << 0, IBV_SRQ_INIT_ATTR_PD = 1 << 1, IBV_SRQ_INIT_ATTR_XRCD = 1 << 2, IBV_SRQ_INIT_ATTR_CQ = 1 << 3, IBV_SRQ_INIT_ATTR_RESERVED = 1 << 4 }; struct ibv_srq_init_attr_ex { void *srq_context; struct ibv_srq_attr attr; uint32_t comp_mask; enum ibv_srq_type srq_type; struct ibv_pd *pd; struct ibv_xrcd *xrcd; struct ibv_cq *cq; }; enum ibv_qp_type { IBV_QPT_RC = 2, IBV_QPT_UC, IBV_QPT_UD, /* XRC compatible code */ IBV_QPT_XRC, IBV_QPT_RAW_PACKET = 8, IBV_QPT_RAW_ETH = 8, IBV_QPT_XRC_SEND = 9, IBV_QPT_XRC_RECV, /* Leave a gap for future qp types before starting with * experimental qp types. */ IBV_EXP_QP_TYPE_START = 32, IBV_EXP_QPT_DC_INI = IBV_EXP_QP_TYPE_START }; struct ibv_qp_cap { uint32_t max_send_wr; uint32_t max_recv_wr; uint32_t max_send_sge; uint32_t max_recv_sge; uint32_t max_inline_data; }; struct ibv_qp_init_attr { void *qp_context; struct ibv_cq *send_cq; struct ibv_cq *recv_cq; struct ibv_srq *srq; struct ibv_qp_cap cap; enum ibv_qp_type qp_type; int sq_sig_all; /* Below is needed for backwards compatabile */ struct ibv_xrc_domain *xrc_domain; }; enum ibv_qp_init_attr_mask { IBV_QP_INIT_ATTR_PD = 1 << 0, IBV_QP_INIT_ATTR_XRCD = 1 << 1, IBV_QP_INIT_ATTR_RESERVED = 1 << 2 }; struct ibv_qp_init_attr_ex { void *qp_context; struct ibv_cq *send_cq; struct ibv_cq *recv_cq; struct ibv_srq *srq; struct ibv_qp_cap cap; enum ibv_qp_type qp_type; int sq_sig_all; uint32_t comp_mask; struct ibv_pd *pd; struct ibv_xrcd *xrcd; }; enum ibv_qp_open_attr_mask { IBV_QP_OPEN_ATTR_NUM = 1 << 0, IBV_QP_OPEN_ATTR_XRCD = 1 << 1, IBV_QP_OPEN_ATTR_CONTEXT = 1 << 2, IBV_QP_OPEN_ATTR_TYPE = 1 << 3, IBV_QP_OPEN_ATTR_RESERVED = 1 << 4 }; struct ibv_qp_open_attr { uint32_t comp_mask; uint32_t qp_num; struct ibv_xrcd *xrcd; void *qp_context; enum ibv_qp_type qp_type; }; enum ibv_qp_attr_mask { IBV_QP_STATE = 1 << 0, IBV_QP_CUR_STATE = 1 << 1, IBV_QP_EN_SQD_ASYNC_NOTIFY = 1 << 2, IBV_QP_ACCESS_FLAGS = 1 << 3, IBV_QP_PKEY_INDEX = 1 << 4, IBV_QP_PORT = 1 << 5, IBV_QP_QKEY = 1 << 6, IBV_QP_AV = 1 << 7, IBV_QP_PATH_MTU = 1 << 8, IBV_QP_TIMEOUT = 1 << 9, IBV_QP_RETRY_CNT = 1 << 10, IBV_QP_RNR_RETRY = 1 << 11, IBV_QP_RQ_PSN = 1 << 12, IBV_QP_MAX_QP_RD_ATOMIC = 1 << 13, IBV_QP_ALT_PATH = 1 << 14, IBV_QP_MIN_RNR_TIMER = 1 << 15, IBV_QP_SQ_PSN = 1 << 16, IBV_QP_MAX_DEST_RD_ATOMIC = 1 << 17, IBV_QP_PATH_MIG_STATE = 1 << 18, IBV_QP_CAP = 1 << 19, IBV_QP_DEST_QPN = 1 << 20 }; enum ibv_qp_state { IBV_QPS_RESET, IBV_QPS_INIT, IBV_QPS_RTR, IBV_QPS_RTS, IBV_QPS_SQD, IBV_QPS_SQE, IBV_QPS_ERR, IBV_QPS_UNKNOWN }; enum ibv_mig_state { IBV_MIG_MIGRATED, IBV_MIG_REARM, IBV_MIG_ARMED }; struct ibv_qp_attr { enum ibv_qp_state qp_state; enum ibv_qp_state cur_qp_state; enum ibv_mtu path_mtu; enum ibv_mig_state path_mig_state; uint32_t qkey; uint32_t rq_psn; uint32_t sq_psn; uint32_t dest_qp_num; int qp_access_flags; struct ibv_qp_cap cap; struct ibv_ah_attr ah_attr; struct ibv_ah_attr alt_ah_attr; uint16_t pkey_index; uint16_t alt_pkey_index; uint8_t en_sqd_async_notify; uint8_t sq_draining; uint8_t max_rd_atomic; uint8_t max_dest_rd_atomic; uint8_t min_rnr_timer; uint8_t port_num; uint8_t timeout; uint8_t retry_cnt; uint8_t rnr_retry; uint8_t alt_port_num; uint8_t alt_timeout; }; enum ibv_wr_opcode { IBV_WR_RDMA_WRITE, IBV_WR_RDMA_WRITE_WITH_IMM, IBV_WR_SEND, IBV_WR_SEND_WITH_IMM, IBV_WR_RDMA_READ, IBV_WR_ATOMIC_CMP_AND_SWP, IBV_WR_ATOMIC_FETCH_AND_ADD }; enum ibv_send_flags { IBV_SEND_FENCE = 1 << 0, IBV_SEND_SIGNALED = 1 << 1, IBV_SEND_SOLICITED = 1 << 2, IBV_SEND_INLINE = 1 << 3 }; struct ibv_sge { uint64_t addr; uint32_t length; uint32_t lkey; }; struct ibv_send_wr { uint64_t wr_id; struct ibv_send_wr *next; struct ibv_sge *sg_list; int num_sge; enum ibv_wr_opcode opcode; int send_flags; uint32_t imm_data; /* in network byte order */ union { struct { uint64_t remote_addr; uint32_t rkey; } rdma; struct { uint64_t remote_addr; uint64_t compare_add; uint64_t swap; uint32_t rkey; } atomic; struct { struct ibv_ah *ah; uint32_t remote_qpn; uint32_t remote_qkey; } ud; } wr; union { union { struct { uint32_t remote_srqn; } xrc; } qp_type; uint32_t xrc_remote_srq_num; }; }; struct ibv_recv_wr { uint64_t wr_id; struct ibv_recv_wr *next; struct ibv_sge *sg_list; int num_sge; }; struct ibv_mw_bind { uint64_t wr_id; struct ibv_mr *mr; void *addr; size_t length; int send_flags; int mw_access_flags; }; struct ibv_srq { struct ibv_context *context; void *srq_context; struct ibv_pd *pd; uint32_t handle; pthread_mutex_t mutex; pthread_cond_t cond; uint32_t events_completed; /* below are for source compatabilty with legacy XRC, * padding based on ibv_srq_legacy. */ uint32_t xrc_srq_num_bin_compat_padding; struct ibv_xrc_domain *xrc_domain_bin_compat_padding; struct ibv_cq *xrc_cq_bin_compat_padding; void *ibv_srq_padding; /* legacy fields */ uint32_t xrc_srq_num; struct ibv_xrc_domain *xrc_domain; struct ibv_cq *xrc_cq; }; /* Not in use in new API, needed for compilation as part of source compat layer */ enum ibv_event_flags { IBV_XRC_QP_EVENT_FLAG = 0x80000000, }; struct ibv_qp { struct ibv_context *context; void *qp_context; struct ibv_pd *pd; struct ibv_cq *send_cq; struct ibv_cq *recv_cq; struct ibv_srq *srq; uint32_t handle; uint32_t qp_num; enum ibv_qp_state state; enum ibv_qp_type qp_type; pthread_mutex_t mutex; pthread_cond_t cond; uint32_t events_completed; }; struct ibv_comp_channel { struct ibv_context *context; int fd; int refcnt; }; struct ibv_cq { struct ibv_context *context; struct ibv_comp_channel *channel; void *cq_context; uint32_t handle; int cqe; pthread_mutex_t mutex; pthread_cond_t cond; uint32_t comp_events_completed; uint32_t async_events_completed; }; struct ibv_ah { struct ibv_context *context; struct ibv_pd *pd; uint32_t handle; }; enum ibv_flow_flags { IBV_FLOW_ATTR_FLAGS_ALLOW_LOOP_BACK = 1, IBV_FLOW_ATTR_FLAGS_DONT_TRAP = 1 << 1, }; enum ibv_flow_attr_type { /* steering according to rule specifications */ IBV_FLOW_ATTR_NORMAL = 0x0, /* default unicast and multicast rule - * receive all Eth traffic which isn't steered to any QP */ IBV_FLOW_ATTR_ALL_DEFAULT = 0x1, /* default multicast rule - * receive all Eth multicast traffic which isn't steered to any QP */ IBV_FLOW_ATTR_MC_DEFAULT = 0x2, }; enum ibv_flow_spec_type { IBV_FLOW_SPEC_ETH = 0x20, IBV_FLOW_SPEC_IPV4 = 0x30, IBV_FLOW_SPEC_TCP = 0x40, IBV_FLOW_SPEC_UDP = 0x41, }; struct ibv_flow_eth_filter { uint8_t dst_mac[6]; uint8_t src_mac[6]; uint16_t ether_type; /* * same layout as 802.1q: prio 3, cfi 1, vlan id 12 */ uint16_t vlan_tag; }; struct ibv_flow_spec_eth { enum ibv_flow_spec_type type; uint16_t size; struct ibv_flow_eth_filter val; struct ibv_flow_eth_filter mask; }; struct ibv_flow_ipv4_filter { uint32_t src_ip; uint32_t dst_ip; }; struct ibv_flow_spec_ipv4 { enum ibv_flow_spec_type type; uint16_t size; struct ibv_flow_ipv4_filter val; struct ibv_flow_ipv4_filter mask; }; struct ibv_flow_tcp_udp_filter { uint16_t dst_port; uint16_t src_port; }; struct ibv_flow_spec_tcp_udp { enum ibv_flow_spec_type type; uint16_t size; struct ibv_flow_tcp_udp_filter val; struct ibv_flow_tcp_udp_filter mask; }; struct ibv_flow_spec { union { struct { enum ibv_flow_spec_type type; uint16_t size; } hdr; struct ibv_flow_spec_eth eth; struct ibv_flow_spec_ipv4 ipv4; struct ibv_flow_spec_tcp_udp tcp_udp; }; }; struct ibv_flow_attr { uint32_t comp_mask; enum ibv_flow_attr_type type; uint16_t size; uint16_t priority; uint8_t num_of_specs; uint8_t port; uint32_t flags; /* Following are the optional layers according to user request * struct ibv_flow_spec_xxx [L2] * struct ibv_flow_spec_yyy [L3/L4] */ }; struct ibv_flow { uint32_t comp_mask; struct ibv_context *context; uint32_t handle; }; struct ibv_device; struct ibv_context; struct ibv_device_ops { struct ibv_context * (*alloc_context)(struct ibv_device *device, int cmd_fd); void (*free_context)(struct ibv_context *context); }; enum { IBV_SYSFS_NAME_MAX = 64, IBV_SYSFS_PATH_MAX = 256 }; struct ibv_device { struct ibv_device_ops ops; enum ibv_node_type node_type; enum ibv_transport_type transport_type; /* Name of underlying kernel IB device, eg "mthca0" */ char name[IBV_SYSFS_NAME_MAX]; /* Name of uverbs device, eg "uverbs0" */ char dev_name[IBV_SYSFS_NAME_MAX]; /* Path to infiniband_verbs class device in sysfs */ char dev_path[IBV_SYSFS_PATH_MAX]; /* Path to infiniband class device in sysfs */ char ibdev_path[IBV_SYSFS_PATH_MAX]; }; struct verbs_device { struct ibv_device device; /* Must be first */ size_t sz; size_t size_of_context; int (*init_context)(struct verbs_device *device, struct ibv_context *ctx, int cmd_fd); void (*uninit_context)(struct verbs_device *device, struct ibv_context *ctx); /* future fields added here */ }; struct ibv_context_ops { int (*query_device)(struct ibv_context *context, struct ibv_device_attr *device_attr); int (*query_port)(struct ibv_context *context, uint8_t port_num, struct ibv_port_attr *port_attr); struct ibv_pd * (*alloc_pd)(struct ibv_context *context); int (*dealloc_pd)(struct ibv_pd *pd); struct ibv_mr * (*reg_mr)(struct ibv_pd *pd, void *addr, size_t length, int access); struct ibv_mr * (*rereg_mr)(struct ibv_mr *mr, int flags, struct ibv_pd *pd, void *addr, size_t length, int access); int (*dereg_mr)(struct ibv_mr *mr); struct ibv_mw * (*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type); int (*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw, struct ibv_mw_bind *mw_bind); int (*dealloc_mw)(struct ibv_mw *mw); struct ibv_cq * (*create_cq)(struct ibv_context *context, int cqe, struct ibv_comp_channel *channel, int comp_vector); int (*poll_cq)(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc); int (*req_notify_cq)(struct ibv_cq *cq, int solicited_only); void (*cq_event)(struct ibv_cq *cq); int (*resize_cq)(struct ibv_cq *cq, int cqe); int (*destroy_cq)(struct ibv_cq *cq); struct ibv_srq * (*create_srq)(struct ibv_pd *pd, struct ibv_srq_init_attr *srq_init_attr); int (*modify_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr, int srq_attr_mask); int (*query_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr); int (*destroy_srq)(struct ibv_srq *srq); int (*post_srq_recv)(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr, struct ibv_recv_wr **bad_recv_wr); struct ibv_qp * (*create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr); int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask); int (*destroy_qp)(struct ibv_qp *qp); int (*post_send)(struct ibv_qp *qp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr); int (*post_recv)(struct ibv_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr); struct ibv_ah * (*create_ah)(struct ibv_pd *pd, struct ibv_ah_attr *attr); int (*destroy_ah)(struct ibv_ah *ah); int (*attach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); int (*detach_mcast)(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid); void (*async_event)(struct ibv_async_event *event); }; struct ibv_context { struct ibv_device *device; struct ibv_context_ops ops; int cmd_fd; int async_fd; int num_comp_vectors; pthread_mutex_t mutex; void *abi_compat; }; enum verbs_context_mask { VERBS_CONTEXT_XRCD = (uint64_t)1 << 0, VERBS_CONTEXT_SRQ = (uint64_t)1 << 1, VERBS_CONTEXT_QP = (uint64_t)1 << 2, VERBS_CONTEXT_RESERVED = (uint64_t)1 << 3, VERBS_CONTEXT_EXP = (uint64_t)1 << 62 }; struct verbs_context { /* "grows up" - new fields go here */ int (*query_port)(struct ibv_context *context, uint8_t port_num, struct ibv_port_attr *port_attr, size_t port_attr_len); int (*_reserved[25]) (void); struct verbs_ex_private *priv; int (*query_device_ex)(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr, size_t attr_size); int (*ibv_destroy_flow) (struct ibv_flow *flow); void (*ABI_placeholder2) (void); /* DO NOT COPY THIS GARBAGE */ struct ibv_flow * (*ibv_create_flow) (struct ibv_qp *qp, struct ibv_flow_attr *flow_attr); void (*ABI_placeholder1) (void); /* DO NOT COPY THIS GARBAGE */ struct ibv_qp * (*open_qp)(struct ibv_context *context, struct ibv_qp_open_attr *attr); struct ibv_qp * (*create_qp_ex)(struct ibv_context *context, struct ibv_qp_init_attr_ex *qp_init_attr_ex); int (*get_srq_num)(struct ibv_srq *srq, uint32_t *srq_num); struct ibv_srq * (*create_srq_ex)(struct ibv_context *context, struct ibv_srq_init_attr_ex *srq_init_attr_ex); struct ibv_xrcd * (*open_xrcd)(struct ibv_context *context, struct ibv_xrcd_init_attr *xrcd_init_attr); int (*close_xrcd)(struct ibv_xrcd *xrcd); uint64_t _ABI_placeholder3; size_t sz; /* Must be immediately before struct ibv_context */ struct ibv_context context; /* Must be last field in the struct */ }; static inline struct verbs_context *verbs_get_ctx(struct ibv_context *ctx) { if (ctx->abi_compat != __VERBS_ABI_IS_EXTENDED) return NULL; /* open code container_of to not pollute the global namespace */ return (struct verbs_context *)(((uintptr_t)ctx) - offsetof(struct verbs_context, context)); } #define verbs_get_ctx_op(ctx, op) ({ \ struct verbs_context *__vctx = verbs_get_ctx(ctx); \ (!__vctx || (__vctx->sz < sizeof(*__vctx) - offsetof(struct verbs_context, op)) || \ !__vctx->op) ? NULL : __vctx; }) #define verbs_set_ctx_op(_vctx, op, ptr) ({ \ struct verbs_context *vctx = _vctx; \ if (vctx && (vctx->sz >= sizeof(*vctx) - offsetof(struct verbs_context, op))) \ vctx->op = ptr; }) static inline struct verbs_device *verbs_get_device(struct ibv_device *dev) { return (dev->ops.alloc_context) ? NULL : container_of(dev, struct verbs_device, device); } static inline int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr) { return qp->context->ops.post_send(qp, wr, bad_wr); } struct ibv_ece { /* * Unique identifier of the provider vendor on the network. * The providers will set IEEE OUI here to distinguish * itself in non-homogenius network. */ uint32_t vendor_id; /* * Provider specific attributes which are supported or * needed to be enabled by ECE users. */ uint32_t options; uint32_t comp_mask; }; /** * ibv_query_port_ex - Get (extended) port properties */ static inline int ibv_query_port_ex(struct ibv_context *context, uint8_t port_num, struct ibv_port_attr *port_attr) { struct verbs_context *vctx = verbs_get_ctx_op(context, query_port); if (vctx) { return vctx->query_port(context, port_num, port_attr, sizeof(*port_attr)); } return -1; } #endif // NCCL_IBV_CORE_H_ nccl-2.29.7-1/src/include/ibvsymbols.h000066400000000000000000000057051515037102200174470ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_IBV_SYMBOLS_H_ #define NCCL_IBV_SYMBOLS_H_ #ifdef NCCL_BUILD_RDMA_CORE #include #else #include "ibvcore.h" #endif #include "nccl.h" /* IB Verbs Function Pointers*/ struct ncclIbvSymbols { int (*ibv_internal_fork_init)(void); struct ibv_device** (*ibv_internal_get_device_list)(int *num_devices); void (*ibv_internal_free_device_list)(struct ibv_device **list); const char * (*ibv_internal_get_device_name)(struct ibv_device *device); struct ibv_context* (*ibv_internal_open_device)(struct ibv_device* device); int (*ibv_internal_close_device)(struct ibv_context *context); int (*ibv_internal_get_async_event)(struct ibv_context *context, struct ibv_async_event *event); void (*ibv_internal_ack_async_event)(struct ibv_async_event *event); int (*ibv_internal_query_device)(struct ibv_context *context, struct ibv_device_attr *device_attr); int (*ibv_internal_query_port)(struct ibv_context *context, uint8_t port_num, struct ibv_port_attr *port_attr); int (*ibv_internal_query_gid)(struct ibv_context *context, uint8_t port_num, int index, union ibv_gid *gid); int (*ibv_internal_query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr); struct ibv_pd * (*ibv_internal_alloc_pd)(struct ibv_context *context); int (*ibv_internal_dealloc_pd)(struct ibv_pd *pd); struct ibv_mr * (*ibv_internal_reg_mr)(struct ibv_pd *pd, void *addr, size_t length, int access); struct ibv_mr * (*ibv_internal_reg_mr_iova2)(struct ibv_pd *pd, void *addr, size_t length, uint64_t iova, unsigned int access); /* DMA-BUF support */ struct ibv_mr * (*ibv_internal_reg_dmabuf_mr)(struct ibv_pd *pd, uint64_t offset, size_t length, uint64_t iova, int fd, int access); int (*ibv_internal_dereg_mr)(struct ibv_mr *mr); struct ibv_cq * (*ibv_internal_create_cq)(struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, int comp_vector); int (*ibv_internal_destroy_cq)(struct ibv_cq *cq); struct ibv_qp * (*ibv_internal_create_qp)(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr); int (*ibv_internal_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask); int (*ibv_internal_destroy_qp)(struct ibv_qp *qp); const char * (*ibv_internal_event_type_str)(enum ibv_event_type event); int (*ibv_internal_query_ece)(struct ibv_qp *qp, struct ibv_ece *ece); int (*ibv_internal_set_ece)(struct ibv_qp *qp, struct ibv_ece *ece); }; /* Constructs IB verbs symbols per rdma-core linking or dynamic loading mode */ ncclResult_t buildIbvSymbols(struct ncclIbvSymbols* ibvSymbols); #endif // NCCL_IBV_SYMBOLS_H_ nccl-2.29.7-1/src/include/ibvwrap.h000066400000000000000000000134001515037102200167170ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_IBVWRAP_H_ #define NCCL_IBVWRAP_H_ #include #include #ifdef NCCL_BUILD_RDMA_CORE #include #else #include "ibvcore.h" #endif #include "core.h" #include #include typedef enum ibv_return_enum { IBV_SUCCESS = 0, //!< The operation was successful } ibv_return_t; ncclResult_t wrap_ibv_symbols(void); /* NCCL wrappers of IB verbs functions */ ncclResult_t wrap_ibv_fork_init(void); ncclResult_t wrap_ibv_get_device_list(struct ibv_device ***ret, int *num_devices); ncclResult_t wrap_ibv_free_device_list(struct ibv_device **list); const char *wrap_ibv_get_device_name(struct ibv_device *device); ncclResult_t wrap_ibv_open_device(struct ibv_context **ret, struct ibv_device *device); ncclResult_t wrap_ibv_close_device(struct ibv_context *context); ncclResult_t wrap_ibv_get_async_event(struct ibv_context *context, struct ibv_async_event *event); ncclResult_t wrap_ibv_ack_async_event(struct ibv_async_event *event); ncclResult_t wrap_ibv_query_device(struct ibv_context *context, struct ibv_device_attr *device_attr); ncclResult_t wrap_ibv_query_port(struct ibv_context *context, uint8_t port_num, struct ibv_port_attr *port_attr); ncclResult_t wrap_ibv_query_gid(struct ibv_context *context, uint8_t port_num, int index, union ibv_gid *gid); ncclResult_t wrap_ibv_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr); ncclResult_t wrap_ibv_alloc_pd(struct ibv_pd **ret, struct ibv_context *context); ncclResult_t wrap_ibv_dealloc_pd(struct ibv_pd *pd); ncclResult_t wrap_ibv_reg_mr(struct ibv_mr **ret, struct ibv_pd *pd, void *addr, size_t length, int access); struct ibv_mr * wrap_direct_ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access); ncclResult_t wrap_ibv_reg_mr_iova2(struct ibv_mr **ret, struct ibv_pd *pd, void *addr, size_t length, uint64_t iova, int access); /* DMA-BUF support */ ncclResult_t wrap_ibv_reg_dmabuf_mr(struct ibv_mr **ret, struct ibv_pd *pd, uint64_t offset, size_t length, uint64_t iova, int fd, int access); struct ibv_mr * wrap_direct_ibv_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, size_t length, uint64_t iova, int fd, int access); ncclResult_t wrap_ibv_dereg_mr(struct ibv_mr *mr); ncclResult_t wrap_ibv_create_comp_channel(struct ibv_comp_channel **ret, struct ibv_context *context); ncclResult_t wrap_ibv_destroy_comp_channel(struct ibv_comp_channel *channel); ncclResult_t wrap_ibv_create_cq(struct ibv_cq **ret, struct ibv_context *context, int cqe, void *cq_context, struct ibv_comp_channel *channel, int comp_vector); ncclResult_t wrap_ibv_destroy_cq(struct ibv_cq *cq); static inline ncclResult_t wrap_ibv_poll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc, int* num_done) { int done = cq->context->ops.poll_cq(cq, num_entries, wc); /*returns the number of wcs or 0 on success, a negative number otherwise*/ if (done < 0) { WARN("Call to ibv_poll_cq() returned %d", done); return ncclSystemError; } *num_done = done; return ncclSuccess; } ncclResult_t wrap_ibv_create_qp(struct ibv_qp **ret, struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr); ncclResult_t wrap_ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask); ncclResult_t wrap_ibv_destroy_qp(struct ibv_qp *qp); ncclResult_t wrap_ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece, int* supported); ncclResult_t wrap_ibv_set_ece(struct ibv_qp *qp, struct ibv_ece *ece, int* supported); static inline ncclResult_t wrap_ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr, struct ibv_send_wr **bad_wr) { int ret = qp->context->ops.post_send(qp, wr, bad_wr); /*returns 0 on success, or the value of errno on failure (which indicates the failure reason)*/ if (ret != IBV_SUCCESS) { WARN("ibv_post_send() failed with error %s, Bad WR %p, First WR %p", strerror(ret), wr, *bad_wr); return ncclSystemError; } return ncclSuccess; } static inline ncclResult_t wrap_ibv_post_recv(struct ibv_qp *qp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr) { int ret = qp->context->ops.post_recv(qp, wr, bad_wr); /*returns 0 on success, or the value of errno on failure (which indicates the failure reason)*/ if (ret != IBV_SUCCESS) { WARN("ibv_post_recv() failed with error %s", strerror(ret)); return ncclSystemError; } return ncclSuccess; } ncclResult_t wrap_ibv_event_type_str(char **ret, enum ibv_event_type event); // converts a GID into a readable string. On success, returns a non-null pointer to gidStr. // NULL is returned if there was an error, with errno set to indicate the error. // errno = ENOSPC if the converted string would exceed strLen. static inline const char* ibvGetGidStr(union ibv_gid* gid, char* gidStr, size_t strLen) { // GID is a 16B handle, to convert it to a readable form, we use inet_ntop // sizeof(ibv_gid) == sizeof(struct in6_addr), so using AF_INET6 static_assert(sizeof(union ibv_gid) == sizeof(struct in6_addr), "the sizeof struct ibv_gid must be the size of struct in6_addr"); return inet_ntop(AF_INET6, gid->raw, gidStr, strLen); } // Helper function to convert IB work completion status to string const char* ibvWcStatusStr(enum ibv_wc_status status); // Helper function to convert IB work completion opcode to string const char* ibvWcOpcodeStr(enum ibv_wc_opcode opcode); // Helper function to convert IB work request opcode to string const char* ibvWrOpcodeStr(enum ibv_wr_opcode opcode); #endif //End include guard nccl-2.29.7-1/src/include/info.h000066400000000000000000000020341515037102200162010ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2019-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_INFO_H_ #define NCCL_INFO_H_ #include "nccl.h" #include "collectives.h" #include "core.h" #include "utils.h" // Used to pass NCCL call information between functions struct ncclInfo { ncclFunc_t coll; const char* opName; // NCCL Coll Args const void* sendbuff; void* recvbuff; size_t count; ncclDataType_t datatype; ncclRedOp_t op; int root; // peer for p2p operations ncclComm_t comm; cudaStream_t stream; // Algorithm details int chunkSteps; int sliceSteps; // One-sided ops size_t peerWinOffset; ncclWindow_t peerWin; int sigIdx; int ctx; unsigned int flags; // WaitSignal descriptors int nDesc; ncclWaitSignalDesc_t* signalDescs; }; #endif nccl-2.29.7-1/src/include/ipcsocket.h000066400000000000000000000026771515037102200172470ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_IPCSOCKET_H #define NCCL_IPCSOCKET_H #include "nccl.h" #include #include #include #include #include #include #include #include #include #include #include #define NCCL_IPC_SOCKNAME_LEN 64 struct ncclIpcSocket { int fd; char socketName[NCCL_IPC_SOCKNAME_LEN]; volatile uint32_t* abortFlag; }; ncclResult_t ncclIpcSocketInit(struct ncclIpcSocket *handle, int rank, uint64_t hash, volatile uint32_t* abortFlag); ncclResult_t ncclIpcSocketClose(struct ncclIpcSocket *handle); ncclResult_t ncclIpcSocketGetFd(struct ncclIpcSocket* handle, int* fd); ncclResult_t ncclIpcSocketRecvFd(struct ncclIpcSocket *handle, int *fd); ncclResult_t ncclIpcSocketSendFd(struct ncclIpcSocket *handle, const int fd, int rank, uint64_t hash); ncclResult_t ncclIpcSocketSendMsg(ncclIpcSocket *handle, void *hdr, int hdrLen, const int sendFd, int rank, uint64_t hash); ncclResult_t ncclIpcSocketRecvMsg(ncclIpcSocket *handle, void *hdr, int hdrLen, int *recvFd); #endif /* NCCL_IPCSOCKET_H */ nccl-2.29.7-1/src/include/mem_manager.h000066400000000000000000000132521515037102200175220ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_MEM_MANAGER_H_ #define NCCL_MEM_MANAGER_H_ #include "nccl.h" #include #include #include #include #ifdef __cplusplus extern "C" { #endif #if CUDART_VERSION < 12030 // MNNVL: FABRIC handle support lifted from CUDA 12.3 #define CU_DEVICE_ATTRIBUTE_HANDLE_TYPE_FABRIC_SUPPORTED ((CUdevice_attribute)128) #define CU_MEM_HANDLE_TYPE_FABRIC ((CUmemAllocationHandleType)0x8ULL) #ifndef CU_IPC_HANDLE_SIZE #define CU_IPC_HANDLE_SIZE 64 #endif typedef struct CUmemFabricHandle_st { unsigned char data[CU_IPC_HANDLE_SIZE]; } CUmemFabricHandle_v1; typedef CUmemFabricHandle_v1 CUmemFabricHandle; #endif struct ncclComm; // Initial capacity for exported peers array #define NCCL_MEM_EXPORT_PEERS_INIT 8 // Memory Type for NCCL allocations typedef enum { ncclMemPersist = 0, // Persistent memory - track stats only, never release/offload ncclMemScratch = 1, // Free without saving ncclMemOffload = 2 // Copy to CPU before free, restore on resume } ncclMemType_t; // Memory entry state typedef enum { ncclDynMemStateActive = 0, // Memory is allocated and usable ncclDynMemStateReleased = 1 // Memory has been released } ncclDynMemState_t; // Local owned memory descriptor typedef struct ncclDynMemLocalDesc { // Shareable handle for P2P exports // TODO: Remove the 'fd' field - POSIX FD handles are converted on-demand via proxy // (ncclProxyClientGetFdBlocking), so we no longer export them upfront. Only FABRIC // handles need upfront export since they can be shared directly via messaging. union { int fd; // For POSIX_FILE_DESCRIPTOR (unused) CUmemFabricHandle fabricHandle; // For FABRIC } shareableHandle; bool shareableHandleValid; // Peer tracking for P2P exports int numExportedPeers; int exportedPeersCapacity; int* exportedPeerRanks; } ncclDynMemLocalDesc; // Imported from peer memory descriptor typedef struct ncclDynMemImportDesc { int ownerRank; // Rank that owns the original buffer int ownerDev; // CUDA device of the owner void* ownerPtr; // Owner's virtual address } ncclDynMemImportDesc; // Individual tracked memory entry (only track scratch and offload allocations) typedef struct ncclDynMemEntry { void* ptr; // GPU virtual address size_t size; // Allocation size CUmemGenericAllocationHandle handle; // Physical memory handle CUmemAllocationHandleType handleType; ncclMemType_t memType; ncclDynMemState_t state; int cudaDev; // CPU backup for OFFLOAD type memory void* cpuBackup; // Host memory for offloaded data // Ownership type and type-specific data bool isImportedFromPeer; // true if this is a peer-imported buffer union { ncclDynMemLocalDesc local; ncclDynMemImportDesc imported; } desc; // Linked list pointer struct ncclDynMemEntry* next; } ncclDynMemEntry; // P2P Handle Exchange Structure typedef struct ncclDynMemP2pHandleInfo { void* ptr; int ownerRank; int ownerDev; size_t size; int handleType; union { uint64_t handleData; CUmemFabricHandle fabricHandle; }; } ncclDynMemP2pHandleInfo; // Memory manager attached to ncclComm typedef struct ncclMemManager { ncclDynMemEntry* entries; // Linked list of tracked allocations, only track scratch and offload allocations int numEntries; std::mutex lock; int released; int initialized; int refCount; size_t totalPersist; size_t totalPersistImported; size_t totalScratch; size_t totalScratchImported; size_t totalOffload; size_t totalOffloadImported; size_t cpuBackupUsage; int commCudaDev; } ncclMemManager; // Initialize memory manager ncclResult_t ncclMemManagerInit(struct ncclComm* comm); // Destroy memory manager and free all resources ncclResult_t ncclMemManagerDestroy(struct ncclComm* comm); // Track a new allocation ncclResult_t ncclMemTrack( struct ncclMemManager* manager, void* ptr, size_t size, CUmemGenericAllocationHandle handle, CUmemAllocationHandleType handleType, ncclMemType_t memType ); // Track imported allocation from peer ncclResult_t ncclMemTrackImportFromPeer( struct ncclMemManager* manager, void* ptr, size_t size, CUmemGenericAllocationHandle handle, CUmemAllocationHandleType handleType, ncclMemType_t memType, int ownerRank, int ownerDev, void* ownerPtr ); // Untrack allocation ncclResult_t ncclMemUntrack(struct ncclMemManager* manager, void* ptr, size_t size); // Add peer info for buffers in the linked list entries (only for dynamic memory: scratch/offload) ncclResult_t ncclDynMemMarkExportToPeer(struct ncclMemManager* manager, void* ptr, int peerRank); ncclResult_t ncclCommMemSuspend(struct ncclComm* comm); ncclResult_t ncclCommMemResume(struct ncclComm* comm); #ifdef __cplusplus } #endif #endif /* NCCL_MEM_MANAGER_H_ */ nccl-2.29.7-1/src/include/mlx5/000077500000000000000000000000001515037102200157635ustar00rootroot00000000000000nccl-2.29.7-1/src/include/mlx5/mlx5dvcore.h000066400000000000000000000037321515037102200202310ustar00rootroot00000000000000/* * Copyright (c) 2017 Mellanox Technologies, Inc. All rights reserved. * * This software is available to you under the OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ /************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 and BSD-3 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_MLX5DV_CORE_H_ #define NCCL_MLX5DV_CORE_H_ /* Basic MLX5 direct verbs structs. Needed to dynamically load MLX5 direct verbs functions without * explicit including of MLX5 direct verbs header. */ #include #include #include #include #include "ibvwrap.h" enum mlx5dv_reg_dmabuf_access { MLX5DV_REG_DMABUF_ACCESS_DATA_DIRECT = (1<<0), }; #endif // NCCL_MLX5DV_CORE_H_ nccl-2.29.7-1/src/include/mlx5/mlx5dvsymbols.h000066400000000000000000000022031515037102200207610ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_MLX5DV_SYMBOLS_H_ #define NCCL_MLX5DV_SYMBOLS_H_ #ifdef NCCL_BUILD_MLX5DV #include #else #include "mlx5/mlx5dvcore.h" #endif #include "nccl.h" /* MLX5 Direct Verbs Function Pointers*/ struct ncclMlx5dvSymbols { bool (*mlx5dv_internal_is_supported)(struct ibv_device *device); int (*mlx5dv_internal_get_data_direct_sysfs_path)(struct ibv_context *context, char *buf, size_t buf_len); /* DMA-BUF support */ struct ibv_mr * (*mlx5dv_internal_reg_dmabuf_mr)(struct ibv_pd *pd, uint64_t offset, size_t length, uint64_t iova, int fd, int access, int mlx5_access); }; /* Constructs MLX5 direct verbs symbols per rdma-core linking or dynamic loading mode */ ncclResult_t buildMlx5dvSymbols(struct ncclMlx5dvSymbols* mlx5dvSymbols); #endif // NCCL_MLX5DV_SYMBOLS_H_ nccl-2.29.7-1/src/include/mlx5/mlx5dvwrap.h000066400000000000000000000025541515037102200202530ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_MLX5DVWRAP_H_ #define NCCL_MLX5DVWRAP_H_ #include #include #ifdef NCCL_BUILD_MLX5DV #include #else #include "mlx5/mlx5dvcore.h" #endif #include "core.h" #include "ibvwrap.h" #include #include typedef enum mlx5dv_return_enum { MLX5DV_SUCCESS = 0, //!< The operation was successful } mlx5dv_return_t; ncclResult_t wrap_mlx5dv_symbols(void); /* NCCL wrappers of MLX5 direct verbs functions */ bool wrap_mlx5dv_is_supported(struct ibv_device *device); ncclResult_t wrap_mlx5dv_get_data_direct_sysfs_path(struct ibv_context *context, char *buf, size_t buf_len); /* DMA-BUF support */ ncclResult_t wrap_mlx5dv_reg_dmabuf_mr(struct ibv_mr **ret, struct ibv_pd *pd, uint64_t offset, size_t length, uint64_t iova, int fd, int access, int mlx5_access); struct ibv_mr * wrap_direct_mlx5dv_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, size_t length, uint64_t iova, int fd, int access, int mlx5_access); #endif // NCCL_MLX5DVWRAP_H_ nccl-2.29.7-1/src/include/mnnvl.h000066400000000000000000000007511515037102200164040ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_MNNVL_H_ #define NCCL_MNNVL_H_ #include "nccl.h" #include "comm.h" ncclResult_t ncclMnnvlCheck(struct ncclComm* comm); #endif nccl-2.29.7-1/src/include/nccl_common.h000066400000000000000000000041201515037102200175330ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_DEBUG_H_ #define NCCL_DEBUG_H_ // Workaround for libstdc++ trying to force public visibility of std:: symbols. We don't want to do that in libnccl.so. #include #undef _GLIBCXX_VISIBILITY #define _GLIBCXX_VISIBILITY(V) #include typedef enum { NCCL_LOG_NONE = 0, NCCL_LOG_VERSION = 1, NCCL_LOG_WARN = 2, NCCL_LOG_INFO = 3, NCCL_LOG_ABORT = 4, NCCL_LOG_TRACE = 5 } ncclDebugLogLevel; typedef enum { NCCL_INIT = 0x1, NCCL_COLL = 0x2, NCCL_P2P = 0x4, NCCL_SHM = 0x8, NCCL_NET = 0x10, NCCL_GRAPH = 0x20, NCCL_TUNING = 0x40, NCCL_ENV = 0x80, NCCL_ALLOC = 0x100, NCCL_CALL = 0x200, NCCL_PROXY = 0x400, NCCL_NVLS = 0x800, NCCL_BOOTSTRAP = 0x1000, NCCL_REG = 0x2000, NCCL_PROFILE = 0x4000, NCCL_RAS = 0x8000, NCCL_ALL = ~0 } ncclDebugLogSubSys; typedef void (*ncclDebugLogger_t)(ncclDebugLogLevel level, unsigned long flags, const char *file, int line, const char *fmt, ...); // NCCL core profiler callback for network defined events instrumentation enum { ncclProfilerNetEventStart = 0, ncclProfilerNetEventStop, ncclProfilerNetEventUpdate, ncclProfilerNetEventUpdateAndStop, }; typedef ncclResult_t (*ncclProfilerCallback_t)(void** eHandle, int type, void* pHandle, int64_t pluginId, void* extData); #define NCCL_NUM_FUNCTIONS 5 // Send/Recv not included for now typedef enum { ncclFuncBroadcast = 0, ncclFuncReduce = 1, ncclFuncAllGather = 2, ncclFuncReduceScatter = 3, ncclFuncAllReduce = 4, ncclFuncSendRecv = 5, ncclFuncSend = 6, ncclFuncRecv = 7, ncclFuncAlltoAll = 8, ncclFuncScatter = 9, ncclFuncGather = 10, ncclFuncAllGatherV = 11, ncclFuncPutSignal = 12, ncclFuncSignal = 13, ncclFuncWaitSignal = 14, ncclNumFuncs = 15 } ncclFunc_t; #endif nccl-2.29.7-1/src/include/nccl_device.h000066400000000000000000000024751515037102200175150ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef NCCL_HOSTLIB_ONLY #include "nccl_device/coop.h" #include "nccl_device/impl/barrier__funcs.h" #include "nccl_device/impl/comm__funcs.h" #include "nccl_device/impl/core__funcs.h" #include "nccl_device/impl/ll_a2a__funcs.h" #include "nccl_device/impl/lsa_barrier__funcs.h" #include "nccl_device/impl/gin__funcs.h" #include "nccl_device/impl/gin_barrier__funcs.h" #include "nccl_device/impl/ptr__funcs.h" #include "nccl_device/impl/reduce_copy__funcs.h" #else // Include the types and declaration if NCCL_HOSTLIB_ONLY is defined #include "nccl_device/coop.h" #include "nccl_device/core.h" #include "nccl_device/ll_a2a.h" #include "nccl_device/barrier.h" #include "nccl_device/ptr.h" #include "nccl_device/reduce_copy.h" #include "nccl_device/impl/comm__types.h" #include "nccl_device/impl/core__types.h" #include "nccl_device/impl/ll_a2a__types.h" #include "nccl_device/impl/barrier__types.h" #include "nccl_device/impl/ptr__types.h" #include "nccl_device/impl/reduce_copy__types.h" #endif nccl-2.29.7-1/src/include/nccl_device/000077500000000000000000000000001515037102200173345ustar00rootroot00000000000000nccl-2.29.7-1/src/include/nccl_device/README.md000066400000000000000000000031511515037102200206130ustar00rootroot00000000000000This directory has been structured to make it easy for user to read the headers to learn the API. The files adjacent to this README are meant for humans. They contain the essential declarations like which types exist and function prototypes and comments indicating the contract/usage. Everything else goes into the "impl/" subdirectory. Most modules are stratified into three layers: 1) "foo.h" Public API declarations. 2) "impl/foo__types.h" struct definitions. Has #include of layer 1. 3) "impl/foo_funcs.h" inline functions. Has #include of layer 2. The include dependencies should be acyclic for layers 1 and 2 since order matters for declarations and types. Layer 3 though can freely have cycles amongst itself ("impl/foo__funcs.h" and "impl/bar__funcs.h" can mutually include each other) since functions can be defined in any order once declared. Translation units should just include "nccl_device.h" to ensure they get all the "impl/foo__funcs.h". But if a translation unit wants to be more specific as to which module it pulls in it should include "impl/foo__funcs.h". One of the nasty reasons this was required is because of C++ defaulted function parameters: ``` // +++ in foo.h +++ struct Foo; // defined in some __types.h // +++ in "impl/foo__types.h" +++ struct Foo { int x; }; // +++ in "bar.h" +++ // Prototype function where default value is default construction of Foo. Since // Foo would be incomplete if just including "foo.h" the compiler errors because // it can't reason about the {}. // I was able to solve this by including "impl/foo__types.h" instead. #include "impl/foo__types.h" void bar(Foo arg = {}); ``` nccl-2.29.7-1/src/include/nccl_device/barrier.h000066400000000000000000000032701515037102200211350ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_DEVICE_BARRIER_H_ #define _NCCL_DEVICE_BARRIER_H_ #include "impl/core__types.h" #include "impl/lsa_barrier__types.h" #include "impl/gin_barrier__types.h" #if NCCL_CHECK_CUDACC template struct ncclBarrierSession_internal; template struct ncclBarrierSession: ncclBarrierSession_internal { // Full featured constructor: NCCL_DEVICE_INLINE ncclBarrierSession( Coop, ncclTeam innerTeam, ncclTeam outerTeam, ncclGin, ncclLsaBarrierHandle innerBarHandle, ncclGinBarrierHandle outerBarHandle, uint32_t index, bool multimem=false, ncclMultimemHandle innerMmHandle={} ); // Convenience constructors for baked in teams: NCCL_DEVICE_INLINE ncclBarrierSession( Coop, ncclTeamTagWorld, ncclGin, uint32_t index, bool multimem=false ); NCCL_DEVICE_INLINE ncclBarrierSession( Coop, ncclTeamTagLsa, ncclDevComm const&, uint32_t index, bool multimem=false ); NCCL_DEVICE_INLINE ncclBarrierSession( Coop, ncclTeamTagRail, ncclGin, uint32_t index ); ncclBarrierSession(ncclBarrierSession const&) = delete; // Sessions are not copyable NCCL_DEVICE_INLINE ncclLsaBarrierSession& lsaBarrier(); NCCL_DEVICE_INLINE ncclGinBarrierSession& ginBarrier(); NCCL_DEVICE_INLINE void sync(Coop, cuda::memory_order, ncclGinFenceLevel); }; #endif #endif // _NCCL_DEVICE_BARRIER_H_ nccl-2.29.7-1/src/include/nccl_device/comm.h000066400000000000000000000006561515037102200204470ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_DEVICE_COMM_H_ #define _NCCL_DEVICE_COMM_H_ #include "core.h" #endif nccl-2.29.7-1/src/include/nccl_device/coop.h000066400000000000000000000203101515037102200204410ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_DEVICE_COOP_H_ #define _NCCL_DEVICE_COOP_H_ #include "utility.h" // ncclCoop[Foo]: NCCL's versions of CUDA's Cooperative Groups. They conform // to just this subset of the CUDA API: // int Coop::thread_rank(); // int Coop::size(); // int Coop::num_threads(); // same as size() // void Coop::sync(); #if NCCL_CHECK_CUDACC struct ncclCoopAny { struct Storage { alignas(alignof(void*)) char space[16]; }; struct VTable { int(*thread_rank)(void const*); int(*size)(void const*); void(*sync)(void*); }; template __device__ static int thread_rank(void const* o) { return static_cast(o)->thread_rank(); } template __device__ static int size(void const* o) { return static_cast(o)->size(); } template __device__ static void sync(void* o) { static_cast(o)->sync(); } template __device__ static VTable const* get_vtable() { static_assert(sizeof(Impl) <= sizeof(Storage), "Incompatible coop type size"); static_assert(alignof(Impl) <= alignof(Storage), "Incompatible coop type alignment"); static constexpr VTable v = { &thread_rank, &size, &sync }; return &v; } Storage storage; VTable const* vtable; ncclCoopAny(ncclCoopAny const&) = default; ncclCoopAny(ncclCoopAny&&) = default; ncclCoopAny() = default; template __device__ ncclCoopAny(Impl impl) { ::new (&this->storage) Impl(impl); this->vtable = get_vtable(); } __device__ int thread_rank() const { return vtable->thread_rank(&storage); } __device__ int size() const { return vtable->size(&storage); } __device__ int num_threads() const { return vtable->size(&storage); } __device__ void sync() { vtable->sync(&storage); } }; #endif #if NCCL_CHECK_CUDACC template struct ncclCoopTile { // An aligned pow2 set of threads within the warp. static_assert(nccl::utility::isPow2(nThreadsPow2) && nThreadsPow2 <= 32, "Condition required"); NCCL_DEVICE_INLINE int thread_rank() const { return nccl::utility::lane() % nThreadsPow2; } NCCL_DEVICE_INLINE constexpr int size() const { return nThreadsPow2; } NCCL_DEVICE_INLINE constexpr int num_threads() const { return nThreadsPow2; } NCCL_DEVICE_INLINE uint32_t laneMask() const { return (-1u>>(32-nThreadsPow2))<<(nccl::utility::lane() & -nThreadsPow2); } NCCL_DEVICE_INLINE void sync() { if (nThreadsPow2 > 1) __syncwarp(laneMask()); } }; #endif #if NCCL_CHECK_CUDACC typedef ncclCoopTile<1> ncclCoopThread; typedef ncclCoopTile<32> ncclCoopWarp; #endif #if NCCL_CHECK_CUDACC struct ncclCoopLanes { // Some lanes of this warp. uint32_t lmask; NCCL_DEVICE_INLINE constexpr ncclCoopLanes(uint32_t lmask=-1u): lmask(lmask) {} NCCL_DEVICE_INLINE int thread_rank() const { return __popc(lmask & nccl::utility::lanemask_lt()); } NCCL_DEVICE_INLINE int size() const { return __popc(lmask); } NCCL_DEVICE_INLINE int num_threads() const { return __popc(lmask); } NCCL_DEVICE_INLINE void sync() { __syncwarp(lmask); } }; #endif #if NCCL_CHECK_CUDACC // A set of consecutive warps that the user has also supplied with a unique // id from [0..15]. It is an error for two different warp spans with the same // id to be in a collective concurrently. struct ncclCoopWarpSpan { uint32_t warp0:8, nWarps:8, id:8; NCCL_DEVICE_INLINE constexpr ncclCoopWarpSpan(int warp0, int nWarps, int id): warp0(warp0), nWarps(nWarps), id(id) { } NCCL_DEVICE_INLINE int thread_rank() const { return threadIdx.x - 32*warp0; } NCCL_DEVICE_INLINE int size() const { return 32*nWarps; } NCCL_DEVICE_INLINE int num_threads() const { return 32*nWarps; } NCCL_DEVICE_INLINE void sync() { //asm volatile("barrier.sync %0, %1;" :: "r"(1+id), "r"(32*nWarps) : "memory"); __barrier_sync_count(1+id, 32*nWarps); } }; #endif #if NCCL_CHECK_CUDACC struct ncclCoopCta { NCCL_DEVICE_INLINE int thread_rank() const { return threadIdx.x; } NCCL_DEVICE_INLINE int size() const { return blockDim.x; } NCCL_DEVICE_INLINE int num_threads() const { return blockDim.x; } NCCL_DEVICE_INLINE void sync() { __syncthreads(); } }; #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE uint32_t ncclCoopGetLaneMask(ncclCoopTile coop) { return coop.laneMask(); } NCCL_DEVICE_INLINE uint32_t ncclCoopGetLaneMask(ncclCoopLanes coop) { return coop.lmask; } NCCL_DEVICE_INLINE uint32_t ncclCoopGetLaneMask(ncclCoopWarpSpan coop) { return -1u; } NCCL_DEVICE_INLINE uint32_t ncclCoopGetLaneMask(ncclCoopCta coop) { return -1u; } #endif #if NCCL_CHECK_CUDACC // ncclCoopIsThread: // At compile time do we know the given coop is a single thread only. template NCCL_DEVICE_INLINE constexpr bool ncclCoopIsThread(ncclCoopTile) { return nThreads == 1; } NCCL_DEVICE_INLINE constexpr bool ncclCoopIsThread(ncclCoopLanes) { return false; } NCCL_DEVICE_INLINE constexpr bool ncclCoopIsThread(ncclCoopWarpSpan) { return false; } NCCL_DEVICE_INLINE constexpr bool ncclCoopIsThread(ncclCoopCta) { return false; } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE constexpr bool ncclCoopWithinWarp(ncclCoopTile) { return true; } NCCL_DEVICE_INLINE constexpr bool ncclCoopWithinWarp(ncclCoopLanes) { return true; } NCCL_DEVICE_INLINE constexpr bool ncclCoopWithinWarp(ncclCoopWarpSpan) { return false; } NCCL_DEVICE_INLINE constexpr bool ncclCoopWithinWarp(ncclCoopCta) { return false; } #endif #if NCCL_CHECK_CUDACC // Pick threads of our warp that are safe to use collectively. NCCL_DEVICE_INLINE ncclCoopLanes ncclCoopCoalesced() { return ncclCoopLanes{__activemask()}; } #endif #if NCCL_CHECK_CUDACC // Pick threads of our warp that are safe to use collectively given that this // is a collective on the provided cooperative group. template NCCL_DEVICE_INLINE ncclCoopTile<32> ncclCoopCoalesced(Coop) { return ncclCoopTile<32>(); } NCCL_DEVICE_INLINE ncclCoopLanes ncclCoopCoalesced(ncclCoopLanes coop) { return coop; } template NCCL_DEVICE_INLINE ncclCoopTile ncclCoopCoalesced(ncclCoopTile coop) { return coop; } #endif #if NCCL_CHECK_CUDACC template NCCL_DEVICE_INLINE T ncclCoopBcast(ncclCoopTile, T value, int root, bool entrySync=true) { constexpr int n = (sizeof(T)+4-1)/4; union { uint32_t u[n]; T v; }; v = value; #pragma unroll for (int i=0; i < n; i++) u[i] = __shfl_sync(-1u, u[i], root, nThreads); return v; } template NCCL_DEVICE_INLINE T ncclCoopBcast(ncclCoopLanes coop, T value, int root, bool entrySync=true) { uint32_t m = coop.lmask; uint32_t r = root == 0 ? __ffs(m)-1 : __fns(m, 0, 1+root); constexpr int n = (sizeof(T)+4-1)/4; union { uint32_t u[n]; T v; }; v = value; #pragma unroll for (int i=0; i < n; i++) u[i] = __shfl_sync(m, u[i], r); return v; } NCCL_DEVICE_INLINE ulong2* ncclCoopBcast_WarpSpan_stash() { __shared__ ulong2 stash[15]; return stash; } template NCCL_DEVICE_INLINE T ncclCoopBcast(ncclCoopWarpSpan coop, T value, int root, bool entrySync=true) { static_assert(sizeof(T) <= sizeof(ncclCoopBcast_WarpSpan_stash()[0]), "Required"); if (entrySync) coop.sync(); if (coop.thread_rank() == root) *(T*)&ncclCoopBcast_WarpSpan_stash()[coop.id] = value; coop.sync(); return *(T*)&ncclCoopBcast_WarpSpan_stash()[coop.id]; } NCCL_DEVICE_INLINE ulong2* ncclCoopBcast_Cta_stash() { __shared__ ulong2 stash; return &stash; } template NCCL_DEVICE_INLINE T ncclCoopBcast(ncclCoopCta coop, T value, int root, bool entrySync=true) { static_assert(sizeof(T) <= sizeof(*ncclCoopBcast_Cta_stash()), "Required"); if (entrySync) coop.sync(); if (coop.thread_rank() == root) *(T*)ncclCoopBcast_Cta_stash() = value; coop.sync(); return *(T*)ncclCoopBcast_Cta_stash(); } #endif #endif nccl-2.29.7-1/src/include/nccl_device/core.h000066400000000000000000000244101515037102200204360ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_DEVICE_CORE_H_ #define _NCCL_DEVICE_CORE_H_ #include #include "coop.h" #include "utility.h" struct ncclDevComm; typedef struct ncclDevComm ncclDevComm_t; struct ncclTeam; typedef struct ncclTeam ncclTeam_t; // typedef struct ncclWindow_vidmem* ncclWindow_t; // in nccl.h struct ncclMultimemHandle; typedef struct ncclMultimemHandle ncclMultimemHandle_t; typedef uint32_t ncclDevResourceHandle; typedef ncclDevResourceHandle ncclDevResourceHandle_t; typedef uint32_t ncclGinSignal_t; typedef uint32_t ncclGinCounter_t; struct ncclLsaBarrierHandle; typedef struct ncclLsaBarrierHandle ncclLsaBarrierHandle_t; struct ncclGinBarrierHandle; typedef struct ncclGinBarrierHandle ncclGinBarrierHandle_t; struct ncclLLA2AHandle; typedef struct ncclLLA2AHandle ncclLLA2AHandle_t; struct ncclTeam { int nRanks, rank, stride; }; #if __cplusplus template struct ncclSymPtr; #endif #if __cplusplus struct ncclTeamTagWorld {}; struct ncclTeamTagLsa {}; struct ncclTeamTagRail {}; #endif struct ncclDevCommRequirements; typedef struct ncclDevCommRequirements ncclDevCommRequirements_t; struct ncclDevResourceRequirements; typedef struct ncclDevResourceRequirements ncclDevResourceRequirements_t; struct ncclTeamRequirements; typedef struct ncclTeamRequirements ncclTeamRequirements_t; struct ncclCommProperties; typedef struct ncclCommProperties ncclCommProperties_t; typedef enum { NCCL_GIN_CONNECTION_NONE, NCCL_GIN_CONNECTION_FULL, NCCL_GIN_CONNECTION_RAIL, } ncclGinConnectionType_t; struct ncclDevCommRequirements { /* attributes that users should never touch. */ size_t size; unsigned int magic; unsigned int version; /* attributes that users are able to customize. */ ncclDevResourceRequirements_t* resourceRequirementsList; ncclTeamRequirements_t* teamRequirementsList; bool lsaMultimem; // Enable multimem on lsa team int barrierCount; int lsaBarrierCount; int railGinBarrierCount; int lsaLLA2ABlockCount, lsaLLA2ASlotCount; bool ginForceEnable; int ginContextCount; // This is a hint, the actual context count in the devcomm may not match. int ginSignalCount; // Guaranteed to start at id=0 int ginCounterCount; // Guaranteed to start at id=0 ncclGinConnectionType_t ginConnectionType; bool ginExclusiveContexts; int ginQueueDepth; }; #define NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER { \ sizeof(ncclDevCommRequirements_t), /* size */ \ NCCL_API_MAGIC, /* magic */ \ NCCL_VERSION(NCCL_MAJOR, NCCL_MINOR, NCCL_PATCH), /* version */ \ nullptr, /* resourceRequirementsList*/ \ nullptr, /* teamRequirementsList */ \ false, /* lsaMultimem */ \ 0, /* barrierCount */ \ 0, /* lsaBarrierCount */ \ 0, /* railGinBarrierCount */ \ 0, /* lsaLLA2ABlockCount */ \ 0, /* lsaLLA2ASlotCount */ \ false, /* ginForceEnable */ \ 4, /* ginContextCount */ \ 0, /* ginSignalCount */ \ 0, /* ginCounterCount */ \ NCCL_GIN_CONNECTION_NONE, /* ginConnectionType */ \ false, /* ginExclusiveContexts */ \ 0, /* ginQueueDepth */ \ } struct ncclDevResourceRequirements { ncclDevResourceRequirements_t* next; size_t bufferSize, bufferAlign; ncclDevResourceHandle_t* outBufferHandle; // If non-null, target assigned during ncclDevCommCreate. int ginSignalCount; int ginCounterCount; ncclGinSignal_t* outGinSignalStart; ncclGinCounter_t* outGinCounterStart; }; struct ncclTeamRequirements { ncclTeamRequirements_t* next; ncclTeam_t team; bool multimem; ncclMultimemHandle_t* outMultimemHandle; // If non-null, target assigned during ncclDevCommCreate. }; #define NCCL_COMM_PROPERTIES_INITIALIZER { \ sizeof(ncclCommProperties_t), /* size */ \ NCCL_API_MAGIC, /* magic */ \ NCCL_VERSION(NCCL_MAJOR, NCCL_MINOR, NCCL_PATCH), /* version */ \ } typedef enum { NCCL_GIN_TYPE_NONE = 0, NCCL_GIN_TYPE_PROXY = 2, // intentially not 1. Must match NCCL_NET_DEVICE_GIN_PROXY for backward compatibility NCCL_GIN_TYPE_GDAKI = 3, // intentially not 2. Must match NCCL_NET_DEVICE_GIN_GDAKI for backward compatibility } ncclGinType_t; struct ncclCommProperties { /* internal use only */ size_t size; unsigned int magic; unsigned int version; /* attributes for users. */ int rank; int nRanks; int cudaDev; int nvmlDev; bool deviceApiSupport; bool multimemSupport; ncclGinType_t ginType; int nLsaTeams; bool hostRmaSupport; ncclGinType_t railedGinType; }; NCCL_EXTERN_C __host__ ncclResult_t ncclCommQueryProperties(ncclComm_t, ncclCommProperties_t*); NCCL_EXTERN_C __host__ ncclResult_t ncclDevCommCreate(ncclComm_t, ncclDevCommRequirements_t const*, ncclDevComm_t* outDevComm); NCCL_EXTERN_C __host__ ncclResult_t ncclDevCommDestroy(ncclComm_t, ncclDevComm_t const* devComm); NCCL_EXTERN_C __host__ ncclResult_t ncclGetLsaMultimemDevicePointer(ncclWindow_t window, size_t offset, void** outPtr); NCCL_EXTERN_C __host__ ncclResult_t ncclGetMultimemDevicePointer(ncclWindow_t window, size_t offset, ncclMultimemHandle multimem, void** outPtr); NCCL_EXTERN_C __host__ ncclResult_t ncclGetLsaDevicePointer(ncclWindow_t window, size_t offset, int lsaRank, void** outPtr); NCCL_EXTERN_C __host__ ncclResult_t ncclGetPeerDevicePointer(ncclWindow_t window, size_t offset, int peer, void** outPtr); //////////////////////////////////////////////////////////////////////////////// // Team API: #if __cplusplus NCCL_IR_EXTERN_C NCCL_HOST_DEVICE_INLINE ncclTeam ncclTeamWorld(ncclDevComm const&); #endif #ifndef __clang_llvm_bitcode_lib__ NCCL_EXTERN_C __host__ ncclTeam_t ncclTeamWorld(ncclComm_t); #endif #if __cplusplus NCCL_IR_EXTERN_C NCCL_HOST_DEVICE_INLINE ncclTeam ncclTeamLsa(ncclDevComm const&); #endif #ifndef __clang_llvm_bitcode_lib__ NCCL_EXTERN_C __host__ ncclTeam_t ncclTeamLsa(ncclComm_t); #endif NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE bool ncclTeamRankIsMember(ncclTeam_t a, ncclTeam_t b, int bPeer); NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE int ncclTeamRankToTeam(ncclTeam_t a, ncclTeam_t b, int bPeer); #if __cplusplus NCCL_IR_EXTERN_C NCCL_HOST_DEVICE_INLINE int ncclTeamRankToWorld(ncclDevComm const&, ncclTeam, int rank); #endif #ifndef __clang_llvm_bitcode_lib__ NCCL_EXTERN_C __host__ int ncclTeamRankToWorld(ncclComm_t, ncclTeam_t, int rank); #endif #if __cplusplus NCCL_IR_EXTERN_C NCCL_HOST_DEVICE_INLINE int ncclTeamRankToLsa(ncclDevComm const&, ncclTeam, int rank); #endif #ifndef __clang_llvm_bitcode_lib__ NCCL_EXTERN_C __host__ int ncclTeamRankToLsa(ncclComm_t, ncclTeam_t, int rank); #endif NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE ncclTeam_t ncclTeamInnerFactor(ncclTeam_t parent, int innerSize); NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE ncclTeam_t ncclTeamOuterFactor(ncclTeam_t parent, int innerSize); // Interpret each team as a set of ranks. This function assumes that `subset` // is a subset of `parent`. Thus the number of ranks in the set difference of // `parent` minus `subset` is `super.nRanks - subset.nRanks`. Given `index` this // function returns the index'th element of `parent` minus `subset`. NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE int ncclTeamRankInDifference(ncclTeam_t parent, ncclTeam_t subset, int index); // Equivalent to ncclTeamOuterFactor of lsa team. #if __cplusplus NCCL_IR_EXTERN_C NCCL_HOST_DEVICE_INLINE ncclTeam ncclTeamRail(ncclDevComm const&); #endif #ifndef __clang_llvm_bitcode_lib__ NCCL_EXTERN_C __host__ ncclTeam_t ncclTeamRail(ncclComm_t); #endif // Get offset of resource buffer within `comm.resourceWindow`. NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE size_t ncclGetResourceBufferOffset(ncclDevResourceHandle_t); #if NCCL_CHECK_CUDACC NCCL_DEVICE_INLINE ncclSymPtr ncclGetResourceBuffer(ncclDevComm const&, ncclDevResourceHandle); #endif //////////////////////////////////////////////////////////////////////////////// // Window API: #if NCCL_CHECK_CUDACC NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetLocalPointer(ncclWindow_t w, size_t offset); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetLsaPointer(ncclWindow_t w, size_t offset, int peer); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetPeerPointer(ncclWindow_t w, size_t offset, int peer); NCCL_DEVICE_INLINE void* ncclGetPeerPointer(ncclWindow_t w, size_t offset, ncclTeam tm, int peer); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetMultimemPointer(ncclWindow_t w, size_t offset, ncclMultimemHandle mmHandle); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetLsaMultimemPointer(ncclWindow_t w, size_t offset, ncclDevComm const&); #endif #if NCCL_CHECK_CUDACC // Convenience for combining ncclGet***Pointer() with resource handle. NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetResourceBufferLocalPointer(ncclDevComm const&, ncclDevResourceHandle); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetResourceBufferLsaPointer(ncclDevComm const&, ncclDevResourceHandle, int peer); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetResourceBufferPeerPointer(ncclDevComm const&, ncclDevResourceHandle, ncclTeam, int peer); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetResourceBufferMultimemPointer(ncclDevComm const&, ncclDevResourceHandle, ncclMultimemHandle); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void* ncclGetResourceBufferLsaMultimemPointer(ncclDevComm const&, ncclDevResourceHandle); #endif #endif // _NCCL_DEVICE_CORE_H_ nccl-2.29.7-1/src/include/nccl_device/gin.h000066400000000000000000000344601515037102200202710ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_DEVICE_GIN_SESSION_H_ #define _NCCL_DEVICE_GIN_SESSION_H_ #include "core.h" #include "gin/gin_device_common.h" #if NCCL_CHECK_CUDACC struct ncclGinCtx; // Definition in nccl_device/gin/gin_device_host_common.h template struct ncclGinCtx_M; // ... struct ncclGinDescriptorSmem; // A type user allocates in __shared__ memory // Used as completion actions for ncclGinSession::put struct ncclGin_None {}; struct ncclGin_VASignalInc { ncclWindow_t signalWindow; size_t signalOffset; }; struct ncclGin_VASignalAdd { ncclWindow_t signalWindow; size_t signalOffset; uint64_t value; }; struct ncclGin_SignalAdd { ncclGinSignal_t signal; uint64_t value; }; // SignalInc: equivalent to SignalAdd{+1} except it may not be mixed with any // other signal operator without intervening signal reset(). Formally: for a // given signal, all operations between successive reset()'s of that signal must // either all be SignalInc or all not SignalInc. struct ncclGin_SignalInc { ncclGinSignal_t signal; }; // Support deferred: // struct ncclGin_SignalSet { ncclGinSignal_t signal; uint64_t value; }; struct ncclGin_CounterInc { ncclGinCounter_t counter; }; struct ncclGin_DescriptorSmem { ncclGinDescriptorSmem* descriptor; }; template struct ncclGin_BackendMask; template using ncclGin_BackendOne = ncclGin_BackendMask<(1u<<(int)backend)>; using ncclGin = ncclGin_BackendMask; #endif #if NCCL_CHECK_CUDACC struct ncclGin_C { ncclDevComm const& comm; uint32_t nConnections:8, connectionId:8, _ginBackend:8; uint32_t contextId; ////////////////////////////////////////////////////////////////////////////// // internal: void* _ginHandle; uint64_t* _signalShadows; unsigned backendMask; NCCL_DEVICE_INLINE ncclGin_C(ncclDevComm const& comm_, unsigned backendMask_, int contextIndex); }; // Helper init function that wraps placement new NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGin_C_init( ncclGin_C* net, unsigned backendMask, ncclDevComm const& comm, int contextIndex); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinPut( ncclGin_C* net, ncclTeam team, int peer, ncclWindow_t dstWin, size_t dstOffset, ncclWindow_t srcWin, size_t srcOffset, size_t bytes, bool isSignal, ncclGinSignal_t signalId, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, bool isCounter, ncclGinCounter_t counterId, ncclCoopAny coop, bool isDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope givenRelease, cuda::thread_scope requiredRelease); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinSignal( ncclGin_C* net, ncclTeam team, int peer, bool isSignal, ncclGinSignal_t signalId, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, ncclCoopAny coop, bool isDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope givenRelease, cuda::thread_scope requiredRelease); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinPut_v2( ncclGin_C* net, ncclTeam team, int peer, ncclWindow_t dstWin, size_t dstOffset, ncclWindow_t srcWin, size_t srcOffset, size_t bytes, bool isSignal, ncclGinSignal_t signalId, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, bool isCounter, ncclGinCounter_t counterId, ncclCoopAny coop, bool isDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope givenRelease, cuda::thread_scope requiredRelease, uint32_t optFlags); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinSignal_v2( ncclGin_C* net, ncclTeam team, int peer, bool isSignal, ncclGinSignal_t signalId, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, ncclCoopAny coop, bool isDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope givenRelease, cuda::thread_scope requiredRelease, uint32_t optFlags); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinFlush( ncclGin_C* net, ncclCoopAny coop, cuda::memory_order ord); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE uint64_t ncclGinReadCounter( ncclGin_C* net, ncclGinCounter_t counter, int bits, cuda::memory_order ord); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinWaitCounter( ncclGin_C* net, ncclCoopAny coop, ncclGinCounter_t counter, uint64_t least, int bits, cuda::memory_order ord); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE uint64_t ncclGinReadSignal( ncclGin_C* net, ncclGinSignal_t signal, int bits, cuda::memory_order ord); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinWaitSignal( ncclGin_C* net, ncclCoopAny coop, ncclGinSignal_t signal, uint64_t least, int bits, cuda::memory_order ord); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinResetCounter( ncclGin_C* net, ncclGinCounter_t counter); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinResetSignal( ncclGin_C* net, ncclGinSignal_t signal); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinPutValue( ncclGin_C* net, ncclTeam team, int peer, ncclWindow_t dstWin, size_t dstOffset, uint64_t value, size_t size, bool isSignal, ncclGinSignal_t signalId, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, ncclCoopAny coop, bool isDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope givenRelease, cuda::thread_scope requiredRelease); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE void ncclGinPutValue_v2( ncclGin_C* net, ncclTeam team, int peer, ncclWindow_t dstWin, size_t dstOffset, uint64_t value, size_t size, bool isSignal, ncclGinSignal_t signalId, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, ncclCoopAny coop, bool isDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope givenRelease, cuda::thread_scope requiredRelease, uint32_t optFlags); NCCL_IR_EXTERN_C NCCL_DEVICE_INLINE uint64_t* ncclGinGetSignalShadowPtr( ncclGin_C* net, ncclGinSignal_t signal); template struct ncclGin_BackendMask { ncclDevComm const& comm; uint32_t nConnections:8, connectionId:8, _ginBackend:8; uint32_t contextId; // Loads GIN context into registers. Each context has one QP per peer. NCCL_DEVICE_INLINE ncclGin_BackendMask(ncclDevComm const&, int contextIndex); template< // Action to take on peer when put completes. If a signalling action is used // then that signal will be visible only after the payload of this put as well as // the payloads of preceding puts on this netContext to the same peer are settled. typename RemoteAction = ncclGin_None, // one of ncclGin_{None|SignalInc|SignalAdd|SignalSet} // Action to take locally when source has been consumed. typename LocalAction = ncclGin_None, // one of ncclGin_{None|CounterInc} // Set of threads participating in this put. Must be a subset of Coop. typename Coop = ncclCoopThread, // Optional smem descriptor space to use. Either ncclGin_{None|DescriptorSmem} typename DescriptorSmem = ncclGin_None > NCCL_DEVICE_INLINE void put( ncclTeam, int peer, ncclWindow_t dstWnd, size_t dstOffset, ncclWindow_t srcWnd, size_t srcOffset, size_t bytes, RemoteAction remoteAction = ncclGin_None{}, LocalAction localAction = ncclGin_None{}, Coop coop = ncclCoopThread{}, DescriptorSmem descriptor = ncclGin_None{}, cuda::thread_scope givenRelease = cuda::thread_scope_thread, cuda::thread_scope requiredRelease = cuda::thread_scope_device, uint32_t optFlags = ncclGinOptFlagsDefault ) const; template< typename T, // Action to take on peer when put completes. If a signalling action is used // then that signal will be visible only after the payload of this put as well as // the payloads of preceding puts on this context to the same peer are settled. typename RemoteAction = ncclGin_None, // one of ncclGin_{None|SignalInc|SignalAdd|SignalSet} // Action to take locally when source has been consumed. typename LocalAction = ncclGin_None, // one of ncclGin_{None|CounterInc} // Set of threads participating in this put. Must be a subset of Coop. typename Coop = ncclCoopThread, // Optional smem descriptor space to use. Either ncclGin_{None|DescriptorSmem} typename DescriptorSmem = ncclGin_None > NCCL_DEVICE_INLINE void put( ncclTeam, int peer, ncclSymPtr dstElts, ncclSymPtr srcElts, size_t nElts, RemoteAction remoteAction = ncclGin_None{}, LocalAction localAction = ncclGin_None{}, Coop coop = ncclCoopThread{}, DescriptorSmem descriptor = ncclGin_None{}, cuda::thread_scope givenRelease = cuda::thread_scope_thread, cuda::thread_scope requiredRelease = cuda::thread_scope_device, uint32_t optFlags = ncclGinOptFlagsDefault ) const; template< typename T, // requires sizeof(T) <= 8 // See put() for all template arguments. typename RemoteAction = ncclGin_None, typename Coop = ncclCoopThread, typename DescriptorSmem = ncclGin_None > NCCL_DEVICE_INLINE void putValue( ncclTeam, int peer, ncclWindow_t dstWnd, size_t dstOffset, T value, RemoteAction remoteAction = ncclGin_None{}, Coop coop = ncclCoopThread{}, DescriptorSmem descriptor = ncclGin_None{}, cuda::thread_scope givenRelease = cuda::thread_scope_thread, cuda::thread_scope requiredRelease = cuda::thread_scope_device, uint32_t optFlags = ncclGinOptFlagsDefault ) const; template< typename T, // requires sizeof(T) <= 8 // See put() for all template arguments. typename RemoteAction = ncclGin_None, typename Coop = ncclCoopThread, typename DescriptorSmem = ncclGin_None > NCCL_DEVICE_INLINE void putValue( ncclTeam, int peer, ncclSymPtr dst, T value, RemoteAction remoteAction = ncclGin_None{}, Coop coop = ncclCoopThread{}, DescriptorSmem descriptor = ncclGin_None{}, cuda::thread_scope givenRelease = cuda::thread_scope_thread, cuda::thread_scope requiredRelease = cuda::thread_scope_device, uint32_t optFlags = ncclGinOptFlagsDefault ) const; template NCCL_DEVICE_INLINE void signal( ncclTeam, int peer, RemoteAction remoteAction, Coop coop = ncclCoopThread(), DescriptorSmem descriptor = ncclGin_None{}, cuda::thread_scope givenRelease = cuda::thread_scope_thread, cuda::thread_scope requiredRelease = cuda::thread_scope_device, uint32_t optFlags = ncclGinOptFlagsDefault ) const; // All source buffers from put's from any thread in this coop will be safe to reuse. // Flush does not guarantee that data has settled in remote memory. template NCCL_DEVICE_INLINE void flush(Coop, cuda::memory_order ord = cuda::memory_order_acquire) const; // Counter and signal wait use "rolling" comparison logic of a given bit-width // such that unsigned overflow does not disturb the property that: x < x+1. // // bool rolling_less_equal(uint64_t a, uint64_t b, int bits) { // uint64_t m = uint64_t(-1)>>(64-bits); // return ((b-a) & m) <= (m>>1); // } // // The condition waited for is that the supplied value is rolling_less_equal // to the internal value. // // Counters are restricted to using a maximum of 56 bits despite that being fewer // than a uint64_t can carry. NCCL_DEVICE_INLINE uint64_t readCounter(ncclGinCounter_t counter, int bits=56, cuda::memory_order ord = cuda::memory_order_acquire) const; template NCCL_DEVICE_INLINE void waitCounter(Coop, ncclGinCounter_t counter, uint64_t least, int bits=56, cuda::memory_order ord = cuda::memory_order_acquire) const; // Each signal has a dedicated "shadow" which the user is free to manipulate for // any reason. The only calls which manipulate the shadow are `increaseSignalShadow` // and `resetSignal`. NCCL_DEVICE_INLINE uint64_t* getSignalShadowPtr(ncclGinSignal_t signal) const; NCCL_DEVICE_INLINE void increaseSignalShadow(ncclGinSignal_t signal, uint64_t delta) const; // Returns current value of signal with all but bottom bits set to zero. NCCL_DEVICE_INLINE uint64_t readSignal(ncclGinSignal_t signal, int bits=64, cuda::memory_order ord = cuda::memory_order_acquire) const; // Returns current value of VA signal at given window and offset with all but bottom bits set to zero. NCCL_DEVICE_INLINE uint64_t readSignal(ncclWindow_t signalWindow, size_t signalOffset, int bits=64, cuda::memory_order ord = cuda::memory_order_acquire) const; // Wait for signal to meet or exceed value. template NCCL_DEVICE_INLINE void waitSignal(Coop, ncclGinSignal_t signal, uint64_t least, int bits=64, cuda::memory_order ord = cuda::memory_order_acquire) const; // Wait for VA signal at given window and offset to meet or exceed value. template NCCL_DEVICE_INLINE void waitSignal(Coop, ncclWindow_t signalWindow, size_t signalOffset, uint64_t least, int bits=64, cuda::memory_order ord = cuda::memory_order_acquire) const; // Wait for signal to meet or exceed shadow value. template NCCL_DEVICE_INLINE void waitSignalMeetShadow(Coop, ncclGinSignal_t signal, int bits=64, cuda::memory_order ord = cuda::memory_order_acquire) const; // Wait until signal exceeds shadow by `leastDelta` (typically 1), updates shadow // with latest value, and returns with `before` equal to previous shadow value // and `delta` equal to difference. template NCCL_DEVICE_INLINE void waitSignalFollowShadow(Coop, ncclGinSignal_t signal, Uint leastDelta, Uint* before, Uint* delta, int bits=64, cuda::memory_order ord = cuda::memory_order_acquire) const; // Sets to zero. May not race with concurrent modifications to counter. NCCL_DEVICE_INLINE void resetCounter(ncclGinCounter_t counter) const; // Sets signal and shadow to zero. May not race with concurrent modifcations to signal. NCCL_DEVICE_INLINE void resetSignal(ncclGinSignal_t signal) const; // Resets a VA signal at the given window and offset. NCCL_DEVICE_INLINE void resetSignal(ncclWindow_t signalWindow, size_t signalOffset) const; ////////////////////////////////////////////////////////////////////////////// // internal: void* _ginHandle; uint64_t* _signalShadows; NCCL_DEVICE_INLINE ncclGinCtx_M _makeCtx() const; }; #endif #endif // _NCCL_DEVICE_GIN_SESSION_H_ nccl-2.29.7-1/src/include/nccl_device/gin/000077500000000000000000000000001515037102200201115ustar00rootroot00000000000000nccl-2.29.7-1/src/include/nccl_device/gin/gdaki/000077500000000000000000000000001515037102200211705ustar00rootroot00000000000000nccl-2.29.7-1/src/include/nccl_device/gin/gdaki/gin_gdaki.h000066400000000000000000000326021515037102200232600ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_DEVICE_GIN_GDAKI_H_ #define _NCCL_DEVICE_GIN_GDAKI_H_ #include #ifndef DOCA_VERBS_USE_CUDA_WRAPPER #define DOCA_VERBS_USE_CUDA_WRAPPER #endif #ifndef DOCA_VERBS_USE_NET_WRAPPER #define DOCA_VERBS_USE_NET_WRAPPER #endif #ifdef NCCL_DEVICE_GIN_GDAKI_ENABLE_DEBUG #define DOCA_GPUNETIO_VERBS_ENABLE_DEBUG 1 #endif #include "../gin_device_common.h" #include "gin_gdaki_device_host_common.h" #include "doca_gpunetio/doca_gpunetio_device.h" #ifdef NCCL_DEVICE_GIN_GDAKI_ENABLE_DEBUG #include #endif namespace nccl { namespace gin { namespace gdaki { template NCCL_DEVICE_INLINE static void putImpl(ncclGinCtx ctx, Coop coop, int peer, bool hasWins, ncclGinWindow_t dstWin, size_t dstOff, ncclGinWindow_t srcWin, size_t srcOff, size_t bytes, bool hasSignal, size_t signalOffset, __be32 signalKey, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, bool hasCounter, ncclGinCounter_t counterId, bool hasDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope required, cuda::thread_scope given, uint32_t optFlags) { using nccl::utility::loadConst; coop.sync(); if (coop.thread_rank() == 0) { ncclGinGdakiGPUContext* gdaki = &((struct ncclGinGdakiGPUContext*)ctx.handle)[ctx.contextId]; doca_gpu_dev_verbs_qp* qp = loadConst(&gdaki->gdqp) + peer; doca_gpu_dev_verbs_qp* companion_qp; ncclGinGdakiMemHandle* dstMh = (ncclGinGdakiMemHandle*)dstWin; ncclGinGdakiMemHandle* srcMh = (ncclGinGdakiMemHandle*)srcWin; uint32_t codeOpt = DOCA_GPUNETIO_VERBS_GPU_CODE_OPT_DEFAULT | (!!(optFlags & ncclGinOptFlagsMaySkipCreditCheck) * DOCA_GPUNETIO_VERBS_GPU_CODE_OPT_SKIP_AVAILABILITY_CHECK) | (!!(optFlags & ncclGinOptFlagsAggregateRequests) * DOCA_GPUNETIO_VERBS_GPU_CODE_OPT_SKIP_DB_RINGING); doca_gpu_dev_verbs_addr raddr, laddr; if (hasWins) { raddr.addr = dstOff; raddr.key = loadConst(loadConst(&dstMh->rkeys) + peer); laddr.addr = srcOff, laddr.key = loadConst(&srcMh->lkey); } doca_gpu_dev_verbs_addr sig_raddr, sig_laddr; if (hasSignal) { if (signalOp == ncclGinSignalInc) signalOpArg = 1; sig_raddr.addr = signalOffset; sig_raddr.key = signalKey; sig_laddr.addr = 0; sig_laddr.key = loadConst(&gdaki->sink_buffer_lkey); } doca_gpu_dev_verbs_addr counter_raddr, counter_laddr; if (hasCounter) { companion_qp = loadConst(&gdaki->companion_gdqp) + peer; counter_raddr.addr = sizeof(uint64_t) * (counterId + loadConst(&gdaki->counters_table.offset)); counter_raddr.key = loadConst(loadConst(&gdaki->counters_table.rkeys) + ctx.rank); counter_laddr.addr = 0; counter_laddr.key = loadConst(&gdaki->sink_buffer_lkey); } // cuda::thread_scope_system has the lowest value if ((required == cuda::thread_scope_system) && (given > required)) { doca_gpu_dev_verbs_fence_release(); } if (hasWins) { if (hasSignal && hasCounter) { doca_gpu_dev_verbs_put_signal_counter( qp, raddr, laddr, bytes, sig_raddr, sig_laddr, signalOpArg, companion_qp, counter_raddr, counter_laddr, 1, codeOpt); } else if (hasSignal) { doca_gpu_dev_verbs_put_signal( qp, raddr, laddr, bytes, sig_raddr, sig_laddr, signalOpArg, codeOpt); } else if (hasCounter) { doca_gpu_dev_verbs_put_counter(qp, raddr, laddr, bytes, companion_qp, counter_raddr, counter_laddr, 1, codeOpt); } else { doca_gpu_dev_verbs_put(qp, raddr, laddr, bytes, codeOpt); } } else { if (hasCounter) { doca_gpu_dev_verbs_signal_counter( qp, sig_raddr, sig_laddr, signalOpArg, companion_qp, counter_raddr, counter_laddr, 1, codeOpt); } else { doca_gpu_dev_verbs_signal( qp, sig_raddr, sig_laddr, signalOpArg, codeOpt); } } #ifdef NCCL_DEVICE_GIN_GDAKI_ENABLE_DEBUG doca_gpu_dev_verbs_wait(qp); if (hasCounter) doca_gpu_dev_verbs_wait(companion_qp); #endif } coop.sync(); } template NCCL_DEVICE_INLINE static void putValueImpl(ncclGinCtx ctx, Coop coop, int peer, ncclGinWindow_t dstWin, size_t dstOff, T srcData, bool hasSignal, size_t signalOffset, __be32 signalKey, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, bool hasDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope required, cuda::thread_scope given, uint32_t optFlags) { using nccl::utility::loadConst; coop.sync(); if (coop.thread_rank() == 0) { ncclGinGdakiGPUContext* gdaki = &((struct ncclGinGdakiGPUContext*)ctx.handle)[ctx.contextId]; doca_gpu_dev_verbs_qp* qp = loadConst(&gdaki->gdqp) + peer; ncclGinGdakiMemHandle* dstMh = (ncclGinGdakiMemHandle*)dstWin; uint32_t codeOpt = DOCA_GPUNETIO_VERBS_GPU_CODE_OPT_DEFAULT | (!!(optFlags & ncclGinOptFlagsMaySkipCreditCheck) * DOCA_GPUNETIO_VERBS_GPU_CODE_OPT_SKIP_AVAILABILITY_CHECK) | (!!(optFlags & ncclGinOptFlagsAggregateRequests) * DOCA_GPUNETIO_VERBS_GPU_CODE_OPT_SKIP_DB_RINGING); doca_gpu_dev_verbs_addr raddr; raddr.addr = dstOff; raddr.key = loadConst(loadConst(&dstMh->rkeys) + peer); doca_gpu_dev_verbs_addr sig_raddr, sig_laddr; if (hasSignal) { if (signalOp == ncclGinSignalInc) signalOpArg = 1; sig_raddr.addr = signalOffset; sig_raddr.key = signalKey; sig_laddr.addr = 0; sig_laddr.key = loadConst(&gdaki->sink_buffer_lkey); } // cuda::thread_scope_system has the lowest value if ((required == cuda::thread_scope_system) && (given > required)) { doca_gpu_dev_verbs_fence_release(); } if (hasSignal) { doca_gpu_dev_verbs_p_signal( qp, raddr, srcData, sig_raddr, sig_laddr, signalOpArg, codeOpt); } else { doca_gpu_dev_verbs_p(qp, raddr, srcData, codeOpt); } #ifdef NCCL_DEVICE_GIN_GDAKI_ENABLE_DEBUG doca_gpu_dev_verbs_wait(qp); #endif } coop.sync(); } } // namespace gdaki } // namespace gin } // namespace nccl template <> struct ncclGinApi_Put { template NCCL_DEVICE_INLINE static void call(ncclGinCtx ctx, Coop coop, int peer, bool hasWins, ncclGinWindow_t dstWin, size_t dstOff, ncclGinWindow_t srcWin, size_t srcOff, size_t bytes, ncclGinSignalDescriptor signal, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, bool hasCounter, ncclGinCounter_t counterId, bool hasDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope required, cuda::thread_scope given, uint32_t optFlags = ncclGinOptFlagsDefault) { using nccl::utility::loadConst; size_t signalOffset = 0; __be32 signalKey = 0; bool hasSignal = signal.type != NCCL_GIN_SIGNAL_TYPE_NONE; if (signal.type == NCCL_GIN_SIGNAL_TYPE_INDEXED) { ncclGinGdakiGPUContext* gdaki = &((struct ncclGinGdakiGPUContext*)ctx.handle)[ctx.contextId]; signalOffset = sizeof(uint64_t) * (signal.indexedSignal.signalId + loadConst(&gdaki->signals_table.offset)); signalKey = loadConst(loadConst(&gdaki->signals_table.rkeys) + peer); } else if (signal.type == NCCL_GIN_SIGNAL_TYPE_VA) { ncclGinGdakiMemHandle* signalMh = (ncclGinGdakiMemHandle*)signal.vaSignal.signalWindow; signalKey = loadConst(loadConst(&signalMh->rkeys) + peer); signalOffset = signal.vaSignal.signalOffset; } nccl::gin::gdaki::putImpl( ctx, coop, peer, hasWins, dstWin, dstOff, srcWin, srcOff, bytes, hasSignal, signalOffset, signalKey, signalOp, signalOpArg, hasCounter, counterId, hasDescriptor, descriptor, required, given, optFlags ); } }; template <> struct ncclGinApi_PutValue { template NCCL_DEVICE_INLINE static void call(ncclGinCtx ctx, Coop coop, int peer, ncclGinWindow_t dstWin, size_t dstOff, T srcVal, ncclGinSignalDescriptor signal, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, bool hasDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope required, cuda::thread_scope given, uint32_t optFlags = ncclGinOptFlagsDefault) { using nccl::utility::loadConst; size_t signalOffset = 0; __be32 signalKey = 0; bool hasSignal = signal.type != NCCL_GIN_SIGNAL_TYPE_NONE; if (signal.type == NCCL_GIN_SIGNAL_TYPE_VA) { ncclGinGdakiMemHandle* signalMh = (ncclGinGdakiMemHandle*)signal.vaSignal.signalWindow; signalKey = loadConst(loadConst(&signalMh->rkeys) + peer); signalOffset = signal.vaSignal.signalOffset; } else if (signal.type == NCCL_GIN_SIGNAL_TYPE_INDEXED) { ncclGinGdakiGPUContext* gdaki = &((struct ncclGinGdakiGPUContext*)ctx.handle)[ctx.contextId]; signalOffset = sizeof(uint64_t) * (signal.indexedSignal.signalId + loadConst(&gdaki->signals_table.offset)); signalKey = loadConst(loadConst(&gdaki->signals_table.rkeys) + peer); } nccl::gin::gdaki::putValueImpl( ctx, coop, peer, dstWin, dstOff, srcVal, hasSignal, signalOffset, signalKey, signalOp, signalOpArg, hasDescriptor, descriptor, required, given, optFlags ); } }; template <> struct ncclGinApi_ResetCounter { NCCL_DEVICE_INLINE static void call(ncclGinCtx ctx, ncclGinCounter_t counterId) { using nccl::utility::loadConst; ncclGinGdakiGPUContext* gdaki = &((struct ncclGinGdakiGPUContext*)ctx.handle)[ctx.contextId]; loadConst(&gdaki->counters_table.buffer)[counterId] = 0; } }; template <> struct ncclGinApi_ResetSignal { NCCL_DEVICE_INLINE static void call(ncclGinCtx ctx, ncclGinSignalDescriptor signal) { using nccl::utility::loadConst; if (signal.type == NCCL_GIN_SIGNAL_TYPE_VA) { uint64_t* signalPtr = (uint64_t*)ncclGetLocalPointer(signal.vaSignal.ncclWindow, signal.vaSignal.signalOffset); *signalPtr = 0; } else { ncclGinGdakiGPUContext* gdaki = &((struct ncclGinGdakiGPUContext*)ctx.handle)[ctx.contextId]; loadConst(&gdaki->signals_table.buffer)[signal.indexedSignal.signalId] = 0; } } }; template <> struct ncclGinApi_GetCounterPtr { NCCL_DEVICE_INLINE static uint64_t* call(ncclGinCtx ctx, ncclGinCounter_t counterId) { using nccl::utility::loadConst; ncclGinGdakiGPUContext* gdaki = &((struct ncclGinGdakiGPUContext*)ctx.handle)[ctx.contextId]; return loadConst(&gdaki->counters_table.buffer) + counterId; } }; template <> struct ncclGinApi_GetSignalPtr { NCCL_DEVICE_INLINE static uint64_t* call(ncclGinCtx ctx, ncclGinSignal_t signalId) { using nccl::utility::loadConst; ncclGinGdakiGPUContext* gdaki = &((struct ncclGinGdakiGPUContext*)ctx.handle)[ctx.contextId]; return loadConst(&gdaki->signals_table.buffer) + signalId; } }; template <> struct ncclGinApi_Flush { template NCCL_DEVICE_INLINE static void call(ncclGinCtx ctx, Coop coop, cuda::memory_order ord, uint32_t* abortFlag) { using nccl::utility::loadConst; using nccl::utility::testAbort; ncclGinGdakiGPUContext* gdaki = &((struct ncclGinGdakiGPUContext*)ctx.handle)[ctx.contextId]; doca_gpu_dev_verbs_qp* qps = loadConst(&gdaki->gdqp); if (abortFlag) { uint32_t steps = 0; #pragma unroll 1 for (int peer = coop.thread_rank(); peer < ctx.nRanks; peer += coop.size()) { int status = EBUSY; uint64_t ticket = doca_gpu_dev_verbs_atomic_read(&qps[peer].sq_rsvd_index); if (ticket == 0) return; --ticket; while (status != 0 && !testAbort(abortFlag, steps)) { status = doca_gpu_dev_verbs_poll_one_cq_at(&qps[peer].cq_sq, ticket); } } } else { #pragma unroll 1 for (int peer = coop.thread_rank(); peer < ctx.nRanks; peer += coop.size()) { doca_gpu_dev_verbs_wait(qps + peer); } } } }; #endif /* _NCCL_DEVICE_GIN_GDAKI_H_ */ nccl-2.29.7-1/src/include/nccl_device/gin/gdaki/gin_gdaki_device_host_common.h000066400000000000000000000021561515037102200272050ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_DEVICE_GIN_GDAKI_DEVICE_HOST_COMMON_H_ #define _NCCL_DEVICE_GIN_GDAKI_DEVICE_HOST_COMMON_H_ #include #define NCCL_GIN_GDAKI_VERSION 100 template struct ncclGinGdakiGlobalGPUBufferTable { T *buffer; __be32 *rkeys; __be32 lkey; unsigned int offset; }; struct ncclGinGdakiGPUContext { struct doca_gpu_dev_verbs_qp *gdqp; struct doca_gpu_dev_verbs_qp *companion_gdqp; struct ncclGinGdakiGlobalGPUBufferTable counters_table; struct ncclGinGdakiGlobalGPUBufferTable signals_table; // Local buffer we don't consume but is required for some operations. __be32 sink_buffer_lkey; }; struct ncclGinGdakiMemHandle { __be32 *rkeys; __be32 lkey; }; #endif /* _NCCL_DEVICE_GIN_GDAKI_DEVICE_HOST_COMMON_H_ */ nccl-2.29.7-1/src/include/nccl_device/gin/gin_device_api.h000066400000000000000000000010771515037102200232140ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_GIN_DEVICE_API_H_ #define _NCCL_GIN_DEVICE_API_H_ #include "gin_device_common.h" #if NCCL_GIN_GDAKI_ENABLE #include "gdaki/gin_gdaki.h" #endif #if NCCL_GIN_PROXY_ENABLE #include "proxy/gin_proxy.h" #endif #endif nccl-2.29.7-1/src/include/nccl_device/gin/gin_device_common.h000066400000000000000000000131171515037102200237310ustar00rootroot00000000000000/************************************************************************* * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * See LICENSE.txt for more license information *************************************************************************/ #ifndef _NCCL_GIN_DEVICE_COMMON_H_ #define _NCCL_GIN_DEVICE_COMMON_H_ #include "../net_device.h" #include "../utility.h" #include "gin_device_host_common.h" #if CUDA_VERSION >= 12080 && __CUDA_ARCH__ >= 900 #define NCCL_GIN_HAS_FENCE_ACQUIRE_RELEASE_PTX 1 #endif #ifndef NCCL_GIN_PROXY_ENABLE #define NCCL_GIN_PROXY_ENABLE 1 #endif #ifndef NCCL_GIN_GDAKI_ENABLE #if CUDA_VERSION >= 12020 && __CUDA_ARCH__ >= 700 #define NCCL_GIN_GDAKI_ENABLE 1 #else #define NCCL_GIN_GDAKI_ENABLE 0 #endif #endif enum ncclGinOptFlags { ncclGinOptFlagsDefault = 0, ncclGinOptFlagsMaySkipCreditCheck = (1 << 0), ncclGinOptFlagsAggregateRequests = (1 << 1), }; #define NCCL_GIN_BACKEND_MASK_ALL \ (((NCCL_GIN_PROXY_ENABLE) ? 1u : 0u) << (unsigned)NCCL_NET_DEVICE_GIN_PROXY | \ ((NCCL_GIN_GDAKI_ENABLE) ? 1u : 0u) << (unsigned)NCCL_NET_DEVICE_GIN_GDAKI) struct ncclGinCtx { unsigned backendMask; ncclNetDeviceType backend; int rank; int nRanks; void* handle; int contextId; }; template struct ncclGinCtx_M : ncclGinCtx {}; struct ncclGinDescriptorSmem { alignas(16) char space[64]; }; enum ncclGinSignalType { NCCL_GIN_SIGNAL_TYPE_NONE, NCCL_GIN_SIGNAL_TYPE_VA, NCCL_GIN_SIGNAL_TYPE_INDEXED, }; struct ncclGinSignalDescriptor { ncclGinSignalType type; union { struct { ncclGinWindow_t signalWindow; size_t signalOffset; ncclWindow_t ncclWindow; } vaSignal; struct { ncclGinSignal_t signalId; } indexedSignal; }; }; #if NCCL_CHECK_CUDACC template struct ncclGinApi_Put { template NCCL_DEVICE_INLINE static void call(ncclGinCtx, Coop coop, int peer, bool hasWins, ncclGinWindow_t dstWin, size_t dstOff, ncclGinWindow_t srcWin, size_t srcOff, size_t bytes, ncclGinSignalDescriptor signal, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, bool hasCounter, ncclGinCounter_t counterId, bool hasDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope required, cuda::thread_scope given, uint32_t optFlags = ncclGinOptFlagsDefault); }; template struct ncclGinApi_PutValue { template NCCL_DEVICE_INLINE static void call(ncclGinCtx, Coop coop, int peer, ncclGinWindow_t dstWin, size_t dstOff, T srcData, ncclGinSignalDescriptor signal, ncclGinSignalOp_t signalOp, uint64_t signalOpArg, bool hasDescriptor, ncclGinDescriptorSmem* descriptor, cuda::thread_scope required, cuda::thread_scope given, uint32_t optFlags = ncclGinOptFlagsDefault); }; template struct ncclGinApi_GetSignalPtr { NCCL_DEVICE_INLINE static uint64_t* call(ncclGinCtx, int peer, ncclGinSignal_t signalId); }; template struct ncclGinApi_GetCounterPtr { NCCL_DEVICE_INLINE static uint64_t* call(ncclGinCtx, int peer, ncclGinCounter_t counterId); }; template struct ncclGinApi_ResetSignal { NCCL_DEVICE_INLINE static void call(ncclGinCtx, ncclGinSignalDescriptor signal); }; template struct ncclGinApi_ResetCounter { NCCL_DEVICE_INLINE static void call(ncclGinCtx, ncclGinCounter_t counterId); }; template struct ncclGinApi_Flush { template NCCL_DEVICE_INLINE static void call(ncclGinCtx, Coop, cuda::memory_order ord, uint32_t* abortFlag); }; #endif #if NCCL_CHECK_CUDACC template