diff options
| -rw-r--r-- | gnu/local.mk | 2 | ||||
| -rw-r--r-- | gnu/packages/machine-learning.scm | 143 | ||||
| -rw-r--r-- | gnu/packages/patches/python-pytorch-runpath.patch | 25 | ||||
| -rw-r--r-- | gnu/packages/patches/python-pytorch-system-libraries.patch | 131 |
4 files changed, 300 insertions, 1 deletions
diff --git a/gnu/local.mk b/gnu/local.mk index 82e5fbc88f..51a76e3638 100644 --- a/gnu/local.mk +++ b/gnu/local.mk @@ -1649,6 +1649,8 @@ dist_patch_DATA = \ %D%/packages/patches/python-pydot-regression-test.patch \ %D%/packages/patches/python2-pygobject-2-deprecation.patch \ %D%/packages/patches/python-pygpgme-fix-pinentry-tests.patch \ + %D%/packages/patches/python-pytorch-runpath.patch \ + %D%/packages/patches/python-pytorch-system-libraries.patch \ %D%/packages/patches/python-robotframework-source-date-epoch.patch \ %D%/packages/patches/python-seaborn-kde-test.patch \ %D%/packages/patches/python2-subprocess32-disable-input-test.patch \ diff --git a/gnu/packages/machine-learning.scm b/gnu/packages/machine-learning.scm index c3480d6523..4149defc47 100644 --- a/gnu/packages/machine-learning.scm +++ b/gnu/packages/machine-learning.scm @@ -65,10 +65,12 @@ #:use-module (gnu packages graphviz) #:use-module (gnu packages gstreamer) #:use-module (gnu packages image) + #:use-module (gnu packages libffi) #:use-module (gnu packages linux) #:use-module (gnu packages llvm) #:use-module (gnu packages maths) #:use-module (gnu packages mpi) + #:use-module (gnu packages ninja) #:use-module (gnu packages ocaml) #:use-module (gnu packages onc-rpc) #:use-module (gnu packages parallel) @@ -2546,7 +2548,7 @@ general non-linear dimension reduction.") (package (name "xnnpack") (version (git-version version revision commit)) - (home-page "https://github.com/google/XNNPACK") + (home-page "https://github.com/google/XNNPACK") ;fork of QNNPACK (source (origin (method git-fetch) (uri (git-reference (url home-page) (commit commit))) @@ -2586,3 +2588,142 @@ high-level machine learning frameworks, such as TensorFlow Lite, TensorFlow.js, PyTorch, and MediaPipe.") (license license:bsd-3)))) +(define-public python-pytorch + (package + (name "python-pytorch") + (version "1.9.0") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/pytorch/pytorch") + (commit (string-append "v" version)) + (recursive? #t))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0cznsh68hwk5761gv7iijb4g6jgjpvs3bbixwpzzmkbkbn2q96c1")) + (patches (search-patches "python-pytorch-system-libraries.patch" + "python-pytorch-runpath.patch")) + (modules '((guix build utils))) + (snippet + '(begin + ;; XXX: Let's be clear: this package is a bundling fest. We + ;; delete as much as we can, but there's still a lot left. + (for-each (lambda (directory) + (delete-file-recursively + (string-append "third_party/" directory))) + '("benchmark" "cpuinfo" "eigen" + + ;; FIXME: QNNPACK (of which XNNPACK is a fork) + ;; needs these. + ;; "FP16" "FXdiv" "gemmlowp" "psimd" + + "gloo" "googletest" "ios-cmake" + "onnx" "protobuf" "pthreadpool" + "pybind11" "python-enum" "python-peachpy" + "python-six" "tbb" "XNNPACK" "zstd")) + + ;; Adjust references to the onnx-optimizer headers. + (substitute* "caffe2/onnx/backend.cc" + (("onnx/optimizer/") + "onnxoptimizer/")))))) + (build-system python-build-system) + (arguments + '(#:phases (modify-phases %standard-phases + (add-before 'build 'use-system-libraries + (lambda* (#:key outputs #:allow-other-keys) + ;; Tell 'setup.py' to let 'CMakeLists.txt' know that we + ;; want to use "system libraries" instead of the bundled + ;; ones. + (setenv "USE_SYSTEM_LIBS" "1") + + ;; XXX: Disable that for simplicity for now. + (setenv "USE_FBGEMM" "0") + + ;; Set 'OUT/lib' rather than '$ORIGIN' as the RUNPATH on + ;; binaries that go to 'lib/python3.8/torch/bin' & co. + #;(substitute* "cmake/Dependencies.cmake" + (("^set\\(CMAKE_INSTALL_RPATH .*") + (string-append "set(CMAKE_INSTALL_RPATH \"" + (assoc-ref outputs "out") + "/lib\")\n"))))) + (add-before 'build 'make-things-writable + (lambda _ + ;; The 'build_caffe2' function in + ;; 'tools/build_pytorch_libs.py', called from the + ;; top-level 'setup.py', needs write access to this + ;; directory. + (for-each make-file-writable + (find-files "caffe2/proto" "." + #:directories? #t)))) + (replace 'check + (lambda* (#:key inputs outputs tests? #:allow-other-keys) + ;; Run the test suite following the instructions in + ;; 'CONTRIBUTING.md'. XXX: Unfortunately this doesn't + ;; work, unless you set PYTHONPATH presumably. + (when tests? + (let ((python-site (site-packages inputs outputs))) + (setenv "PYTHONPATH" + (string-append python-site ":" + (getenv "PYTHONPATH"))) + (invoke "python" "test/run_test.py"))))) + (add-after 'install 'remove-test-executables + (lambda* (#:key inputs outputs #:allow-other-keys) + ;; Remove test executables, but keep other executables + ;; such as 'torch_shm_manager' and and .so files such as + ;; 'libtorch_global_deps.so'. + (let ((python-site (site-packages inputs outputs))) + (for-each delete-file + (find-files python-site + "(^test_cpp_rpc|_test)$")))))) + + ;; XXX: Tests attempt to download data such as + ;; <https://raw.githubusercontent.com/pytorch/test-infra/master/stats/slow-tests.json>. + #:tests? #f)) + (native-inputs + `(("cmake" ,cmake) + ("ninja" ,ninja))) + (inputs + `(("eigen" ,eigen) + ;; ("fmt" ,fmt) + ("fp16" ,fp16) + ("gemmlowp" ,gemmlowp) + ("googletest" ,googletest) + ("googlebenchmark" ,googlebenchmark) + ("gloo" ,gloo) + ("openblas" ,openblas) + ("openmpi" ,openmpi) + ("pthreadpool" ,pthreadpool) + ("protobuf" ,protobuf) + ("pybind11" ,pybind11) + ("sleef" ,sleef) + ("xnnpack" ,xnnpack) + ("zstd" ,zstd))) + (propagated-inputs + `(("python-astunparse" ,python-astunparse) + ("python-numpy" ,python-numpy) + ("python-pyyaml" ,python-pyyaml) + ("python-cffi" ,python-cffi) + ("python-peachpy" ,python-peachpy) + ("python-typing-extensions" ,python-typing-extensions) + ("python-future" ,python-future) + ("python-six" ,python-six) + ("python-requests" ,python-requests) + ("onnx" ,onnx) ;propagated for its Python modules + ("onnx-optimizer" ,onnx-optimizer) + ("cpuinfo" ,cpuinfo))) + (home-page "https://pytorch.org/") + (synopsis "Python library for tensor computation and deep neural networks") + (description + "PyTorch is a Python package that provides two high-level features: + +@itemize +@item tensor computation (like NumPy) with strong GPU acceleration; +@item deep neural networks (DNNs) built on a tape-based autograd system. +@end itemize + +You can reuse Python packages such as NumPy, SciPy, and Cython to extend +PyTorch when needed. + +Note: currently this package does not provide GPU support.") + (license license:bsd-3))) diff --git a/gnu/packages/patches/python-pytorch-runpath.patch b/gnu/packages/patches/python-pytorch-runpath.patch new file mode 100644 index 0000000000..6f270ef9b1 --- /dev/null +++ b/gnu/packages/patches/python-pytorch-runpath.patch @@ -0,0 +1,25 @@ +Libraries (such as 'libtorch_cpu.so') and executables (such as 'torch_shm_manager') +get installed, quite surprisingly, to 'lib/python3.8/site-packages/{bin,lib}'. +Make sure RUNPATH matches that. + +--- a/cmake/Dependencies.cmake ++++ b/cmake/Dependencies.cmake +@@ -4,7 +4,7 @@ if(APPLE) + set(CMAKE_MACOSX_RPATH ON) + set(_rpath_portable_origin "@loader_path") + else() +- set(_rpath_portable_origin $ORIGIN) ++ set(_rpath_portable_origin $ORIGIN/../lib) + endif(APPLE) + # Use separate rpaths during build and install phases + set(CMAKE_SKIP_BUILD_RPATH FALSE) + +--- a/caffe2/CMakeLists.txt ++++ b/caffe2/CMakeLists.txt +@@ -1797,5 +1797,5 @@ if(BUILD_PYTHON) + if(${BUILDING_WITH_TORCH_LIBS}) + # site-packages/caffe2/python/caffe2_pybind11_state + # site-packages/torch/lib +- set(caffe2_pybind11_rpath "${_rpath_portable_origin}/../../torch/lib") ++ set(caffe2_pybind11_rpath $ORIGIN/../../torch/lib) + endif(${BUILDING_WITH_TORCH_LIBS}) diff --git a/gnu/packages/patches/python-pytorch-system-libraries.patch b/gnu/packages/patches/python-pytorch-system-libraries.patch new file mode 100644 index 0000000000..c8d14b3f56 --- /dev/null +++ b/gnu/packages/patches/python-pytorch-system-libraries.patch @@ -0,0 +1,131 @@ +Use our own googletest rather than the bundled one. +Get NNPACK to use our own PeachPy rather than the bundled one. + +diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake +index 5d57b9ca78..620cca4e60 100644 +--- a/cmake/Dependencies.cmake ++++ b/cmake/Dependencies.cmake +@@ -644,11 +644,6 @@ if(BUILD_TEST OR BUILD_MOBILE_BENCHMARK OR BUILD_MOBILE_TEST) + # this shouldn't be necessary anymore. + get_property(INC_DIR_temp DIRECTORY PROPERTY INCLUDE_DIRECTORIES) + set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES "") +- add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest) +- set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES ${INC_DIR_temp}) +- +- include_directories(BEFORE SYSTEM ${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest/googletest/include) +- include_directories(BEFORE SYSTEM ${CMAKE_CURRENT_LIST_DIR}/../third_party/googletest/googlemock/include) + + # We will not need to test benchmark lib itself. + set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing as we don't need it.") +@@ -1485,7 +1480,7 @@ if(CAFFE2_CMAKE_BUILDING_WITH_MAIN_REPO AND NOT INTERN_DISABLE_ONNX) + endif() + set_property(TARGET onnx_proto PROPERTY IMPORTED_LOCATION ${ONNX_PROTO_LIBRARY}) + message("-- Found onnx: ${ONNX_LIBRARY} ${ONNX_PROTO_LIBRARY}") +- list(APPEND Caffe2_DEPENDENCY_LIBS onnx_proto onnx) ++ list(APPEND Caffe2_DEPENDENCY_LIBS onnx_proto onnx onnx_optimizer) + endif() + include_directories(${FOXI_INCLUDE_DIRS}) + list(APPEND Caffe2_DEPENDENCY_LIBS foxi_loader) + +diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt +index 50ebb224ce..5953d9ddf7 100644 +--- a/caffe2/CMakeLists.txt ++++ b/caffe2/CMakeLists.txt +@@ -1632,7 +1632,7 @@ if(BUILD_TEST) + if(NOT MSVC) + add_executable(${test_name}_${CPU_CAPABILITY} "${test_src}" ../aten/src/ATen/native/quantized/affine_quantizer_base.cpp) + # TODO: Get rid of c10 dependency (which is only needed for the implementation of AT_ERROR) +- target_link_libraries(${test_name}_${CPU_CAPABILITY} c10 sleef gtest_main) ++ target_link_libraries(${test_name}_${CPU_CAPABILITY} c10 sleef gtest_main gtest) + if(USE_FBGEMM) + target_link_libraries(${test_name}_${CPU_CAPABILITY} fbgemm) + endif() +@@ -1655,7 +1655,7 @@ if(BUILD_TEST) + foreach(test_src ${Caffe2_CPU_TEST_SRCS}) + get_filename_component(test_name ${test_src} NAME_WE) + add_executable(${test_name} "${test_src}") +- target_link_libraries(${test_name} torch_library gtest_main) ++ target_link_libraries(${test_name} torch_library gtest_main gtest) + target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>) + target_include_directories(${test_name} PRIVATE $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/include>) + target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE}) +@@ -1673,7 +1673,7 @@ if(BUILD_TEST) + foreach(test_src ${Caffe2_GPU_TEST_SRCS}) + get_filename_component(test_name ${test_src} NAME_WE) + cuda_add_executable(${test_name} "${test_src}") +- target_link_libraries(${test_name} torch_library gtest_main) ++ target_link_libraries(${test_name} torch_library gtest_main gtest) + target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>) + target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE}) + add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>) +@@ -1691,7 +1691,7 @@ if(BUILD_TEST) + foreach(test_src ${Caffe2_VULKAN_TEST_SRCS}) + get_filename_component(test_name ${test_src} NAME_WE) + add_executable(${test_name} "${test_src}") +- target_link_libraries(${test_name} torch_library gtest_main) ++ target_link_libraries(${test_name} torch_library gtest_main gtest) + target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>) + target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE}) + add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>) +@@ -1709,7 +1709,7 @@ if(BUILD_TEST) + foreach(test_src ${Caffe2_HIP_TEST_SRCS}) + get_filename_component(test_name ${test_src} NAME_WE) + add_executable(${test_name} "${test_src}") +- target_link_libraries(${test_name} torch_library gtest_main) ++ target_link_libraries(${test_name} torch_library gtest_main gtest) + target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>) + target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE} ${Caffe2_HIP_INCLUDE}) + target_compile_options(${test_name} PRIVATE ${HIP_CXX_FLAGS}) + +diff --git a/torch/lib/c10d/test/CMakeLists.txt b/torch/lib/c10d/test/CMakeLists.txt +index b74d4b65f7..fc7c207505 100644 +--- a/torch/lib/c10d/test/CMakeLists.txt ++++ b/torch/lib/c10d/test/CMakeLists.txt +@@ -16,25 +16,25 @@ function(c10d_add_test test_src) + add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>) + endfunction() + +-c10d_add_test(FileStoreTest.cpp c10d gtest_main) +-c10d_add_test(TCPStoreTest.cpp c10d gtest_main) ++c10d_add_test(FileStoreTest.cpp c10d gtest_main gtest) ++c10d_add_test(TCPStoreTest.cpp c10d gtest_main gtest) + if(NOT WIN32) +- c10d_add_test(HashStoreTest.cpp c10d gtest_main) ++ c10d_add_test(HashStoreTest.cpp c10d gtest_main gtest) + endif() + + if(USE_CUDA) + if(USE_C10D_GLOO) +- c10d_add_test(ProcessGroupGlooTest.cpp c10d c10d_cuda_test gtest_main) +- c10d_add_test(ProcessGroupGlooAsyncTest.cpp c10d c10d_cuda_test gtest_main) ++ c10d_add_test(ProcessGroupGlooTest.cpp c10d c10d_cuda_test gtest_main gtest) ++ c10d_add_test(ProcessGroupGlooAsyncTest.cpp c10d c10d_cuda_test gtest_main gtest) + endif() + if(USE_C10D_NCCL) +- c10d_add_test(ProcessGroupNCCLTest.cpp c10d c10d_cuda_test gtest_main) ++ c10d_add_test(ProcessGroupNCCLTest.cpp c10d c10d_cuda_test gtest_main gtest) + c10d_add_test(ProcessGroupNCCLErrorsTest.cpp c10d c10d_cuda_test +- gtest_main) ++ gtest_main gtest) + endif() + else() + if(USE_C10D_GLOO) +- c10d_add_test(ProcessGroupGlooTest.cpp c10d gtest_main) ++ c10d_add_test(ProcessGroupGlooTest.cpp c10d gtest_main gtest) + endif() + endif() + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 5ecd2df..24feae3 100644 +--- a/third_party/NNPACK/CMakeLists.txt ++++ b/third_party/NNPACK/CMakeLists.txt +@@ -427,8 +427,7 @@ IF(NNPACK_BACKEND STREQUAL "x86-64") + FILE(MAKE_DIRECTORY ${obj_dir}) + ADD_CUSTOM_COMMAND( + OUTPUT ${obj} +- COMMAND "PYTHONPATH=${PEACHPY_PYTHONPATH}" +- ${PYTHON_EXECUTABLE} -m peachpy.x86_64 ++ COMMAND ${PYTHON_EXECUTABLE} -m peachpy.x86_64 + -mabi=sysv -g4 -mimage-format=${PEACHPY_IMAGE_FORMAT} + "-I${PROJECT_SOURCE_DIR}/src" "-I${PROJECT_SOURCE_DIR}/src/x86_64-fma" "-I${FP16_SOURCE_DIR}/include" + -o ${obj} "${PROJECT_SOURCE_DIR}/${src}" |