From 0821d4026db2f5fb063dd5d0719f4dc95ebe72bd Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 17 Jan 2023 10:54:44 +0300 Subject: [PATCH 1/2] OpenCV package does not distribute zlib. Roll back to manylinux version to presume binary compatibility. --- docker/manylinux2014/Dockerfile_aarch64 | 18 ++--------- docker/manylinux2014/Dockerfile_x86_64 | 17 ++-------- patch_auditwheel_whitelist.py | 42 ------------------------- travis_config.sh | 1 - 4 files changed, 6 insertions(+), 72 deletions(-) delete mode 100644 patch_auditwheel_whitelist.py diff --git a/docker/manylinux2014/Dockerfile_aarch64 b/docker/manylinux2014/Dockerfile_aarch64 index bbb06b72..b6fc85bf 100644 --- a/docker/manylinux2014/Dockerfile_aarch64 +++ b/docker/manylinux2014/Dockerfile_aarch64 @@ -1,4 +1,4 @@ -# Version: 20221205 +# Version: 20230117 # Image name: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64 FROM quay.io/pypa/manylinux2014_aarch64:latest @@ -11,28 +11,16 @@ ARG NASM_VERSION=2.15.04 ARG OPENSSL_VERSION=1_1_1s ARG QT_VERSION=5.15.0 ARG YASM_VERSION=1.3.0 -ARG ZLIB_VERSION=1.2.13 ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH # epel-release need for aarch64 to get openblas packages -RUN yum install bzip2-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ +RUN yum install curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ yum install openblas-devel dejavu-sans-fonts -y && \ cp /usr/include/lapacke/lapacke*.h /usr/include/ && \ curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \ # libpng will be built from source - yum remove libpng zlib-devel bzip2-devel -y - -RUN mkdir ~/zlib_sources && \ - cd ~/zlib_sources && \ - curl -O -L https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz && \ - tar -xf zlib-${ZLIB_VERSION}.tar.gz && \ - cd zlib-${ZLIB_VERSION} && \ - ./configure --prefix=/usr/local && \ - make && \ - make install && \ - cd .. && \ - rm -rf ~/zlib_sources + yum remove libpng -y RUN mkdir ~/libpng_sources && \ cd ~/libpng_sources && \ diff --git a/docker/manylinux2014/Dockerfile_x86_64 b/docker/manylinux2014/Dockerfile_x86_64 index 89eb353f..ea129b0d 100644 --- a/docker/manylinux2014/Dockerfile_x86_64 +++ b/docker/manylinux2014/Dockerfile_x86_64 @@ -1,4 +1,4 @@ -# Version: 20221205 +# Version: 20230117 # Image name: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64 FROM quay.io/pypa/manylinux2014_x86_64:latest @@ -16,23 +16,12 @@ ARG ZLIB_VERSION=1.2.13 ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH # epel-release need for aarch64 to get openblas packages -RUN yum install bzip2-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ +RUN yum install curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ yum install openblas-devel dejavu-sans-fonts -y && \ cp /usr/include/lapacke/lapacke*.h /usr/include/ && \ curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \ # libpng will be built from source - yum remove libpng zlib-devel bzip2-devel -y - -RUN mkdir ~/zlib_sources && \ - cd ~/zlib_sources && \ - curl -O -L https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz && \ - tar -xf zlib-${ZLIB_VERSION}.tar.gz && \ - cd zlib-${ZLIB_VERSION} && \ - ./configure --prefix=/usr/local && \ - make && \ - make install && \ - cd .. && \ - rm -rf ~/zlib_sources + yum remove libpng -y RUN mkdir ~/libpng_sources && \ cd ~/libpng_sources && \ diff --git a/patch_auditwheel_whitelist.py b/patch_auditwheel_whitelist.py deleted file mode 100644 index d1345911..00000000 --- a/patch_auditwheel_whitelist.py +++ /dev/null @@ -1,42 +0,0 @@ -from os.path import join, dirname, abspath -import json - -from auditwheel import policy - - -def add_zlib_versions(): - with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), 'r') as manylinux_policy: - manylinux_policy_lines = manylinux_policy.readlines() - - manylinux_policy_data = "" - for line in manylinux_policy_lines: - if "ZLIB" in line: - if len(line) > 22: - updated_line = line[:-2] + ', "1.2.9", "1.2.12", "1.2.13"]' - else: - updated_line = line[:-2] + '"1.2.9", "1.2.12", "1.2.13"]' - print("auditwheel patch: replace policy line \"%s\" with \"%s\"" % (line, updated_line)) - manylinux_policy_replacement = line.replace(line, updated_line) - else: - manylinux_policy_replacement = line - manylinux_policy_data = manylinux_policy_data + manylinux_policy_replacement - - with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), 'w') as manylinux_final_policy: - manylinux_final_policy.write(manylinux_policy_data) - -def add_whitelisted_libs(): - policies = None - - with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json")) as f: - policies = json.load(f) - - for p in policies: - p["lib_whitelist"].append("libxcb.so.1") - - with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), "w") as f: - f.write(json.dumps(policies)) - - -if __name__ == '__main__': - add_zlib_versions() - add_whitelisted_libs() diff --git a/travis_config.sh b/travis_config.sh index b91d63c4..ca5b8836 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -23,7 +23,6 @@ function bdist_wheel_cmd { TOOLS_PATH=/opt/_internal/pipx/venvs/auditwheel /opt/python/cp39-cp39/bin/python -m venv $TOOLS_PATH source $TOOLS_PATH/bin/activate - python patch_auditwheel_whitelist.py deactivate fi if [ -n "$USE_CCACHE" -a -z "$BREW_BOOTSTRAP_MODE" ]; then ccache -s; fi From aa095f7cd910cf6ade3c510ebc691875ed518f59 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 18 Jan 2023 10:02:23 +0300 Subject: [PATCH 2/2] Restored patch auditwheels. --- .github/workflows/build_wheels_linux.yml | 4 ++-- .github/workflows/build_wheels_linux_arm.yml | 2 +- docker/manylinux2014/Dockerfile_aarch64 | 2 +- docker/manylinux2014/Dockerfile_x86_64 | 3 +-- patch_auditwheel_whitelist.py | 19 +++++++++++++++++++ travis_config.sh | 1 + 6 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 patch_auditwheel_whitelist.py diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index 95e35898..6fa602fb 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -41,7 +41,7 @@ jobs: MB_ML_VER: 2014 TRAVIS_BUILD_DIR: ${{ github.workspace }} CONFIG_PATH: travis_config.sh - DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64:20221205 + DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64:20230117 USE_CCACHE: 0 UNICODE_WIDTH: 32 PLAT: x86_64 @@ -139,7 +139,7 @@ jobs: NP_TEST_DEP: numpy==1.19.4 TRAVIS_BUILD_DIR: ${{ github.workspace }} CONFIG_PATH: travis_config.sh - DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64:20220628 + DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64:20230117 USE_CCACHE: 1 UNICODE_WIDTH: 32 SDIST: ${{ matrix.build_sdist || 0 }} diff --git a/.github/workflows/build_wheels_linux_arm.yml b/.github/workflows/build_wheels_linux_arm.yml index b025e285..ea5e2bb8 100644 --- a/.github/workflows/build_wheels_linux_arm.yml +++ b/.github/workflows/build_wheels_linux_arm.yml @@ -42,7 +42,7 @@ jobs: MB_ML_VER: 2014 TRAVIS_BUILD_DIR: ${{ github.workspace }} CONFIG_PATH: travis_config.sh - DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64:20221205 + DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64:20230117 USE_CCACHE: 0 UNICODE_WIDTH: 32 SDIST: ${{ matrix.build_sdist || 0 }} diff --git a/docker/manylinux2014/Dockerfile_aarch64 b/docker/manylinux2014/Dockerfile_aarch64 index b6fc85bf..6a298f61 100644 --- a/docker/manylinux2014/Dockerfile_aarch64 +++ b/docker/manylinux2014/Dockerfile_aarch64 @@ -15,7 +15,7 @@ ARG YASM_VERSION=1.3.0 ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH # epel-release need for aarch64 to get openblas packages -RUN yum install curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ +RUN yum install zlib-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ yum install openblas-devel dejavu-sans-fonts -y && \ cp /usr/include/lapacke/lapacke*.h /usr/include/ && \ curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \ diff --git a/docker/manylinux2014/Dockerfile_x86_64 b/docker/manylinux2014/Dockerfile_x86_64 index ea129b0d..5cc8251c 100644 --- a/docker/manylinux2014/Dockerfile_x86_64 +++ b/docker/manylinux2014/Dockerfile_x86_64 @@ -11,12 +11,11 @@ ARG NASM_VERSION=2.15.04 ARG OPENSSL_VERSION=1_1_1s ARG QT_VERSION=5.15.0 ARG YASM_VERSION=1.3.0 -ARG ZLIB_VERSION=1.2.13 ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH # epel-release need for aarch64 to get openblas packages -RUN yum install curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ +RUN yum install zlib-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ yum install openblas-devel dejavu-sans-fonts -y && \ cp /usr/include/lapacke/lapacke*.h /usr/include/ && \ curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \ diff --git a/patch_auditwheel_whitelist.py b/patch_auditwheel_whitelist.py new file mode 100644 index 00000000..b396ab55 --- /dev/null +++ b/patch_auditwheel_whitelist.py @@ -0,0 +1,19 @@ +from os.path import join, dirname, abspath +import json + +from auditwheel import policy + +def add_whitelisted_libs(): + policies = None + + with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json")) as f: + policies = json.load(f) + + for p in policies: + p["lib_whitelist"].append("libxcb.so.1") + + with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), "w") as f: + f.write(json.dumps(policies)) + +if __name__ == '__main__': + add_whitelisted_libs() diff --git a/travis_config.sh b/travis_config.sh index ca5b8836..b91d63c4 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -23,6 +23,7 @@ function bdist_wheel_cmd { TOOLS_PATH=/opt/_internal/pipx/venvs/auditwheel /opt/python/cp39-cp39/bin/python -m venv $TOOLS_PATH source $TOOLS_PATH/bin/activate + python patch_auditwheel_whitelist.py deactivate fi if [ -n "$USE_CCACHE" -a -z "$BREW_BOOTSTRAP_MODE" ]; then ccache -s; fi