From bee9c49900097937a3b3d07e2007344488196f06 Mon Sep 17 00:00:00 2001 From: Vadim Lebedev Date: Fri, 19 Aug 2022 12:44:44 +0200 Subject: [PATCH 1/8] Intrgrate embedding of libarchive.so into the deploy workflow --- .github/workflows/deploy.yml | 56 +++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 9d49b69..919fda3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -3,24 +3,66 @@ on: push: tags: - '*' +env: + libarchive_tag: v3.6.1 jobs: deploy: name: Deploy to PyPI if: startsWith(github.ref, 'refs/tags') runs-on: ubuntu-latest + container: quay.io/pypa/manylinux2014_x86_64 steps: - - name: Setup python - uses: actions/setup-python@v4 - with: - python-version: '3.8' + - name: Print System Information + run: | + echo "uname -a: $( uname -a )" + cat /etc/issue + echo "Shell: $SHELL" + echo "Mount points:"; mount + echo "nproc: $( nproc )" + env + + - name: Chose Python version + run: | + ln -s /opt/python/cp39-cp39/bin/python3 /usr/local/bin/python3 + export PATH="/opt/python/cp39-cp39/bin:$PATH" + + - name: Install dependencies + run: | + yum install -y epel-release libxml2-devel libzstd-devel xz-devel bzip2-devel + yum install -y libacl-devel lz4-devel e2fsprogs-devel libb2-devel lzo-devel openssl-devel + yum install -y librichacl-devel swig strace + + - name: Install python tools + run: | + python3 -m pip install pytest auditwheel + + - name: Build libarchive C sources + run: | + cd /tmp + git clone https://github.com/libarchive/libarchive.git libarchive-src + cd libarchive-src; git checkout ${{ env.libarchive_tag }} + mkdir build-libarchive; cd build-libarchive + cmake ../ + make -j$(nproc); make install + cd $GITHUB_WORKSPACE - name: Check out code uses: actions/checkout@v3 - - name: Build the library - run: python3 setup.py sdist + - name: SWIG and Compile python modules + run: | + touch libarchive/_libarchive.i + make PYVER=3.9 + + - name: Build wheels + run: | + python3 -m pip wheel --wheel-dir dist/ . + auditwheel repair dist/*.whl - - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Deploy wheels to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} + wheel_files: wheelhouse/*.whl + dry_run: true From 664b6836102614ca6ca76423d68ab181726bc8e4 Mon Sep 17 00:00:00 2001 From: Vadim Lebedev Date: Fri, 19 Aug 2022 15:06:22 +0200 Subject: [PATCH 2/8] Fixes for deployment workflow --- .github/workflows/deploy.yml | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 919fda3..1ca36da 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -8,31 +8,25 @@ env: jobs: deploy: + strategy: + matrix: + python_version: [3.8, 3.9, 3.10] + name: Deploy to PyPI if: startsWith(github.ref, 'refs/tags') runs-on: ubuntu-latest container: quay.io/pypa/manylinux2014_x86_64 - steps: - - name: Print System Information - run: | - echo "uname -a: $( uname -a )" - cat /etc/issue - echo "Shell: $SHELL" - echo "Mount points:"; mount - echo "nproc: $( nproc )" - env - - - name: Chose Python version - run: | - ln -s /opt/python/cp39-cp39/bin/python3 /usr/local/bin/python3 - export PATH="/opt/python/cp39-cp39/bin:$PATH" - + steps: - name: Install dependencies run: | yum install -y epel-release libxml2-devel libzstd-devel xz-devel bzip2-devel yum install -y libacl-devel lz4-devel e2fsprogs-devel libb2-devel lzo-devel openssl-devel yum install -y librichacl-devel swig strace + - name: Setup Python + uses: actions/setup-python@v2 + with: + python_version: ${{ matrix.python_version }} - name: Install python tools run: | python3 -m pip install pytest auditwheel @@ -53,7 +47,7 @@ jobs: - name: SWIG and Compile python modules run: | touch libarchive/_libarchive.i - make PYVER=3.9 + make PYVER=${{ matrix.python_version }} - name: Build wheels run: | @@ -64,5 +58,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} - wheel_files: wheelhouse/*.whl - dry_run: true + wheel_files: wheelhouse/*.whl \ No newline at end of file From 1a6c5699189db06a7856d01e2c56582b7052a9ab Mon Sep 17 00:00:00 2001 From: Vadim Lebedev Date: Fri, 19 Aug 2022 15:12:55 +0200 Subject: [PATCH 3/8] pytest is not needed during deployement --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1ca36da..beace1b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -29,7 +29,7 @@ jobs: python_version: ${{ matrix.python_version }} - name: Install python tools run: | - python3 -m pip install pytest auditwheel + python3 -m pip install auditwheel - name: Build libarchive C sources run: | From 72db47490f187eb5e5453db3036b20cc87aeb076 Mon Sep 17 00:00:00 2001 From: Vadim Lebedev Date: Fri, 19 Aug 2022 17:22:58 +0200 Subject: [PATCH 4/8] Fixes to deployment workflow --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index beace1b..d9e49ea 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -26,11 +26,11 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python_version: ${{ matrix.python_version }} + python-version: ${{ matrix.python_version }} - name: Install python tools run: | python3 -m pip install auditwheel - + - name: Build libarchive C sources run: | cd /tmp @@ -58,4 +58,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} - wheel_files: wheelhouse/*.whl \ No newline at end of file + packages_dir: wheelhouse/ \ No newline at end of file From 7a3cfc965fd79783fbbf7e12fa7e998c8151ac7a Mon Sep 17 00:00:00 2001 From: Vadim Lebedev Date: Mon, 22 Aug 2022 14:42:45 +0200 Subject: [PATCH 5/8] Add shell script to build libarchive.so --- build-libarchive-lib.sh | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 build-libarchive-lib.sh diff --git a/build-libarchive-lib.sh b/build-libarchive-lib.sh new file mode 100644 index 0000000..6965d5f --- /dev/null +++ b/build-libarchive-lib.sh @@ -0,0 +1,36 @@ +#!/usr/bin/bash + +function build_libarchive() { + tag=$1 + + dd=$PWD + cd /tmp + git clone https://github.com/libarchive/libarchive.git libarchive-src + cd libarchive-src; git checkout $tag + cd /tmp + mkdir build-libarchive; cd build-libarchive + cmake ../libarchive-src + make -j$(nproc); make install + cd $dd + +} + +function install_deps_centos() { + + yum install -y epel-release libxml2-devel libzstd-devel xz-devel bzip2-devel + yum install -y libacl-devel lz4-devel e2fsprogs-devel libb2-devel lzo-devel openssl-devel + yum install -y librichacl-devel swig strace cmake +} + +function install_deps_ubuntu() { + + apt-get install -y libxml2-dev libzstd-dev xz-dev bzip2-dev + apt-get install -y libacl1-dev liblz4-dev libext2fs-dev libb2-dev lzo-dev libssl-dev + apt-get install -y swig strace cmake +} + +os=$1 +tag=$2 +install_deps_$os +build_libarchive $tag + From edfb8f850c42bdd5e369c3a6e179106a7ccc5a73 Mon Sep 17 00:00:00 2001 From: Vadim Lebedev Date: Mon, 22 Aug 2022 14:43:30 +0200 Subject: [PATCH 6/8] Fix cibuildwheel action activation --- .github/workflows/deploy.yml | 80 ++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d9e49ea..915da5f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -7,55 +7,53 @@ env: libarchive_tag: v3.6.1 jobs: - deploy: - strategy: - matrix: - python_version: [3.8, 3.9, 3.10] - - name: Deploy to PyPI - if: startsWith(github.ref, 'refs/tags') + build_wheels: + name: Build wheels on runs-on: ubuntu-latest - container: quay.io/pypa/manylinux2014_x86_64 - steps: - - name: Install dependencies - run: | - yum install -y epel-release libxml2-devel libzstd-devel xz-devel bzip2-devel - yum install -y libacl-devel lz4-devel e2fsprogs-devel libb2-devel lzo-devel openssl-devel - yum install -y librichacl-devel swig strace - - - name: Setup Python - uses: actions/setup-python@v2 + steps: + - uses: actions/checkout@v3 + - name: Build wheels + uses: pypa/cibuildwheel@v2.9.0 + env: + CIBW_ENVIRONMENT: INCLUDE=/usr/local/include LIBARCHIVE_PREFIX=/usr/local + CIBW_PYTHON_VERSIONS: 3.8,3.9,3.10 + CIBW_BEFORE_ALL: bash -x build-libarchive-lib.sh centos ${{ env.libarchive_tag }} + + + - uses: actions/upload-artifact@v3 with: - python-version: ${{ matrix.python_version }} - - name: Install python tools - run: | - python3 -m pip install auditwheel + path: ./wheelhouse/*.whl - - name: Build libarchive C sources - run: | - cd /tmp - git clone https://github.com/libarchive/libarchive.git libarchive-src - cd libarchive-src; git checkout ${{ env.libarchive_tag }} - mkdir build-libarchive; cd build-libarchive - cmake ../ - make -j$(nproc); make install - cd $GITHUB_WORKSPACE + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 - - name: Check out code - uses: actions/checkout@v3 + - name: Build sdist + run: pipx run build --sdist - - name: SWIG and Compile python modules - run: | - touch libarchive/_libarchive.i - make PYVER=${{ matrix.python_version }} + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz - - name: Build wheels - run: | - python3 -m pip wheel --wheel-dir dist/ . - auditwheel repair dist/*.whl + upload_pypi: + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + # upload to PyPI on every tag starting with 'v' + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + # alternatively, to publish when a GitHub Release is created, use the following rule: + # if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v3 + with: + # unpacks default artifact into dist/ + # if `name: artifact` is omitted, the action will create extra parent dir + name: artifact + path: dist - name: Deploy wheels to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} - packages_dir: wheelhouse/ \ No newline at end of file + From bb934bb8ed776951cebaf033111b5a21e7b5506b Mon Sep 17 00:00:00 2001 From: Vadim Lebedev Date: Mon, 22 Aug 2022 15:00:00 +0200 Subject: [PATCH 7/8] Workflow syntax fix --- .github/workflows/deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 915da5f..5fec9a1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -52,8 +52,8 @@ jobs: name: artifact path: dist - - name: Deploy wheels to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} + - name: Deploy wheels to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} From 0fa42bbdca97c7239f94a256967cd8e87ac3ca99 Mon Sep 17 00:00:00 2001 From: Vadim Lebedev Date: Mon, 22 Aug 2022 15:59:14 +0200 Subject: [PATCH 8/8] Fixes for upstream merging --- .github/workflows/deploy.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5fec9a1..ce15ca8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -8,7 +8,7 @@ env: jobs: build_wheels: - name: Build wheels on + name: Build wheels runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -40,10 +40,7 @@ jobs: upload_pypi: needs: [build_wheels, build_sdist] runs-on: ubuntu-latest - # upload to PyPI on every tag starting with 'v' - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - # alternatively, to publish when a GitHub Release is created, use the following rule: - # if: github.event_name == 'release' && github.event.action == 'published' + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') steps: - uses: actions/download-artifact@v3 with: