From 1f1f450177bb40040cdc8143e68899bcd2d2431d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tam=C3=A1s=20B=C3=A1lint=20Misius?= Date: Sun, 11 Sep 2022 00:06:55 +0200 Subject: [PATCH] Fix starcatcher uploads --- .github/prepare.py | 78 ++++++++++++++++++++-------------- .github/starcatcher-publish.sh | 6 ++- .github/starcatcher-release.sh | 3 ++ .github/workflows/build.yaml | 34 +++++++++------ 4 files changed, 74 insertions(+), 47 deletions(-) diff --git a/.github/prepare.py b/.github/prepare.py index 4d079de96..0490df6fe 100755 --- a/.github/prepare.py +++ b/.github/prepare.py @@ -33,54 +33,66 @@ print('::set-output name=release_name::' + release_name) with open('.github/mod_id.txt') as f: print('::set-output name=mod_id::' + f.read()) -configurations = [] -for bsh_host_arch, bsh_host_platform, bsh_host_libc, bsh_static_dynamic, bsh_build_platform, runs_on, package_suffix, publish, artifact, debug_suffix, starcatcher_name in [ - ( 'x86_64', 'linux', 'gnu', 'static', 'linux', 'ubuntu-18.04', '', True, True, '.dbg', '' ), - ( 'x86_64', 'linux', 'gnu', 'dynamic', 'linux', 'ubuntu-18.04', '', False, False, None, '' ), -# ubuntu-20.04 doesn't have windows TLS headers somehow and I haven't yet figured out how to get them -# ( 'x86_64', 'windows', 'mingw', 'static', 'linux', 'ubuntu-20.04', '', False, True, '.dbg', '' ), - ( 'x86_64', 'windows', 'mingw', 'dynamic', 'linux', 'ubuntu-20.04', '', False, False, None, '' ), - ( 'x86_64', 'windows', 'mingw', 'static', 'windows', 'windows-2019', '.exe', False, True, '.dbg', '' ), - ( 'x86_64', 'windows', 'mingw', 'dynamic', 'windows', 'windows-2019', '.exe', False, False, None, '' ), - ( 'x86_64', 'windows', 'msvc', 'static', 'windows', 'windows-2019', '.exe', True, True, '.pdb', '' ), - ( 'x86_64', 'windows', 'msvc', 'dynamic', 'windows', 'windows-2019', '.exe', False, False, None, '' ), - ( 'x86', 'windows', 'msvc', 'static', 'windows', 'windows-2019', '.exe', True, True, '.pdb', '' ), - ( 'x86', 'windows', 'msvc', 'dynamic', 'windows', 'windows-2019', '.exe', False, False, None, '' ), -# I have no idea how to separate debug info on macos - ( 'x86_64', 'darwin', 'macos', 'static', 'darwin', 'macos-11.0', '', True, True, None, '' ), - ( 'x86_64', 'darwin', 'macos', 'dynamic', 'darwin', 'macos-11.0', '', False, False, None, '' ), - ( 'aarch64', 'darwin', 'macos', 'static', 'darwin', 'macos-11.0', '', True, True, None, '' ), -# macos-11.0 is x86_64 and I haven't yet figured out how to get homebrew to install aarch64 libs on x86_64 -# ( 'aarch64', 'darwin', 'macos', 'dynamic', 'darwin', 'macos-11.0', '', False, False, None, '' ), - ( 'x86', 'android', 'bionic', 'static', 'linux', 'ubuntu-18.04', '.apk', True, True, '.dbg', '' ), - ( 'x86_64', 'android', 'bionic', 'static', 'linux', 'ubuntu-18.04', '.apk', True, True, '.dbg', '' ), - ( 'arm', 'android', 'bionic', 'static', 'linux', 'ubuntu-18.04', '.apk', True, True, '.dbg', '' ), - ( 'aarch64', 'android', 'bionic', 'static', 'linux', 'ubuntu-18.04', '.apk', True, True, '.dbg', '' ), +build_matrix = [] +publish_matrix = [] +for bsh_host_arch, bsh_host_platform, bsh_host_libc, bsh_static_dynamic, bsh_build_platform, runs_on, package_suffix, publish, artifact, debug_suffix, starcatcher_name in [ + ( 'x86_64', 'linux', 'gnu', 'static', 'linux', 'ubuntu-18.04', '', True, True, '.dbg', 'x86_64-lin-gcc-static' ), + ( 'x86_64', 'linux', 'gnu', 'dynamic', 'linux', 'ubuntu-18.04', '', False, False, None, '' ), +# ( 'x86_64', 'windows', 'mingw', 'static', 'linux', 'ubuntu-20.04', '', False, True, '.dbg', '' ), # ubuntu-20.04 doesn't have windows TLS headers somehow and I haven't yet figured out how to get them + ( 'x86_64', 'windows', 'mingw', 'dynamic', 'linux', 'ubuntu-20.04', '', False, False, None, '' ), + ( 'x86_64', 'windows', 'mingw', 'static', 'windows', 'windows-2019', '.exe', False, True, '.dbg', '' ), + ( 'x86_64', 'windows', 'mingw', 'dynamic', 'windows', 'windows-2019', '.exe', False, False, None, '' ), + ( 'x86_64', 'windows', 'msvc', 'static', 'windows', 'windows-2019', '.exe', True, True, '.pdb', 'x86_64-win-msvc-static' ), + ( 'x86_64', 'windows', 'msvc', 'dynamic', 'windows', 'windows-2019', '.exe', False, False, None, '' ), + ( 'x86', 'windows', 'msvc', 'static', 'windows', 'windows-2019', '.exe', True, True, '.pdb', 'i686-win-msvc-static' ), + ( 'x86', 'windows', 'msvc', 'dynamic', 'windows', 'windows-2019', '.exe', False, False, None, '' ), + ( 'x86_64', 'darwin', 'macos', 'static', 'darwin', 'macos-11.0', '', True, True, None, 'x86_64-mac-gcc-static' ), # I have no idea how to separate debug info on macos + ( 'x86_64', 'darwin', 'macos', 'dynamic', 'darwin', 'macos-11.0', '', False, False, None, '' ), + ( 'aarch64', 'darwin', 'macos', 'static', 'darwin', 'macos-11.0', '', True, True, None, 'arm64-mac-gcc-static' ), +# ( 'aarch64', 'darwin', 'macos', 'dynamic', 'darwin', 'macos-11.0', '', False, False, None, '' ), # macos-11.0 is x86_64 and I haven't yet figured out how to get homebrew to install aarch64 libs on x86_64 + ( 'x86', 'android', 'bionic', 'static', 'linux', 'ubuntu-18.04', '.apk', True, True, '.dbg', 'i686-and-gcc-static' ), + ( 'x86_64', 'android', 'bionic', 'static', 'linux', 'ubuntu-18.04', '.apk', True, True, '.dbg', 'x86_64-and-gcc-static' ), + ( 'arm', 'android', 'bionic', 'static', 'linux', 'ubuntu-18.04', '.apk', True, True, '.dbg', 'arm-and-gcc-static' ), + ( 'aarch64', 'android', 'bionic', 'static', 'linux', 'ubuntu-18.04', '.apk', True, True, '.dbg', 'arm64-and-gcc-static' ), ]: separate_debug = True if not debug_suffix: debug_suffix = '' separate_debug = False + if publish: + assert artifact for debug_release in [ 'debug', 'release' ]: - configurations.append({ - 'bsh_build_platform': bsh_build_platform, - 'bsh_host_arch': bsh_host_arch, - 'bsh_host_platform': bsh_host_platform, - 'bsh_host_libc': bsh_host_libc, - 'bsh_static_dynamic': bsh_static_dynamic, - 'bsh_debug_release': debug_release, + publish_release = publish and debug_release == 'release' + build_matrix.append({ + 'bsh_build_platform': bsh_build_platform, # part of the unique portion of the matrix + 'bsh_host_arch': bsh_host_arch, # part of the unique portion of the matrix + 'bsh_host_platform': bsh_host_platform, # part of the unique portion of the matrix + 'bsh_host_libc': bsh_host_libc, # part of the unique portion of the matrix + 'bsh_static_dynamic': bsh_static_dynamic, # part of the unique portion of the matrix + 'bsh_debug_release': debug_release, # part of the unique portion of the matrix 'runs_on': runs_on, 'package_suffix': package_suffix, - 'publish': publish and 'yes' or 'no', + 'publish': publish_release and 'yes' or 'no', 'artifact': artifact and 'yes' or 'no', 'separate_debug': separate_debug and 'yes' or 'no', 'asset_path': f'powder{package_suffix}', 'asset_name': f'powder-{release_name}-{bsh_host_arch}-{bsh_host_platform}-{bsh_host_libc}{package_suffix}', 'debug_asset_path': f'powder{debug_suffix}', 'debug_asset_name': f'powder-{release_name}-{bsh_host_arch}-{bsh_host_platform}-{bsh_host_libc}{debug_suffix}', - 'starcatcher_name': starcatcher_name, }) + if publish_release: + publish_matrix.append({ + 'bsh_build_platform': bsh_build_platform, # part of the unique portion of the matrix + 'bsh_host_arch': bsh_host_arch, # part of the unique portion of the matrix + 'bsh_host_platform': bsh_host_platform, # part of the unique portion of the matrix + 'bsh_host_libc': bsh_host_libc, # part of the unique portion of the matrix + 'bsh_static_dynamic': bsh_static_dynamic, # part of the unique portion of the matrix + 'asset_path': f'powder{package_suffix}', + 'asset_name': f'powder-{release_name}-{bsh_host_arch}-{bsh_host_platform}-{bsh_host_libc}{package_suffix}', + 'starcatcher_name': f'powder-{release_name}-{starcatcher_name}{package_suffix}', + }) -print('::set-output name=matrix::' + json.dumps({ 'include': configurations })) +print('::set-output name=build_matrix::' + json.dumps({ 'include': build_matrix })) +print('::set-output name=publish_matrix::' + json.dumps({ 'include': publish_matrix })) print('::set-output name=do_release::' + (ref.startswith('refs/tags/v') and 'yes' or 'no')) print('::set-output name=do_publish::' + ((publish_hostport and (match_stable or match_beta or match_snapshot)) and 'yes' or 'no')) diff --git a/.github/starcatcher-publish.sh b/.github/starcatcher-publish.sh index 19e733bfc..5ced1861a 100755 --- a/.github/starcatcher-publish.sh +++ b/.github/starcatcher-publish.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash -cp "$ASSET_PATH" "$PUBLISH_FILENAME" -LFTP_PASSWORD=$PUBLISH_PASSWORD lftp -c "open --user '$PUBLISH_USERNAME' --env-password -e 'set ftp:ssl-protect-data true; set ssl:verify-certificate false; put \"$PUBLISH_FILENAME\";' ftp://$PUBLISH_HOSTPORT" +set -euo pipefail +IFS=$'\n\t' + +LFTP_PASSWORD=$PUBLISH_PASSWORD lftp -c "open --user '$PUBLISH_USERNAME' --env-password -e 'set ftp:ssl-protect-data true; set ssl:verify-certificate false; cd "${PUBLISH_DIRECTORY:-.}"; put \"$PUBLISH_FILENAME\";' ftp://$PUBLISH_HOSTPORT" diff --git a/.github/starcatcher-release.sh b/.github/starcatcher-release.sh index 7e99bf78a..e22ff9cdd 100755 --- a/.github/starcatcher-release.sh +++ b/.github/starcatcher-release.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + curl "https://starcatcher.us/TPT/perform-release.lua?mod=$MOD_ID&type=$RELEASE_TYPE&name=$RELEASE_NAME&commit=$GITHUB_SHA" > perform_release if ! grep "Release complete" < perform_release > /dev/null; then cat perform_release diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9c49886ab..3c1ad1830 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -18,7 +18,8 @@ jobs: outputs: release_upload_url: ${{ steps.create_release.outputs.upload_url }} do_release: ${{ steps.prepare.outputs.do_release }} - matrix: ${{ steps.prepare.outputs.matrix }} + build_matrix: ${{ steps.prepare.outputs.build_matrix }} + publish_matrix: ${{ steps.prepare.outputs.publish_matrix }} release_type: ${{ steps.prepare.outputs.release_type }} release_name: ${{ steps.prepare.outputs.release_name }} mod_id: ${{ steps.prepare.outputs.mod_id }} @@ -48,7 +49,7 @@ jobs: needs: [prepare] strategy: fail-fast: false - matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} + matrix: ${{ fromJSON(needs.prepare.outputs.build_matrix) }} steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -105,23 +106,32 @@ jobs: with: path: build/${{ matrix.debug_asset_path }} name: ${{ matrix.debug_asset_name }} - - if: needs.prepare.outputs.do_publish == 'yes' && matrix.publish == 'yes' - run: sudo apt update && sudo apt install lftp && bash -c './.github/starcatcher-publish.sh - env: - ASSET_PATH: build/${{ matrix.asset_path }} - PUBLISH_HOSTPORT: ${{ secrets.STARCATCHER_PUBLISH_HOSTPORT }} - PUBLISH_USERNAME: ${{ secrets.STARCATCHER_PUBLISH_USERNAME }} - PUBLISH_PASSWORD: ${{ secrets.STARCATCHER_PUBLISH_PASSWORD }} - PUBLISH_FILENAME: ${{ matrix.starcatcher_name }} publish: runs-on: ubuntu-latest needs: [build, prepare] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.prepare.outputs.publish_matrix) }} if: needs.prepare.outputs.do_publish == 'yes' steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/download-artifact@v1 with: - python-version: '3.10' + name: ${{ matrix.asset_name }} + - run: mv ${{ matrix.asset_name }}/${{ matrix.asset_path }} ${{ matrix.starcatcher_name }} + - run: sudo apt update && sudo apt install lftp && bash -c './.github/starcatcher-publish.sh' + env: + PUBLISH_HOSTPORT: ${{ secrets.STARCATCHER_PUBLISH_HOSTPORT }} + PUBLISH_USERNAME: ${{ secrets.STARCATCHER_PUBLISH_USERNAME }} + PUBLISH_PASSWORD: ${{ secrets.STARCATCHER_PUBLISH_PASSWORD }} + PUBLISH_DIRECTORY: ${{ secrets.STARCATCHER_PUBLISH_DIRECTORY }} + PUBLISH_FILENAME: ${{ matrix.starcatcher_name }} + release: + runs-on: ubuntu-latest + needs: [build, publish, prepare] + if: needs.prepare.outputs.do_publish == 'yes' + steps: + - uses: actions/checkout@v2 - run: ./.github/starcatcher-release.sh env: RELEASE_NAME: ${{ needs.prepare.outputs.release_name }}