Skip to content

Commit

Permalink
Merge pull request #3622 from myk002/myk_test_test
Browse files Browse the repository at this point in the history
first attempt to run tests on windows
  • Loading branch information
myk002 authored Aug 4, 2023
2 parents 5edf85d + d3884bd commit 3599403
Show file tree
Hide file tree
Showing 16 changed files with 271 additions and 168 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/build-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ on:
extras:
type: boolean
default: false
tests:
type: boolean
default: false
gcc-ver:
type: string
default: "10"
Expand Down Expand Up @@ -112,6 +115,7 @@ jobs:
-DBUILD_SIZECHECK:BOOL=${{ inputs.extras }} \
-DBUILD_SKELETON:BOOL=${{ inputs.extras }} \
-DBUILD_DOCS:BOOL=${{ inputs.docs }} \
-DBUILD_TESTS:BOOL=${{ inputs.tests }} \
-DINSTALL_DATA_FILES:BOOL=${{ inputs.common-files }} \
-DINSTALL_SCRIPTS:BOOL=${{ inputs.common-files }}
- name: Build DFHack
Expand Down
22 changes: 20 additions & 2 deletions .github/workflows/build-windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ on:
inputs:
dfhack_ref:
type: string
scripts_ref:
type: string
structures_ref:
type: string
artifact-name:
Expand All @@ -21,6 +23,15 @@ on:
common-files:
type: boolean
default: true
docs:
type: boolean
default: false
stonesense:
type: boolean
default: false
tests:
type: boolean
default: false
launchdf:
type: boolean
default: false
Expand All @@ -40,6 +51,13 @@ jobs:
ref: ${{ inputs.dfhack_ref }}
submodules: true
fetch-depth: 0
- name: Clone scripts
if: inputs.scripts_ref
uses: actions/checkout@v3
with:
repository: 'DFHack/scripts'
ref: ${{ inputs.scripts_ref }}
path: scripts
- name: Clone structures
if: inputs.structures_ref
uses: actions/checkout@v3
Expand All @@ -65,14 +83,14 @@ jobs:
win-msvc
- name: Cross-compile
env:
CMAKE_EXTRA_ARGS: '-DBUILD_STONESENSE:BOOL=1 -DBUILD_DOCS:BOOL=${{ inputs.common-files }} -DINSTALL_DATA_FILES:BOOL=${{ inputs.common-files }} -DINSTALL_SCRIPTS:BOOL=${{ inputs.common-files }} -DBUILD_DFLAUNCH:BOOL=${{ inputs.launchdf }}'
CMAKE_EXTRA_ARGS: '-DBUILD_STONESENSE:BOOL=${{ inputs.stonesense }} -DBUILD_DOCS:BOOL=${{ inputs.docs }} -DINSTALL_DATA_FILES:BOOL=${{ inputs.common-files }} -DINSTALL_SCRIPTS:BOOL=${{ inputs.common-files }} -DBUILD_DFLAUNCH:BOOL=${{ inputs.launchdf }} -DBUILD_TESTS:BOOL=${{ inputs.tests }}'
run: |
cd build
bash -x build-win64-from-linux.sh
- name: Trim cache
run: |
cd build
ccache -d win64-cross/ccache --max-size 200M
ccache -d win64-cross/ccache --max-size 150M
ccache -d win64-cross/ccache --cleanup
ccache -d win64-cross/ccache --show-stats --verbose
- name: Save ccache
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ jobs:
artifact-name: dfhack-win64-build
append-date-and-hash: true
cache-id: release
stonesense: true
docs: true
secrets: inherit

docs:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/github-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ jobs:
dfhack_ref: ${{ github.event.inputs && github.event.inputs.ref || github.event.ref }}
cache-id: release
cache-readonly: true
stonesense: true
docs: true
launchdf: true
secrets: inherit

Expand Down
1 change: 1 addition & 0 deletions .github/workflows/steam-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ jobs:
cache-id: release
cache-readonly: true
common-files: false
stonesense: true
launchdf: true
secrets: inherit

Expand Down
119 changes: 79 additions & 40 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,17 @@ on:
type: string

jobs:
build-windows:
name: Windows MSVC
uses: ./.github/workflows/build-windows.yml
with:
dfhack_ref: ${{ inputs.dfhack_ref }}
scripts_ref: ${{ inputs.scripts_ref }}
structures_ref: ${{ inputs.structures_ref }}
artifact-name: test-msvc
cache-id: test
tests: true

build-linux:
name: Linux gcc-${{ matrix.gcc }}
uses: ./.github/workflows/build-linux.yml
Expand All @@ -22,6 +33,7 @@ jobs:
cache-id: test
stonesense: ${{ matrix.plugins == 'all' }}
extras: ${{ matrix.plugins == 'all' }}
tests: true
gcc-ver: ${{ matrix.gcc }}
secrets: inherit
strategy:
Expand All @@ -33,51 +45,78 @@ jobs:
- gcc: 12
plugins: "all"

test-linux:
name: Test (Linux, GCC ${{ matrix.gcc }}, ${{ matrix.plugins }} plugins)
needs: build-linux
runs-on: ubuntu-latest
test-windows:
name: Test (${{ matrix.os }}, ${{ matrix.compiler }}, ${{ matrix.plugins }} plugins)
needs:
- build-windows
- build-linux
runs-on: ${{ matrix.os }}-latest
strategy:
fail-fast: false
matrix:
include:
- gcc: 10
- os: windows
compiler: msvc
plugins: "default"
- gcc: 12
plugins: "all"
# TODO: uncomment once we have a linux build we can download from bay12
# - os: ubuntu
# compiler: gcc-10
# gcc: 10
# plugins: "default"
# - os: ubuntu
# compiler: gcc-10
# gcc: 12
# plugins: "all"
steps:
- name: Download artifact
- name: Set env (windows)
if: matrix.os == 'windows'
run: echo "DF_FOLDER=DF" >> $env:GITHUB_ENV
- name: Set env (posix)
if: matrix.os != 'windows'
run: echo "DF_FOLDER=DF" >> $GITHUB_ENV
- name: Clone DFHack
uses: actions/checkout@v3
with:
repository: 'DFHack/dfhack'
ref: ${{ inputs.dfhack_ref }}
- name: Detect DF version (windows)
if: matrix.os == 'windows'
run: echo DF_VERSION="$(sh ci/get-df-version.sh)" >> $env:GITHUB_ENV
- name: Detect DF version (posix)
if: matrix.os != 'windows'
run: echo DF_VERSION="$(sh ci/get-df-version.sh)" >> $GITHUB_ENV
- name: Fetch DF cache
id: restore-df
uses: actions/cache/restore@v3
with:
path: ${{ env.DF_FOLDER }}
key: df-${{ matrix.os }}-${{ env.DF_VERSION }}-${{ hashFiles('ci/download-df.sh') }}
- name: Download DF
if: steps.restore-df.outputs.cache-hit != 'true'
run: sh ci/download-df.sh ${{ env.DF_FOLDER }} ${{ matrix.os }} ${{ env.DF_VERSION }}
- name: Save DF cache
if: steps.restore-df.outputs.cache-hit != 'true'
uses: actions/cache/save@v3
with:
path: ${{ env.DF_FOLDER }}
key: df-${{ matrix.os }}-${{ env.DF_VERSION }}-${{ hashFiles('ci/download-df.sh') }}
- name: Download DFHack
uses: actions/download-artifact@v3
with:
name: test-gcc-${{ matrix.gcc }}
# - name: Fetch DF cache
# uses: actions/cache@v3
# with:
# path: ~/DF
# key: df-${{ hashFiles('ci/download-df.sh') }}
# - name: Download DF
# run: |
# sh ci/download-df.sh
# - name: Run lua tests
# id: run_tests_lua
# run: |
# export TERM=dumb
# status=0
# script -qe -c "python ci/run-tests.py --headless --keep-status \"$DF_FOLDER\"" || status=$((status + 1))
# python ci/check-rpc.py "$DF_FOLDER/dfhack-rpc.txt" || status=$((status + 2))
# mkdir -p artifacts
# cp "$DF_FOLDER"/test*.json "$DF_FOLDER"/*.log artifacts || status=$((status + 4))
# exit $status
# - name: Upload test artifacts
# uses: actions/upload-artifact@v3
# if: (success() || failure()) && steps.run_tests.outcome != 'skipped'
# continue-on-error: true
# with:
# name: test-artifacts-${{ matrix.gcc }}
# path: artifacts
# - name: Clean up DF folder
# # prevent DFHack-generated files from ending up in the cache
# # (download-df.sh also removes them, this is just to save cache space)
# if: success() || failure()
# run: |
# rm -rf "$DF_FOLDER"
name: test-${{ matrix.compiler }}
path: ${{ env.DF_FOLDER }}
- name: Run lua tests
timeout-minutes: 10
run: python ci/run-tests.py --keep-status "${{ env.DF_FOLDER }}"
- name: Check RPC interface
run: python ci/check-rpc.py "${{ env.DF_FOLDER }}/dfhack-rpc.txt"
- name: Upload test artifacts
uses: actions/upload-artifact@v3
if: always()
continue-on-error: true
with:
name: test-artifacts-msvc
path: |
${{ env.DF_FOLDER }}/dfhack-rpc.txt
${{ env.DF_FOLDER }}/test*.json
${{ env.DF_FOLDER }}/*.log
9 changes: 8 additions & 1 deletion ci/check-rpc.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
#!/usr/bin/env python3
import glob
import itertools
import sys

actual = {'': {}}
SEP = ('=' * 80)

with open(sys.argv[1]) as f:
plugin_name = ''
Expand All @@ -26,7 +28,7 @@
parts = line.split(' ')
expected[''][parts[2]] = (parts[4], parts[6])

for p in glob.iglob('plugins/proto/*.proto'):
for p in itertools.chain(glob.iglob('plugins/proto/*.proto'), glob.iglob('plugins/*/proto/*.proto')):
plugin_name = ''
with open(p) as f:
for line in f:
Expand All @@ -53,6 +55,7 @@
methods = actual[plugin_name]

if plugin_name not in expected:
print(SEP)
print('Missing documentation for plugin proto files: ' + plugin_name)
print('Add the following lines:')
print('// Plugin: ' + plugin_name)
Expand All @@ -73,12 +76,14 @@
missing.append('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1])

if len(missing) > 0:
print(SEP)
print('Incomplete documentation for ' + ('core' if plugin_name == '' else 'plugin "' + plugin_name + '"') + ' proto files. Add the following lines:')
for m in missing:
print(m)
error_count += 1

if len(wrong) > 0:
print(SEP)
print('Incorrect documentation for ' + ('core' if plugin_name == '' else 'plugin "' + plugin_name + '"') + ' proto files. Replace the following comments:')
for m in wrong:
print(m)
Expand All @@ -88,6 +93,7 @@
methods = expected[plugin_name]

if plugin_name not in actual:
print(SEP)
print('Incorrect documentation for plugin proto files: ' + plugin_name)
print('The following methods are documented, but the plugin does not provide any RPC methods:')
for m in methods:
Expand All @@ -102,6 +108,7 @@
missing.append('// RPC ' + m + ' : ' + io[0] + ' -> ' + io[1])

if len(missing) > 0:
print(SEP)
print('Incorrect documentation for ' + ('core' if plugin_name == '' else 'plugin "' + plugin_name + '"') + ' proto files. Remove the following lines:')
for m in missing:
print(m)
Expand Down
80 changes: 39 additions & 41 deletions ci/download-df.sh
Original file line number Diff line number Diff line change
@@ -1,52 +1,50 @@
#!/bin/sh

DF_FOLDER=$1
OS_TARGET=$2
DF_VERSION=$3

set -e

df_tardest="df.tar.bz2"
save_tardest="test_save.tgz"

cd "$(dirname "$0")"
echo "DF_VERSION: $DF_VERSION"
echo "DF_FOLDER: $DF_FOLDER"
mkdir -p "$DF_FOLDER"
# back out of df_linux
cd "$DF_FOLDER/.."

if ! test -f "$df_tardest"; then
minor=$(echo "$DF_VERSION" | cut -d. -f2)
patch=$(echo "$DF_VERSION" | cut -d. -f3)
echo "Downloading DF $DF_VERSION"
while read url; do
echo "Attempting download: ${url}"
if wget -v "$url" -O "$df_tardest"; then
break
fi
done <<URLS
https://www.bay12games.com/dwarves/df_${minor}_${patch}_linux.tar.bz2
https://files.dfhack.org/DF/${minor}.${patch}/df_${minor}_${patch}_linux.tar.bz2
URLS
echo $df_tardest
if ! test -f "$df_tardest"; then
echo "DF failed to download: $df_tardest not found"
exit 1
fi

echo "Downloading test save"
#test_save_url="https://files.dfhack.org/DF/0.${minor}.${patch}/test_save.tgz"
test_save_url="https://drive.google.com/uc?export=download&id=1XvYngl-DFONiZ9SD9OC4B2Ooecu8rPFz"
if ! wget -v "$test_save_url" -O "$save_tardest"; then
echo "failed to download test save"
exit 1
fi
echo $save_tardest
minor=$(echo "$DF_VERSION" | cut -d. -f1)
patch=$(echo "$DF_VERSION" | cut -d. -f2)
df_url="https://www.bay12games.com/dwarves/df_${minor}_${patch}"
if test "$OS_TARGET" = "windows"; then
WGET="C:/msys64/usr/bin/wget.exe"
df_url="${df_url}_win_s.zip"
df_archive_name="df.zip"
df_extract_cmd="unzip -d ${DF_FOLDER}"
elif test "$OS_TARGET" = "ubuntu"; then
WGET=wget
df_url="${df_url}_linux.tar.bz2"
df_archive_name="df.tar.bz2"
df_extract_cmd="tar -x -j --strip-components=1 -f"
else
echo "Unhandled OS target: ${OS_TARGET}"
exit 1
fi

if ! $WGET -v "$df_url" -O "$df_archive_name"; then
echo "Failed to download DF from $df_url"
exit 1
fi

rm -rf df_linux
mkdir -p df_linux/save
save_url="https://dffd.bay12games.com/download.php?id=15434&f=dreamfort.7z"
save_archive_name="test_save.7z"
save_extract_cmd="7z x -oDF/save"

if ! $WGET -v "$save_url" -O "$save_archive_name"; then
echo "Failed to download test save from $save_url"
exit 1
fi

echo Extracting
tar xf "$df_tardest" --strip-components=1 -C df_linux
tar xf "$save_tardest" -C df_linux/save
$df_extract_cmd "$df_archive_name"
$save_extract_cmd "$save_archive_name"
mv DF/save/* DF/save/region1

echo Done

ls -l

md5sum "$df_archive_name" "$save_archive_name"
Loading

0 comments on commit 3599403

Please sign in to comment.