Index: ps/trunk/build/jenkins/build-patch.sh =================================================================== --- ps/trunk/build/jenkins/build-patch.sh +++ ps/trunk/build/jenkins/build-patch.sh @@ -1,98 +0,0 @@ -#!/bin/sh - -set -e - -# This script allows Jenkins to build and test a patch inside the phabricator -# workspace. - -# The important thing is determining how much cleanup is needed. For instance, -# if a patch adds a source file, it is necessary to delete it afterwards and -# remove all related build artifacts. It will also be necessary to rebuild the game -# for testing the next patch, even if that new patch doesn't change the code. - -# However, rebuilding everything each time is expensive and prevents us from -# handling a decent input of patches on Phabricator. - -# This script does its best to determine a somewhat minimal necessary amount of -# cleanup, and resorts to a clean checkout in case of failure. - -if [ -z "$DIFF_ID" ]; then - echo 'No patch ID provided, aborting' - exit 1 -fi - -# Build environment -JOBS=${JOBS:="-j2"} - -# Move to the root of the repository (this script is in build/jenkins/) -cd "$(dirname $0)"/../../ - -# Utility functions - -fully_cleaned=false - -patch() -{ - # Revert previously applied changes or deleted files (see full_cleanup). - svn revert -R . - - # Removing non-ignored files from binaries/data/ might be necessary and - # doesn't impact the build, so do it each time. - svn status binaries/data/ | cut -c 9- | xargs rm -rf - - # If we have non-ignored files under source/, we must delete them and - # clean the workspaces. - if [ "$(svn status source/)" ]; then - svn status source/ | cut -c 9- | xargs rm -rf - build/workspaces/clean-workspaces.sh --preserve-libs - fi - - # Apply the patch - # Patch errors (including successful hunks) are written to the Phab comment. - # If patching is successful, this will be overwritten in the build() step. - # If patching fails, the comment will be explicit about it. - arc patch --diff "$DIFF_ID" --force 2> .phabricator-comment -} - -full_cleanup() -{ - fully_cleaned=true - svn st --no-ignore | cut -c 9- | xargs rm -rf - patch # patch will revert everything, so don't worry about deleted files -} - -build() -{ - { - echo 'Updating workspaces...' - build/workspaces/update-workspaces.sh "${JOBS}" --with-system-nvtt --without-miniupnpc >/dev/null || echo 'Updating workspaces failed!' - cd build/workspaces/gcc - echo 'Build (release)...' - make "${JOBS}" config=release 2>&1 1>/dev/null - echo 'Build (debug)...' - make "${JOBS}" config=debug 2>&1 1>/dev/null - cd ../../../binaries/system - echo 'Running release tests...' - ./test 2>&1 - echo 'Running debug tests...' - ./test_dbg 2>&1 - cd ../../source/tools/xmlvalidator - echo 'Checking XML files...' - perl validate.pl 2>&1 1>/dev/null - } > .phabricator-comment -} - - -# Try to patch, if that manages to fail, clean up everything. -# If it patches, make sure the libraries are not touched. -if ! patch; then - full_cleanup -elif svn status libraries/source/ | grep -v '^?' >/dev/null 2>&1; then - full_cleanup -fi - -# Try to build, if that fails, clean up everything. -if ! build && [ "$fully_cleaned" != true ]; then - full_cleanup - build -fi Index: ps/trunk/build/jenkins/dockerfiles/coala.Dockerfile =================================================================== --- ps/trunk/build/jenkins/dockerfiles/coala.Dockerfile +++ ps/trunk/build/jenkins/dockerfiles/coala.Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.7-alpine + +RUN adduser -u 1006 -D builder + +RUN apk add subversion cppcheck npm + +RUN pip3 install coala-bears svn +RUN npm install -g eslint@5.16.0 jshint eslint-plugin-brace-rules + +USER builder Index: ps/trunk/build/jenkins/dockerfiles/gcc6.Dockerfile =================================================================== --- ps/trunk/build/jenkins/dockerfiles/gcc6.Dockerfile +++ ps/trunk/build/jenkins/dockerfiles/gcc6.Dockerfile @@ -0,0 +1,43 @@ +FROM gcc:6 + +RUN useradd -ms /bin/bash --uid 1006 builder + +RUN apt-get -qq update && apt-get -qq upgrade +RUN apt-get install -qqy cmake libcurl4-gnutls-dev libenet-dev \ + libgnutls28-dev libgtk-3-dev libicu-dev libidn11-dev libjson-perl \ + libminiupnpc-dev libnspr4-dev libpython-dev libogg-dev \ + libopenal-dev libpng-dev libsdl2-dev libvorbis-dev libxcursor-dev \ + libxml2-dev libxml-simple-perl subversion zlib1g-dev + +ADD libsodium-1.0.17.tar.gz /root/libsodium/ +RUN cd /root/libsodium/libsodium-1.0.17 \ + && ./configure \ + && make && make check \ + && make install + +ADD boost_1_67_0.tar.bz2 /root/boost/ +RUN cd /root/boost/boost_1_67_0 \ + && ./bootstrap.sh --with-libraries=filesystem \ + && ./b2 install + +ADD gloox-1.0.22.tar.bz2 /root/gloox/ +RUN cd /root/gloox/gloox-1.0.22 \ + && ./configure \ + && make && make check \ + && make install + +ADD wxWidgets-3.0.4.tar.bz2 /root/wxWidgets/ +RUN cd /root/wxWidgets/wxWidgets-3.0.4 \ + && ./configure \ + && make \ + && make install + +RUN ldconfig + +USER builder +ENV SHELL /bin/bash + +ENV LIBCC gcc +ENV LIBCXX g++ +ENV PSCC gcc +ENV PSCXX g++ Index: ps/trunk/build/jenkins/dockerfiles/translations.Dockerfile =================================================================== --- ps/trunk/build/jenkins/dockerfiles/translations.Dockerfile +++ ps/trunk/build/jenkins/dockerfiles/translations.Dockerfile @@ -0,0 +1,17 @@ +FROM gcc:8 + +RUN useradd -ms /bin/bash --uid 1006 builder + +RUN apt-get -qq update && apt-get -qq upgrade +RUN apt-get install -qqy cmake git gettext libxml2-dev python python-pip subversion + +RUN pip2 install transifex-client lxml + +RUN git clone https://anongit.kde.org/pology.git /root/pology +RUN mkdir /root/pology/build +WORKDIR /root/pology/build +RUN cmake .. +RUN make && make install + +USER builder +COPY --chown=builder transifexrc /home/builder/.transifexrc Index: ps/trunk/build/jenkins/pipelines/docker-differential.Jenkinsfile =================================================================== --- ps/trunk/build/jenkins/pipelines/docker-differential.Jenkinsfile +++ ps/trunk/build/jenkins/pipelines/docker-differential.Jenkinsfile @@ -0,0 +1,160 @@ +/* Copyright (C) 2019 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +// This pipeline is used to build patches on various compilers. + +def compilers = ["gcc6"] + +def patchesMap = compilers.collectEntries { + ["${it}": patch(it)] +} +def patch(compiler) { + return { + stage("Patch: ${compiler}") { + try { + ws("/zpool0/${compiler}") { + sh "arc patch --diff ${params.DIFF_ID} --force" + } + } catch(e) { + sh "sudo zfs rollback zpool0/${compiler}@latest" + throw e + } + } + } +} + +def buildsMap = compilers.collectEntries { + ["${it}": build(it)] +} +def build(compiler) { + return { + stage("Build: ${compiler}") { + try { + ws("/zpool0/${compiler}") { + docker.image("0ad-${compiler}:latest").inside { + sh "build/workspaces/update-workspaces.sh -j1 --jenkins-tests" + + // TODO: Mark build as unstable in case of warnings and tweak the plugin accordingly. + + sh "cd build/workspaces/gcc/ && make -j1 config=debug" + try { + sh "binaries/system/test_dbg > cxxtest-debug-${compiler}.xml" + } catch (e) { + echo (message: readFile (file: "cxxtest-debug-${compiler}.xml")) + throw e + } finally { + stash includes: "cxxtest-debug-${compiler}.xml", name: "tests-debug-${compiler}" + } + + sh "cd build/workspaces/gcc/ && make -j1 config=release" + try { + sh "binaries/system/test > cxxtest-release-${compiler}.xml" + } catch (e) { + echo (message: readFile (file: "cxxtest-release-${compiler}.xml")) + throw e + } finally { + stash includes: "cxxtest-release-${compiler}.xml", name: "tests-release-${compiler}" + } + } + } + } catch (e) { + throw e + } finally { + sh "sudo zfs rollback zpool0/${compiler}@latest" + } + } + } +} + +pipeline { + agent { + node { + label 'LinuxSlave' + customWorkspace '/zpool0/trunk' + } + } + parameters { + string(name: 'DIFF_ID', defaultValue: '', description: 'ID of the Phabricator Differential.') + string(name: 'PHID', defaultValue: '', description: 'Phabricator ID') + } + + stages { + stage("Patch") { + steps { + sh "arc patch --diff ${params.DIFF_ID} --force" + script { parallel patchesMap } + } + } + stage("Build") { + steps { + script { parallel buildsMap } + } + post { + always { + script { + for(compiler in compilers) { + catchError { unstash "tests-debug-${compiler}" } + catchError { unstash "tests-release-${compiler}" } + } + } + catchError { junit 'cxxtest*.xml' } + } + } + } + stage("Lint") { + steps { + script { + try { + withDockerContainer("0ad-coala:latest") { + sh "svn st | grep '^[AM]' | cut -c 9- | xargs coala -d build/coala --ci --flush-cache --limit-files > coala-report" + } + } catch (e) { + sh ''' + echo "Linter detected issues:" >> phabricator-comment + cat coala-report >> phabricator-comment + echo "\n" >> phabricator-comment + ''' + } + } + echo (message: readFile (file: "coala-report")) + } + } + stage("Data checks") { + steps { + sh "cd source/tools/entity/ && perl checkrefs.pl --check-map-xml --validate-templates" + } + } + } + + post { + always { + script { + try { + if (fileExists("phabricator-comment")) { + step([$class: 'PhabricatorNotifier', commentOnSuccess: true, commentWithConsoleLinkOnFailure: true, customComment: true, commentFile: "phabricator-comment"]) + } else { + step([$class: 'PhabricatorNotifier', commentOnSuccess: true, commentWithConsoleLinkOnFailure: true]) + } + } catch(e) { + throw e + } finally { + sh "sudo zfs rollback zpool0/trunk@latest" + } + } + } + } +} Index: ps/trunk/build/jenkins/pipelines/docker-rebuild.Jenkinsfile =================================================================== --- ps/trunk/build/jenkins/pipelines/docker-rebuild.Jenkinsfile +++ ps/trunk/build/jenkins/pipelines/docker-rebuild.Jenkinsfile @@ -0,0 +1,87 @@ +/* Copyright (C) 2019 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +// This pipeline is used to build a clean base from scratch in order to +// use ZFS snapshots efficiently. + +def compilers = ["gcc6"] + +def volumeUpdatesMap = compilers.collectEntries { + ["${it}": volumeUpdate(it)] +} +def volumeUpdate(compiler) { + return { + stage("Recreate: ${compiler}") { + sh "sudo zfs clone zpool0/trunk@base zpool0/${compiler}" + } + } +} + +def buildsMap = compilers.collectEntries { + ["${it}": build(it)] +} +def build(compiler) { + return { + stage("Build: ${compiler}") { + try { + ws("/zpool0/${compiler}") { + docker.image("0ad-${compiler}:latest").inside { + sh "build/workspaces/update-workspaces.sh -j1 --jenkins-tests" + + sh "cd build/workspaces/gcc/ && make -j1 config=debug" + sh "binaries/system/test_dbg" + + sh "cd build/workspaces/gcc/ && make -j1 config=release" + sh "binaries/system/test" + } + } + } catch (e) { + throw e + } finally { + sh "sudo zfs snapshot zpool0/${compiler}@latest" + } + } + } +} + +pipeline { + agent { + node { + label 'LinuxSlave' + customWorkspace '/zpool0/trunk' + } + } + + stages { + stage("Volume updates") { + steps { + // Note: latest must always be the last snapshot in order to be able to rollback to it + sh "sudo zfs rollback zpool0/trunk@latest" + sh "sudo zfs destroy -R zpool0/trunk@latest" + sh "sudo zfs destroy -R zpool0/trunk@base" + sh "sudo zfs snapshot zpool0/trunk@base" + sh "sudo zfs snapshot zpool0/trunk@latest" + script { parallel volumeUpdatesMap } + } + } + stage("Build") { + steps { + script { parallel buildsMap } + } + } + } +} Index: ps/trunk/build/jenkins/pipelines/docker-reset.Jenkinsfile =================================================================== --- ps/trunk/build/jenkins/pipelines/docker-reset.Jenkinsfile +++ ps/trunk/build/jenkins/pipelines/docker-reset.Jenkinsfile @@ -0,0 +1,81 @@ +/* Copyright (C) 2019 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +// This is a helper pipeline to build Docker images and setup ZFS volumes. + +pipeline { + agent { + node { + label 'LinuxSlave' + customWorkspace '/zpool0/trunk' + } + } + parameters { + booleanParam(name: 'no-cache', defaultValue: false, description: 'Rebuild containers from scratch') + booleanParam(name: 'reset-volumes', defaultValue: false, description: 'Reset ZFS volumes') + } + + stages { + stage("Cleanup") { + steps { + sh 'docker system prune -f' + } + } + stage("Full Rebuild") { + when { + environment name: 'no-cache', value: 'true' + } + steps { + sh 'docker build --no-cache -t 0ad-gcc6 ~/dockerfiles/gcc6' + // WIP: clang and recent gccs + sh 'docker build --no-cache -t 0ad-coala ~/dockerfiles/coala' + sh 'docker build --no-cache -t 0ad-translations ~/dockerfiles/translations' + } + } + stage("Build") { + steps { + sh 'docker build -t 0ad-gcc6 ~/dockerfiles/gcc6' + // WIP: clang and recent gccs + sh 'docker build -t 0ad-coala ~/dockerfiles/coala' + sh 'docker build -t 0ad-translations ~/dockerfiles/translations' + } + } + stage("Update") { + steps { + sh "svn cleanup 2>/dev/null || true" + svn "https://svn.wildfiregames.com/public/ps/trunk" + sh "svn st --no-ignore | cut -c 9- | xargs rm -rf" + sh "svn revert -R ." + } + } + stage("Volumes") { + when { + environment name: 'reset-volumes', value: 'true' + } + steps { + sh "sudo zfs destroy -R zpool0/trunk@base || true" + sh "sudo zfs destroy -R zpool0/trunk@latest || true" + + sh "sudo zfs snapshot zpool0/trunk@base" + sh "sudo zfs clone zpool0/trunk@base zpool0/gcc6" + sh "sudo zfs clone zpool0/trunk@base zpool0/gcc7" + + sh "sudo zfs snapshot zpool0/trunk@latest" + } + } + } +} Index: ps/trunk/build/jenkins/pipelines/docker-svn.Jenkinsfile =================================================================== --- ps/trunk/build/jenkins/pipelines/docker-svn.Jenkinsfile +++ ps/trunk/build/jenkins/pipelines/docker-svn.Jenkinsfile @@ -0,0 +1,141 @@ +/* Copyright (C) 2019 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +// This is the post-commit pipeline. +// In case of success, it provides a clean base for incremental builds +// with the `differential` pipeline. + +def compilers = ["gcc6"] + +def volumeUpdatesMap = compilers.collectEntries { + ["${it}": volumeUpdate(it)] +} +def volumeUpdate(compiler) { + return { + stage("Update: ${compiler}") { + ws("/zpool0/${compiler}") { + sh "svn up -r ${params.SVN_REV}" + } + } + } +} + +def buildsMap = compilers.collectEntries { + ["${it}": build(it)] +} +def build(compiler) { + return { + stage("Build: ${compiler}") { + try { + ws("/zpool0/${compiler}") { + docker.image("0ad-${compiler}:latest").inside { + sh "build/workspaces/update-workspaces.sh -j1 --jenkins-tests" + + sh "cd build/workspaces/gcc/ && make -j1 config=debug" + try { + sh "binaries/system/test_dbg > cxxtest-debug-${compiler}.xml" + } catch (e) { + echo (message: readFile (file: "cxxtest-debug-${compiler}.xml")) + throw e + } finally { + stash includes: "cxxtest-debug-${compiler}.xml", name: "tests-debug-${compiler}" + } + + sh "cd build/workspaces/gcc/ && make -j1 config=release" + try { + sh "binaries/system/test > cxxtest-release-${compiler}.xml" + } catch (e) { + echo (message: readFile (file: "cxxtest-release-${compiler}.xml")) + throw e + } finally { + stash includes: "cxxtest-release-${compiler}.xml", name: "tests-release-${compiler}" + } + } + } + + sh "sudo zfs destroy zpool0/${compiler}@latest" + sh "sudo zfs snapshot zpool0/${compiler}@latest" + } catch (e) { + sh "sudo zfs rollback zpool0/${compiler}@latest" + throw e + } + } + } +} + +pipeline { + agent { + node { + label 'LinuxSlave' + customWorkspace '/zpool0/trunk' + } + } + parameters { + string(name: 'SVN_REV', defaultValue: '', description: 'For instance 21000') + string(name: 'PHID', defaultValue: '', description: 'Phabricator ID') + } + + stages { + stage("Update") { + steps { + sh "svn cleanup 2>/dev/null || true" + svn "https://svn.wildfiregames.com/public/ps/trunk@${params.SVN_REV}" + sh "svn st --no-ignore | cut -c 9- | xargs rm -rf" + sh "svn revert -R ." + } + } + stage("Volume updates") { + steps { + script { parallel volumeUpdatesMap } + } + } + stage("Build") { + steps { + script { parallel buildsMap } + } + post { + always { + script { + for(compiler in compilers) { + catchError { unstash "tests-debug-${compiler}" } + catchError { unstash "tests-release-${compiler}" } + } + } + catchError { junit 'cxxtest*.xml' } + } + failure { + sh "sudo zfs rollback zpool0/trunk@latest" + } + success { + sh "sudo zfs destroy -R zpool0/trunk@latest" + sh "sudo zfs snapshot zpool0/trunk@latest" + } + } + } + stage("Data checks") { + steps { + sh "cd source/tools/entity/ && perl checkrefs.pl --check-map-xml --validate-templates" + } + } + } + + post { + always { + step([$class: 'PhabricatorNotifier']) + } + } +} Index: ps/trunk/build/jenkins/pipelines/docker-translations.Jenkinsfile =================================================================== --- ps/trunk/build/jenkins/pipelines/docker-translations.Jenkinsfile +++ ps/trunk/build/jenkins/pipelines/docker-translations.Jenkinsfile @@ -0,0 +1,64 @@ +/* Copyright (C) 2019 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +// This pipeline is used to update translations to and from Transifex. + +pipeline { + agent { + node { + label 'LinuxSlave' + customWorkspace '/zpool0/trunk' + } + } + + stages { + stage("Prepare volume") { + steps { + sh "sudo zfs clone zpool0/trunk@latest zpool0/translations" + } + } + + stage("Update translations") { + steps { + ws('/zpool0/translations') { + withDockerContainer("0ad-translations:latest") { + sh "sh source/tools/i18n/maintenanceTasks.sh" + } + } + } + } + + stage("Commit") { + steps { + ws('/zpool0/translations') { + withCredentials([usernamePassword(credentialsId: 'redacted', passwordVariable: 'SVNPASS', usernameVariable: 'SVNUSER')]) { + sh "svn relocate --username ${SVNUSER} --password ${SVNPASS} --no-auth-cache https://svn.wildfiregames.com/svn/ps/trunk" + sh "svn add --force binaries/" + sh "svn commit --username ${SVNUSER} --password ${SVNPASS} --no-auth-cache --non-interactive -m '[i18n] Updated POT and PO files.'" + } + } + } + } + } + + post { + always { + sleep 10 + sh "sudo zfs destroy zpool0/translations" + } + } +} Index: ps/trunk/build/jenkins/pipelines/vs2015-autobuild.Jenkinsfile =================================================================== --- ps/trunk/build/jenkins/pipelines/vs2015-autobuild.Jenkinsfile +++ ps/trunk/build/jenkins/pipelines/vs2015-autobuild.Jenkinsfile @@ -1,3 +1,23 @@ +/* Copyright (C) 2019 Wildfire Games. + * This file is part of 0 A.D. + * + * 0 A.D. is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * 0 A.D. is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with 0 A.D. If not, see . + */ + +// This pipeline builds Windows binaries. It is run at most daily, every +// time a commit touches the source code. + def AtlasOption = "" def GlooxOption = "" def AtlasPrj = ""