From 9ea73c0ff089b6f84c5e134a1b67f88eff45dbf6 Mon Sep 17 00:00:00 2001 From: tobtoht Date: Sun, 2 Jul 2023 21:00:24 +0200 Subject: [PATCH] Bootstrappable Builds --- .github/workflows/gitian.yml | 49 -- .github/workflows/guix.yml | 86 ++ .gitignore | 6 +- CMakeLists.txt | 2 +- README.md | 6 +- contrib/depends/Makefile | 11 +- contrib/depends/funcs.mk | 38 +- contrib/depends/hosts/darwin.mk | 62 +- contrib/depends/hosts/freebsd.mk | 13 +- contrib/depends/hosts/linux.mk | 2 + contrib/depends/packages/android_ndk.mk | 7 +- contrib/depends/packages/boost.mk | 5 +- contrib/depends/packages/darwin_sdk.mk | 1 + contrib/depends/packages/freebsd_base.mk | 10 +- contrib/depends/packages/native_cctools.mk | 10 +- contrib/depends/packages/native_clang.mk | 28 - contrib/depends/packages/native_libtapi.mk | 18 +- contrib/depends/packages/packages.mk | 2 +- .../depends/patches/android_ndk/fix_env.patch | 30 + .../depends/patches/boost/fix_coalesce.patch | 17 + contrib/depends/toolchain.cmake.in | 11 +- contrib/gitian/DOCKRUN.md | 114 --- contrib/gitian/README.md | 272 ------ contrib/gitian/dockrun.sh | 128 --- contrib/gitian/gitian-android.yml | 131 --- contrib/gitian/gitian-build.py | 203 ----- contrib/gitian/gitian-freebsd.yml | 128 --- contrib/gitian/gitian-linux.yml | 180 ---- contrib/gitian/gitian-osx.yml | 119 --- contrib/gitian/gitian-win.yml | 136 --- contrib/guix/INSTALL.md | 798 ++++++++++++++++++ contrib/guix/README.md | 345 ++++++++ contrib/guix/guix-attest | 263 ++++++ contrib/guix/guix-build | 466 ++++++++++ contrib/guix/guix-clean | 83 ++ contrib/guix/guix-verify | 174 ++++ contrib/guix/libexec/build.sh | 378 +++++++++ contrib/guix/libexec/prelude.bash | 84 ++ contrib/guix/manifest.scm | 271 ++++++ .../patches/gcc-fix-win32-vsnprintf.patch | 83 ++ .../guix/patches/gcc-remap-guix-store.patch | 25 + .../guix/patches/glibc-2.27-guix-prefix.patch | 22 + .../guix/patches/glibc-2.27-no-librt.patch | 53 ++ ...as_include-to-include-asm-syscalls.h.patch | 76 ++ contrib/shell/git-utils.bash | 14 + contrib/shell/realpath.bash | 71 ++ docs/RELEASE_CHECKLIST.md | 2 - 47 files changed, 3470 insertions(+), 1563 deletions(-) delete mode 100644 .github/workflows/gitian.yml create mode 100644 .github/workflows/guix.yml delete mode 100644 contrib/depends/packages/native_clang.mk create mode 100644 contrib/depends/patches/android_ndk/fix_env.patch create mode 100644 contrib/depends/patches/boost/fix_coalesce.patch delete mode 100644 contrib/gitian/DOCKRUN.md delete mode 100644 contrib/gitian/README.md delete mode 100755 contrib/gitian/dockrun.sh delete mode 100644 contrib/gitian/gitian-android.yml delete mode 100755 contrib/gitian/gitian-build.py delete mode 100644 contrib/gitian/gitian-freebsd.yml delete mode 100644 contrib/gitian/gitian-linux.yml delete mode 100644 contrib/gitian/gitian-osx.yml delete mode 100644 contrib/gitian/gitian-win.yml create mode 100644 contrib/guix/INSTALL.md create mode 100644 contrib/guix/README.md create mode 100755 contrib/guix/guix-attest create mode 100755 contrib/guix/guix-build create mode 100755 contrib/guix/guix-clean create mode 100755 contrib/guix/guix-verify create mode 100644 contrib/guix/libexec/build.sh create mode 100644 contrib/guix/libexec/prelude.bash create mode 100644 contrib/guix/manifest.scm create mode 100644 contrib/guix/patches/gcc-fix-win32-vsnprintf.patch create mode 100644 contrib/guix/patches/gcc-remap-guix-store.patch create mode 100644 contrib/guix/patches/glibc-2.27-guix-prefix.patch create mode 100644 contrib/guix/patches/glibc-2.27-no-librt.patch create mode 100644 contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch create mode 100644 contrib/shell/git-utils.bash create mode 100644 contrib/shell/realpath.bash diff --git a/.github/workflows/gitian.yml b/.github/workflows/gitian.yml deleted file mode 100644 index 91e60a88f5..0000000000 --- a/.github/workflows/gitian.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: ci/gh-actions/gitian - -on: - push: - tags: - - '*' - -jobs: - build-gitian: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - operating-system: - - name: "Linux" - option: "l" - - name: "Windows" - option: "w" - - name: "Android" - option: "a" - - name: "FreeBSD" - option: "f" - - name: "macOS" - option: "m" - name: ${{ matrix.operating-system.name }} - steps: - - name: prepare - run: | - sudo apt update - curl -O https://raw.githubusercontent.com/monero-project/monero/${{ github.ref_name }}/contrib/gitian/gitian-build.py - chmod +x gitian-build.py - - name: setup - run: | - ./gitian-build.py --setup --docker github-actions ${{ github.ref_name }} - - name: build - run: | - ./gitian-build.py --docker --detach-sign --no-commit --build -j 3 -o ${{ matrix.operating-system.option }} github-actions ${{ github.ref_name }} - - name: post build - run: | - cd out/${{ github.ref_name }} - shasum -a256 * - echo \`\`\` >> $GITHUB_STEP_SUMMARY - shasum -a256 * >> $GITHUB_STEP_SUMMARY - echo \`\`\` >> $GITHUB_STEP_SUMMARY - - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.operating-system.name }} - path: | - out/${{ github.ref_name }}/* diff --git a/.github/workflows/guix.yml b/.github/workflows/guix.yml new file mode 100644 index 0000000000..5898bb3c06 --- /dev/null +++ b/.github/workflows/guix.yml @@ -0,0 +1,86 @@ +name: ci/gh-actions/guix + +on: + push: + paths: + - 'contrib/depends/**' + - 'contrib/guix/**' + - '!contrib/**.md' + pull_request: + paths: + - 'contrib/depends/**' + - 'contrib/guix/**' + - '!contrib/**.md' + +jobs: + cache-sources: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: depends sources cache + id: cache + uses: actions/cache@v4 + with: + path: contrib/depends/sources + key: sources-${{ hashFiles('contrib/depends/packages/*') }} + - name: download depends sources + if: steps.cache.outputs.cache-hit != 'true' + run: make -C contrib/depends download + + build-guix: + runs-on: ubuntu-latest + needs: [cache-sources] + strategy: + fail-fast: false + matrix: + toolchain: + - target: "x86_64-linux-gnu" + - target: "aarch64-linux-gnu" + - target: "arm-linux-gnueabihf" + - target: "riscv64-linux-gnu" + - target: "i686-linux-gnu" + - target: "x86_64-w64-mingw32" + - target: "i686-w64-mingw32" + - target: "x86_64-unknown-freebsd" + - target: "x86_64-apple-darwin" + - target: "aarch64-apple-darwin" + - target: "aarch64-linux-android" + - target: "arm-linux-androideabi" + + name: ${{ matrix.toolchain.target }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + - name: remove bundled packages + run: sudo rm -rf /usr/local + - name: depends sources cache + uses: actions/cache/restore@v4 + with: + path: contrib/depends/sources + key: sources-${{ hashFiles('contrib/depends/packages/*') }} + - name: install dependencies + run: sudo apt update; sudo apt -y install guix git ca-certificates + - name: build + run: SUBSTITUTE_URLS='http://ci.guix.gnu.org' HOSTS="${{ matrix.toolchain.target }}" ./contrib/guix/guix-build + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.toolchain.target }} + path: | + guix/guix-build-*/output/${{ matrix.toolchain.target }}/* + guix/guix-build-*/logs/${{ matrix.toolchain.target }}/* + + bundle-logs: + runs-on: ubuntu-latest + needs: [build-guix] + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + - uses: actions/upload-artifact@v4 + with: + name: "logs" + path: '**/logs/**' diff --git a/.gitignore b/.gitignore index 9f62575e5a..7da4837ccd 100644 --- a/.gitignore +++ b/.gitignore @@ -25,10 +25,8 @@ miniupnpcstrings.h version/ ClangBuildAnalyzerSession.txt -# gitian -contrib/gitian/builder/ -contrib/gitian/docker/ -contrib/gitian/sigs/ +# guix +/guix # Created by https://www.gitignore.io diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ffdaeb55e..c96d3ab048 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1113,7 +1113,7 @@ if(MINGW) elseif(APPLE OR OPENBSD OR ANDROID) set(EXTRA_LIBRARIES "") elseif(FREEBSD) - set(EXTRA_LIBRARIES execinfo) + set(EXTRA_LIBRARIES execinfo elf) elseif(DRAGONFLY) find_library(COMPAT compat) set(EXTRA_LIBRARIES execinfo ${COMPAT}) diff --git a/README.md b/README.md index f0c0fe7fde..c432da77de 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. - [Release staging schedule and protocol](#release-staging-schedule-and-protocol) - [Compiling Monero from source](#compiling-monero-from-source) - [Dependencies](#dependencies) - - [Gitian builds](#gitian-builds) + - [Guix builds](#guix-builds) - [Internationalization](#Internationalization) - [Using Tor](#using-tor) - [Pruning](#Pruning) @@ -599,9 +599,9 @@ USE_DEVICE_TREZOR=OFF make release For more information, please check out Trezor [src/device_trezor/README.md](src/device_trezor/README.md). -### Gitian builds +### Guix builds -See [contrib/gitian/README.md](contrib/gitian/README.md). +See [contrib/guix/README.md](contrib/guix/README.md). ## Installing Monero from a package diff --git a/contrib/depends/Makefile b/contrib/depends/Makefile index f612dbffbe..a86281a554 100644 --- a/contrib/depends/Makefile +++ b/contrib/depends/Makefile @@ -85,6 +85,7 @@ include builders/$(build_os).mk include builders/default.mk include packages/packages.mk +ifeq ($(GUIX_ENVIRONMENT),) build_id_string:=$(BUILD_ID_SALT) build_id_string+=$(shell $(build_CC) --version 2>/dev/null) build_id_string+=$(shell $(build_AR) --version 2>/dev/null) @@ -98,6 +99,10 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_AR) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) +else +build_id_string:=$(realpath $(GUIX_ENVIRONMENT)) +$(host_arch)_$(host_os)_id_string:=$(realpath $(GUIX_ENVIRONMENT)) +endif packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) @@ -111,7 +116,7 @@ $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) include funcs.mk toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) -final_build_id_long+=$(shell $(build_SHA256SUM) toolchain.cmake.in) +final_build_id_long+=:[sha256sum]:$(shell $(build_SHA256SUM) toolchain.cmake.in) final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)rm -rf $(@D) @@ -124,8 +129,8 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(host_prefix)/share/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_$(final_build_id) $(AT)@mkdir -p $(@D) $(AT)sed -e 's|@HOST@|$(host)|' \ - -e 's|@CC@|$(toolchain_path)$(host_CC)|' \ - -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \ + -e 's|@CC@|$(host_CC)|' \ + -e 's|@CXX@|$(host_CXX)|' \ -e 's|@AR@|$(toolchain_path)$(host_AR)|' \ -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \ -e 's|@NM@|$(toolchain_path)$(host_NM)|' \ diff --git a/contrib/depends/funcs.mk b/contrib/depends/funcs.mk index e08669c981..be928c192b 100644 --- a/contrib/depends/funcs.mk +++ b/contrib/depends/funcs.mk @@ -1,18 +1,25 @@ define int_vars #Set defaults for vars which may be overridden per-package -$(1)_cc=$($($(1)_type)_CC) -$(1)_cxx=$($($(1)_type)_CXX) -$(1)_objc=$($($(1)_type)_OBJC) -$(1)_objcxx=$($($(1)_type)_OBJCXX) -$(1)_ar=$($($(1)_type)_AR) -$(1)_ranlib=$($($(1)_type)_RANLIB) -$(1)_libtool=$($($(1)_type)_LIBTOOL) -$(1)_nm=$($($(1)_type)_NM) -$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS) -$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS) -$(1)_arflags=$($($(1)_type)_ARFLAGS) $($($(1)_type)_$(release_type)_ARFLAGS) -$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib -$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include +$(1)_cc=$$($$($(1)_type)_CC) +$(1)_cxx=$$($$($(1)_type)_CXX) +$(1)_objc=$$($$($(1)_type)_OBJC) +$(1)_objcxx=$$($$($(1)_type)_OBJCXX) +$(1)_ar=$$($$($(1)_type)_AR) +$(1)_ranlib=$$($$($(1)_type)_RANLIB) +$(1)_libtool=$$($$($(1)_type)_LIBTOOL) +$(1)_nm=$$($$($(1)_type)_NM) +$(1)_cflags=$$($$($(1)_type)_CFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CFLAGS) +$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CXXFLAGS) +$(1)_arflags=$$($$($(1)_type)_ARFLAGS) \ + $$($$($(1)_type)_$(release_type)_ARFLAGS) +$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \ + $$($$($(1)_type)_$$(release_type)_LDFLAGS) \ + -L$$($($(1)_type)_prefix)/lib +$(1)_cppflags=$$($$($(1)_type)_CPPFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CPPFLAGS) \ + -I$$($$($(1)_type)_prefix)/include $(1)_recipe_hash:= endef @@ -37,6 +44,7 @@ endef define int_get_build_recipe_hash $(eval $(1)_all_file_checksums:=$(shell $(build_SHA256SUM) $(meta_depends) packages/$(1).mk $(addprefix $(PATCHES_PATH)/$(1)/,$($(1)_patches)) | cut -d" " -f1)) +final_build_id_long+=:[$(1)_all_file_checksums]$(foreach checksum,$($(1)_all_file_checksums),$(shell echo ":$(checksum)")): $(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | $(build_SHA256SUM) | cut -d" " -f1)) endef @@ -46,7 +54,7 @@ $(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash))) $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string)) $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))) -final_build_id_long+=$($(package)_build_id_long) +final_build_id_long+=:[recipe]:$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type):[deps]$(foreach dep,$($(1)_build_id_deps),$(shell echo ":$(dep)")):[$($(1)_type)_id]:$($($(1)_type)_id_string): #compute package-specific paths $(1)_build_subdir?=. @@ -267,4 +275,4 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$ $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package)))) #special exception: if a toolchain package exists, all non-native packages depend on it -$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) )) +$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) )) diff --git a/contrib/depends/hosts/darwin.mk b/contrib/depends/hosts/darwin.mk index 9a96e26019..58ff5dc32c 100644 --- a/contrib/depends/hosts/darwin.mk +++ b/contrib/depends/hosts/darwin.mk @@ -1,23 +1,65 @@ OSX_MIN_VERSION=10.13 +OSX_SDK_VERSION=11.0 +XCODE_VERSION=12.2 +XCODE_BUILD_ID=12B45b LD64_VERSION=609 -ifeq (aarch64, $(host_arch)) -CC_target=arm64-apple-$(host_os) -else -CC_target=$(host) -endif -darwin_CC=clang -target $(CC_target) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(host_prefix)/native/SDK/ -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks -mlinker-version=$(LD64_VERSION) -B$(host_prefix)/native/bin/$(host)- -darwin_CXX=clang++ -target $(CC_target) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(host_prefix)/native/SDK/ -iwithsysroot/usr/include/c++/v1 -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks -mlinker-version=$(LD64_VERSION) -stdlib=libc++ -B$(host_prefix)/native/bin/$(host)- + +OSX_SDK=$(host_prefix)/native/SDK + +darwin_native_toolchain=darwin_sdk native_cctools + +clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") +clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") + +# Flag explanations: +# +# -mlinker-version +# +# Ensures that modern linker features are enabled. See here for more +# details: https://github.com/bitcoin/bitcoin/pull/19407. +# +# -B$(build_prefix)/bin +# +# Explicitly point to our binaries (e.g. cctools) so that they are +# ensured to be found and preferred over other possibilities. +# +# -isysroot$(OSX_SDK) -nostdlibinc +# +# Disable default include paths built into the compiler as well as +# those normally included for libc and libc++. The only path that +# remains implicitly is the clang resource dir. +# +# -iwithsysroot / -iframeworkwithsysroot +# +# Adds the desired paths from the SDK +# + +darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH \ + $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ + -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + -isysroot$(OSX_SDK) \ + -isysroot$(OSX_SDK) -nostdlibinc \ + -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks + +darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH \ + $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ + -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + -isysroot$(OSX_SDK) -nostdlibinc \ + -iwithsysroot/usr/include/c++/v1 \ + -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) darwin_ARFLAGS=cr -darwin_release_CFLAGS=-O1 +darwin_release_CFLAGS=-O2 darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) -darwin_native_toolchain=native_cctools darwin_sdk - darwin_cmake_system=Darwin diff --git a/contrib/depends/hosts/freebsd.mk b/contrib/depends/hosts/freebsd.mk index d3c6c6149b..aeaccbf273 100644 --- a/contrib/depends/hosts/freebsd.mk +++ b/contrib/depends/hosts/freebsd.mk @@ -1,5 +1,14 @@ -freebsd_CC=clang-8 -freebsd_CXX=clang++-8 +clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") +clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") + +freebsd_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH $(clang_prog) --target=$(host) --sysroot=$(host_prefix)/native -iwithsysroot/usr/include +freebsd_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH $(clangxx_prog) --target=$(host) -stdlib=libc++ --sysroot=$(host_prefix)/native \ + -iwithsysroot/usr/include/c++/v1 -iwithsysroot/usr/include + freebsd_AR=ar freebsd_RANLIB=ranlib freebsd_NM=nm diff --git a/contrib/depends/hosts/linux.mk b/contrib/depends/hosts/linux.mk index 0f9379f4ae..ba0c4c3027 100644 --- a/contrib/depends/hosts/linux.mk +++ b/contrib/depends/hosts/linux.mk @@ -10,6 +10,7 @@ linux_debug_CXXFLAGS=$(linux_debug_CFLAGS) linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC +ifeq ($(GUIX_ENVIRONMENT),) ifeq (86,$(findstring 86,$(build_arch))) i686_linux_CC=gcc -m32 i686_linux_CXX=g++ -m32 @@ -30,5 +31,6 @@ i686_linux_CXX=$(default_host_CXX) -m32 x86_64_linux_CC=$(default_host_CC) -m64 x86_64_linux_CXX=$(default_host_CXX) -m64 endif +endif linux_cmake_system=Linux diff --git a/contrib/depends/packages/android_ndk.mk b/contrib/depends/packages/android_ndk.mk index 65fa89de6c..e32e47739b 100644 --- a/contrib/depends/packages/android_ndk.mk +++ b/contrib/depends/packages/android_ndk.mk @@ -3,7 +3,7 @@ $(package)_version=18b $(package)_download_path=https://dl.google.com/android/repository/ $(package)_file_name=android-ndk-r$($(package)_version)-linux-x86_64.zip $(package)_sha256_hash=4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd -$(package)_patches=api_definition.patch +$(package)_patches=api_definition.patch fix_env.patch define $(package)_set_vars $(package)_config_opts_arm=--arch arm @@ -17,11 +17,12 @@ endef define $(package)_preprocess_cmds cd android-ndk-r$($(package)_version) && \ - patch -p1 < $($(package)_patch_dir)/api_definition.patch + patch -p1 < $($(package)_patch_dir)/api_definition.patch && \ + patch -p1 < $($(package)_patch_dir)/fix_env.patch endef define $(package)_stage_cmds - android-ndk-r$($(package)_version)/build/tools/make_standalone_toolchain.py --api 21 \ + python3 android-ndk-r$($(package)_version)/build/tools/make_standalone_toolchain.py --api 21 \ --install-dir $(build_prefix) --stl=libc++ $($(package)_config_opts) &&\ mv $(build_prefix) $($(package)_staging_dir)/$(host_prefix) endef diff --git a/contrib/depends/packages/boost.mk b/contrib/depends/packages/boost.mk index d81bca973a..1598295392 100644 --- a/contrib/depends/packages/boost.mk +++ b/contrib/depends/packages/boost.mk @@ -1,9 +1,9 @@ -package=boost +package=boost $(package)_version=1_64_0 $(package)_download_path=https://downloads.sourceforge.net/project/boost/boost/1.64.0/ $(package)_file_name=$(package)_$($(package)_version).tar.bz2 $(package)_sha256_hash=7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 -$(package)_patches=fix_aroptions.patch fix_arm_arch.patch +$(package)_patches=fix_aroptions.patch fix_arm_arch.patch fix_coalesce.patch define $(package)_set_vars $(package)_config_opts_release=variant=release @@ -31,6 +31,7 @@ endef define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/fix_aroptions.patch &&\ patch -p1 < $($(package)_patch_dir)/fix_arm_arch.patch &&\ + patch -p1 < $($(package)_patch_dir)/fix_coalesce.patch &&\ echo "using $(boost_toolset_$(host_os)) : : $($(package)_cxx) : \"$($(package)_cxxflags) $($(package)_cppflags)\" \"$($(package)_ldflags)\" \"$(boost_archiver_$(host_os))\" \"$($(package)_arflags)\" \"$(host_STRIP)\" \"$(host_RANLIB)\" \"$(host_WINDRES)\" : ;" > user-config.jam endef diff --git a/contrib/depends/packages/darwin_sdk.mk b/contrib/depends/packages/darwin_sdk.mk index 3355dcf3a6..ac5597a25e 100644 --- a/contrib/depends/packages/darwin_sdk.mk +++ b/contrib/depends/packages/darwin_sdk.mk @@ -6,5 +6,6 @@ $(package)_sha256_hash=df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717 define $(package)_stage_cmds mkdir -p $($(package)_staging_dir)/$(host_prefix)/native/SDK &&\ + rm -rf usr/include/readline && \ mv * $($(package)_staging_dir)/$(host_prefix)/native/SDK endef diff --git a/contrib/depends/packages/freebsd_base.mk b/contrib/depends/packages/freebsd_base.mk index ad9975f8d5..4484a4afe4 100644 --- a/contrib/depends/packages/freebsd_base.mk +++ b/contrib/depends/packages/freebsd_base.mk @@ -10,14 +10,8 @@ define $(package)_extract_cmds tar xf $($(1)_source_dir)/$($(package)_file_name) ./lib/ ./usr/lib/ ./usr/include/ endef -define $(package)_build_cmds - mkdir bin &&\ - echo "#!/bin/sh\n\nexec /usr/bin/clang-8 -target x86_64-unknown-freebsd$($(package)_version) --sysroot=$(host_prefix)/native $$$$""@" > bin/clang-8 &&\ - echo "#!/bin/sh\n\nexec /usr/bin/clang++-8 -target x86_64-unknown-freebsd$($(package)_version) --sysroot=$(host_prefix)/native $$$$""@" > bin/clang++-8 &&\ - chmod 755 bin/* -endef - define $(package)_stage_cmds mkdir $($(package)_staging_dir)/$(host_prefix)/native &&\ - mv bin lib usr $($(package)_staging_dir)/$(host_prefix)/native + rm -rf usr/include/openssl &&\ + mv lib usr $($(package)_staging_dir)/$(host_prefix)/native endef diff --git a/contrib/depends/packages/native_cctools.mk b/contrib/depends/packages/native_cctools.mk index 8c1ea4c62a..a019b1649e 100644 --- a/contrib/depends/packages/native_cctools.mk +++ b/contrib/depends/packages/native_cctools.mk @@ -5,14 +5,18 @@ $(package)_download_file=$($(package)_version).tar.gz $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=70a7189418c2086d20c299c5d59250cf5940782c778892ccc899c66516ed240e $(package)_build_subdir=cctools -$(package)_dependencies=native_clang native_libtapi $(package)_patches=no-build-date.patch +$(package)_dependencies=native_libtapi define $(package)_set_vars $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$(host_prefix) $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib -$(package)_cc=$(host_prefix)/native/bin/clang -$(package)_cxx=$(host_prefix)/native/bin/clang++ +$(package)_cc=$(clang_prog) +$(package)_cxx=$(clangxx_prog) +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools endef define $(package)_preprocess_cmds diff --git a/contrib/depends/packages/native_clang.mk b/contrib/depends/packages/native_clang.mk deleted file mode 100644 index 4ff21ada0d..0000000000 --- a/contrib/depends/packages/native_clang.mk +++ /dev/null @@ -1,28 +0,0 @@ -package=native_clang -$(package)_version=9.0.0 -$(package)_download_path=https://releases.llvm.org/$($(package)_version) -$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-18.04.tar.xz -$(package)_file_name=clang-llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-18.04.tar.xz -$(package)_sha256_hash=a23b082b30c128c9831dbdd96edad26b43f56624d0ad0ea9edec506f5385038d - -define $(package)_extract_cmds - echo $($(package)_sha256_hash) $($(package)_source) | sha256sum -c &&\ - mkdir -p toolchain/bin toolchain/lib/clang/3.5/include && \ - tar --strip-components=1 -C toolchain -xf $($(package)_source) && \ - rm -f toolchain/lib/libc++abi.so* && \ - echo "#!/bin/sh" > toolchain/bin/$(host)-dsymutil && \ - echo "exit 0" >> toolchain/bin/$(host)-dsymutil && \ - chmod +x toolchain/bin/$(host)-dsymutil -endef - -define $(package)_stage_cmds - cd $($(package)_extract_dir)/toolchain && \ - mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \ - mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \ - cp bin/clang $($(package)_staging_prefix_dir)/bin/ &&\ - cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\ - cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ - cp -rf lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/ && \ - cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ - if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi -endef diff --git a/contrib/depends/packages/native_libtapi.mk b/contrib/depends/packages/native_libtapi.mk index c5625501a2..36234a001c 100644 --- a/contrib/depends/packages/native_libtapi.mk +++ b/contrib/depends/packages/native_libtapi.mk @@ -4,30 +4,16 @@ $(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive $(package)_download_file=$($(package)_version).tar.gz $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=62e419c12d1c9fad67cc1cd523132bc00db050998337c734c15bc8d73cc02b61 -$(package)_build_subdir=build -$(package)_dependencies=native_clang $(package)_patches=no_embed_git_rev.patch define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/no_embed_git_rev.patch endef -define $(package)_config_cmds - echo -n $(build_prefix) > INSTALLPREFIX; \ - CC=$(host_prefix)/native/bin/clang CXX=$(host_prefix)/native/bin/clang++ \ - cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DCMAKE_BUILD_TYPE=RELEASE \ - -DTAPI_REPOSITORY_STRING="1100.0.11" \ - -DTAPI_FULL_VERSION="11.0.0" \ - -DCMAKE_CXX_FLAGS="-I $($(package)_extract_dir)/src/llvm/projects/clang/include -I $($(package)_build_dir)/projects/clang/include" \ - $($(package)_extract_dir)/src/llvm -endef - define $(package)_build_cmds - $(MAKE) clangBasic && $(MAKE) libtapi + CC=$(clang_prog) CXX=$(clangxx_prog) INSTALLPREFIX=$($(package)_staging_prefix_dir) ./build.sh endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install-libtapi install-tapi-headers + ./install.sh endef diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 14561964f5..cb4e8e228a 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -24,5 +24,5 @@ mingw32_packages = $(hardware_packages) mingw32_native_packages = $(hardware_native_packages) ifneq ($(build_os),darwin) -darwin_native_packages += darwin_sdk native_clang native_cctools native_libtapi +darwin_native_packages += darwin_sdk native_cctools native_libtapi endif diff --git a/contrib/depends/patches/android_ndk/fix_env.patch b/contrib/depends/patches/android_ndk/fix_env.patch new file mode 100644 index 0000000000..01928dd485 --- /dev/null +++ b/contrib/depends/patches/android_ndk/fix_env.patch @@ -0,0 +1,30 @@ +diff --git a/build/tools/make_standalone_toolchain.py b/build/tools/make_standalone_toolchain.py +index b8172b2..19c0ad6 100755 +--- a/build/tools/make_standalone_toolchain.py ++++ b/build/tools/make_standalone_toolchain.py +@@ -224,10 +224,10 @@ def make_clang_scripts(install_dir, triple, api, windows): + clang.write(textwrap.dedent("""\ + #!/bin/bash + if [ "$1" != "-cc1" ]; then +- `dirname $0`/clang{version} {flags} "$@" ++ env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH -u LIBRARY_PATH `dirname $0`/clang{version} {flags} "$@" + else + # target/triple already spelled out. +- `dirname $0`/clang{version} "$@" ++ env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH -u LIBRARY_PATH `dirname $0`/clang{version} "$@" + fi + """.format(version=version_number, flags=unix_flags))) + +@@ -239,10 +239,10 @@ def make_clang_scripts(install_dir, triple, api, windows): + clangpp.write(textwrap.dedent("""\ + #!/bin/bash + if [ "$1" != "-cc1" ]; then +- `dirname $0`/clang{version}++ {flags} "$@" ++ env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH -u LIBRARY_PATH `dirname $0`/clang{version}++ {flags} "$@" + else + # target/triple already spelled out. +- `dirname $0`/clang{version}++ "$@" ++ env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH -u LIBRARY_PATH `dirname $0`/clang{version}++ "$@" + fi + """.format(version=version_number, flags=unix_flags))) + diff --git a/contrib/depends/patches/boost/fix_coalesce.patch b/contrib/depends/patches/boost/fix_coalesce.patch new file mode 100644 index 0000000000..8ee29283e2 --- /dev/null +++ b/contrib/depends/patches/boost/fix_coalesce.patch @@ -0,0 +1,17 @@ +--- boost_1_64_0/tools/build/src/tools/darwin.jam ++++ boost_1_64_0/tools/build/src/tools/darwin.jam +@@ -138,10 +138,10 @@ rule init ( version ? : command * : options * : requirement * ) + common.handle-options darwin : $(condition) : $(command) : $(options) ; + + # - GCC 4.0 and higher in Darwin does not have -fcoalesce-templates. +- if $(real-version) < "4.0.0" +- { +- flags darwin.compile.c++ OPTIONS $(condition) : -fcoalesce-templates ; +- } ++ #if $(real-version) < "4.0.0" ++ #{ ++ # flags darwin.compile.c++ OPTIONS $(condition) : -fcoalesce-templates ; ++ #} + # - GCC 4.2 and higher in Darwin does not have -Wno-long-double. + if $(real-version) < "4.2.0" + { diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index e44882f4c9..29d90dfbb0 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -78,12 +78,14 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") SET(CMAKE_OSX_ARCHITECTURES "x86_64") endif() SET(_CMAKE_TOOLCHAIN_PREFIX @prefix@/native/bin/${CONF_TRIPLE}-) - SET(CMAKE_C_COMPILER @prefix@/native/bin/clang) + SET(CMAKE_C_COMPILER @CC@) SET(CMAKE_C_COMPILER_TARGET ${CLANG_TARGET}) SET(CMAKE_C_FLAGS_INIT -B${_CMAKE_TOOLCHAIN_PREFIX}) - SET(CMAKE_CXX_COMPILER @prefix@/native/bin/clang++ -stdlib=libc++) + SET(CMAKE_CXX_COMPILER @CXX@ -stdlib=libc++) SET(CMAKE_CXX_COMPILER_TARGET ${CLANG_TARGET}) SET(CMAKE_CXX_FLAGS_INIT -B${_CMAKE_TOOLCHAIN_PREFIX}) + SET(CMAKE_ASM_COMPILER clang) + SET(CMAKE_ASM-ATT_COMPILER as) SET(CMAKE_ASM_COMPILER_TARGET ${CLANG_TARGET}) SET(CMAKE_ASM-ATT_COMPILER_TARGET ${CLANG_TARGET}) SET(APPLE True) @@ -112,6 +114,11 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Android") SET(CMAKE_C_COMPILER "${_CMAKE_TOOLCHAIN_PREFIX}clang") SET(CMAKE_CXX_COMPILER "${_CMAKE_TOOLCHAIN_PREFIX}clang++") else() + if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + SET(CMAKE_ASM_COMPILER clang) + SET(CMAKE_ASM-ATT_COMPILER as) + endif() + SET(CMAKE_C_COMPILER @CC@) SET(CMAKE_CXX_COMPILER @CXX@) endif() diff --git a/contrib/gitian/DOCKRUN.md b/contrib/gitian/DOCKRUN.md deleted file mode 100644 index 06e4894a24..0000000000 --- a/contrib/gitian/DOCKRUN.md +++ /dev/null @@ -1,114 +0,0 @@ -Quick Gitian building with docker -================================= - -*Setup instructions for a Gitian build of Monero using Docker.* - -Gitian supports other container mechanisms too but if you have a Debian or -Ubuntu-based host the steps can be greatly simplified. - -Preparing the Gitian builder host ---------------------------------- - -The procedure here will create a docker container for build preparation, as well as -for actually running the builds. The only items you must install on your own host -are docker and apt-cacher-ng. With docker installed, you should also give yourself -permission to use docker by adding yourself to the docker group. - -```bash -sudo apt-get install docker.io apt-cacher-ng -sudo usermod -aG docker $USER -su $USER -``` - -The final `su` command is needed to start a new shell with your new group membership, -since the `usermod` command doesn't affect any existing sessions. - -You'll also need to clone the monero repository and navigate to the `contrib/gitian` directory: - -```bash -git clone https://github.com/monero-project/monero.git -cd monero/contrib/gitian -``` - -Other User Preparation ----------------------- - -The final step will be to `gpg` sign the results of your build and upload them to GitHub. -Before you can do that, you'll need -* a GitHub account. -If your GitHub account name is different from your local account name, you must -set your GitHub account name for the script to use: - -```bash -export GH_USER= -``` - -* PGP keys - if you don't have one already, you can use `gpg --quick-gen-key` to generate it. -* a fork of the [gitian.sigs](https://github.com/monero-project/gitian.sigs/) repo on your GitHub account. -Please follow the directions there for uploading your key first. - -**Note:** Please ensure your gpg public key is available to check signatures by adding it to the [gitian.sigs/gitian-pubkeys/](https://github.com/monero-project/gitian.sigs/tree/master/gitian-pubkeys) directory in a pull request. - - -Building the Binaries ---------------------- - -The dockrun.sh script will do everything to build the binaries. Just specify the -version to build as its only argument, e.g. - -```bash -VERSION=v0.18.1.0 -./dockrun.sh $VERSION -``` - -The build should run to completion with no errors, and will display the SHA256 checksums -of the resulting binaries. You'll be prompted to check if the sums look good, and if so -then the results will be signed, and the signatures will be pushed to GitHub. - -***Note: In order to publish the signed assertions via this script, you need to have your SSH key uploaded to GitHub beforehand. See https://docs.github.com/articles/generating-an-ssh-key/ for more info.*** - -You can also look in the [gitian.sigs](https://github.com/monero-project/gitian.sigs/) repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / GitHub to figure out what is going on. - - -Other Options -------------- - -This script just runs the [gitian-build.py](gitian-build.py) inside a container named `gitrun`. -You can set other options for that script by setting the OPT variable when running `dockrun.sh` -e.g. - -```bash -# Run build processes with 8 threads -OPT="-j 8" ./dockrun.sh $VERSION -``` - -Post-build ----------- - -You can examine the build and install logs by running a shell in the container, e.g. - -```bash -# Tail running logs -docker exec -it gitrun /bin/bash -tail -F builder/var/install.log -tail -F builder/var/build.log - -# Inspect logs, in format install-.log and build-.log -docker exec -it gitrun /bin/bash -more builder/var/install-linux.log -more builder/var/build-linux.log -``` - -You can find the compiled archives inside of the container at the following directory: - -```bash -docker exec -it gitrun /bin/bash -ls -la out/$VERSION/ -``` - -To copy the compiled archives to the local host out of the Docker container, you can run the following: - -```bash -mkdir out -docker cp gitrun:/home/ubuntu/out/$VERSION out -``` diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md deleted file mode 100644 index 5211b84094..0000000000 --- a/contrib/gitian/README.md +++ /dev/null @@ -1,272 +0,0 @@ -Gitian building -================ - -*Setup instructions for a Gitian build of Monero.* - -Gitian is the deterministic build process that is used to build the Monero CLI -executables. It provides a way to be reasonably sure that the -executables are really built from the git source. It also makes sure that -the same, tested dependencies are used and statically built into the executable. - -Multiple developers build the source code by following a specific descriptor -("recipe"), cryptographically sign the result, and upload the resulting signature. -These results are compared and only if they match, the build is accepted and provided -for download. - -Gitian runs compilation steps in an isolated container. It is flexible and gives you full -control over the build environment, while still ensuring reproducibility and consistent output -formats. - -More independent Gitian builders are needed, which is why this guide exists. -It is preferred you follow these steps yourself instead of using someone else's -VM image to avoid 'contaminating' the build. - -Preparing the Gitian builder host ---------------------------------- - -The first step is to prepare the host environment that will be used to perform the Gitian builds. -This guide explains how to set up the environment, and how to start the builds. - -* Gitian host OS should be Ubuntu 18.04 "Bionic Beaver". If you are on a mac or windows for example, you can run it in a VM but will be slower. - -* Gitian gives you the option of using any of 3 different virtualization tools: `kvm`, `docker` or `lxc`. This documentation will only show how to build with `lxc` and `docker` (documentation for `kvm` is welcome). Building with `lxc` is the default, but is more complicated, so we recommend docker your first time. - -* For a shortcut using `docker` follow the instructions in [DOCKRUN.md](DOCKRUN.md) instead -of following the rest of this document.. - -## Create the gitianuser account - -You need to create a new user called `gitianuser` and be logged in as that user. The user needs `sudo` access. - -```bash -sudo adduser gitianuser -sudo usermod -aG sudo gitianuser -``` - -LXC ---- - -LXC builds should be run on Ubuntu 18.04 "Bionic Beaver". - -Note that a version of `lxc-execute` higher or equal to 2.1.1 is required. -You can check the version with `lxc-execute --version`. - -First we need to set up dependencies. Type/paste the following in the terminal: - -```bash -sudo apt-get install git ruby apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils make ubuntu-archive-keyring curl firewalld -``` - -Then set up LXC and the rest with the following, which is a complex jumble of settings and workarounds: - -```bash -sudo -s -# the version of lxc-start in Debian needs to run as root, so make sure -# that the build script can execute it without providing a password -echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-start" > /etc/sudoers.d/gitian-lxc -echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-execute" >> /etc/sudoers.d/gitian-lxc -# make /etc/rc.local script that sets up bridge between guest and host -echo '#!/bin/sh -e' > /etc/rc.local -echo 'brctl addbr br0' >> /etc/rc.local -echo 'ip addr add 10.0.2.2/24 broadcast 10.0.2.255 dev br0' >> /etc/rc.local -echo 'ip link set br0 up' >> /etc/rc.local -echo 'firewall-cmd --zone=trusted --add-interface=br0' >> /etc/rc.local -echo 'exit 0' >> /etc/rc.local -chmod +x /etc/rc.local -# make sure that USE_LXC is always set when logging in as gitianuser, -# and configure LXC IP addresses -echo 'export USE_LXC=1' >> /home/gitianuser/.profile -echo 'export GITIAN_HOST_IP=10.0.2.2' >> /home/gitianuser/.profile -echo 'export LXC_GUEST_IP=10.0.2.5' >> /home/gitianuser/.profile -reboot -``` - -This setup is required to enable networking in the container. - -Docker ------- - -Prepare for building with docker: - -```bash -sudo bash -c 'apt-get update && apt-get upgrade -y && apt-get install git curl docker.io' -``` - -Consider adding `gitianuser` to the `docker` group after reading about [the security implications](https://docs.docker.com/v17.09/engine/installation/linux/linux-postinstall/): - -```bash -sudo groupadd docker -sudo usermod -aG docker gitianuser -``` - -Optionally add yourself to the docker group. Note that this will give docker root access to your system. - -```bash -sudo usermod -aG docker $USER -``` - -Manual Building -------------------- - -======= -The script automatically installs some packages with apt. If you are not running it on a debian-like system, pass `--no-apt` along with the other -arguments to it. It calls all available .yml descriptors, which in turn pass the build configurations for different platforms to gitian. -The instructions below use the automated script [gitian-build.py](gitian-build.py) which is tested to work on Ubuntu. - -It calls all available .yml descriptors, which in turn pass the build configurations for different platforms to gitian. -Help for the build steps taken can be accessed with `./gitian-build.py --help`. - -Initial Gitian Setup --------------------- - -The `gitian-build.py` script will checkout different release tags, so it's best to copy it to the top level directory: - -```bash -cp monero/contrib/gitian/gitian-build.py . -``` - -### Setup the required environment - -Common setup part: - -```bash -su - gitianuser - -GH_USER=YOUR_GITHUB_USER_NAME -VERSION=v0.18.0.0 -``` - -Where `GH_USER` is your GitHub user name and `VERSION` is the version tag you want to build. -The `gitian-build.py`'s `--setup` switch will also refresh the environment of any stale files and submodules. - -Setup for LXC: - -```bash -./gitian-build.py --setup $GH_USER $VERSION -``` - -Setup for docker: - -```bash -./gitian-build.py --setup --docker $GH_USER $VERSION -``` - -While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a separate step. This script is only there for convenience. Separate steps for building can still be taken. -In order to sign gitian builds on your host machine, which has your PGP key, -fork the [gitian.sigs repository](https://github.com/monero-project/gitian.sigs) and clone it on your host machine, -or pass the signed assert file back to your build machine. - -```bash -git clone https://github.com/monero-project/gitian.sigs/ -pushd gitian.sigs -git remote add $GH_USER https://github.com/$GH_USER/gitian.sigs -popd -``` - -Build the binaries ------------------- - -To build the most recent tag (pass in `--docker` if using docker): - -```bash -./gitian-build.py --detach-sign --no-commit --build $GH_USER $VERSION -``` - -To speed up the build, use `-j 5 --memory 10000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 10000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. A good rule of thumb is, that Monero currently needs about 2 GB of RAM per core. - -A full example for `docker` would look like the following: - -```bash -./gitian-build.py -j 5 --memory 10000 --docker --detach-sign --no-commit --build $GH_USER $VERSION -``` - -If all went well, this produces a number of (uncommitted) `.assert` files in the gitian.sigs directory. - -Checking your work ------------------- - -Take a look in the assert files and note the SHA256 checksums listed there. - -You should verify that the checksum that is listed matches each of the binaries you actually built. -This may be done on Linux using the `sha256sum` command or on MacOS using `shasum --algorithm 256` for example. -An example script to verify the checksums would be: - -```bash -pushd out/${VERSION} - -for ASSERT in ../../sigs/${VERSION}-*/*/*.assert; do - if ! sha256sum --ignore-missing -c "${ASSERT}" ; then - echo "FAILED for ${ASSERT} ! Please inspect manually." - fi -done - -popd -``` - -Don't ignore the incorrect formatting of the found assert files. These files you'll have to compare manually (currently OSX and FreeBSD). - - -You can also look in the [gitian.sigs](https://github.com/monero-project/gitian.sigs/) repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / github to figure out what is going on. - - -Signing assert files --------------------- - -If you chose to do detached signing using `--detach-sign` above (recommended), you need to copy these uncommitted changes to your host machine, then sign them using your gpg key like so: - -```bash -for ASSERT in sigs/${VERSION}-*/*/*.assert; do gpg --detach-sign ${ASSERT}; done -``` - -This will create a `.sig` file for each `.assert` file above (2 files for each platform). - - -Submitting your signed assert files ------------------------------------ - -Make a pull request (both the `.assert` and `.assert.sig` files) to the -[monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: - -```bash -cd gitian.sigs -git checkout -b $VERSION -# add your assert and sig files... -git commit -S -a -m "Add $GH_USER $VERSION" -git push --set-upstream $GH_USER $VERSION -``` - -**Note:** Please ensure your gpg public key is available to check signatures by adding it to the [gitian.sigs/gitian-pubkeys/](https://github.com/monero-project/gitian.sigs/tree/master/gitian-pubkeys) directory in a pull request. - - -More Build Options ------------------- - -You can choose your own remote and commit hash by running for example: -```bash -./gitian-build.py --detach-sign --no-commit --url https://github.com/moneromooo-monero/bitmonero -b moneromooo 1f5680c8db8f4cc7acc04a04c724b832003440fd -``` - -Note that you won't be able to build commits authored before the gitian scripts -were added. Gitian clones the source files from the given url, be sure to push -to the remote you provide before building. -To get all build options run: -```bash -./gitian-build.py --help -``` - -Doing Successive Builds ------------------------ - -If you need to do multiple iterations (while developing/testing) you can use the -`--rebuild` option instead of `--build` on subsequent iterations. This skips the -initial check for the freshness of the depends tools. In particular, doing this -check all the time prevents rebuilding when you have no network access. - - -Local-Only Builds ------------------ - -If you need to run builds while disconnected from the internet, make sure you have -local up-to-date repos in advance. Then specify your local repo using the `--url` -option when building. This will avoid attempts to git pull across a network. - diff --git a/contrib/gitian/dockrun.sh b/contrib/gitian/dockrun.sh deleted file mode 100755 index 396db126b0..0000000000 --- a/contrib/gitian/dockrun.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/sh - -if [ $# -ne 1 ]; then - echo "usage: $0 " - exit 1 -fi -VERSION=$1 - -DOCKER=`command -v docker` -CACHER=`command -v apt-cacher-ng` - -if [ -z "$DOCKER" -o -z "$CACHER" ]; then - echo "$0: you must first install docker.io and apt-cacher-ng" - echo " e.g. sudo apt-get install docker.io apt-cacher-ng" - exit 1 -fi - -GH_USER=${GH_USER-$USER} - -TAG=gitrun-bionic -TAG2=base-bionic-amd64 -IMAGE=`docker images | grep $TAG` - -WORKDIR=/home/ubuntu - -if [ -z "$IMAGE" ]; then -GID=`getent group docker` -mkdir -p docker -cd docker - -# container for running gitian-build.py -cat < ${TAG}.Dockerfile -FROM ubuntu:bionic - -ENV DEBIAN_FRONTEND=noninteractive -RUN echo 'Acquire::http { Proxy "http://172.17.0.1:3142"; };' > /etc/apt/apt.conf.d/50cacher -RUN echo "$GID" >> /etc/group -RUN apt-get update && apt-get --no-install-recommends -y install lsb-release ruby git make wget docker.io python3 curl - -RUN useradd -ms /bin/bash -U ubuntu -G docker -USER ubuntu:docker -WORKDIR $WORKDIR - -RUN git clone https://github.com/monero-project/gitian.sigs.git sigs; \ - git clone https://github.com/devrandom/gitian-builder.git builder; \ - cd builder; git checkout c0f77ca018cb5332bfd595e0aff0468f77542c23; mkdir -p inputs var; cd inputs; \ - git clone https://github.com/monero-project/monero - -CMD ["sleep", "infinity"] -EOF - -docker build --pull -f ${TAG}.Dockerfile -t $TAG . - -cd .. -docker run -v /var/run/docker.sock:/var/run/docker.sock -d --name gitrun $TAG - -fi - -IMAGE=`docker images | grep $TAG2` -if [ -z "$IMAGE" ]; then -mkdir -p docker -cd docker - -# container for actually running each build -cat < ${TAG2}.Dockerfile -FROM ubuntu:bionic - -ENV DEBIAN_FRONTEND=noninteractive -RUN echo 'Acquire::http { Proxy "http://172.17.0.1:3142"; };' > /etc/apt/apt.conf.d/50cacher -RUN apt-get update && apt-get --no-install-recommends -y install build-essential git language-pack-en \ - wget lsb-release curl gcc-7 g++-7 gcc g++ binutils-gold pkg-config autoconf libtool automake faketime \ - bsdmainutils ca-certificates python cmake gperf - -RUN useradd -ms /bin/bash -U ubuntu -USER ubuntu:ubuntu -WORKDIR $WORKDIR - -CMD ["sleep", "infinity"] -EOF - -docker build --pull -f ${TAG2}.Dockerfile -t $TAG2 . - -cd .. - -fi - -RUNNING=`docker ps | grep gitrun` -if [ -z "$RUNNING" ]; then - BUILT=`docker ps -a | grep gitrun` - if [ -z "$BUILT" ]; then - docker run -v /var/run/docker.sock:/var/run/docker.sock -d --name gitrun $TAG - else - docker start gitrun - fi -fi -docker cp gitian-build.py gitrun:$WORKDIR/ -docker exec -t gitrun ./gitian-build.py -d -b -D -n $OPT $GH_USER $VERSION -RC=$? -if [ $RC != 0 ]; then - exit $RC -fi -echo "\nBuild Results:\n" -docker exec gitrun sh -c "sha256sum out/$VERSION/*" -echo "\nIf these results look correct, type \"sign\" to sign them, otherwise ^C to stop now." -read check -if [ "$check" != "sign" ]; then - echo "Not signing, bye." - exit 1 -fi - -if [ ! -d sigs ]; then - git clone https://github.com/monero-project/gitian.sigs.git sigs - cd sigs - git remote add $GH_USER git@github.com:$GH_USER/gitian.sigs.git - cd .. -fi - -DIRS=`docker exec gitrun sh -c "echo sigs/$VERSION-*"` -for i in $DIRS; do - docker cp gitrun:$WORKDIR/$i sigs - gpg --detach-sign $i/$GH_USER/*.assert -done - -cd sigs -git checkout -b $VERSION -git add $VERSION-* -git commit -S -m "Add $GH_USER $VERSION" -git push --set-upstream $GH_USER $VERSION diff --git a/contrib/gitian/gitian-android.yml b/contrib/gitian/gitian-android.yml deleted file mode 100644 index 7e9ca81782..0000000000 --- a/contrib/gitian/gitian-android.yml +++ /dev/null @@ -1,131 +0,0 @@ ---- -name: "monero-android-0.18" -enable_cache: true -suites: -- "bionic" -architectures: -- "amd64" -packages: -- "curl" -- "gperf" -- "gcc-7" -- "g++-7" -- "gcc" -- "g++" -- "binutils-gold" -- "git" -- "pkg-config" -- "build-essential" -- "autoconf" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "ca-certificates" -- "python" -- "cmake" -- "unzip" -remotes: -- "url": "https://github.com/monero-project/monero.git" - "dir": "monero" -files: [] -script: | - - WRAP_DIR=$HOME/wrapped - HOSTS="arm-linux-android aarch64-linux-android" - FAKETIME_HOST_PROGS="clang clang++ ar nm" - FAKETIME_PROGS="date" - HOST_CFLAGS="-O2 -g" - HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS=-static-libstdc++ - - export GZIP="-9n" - export TZ="UTC" - export BUILD_DIR=`pwd` - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - export GITIAN=1 - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - export ZERO_AR_DATE=1 - - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - ABI=$i - if expr $i : arm- > /dev/null - then - ABI=$i"eabi" - fi - NDKDIR="${BUILD_DIR}/monero/contrib/depends/$i/native/bin" - for prog in ${FAKETIME_HOST_PROGS}; do - WRAPPER=${WRAP_DIR}/${ABI}-${prog} - echo '#!/usr/bin/env bash' > ${WRAPPER} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAPPER} - echo "export FAKETIME=\"$1\"" >> ${WRAPPER} - echo "$NDKDIR/${ABI}-$prog \$@" >> ${WRAPPER} - chmod +x ${WRAPPER} - done - done - } - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - # gcc 7+ honors SOURCE_DATE_EPOCH, no faketime needed - export SOURCE_DATE_EPOCH=`date -d 2000-01-01T12:00:00 +%s` - - git config --global core.abbrev 9 - cd monero - # Set the version string that gets added to the tar archive name - version="`git describe`" - if [[ $version == *"-"*"-"* ]]; then - version="`git rev-parse --short=9 HEAD`" - version="`echo $version | head -c 9`" - fi - - BASEPREFIX=`pwd`/contrib/depends - # Build dependencies for each host - export TAR_OPTIONS=--mtime=2000-01-01T12:00:00 - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - - # Build in a new dir for each host - export TAR_OPTIONS=--mtime=${REFERENCE_DATE}T${REFERENCE_TIME} - for i in ${HOSTS}; do - export PATH=${WRAP_DIR}:${BASEPREFIX}/${i}/native/bin:${PATH_orig} - mkdir build && cd build - cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release - make ${MAKEOPTS} - chmod 755 bin/* - cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin - chmod 644 bin/LICENSE bin/*.md - DISTNAME=monero-${i}-${version} - mv bin ${DISTNAME} - find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 - cd .. - rm -rf build - done - diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py deleted file mode 100755 index 859f460a78..0000000000 --- a/contrib/gitian/gitian-build.py +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import subprocess -import sys - -gsigs = 'https://github.com/monero-project/gitian.sigs.git' -gbrepo = 'https://github.com/devrandom/gitian-builder.git' - -platforms = {'l': ['Linux', 'linux', 'tar.bz2'], - 'a': ['Android', 'android', 'tar.bz2'], - 'f': ['FreeBSD', 'freebsd', 'tar.bz2'], - 'w': ['Windows', 'win', 'zip'], - 'm': ['MacOS', 'osx', 'tar.bz2'] } - -def setup(): - global args, workdir - programs = ['apt-cacher-ng', 'ruby', 'git', 'make', 'wget'] - if args.kvm: - programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils'] - else: - programs += ['lxc', 'debootstrap'] - if not args.no_apt: - subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) - if not os.path.isdir('sigs'): - subprocess.check_call(['git', 'clone', gsigs, 'sigs']) - if not os.path.isdir('builder'): - subprocess.check_call(['git', 'clone', gbrepo, 'builder']) - os.chdir('builder') - subprocess.check_call(['git', 'checkout', 'c0f77ca018cb5332bfd595e0aff0468f77542c23']) - os.makedirs('inputs', exist_ok=True) - os.chdir('inputs') - if os.path.isdir('monero'): - # Remove the potentially stale monero dir. Otherwise you might face submodule mismatches. - subprocess.check_call(['rm', 'monero', '-fR']) - subprocess.check_call(['git', 'clone', args.url, 'monero']) - os.chdir('..') - make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] - if args.docker: - try: - subprocess.check_output(['docker', '--help']) - except: - print("ERROR: Could not find 'docker' command. Ensure this is in your PATH.") - sys.exit(1) - make_image_prog += ['--docker'] - elif not args.kvm: - make_image_prog += ['--lxc'] - subprocess.check_call(make_image_prog) - os.chdir(workdir) - if args.is_bionic and not args.kvm and not args.docker: - subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) - print('Reboot is required') - sys.exit(0) - -def rebuild(): - global args, workdir - - print('\nBuilding Dependencies\n') - os.makedirs('../out/' + args.version, exist_ok=True) - - - for i in args.os: - os_name = platforms[i][0] - tag_name = platforms[i][1] - suffix = platforms[i][2] - - print('\nCompiling ' + args.version + ' ' + os_name) - infile = 'inputs/monero/contrib/gitian/gitian-' + tag_name + '.yml' - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, infile]) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-'+tag_name, '--destination', '../sigs/', infile]) - subprocess.check_call('mv build/out/monero-*.' + suffix + ' ../out/'+args.version, shell=True) - print('Moving var/install.log to var/install-' + tag_name + '.log') - subprocess.check_call('mv var/install.log var/install-' + tag_name + '.log', shell=True) - print('Moving var/build.log to var/build-' + tag_name + '.log') - subprocess.check_call('mv var/build.log var/build-' + tag_name + '.log', shell=True) - - os.chdir(workdir) - - if args.commit_files: - print('\nCommitting '+args.version+' Unsigned Sigs\n') - os.chdir('sigs') - for i, v in platforms: - subprocess.check_call(['git', 'add', args.version+'-'+v[1]+'/'+args.signer]) - subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) - os.chdir(workdir) - - -def build(): - global args, workdir - - print('\nChecking Depends Freshness\n') - os.chdir('builder') - os.makedirs('inputs', exist_ok=True) - - subprocess.check_call(['make', '-C', 'inputs/monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) - - rebuild() - - -def verify(): - global args, workdir - os.chdir('builder') - - for i, v in platforms: - print('\nVerifying v'+args.version+' '+v[0]+'\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../sigs/', '-r', args.version+'-'+v[1], 'inputs/monero/contrib/gitian/gitian-'+v[1]+'.yml']) - os.chdir(workdir) - -def main(): - global args, workdir - - parser = argparse.ArgumentParser(description='Script for running full Gitian builds.', usage='%(prog)s [options] signer version') - parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') - parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') - parser.add_argument('-u', '--url', dest='url', default='https://github.com/monero-project/monero', help='Specify the URL of the repository. Default is %(default)s') - parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') - parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') - parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') - parser.add_argument('-o', '--os', dest='os', default='lafwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, a for Android, f for FreeBSD, w for Windows, m for MacOS') - parser.add_argument('-r', '--rebuild', action='store_true', dest='rebuild', help='Redo a Gitian build') - parser.add_argument('-R', '--rebuildsign', action='store_true', dest='rebuildsign', help='Redo and sign a Gitian build') - parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') - parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') - parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') - parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') - parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. If you run this script on a non-debian based system, pass the --no-apt flag') - parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') - parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') - parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file') - parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build.') - parser.add_argument('-a', '--no-apt', action='store_true', dest='no_apt', help='Indicate that apt is not installed on the system') - - args = parser.parse_args() - workdir = os.getcwd() - - args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) - - if args.buildsign: - args.build = True - args.sign = True - - if args.rebuildsign: - args.rebuild = True - args.sign = True - - if args.kvm and args.docker: - raise Exception('Error: cannot have both kvm and docker') - - args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' - - # Set enviroment variable USE_LXC or USE_DOCKER, let gitian-builder know that we use lxc or docker - if args.docker: - os.environ['USE_DOCKER'] = '1' - elif not args.kvm: - os.environ['USE_LXC'] = '1' - if not 'GITIAN_HOST_IP' in os.environ.keys(): - os.environ['GITIAN_HOST_IP'] = '10.0.2.2' - if not 'LXC_GUEST_IP' in os.environ.keys(): - os.environ['LXC_GUEST_IP'] = '10.0.2.5' - - script_name = os.path.basename(sys.argv[0]) - # Signer and version shouldn't be empty - if args.signer == '': - print(script_name+': Missing signer.') - print('Try '+script_name+' --help for more information') - sys.exit(1) - if args.version == '': - print(script_name+': Missing version.') - print('Try '+script_name+' --help for more information') - sys.exit(1) - - # Add leading 'v' for tags - if args.commit and args.pull: - raise Exception('Cannot have both commit and pull') - args.commit = args.commit if args.commit else args.version - - if args.setup: - setup() - - os.makedirs('builder/inputs/monero', exist_ok=True) - os.chdir('builder/inputs/monero') - if args.pull: - subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) - args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True).strip() - args.version = 'pull-' + args.version - print(args.commit) - subprocess.check_call(['git', 'fetch']) - subprocess.check_call(['git', 'checkout', args.commit]) - os.chdir(workdir) - - if args.build: - build() - - if args.rebuild: - os.chdir('builder') - rebuild() - - if args.verify: - verify() - -if __name__ == '__main__': - main() diff --git a/contrib/gitian/gitian-freebsd.yml b/contrib/gitian/gitian-freebsd.yml deleted file mode 100644 index 7a17f0750c..0000000000 --- a/contrib/gitian/gitian-freebsd.yml +++ /dev/null @@ -1,128 +0,0 @@ ---- -name: "monero-freebsd-0.18" -enable_cache: true -suites: -- "bionic" -architectures: -- "amd64" -packages: -- "curl" -- "clang-8" -- "gperf" -- "gcc-7" -- "g++-7" -- "gcc" -- "g++" -- "binutils-gold" -- "git" -- "pkg-config" -- "build-essential" -- "autoconf" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "ca-certificates" -- "python" -- "cmake" -remotes: -- "url": "https://github.com/monero-project/monero.git" - "dir": "monero" -files: [] -script: | - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-unknown-freebsd" - FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="clang-8 clang++-8 date" - HOST_CFLAGS="-O2 -g" - HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS=-static-libstdc++ - - export GZIP="-9n" - export TZ="UTC" - export BUILD_DIR=`pwd` - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - export GITIAN=1 - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - export ZERO_AR_DATE=1 - - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - WRAPPER=${WRAP_DIR}/${i}-${prog} - echo '#!/usr/bin/env bash' > ${WRAPPER} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAPPER} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAPPER} - echo "export FAKETIME=\"$1\"" >> ${WRAPPER} - echo "$NDKDIR/${ABI}-$prog \$@" >> ${WRAPPER} - chmod +x ${WRAPPER} - done - done - } - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - # gcc 7+ honors SOURCE_DATE_EPOCH, no faketime needed - export SOURCE_DATE_EPOCH=`date -d 2000-01-01T12:00:00 +%s` - - git config --global core.abbrev 9 - cd monero - # Set the version string that gets added to the tar archive name - version="`git describe`" - if [[ $version == *"-"*"-"* ]]; then - version="`git rev-parse --short=9 HEAD`" - version="`echo $version | head -c 9`" - fi - - BASEPREFIX=`pwd`/contrib/depends - # Build dependencies for each host - export TAR_OPTIONS=--mtime=2000-01-01T12:00:00 - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - - ORIGPATH="$PATH" - # Build in a new dir for each host - export SOURCE_DATE_EPOCH=`date -d ${REFERENCE_DATE}T${REFERENCE_TIME} +%s` - export TAR_OPTIONS=--mtime=${REFERENCE_DATE}T${REFERENCE_TIME} - for i in ${HOSTS}; do - export PATH=${WRAP_DIR}:${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir build && cd build - cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_SKIP_RPATH=ON - make ${MAKEOPTS} - chmod 755 bin/* - cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin - chmod 644 bin/LICENSE bin/*.md - DISTNAME=monero-${i}-${version} - mv bin ${DISTNAME} - find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 - cd .. - rm -rf build - done - diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml deleted file mode 100644 index 41915deb9e..0000000000 --- a/contrib/gitian/gitian-linux.yml +++ /dev/null @@ -1,180 +0,0 @@ ---- -name: "monero-linux-0.18" -enable_cache: true -suites: -- "bionic" -architectures: -- "amd64" -packages: -- "curl" -- "gperf" -- "gcc-7" -- "g++-7" -- "gcc" -- "g++" -- "gcc-7-aarch64-linux-gnu" -- "g++-7-aarch64-linux-gnu" -- "gcc-aarch64-linux-gnu" -- "g++-aarch64-linux-gnu" -- "binutils-aarch64-linux-gnu" -- "gcc-7-arm-linux-gnueabihf" -- "g++-7-arm-linux-gnueabihf" -- "gcc-arm-linux-gnueabihf" -- "g++-arm-linux-gnueabihf" -- "g++-riscv64-linux-gnu" -- "g++-7-multilib" -- "gcc-7-multilib" -- "binutils-arm-linux-gnueabihf" -- "binutils-gold" -- "git" -- "pkg-config" -- "build-essential" -- "autoconf" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "ca-certificates" -- "python" -- "cmake" -remotes: -- "url": "https://github.com/monero-project/monero.git" - "dir": "monero" -files: [] -script: | - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu riscv64-linux-gnu" - FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="date" - HOST_CFLAGS="-O2 -g" - HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS=-static-libstdc++ - - export GZIP="-9n" - export TZ="UTC" - export BUILD_DIR=`pwd` - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - export GITIAN=1 - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - if which ${i}-${prog}-7 - then - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog}-7 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - fi - done - done - } - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes - mkdir -p $EXTRA_INCLUDES_BASE - - # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, - # but we can't write there. Instead, create a link here and force it to be included in the - # search paths. - # This problem goes away if linux-libc-dev:i386 pkg exists, but it's not in bionic. - - mkdir -p $EXTRA_INCLUDES_BASE/i686-linux-gnu - rm -f $WRAP_DIR/extra_includes/i686-linux-gnu/asm - ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-linux-gnu/asm - - # glibc 2.23 breaks compatibility with <=2.19 use of lgamma function. - # Hack the math header to restore the old behavior. - mkdir $EXTRA_INCLUDES_BASE/bits - sed -e '/__REDIRFROM .lgamma,/,+3s/_USE_/_DONTUSE_/g' /usr/include/x86_64-linux-gnu/bits/math-finite.h > $EXTRA_INCLUDES_BASE/bits/math-finite.h - - # gcc 7+ honors SOURCE_DATE_EPOCH, no faketime needed - export SOURCE_DATE_EPOCH=`date -d 2000-01-01T12:00:00 +%s` - - git config --global core.abbrev 9 - cd monero - # Set the version string that gets added to the tar archive name - version="`git describe`" - if [[ $version == *"-"*"-"* ]]; then - version="`git rev-parse --short=9 HEAD`" - version="`echo $version | head -c 9`" - fi - - BASEPREFIX=`pwd`/contrib/depends - # Build dependencies for each host - export TAR_OPTIONS=--mtime=2000-01-01T12:00:00 - for i in $HOSTS; do - ARCH_INCLUDES="$EXTRA_INCLUDES_BASE/$i" - if [ -d "$ARCH_INCLUDES" ]; then - EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}:${ARCH_INCLUDES}" - else - EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}" - fi - export C_INCLUDE_PATH="$EXTRA_INCLUDES" - export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES" - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" V=1 - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - ORIGPATH="$PATH" - # Build in a new dir for each host - export SOURCE_DATE_EPOCH=`date -d ${REFERENCE_DATE}T${REFERENCE_TIME} +%s` - export TAR_OPTIONS=--mtime=${REFERENCE_DATE}T${REFERENCE_TIME} - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir build && cd build - ARCH_INCLUDES="$EXTRA_INCLUDES_BASE/$i" - if [ -d "$ARCH_INCLUDES" ]; then - EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}:${ARCH_INCLUDES}" - else - EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}" - fi - export C_INCLUDE_PATH="$EXTRA_INCLUDES" - export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES" - # glibc only added riscv support in 2.27, disable backwards compatibility - if [ "$i" == "riscv64-linux-gnu" ]; then - BACKCOMPAT_OPTION=OFF - else - BACKCOMPAT_OPTION=ON - fi - cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=${BACKCOMPAT_OPTION} -DCMAKE_SKIP_RPATH=ON - make ${MAKEOPTS} - chmod 755 bin/* - cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin - chmod 644 bin/LICENSE bin/*.md - DISTNAME=monero-${i}-${version} - mv bin ${DISTNAME} - find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 - cd .. - rm -rf build - done - diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml deleted file mode 100644 index 74ad21ea3a..0000000000 --- a/contrib/gitian/gitian-osx.yml +++ /dev/null @@ -1,119 +0,0 @@ ---- -name: "monero-osx-0.18" -enable_cache: true -suites: -- "bionic" -architectures: -- "amd64" -packages: -- "ca-certificates" -- "curl" -- "g++" -- "git" -- "pkg-config" -- "autoconf" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "cmake" -- "libcap-dev" -- "libz-dev" -- "libbz2-dev" -- "python" -- "python-dev" -- "python-setuptools" -remotes: -- "url": "https://github.com/monero-project/monero.git" - "dir": "monero" -files: [] -script: | - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin aarch64-apple-darwin" - FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="ar ranlib date dmg genisoimage python" - - export GZIP="-9n" - export TZ="UTC" - export BUILD_DIR=`pwd` - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - export GITIAN=1 - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - export ZERO_AR_DATE=1 - - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - git config --global core.abbrev 9 - cd monero - # Set the version string that gets added to the tar archive name - version="`git describe`" - if [[ $version == *"-"*"-"* ]]; then - version="`git rev-parse --short=9 HEAD`" - version="`echo $version | head -c 9`" - fi - - BASEPREFIX=`pwd`/contrib/depends - - # Build dependencies for each host - export TAR_OPTIONS=--mtime=2000-01-01T12:00:00 - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - ORIGPATH="$PATH" - # Build in a new dir for each host - export TAR_OPTIONS=--mtime=${REFERENCE_DATE}T${REFERENCE_TIME} - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir build && cd build - cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake - make ${MAKEOPTS} - chmod 755 bin/* - cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin - chmod 644 bin/LICENSE bin/*.md - DISTNAME=monero-${i}-${version} - mv bin ${DISTNAME} - find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2 - cd .. - rm -rf build - done - diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml deleted file mode 100644 index 4c607898eb..0000000000 --- a/contrib/gitian/gitian-win.yml +++ /dev/null @@ -1,136 +0,0 @@ ---- -name: "monero-win-0.18" -enable_cache: true -suites: -- "bionic" -architectures: -- "amd64" -packages: -- "curl" -- "g++" -- "git" -- "pkg-config" -- "autoconf" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "mingw-w64" -- "g++-mingw-w64" -- "zip" -- "ca-certificates" -- "python" -- "cmake" -alternatives: -- - package: "i686-w64-mingw32-g++" - path: "/usr/bin/i686-w64-mingw32-g++-posix" -- - package: "i686-w64-mingw32-gcc" - path: "/usr/bin/i686-w64-mingw32-gcc-posix" -- - package: "x86_64-w64-mingw32-g++" - path: "/usr/bin/x86_64-w64-mingw32-g++-posix" -- - package: "x86_64-w64-mingw32-gcc" - path: "/usr/bin/x86_64-w64-mingw32-gcc-posix" -remotes: -- "url": "https://github.com/monero-project/monero.git" - "dir": "monero" -files: [] -script: | - WRAP_DIR=$HOME/wrapped - HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" - FAKETIME_HOST_PROGS="windres objcopy" - FAKETIME_PROGS="date zip" - HOST_CFLAGS="-O2 -g" - HOST_CXXFLAGS="-O2 -g" - - export GZIP="-9n" - export TZ="UTC" - export BUILD_DIR=`pwd` - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - export GITIAN=1 - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - # gcc 7+ honors SOURCE_DATE_EPOCH, no faketime needed - export SOURCE_DATE_EPOCH=`date -d 2000-01-01T12:00:00 +%s` - - git config --global core.abbrev 9 - cd monero - # Set the version string that gets added to the tar archive name - version="`git describe`" - if [[ $version == *"-"*"-"* ]]; then - version="`git rev-parse --short=9 HEAD`" - version="`echo $version | head -c 9`" - fi - - BASEPREFIX=`pwd`/contrib/depends - # Build dependencies for each host - export TAR_OPTIONS=--mtime=2000-01-01T12:00:00 - for i in $HOSTS; do - EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" - if [ -d "$EXTRA_INCLUDES" ]; then - export HOST_ID_SALT="$EXTRA_INCLUDES" - fi - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" V=1 - unset HOST_ID_SALT - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - ORIGPATH="$PATH" - # Run cmake and make, for each create a new build/ directory, - # compile from there, archive, export and delete the archive again - export SOURCE_DATE_EPOCH=`date -d ${REFERENCE_DATE}T${REFERENCE_TIME} +%s` - export TAR_OPTIONS=--mtime=${REFERENCE_DATE}T${REFERENCE_TIME} - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir build && cd build - cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake - make ${MAKEOPTS} - cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin - DISTNAME=monero-${i}-${version} - mv bin ${DISTNAME} - find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip - cd .. && rm -rf build - done - diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md new file mode 100644 index 0000000000..c6b6a3cb72 --- /dev/null +++ b/contrib/guix/INSTALL.md @@ -0,0 +1,798 @@ +# Guix Installation and Setup + +This only needs to be done once per machine. If you have already completed the +installation and setup, please proceed to [perform a build](./README.md). + +Otherwise, you may choose from one of the following options to install Guix: + +1. Using the official **shell installer script** [⤓ skip to section][install-script] + - Maintained by Guix developers + - Easiest (automatically performs *most* setup) + - Works on nearly all Linux distributions + - Only installs latest release + - Binary installation only, requires high level of trust + - Note: The script needs to be run as root, so it should be inspected before it's run +2. Using the official **binary tarball** [⤓ skip to section][install-bin-tarball] + - Maintained by Guix developers + - Normal difficulty (full manual setup required) + - Works on nearly all Linux distributions + - Installs any release + - Binary installation only, requires high level of trust +3. Using a **distribution-maintained package** [⤓ skip to section][install-distro-pkg] + - Maintained by distribution's Guix package maintainer + - Normal difficulty (manual setup required) + - Works only on distributions with Guix packaged, see: https://repology.org/project/guix/versions + - Installs a release decided on by package maintainer + - Source or binary installation depending on the distribution +4. Building **from source** [⤓ skip to section][install-source] + - Maintained by you + - Hard, but rewarding + - Can be made to work on most Linux distributions + - Installs any commit (more granular) + - Source installation, requires lower level of trust + +## Options 1 and 2: Using the official shell installer script or binary tarball + +The installation instructions for both the official shell installer script and +the binary tarballs can be found in the GNU Guix Manual's [Binary Installation +section](https://guix.gnu.org/manual/en/html_node/Binary-Installation.html). + +Note that running through the binary tarball installation steps is largely +equivalent to manually performing what the shell installer script does. + +Note that at the time of writing (July 5th, 2021), the shell installer script +automatically creates an `/etc/profile.d` entry which the binary tarball +installation instructions do not ask you to create. However, you will likely +need this entry for better desktop integration. Please see [this +section](#add-an-etcprofiled-entry) for instructions on how to add a +`/etc/profile.d/guix.sh` entry. + +Regardless of which installation option you chose, the changes to +`/etc/profile.d` will not take effect until the next shell or desktop session, +so you should log out and log back in. + +## Option 3: Using a distribution-maintained package + +Note that this section is based on the distro packaging situation at the time of +writing (July 2021). Guix is expected to be more widely packaged over time. For +an up-to-date view on Guix's package status/version across distros, please see: +https://repology.org/project/guix/versions + +### Debian / Ubuntu + +Guix is available as a distribution package in [Debian +](https://packages.debian.org/search?keywords=guix) and [Ubuntu +](https://packages.ubuntu.com/search?keywords=guix). + +To install: +```sh +sudo apt install guix +``` + +### Arch Linux + +Guix is available in the AUR as +[`guix`](https://aur.archlinux.org/packages/guix/), please follow the +installation instructions in the Arch Linux Wiki ([live +link](https://wiki.archlinux.org/index.php/Guix#AUR_Package_Installation), +[2021/03/30 +permalink](https://wiki.archlinux.org/index.php?title=Guix&oldid=637559#AUR_Package_Installation)) +to install Guix. + +At the time of writing (2021/03/30), the `check` phase will fail if the path to +guix's build directory is longer than 36 characters due to an anachronistic +character limit on the shebang line. Since the `check` phase happens after the +`build` phase, which may take quite a long time, it is recommended that users +either: + +1. Skip the `check` phase + - For `makepkg`: `makepkg --nocheck ...` + - For `yay`: `yay --mflags="--nocheck" ...` + - For `paru`: `paru --nocheck ...` +2. Or, check their build directory's length beforehand + - For those building with `makepkg`: `pwd | wc -c` + +## Option 4: Building from source + +Building Guix from source is a rather involved process but a rewarding one for +those looking to minimize trust and maximize customizability (e.g. building a +particular commit of Guix). Previous experience with using autotools-style build +systems to build packages from source will be helpful. *hic sunt dracones.* + +I strongly urge you to at least skim through the entire section once before you +start issuing commands, as it will save you a lot of unnecessary pain and +anguish. + +### Installing common build tools + +There are a few basic build tools that are required for most things we'll build, +so let's install them now: + +Text transformation/i18n: +- `autopoint` (sometimes packaged in `gettext`) +- `help2man` +- `po4a` +- `texinfo` + +Build system tools: +- `g++` w/ C++11 support +- `libtool` +- `autoconf` +- `automake` +- `pkg-config` (sometimes packaged as `pkgconf`) +- `make` +- `cmake` + +Miscellaneous: +- `git` +- `gnupg` +- `python3` + +### Building and Installing Guix's dependencies + +In order to build Guix itself from source, we need to first make sure that the +necessary dependencies are installed and discoverable. The most up-to-date list +of Guix's dependencies is kept in the ["Requirements" +section](https://guix.gnu.org/manual/en/html_node/Requirements.html) of the Guix +Reference Manual. + +Depending on your distribution, most or all of these dependencies may already be +packaged and installable without manually building and installing. + +For reference, the graphic below outlines Guix v1.3.0's dependency graph: + +![bootstrap map](https://user-images.githubusercontent.com/6399679/125064185-a9a59880-e0b0-11eb-82c1-9b8e5dc9950d.png) + +If you do not care about building each dependency from source, and Guix is +already packaged for your distribution, you can easily install only the build +dependencies of Guix. For example, to enable deb-src and install the Guix build +dependencies on Ubuntu/Debian: + +```sh +sed -i 's|# deb-src|deb-src|g' /etc/apt/sources.list +apt update +apt-get build-dep -y guix +``` + +If this succeeded, you can likely skip to section +["Building and Installing Guix itself"](#building-and-installing-guix-itself). + +#### Guile + +###### Corner case: Multiple versions of Guile on one system + +It is recommended to only install the required version of Guile, so that build systems do +not get confused about which Guile to use. + +However, if you insist on having more versions of Guile installed on +your system, then you need to **consistently** specify +`GUILE_EFFECTIVE_VERSION=3.0` to all +`./configure` invocations for Guix and its dependencies. + +##### Installing Guile + +If your distribution splits packages into `-dev`-suffixed and +non-`-dev`-suffixed sub-packages (as is the case for Debian-derived +distributions), please make sure to install both. For example, to install Guile +v3.0 on Debian/Ubuntu: + +```sh +apt install guile-3.0 guile-3.0-dev +``` + +#### Mixing distribution packages and source-built packages + +At the time of writing, most distributions have _some_ of Guix's dependencies +packaged, but not all. This means that you may want to install the distribution +package for some dependencies, and manually build-from-source for others. + +Distribution packages usually install to `/usr`, which is different from the +default `./configure` prefix of source-built packages: `/usr/local`. + +This means that if you mix-and-match distribution packages and source-built +packages and do not specify exactly `--prefix=/usr` to `./configure` for +source-built packages, you will need to augment the `GUILE_LOAD_PATH` and +`GUILE_LOAD_COMPILED_PATH` environment variables so that Guile will look +under the right prefix and find your source-built packages. + +For example, if you are using Guile v3.0, and have Guile packages in the +`/usr/local` prefix, either add the following lines to your `.profile` or +`.bash_profile` so that the environment variable is properly set for all future +shell logins, or paste the lines into a POSIX-style shell to temporarily modify +the environment variables of your current shell session. + +```sh +# Help Guile v3.0.x find packages in /usr/local +export GUILE_LOAD_PATH="/usr/local/share/guile/site/3.0${GUILE_LOAD_PATH:+:}$GUILE_LOAD_PATH" +export GUILE_LOAD_COMPILED_PATH="/usr/local/lib/guile/3.0/site-ccache${GUILE_LOAD_COMPILED_PATH:+:}$GUILE_COMPILED_LOAD_PATH" +``` + +Note that these environment variables are used to check for packages during +`./configure`, so they should be set as soon as possible should you want to use +a prefix other than `/usr`. + +#### Building and installing source-built packages + +***IMPORTANT**: A few dependencies have non-obvious quirks/errata which are +documented in the sub-sections immediately below. Please read these sections +before proceeding to build and install these packages.* + +Although you should always refer to the README or INSTALL files for the most +accurate information, most of these dependencies use autoconf-style build +systems (check if there's a `configure.ac` file), and will likely do the right +thing with the following: + +Clone the repository and check out the latest release: +```sh +git clone /.git +cd +git tag -l # check for the latest release +git checkout +``` + +For autoconf-based build systems (if `./autogen.sh` or `configure.ac` exists at +the root of the repository): + +```sh +./autogen.sh || autoreconf -vfi +./configure --prefix= +make +sudo make install +``` + +For CMake-based build systems (if `CMakeLists.txt` exists at the root of the +repository): + +```sh +mkdir build && cd build +cmake .. -DCMAKE_INSTALL_PREFIX= +sudo cmake --build . --target install +``` + +If you choose not to specify exactly `--prefix=/usr` to `./configure`, please +make sure you've carefully read the [previous section] on mixing distribution +packages and source-built packages. + +##### Binding packages require `-dev`-suffixed packages + +Relevant for: +- Everyone + +When building bindings, the `-dev`-suffixed version of the original package +needs to be installed. For example, building `Guile-zlib` on Debian-derived +distributions requires that `zlib1g-dev` is installed. + +When using bindings, the `-dev`-suffixed version of the original package still +needs to be installed. This is particularly problematic when distribution +packages are mispackaged like `guile-sqlite3` is in Ubuntu Focal such that +installing `guile-sqlite3` does not automatically install `libsqlite3-dev` as a +dependency. + +Below is a list of relevant Guile bindings and their corresponding `-dev` +packages in Debian at the time of writing. + +| Guile binding package | -dev Debian package | +|-----------------------|---------------------| +| guile-gcrypt | libgcrypt-dev | +| guile-git | libgit2-dev | +| guile-gnutls | (none) | +| guile-json | (none) | +| guile-lzlib | liblz-dev | +| guile-ssh | libssh-dev | +| guile-sqlite3 | libsqlite3-dev | +| guile-zlib | zlib1g-dev | + +##### `guile-git` actually depends on `libgit2 >= 1.1` + +Relevant for: +- Those building `guile-git` from source against `libgit2 < 1.1` +- Those installing `guile-git` from their distribution where `guile-git` is + built against `libgit2 < 1.1` + +As of v0.5.2, `guile-git` claims to only require `libgit2 >= 0.28.0`, however, +it actually requires `libgit2 >= 1.1`, otherwise, it will be confused by a +reference of `origin/keyring`: instead of interpreting the reference as "the +'keyring' branch of the 'origin' remote", the reference is interpreted as "the +branch literally named 'origin/keyring'" + +This is especially notable because Ubuntu Focal packages `libgit2 v0.28.4`, and +`guile-git` is built against it. + +Should you be in this situation, you need to build both `libgit2 v1.1.x` and +`guile-git` from source. + +Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 + +### Building and Installing Guix itself + +Start by cloning Guix: + +``` +git clone https://git.savannah.gnu.org/git/guix.git +cd guix +``` + +You will likely want to build the latest release. +At the time of writing (November 2023), the latest release was `v1.4.0`. + +``` +git branch -a -l 'origin/version-*' # check for the latest release +git checkout +``` + +Bootstrap the build system: +``` +./bootstrap +``` + +Configure with the recommended `--localstatedir` flag: +``` +./configure --localstatedir=/var +``` + +Note: If you intend to hack on Guix in the future, you will need to supply the +same `--localstatedir=` flag for all future Guix `./configure` invocations. See +the last paragraph of this +[section](https://guix.gnu.org/manual/en/html_node/Requirements.html) for more +details. + +Build Guix (this will take a while): +``` +make -j$(nproc) +``` + +Install Guix: + +``` +sudo make install +``` + +### Post-"build from source" Setup + +#### Creating and starting a `guix-daemon-original` service with a fixed `argv[0]` + +At this point, guix will be installed to `${bindir}`, which is likely +`/usr/local/bin` if you did not override directory variables at +`./configure`-time. More information on standard Automake directory variables +can be found +[here](https://www.gnu.org/software/automake/manual/html_node/Standard-Directory-Variables.html). + +However, the Guix init scripts and service configurations for Upstart, systemd, +SysV, and OpenRC are installed (in `${libdir}`) to launch +`${localstatedir}/guix/profiles/per-user/root/current-guix/bin/guix-daemon`, +which does not yet exist, and will only exist after [`root` performs their first +`guix pull`](#guix-pull-as-root). + +We need to create a `-original` version of these init scripts that's pointed to +the binaries we just built and `make install`'ed in `${bindir}` (normally, +`/usr/local/bin`). + +Example for `systemd`, run as `root`: + +```sh +# Create guix-daemon-original.service by modifying guix-daemon.service +libdir=# set according to your PREFIX (default is /usr/local/lib) +bindir="$(dirname $(command -v guix-daemon))" +sed -E -e "s|/\S*/guix/profiles/per-user/root/current-guix/bin/guix-daemon|${bindir}/guix-daemon|" "${libdir}"/systemd/system/guix-daemon.service > /etc/systemd/system/guix-daemon-original.service +chmod 664 /etc/systemd/system/guix-daemon-original.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the non-working guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the working guix-daemon-original.service is started and enabled +systemctl enable guix-daemon-original +systemctl start guix-daemon-original +``` + +#### Creating `guix-daemon` users / groups + +Please see the [relevant +section](https://guix.gnu.org/manual/en/html_node/Build-Environment-Setup.html) +in the Guix Reference Manual for more details. + +## Optional setup + +At this point, you are set up to [use Guix to build Monero](./README.md#usage). However, if you want to polish your setup a bit and +make it "what Guix intended", then read the next few subsections. + +### Add an `/etc/profile.d` entry + +This section definitely does not apply to you if you installed Guix using: +1. The shell installer script +2. fanquake's Docker image +3. Debian's `guix` package + +#### Background + +Although Guix knows how to update itself and its packages, it does so in a +non-invasive way (it does not modify `/usr/local/bin/guix`). + +Instead, it does the following: + +- After a `guix pull`, it updates + `/var/guix/profiles/per-user/$USER/current-guix`, and creates a symlink + targeting this directory at `$HOME/.config/guix/current` + +- After a `guix install`, it updates + `/var/guix/profiles/per-user/$USER/guix-profile`, and creates a symlink + targeting this directory at `$HOME/.guix-profile` + +Therefore, in order for these operations to affect your shell/desktop sessions +(and for the principle of least astonishment to hold), their corresponding +directories have to be added to well-known environment variables like `$PATH`, +`$INFOPATH`, `$XDG_DATA_DIRS`, etc. + +In other words, if `$HOME/.config/guix/current/bin` does not exist in your +`$PATH`, a `guix pull` will have no effect on what `guix` you are using. Same +goes for `$HOME/.guix-profile/bin`, `guix install`, and installed packages. + +Helpfully, after a `guix pull` or `guix install`, a message will be printed like +so: + +``` +hint: Consider setting the necessary environment variables by running: + + GUIX_PROFILE="$HOME/.guix-profile" + . "$GUIX_PROFILE/etc/profile" + +Alternately, see `guix package --search-paths -p "$HOME/.guix-profile"'. +``` + +However, this is somewhat tedious to do for both `guix pull` and `guix install` +for each user on the system that wants to properly use `guix`. I recommend that +you instead add an entry to `/etc/profile.d` instead. This is done by default +when installing the Debian package later than 1.2.0-4 and when using the shell +script installer. + +#### Instructions + +Create `/etc/profile.d/guix.sh` with the following content: +```sh +# _GUIX_PROFILE: `guix pull` profile +_GUIX_PROFILE="$HOME/.config/guix/current" +if [ -L $_GUIX_PROFILE ]; then + export PATH="$_GUIX_PROFILE/bin${PATH:+:}$PATH" + # Export INFOPATH so that the updated info pages can be found + # and read by both /usr/bin/info and/or $GUIX_PROFILE/bin/info + # When INFOPATH is unset, add a trailing colon so that Emacs + # searches 'Info-default-directory-list'. + export INFOPATH="$_GUIX_PROFILE/share/info:$INFOPATH" +fi + +# GUIX_PROFILE: User's default profile +GUIX_PROFILE="$HOME/.guix-profile" +[ -L $GUIX_PROFILE ] || return +GUIX_LOCPATH="$GUIX_PROFILE/lib/locale" +export GUIX_PROFILE GUIX_LOCPATH + +[ -f "$GUIX_PROFILE/etc/profile" ] && . "$GUIX_PROFILE/etc/profile" + +# set XDG_DATA_DIRS to include Guix installations +export XDG_DATA_DIRS="$GUIX_PROFILE/share:${XDG_DATA_DIRS:-/usr/local/share/:/usr/share/}" +``` + +Please note that this will not take effect until the next shell or desktop +session (log out and log back in). + +### `guix pull` as root + +Before you do this, you need to read the section on [choosing your security +model][security-model] and adjust `guix` and `guix-daemon` flags according to +your choice, as invoking `guix pull` may pull substitutes from substitute +servers (which you may not want). + +As mentioned in a previous section, Guix expects +`${localstatedir}/guix/profiles/per-user/root/current-guix` to be populated with +`root`'s Guix profile, `guix pull`-ed and built by some former version of Guix. +However, this is not the case when we build from source. Therefore, we need to +perform a `guix pull` as `root`: + +```sh +sudo --login guix pull --branch=version- +# or +sudo --login guix pull --commit= +``` + +`guix pull` is quite a long process (especially if you're using +`--no-substitutes`). If you encounter build problems, please refer to the +[troubleshooting section](#troubleshooting). + +Note that running a bare `guix pull` with no commit or branch specified will +pull the latest commit on Guix's master branch, which is likely fine, but not +recommended. + +If you installed Guix from source, you may get an error like the following: +```sh +error: while creating symlink '/root/.config/guix/current' No such file or directory +``` +To resolve this, simply: +``` +sudo mkdir -p /root/.config/guix +``` +Then try the `guix pull` command again. + +After the `guix pull` finishes successfully, +`${localstatedir}/guix/profiles/per-user/root/current-guix` should be populated. + +#### Using the newly-pulled `guix` by restarting the daemon + +Depending on how you installed Guix, you should now make sure that your init +scripts and service configurations point to the newly-pulled `guix-daemon`. + +##### If you built Guix from source + +If you followed the instructions for [fixing argv\[0\]][fix-argv0], you can now +do the following: + +```sh +systemctl stop guix-daemon-original +systemctl disable guix-daemon-original + +systemctl enable guix-daemon +systemctl start guix-daemon +``` + +Remember to set `--no-substitutes` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. + +##### If you installed Guix via the Debian/Ubuntu distribution packages + +You will need to create a `guix-daemon-latest` service which points to the new +`guix` rather than a pinned one. + +```sh +# Create guix-daemon-latest.service by modifying guix-daemon.service +sed -E -e "s|/usr/bin/guix-daemon|/var/guix/profiles/per-user/root/current-guix/bin/guix-daemon|" /etc/systemd/system/guix-daemon.service > /lib/systemd/system/guix-daemon-latest.service +chmod 664 /lib/systemd/system/guix-daemon-latest.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the old guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the new guix-daemon-latest.service is started and enabled +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### If you installed Guix via lantw44's Arch Linux AUR package + +At the time of writing (July 5th, 2021) the systemd unit for "updated Guix" is +`guix-daemon-latest.service`, therefore, you should do the following: + +```sh +systemctl stop guix-daemon +systemctl disable guix-daemon + +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### Otherwise... + +Simply do: + +```sh +systemctl restart guix-daemon +``` + +### Checking everything + +If you followed all the steps above to make your Guix setup "prim and proper," +you can check that you did everything properly by running through this +checklist. + +1. `/etc/profile.d/guix.sh` should exist and be sourced at each shell login + +2. `guix describe` should not print `guix describe: error: failed to determine + origin`, but rather something like: + + ``` + Generation 38 Feb 22 2021 16:39:31 (current) + guix f350df4 + repository URL: https://git.savannah.gnu.org/git/guix.git + branch: version-1.2.0 + commit: f350df405fbcd5b9e27e6b6aa500da7f101f41e7 + ``` + +3. `guix-daemon` should be running from `${localstatedir}/guix/profiles/per-user/root/current-guix` + +# Troubleshooting + +## Derivation failed to build + +When you see a build failure like below: + +``` +building /gnu/store/...-foo-3.6.12.drv... +/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' +builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 +build of /gnu/store/...-foo-3.6.12.drv failed +View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. +cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built +guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed +``` + +It means that `guix` failed to build a package named `foo`, which was a +dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" +line is not necessarily the root cause, the first "failed" line is. + +Most of the time, the build failure is due to a spurious test failure or the +package's build system/test suite breaking when running multi-threaded. To +rebuild _just_ this derivation in a single-threaded fashion (please don't forget +to add other `guix` flags like `--no-substitutes` as appropriate): + +```sh +$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv +``` + +If the single-threaded rebuild did not succeed, you may need to dig deeper. +You may view `foo`'s build logs in `less` like so (please replace paths with the +path you see in the build failure output): + +```sh +$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less +``` + +`foo`'s build directory is also preserved and available at +`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple +times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build +failure output for the most accurate, up-to-date information. + +### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character + +This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem +which rejects characters not present in the UTF-8 character code set. An example +is ZFS with the utf8only=on option set. + +More information: https://github.com/python/cpython/issues/81765 + +### openssl-1.1.1l and openssl-1.1.1n + +OpenSSL includes tests that will fail once some certificate has expired. +The workarounds from the GnuTLS section immediately below can be used. + +### GnuTLS: test-suite FAIL: status-request-revoked + +*The derivation is likely identified by: `/gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv`* + +This unfortunate error is most common for non-substitute builders who installed +Guix v1.2.0. The problem stems from the fact that one of GnuTLS's tests uses a +hardcoded certificate which expired on 2020-10-24. + +What's more unfortunate is that this GnuTLS derivation is somewhat special in +Guix's dependency graph and is not affected by the package transformation flags +like `--without-tests=`. + +The easiest solution for those encountering this problem is to install a newer +version of Guix. However, there are ways to work around this issue: + +#### Workaround 1: Using substitutes for this single derivation + +If you've authorized the official Guix build farm's key (more info +[here](./README.md#step-1-authorize-the-signing-keys)), then you can use +substitutes just for this single derivation by invoking the following: + +```sh +guix build --substitute-urls="https://ci.guix.gnu.org" /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +``` + +See [this section](./README.md#removing-authorized-keys) for instructions on how +to remove authorized keys if you don't want to keep the build farm's key +authorized. + +#### Workaround 2: Temporarily setting the system clock back + +This workaround was described [here](https://issues.guix.gnu.org/44559#5). + +Basically: +2. Turn off NTP +3. Set system time to 2020-10-01 +4. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +5. Set system time back to accurate current time +6. Turn NTP back on + +For example, + +```sh +sudo timedatectl set-ntp no +sudo date --set "01 oct 2020 15:00:00" +guix build /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +sudo timedatectl set-ntp yes +``` + +#### Workaround 3: Disable the tests in the Guix source code for this single derivation + +If all of the above workarounds fail, you can also disable the `tests` phase of +the derivation via the `arguments` option, as described in the official +[`package` +reference](https://guix.gnu.org/manual/en/html_node/package-Reference.html). + +For example, to disable the openssl-1.1 check phase: + +```diff +diff --git a/gnu/packages/tls.scm b/gnu/packages/tls.scm +index f1e844b..1077c4b 100644 +--- a/gnu/packages/tls.scm ++++ b/gnu/packages/tls.scm +@@ -494,4 +494,5 @@ (define-public openssl-1.1 + (arguments + `(#:parallel-tests? #f ++ #:tests? #f + #:test-target "test" +``` + +### coreutils: FAIL: tests/tail-2/inotify-dir-recreate + +The inotify-dir-create test fails on "remote" filesystems such as overlayfs +(Docker's default filesystem) due to the filesystem being mistakenly recognized +as non-remote. + +A relatively easy workaround to this is to make sure that a somewhat traditional +filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For +Docker users, this might mean [using a volume][docker/volumes], [binding +mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) +[mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. + +Please see the following links for more details: + +- An upstream coreutils bug has been filed: [debbugs#47940](https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47940) +- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935), [guix-issues#49985](https://issues.guix.gnu.org/49985#5) +- A commit to skip this test in Guix has been merged into the core-updates branch: + [savannah/guix@6ba1058](https://git.savannah.gnu.org/cgit/guix.git/commit/?id=6ba1058df0c4ce5611c2367531ae5c3cdc729ab4) + + +[install-script]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-bin-tarball]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-distro-pkg]: #option-4-using-a-distribution-maintained-package +[install-source]: #option-5-building-from-source + +[fix-argv0]: #creating-and-starting-a-guix-daemon-original-service-with-a-fixed-argv0 +[security-model]: ./README.md#choosing-your-security-model + +[docker/volumes]: https://docs.docker.com/storage/volumes/ +[docker/bind-mnt]: https://docs.docker.com/storage/bind-mounts/ +[docker/tmpfs]: https://docs.docker.com/storage/tmpfs/ + +# Purging/Uninstalling Guix + +In the extraordinarily rare case where you messed up your Guix installation in +an irreversible way, you may want to completely purge Guix from your system and +start over. + +1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt + purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). +2. Remove all build users and groups + + You may check for relevant users and groups using: + + ``` + getent passwd | grep guix + getent group | grep guix + ``` + + Then, you may remove users and groups using: + + ``` + sudo userdel + sudo groupdel + ``` + +3. Remove all possible Guix-related directories + - `/var/guix/` + - `/var/log/guix/` + - `/gnu/` + - `/etc/guix/` + - `/home/*/.config/guix/` + - `/home/*/.cache/guix/` + - `/home/*/.guix-profile/` + - `/root/.config/guix/` + - `/root/.cache/guix/` + - `/root/.guix-profile/` diff --git a/contrib/guix/README.md b/contrib/guix/README.md new file mode 100644 index 0000000000..83f42c1b61 --- /dev/null +++ b/contrib/guix/README.md @@ -0,0 +1,345 @@ +# Bootstrappable Monero Builds + +This directory contains the files necessary to perform bootstrappable Monero +builds. + +[Bootstrappability][b17e] furthers our binary security guarantees by allowing us +to _audit and reproduce_ our toolchain instead of blindly _trusting_ binary +downloads. + +We achieve bootstrappability by using Guix as a functional package manager. + +# Requirements + +Conservatively, you will need an x86_64 machine with: + +- 16GB of free disk space on the partition that /gnu/store will reside in +- 8GB of free disk space **per platform triple** you're planning on building + (see the `HOSTS` [environment variable description][env-vars-list]) + +# Installation and Setup + +If you don't have Guix installed and set up, please follow the instructions in +[INSTALL.md](./INSTALL.md) + +# Usage + +If you haven't considered your security model yet, please read [the relevant +section](#choosing-your-security-model) before proceeding to perform a build. + +## Building + +*The author highly recommends at least reading over the [common usage patterns +and examples](#common-guix-build-invocation-patterns-and-examples) section below +before starting a build. For a full list of customization options, see the +[recognized environment variables][env-vars-list] section.* + +To build Monero reproducibly with all default options, invoke the +following from the top of a clean repository: + +```sh +./contrib/guix/guix-build +``` + +## Cleaning intermediate work directories + +By default, `guix-build` leaves all intermediate files or "work directories" +(e.g. `depends/work`, `guix-build-*/distsrc-*`) intact at the end of a build so +that they are available to the user (to aid in debugging, etc.). However, these +directories usually take up a large amount of disk space. Therefore, a +`guix-clean` convenience script is provided which cleans the current `git` +worktree to save disk space: + +``` +./contrib/guix/guix-clean +``` + +## Attesting to build outputs + +Much like how Gitian build outputs are attested to in a `gitian.sigs` +repository, Guix build outputs are attested to in the [`guix.sigs` +repository](https://github.com/monero-project/guix.sigs). + +After you've cloned the `guix.sigs` repository, to attest to the current +worktree's commit/tag: + +``` +env GUIX_SIGS_REPO= SIGNER= ./contrib/guix/guix-attest +``` + +See `./contrib/guix/guix-attest --help` for more information on the various ways +`guix-attest` can be invoked. + +## Verifying build output attestations + +After at least one other signer has uploaded their signatures to the `guix.sigs` +repository: + +``` +git -C pull +env GUIX_SIGS_REPO= ./contrib/guix/guix-verify +``` + +## Common `guix-build` invocation patterns and examples + +### Keeping caches outside of the worktree + +If you perform a lot of builds and have a bunch of worktrees, you may find it +more efficient to keep the depends tree's download cache and build cache +outside of the worktrees to avoid duplicate downloads and unnecessary builds. To +help with this situation, the `guix-build` script honours the `SOURCES_PATH`, +`BASE_CACHE` environment variables and will pass them on to the +depends tree so that you can do something like: + +```sh +env SOURCES_PATH="$HOME/depends-SOURCES_PATH" BASE_CACHE="$HOME/depends-BASE_CACHE" ./contrib/guix/guix-build +``` + +Note that the paths that these environment variables point to **must be +directories**, and **NOT symlinks to directories**. + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Building a subset of platform triples + +Sometimes you only want to build a subset of the supported platform triples, in +which case you can override the default list by setting the space-separated +`HOSTS` environment variable: + +```sh +env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin' ./contrib/guix/guix-build +``` + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Controlling the number of threads used by `guix` build commands + +Depending on your system's RAM capacity, you may want to decrease the number of +threads used to decrease RAM usage or vice versa. + +By default, the scripts under `./contrib/guix` will invoke all `guix` build +commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not +specified. However, astute manual readers will also notice that `guix` build +commands also accept a `--max-jobs=` flag (which defaults to 1 if unspecified). + +Here is the difference between `--cores=` and `--max-jobs=`: + +> Note: When I say "derivation," think "package" + +`--cores=` + +- controls the number of CPU cores to build each derivation. This is the value + passed to `make`'s `--jobs=` flag. + +`--max-jobs=` + +- controls how many derivations can be built in parallel +- defaults to 1 + +Therefore, the default is for `guix` build commands to build one derivation at a +time, utilizing `$JOBS` threads. + +Specifying the `$JOBS` environment variable will only modify `--cores=`, but you +can also modify the value for `--max-jobs=` by specifying +`$ADDITIONAL_GUIX_COMMON_FLAGS`. For example, if you have a LOT of memory, you +may want to set: + +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +Which allows for a maximum of 8 derivations to be built at the same time, each +utilizing `$JOBS` threads. + +Or, if you'd like to avoid spurious build failures caused by issues with +parallelism within a single package, but would still like to build multiple +packages when the dependency graph allows for it, you may want to try: + +```sh +export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' +``` + +See the [recognized environment variables][env-vars-list] section for more +details. + +## Recognized environment variables + +* _**HOSTS**_ + + Override the space-separated list of platform triples for which to perform a + bootstrappable build. + + _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu + riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_ + +* _**SOURCES_PATH**_ + + Set the depends tree download cache for sources. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary redownloading of package sources. + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**BASE_CACHE**_ + + Set the depends tree cache for built packages. This is passed through to the + depends tree. Setting this to the same directory across multiple builds of the + depends tree can eliminate unnecessary building of packages. + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + +* _**JOBS**_ + + Override the number of jobs to run simultaneously, you might want to do so on + a memory-limited machine. This may be passed to: + + - `guix` build commands as in `guix shell --cores="$JOBS"` + - `make` as in `make --jobs="$JOBS"` + - `xargs` as in `xargs -P"$JOBS"` + + See [here](#controlling-the-number-of-threads-used-by-guix-build-commands) for + more details. + + _(defaults to the value of `nproc` outside the container)_ + +* _**SOURCE_DATE_EPOCH**_ + + Override the reference UNIX timestamp used for bit-for-bit reproducibility, + the variable name conforms to [standard][r12e/source-date-epoch]. + + _(defaults to the output of `$(git log --format=%at -1)`)_ + +* _**V**_ + + If non-empty, will pass `V=1` to all `make` invocations, making `make` output + verbose. + + Note that any given value is ignored. The variable is only checked for + emptiness. More concretely, this means that `V=` (setting `V` to the empty + string) is interpreted the same way as not setting `V` at all, and that `V=0` + has the same effect as `V=1`. + +* _**SUBSTITUTE_URLS**_ + + A whitespace-delimited list of URLs from which to download pre-built packages. + A URL is only used if its signing key is authorized (refer to the [substitute + servers section](#option-1-building-with-substitutes) for more details). + +* _**ADDITIONAL_GUIX_COMMON_FLAGS**_ + + Additional flags to be passed to all `guix` commands. + +* _**ADDITIONAL_GUIX_TIMEMACHINE_FLAGS**_ + + Additional flags to be passed to `guix time-machine`. + +* _**ADDITIONAL_GUIX_ENVIRONMENT_FLAGS**_ + + Additional flags to be passed to the invocation of `guix shell` inside + `guix time-machine`. + +# Choosing your security model + +No matter how you installed Guix, you need to decide on your security model for +building packages with Guix. + +Guix allows us to achieve better binary security by using our CPU time to build +everything from scratch. However, it doesn't sacrifice user choice in pursuit of +this: users can decide whether or not to use **substitutes** (pre-built +packages). + +## Option 1: Building with substitutes + +### Step 1: Authorize the signing keys + +Depending on the installation procedure you followed, you may have already +authorized the Guix build farm key. In particular, the official shell installer +script asks you if you want the key installed, and the debian distribution +package authorized the key during installation. + +You can check the current list of authorized keys at `/etc/guix/acl`. + +At the time of writing, a `/etc/guix/acl` with just the Guix build farm key +authorized looks something like: + +```lisp +(acl + (entry + (public-key + (ecc + (curve Ed25519) + (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#) + ) + ) + (tag + (guix import) + ) + ) + ) +``` + +If you've determined that the official Guix build farm key hasn't been +authorized, and you would like to authorize it, run the following as root: + +``` +guix archive --authorize < /var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub +``` + +If +`/var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub` +doesn't exist, try: + +```sh +guix archive --authorize < /share/guix/ci.guix.gnu.org.pub +``` + +Where `` is likely: +- `/usr` if you installed from a distribution package +- `/usr/local` if you installed Guix from source and didn't supply any + prefix-modifying flags to Guix's `./configure` + +#### Removing authorized keys + +To remove previously authorized keys, simply edit `/etc/guix/acl` and remove the +`(entry (public-key ...))` entry. + +### Step 2: Specify the substitute servers + +Once its key is authorized, the official Guix build farm at +https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag +is supplied. This default list of substitute servers is overridable both on a +`guix-daemon` level and when you invoke `guix` commands. + +## Option 2: Disabling substitutes on an ad-hoc basis + +If you prefer not to use any substitutes, make sure to supply `--no-substitutes` +like in the following snippet. The first build will take a while, but the +resulting packages will be cached for future builds. + +For direct invocations of `guix`: +```sh +guix --no-substitutes +``` + +For the scripts under `./contrib/guix/`: +```sh +export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' +``` + +## Option 3: Disabling substitutes by default + +`guix-daemon` accepts a `--no-substitutes` flag, which will make sure that, +unless otherwise overridden by a command line invocation, no substitutes will be +used. + +If you start `guix-daemon` using an init script, you can edit said script to +supply this flag. + +[b17e]: https://bootstrappable.org/ +[r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ +[env-vars-list]: #recognized-environment-variables diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest new file mode 100755 index 0000000000..f494eb99c9 --- /dev/null +++ b/contrib/guix/guix-attest @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the monero repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat env basename mkdir diff sort + +if [ -z "$NO_SIGN" ]; then + # make it possible to override the gpg binary + GPG=${GPG:-gpg} + + # $GPG can contain extra arguments passed to the binary + # so let's check only the existence of arg[0] + # shellcheck disable=SC2206 + GPG_ARRAY=($GPG) + check_tools "${GPG_ARRAY[0]}" +fi + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat < \\ + SIGNER=GPG_KEY_NAME[=SIGNER_NAME] \\ + [ NO_SIGN=1 ] + ./contrib/guix/guix-attest + +Example w/o overriding signing name: + + env GUIX_SIGS_REPO=/home/user/guix.sigs \\ + SIGNER=achow101 \\ + ./contrib/guix/guix-attest + +Example overriding signing name: + + env GUIX_SIGS_REPO=/home/user/guix.sigs \\ + SIGNER=0x96AB007F1A7ED999=dongcarl \\ + ./contrib/guix/guix-attest + +Example w/o signing, just creating SHA256SUMS: + + env GUIX_SIGS_REPO=/home/user/guix.sigs \\ + SIGNER=achow101 \\ + NO_SIGN=1 \\ + ./contrib/guix/guix-attest + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ] || [ -z "$SIGNER" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +################ +# The key specified in SIGNER should be usable +################ + +IFS='=' read -r gpg_key_name signer_name <<< "$SIGNER" +if [ -z "${signer_name}" ]; then + signer_name="$gpg_key_name" +fi + +if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then + echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" + exit 1 +fi + +################ +# We should be able to find at least one output +################ + +echo "Looking for build output SHA256SUMS fragments in ${LOGDIR_BASE}" + +shopt -s nullglob +sha256sum_fragments=( "$LOGDIR_BASE"/*/SHA256SUMS.part ) # This expands to an array of directories... +shopt -u nullglob + +noncodesigned_fragments=() +codesigned_fragments=() + +if (( ${#sha256sum_fragments[@]} )); then + echo "Found build output SHA256SUMS fragments:" + for logdir in "${sha256sum_fragments[@]}"; do + echo " '$logdir'" + case "$logdir" in + "$LOGDIR_BASE"/*-codesigned/SHA256SUMS.part) + codesigned_fragments+=("$logdir") + ;; + *) + noncodesigned_fragments+=("$logdir") + ;; + esac + done + echo +else + echo "ERR: Could not find any build output SHA256SUMS fragments in ${LOGDIR_BASE}" + exit 1 +fi + +############## +## Attest ## +############## + +# Usage: out_name $logdir +# +# HOST: The output directory being attested +# +out_name() { + basename "$(dirname "$1")" +} + +shasum_already_exists() { +cat < "$temp_noncodesigned" + if [ -e noncodesigned.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u noncodesigned.SHA256SUMS "$temp_noncodesigned"; then + echo "A noncodesigned.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists noncodesigned.SHA256SUMS + exit 1 + fi + else + mv "$temp_noncodesigned" noncodesigned.SHA256SUMS + fi + else + echo "ERR: No noncodesigned outputs found for '${VERSION}', exiting..." + exit 1 + fi + + temp_all="$(mktemp)" + trap 'rm -rf -- "$temp_all"' EXIT + + if (( ${#codesigned_fragments[@]} )); then + # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is + # not needed if there are no $codesigned_fragments + cat "${sha256sum_fragments[@]}" \ + | sort -u \ + | sort -k2 \ + | basenameify_SHA256SUMS \ + > "$temp_all" + if [ -e all.SHA256SUMS ]; then + # The SHA256SUMS already exists, make sure it's exactly what we + # expect, error out if not + if diff -u all.SHA256SUMS "$temp_all"; then + echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." + else + shasum_already_exists all.SHA256SUMS + exit 1 + fi + else + mv "$temp_all" all.SHA256SUMS + fi + else + # It is fine to have the codesigned outputs be missing (perhaps the + # detached codesigs have not been published yet), just print a log + # message instead of erroring out + echo "INFO: No codesigned outputs found for '${VERSION}', skipping..." + fi + + if [ -z "$NO_SIGN" ]; then + echo "Signing SHA256SUMS to produce SHA256SUMS.asc" + for i in *.SHA256SUMS; do + if [ ! -e "$i".asc ]; then + ${GPG} --detach-sign \ + --digest-algo sha256 \ + --local-user "$gpg_key_name" \ + --armor \ + --output "$i".asc "$i" + else + echo "Signature already there" + fi + done + else + echo "Not signing SHA256SUMS as \$NO_SIGN is not empty" + fi + echo "" +) diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build new file mode 100755 index 0000000000..21c5b5e2e3 --- /dev/null +++ b/contrib/guix/guix-build @@ -0,0 +1,466 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Monero repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + +################### +## SANITY CHECKS ## +################### + +################ +# Required non-builtin commands should be invocable +################ + +check_tools cat mkdir make getent curl git guix + +################ +# GUIX_BUILD_OPTIONS should be empty +################ +# +# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that +# can perform builds. This seems like what we want instead of +# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually +# _appended_ to normal command-line options. Meaning that they will take +# precedence over the command-specific ADDITIONAL_GUIX__FLAGS. +# +# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's +# existence here and direct users of this script to use our (more flexible) +# custom environment variables. +if [ -n "$GUIX_BUILD_OPTIONS" ]; then +cat << EOF +Error: Environment variable GUIX_BUILD_OPTIONS is not empty: + '$GUIX_BUILD_OPTIONS' + +Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset +GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options +across guix commands or ADDITIONAL_GUIX__FLAGS to set build options for a +specific guix command. + +See contrib/guix/README.md for more details. +EOF +exit 1 +fi + +################ +# Checkout git submodules if we haven't already +################ + +git submodule update --init --recursive --progress + +################ +# The git worktree should not be dirty +################ + +if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then +cat << EOF +ERR: The current git worktree is dirty, which may lead to broken builds. + + Aborting... + +Hint: To make your git worktree clean, You may want to: + 1. Commit your changes, + 2. Stash your changes, or + 3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on + using a dirty worktree +EOF +exit 1 +fi + +mkdir -p "$VERSION_BASE" + +################ +# Build directories should not exist +################ + +# Default to building for all supported HOSTs (overridable by environment) +export HOSTS="${HOSTS:-x86_64-linux-gnu + aarch64-linux-gnu + arm-linux-gnueabihf + riscv64-linux-gnu + i686-linux-gnu + x86_64-w64-mingw32 + i686-w64-mingw32 + x86_64-unknown-freebsd + x86_64-apple-darwin + aarch64-apple-darwin + aarch64-linux-android + arm-linux-androideabi}" + +# Usage: distsrc_for_host HOST +# +# HOST: The current platform triple we're building for +# +distsrc_for_host() { + echo "${DISTSRC_BASE}/build/distsrc-${VERSION}-${1}" +} + +# Accumulate a list of build directories that already exist... +hosts_distsrc_exists="" +for host in $HOSTS; do + if [ -e "$(distsrc_for_host "$host")" ]; then + hosts_distsrc_exists+=" ${host}" + fi +done + +if [ -n "$hosts_distsrc_exists" ]; then +# ...so that we can print them out nicely in an error message +cat << EOF +ERR: Build directories for this commit already exist for the following platform + triples you're attempting to build, probably because of previous builds. + Please remove, or otherwise deal with them prior to starting another build. + + Aborting... + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +for host in $hosts_distsrc_exists; do + echo " ${host} '$(distsrc_for_host "$host")'" +done +exit 1 +else + mkdir -p "$DISTSRC_BASE" +fi + +################ +# Check that we can connect to the guix-daemon +################ + +cat << EOF +Checking that we can connect to the guix-daemon... + +Hint: If this hangs, you may want to try turning your guix-daemon off and on + again. + +EOF +if ! guix gc --list-failures > /dev/null; then +cat << EOF + +ERR: Failed to connect to the guix-daemon, please ensure that one is running and + reachable. +EOF +exit 1 +fi + +# Developer note: we could use `guix repl` for this check and run: +# +# (import (guix store)) (close-connection (open-connection)) +# +# However, the internal API is likely to change more than the CLI invocation + +################ +# Services database must have basic entries +################ + +if ! getent services http https ftp > /dev/null 2>&1; then +cat << EOF +ERR: Your system's C library cannot find service database entries for at least + one of the following services: http, https, ftp. + +Hint: Most likely, /etc/services does not exist yet (common for docker images + and minimal distros), or you don't have permissions to access it. + + If /etc/services does not exist yet, you may want to install the + appropriate package for your distro which provides it. + + On Debian/Ubuntu: netbase + On Arch Linux: iana-etc + + For more information, see: getent(1), services(5) + +EOF + +fi + +######### +# SETUP # +######### + +# Determine the maximum number of jobs to run simultaneously (overridable by +# environment) +JOBS="${JOBS:-$(nproc)}" + +# Usage: host_to_commonname HOST +# +# HOST: The current platform triple we're building for +# +host_to_commonname() { + case "$1" in + *darwin*) echo osx ;; + *mingw*) echo win ;; + *android*) echo android ;; + *linux*) echo linux ;; + *freebsd*) echo freebsd ;; + *) exit 1 ;; + esac +} + +COMMIT_TIMESTAMP="$(git -c log.showSignature=false log --format=%at -1)" + +# Precious directories are those which should not be cleaned between successive +# guix builds +depends_precious_dir_names='SOURCES_PATH BASE_CACHE' +precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE LOGDIR_BASE PROFILES_BASE" + +# Usage: contains IFS-SEPARATED-LIST ITEM +contains() { + for i in ${1}; do + if [ "$i" = "${2}" ]; then + return 0 # Found! + fi + done + return 1 +} + +# If the user explicitly specified a precious directory, create it so we +# can map it into the container +for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if [ -n "$precious_dir_path" ]; then + if [ ! -e "$precious_dir_path" ]; then + mkdir -p "$precious_dir_path" + elif [ -L "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} cannot be a symbolic link" + exit 1 + elif [ ! -d "$precious_dir_path" ]; then + echo "ERR: ${precious_dir_name} must be a directory" + exit 1 + fi + fi +done + +mkdir -p "$VAR_BASE" + +# Record the _effective_ values of precious directories such that guix-clean can +# avoid clobbering them if appropriate. +# +# shellcheck disable=SC2046,SC2086 +{ + # Get depends precious dir definitions from depends + for precious_dir_name in $depends_precious_dir_names; do + precious_dir_path="$(make -C "${PWD}/contrib/depends" --no-print-directory print-${precious_dir_name})" + echo "${precious_dir_name}=${precious_dir_path}" + done + + # Get remaining precious dir definitions from the environment + for precious_dir_name in $precious_dir_names; do + precious_dir_path="${!precious_dir_name}" + if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then + echo "${precious_dir_name}=${precious_dir_path}" + fi + done +} > "${VAR_BASE}/precious_dirs" + +# Make sure an output and logs directory exists for our builds +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" +mkdir -p "$OUTDIR_BASE" +LOGDIR_BASE="${LOGDIR_BASE:-${VERSION_BASE}/logs}" +mkdir -p "$LOGDIR_BASE" + +# Download the depends sources now as we won't have internet access in the build +# container +for host in $HOSTS; do + make -C "${PWD}/contrib/depends" -j"$JOBS" download-"$(host_to_commonname "${host}")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} +done + +# Usage: outdir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +outdir_for_host() { + echo "${OUTDIR_BASE}/${1}${2:+-${2}}" +} + +# Usage: logdir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +logdir_for_host() { + echo "${LOGDIR_BASE}/${1}${2:+-${2}}" +} + +# Usage: profiledir_for_host HOST SUFFIX +# +# HOST: The current platform triple we're building for +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${1}${2:+-${2}}" +} + + +######### +# BUILD # +######### + +# Function to be called when building for host ${1} and the user interrupts the +# build +int_trap() { +cat << EOF +** INT received while building ${1}, you may want to clean up the relevant + work directories (e.g. distsrc-*) before rebuilding + +Hint: To blow everything away, you may want to use: + + $ ./contrib/guix/guix-clean + +Specifically, this will remove all files without an entry in the index, +excluding the depends download cache, the depends built +packages cache, the garbage collector roots for Guix environments, and the +output directory. +EOF +} + +# Deterministically build Monero +# shellcheck disable=SC2153 +for host in $HOSTS; do + + # Display proper warning when the user interrupts the build + trap 'int_trap ${host}' INT + + ( + # Required for 'contrib/guix/manifest.scm' to output the right manifest + # for the particular $HOST we're building for + export HOST="$host" + + + # shellcheck disable=SC2030 +cat << EOF +INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: + ...using commit timestamp: ${COMMIT_TIMESTAMP:?not set} + ...running at most ${JOBS:?not set} jobs + ...from worktree directory: '${PWD}' + ...bind-mounted in container to: '/monero' + ...in build directory: '$(distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...outputting in: '$(outdir_for_host "$HOST")' + ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' +EOF + + rm -f "$(profiledir_for_host "${HOST}")" + + # First run produces a different GUIX_ENVIRONMENT. + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + -- echo "$HOST" + + if [[ -v DRY_RUN ]]; then + echo "Dry run, exiting.." + exit 0 + fi + + # Run the build script 'contrib/guix/libexec/build.sh' in the build + # container specified by 'contrib/guix/manifest.scm'. + # + # Explanation of `guix environment` flags: + # + # --container run command within an isolated container + # + # Running in an isolated container minimizes build-time differences + # between machines and improves reproducibility + # + # --pure unset existing environment variables + # + # Same rationale as --container + # + # --no-cwd do not share current working directory with an + # isolated container + # + # When --container is specified, the default behavior is to share + # the current working directory with the isolated container at the + # same exact path (e.g. mapping '/home/user/monero/' to + # '/home/user/monero/'). This means that the $PWD inside the + # container becomes a source of irreproducibility. --no-cwd disables + # this behaviour. + # + # --share=SPEC for containers, share writable host file system + # according to SPEC + # + # --share="$PWD"=/monero + # + # maps our current working directory to /monero + # inside the isolated container, which we later cd + # into. + # + # While we don't want to map our current working directory to the + # same exact path (as this introduces irreproducibility), we do want + # it to be at a _fixed_ path _somewhere_ inside the isolated + # container so that we have something to build. '/monero' was + # chosen arbitrarily. + # + # ${SOURCES_PATH:+--share="$SOURCES_PATH"} + # + # make the downloaded depends sources path available + # inside the isolated container + # + # The isolated container has no network access as it's in a + # different network namespace from the main machine, so we have to + # make the downloaded depends sources available to it. The sources + # should have been downloaded prior to this invocation. + # + # --keep-failed keep build tree of failed builds + # + # When builds of the Guix environment itself (not Monero) + # fail, it is useful for the build tree to be kept for debugging + # purposes. + # + # ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} + # + # fetch substitute from SUBSTITUTE_URLS if they are + # authorized + # + # Depending on the user's security model, it may be desirable to use + # substitutes (pre-built packages) from servers that the user trusts. + # Please read the README.md in the same directory as this file for + # more information. + # + # shellcheck disable=SC2086,SC2031 + time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ + --container \ + --pure \ + --no-cwd \ + --share="$PWD"=/monero \ + --share="$DISTSRC_BASE"=/distsrc-base \ + --share="$OUTDIR_BASE"=/outdir-base \ + --share="$LOGDIR_BASE"=/logdir-base \ + --expose="$(git rev-parse --git-common-dir)" \ + ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ + ${BASE_CACHE:+--share="$BASE_CACHE"} \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + --link-profile \ + --user="user" \ + --root="$(profiledir_for_host "${HOST}")" \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ + -- env HOST="$HOST" \ + VERSION="$VERSION" \ + JOBS="$JOBS" \ + COMMIT_TIMESTAMP="${COMMIT_TIMESTAMP:?unable to determine value}" \ + ${V:+V=1} \ + ${DEPENDS_ONLY:+DEPENDS_ONLY=1} \ + ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ + DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$host")" \ + OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$host")" \ + LOGDIR="$(LOGDIR_BASE=/logdir-base && logdir_for_host "$host")" \ + DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ + bash -c "cd /monero && bash contrib/guix/libexec/build.sh" + ) + +done diff --git a/contrib/guix/guix-clean b/contrib/guix/guix-clean new file mode 100755 index 0000000000..7ecff1e406 --- /dev/null +++ b/contrib/guix/guix-clean @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Monero repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat mkdir make git guix + + +############# +## Clean ## +############# + +# Usage: under_dir MAYBE_PARENT MAYBE_CHILD +# +# If MAYBE_CHILD is a subdirectory of MAYBE_PARENT, print the relative path +# from MAYBE_PARENT to MAYBE_CHILD. Otherwise, return 1 as the error code. +# +# NOTE: This does not perform any symlink-resolving or path canonicalization. +# +under_dir() { + local path_residue + path_residue="${2##"${1}"}" + if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then + return 1 + else + echo "$path_residue" + fi +} + +# Usage: dir_under_git_root MAYBE_CHILD +# +# If MAYBE_CHILD is under the current git repository and exists, print the +# relative path from the git repository's top-level directory to MAYBE_CHILD, +# otherwise, exit with an error code. +# +dir_under_git_root() { + local rv + rv="$(under_dir "$(git_root)" "$1")" + [ -n "$rv" ] && echo "$rv" +} + +shopt -s nullglob +found_precious_dirs_files=( "${version_base_prefix}"*/"${var_base_basename}/precious_dirs" ) # This expands to an array of directories... +shopt -u nullglob + +exclude_flags=() + +for precious_dirs_file in "${found_precious_dirs_files[@]}"; do + # Make sure the precious directories (e.g. SOURCES_PATH, BASE_CACHE, SDK_PATH) + # are excluded from git-clean + echo "Found precious_dirs file: '${precious_dirs_file}'" + + # Exclude the precious_dirs file itself + if dirs_file_exclude_fragment=$(dir_under_git_root "$(dirname "$precious_dirs_file")"); then + exclude_flags+=( --exclude="${dirs_file_exclude_fragment}/precious_dirs" ) + fi + + # Read each 'name=dir' pair from the precious_dirs file + while IFS='=' read -r name dir; do + # Add an exclusion flag if the precious directory is under the git root. + if under=$(dir_under_git_root "$dir"); then + echo "Avoiding ${name}: ${under}" + exclude_flags+=( --exclude="$under" ) + fi + done < "$precious_dirs_file" +done + +git clean -xdff "${exclude_flags[@]}" diff --git a/contrib/guix/guix-verify b/contrib/guix/guix-verify new file mode 100755 index 0000000000..de158ddd0a --- /dev/null +++ b/contrib/guix/guix-verify @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the monero repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat diff gpg + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat < [ SIGNER= ] ./contrib/guix/guix-verify + +Example overriding signer's manifest to use as base + + env GUIX_SIGS_REPO=/home/user/guix.sigs SIGNER=achow101 ./contrib/guix/guix-verify + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +############## +## Verify ## +############## + +OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}" +echo "Looking for signature directories in '${OUTSIGDIR_BASE}'" +echo "" + +# Usage: verify compare_manifest current_manifest +verify() { + local compare_manifest="$1" + local current_manifest="$2" + if ! gpg --quiet --batch --verify "$current_manifest".asc "$current_manifest" 1>&2; then + echo "ERR: Failed to verify GPG signature in '${current_manifest}'" + echo "" + echo "Hint: Either the signature is invalid or the public key is missing" + echo "" + failure=1 + elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then + echo "ERR: The SHA256SUMS attestation in these two directories differ:" + echo " '${compare_manifest}'" + echo " '${current_manifest}'" + echo "" + failure=1 + else + echo "Verified: '${current_manifest}'" + echo "" + fi +} + +shopt -s nullglob +all_noncodesigned=( "$OUTSIGDIR_BASE"/*/noncodesigned.SHA256SUMS ) +shopt -u nullglob + +echo "--------------------" +echo "" +if (( ${#all_noncodesigned[@]} )); then + compare_noncodesigned="${all_noncodesigned[0]}" + if [[ -n "$SIGNER" ]]; then + signer_noncodesigned="$OUTSIGDIR_BASE/$SIGNER/noncodesigned.SHA256SUMS" + if [[ -f "$signer_noncodesigned" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_noncodesigned="$signer_noncodesigned" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi + + for current_manifest in "${all_noncodesigned[@]}"; do + verify "$compare_noncodesigned" "$current_manifest" + done + + echo "DONE: Checking output signatures for noncodesigned.SHA256SUMS" + echo "" +else + echo "WARN: No signature directories with noncodesigned.SHA256SUMS found" + echo "" +fi + +shopt -s nullglob +all_all=( "$OUTSIGDIR_BASE"/*/all.SHA256SUMS ) +shopt -u nullglob + +echo "--------------------" +echo "" +if (( ${#all_all[@]} )); then + compare_all="${all_all[0]}" + if [[ -n "$SIGNER" ]]; then + signer_all="$OUTSIGDIR_BASE/$SIGNER/all.SHA256SUMS" + if [[ -f "$signer_all" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_all="$signer_all" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi + + for current_manifest in "${all_all[@]}"; do + verify "$compare_all" "$current_manifest" + done + + # Sanity check: there should be no entries that exist in + # noncodesigned.SHA256SUMS that doesn't exist in all.SHA256SUMS + if [[ "$(comm -23 <(sort "$compare_noncodesigned") <(sort "$compare_all") | wc -c)" -ne 0 ]]; then + echo "ERR: There are unique lines in noncodesigned.SHA256SUMS which" + echo " do not exist in all.SHA256SUMS, something went very wrong." + exit 1 + fi + + echo "DONE: Checking output signatures for all.SHA256SUMS" + echo "" +else + echo "WARN: No signature directories with all.SHA256SUMS found" + echo "" +fi + +echo "====================" +echo "" +if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then + echo "ERR: Unable to perform any verifications as no signature directories" + echo " were found" + echo "" + exit 1 +fi + +if [ -n "$failure" ]; then + exit 1 +fi diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh new file mode 100644 index 0000000000..17db90f996 --- /dev/null +++ b/contrib/guix/libexec/build.sh @@ -0,0 +1,378 @@ +#!/usr/bin/env bash +# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2022-2024 The Monero Project +# Distributed under the MIT software license, see the accompanying +# file ../LICENSE.txt or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C +set -e -o pipefail +export TZ=UTC + +# shellcheck source=contrib/shell/git-utils.bash +source contrib/shell/git-utils.bash + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# environment`. It does make sense for at least `guix environment --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + VERSION: ${VERSION:?not set} + HOST: ${HOST:?not set} + COMMIT_TIMESTAMP: ${COMMIT_TIMESTAMP:?not set} + JOBS: ${JOBS:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} + LOGDIR: ${LOGDIR:?not set} + OPTIONS: ${OPTIONS} +EOF + +ACTUAL_OUTDIR="${OUTDIR}" +OUTDIR="${DISTSRC}/output" +DISTNAME="monero-${HOST}-${VERSION}" + +# Use a fixed timestamp for depends builds so hashes match across commits that don't make changes to the build system +export SOURCE_DATE_EPOCH=1397818193 + +##################### +# Environment Setup # +##################### + +# Collect some information about the build environment to help debug potential reproducibility issues +mkdir -p "${LOGDIR}" +ls -1 /gnu/store | sort > ${LOGDIR}/guix-hashes.txt +printenv | sort | grep -v '^\(BASE_CACHE=\|DISTNAME=\|DISTSRC=\|OUTDIR=\|LOGDIR=\|SOURCES_PATH=\|JOBS=\|OPTIONS=\|DEPENDS_ONLY=\)' > ${LOGDIR}/guix-env.txt + +# The depends folder also serves as a base-prefix for depends packages for +# $HOSTs after successfully building. +BASEPREFIX="${PWD}/contrib/depends" + +# Given a package name and an output name, return the path of that output in our +# current guix environment +store_path() { + grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ + | head --lines=1 \ + | sed --expression='s|\x29*$||' \ + --expression='s|^[[:space:]]*"||' \ + --expression='s|"[[:space:]]*$||' +} + + +# Set environment variables to point the NATIVE toolchain to the right +# includes/libs +NATIVE_GCC="$(store_path gcc-toolchain)" +NATIVE_GCC_STATIC="$(store_path gcc-toolchain static)" + +unset LIBRARY_PATH +unset CPATH +unset C_INCLUDE_PATH +unset CPLUS_INCLUDE_PATH +unset OBJC_INCLUDE_PATH +unset OBJCPLUS_INCLUDE_PATH + +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64:${NATIVE_GCC_STATIC}/lib:${NATIVE_GCC_STATIC}/lib64" +export C_INCLUDE_PATH="${NATIVE_GCC}/include" +export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" +export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" +export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" + +prepend_to_search_env_var() { + export "${1}=${2}${!1:+:}${!1}" +} + +# Set environment variables to point the CROSS toolchain to the right +# includes/libs for $HOST +case "$HOST" in + *mingw*) + # Determine output paths to use in CROSS_* environment variables + case "$HOST" in + i686-*) CROSS_GLIBC="$(store_path "mingw-w64-i686-winpthreads")" ;; + x86_64-*) CROSS_GLIBC="$(store_path "mingw-w64-x86_64-winpthreads")" ;; + *) exit 1 ;; + esac + + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + # The search path ordering is generally: + # 1. gcc-related search paths + # 2. libc-related search paths + # 2. kernel-header-related search paths (not applicable to mingw-w64 hosts) + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib" + ;; + *darwin*) + # The CROSS toolchain for darwin uses the SDK and ignores environment variables. + # See depends/hosts/darwin.mk for more details. + ;; + *android*) + export LD_LIBRARY_PATH="$(find /gnu/store -maxdepth 1 -name "*zlib*" | sort | head -n 1)/lib:$(find /gnu/store -maxdepth 1 -name "*gcc-11*-lib" | sort | head -n 1)/lib" + ;; + *linux-gnu*) + CROSS_GLIBC="$(store_path "glibc-cross-${HOST}")" + CROSS_GLIBC_STATIC="$(store_path "glibc-cross-${HOST}" static)" + CROSS_KERNEL="$(store_path "linux-libre-headers-cross-${HOST}")" + CROSS_GCC="$(store_path "gcc-cross-${HOST}")" + CROSS_GCC_LIB_STORE="$(store_path "gcc-cross-${HOST}" lib)" + CROSS_GCC_LIBS=( "${CROSS_GCC_LIB_STORE}/lib/gcc/${HOST}"/* ) # This expands to an array of directories... + CROSS_GCC_LIB="${CROSS_GCC_LIBS[0]}" # ...we just want the first one (there should only be one) + + export CROSS_C_INCLUDE_PATH="${CROSS_GCC_LIB}/include:${CROSS_GCC_LIB}/include-fixed:${CROSS_GLIBC}/include:${CROSS_KERNEL}/include" + export CROSS_CPLUS_INCLUDE_PATH="${CROSS_GCC}/include/c++:${CROSS_GCC}/include/c++/${HOST}:${CROSS_GCC}/include/c++/backward:${CROSS_C_INCLUDE_PATH}" + export CROSS_LIBRARY_PATH="${CROSS_GCC_LIB_STORE}/lib:${CROSS_GCC_LIB}:${CROSS_GLIBC}/lib:${CROSS_GLIBC_STATIC}/lib" + ;; + *freebsd*) + ;; + *) + exit 1 ;; +esac + + +# Sanity check CROSS_*_PATH directories +IFS=':' read -ra PATHS <<< "${CROSS_C_INCLUDE_PATH}:${CROSS_CPLUS_INCLUDE_PATH}:${CROSS_LIBRARY_PATH}" +for p in "${PATHS[@]}"; do + if [ -n "$p" ] && [ ! -d "$p" ]; then + echo "'$p' doesn't exist or isn't a directory... Aborting..." + exit 1 + fi +done + +# Disable Guix ld auto-rpath behavior +case "$HOST" in + *darwin*) + # The auto-rpath behavior is necessary for darwin builds as some native + # tools built by depends refer to and depend on Guix-built native + # libraries + # + # After the native packages in depends are built, the ld wrapper should + # no longer affect our build, as clang would instead reach for + # x86_64-apple-darwin-ld from cctools + ;; + *android*) + ;; + *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;; +esac + +# Make /usr/bin if it doesn't exist +[ -e /usr/bin ] || mkdir -p /usr/bin +[ -e /lib64 ] || mkdir /lib64 + +# Symlink file and env to a conventional path +[ -e /usr/bin/file ] || ln -s --no-dereference "$(command -v file)" /usr/bin/file +[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env +[ -e /bin/bash ] || ln -s --no-dereference "$(command -v bash)" /bin/bash +[ -e /bin/sh ] || ln -s --no-dereference "$(command -v sh)" /bin/sh +[ -e /lib64/ld-linux-x86-64.so.2 ] || ln -s --no-dereference "${NATIVE_GCC}/lib/ld-linux-x86-64.so.2" /lib64/ld-linux-x86-64.so.2 + +# Determine the correct value for -Wl,--dynamic-linker for the current $HOST +case "$HOST" in + *linux-gnu*) + glibc_dynamic_linker=$( + case "$HOST" in + x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; + arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; + aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; + riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; + i686-linux-gnu) echo /lib/ld-linux.so.2 ;; + *) exit 1 ;; + esac + ) + ;; +esac + +export GLIBC_DYNAMIC_LINKER=${glibc_dynamic_linker} + +# Environment variables for determinism +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ="UTC" +case "$HOST" in + *darwin*) + # cctools AR, unlike GNU binutils AR, does not have a deterministic mode + # or a configure flag to enable determinism by default, it only + # understands if this env-var is set or not. See: + # + # https://github.com/tpoechtrager/cctools-port/blob/55562e4073dea0fbfd0b20e0bf69ffe6390c7f97/cctools/ar/archive.c#L334 + export ZERO_AR_DATE=yes + ;; +esac + +#################### +# Depends Building # +#################### + +# LDFLAGS +case "$HOST" in + *linux-gnu*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; + *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; +esac + +mkdir -p "${OUTDIR}" + +# Log the depends build ids +make -C contrib/depends --no-print-directory HOST="$HOST" print-final_build_id_long | tr ':' '\n' > ${LOGDIR}/depends-hashes.txt + +# Build the depends tree, overriding variables that assume multilib gcc +make -C contrib/depends --jobs="$JOBS" HOST="$HOST" \ + ${V:+V=1} \ + ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ + ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ + ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ + OUTDIR="$OUTDIR" \ + LOGDIR="$LOGDIR" \ + x86_64_linux_CC=x86_64-linux-gnu-gcc \ + x86_64_linux_CXX=x86_64-linux-gnu-g++ \ + x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ + x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ + x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ + x86_64_linux_STRIP=x86_64-linux-gnu-strip + +# Log the depends package hashes +DEPENDS_PACKAGES="$(make -C contrib/depends --no-print-directory HOST="$HOST" print-all_packages)" +DEPENDS_CACHE="$(make -C contrib/depends --no-print-directory ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} print-BASE_CACHE)" + +{ + for package in ${DEPENDS_PACKAGES}; do + cat "${DEPENDS_CACHE}/${HOST}/${package}"/*.hash + done +} | sort -k2 > "${LOGDIR}/depends-packages.txt" + +# Stop here if we're only building depends packages +if [[ -n "$DEPENDS_ONLY" ]]; then + exit 0 +fi + +########################### +# Source Tarball Building # +########################### + +# Use COMMIT_TIMESTAMP for the source and release binary archives +export SOURCE_DATE_EPOCH=${COMMIT_TIMESTAMP} +export TAR_OPTIONS="--owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" + +GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/monero-source-${VERSION}.tar.gz" + +# Create the source tarball if not already there +if [ ! -e "$GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git ls-files --recurse-submodules \ + | cat \ + | sort \ + | tar --create --transform "s,^,monero-source-${VERSION}/," --mode='u+rw,go+r-w,a+X' --files-from=- \ + | gzip -9n > ${GIT_ARCHIVE} + sha256sum "$GIT_ARCHIVE" +fi + +########################### +# Binary Tarball Building # +########################### + +# CFLAGS +HOST_CFLAGS="-O2" +HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) +case "$HOST" in + *linux-gnu*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; + *darwin*) unset HOST_CFLAGS ;; + *android*) unset HOST_CFLAGS ;; +esac + +# CXXFLAGS +HOST_CXXFLAGS="$HOST_CFLAGS" + +case "$HOST" in + arm-linux-gnueabihf) HOST_CXXFLAGS+=" -Wno-psabi" ;; +esac + +export GIT_DISCOVERY_ACROSS_FILESYSTEM=1 +export USE_DEVICE_TREZOR_MANDATORY=1 + +# Make $HOST-specific native binaries from depends available in $PATH +export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" +mkdir -p "$DISTSRC" +( + cd "$DISTSRC" + + # Extract the source tarball + tar --strip-components=1 -xf "${GIT_ARCHIVE}" + + # Setup the directory where our Monero build for HOST will be + # installed. This directory will also later serve as the input for our + # binary tarballs. + INSTALLPATH="${DISTSRC}/installed/${DISTNAME}" + mkdir -p "${INSTALLPATH}" + + # Configure this DISTSRC for $HOST + # shellcheck disable=SC2086 + env CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" \ + cmake --toolchain "${BASEPREFIX}/${HOST}/share/toolchain.cmake" -S . -B build \ + -DCMAKE_INSTALL_PREFIX="${INSTALLPATH}" \ + -DCMAKE_EXE_LINKER_FLAGS="${HOST_LDFLAGS}" \ + -DCMAKE_SHARED_LINKER_FLAGS="${HOST_LDFLAGS}" \ + -DCMAKE_SKIP_RPATH=ON \ + -DMANUAL_SUBMODULES=1 + + make -C build --jobs="$JOBS" + + # Copy docs + cp README.md LICENSE docs/ANONYMITY_NETWORKS.md "${INSTALLPATH}" + + # Copy binaries + cp -a build/bin/* "${INSTALLPATH}" + + ( + cd installed + + # Finally, deterministically produce binary tarballs ready for release + case "$HOST" in + *mingw*) + find "${DISTNAME}/" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}/" \ + | sort \ + | zip -X@ "${OUTDIR}/${DISTNAME}.zip" \ + || ( rm -f "${OUTDIR}/${DISTNAME}.zip" && exit 1 ) + ;; + *) + find "${DISTNAME}/" -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" + find "${DISTNAME}/" \ + | sort \ + | tar --no-recursion --owner=0 --group=0 -c -T - \ + | bzip2 -9 > "${OUTDIR}/${DISTNAME}.tar.bz2" \ + || ( rm -f "${OUTDIR}/${DISTNAME}.tar.bz2" && exit 1 ) + ;; + esac + ) +) # $DISTSRC + +rm -rf "$ACTUAL_OUTDIR" +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ + || ( rm -rf "$ACTUAL_OUTDIR" && exit 1 ) + +( + cd /outdir-base + { + echo "$GIT_ARCHIVE" + find "$ACTUAL_OUTDIR" -type f + } | xargs realpath --relative-base="$PWD" \ + | xargs sha256sum \ + | sort -k2 \ + | sponge "$LOGDIR"/SHA256SUMS.part +) diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash new file mode 100644 index 0000000000..30c2c07d59 --- /dev/null +++ b/contrib/guix/libexec/prelude.bash @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# shellcheck source=contrib/shell/realpath.bash +source contrib/shell/realpath.bash + +# shellcheck source=contrib/shell/git-utils.bash +source contrib/shell/git-utils.bash + +################ +# Required non-builtin commands should be invocable +################ + +check_tools() { + for cmd in "$@"; do + if ! command -v "$cmd" > /dev/null 2>&1; then + echo "ERR: This script requires that '$cmd' is installed and available in your \$PATH" + exit 1 + fi + done +} + +check_tools cat env readlink dirname basename git + +################ +# We should be at the top directory of the repository +################ + +same_dir() { + local resolved1 resolved2 + resolved1="$(bash_realpath "${1}")" + resolved2="$(bash_realpath "${2}")" + [ "$resolved1" = "$resolved2" ] +} + +if ! same_dir "${PWD}" "$(git_root)"; then +cat << EOF +ERR: This script must be invoked from the top level of the git repository + +Hint: This may look something like: + env FOO=BAR ./contrib/guix/guix- + +EOF +exit 1 +fi + +################ +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://github.com/tobtoht/guix.git \ + --commit=d5ca4d4fd713a9f7e17e074a1e37dda99bbb09fc \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + + +################ +# Set common variables +################ + +VERSION="${FORCE_VERSION:-$(git_head_version)}" + +VERSION_BASE_DIR="${VERSION_BASE_DIR:-${PWD}}" +version_base_prefix="${VERSION_BASE_DIR}/guix/guix-build-" +VERSION_BASE="${version_base_prefix}${VERSION}" # TOP + +DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}" + +OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" + +LOGDIR_BASE="${LOGDIR_BASE:-${VERSION_BASE}/logs}" + +var_base_basename="var" +VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}" + +profiles_base_basename="profiles" +PROFILES_BASE="${PROFILES_BASE:-${VAR_BASE}/${profiles_base_basename}}" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm new file mode 100644 index 0000000000..70ca584107 --- /dev/null +++ b/contrib/guix/manifest.scm @@ -0,0 +1,271 @@ +(use-modules (gnu packages) + (gnu packages autotools) + (gnu packages bash) + ((gnu packages cmake) #:select (cmake-minimal)) + (gnu packages commencement) + (gnu packages compression) + (gnu packages cross-base) + ((gnu packages elf) #:select (patchelf)) + (gnu packages file) + (gnu packages gawk) + (gnu packages gcc) + ((gnu packages gettext) #:select (gettext-minimal)) + (gnu packages gperf) + ((gnu packages libusb) #:select (libplist)) + ((gnu packages linux) #:select (linux-libre-headers-6.1 util-linux)) + (gnu packages llvm) + (gnu packages mingw) + (gnu packages moreutils) + (gnu packages perl) + (gnu packages pkg-config) + ((gnu packages python) #:select (python-minimal)) + ((gnu packages tls) #:select (openssl)) + ((gnu packages version-control) #:select (git-minimal)) + (guix build-system gnu) + (guix build-system trivial) + (guix download) + (guix gexp) + (guix git-download) + ((guix licenses) #:prefix license:) + (guix packages) + ((guix utils) #:select (substitute-keyword-arguments))) + +(define-syntax-rule (search-our-patches file-name ...) + "Return the list of absolute file names corresponding to each +FILE-NAME found in ./patches relative to the current file." + (parameterize + ((%patch-path (list (string-append (dirname (current-filename)) "/patches")))) + (list (search-patch file-name) ...))) + +(define building-on (string-append "--build=" (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) + +(define (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + ;; 1. Build a cross-compiling gcc without targeting any libc, derived + ;; from BASE-GCC-FOR-LIBC + (xgcc-sans-libc (cross-gcc target + #:xgcc base-gcc-for-libc + #:xbinutils xbinutils)) + ;; 2. Build cross-compiled kernel headers with XGCC-SANS-LIBC, derived + ;; from BASE-KERNEL-HEADERS + (xkernel (cross-kernel-headers target + #:linux-headers base-kernel-headers + #:xgcc xgcc-sans-libc + #:xbinutils xbinutils)) + ;; 3. Build a cross-compiled libc with XGCC-SANS-LIBC and XKERNEL, + ;; derived from BASE-LIBC + (xlibc (cross-libc target + #:libc base-libc + #:xgcc xgcc-sans-libc + #:xbinutils xbinutils + #:xheaders xkernel)) + ;; 4. Build a cross-compiling gcc targeting XLIBC, derived from + ;; BASE-GCC + (xgcc (cross-gcc target + #:xgcc base-gcc + #:xbinutils xbinutils + #:libc xlibc))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-toolchain")) + (version (package-version xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + (list xbinutils + xlibc + xgcc + `(,xlibc "static") + `(,xgcc "lib"))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page xgcc)) + (license (package-license xgcc))))) + +(define base-gcc gcc-10) +(define base-linux-kernel-headers linux-libre-headers-6.1) + +(define* (make-monero-cross-toolchain target + #:key + (base-gcc-for-libc linux-base-gcc) + (base-kernel-headers base-linux-kernel-headers) + (base-libc glibc-2.27) + (base-gcc linux-base-gcc)) + "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values +desirable for building Monero release binaries." + (make-cross-toolchain target + base-gcc-for-libc + base-kernel-headers + base-libc + base-gcc)) + +(define (gcc-mingw-patches gcc) + (package-with-extra-patches gcc + (search-our-patches "gcc-remap-guix-store.patch" + "gcc-fix-win32-vsnprintf.patch"))) + +(define (make-mingw-pthreads-cross-toolchain target) + "Create a cross-compilation toolchain package for TARGET" + (let* ((xbinutils (cross-binutils target)) + (pthreads-xlibc (cond ((string-prefix? "i686-" target) + mingw-w64-i686-winpthreads) + (else mingw-w64-x86_64-winpthreads))) + (pthreads-xgcc (cross-gcc target + #:xgcc (gcc-mingw-patches mingw-w64-base-gcc) + #:xbinutils xbinutils + #:libc pthreads-xlibc))) + ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and + ;; XGCC + (package + (name (string-append target "-posix-toolchain")) + (version (package-version pthreads-xgcc)) + (source #f) + (build-system trivial-build-system) + (arguments '(#:builder (begin (mkdir %output) #t))) + (propagated-inputs + (list xbinutils + pthreads-xlibc + pthreads-xgcc + `(,pthreads-xgcc "lib"))) + (synopsis (string-append "Complete GCC tool chain for " target)) + (description (string-append "This package provides a complete GCC tool +chain for " target " development.")) + (home-page (package-home-page pthreads-xgcc)) + (license (package-license pthreads-xgcc))))) + +(define-public mingw-w64-base-gcc + (package + (inherit base-gcc) + (arguments + (substitute-keyword-arguments (package-arguments base-gcc) + ((#:configure-flags flags) + `(append ,flags + ;; https://gcc.gnu.org/install/configure.html + (list "--enable-threads=posix", + building-on))))))) + +(define-public linux-base-gcc + (package + (inherit base-gcc) + (arguments + (substitute-keyword-arguments (package-arguments base-gcc) + ((#:configure-flags flags) + `(append ,flags + ;; https://gcc.gnu.org/install/configure.html + (list "--enable-initfini-array=yes", + "--enable-default-ssp=yes", + "--enable-default-pie=yes", + building-on))) + ((#:phases phases) + `(modify-phases ,phases + ;; Given a XGCC package, return a modified package that replace each instance of + ;; -rpath in the default system spec that's inserted by Guix with -rpath-link + (add-after 'pre-configure 'replace-rpath-with-rpath-link + (lambda _ + (substitute* (cons "gcc/config/rs6000/sysv4.h" + (find-files "gcc/config" + "^gnu-user.*\\.h$")) + (("-rpath=") "-rpath-link=")) + #t)))))))) + +(define-public glibc-2.27 + (package + (inherit glibc-2.31) + (version "2.27") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit "73886db6218e613bd6d4edf529f11e008a6c2fa6"))) + (file-name (git-file-name "glibc" "73886db6218e613bd6d4edf529f11e008a6c2fa6")) + (sha256 + (base32 + "0azpb9cvnbv25zg8019rqz48h8i2257ngyjg566dlnp74ivrs9vq")) + (patches (search-our-patches "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" + "glibc-2.27-guix-prefix.patch" + "glibc-2.27-no-librt.patch")))) + (arguments + (substitute-keyword-arguments (package-arguments glibc) + ((#:configure-flags flags) + `(append ,flags + ;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html + (list "--enable-stack-protector=all", + "--enable-bind-now", + "--disable-werror", + building-on))) + ((#:phases phases) + `(modify-phases ,phases + (add-before 'configure 'set-etc-rpc-installation-directory + (lambda* (#:key outputs #:allow-other-keys) + ;; Install the rpc data base file under `$out/etc/rpc'. + ;; Otherwise build will fail with "Permission denied." + (let ((out (assoc-ref outputs "out"))) + (substitute* "sunrpc/Makefile" + (("^\\$\\(inst_sysconfdir\\)/rpc(.*)$" _ suffix) + (string-append out "/etc/rpc" suffix "\n")) + (("^install-others =.*$") + (string-append "install-others = " out "/etc/rpc\n")))))))))))) + +(packages->manifest + (append + (list ;; The Basics + bash + which + coreutils-minimal + util-linux + ;; File(system) inspection + file + grep + diffutils + findutils + ;; File transformation + patch + gawk + sed + moreutils + patchelf + ;; Compression and archiving + tar + bzip2 + gzip + xz + p7zip + zip + unzip + ;; Build tools + gnu-make + libtool + autoconf-2.71 + automake + pkg-config + gperf + gettext-minimal + cmake-minimal + ;; Native GCC 10 toolchain + gcc-toolchain-10 + (list gcc-toolchain-10 "static") + ;; Scripting + perl + python-minimal + ;; Git + git-minimal + ) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + ;; Windows + (list (make-mingw-pthreads-cross-toolchain target))) + ((string-contains target "-linux-gnu") + (list (make-monero-cross-toolchain target))) + ((string-contains target "freebsd") + (list clang-toolchain-11 binutils)) + ((string-contains target "darwin") + (list clang-toolchain-11 binutils)) + (else '()))))) diff --git a/contrib/guix/patches/gcc-fix-win32-vsnprintf.patch b/contrib/guix/patches/gcc-fix-win32-vsnprintf.patch new file mode 100644 index 0000000000..33d6a4ae10 --- /dev/null +++ b/contrib/guix/patches/gcc-fix-win32-vsnprintf.patch @@ -0,0 +1,83 @@ +https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105935 +We can remove this patch when we drop support for i686-w64-mingw32 + +diff --git a/libgfortran/configure.ac b/libgfortran/configure.ac +index 8bd2af966c8..655858ebb15 100644 +--- a/libgfortran/configure.ac ++++ b/libgfortran/configure.ac +@@ -321,8 +321,6 @@ if test "${hardwire_newlib:-0}" -eq 1; then + # link executables. + AC_DEFINE(HAVE_MKSTEMP, 1, [Define if you have mkstemp.]) + AC_DEFINE(HAVE_STRTOF, 1, [Define if you have strtof.]) +- AC_DEFINE(HAVE_SNPRINTF, 1, [Define if you have snprintf.]) +- AC_DEFINE(HAVE_VSNPRINTF, 1, [Define if you have vsnprintf.]) + AC_DEFINE(HAVE_LOCALTIME_R, 1, [Define if you have localtime_r.]) + AC_DEFINE(HAVE_GMTIME_R, 1, [Define if you have gmtime_r.]) + AC_DEFINE(HAVE_STRNLEN, 1, [Define if you have strnlen.]) +@@ -335,11 +333,11 @@ if test "${hardwire_newlib:-0}" -eq 1; then + AC_DEFINE(HAVE_STRTOLD, 1, [Define if you have strtold.]) + fi + else +- AC_CHECK_FUNCS_ONCE(getrusage times mkstemp strtof strtold snprintf \ ++ AC_CHECK_FUNCS_ONCE(getrusage times mkstemp strtof strtold \ + ftruncate chsize chdir getentropy getlogin gethostname kill link symlink \ + sleep ttyname sigaction waitpid \ + alarm access fork posix_spawn setmode fcntl writev \ +- gettimeofday stat fstat lstat getpwuid vsnprintf dup \ ++ gettimeofday stat fstat lstat getpwuid dup \ + getcwd localtime_r gmtime_r getpwuid_r ttyname_r clock_gettime \ + getgid getpid getuid geteuid umask getegid \ + secure_getenv __secure_getenv mkostemp strnlen strndup newlocale \ +diff --git a/libgfortran/libgfortran.h b/libgfortran/libgfortran.h +index 6d92089e34b..b79718bb113 100644 +--- a/libgfortran/libgfortran.h ++++ b/libgfortran/libgfortran.h +@@ -155,10 +155,6 @@ extern int __mingw_snprintf (char *, size_t, const char *, ...) + __attribute__ ((format (gnu_printf, 3, 4))); + #undef snprintf + #define snprintf __mingw_snprintf +-/* Fallback to sprintf if target does not have snprintf. */ +-#elif !defined(HAVE_SNPRINTF) +-#undef snprintf +-#define snprintf(str, size, ...) sprintf (str, __VA_ARGS__) + #endif + + +diff --git a/libgfortran/runtime/error.c b/libgfortran/runtime/error.c +index bc811c0387b..ccfdcda1c2f 100644 +--- a/libgfortran/runtime/error.c ++++ b/libgfortran/runtime/error.c +@@ -136,33 +136,6 @@ estr_writev (const struct iovec *iov, int iovcnt) + } + + +-#ifndef HAVE_VSNPRINTF +-static int +-gf_vsnprintf (char *str, size_t size, const char *format, va_list ap) +-{ +- int written; +- +- written = vsprintf(buffer, format, ap); +- +- if (written >= size - 1) +- { +- /* The error message was longer than our buffer. Ouch. Because +- we may have messed up things badly, report the error and +- quit. */ +-#define ERROR_MESSAGE "Internal error: buffer overrun in gf_vsnprintf()\n" +- write (STDERR_FILENO, buffer, size - 1); +- write (STDERR_FILENO, ERROR_MESSAGE, strlen (ERROR_MESSAGE)); +- sys_abort (); +-#undef ERROR_MESSAGE +- +- } +- return written; +-} +- +-#define vsnprintf gf_vsnprintf +-#endif +- +- + /* printf() like function for for printing to stderr. Uses a stack + allocated buffer and doesn't lock stderr, so it should be safe to + use from within a signal handler. */ diff --git a/contrib/guix/patches/gcc-remap-guix-store.patch b/contrib/guix/patches/gcc-remap-guix-store.patch new file mode 100644 index 0000000000..a47ef7a2df --- /dev/null +++ b/contrib/guix/patches/gcc-remap-guix-store.patch @@ -0,0 +1,25 @@ +From aad25427e74f387412e8bc9a9d7bbc6c496c792f Mon Sep 17 00:00:00 2001 +From: Andrew Chow +Date: Wed, 6 Jul 2022 16:49:41 -0400 +Subject: [PATCH] guix: remap guix store paths to /usr + +--- + libgcc/Makefile.in | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/libgcc/Makefile.in b/libgcc/Makefile.in +index 851e7657d07..476c2becd1c 100644 +--- a/libgcc/Makefile.in ++++ b/libgcc/Makefile.in +@@ -854,7 +854,7 @@ endif + # libgcc_eh.a, only LIB2ADDEH matters. If we do, only LIB2ADDEHSTATIC and + # LIB2ADDEHSHARED matter. (Usually all three are identical.) + +-c_flags := -fexceptions ++c_flags := -fexceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + + ifeq ($(enable_shared),yes) + +-- +2.37.0 + diff --git a/contrib/guix/patches/glibc-2.27-guix-prefix.patch b/contrib/guix/patches/glibc-2.27-guix-prefix.patch new file mode 100644 index 0000000000..6648bc6c05 --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-guix-prefix.patch @@ -0,0 +1,22 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +We might be able to drop this in favour of using --with-nonshared-cflags +when we being using newer versions of glibc. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -992,6 +992,10 @@ object-suffixes := + CPPFLAGS-.o = $(pic-default) + # libc.a must be compiled with -fPIE/-fpie for static PIE. + CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) ++ ++# Map Guix store paths to /usr ++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + libtype.o := lib%.a + object-suffixes += .o + ifeq (yes,$(build-shared)) diff --git a/contrib/guix/patches/glibc-2.27-no-librt.patch b/contrib/guix/patches/glibc-2.27-no-librt.patch new file mode 100644 index 0000000000..4f2092ba7e --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-no-librt.patch @@ -0,0 +1,53 @@ +This patch can be dropped when we are building with glibc 2.30+. + +commit 6e41ef56c9baab719a02f1377b1e7ce7bff61e73 +Author: Florian Weimer +Date: Fri Feb 8 10:21:56 2019 +0100 + + rt: Turn forwards from librt to libc into compat symbols [BZ #24194] + + As the result of commit 6e6249d0b461b952d0f544792372663feb6d792a + ("BZ#14743: Move clock_* symbols from librt to libc."), in glibc 2.17, + clock_gettime, clock_getres, clock_settime, clock_getcpuclockid, + clock_nanosleep were added to libc, and the file rt/clock-compat.c + was added with forwarders to the actual implementations in libc. + These forwarders were wrapped in + + #if SHLIB_COMPAT (librt, GLIBC_2_2, GLIBC_2_17) + + so that they are not present for newer architectures (such as + powerpc64le) with a 2.17 or later ABI baseline. But the forwarders + were not marked as compatibility symbols. As a result, on older + architectures, historic configure checks such as + + AC_CHECK_LIB(rt, clock_gettime) + + still cause linking against librt, even though this is completely + unnecessary. It also creates a needless porting hazard because + architectures behave differently when it comes to symbol availability. + + Reviewed-by: Carlos O'Donell + +diff --git a/rt/clock-compat.c b/rt/clock-compat.c +index f816973c05..11e71aa890 100644 +--- a/rt/clock-compat.c ++++ b/rt/clock-compat.c +@@ -30,14 +30,16 @@ + #if HAVE_IFUNC + # undef INIT_ARCH + # define INIT_ARCH() +-# define COMPAT_REDIRECT(name, proto, arglist) libc_ifunc (name, &__##name) ++# define COMPAT_REDIRECT(name, proto, arglist) libc_ifunc (name, &__##name) \ ++ compat_symbol (librt, name, name, GLIBC_2_2); + #else + # define COMPAT_REDIRECT(name, proto, arglist) \ + int \ + name proto \ + { \ + return __##name arglist; \ +- } ++ } \ ++ compat_symbol (librt, name, name, GLIBC_2_2); + #endif + + COMPAT_REDIRECT (clock_getres, diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch new file mode 100644 index 0000000000..c0f8495c41 --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch @@ -0,0 +1,76 @@ +Note that this has been modified from the original commit, to use __has_include +instead of __has_include__, as the later was causing build failures with GCC 10. +See also: http://lists.busybox.net/pipermail/buildroot/2020-July/590376.html. + +https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e + +From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" +Date: Mon, 31 Dec 2018 09:26:42 -0800 +Subject: [PATCH] riscv: Use __has_include__ to include [BZ + #24022] + + has been removed by + +commit 27f8899d6002e11a6e2d995e29b8deab5aa9cc25 +Author: David Abdurachmanov +Date: Thu Nov 8 20:02:39 2018 +0100 + + riscv: add asm/unistd.h UAPI header + + Marcin Juszkiewicz reported issues while generating syscall table for riscv + using 4.20-rc1. The patch refactors our unistd.h files to match some other + architectures. + + - Add asm/unistd.h UAPI header, which has __ARCH_WANT_NEW_STAT only for 64-bit + - Remove asm/syscalls.h UAPI header and merge to asm/unistd.h + - Adjust kernel asm/unistd.h + + So now asm/unistd.h UAPI header should show all syscalls for riscv. + + may be restored by + +Subject: [PATCH] riscv: restore asm/syscalls.h UAPI header +Date: Tue, 11 Dec 2018 09:09:35 +0100 + +UAPI header asm/syscalls.h was merged into UAPI asm/unistd.h header, +which did resolve issue with missing syscalls macros resulting in +glibc (2.28) build failure. It also broke glibc in a different way: +asm/syscalls.h is being used by glibc. I noticed this while doing +Fedora 30/Rawhide mass rebuild. + +The patch returns asm/syscalls.h header and incl. it into asm/unistd.h. +I plan to send a patch to glibc to use asm/unistd.h instead of +asm/syscalls.h + +In the meantime, we use __has_include__, which was added to GCC 5, to +check if exists before including it. Tested with +build-many-glibcs.py for riscv against kernel 4.19.12 and 4.20-rc7. + + [BZ #24022] + * sysdeps/unix/sysv/linux/riscv/flush-icache.c: Check if + exists with __has_include__ before including it. +--- + sysdeps/unix/sysv/linux/riscv/flush-icache.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/sysdeps/unix/sysv/linux/riscv/flush-icache.c b/sysdeps/unix/sysv/linux/riscv/flush-icache.c +index d612ef4c6c..0b2042620b 100644 +--- a/sysdeps/unix/sysv/linux/riscv/flush-icache.c ++++ b/sysdeps/unix/sysv/linux/riscv/flush-icache.c +@@ -21,7 +21,11 @@ + #include + #include + #include +-#include ++#if __has_include () ++# include ++#else ++# include ++#endif + + typedef int (*func_type) (void *, void *, unsigned long int); + +-- +2.31.1 + diff --git a/contrib/shell/git-utils.bash b/contrib/shell/git-utils.bash new file mode 100644 index 0000000000..0bca2d2438 --- /dev/null +++ b/contrib/shell/git-utils.bash @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +git_root() { + git rev-parse --show-toplevel 2> /dev/null +} + +git_head_version() { + local recent_tag + if recent_tag="$(git describe --exact-match HEAD 2> /dev/null)"; then + echo "${recent_tag}" + else + git rev-parse --short=12 HEAD + fi +} diff --git a/contrib/shell/realpath.bash b/contrib/shell/realpath.bash new file mode 100644 index 0000000000..389b77b562 --- /dev/null +++ b/contrib/shell/realpath.bash @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Based on realpath.sh written by Michael Kropat +# Found at: https://github.com/mkropat/sh-realpath/blob/65512368b8155b176b67122aa395ac580d9acc5b/realpath.sh + +bash_realpath() { + canonicalize_path "$(resolve_symlinks "$1")" +} + +resolve_symlinks() { + _resolve_symlinks "$1" +} + +_resolve_symlinks() { + _assert_no_path_cycles "$@" || return + + local dir_context path + if path=$(readlink -- "$1"); then + dir_context=$(dirname -- "$1") + _resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@" + else + printf '%s\n' "$1" + fi +} + +_prepend_dir_context_if_necessary() { + if [ "$1" = . ]; then + printf '%s\n' "$2" + else + _prepend_path_if_relative "$1" "$2" + fi +} + +_prepend_path_if_relative() { + case "$2" in + /* ) printf '%s\n' "$2" ;; + * ) printf '%s\n' "$1/$2" ;; + esac +} + +_assert_no_path_cycles() { + local target path + + target=$1 + shift + + for path in "$@"; do + if [ "$path" = "$target" ]; then + return 1 + fi + done +} + +canonicalize_path() { + if [ -d "$1" ]; then + _canonicalize_dir_path "$1" + else + _canonicalize_file_path "$1" + fi +} + +_canonicalize_dir_path() { + (cd "$1" 2>/dev/null && pwd -P) +} + +_canonicalize_file_path() { + local dir file + dir=$(dirname -- "$1") + file=$(basename -- "$1") + (cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file") +} diff --git a/docs/RELEASE_CHECKLIST.md b/docs/RELEASE_CHECKLIST.md index cfd04f98d0..b36fa660fc 100644 --- a/docs/RELEASE_CHECKLIST.md +++ b/docs/RELEASE_CHECKLIST.md @@ -41,9 +41,7 @@ - [ ] https://miningpoolstats.stream/monero - [ ] Release branch created - [ ] Update src/version.cpp.in with new version AND new name (if necessary) - - [ ] Update Gitian YML files in contrib/gitian/ to the new version number - [ ] Update README.md with new fork table entry (or at least update the Recommended Monero version) - - [ ] Update contrib/gitian/README.md so that the instructions reflect the current version - [ ] Update src/checkpoints/checkpoints.cpp with a recent hardcoded checkpoint - [ ] Update src/blocks/checkpoints.dat with ./monero-blockchain-export --output-file checkpoints.dat --block-stop --blocksdat - [ ] Update expected_block_hashes_hash in src/cryptonote_core/blockchain.cpp with checkpoints.dat sha256 hash