Change:
There is an issue in the test by . This line introduces a dependency on the contents of nixpkgs, making every change evaluate with a different hash. cc
Steps to reproduce:
You're absolutely right, but I don't think I see the impact of this fact. It's not recursed into (no /) and it's not even built by ofborg. The test needs to compare results, but perhaps the actual evaluation can be removed from it; injected as store paths by the test runner instead. It won't make the test easier to read :(
This currently impacts every ofborg evaluation. For example:
I have no idea how to solve this. Why would ofborg have a different hash for multiple evaluations? Cause it puts stuff in the nixpkgs path?
is the whole Nixpkgs source tree, which kinda always changes. Here's a solution . Not pretty but still quick to run outside the sandbox and solves this issue.
Still an issue ?
Ofborg doesn't rebase PRs so if the PR is based on a master that's of the wrong age, you'll still see this happen. Please check that the PR includes the improved test.
I did, and it's been more than 12 hours since the commit landed. At the same time my original repro doesn't see the issue:
It's a broader issue that applies to all NixOS tests, , but it went under the radar.
This seems to be fixed now. Thanks for looking into this!

Discussion:
<del> { lib, nixosTest, path, writeText, hello, figlet, stdenvNoCC }: </del> <ins> { lib, nixosTest, pkgs, writeText, hello, figlet, stdenvNoCC }: # -------------------------------------------------------------------------- # # #                         trivial-builders test # # -------------------------------------------------------------------------- # # #  This file can be run independently (quick): # #      $ pkgs/build-support/trivial-builders/test.sh # #  or in the build sandbox with a ~20s VM overhead # #      $ nix-build -A tests.trivial-builders # # -------------------------------------------------------------------------- # let invokeSamples = file: lib.concatStringsSep " " ( lib.attrValues (import file { inherit pkgs; }) ); in </ins> nixosTest { name = "nixpkgs-trivial-builders"; nodes.machine = { ... }: {

environment.etc."pre-built-paths".source = writeText "pre-built-paths" ( builtins.toJSON [hello figlet stdenvNoCC] ); <ins> environment.variables = { SAMPLE = invokeSamples ./test/sample.nix; REFERENCES = invokeSamples ./test/invoke-writeReferencesToFile.nix; DIRECT_REFS = invokeSamples ./test/invoke-writeDirectReferencesToFile.nix; }; </ins> }; testScript = '' machine.succeed(""" <del> cd ${lib.cleanSource path} ./pkgs/build-support/trivial-builders/test.sh 2>/dev/console </del> <ins> ${./test.sh} 2>/dev/console </ins> """) ''; <ins> meta = { license = lib.licenses.mit; # nixpkgs license maintainers = with lib.maintainers; [ roberth ]; description = "Run the Nixpkgs trivial builders tests"; }; </ins> }

cd "$(dirname ${BASH_SOURCE[0]})"  # nixpkgs root <del> testDirectReferences() { expr="$1" </del> <ins> if [[ -z ${SAMPLE:-} ]]; then sample=( `nix-build test/sample.nix` ) directRefs=( `nix-build test/invoke-writeDirectReferencesToFile.nix` ) references=( `nix-build test/invoke-writeReferencesToFile.nix` ) else # Injected by Nix (to avoid evaluating in a derivation) # turn them into arrays sample=($SAMPLE) directRefs=($DIRECT_REFS) references=($REFERENCES) fi echo >&2 Testing direct references... for i in "${!sample[@]}"; do echo >&2 Checking '#'$i ${sample[$i]} ${directRefs[$i]} </ins> diff -U3  <del> <(sort <$(nix-build --no-out-link --expr "with import ../../.. {}; writeDirectReferencesToFile ($expr)"))  <(nix-store -q --references $(nix-build --no-out-link --expr "with import ../../.. {}; ($expr)") | sort) } </del> <ins> <(sort <${directRefs[$i]})  <(nix-store -q --references ${sample[$i]} | sort) done </ins> <del> testDirectReferences 'hello' testDirectReferences 'figlet' testDirectReferences 'writeText "hi" "hello"' testDirectReferences 'writeText "hi" "hello ${hello}"' testDirectReferences 'writeText "hi" "hello ${hello} ${figlet}"' testClosure() { expr="$1" </del> <ins> echo >&2 Testing closure... for i in "${!sample[@]}"; do echo >&2 Checking '#'$i ${sample[$i]} ${references[$i]} </ins> diff -U3  <del> <(sort <$(nix-build --no-out-link --expr "with import ../../.. {}; writeReferencesToFile ($expr)"))  <(nix-store -q --requisites $(nix-build --no-out-link --expr "with import ../../.. {}; ($expr)") | sort) } testClosure 'hello' testClosure 'figlet' testClosure 'writeText "hi" "hello"' testClosure 'writeText "hi" "hello ${hello}"' testClosure 'writeText "hi" "hello ${hello} ${figlet}"' </del> <ins> <(sort <${references[$i]})  <(nix-store -q --requisites ${sample[$i]} | sort) done </ins> echo 'OK!'

<ins> { pkgs ? import ../../../.. { config = {}; overlays = []; } }: pkgs.lib.mapAttrs (k: v: pkgs.writeDirectReferencesToFile v) (import ./sample.nix { inherit pkgs; }) </ins>

<ins> { pkgs ? import ../../../.. { config = {}; overlays = []; } }: pkgs.lib.mapAttrs (k: v: pkgs.writeReferencesToFile v) (import ./sample.nix { inherit pkgs; }) </ins>

<ins> { pkgs ? import ../../../.. { config = {}; overlays = []; } }: let inherit (pkgs) figlet hello writeText ; in { hello = hello; figlet = figlet; norefs = writeText "hi" "hello"; helloRef = writeText "hi" "hello ${hello}"; helloFigletRef = writeText "hi" "hello ${hello} ${figlet}"; } </ins>