Change:
Project description is like curl, but doesn't get you blocked by cloudflare and other WAPs that insist you have the TLS fingerprint of a web browser. Metadata homepage URL: source URL: license: MIT platforms: linux
the build process produces a separate 'Chrome' and 'Firefox' impersonation binary
I started packaging the firefox variant of curl-impersonate in , but it has unresolved issues, including: missing the curl-impersonate wrapper scripts; PATH conflicts with ; disabled tests; missing shell completions; possible issues with getting an HTTP response from some sites including But it does sort of work and may be useful as a starting point. (Note that I have not checked the TLS fingerprint.)
I've opened which should add a fully working build from source of curl-impersonate (with both firefox and chrome variants)

Discussion:
couchdb = handleTest ./couchdb.nix {}; cri-o = handleTestOn ["aarch64-linux" "x86_64-linux"] ./cri-o.nix {}; cups-pdf = handleTest ./cups-pdf.nix {}; <ins> curl-impersonate = handleTest ./curl-impersonate.nix {}; </ins> custom-ca = handleTest ./custom-ca.nix {}; croc = handleTest ./croc.nix {}; deluge = handleTest ./deluge.nix {};

<ins> /* Test suite for curl-impersonate Abstract: Uses the test suite from the curl-impersonate source repo which: 1. Performs requests with libcurl and captures the TLS client-hello packets with tcpdump to compare against known-good signatures 2. Spins up an nghttpd2 server to test client HTTP/2 headers against known-good headers See https://github.com/lwthiker/curl-impersonate/tree/main/tests/signatures for details. Notes: - We need to have our own web server running because the tests expect to be able to hit domains like wikipedia.org and the sandbox has no internet - We need to be able to do (verifying) TLS handshakes without internet access. We do that by creating a trusted CA and issuing a cert that includes all of the test domains as subject-alternative names and then spoofs the hostnames in /etc/hosts. */ import ./make-test-python.nix ({ pkgs, lib, ... }: let # Update with domains in TestImpersonate.TEST_URLS if needed from: # https://github.com/lwthiker/curl-impersonate/blob/main/tests/test_impersonate.py domains = [ "www.wikimedia.org" "www.wikipedia.org" "www.mozilla.org" "www.apache.org" "www.kernel.org" "git-scm.com" ]; tls-certs = let # Configure CA with X.509 v3 extensions that would be trusted by curl ca-cert-conf = pkgs.writeText "curl-impersonate-ca.cnf" '' basicConstraints = critical, CA:TRUE subjectKeyIdentifier = hash authorityKeyIdentifier = keyid:always, issuer:always keyUsage = critical, cRLSign, digitalSignature, keyCertSign ''; # Configure leaf certificate with X.509 v3 extensions that would be trusted # by curl and set subject-alternative names for test domains tls-cert-conf = pkgs.writeText "curl-impersonate-tls.cnf" '' basicConstraints = critical, CA:FALSE subjectKeyIdentifier = hash authorityKeyIdentifier = keyid:always, issuer:always keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment, keyAgreement extendedKeyUsage = critical, serverAuth subjectAltName = @alt_names [alt_names] ${lib.concatStringsSep "n" (lib.imap0 (idx: domain: "DNS.${toString idx} = ${domain}") domains)} ''; in pkgs.runCommand "curl-impersonate-test-certs" { nativeBuildInputs = [ pkgs.openssl ]; } '' # create CA certificate and key openssl req -newkey rsa:4096 -keyout ca-key.pem -out ca-csr.pem -nodes -subj '/CN=curl-impersonate-ca.nixos.test' openssl x509 -req -sha512 -in ca-csr.pem -key ca-key.pem -out ca.pem -extfile ${ca-cert-conf} -days 36500 openssl x509 -in ca.pem -text # create server certificate and key openssl req -newkey rsa:4096 -keyout key.pem -out csr.pem -nodes -subj '/CN=curl-impersonate.nixos.test' openssl x509 -req -sha512 -in csr.pem -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -extfile ${tls-cert-conf} -days 36500 openssl x509 -in cert.pem -text # output CA cert and server cert and key mkdir -p $out cp key.pem cert.pem ca.pem $out ''; # Test script curl-impersonate-test = let # Build miniature libcurl client used by test driver minicurl = pkgs.runCommandCC "minicurl" { buildInputs = [ pkgs.curl ]; } '' mkdir -p $out/bin $CC -Wall -Werror -o $out/bin/minicurl ${pkgs.curl-impersonate.src}/tests/minicurl.c `curl-config --libs` ''; in pkgs.writeShellScript "curl-impersonate-test" '' set -euxo pipefail # Test driver requirements export PATH="${with pkgs; lib.makeBinPath [ bash coreutils python3Packages.pytest nghttp2 tcpdump ]}" export PYTHONPATH="${with pkgs.python3Packages; makePythonPath [ pyyaml pytest-asyncio dpkt ]}" # Prepare test root prefix mkdir -p usr/{bin,lib} cp -rs ${pkgs.curl-impersonate}/* ${minicurl}/* usr/ cp -r ${pkgs.curl-impersonate.src}/tests ./ # Run tests cd tests pytest . --install-dir ../usr --capture-interface eth1 ''; in { name = "curl-impersonate"; meta = with lib.maintainers; { maintainers = [ lilyinstarlight ]; }; nodes = { web = { nodes, pkgs, lib, config, ... }: { networking.firewall.allowedTCPPorts = [ 80 443 ]; services = { nginx = { enable = true; virtualHosts."curl-impersonate.nixos.test" = { default = true; addSSL = true; sslCertificate = "${tls-certs}/cert.pem"; sslCertificateKey = "${tls-certs}/key.pem"; }; }; }; }; curl = { nodes, pkgs, lib, config, ... }: { networking.extraHosts = lib.concatStringsSep "n" (map (domain: "${nodes.web.networking.primaryIPAddress}  ${domain}") domains); security.pki.certificateFiles = [ "${tls-certs}/ca.pem" ]; }; }; testScript = { nodes, ... }: '' start_all() with subtest("Wait for network"): web.wait_for_unit("network-online.target") curl.wait_for_unit("network-online.target") with subtest("Wait for web server"): web.wait_for_unit("nginx.service") web.wait_for_open_port(443) with subtest("Run curl-impersonate tests"): curl.succeed("${curl-impersonate-test}") ''; }) </ins>

<ins> diff --git a/Makefile.in b/Makefile.in index 877c54f..3e39ed1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -209,6 +209,8 @@ $(NSS_VERSION).tar.gz: $(nss_static_libs): $(NSS_VERSION).tar.gz tar xf $(NSS_VERSION).tar.gz +	sed -i -e "1s@#!/usr/bin/env bash@#!$$(type -p bash)@" $(NSS_VERSION)/nss/build.sh +	sed -i -e "s@/usr/bin/env grep@$$(type -p grep)@" $(NSS_VERSION)/nss/coreconf/config.gypi ifeq ($(host),$(build)) # Native build, use NSS' build script. </ins>

<del> #TODO: It should be possible to build this from source, but it's currently a lot faster to just package the binaries. { lib, stdenv, fetchzip, zlib, autoPatchelfHook }: stdenv.mkDerivation rec { pname = "curl-impersonate-bin"; version = "v0.5.3"; src = fetchzip { url = "https://github.com/lwthiker/curl-impersonate/releases/download/${version}/curl-impersonate-${version}.x86_64-linux-gnu.tar.gz"; sha256 = "sha256-+cH1swAIadIrWG9anzf0dcW6qyBjcKsUHFWdv75F49g="; stripRoot = false; </del> <ins> { lib , stdenv , fetchFromGitHub , fetchpatch , callPackage , buildGoModule , installShellFiles , symlinkJoin , zlib , sqlite , cmake , python3 , ninja , perl , autoconf , automake , libtool , darwin , cacert , unzip , go , p11-kit , nixosTests }: let makeCurlImpersonate = { name, target }: stdenv.mkDerivation rec { pname = "curl-impersonate-${name}"; version = "0.5.4"; src = fetchFromGitHub { owner = "lwthiker"; repo = "curl-impersonate"; rev = "v${version}"; hash = "sha256-LBGWFal2szqgURIBCLB84kHWpdpt5quvBBZu6buGj2A="; }; patches = [ # Fix shebangs in the NSS build script # (can't just patchShebangs since makefile unpacks it) ./curl-impersonate-0.5.2-fix-shebangs.patch ]; strictDeps = true; nativeBuildInputs = lib.optionals stdenv.isDarwin [ # Must come first so that it shadows the 'libtool' command but leaves 'libtoolize' darwin.cctools ] ++ [ installShellFiles cmake python3 python3.pkgs.gyp ninja perl autoconf automake libtool unzip go ]; buildInputs = [ zlib sqlite ]; configureFlags = [ "--with-ca-bundle=${if stdenv.isDarwin then "/etc/ssl/cert.pem" else "/etc/ssl/certs/ca-certificates.crt"}" "--with-ca-path=${cacert}/etc/ssl/certs" ]; buildFlags = [ "${target}-build" ]; checkTarget = "${target}-checkbuild"; installTargets = [ "${target}-install" ]; doCheck = true; dontUseCmakeConfigure = true; dontUseNinjaBuild = true; dontUseNinjaInstall = true; dontUseNinjaCheck = true; postUnpack = lib.concatStringsSep "n" (lib.mapAttrsToList (name: dep: "ln -sT ${dep.outPath} source/${name}") (lib.filterAttrs (n: v: v ? outPath) passthru.deps)); preConfigure = '' export GOCACHE=$TMPDIR/go-cache export GOPATH=$TMPDIR/go export GOPROXY=file://${passthru.boringssl-go-modules} export GOSUMDB=off # Need to get value of $out for this flag configureFlagsArray+=("--with-libnssckbi=$out/lib") ''; postInstall = '' # Remove vestigial *-config script rm $out/bin/curl-impersonate-${name}-config # Patch all shebangs of installed scripts patchShebangs $out/bin # Build and install completions for each curl binary # Patch in correct binary name and alias it to all scripts perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell zsh >$TMPDIR/curl-impersonate-${name}.zsh substituteInPlace $TMPDIR/curl-impersonate-${name}.zsh  --replace  '#compdef curl'  "#compdef curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' %f=curl-impersonate-${name}')" perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell fish >$TMPDIR/curl-impersonate-${name}.fish substituteInPlace $TMPDIR/curl-impersonate-${name}.fish  --replace  '--command curl'  "--command curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' --command %f')" # Install zsh and fish completions installShellCompletion $TMPDIR/curl-impersonate-${name}.{zsh,fish} ''; preFixup = let libext = stdenv.hostPlatform.extensions.sharedLibrary; in '' # If libnssckbi.so is needed, link libnssckbi.so without needing nss in closure if grep -F nssckbi $out/lib/libcurl-impersonate-*${libext} &>/dev/null; then # NOTE: "p11-kit-trust" always ends in ".so" even when on darwin ln -s ${p11-kit}/lib/pkcs11/p11-kit-trust.so $out/lib/libnssckbi${libext} ${lib.optionalString stdenv.isLinux "patchelf --add-needed libnssckbi${libext} $out/lib/libcurl-impersonate-*${libext}"} fi ''; disallowedReferences = [ go ]; passthru = { deps = callPackage ./deps.nix {}; boringssl-go-modules = (buildGoModule { inherit (passthru.deps."boringssl.zip") name; src = passthru.deps."boringssl.zip"; vendorHash = "sha256-ISmRdumckvSu7hBXrjvs5ZApShDiGLdD3T5B0fJ1x2Q="; nativeBuildInputs = [ unzip ]; proxyVendor = true; }).go-modules; }; meta = with lib; { description = "A special build of curl that can impersonate Chrome & Firefox"; homepage = "https://github.com/lwthiker/curl-impersonate"; license = with licenses; [ curl mit ]; maintainers = with maintainers; [ deliciouslytyped lilyinstarlight ]; platforms = platforms.unix; knownVulnerabilities = [ "CVE-2023-32001"  # fopen TOCTOU race condition - https://curl.se/docs/CVE-2023-32001.html "CVE-2022-43551"  # HSTS bypass - https://curl.se/docs/CVE-2022-43551.html "CVE-2022-42916"  # HSTS bypass - https://curl.se/docs/CVE-2022-42916.html ]; }; </ins> }; <ins> in symlinkJoin rec { pname = "curl-impersonate"; inherit (passthru.curl-impersonate-ff) version meta; name = "${pname}-${version}"; paths = [ passthru.curl-impersonate-ff passthru.curl-impersonate-chrome ]; passthru = { curl-impersonate-ff = makeCurlImpersonate { name = "ff"; target = "firefox"; }; curl-impersonate-chrome = makeCurlImpersonate { name = "chrome"; target = "chrome"; }; </ins> <del> nativeBuildInputs = [ autoPatchelfHook zlib ]; </del> <ins> updateScript = ./update.sh; </ins> <del> installPhase = '' mkdir -p $out/bin cp * $out/bin ''; </del> <ins> inherit (passthru.curl-impersonate-ff) src; </ins> <del> meta = with lib; { description = "curl-impersonate: A special build of curl that can impersonate Chrome & Firefox "; homepage = "https://github.com/lwthiker/curl-impersonate"; license = with licenses; [ curl mit ]; maintainers = with maintainers; [ deliciouslytyped ]; platforms = platforms.linux; #TODO I'm unsure about the restrictions here, feel free to expand the platforms it if it works elsewhere. </del> <ins> tests = { inherit (nixosTests) curl-impersonate; }; </ins> }; }

<ins> # Generated by update.sh { fetchurl }: { "curl-7.84.0.tar.xz" = fetchurl { url = "https://curl.se/download/curl-7.84.0.tar.xz"; hash = "sha256-LRGLQ/VHv+W66AbY1HtOWW6lslpsHwgK70n7zYF8Xbg="; }; "brotli-1.0.9.tar.gz" = fetchurl { url = "https://github.com/google/brotli/archive/refs/tags/v1.0.9.tar.gz"; hash = "sha256-+ejYHQQFumbRgVKa9CozVPg4yTkJX/mZMNpqqc32/kY="; }; "nss-3.87.tar.gz" = fetchurl { url = "https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_87_RTM/src/nss-3.87-with-nspr-4.35.tar.gz"; hash = "sha256-63DqC1jc5pqkkOnp/s0TKn1kTh2j1jHhYzdqDcwRoCI="; }; "boringssl.zip" = fetchurl { url = "https://github.com/google/boringssl/archive/3a667d10e94186fd503966f5638e134fe9fb4080.zip"; hash = "sha256-HsDIkd1x5IH49fUF07dJaabMIMsQygW+NI7GneULpA8="; }; "nghttp2-1.46.0.tar.bz2" = fetchurl { url = "https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2"; hash = "sha256-moKXjIcAcbdp8n0riBkct3/clFpRwdaFx/YafhP8Ryk="; }; } </ins>

<ins> #!/usr/bin/env nix-shell #!nix-shell -i bash -p git nix jq coreutils gnugrep gnused curl common-updater-scripts set -euo pipefail nixpkgs="$(git rev-parse --show-toplevel || (printf 'Could not find root of nixpkgs reponAre we running from within the nixpkgs git repo?n' >&2; exit 1))" stripwhitespace() { sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' } narhash() { nix --extra-experimental-features nix-command store prefetch-file --json "$1" | jq -r .hash } nixeval() { nix --extra-experimental-features nix-command eval --json --impure -f "$nixpkgs" "$1" | jq -r . } vendorhash() { (nix --extra-experimental-features nix-command build --no-link -f "$nixpkgs" --no-link "$1" 2>&1 >/dev/null | tail -n3 | grep -F got: | cut -d: -f2- | stripwhitespace) 2>/dev/null || true } findpath() { path="$(nix --extra-experimental-features nix-command eval --json --impure -f "$nixpkgs" "$1.meta.position" | jq -r . | cut -d: -f1)" outpath="$(nix --extra-experimental-features nix-command eval --json --impure --expr "builtins.fetchGit "$nixpkgs"")" if [ -n "$outpath" ]; then path="${path/$(echo "$outpath" | jq -r .)/$nixpkgs}" fi echo "$path" } getvar() { echo "$2" | grep -F "$1" | sed -e 's/:=/:/g' | cut -d: -f2- | stripwhitespace } attr="${UPDATE_NIX_ATTR_PATH:-curl-impersonate}" version="$(curl -sSL "https://api.github.com/repos/lwthiker/curl-impersonate/releases/latest" | jq -r .tag_name | sed -e 's/^v//')" pkgpath="$(findpath "$attr")" updated="$(cd "$nixpkgs" && update-source-version "$attr" "$version" --file="$pkgpath" --print-changes | jq -r length)" if [ "$updated" -eq 0 ]; then echo 'update.sh: Package version not updated, nothing to do.' exit 0 fi vars="$(curl -sSL "https://github.com/lwthiker/curl-impersonate/raw/v$version/Makefile.in" | grep '^ *[^ ]*_(VERSION|URL|COMMIT) *:=')" cat >"$(dirname "$pkgpath")"/deps.nix <<EOF # Generated by update.sh { fetchurl }: { "$(getvar CURL_VERSION "$vars").tar.xz" = fetchurl { url = "https://curl.se/download/$(getvar CURL_VERSION "$vars").tar.xz"; hash = "$(narhash "https://curl.se/download/$(getvar CURL_VERSION "$vars").tar.xz")"; }; "brotli-$(getvar BROTLI_VERSION "$vars").tar.gz" = fetchurl { url = "https://github.com/google/brotli/archive/refs/tags/v$(getvar BROTLI_VERSION "$vars").tar.gz"; hash = "$(narhash "https://github.com/google/brotli/archive/refs/tags/v$(getvar BROTLI_VERSION "$vars").tar.gz")"; }; "$(getvar NSS_VERSION "$vars").tar.gz" = fetchurl { url = "$(getvar NSS_URL "$vars")"; hash = "$(narhash "$(getvar NSS_URL "$vars")")"; }; "boringssl.zip" = fetchurl { url = "https://github.com/google/boringssl/archive/$(getvar BORING_SSL_COMMIT "$vars").zip"; hash = "$(narhash "https://github.com/google/boringssl/archive/$(getvar BORING_SSL_COMMIT "$vars").zip")"; }; "$(getvar NGHTTP2_VERSION "$vars").tar.bz2" = fetchurl { url = "$(getvar NGHTTP2_URL "$vars")"; hash = "$(narhash "$(getvar NGHTTP2_URL "$vars")")"; }; } EOF curhash="$(nixeval "$attr.curl-impersonate-chrome.boringssl-go-modules.outputHash")" newhash="$(vendorhash "$attr.curl-impersonate-chrome.boringssl-go-modules")" if [ -n "$newhash" ] && [ "$curhash" != "$newhash" ]; then sed -i -e "s|"$curhash"|"$newhash"|" "$pkgpath" else echo 'update.sh: New vendorHash same as old vendorHash, nothing to do.' fi </ins>

cups-kyodialog3 = cups-kyodialog; # Added 2022-11-12 cupsBjnp = throw "'cupsBjnp' has been renamed to/replaced by 'cups-bjnp'"; # Converted to throw 2022-02-22 cups_filters = throw "'cups_filters' has been renamed to/replaced by 'cups-filters'"; # Converted to throw 2022-02-22 <ins> curl-impersonate-bin = throw "'curl-impersonate-bin' has been replaced by 'curl-impersonate'"; # Added 2022-10-08 </ins> curlcpp = throw "curlcpp has been removed, no active maintainers and no usage within nixpkgs"; # Added 2022-05-10 curaByDagoma = throw "curaByDagoma has been removed from nixpkgs, because it was unmaintained and dependent on python2 packages"; # Added 2022-01-12 curaLulzbot = throw "curaLulzbot has been removed due to insufficient upstream support for a modern dependency chain"; # Added 2021-10-23

curlWithGnuTls = curl.override { gnutlsSupport = true; opensslSupport = false; }; <del> curl-impersonate-bin = callPackage ../tools/networking/curl-impersonate { }; </del> <ins> curl-impersonate = darwin.apple_sdk_11_0.callPackage ../tools/networking/curl-impersonate { }; inherit (curl-impersonate) curl-impersonate-ff curl-impersonate-chrome; </ins> curlie = callPackage ../tools/networking/curlie { };