diff --git a/.github/workflows/test-using-nix.yml b/.github/workflows/test-using-nix.yml new file mode 100644 index 00000000..6ffc1d19 --- /dev/null +++ b/.github/workflows/test-using-nix.yml @@ -0,0 +1,104 @@ +name: build and test (using nix) + +on: + pull_request: + merge_group: + workflow_dispatch: + push: + branches: + - main + +permissions: read-all + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +defaults: + run: + shell: nix develop .#ci --ignore-env --keep-env-var CI --command bash -e {0} + +jobs: + build_test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + with: + submodules: true + + - uses: cachix/install-nix-action@v31 + - run: echo "Preparing Nix shell" + + - run: dune build --profile=release + - run: dune runtest --force + + format: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + with: + submodules: true + + - uses: cachix/install-nix-action@v31 + - run: echo "Preparing Nix shell" + + - run: dune build @fmt --display=quiet + if: "!cancelled()" + - run: dune build '@treesit' + if: "!cancelled()" + + build_test_slower: + timeout-minutes: 15 + runs-on: ubuntu-latest + needs: [ build_test, format ] + if: "! ( cancelled() || success() ) || github.ref == 'refs/heads/main'" + + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v6 + with: + submodules: true + + - uses: ocaml/setup-ocaml@v3 + + - run: opam option --global 'archive-mirrors+="https://opam.ocaml.org/cache"' + - run: opam repository --all add pac https://github.com/uq-pac/opam-repository.git + - run: opam install . --deps-only --with-test + - run: echo LIBRARY_PATH="$LIBRARY_PATH:/opt/homebrew/lib" >> "$GITHUB_ENV" + if: matrix.os == 'macos-latest' + - run: opam lint + - run: opam exec -- dune build @fmt --display=quiet + - run: opam exec -- dune build --ignore-promoted-rules --profile=release + - run: opam exec -- dune build --ignore-promoted-rules '@treesit' + - run: opam exec -- dune runtest --ignore-promoted-rules --force + + at_least_one_success: + runs-on: ubuntu-latest + needs: [ build_test_slower ] + if: "!cancelled()" + + defaults: + run: + shell: bash + + steps: + - run: ${{ needs.build_test_slower.result == 'success' || needs.build_test_slower.result == 'skipped' }} + name: "Check that at least one build & test job succeeded" + + docs-build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + with: + submodules: true + + - uses: cachix/install-nix-action@v31 + - run: echo "Preparing Nix shell" + + - run: odoc_driver bincaml # TODO: fixme. need to register packages somehow? diff --git a/bincaml.opam b/bincaml.opam index 75a65e1e..7e351ad7 100644 --- a/bincaml.opam +++ b/bincaml.opam @@ -11,7 +11,7 @@ homepage: "https://github.com/username/reponame" doc: "https://url/to/documentation" bug-reports: "https://github.com/username/reponame/issues" depends: [ - "dune" {>= "3.24"} + "dune" {>= "3.20"} "ocaml" "unionFind" "capstone_arm64_disas" diff --git a/bincaml_lsp.opam b/bincaml_lsp.opam index 6e682864..c93d04c3 100644 --- a/bincaml_lsp.opam +++ b/bincaml_lsp.opam @@ -10,7 +10,7 @@ homepage: "https://github.com/username/reponame" doc: "https://url/to/documentation" bug-reports: "https://github.com/username/reponame/issues" depends: [ - "dune" {>= "3.24"} + "dune" {>= "3.20"} "menhir" {>= "20180523"} "ppx_expect" "bincaml" diff --git a/capstone_arm64_disas.opam b/capstone_arm64_disas.opam index 3d89ce4e..7f8b5ed0 100644 --- a/capstone_arm64_disas.opam +++ b/capstone_arm64_disas.opam @@ -9,7 +9,7 @@ homepage: "https://github.com/username/reponame" doc: "https://url/to/documentation" bug-reports: "https://github.com/username/reponame/issues" depends: [ - "dune" {>= "3.24"} + "dune" {>= "3.20"} "ocaml" "ppx_expect" "odoc" {with-doc} diff --git a/dune-project b/dune-project index bfbe8448..d2c8ffe2 100644 --- a/dune-project +++ b/dune-project @@ -1,4 +1,4 @@ -(lang dune 3.24) +(lang dune 3.20) (using menhir 3.0) diff --git a/flake.nix b/flake.nix index 0fdab666..3c170fb8 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,10 @@ { + nixConfig.extra-substituters = [ "https://pac-nix.cachix.org/" ]; + nixConfig.extra-trusted-public-keys = [ + "pac-nix.cachix.org-1:l29Pc2zYR5yZyfSzk1v17uEZkhEw0gI4cXuOIsxIGpc=" + ]; + inputs = { - self.submodules = true; nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; @@ -37,7 +41,7 @@ in flake-for-all-systems args { overlays = { - addBincamlPackages = ofinal: _: { + addBincamlPackages = ofinal: oprev: { buildDune324Package = ofinal.buildDunePackage.override { dune_3 = ofinal.dune_3_24; }; @@ -53,6 +57,7 @@ capstone_arm64_disas = ofinal.callPackage ./nix/capstone_arm64_disas.nix { buildDunePackage = ofinal.buildDune324Package; }; + bincaml-docs = ofinal.callPackage ./nix/bincaml-docs.nix { }; ocaml-protoc-plugin-6-1-0 = ofinal.callPackage ./nix/ocaml-protoc-plugin.nix { }; aslp_lifter_ocaml = ofinal.callPackage ./nix/aslp-lifter-ocaml.nix { }; @@ -61,6 +66,20 @@ kittyimg = ofinal.callPackage ./nix/kittyimg.nix { }; stb_image = ofinal.callPackage ./nix/stb_image.nix { }; containers = ofinal.callPackage ./nix/containers.nix { }; + + odoc = + (oprev.odoc.override { + cmdliner = ofinal.cmdliner; + }).overrideAttrs + ( + f: p: { + doCheck = false; + propagatedBuildInputs = (p.propagatedBuildInputs or [ ]) ++ (p.buildInputs or [ ]) ++ [ofinal.ppx_expect]; + } + ); + sherlodoc = ofinal.callPackage ./nix/sherlodoc.nix { }; + odoc-md = ofinal.callPackage ./nix/odoc-md.nix { }; + odoc-driver = ofinal.callPackage ./nix/odoc-driver.nix { }; }; enableOcamlFramePointer = @@ -88,16 +107,22 @@ ... }: let - inherit (pac-nix.legacyPackages) bnfc-treesitter; - pkgs = nixpkgs.legacyPackages; - selfOcamlPackages = pkgs.ocamlPackages.overrideScope self.overlays.addBincamlPackages; + ocamlPackages = pkgs.ocamlPackages.overrideScope ( + _: _: { + z3-bin = pkgs.z3; + inherit (pac-nix.legacyPackages) bnfc-treesitter; + } + ); + selfOcamlPackages = ocamlPackages.overrideScope self.overlays.addBincamlPackages; fpOcamlPackages = selfOcamlPackages.overrideScope self.overlays.enableOcamlFramePointer; in { defaultPackage = selfOcamlPackages.bincaml; legacyPackages = { + ocamlPackages = selfOcamlPackages; + bincaml = selfOcamlPackages.bincaml; bincaml_lsp = selfOcamlPackages.bincaml_lsp; aslp_lifter_ocaml = selfOcamlPackages.aslp_lifter_ocaml; @@ -123,12 +148,13 @@ devShells = { default = self.devShells.fp; fp = fpOcamlPackages.callPackage ./nix/shell.nix { - inherit bnfc-treesitter; - z3 = pkgs.z3.out; + isShellForCI = false; }; no-fp = selfOcamlPackages.callPackage ./nix/shell.nix { - inherit bnfc-treesitter; - z3 = pkgs.z3.out; + isShellForCI = false; + }; + ci = selfOcamlPackages.callPackage ./nix/shell.nix { + isShellForCI = true; }; }; }; diff --git a/nix/aslp-lifter-ocaml.nix b/nix/aslp-lifter-ocaml.nix index 60ae3117..3cf680af 100644 --- a/nix/aslp-lifter-ocaml.nix +++ b/nix/aslp-lifter-ocaml.nix @@ -24,7 +24,6 @@ buildDunePackage (self: { outputs = [ "out" - "dev" ]; meta = { diff --git a/nix/bincaml-docs.nix b/nix/bincaml-docs.nix new file mode 100644 index 00000000..71553765 --- /dev/null +++ b/nix/bincaml-docs.nix @@ -0,0 +1,55 @@ +{ +buildEnv, +runCommand, +writeShellScriptBin, +odoc-driver, + bincaml, + bincaml_lsp, +odoc, +findlib, +ocaml, +fmt, +sherlodoc, +runtimeShell +}: + +let + dummy-opam = runCommand "dummy-opam" { + } '' + mkdir -pv $out/lib $out/bin + cp ${findlib}/etc/findlib.conf $out/lib + cp -r ${ocaml}/lib/ocaml $out/lib/ocaml + + cat < $out/bin/opam + #!${runtimeShell} + echo $out + EOF + chmod +x $out/bin/opam + ''; + +env = buildEnv { +name = "ajidso"; + paths = [ fmt ]; + includeClosures = true; + pathsToLink = [ "/share/doc" "/lib/ocaml/5.4.1/site-lib" ]; + + postBuild = '' + mv $out/share/doc doc + mv $out/lib/ocaml/*/site-lib lib + ln -s ${ocaml}/lib/ocaml lib + + rm -rf $out + mkdir -v $out + mv doc lib $out + ''; +}; + +in runCommand "bincaml-docs" { + nativeBuildInputs = [ dummy-opam odoc-driver odoc sherlodoc ocaml ]; +} '' + dune_prefix=$(mktemp -d)/_build/install/default + mkdir -p $(dirname $dune_prefix) + ln -s ${env} $dune_prefix + ls -l $(dirname $dune_prefix) + OCAMLPATH=$dune_prefix/lib odoc_driver --html-dir=$out -v $(cd $dune_prefix/lib && echo *) +'' diff --git a/nix/bincaml-lsp.nix b/nix/bincaml-lsp.nix index 94cdd1e3..85a1ecbc 100644 --- a/nix/bincaml-lsp.nix +++ b/nix/bincaml-lsp.nix @@ -34,18 +34,14 @@ buildDunePackage { logs fmt iter - linol - linol-lwt containers ppx_deriving + linol + linol-lwt ]; - propagatedBuildInputs = [ ]; doCheck = true; - outputs = [ - "out" - "dev" - ]; + outputs = [ "out" ]; meta = { homepage = "https://github.com/agle/bincaml"; diff --git a/nix/bincaml.nix b/nix/bincaml.nix index 96f5d737..07773278 100644 --- a/nix/bincaml.nix +++ b/nix/bincaml.nix @@ -40,6 +40,12 @@ qcheck-core, qcheck-alcotest, qcheck-stm, + tree-sitter, + nodejs-slim, + bnfc-treesitter, + boogie, + cvc5, + z3-bin, # custom name for z3 binary, since `z3` is the ocaml library # dev: # , odig @@ -66,6 +72,14 @@ buildDunePackage { qcheck-alcotest qcheck-stm ]; + nativeCheckInputs = [ + tree-sitter + nodejs-slim + bnfc-treesitter + boogie + cvc5 + z3-bin.out + ]; nativeBuildInputs = [ writableTmpDirAsHomeHook protobuf @@ -110,10 +124,7 @@ buildDunePackage { ''; doCheck = true; - outputs = [ - "out" - "dev" - ]; + outputs = [ "out" ]; meta = { homepage = "https://github.com/agle/bincaml"; diff --git a/nix/capstone_arm64_disas.nix b/nix/capstone_arm64_disas.nix index 8b9954f3..22237a85 100644 --- a/nix/capstone_arm64_disas.nix +++ b/nix/capstone_arm64_disas.nix @@ -43,10 +43,7 @@ buildDunePackage { propagatedBuildInputs = [ capstone ]; doCheck = false; - outputs = [ - "out" - "dev" - ]; + outputs = [ "out" ]; meta = { homepage = "https://github.com/agle/bincaml"; diff --git a/nix/odoc-driver.nix b/nix/odoc-driver.nix new file mode 100644 index 00000000..9bcee2ce --- /dev/null +++ b/nix/odoc-driver.nix @@ -0,0 +1,64 @@ +{ + lib, + fetchpatch2, + buildDunePackage, + + odoc, + odoc-md, + fpath, + bos, + yojson, + findlib, + opam-format, + logs, + eio_main, + eio, + progress, + cmdliner, + sexplib, + ppx_sexp_conv, + sherlodoc, +}: + +buildDunePackage { + pname = "odoc-driver"; + inherit (odoc) version src; + + patches = [ + (fetchpatch2 { + url = "https://github.com/ocaml/odoc/commit/02408309dc223f8ab97a023dfec4e4641ec55736.patch"; + hash = "sha256-FO2QaK1TaxCNBpttFy/GHnePa6uvKOvxE+rstKcPY/Q="; + }) + ]; + + nativeBuildInputs = [ sherlodoc ]; + buildInputs = [ + odoc + odoc-md + fpath + bos + yojson + findlib + opam-format + logs + eio_main + eio + progress + cmdliner + sexplib + ppx_sexp_conv + ]; + + nativeCheckInputs = [ ]; + checkInputs = [ ]; + doCheck = true; + + meta = { + description = "OCaml Documentation Generator - Driver"; + mainProgram = "odoc_driver"; + license = lib.licenses.isc; + maintainers = [ lib.maintainers.katrinafyi ]; + homepage = "https://github.com/ocaml/odoc"; + changelog = "https://github.com/ocaml/odoc/blob/${odoc.version}/CHANGES.md"; + }; +} diff --git a/nix/odoc-md.nix b/nix/odoc-md.nix new file mode 100644 index 00000000..ac40205e --- /dev/null +++ b/nix/odoc-md.nix @@ -0,0 +1,32 @@ +{ + lib, + buildDunePackage, + + odoc, + cmdliner, + cmarkit, +}: + +buildDunePackage { + pname = "odoc-md"; + inherit (odoc) version src; + + nativeBuildInputs = [ ]; + buildInputs = [ + odoc + cmdliner + cmarkit + ]; + + nativeCheckInputs = [ ]; + checkInputs = [ ]; + doCheck = true; + + meta = { + description = "OCaml Documentation Generator - Markdown support"; + license = lib.licenses.isc; + maintainers = [ lib.maintainers.katrinafyi ]; + homepage = "https://github.com/ocaml/odoc"; + changelog = "https://github.com/ocaml/odoc/blob/${odoc.version}/CHANGES.md"; + }; +} diff --git a/nix/shell.nix b/nix/shell.nix index 381d54df..28debac8 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -1,4 +1,6 @@ { + isShellForCI, + lib, stdenv, mkShell, @@ -6,55 +8,48 @@ # ocaml packages bincaml, bincaml_lsp, + capstone_arm64_disas, odoc, + odoc-driver, odig, ocaml-lsp, ocamlformat, + opam, # dev packages - tree-sitter, - nodejs-slim, perf, - bnfc-treesitter, - boogie, - cvc5, - linol-lwt, - linol, - capstone, - - # lsp - logs, - mtime, - z3, }: mkShell { - buildInputs = [ - capstone - ]; - packages = [ odoc + odoc-driver odig - ocaml-lsp ocamlformat - tree-sitter - nodejs-slim - bnfc-treesitter - boogie - cvc5 - bincaml_lsp - linol - linol-lwt - logs - mtime - z3.out - # sherlodoc - not in nixpkgs? ] - ++ lib.optional stdenv.hostPlatform.isLinux perf; + + ++ lib.optionals (!isShellForCI) ( + [ + bincaml_lsp + ocaml-lsp + ] + ++ lib.optional stdenv.hostPlatform.isLinux perf + ) + + ++ lib.optionals (isShellForCI) [ + opam + ]; inputsFrom = [ (bincaml.overrideAttrs { doCheck = true; }) + (capstone_arm64_disas.overrideAttrs { doCheck = true; }) + (bincaml_lsp.overrideAttrs { doCheck = true; }) + + # including these unchanged will subtract them from the dependencies of each other: + # https://github.com/NixOS/nixpkgs/blob/f9bb1890175874edf242921789e8e9fdfcc2023c/pkgs/build-support/mkshell/default.nix#L32-L34 + bincaml + capstone_arm64_disas + bincaml_lsp ]; shellHook = '' @@ -75,5 +70,8 @@ mkShell { ln -sf $i/* "$ODIG_LIB_DIR" done fi + '' + lib.optionalString isShellForCI '' + opam init --bare --disable-sandboxing $(mktemp -d) --quiet --no + export OPAMCOLOR=never ''; } diff --git a/nix/sherlodoc.nix b/nix/sherlodoc.nix new file mode 100644 index 00000000..671ec8aa --- /dev/null +++ b/nix/sherlodoc.nix @@ -0,0 +1,89 @@ +{ + lib, + fetchpatch2, + buildDunePackage, + writableTmpDirAsHomeHook, + + opam, + odoc, + base64, + bigstringaf, + js_of_ocaml, + brr, + cmdliner, + decompress, + fpath, + lwt, + menhir, + ppx_blob, + tyxml, + odig, + base, + alcotest, + findlib, +}: + +buildDunePackage (self: { + pname = "sherlodoc"; + inherit (odoc) version src; + + patches = [ + (fetchpatch2 { + url = "https://github.com/ocaml/odoc/commit/9695cd79ec29c082f62c768a99a650ebf06892d3.patch"; + hash = "sha256-8eyclSV4MnMkVcT+0NjPKWARzITm6tJpg0n/Z7Yi/L4="; + }) + ]; + + nativeBuildInputs = [ + menhir + js_of_ocaml + writableTmpDirAsHomeHook + ]; + + buildInputs = [ + odoc + findlib + base64 + bigstringaf + brr + cmdliner + decompress + fpath + lwt + ppx_blob + tyxml + base + alcotest + ]; + + nativeCheckInputs = [ + odig + odoc + opam + ]; + checkInputs = [ alcotest ]; + doCheck = true; + + preCheck = '' + substituteInPlace sherlodoc/test/dune --replace-quiet --quiet "" + + cat <> sherlodoc/test/dune + (env + (_ + (env-vars + (ODIG_LIB_DIR $(echo ${tyxml}/lib/ocaml/*/site-lib)) + (ODIG_DOC_DIR ${tyxml}/share/doc) + (LOG_LEVEL info) + ))) + EOF + + opam init --bare --disable-sandboxing $(mktemp -d) --quiet --no + ''; + + meta = { + description = "Search engine for OCaml documentation"; + license = lib.licenses.mit; + maintainers = [ lib.maintainers.katrinafyi ]; + homepage = "https://github.com/ocaml/odoc"; + }; +}) diff --git a/run-in-nix-shell.sh b/run-in-nix-shell.sh new file mode 100755 index 00000000..e95d089f --- /dev/null +++ b/run-in-nix-shell.sh @@ -0,0 +1,3 @@ +#!/bin/bash -eu + +exec nix develop .#ci --ignore-env --keep-env-var CI --command "$@"