more clean

Author: syphar

.env.tests

@@ -0,0 +1,7 @@
+# environment variables to run our tests
+# when running tests outside docker, locally or in CI, 
+# makes the test use the needed resources defined in our docker-compose file
+DOCSRS_DATABASE_URL=postgresql://cratesfyi:password@localhost:15432
+AWS_ACCESS_KEY_ID=cratesfyi
+AWS_SECRET_ACCESS_KEY=secret_key
+S3_ENDPOINT=http://localhost:9000

.github/workflows/ci.yml

@@ -9,17 +9,7 @@ on:
     - cron: "0 0 * * *"
 
 env:
-  RUST_BACKTRACE: 1
   RUST_CACHE_KEY: rust-cache-20241114
-  DOCSRS_PREFIX: ignored/cratesfyi-prefix
-  DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@localhost:15432
-  DOCSRS_LOG: docs_rs=debug,rustwide=info
-  AWS_ACCESS_KEY_ID: cratesfyi
-  AWS_SECRET_ACCESS_KEY: secret_key
-  S3_ENDPOINT: http://localhost:9000
-  DOCSRS_INCLUDE_DEFAULT_TARGETS: false
-  DOCSRS_DOCKER_IMAGE: ghcr.io/rust-lang/crates-build-env/linux-micro
-  SENTRY_ENVIRONMENT: dev
 
 jobs:
   sqlx:
@@ -48,7 +38,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Launch postgres
-        run: just ensure_db_and_s3_are_running
+        run: just _ensure_db_and_s3_are_running
 
       - name: install SQLX CLI
         run: cargo binstall sqlx-cli
@@ -98,11 +88,6 @@ jobs:
     steps:
       - uses: actions/checkout@v5
 
-      - name: restore build & cargo cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: ${{ env.RUST_CACHE_KEY }}
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -116,7 +101,7 @@ jobs:
         if: ${{ always() }}
         run: just compose-down-and-wipe
 
-  rslint:
+  lint-rs:
     name: rust linters
     runs-on: ubuntu-latest
 
@@ -136,11 +121,14 @@ jobs:
 
       - run: just lint
 
-  eslint:
+  lint-js:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
+
       - uses: denoland/setup-deno@v2
+
       - name: install `just`
         run: sudo snap install --edge --classic just
+
       - run: just lint-js

justfiles/cli.just

@@ -8,18 +8,18 @@
 
 # low-level helper to run any CLI command in its own one-off docker container, 
 # ensuring that `db` and `s3` are running.
-_cli service_name *args: _touch-docker-env ensure_db_and_s3_are_running
+_cli service_name *args: _touch-docker-env _ensure_db_and_s3_are_running
   # dependencies in the docker-compose file are ignored 
   # when running a one-off service with `docker compose run`.
   # Instead we explicitly start any dependent services first via 
-  # `ensure_db_and_s3_are_running`.
+  # `_ensure_db_and_s3_are_running`.
 
   docker compose run --build --rm {{ service_name }} {{ args }}
 
 # run any CLI command in its own one-off `cli` docker container. Args are passed to the container.
 # Only for commands that just need `db` and `s3` and minimal system dependencies.
 [group('cli')]
-cli +args: _touch-docker-env  cli-db-migrate
+cli +args: _touch-docker-env cli-db-migrate
   just _cli cli {{ args }}
 
 # Initialize the `docs.rs` database
@@ -36,48 +36,51 @@ cli-db-migrate:
 # add a release to the build queue
 [group('cli')]
 [group('queue')]
-cli-queue-add +args:
+cli-queue-add crate_name crate_version:
   # only does things with the database, so can use the lightweight `cli` container.
-  just cli queue add {{ args }}
+  just cli queue add {{ crate_name }} {{ crate_version }}
 
 # run builder CLI command in its own one-off `build-server` docker container.
 # Uses a separate builder-cli container & workspace that doesn't conflict 
 # with the continiously running build-servers.
 [group('cli')]
-[group('build')]
+[group('builder')]
 cli-build +args: _touch-docker-env cli-db-migrate
   just _cli builder-cli {{ args }}
 
 # set the nightly rust version to be used for builds. Format: `nightly-YYYY-MM-DD` 
 # or just `nightly` for always using the latest nightly.  
 [group('cli')]
-[group('build')]
+[group('builder')]
 cli-build-set-toolchain name only_first_time="false":
+  #!/usr/bin/env bash
+  set -euo pipefail
+
   FLAG=""
   if [ "{{only_first_time}}" = "true" ]; then FLAG="--only-first-time"; fi
   just cli-build build set-toolchain {{ name }} $FLAG
 
 # update the toolchain in the builders
 [group('cli')]
-[group('build')]
+[group('builder')]
 cli-build-update-toolchain:
   just cli-build build update-toolchain
 
 # build & upload toolchain shared static resources
 [group('cli')]
-[group('build')]
+[group('builder')]
 cli-build-add-essential-files:
   just cli-build build add-essential-files
 
 # build a release
 [group('cli')]
-[group('build')]
+[group('builder')]
 cli-build-crate name version:
   just cli-build build crate {{ name }} {{ version }}
 
 # run registry-watcher CLI command in its own one-off `registry-watcher` docker container.
 [group('cli')]
-[group('registry-watcher')]
+[group('watcher')]
 cli-watcher +args: _touch-docker-env cli-db-migrate
   just _cli registry-watcher-cli {{ args }}
 

justfiles/services.just

@@ -10,11 +10,13 @@ compose-up-web:
 
 # Launch two build servers in the background 
 [group('compose')]
+[group('builder')]
 compose-up-builder: 
   just compose-up builder
 
 # Launch registry watcher in the background 
 [group('compose')]
+[group('watcher')]
 compose-up-watcher: 
   just compose-up watcher
 
@@ -37,6 +39,7 @@ compose-down:
 [group('compose')]
 compose-down-and-wipe:
   #!/usr/bin/env bash
+  set -euo pipefail
 
   docker compose --profile full --profile manual down --volumes --remove-orphans --rmi local
 
@@ -58,4 +61,4 @@ compose-down-and-wipe:
 # stream logs from all services running in docker-compose. Optionally specify services to tail logs from.
 [group('compose')]
 compose-logs *services:
-  docker compose --profile full --profile manual logs -f {{ services }}
+  docker compose --profile full logs -f {{ services }}

justfiles/testing.just

@@ -1,21 +1,28 @@
 # just commands for CI & local development
 
+
+
+[group('testing')]
 sqlx-prepare *args:
   cargo sqlx prepare \
     --database-url $DOCSRS_DATABASE_URL \
     --workspace {{ args }} \
     -- --all-targets --all-features
 
+[group('testing')]
 sqlx-check:
   just sqlx-prepare -check
 
 # Format the code using `cargo fmt`. 
+[group('testing')]
 format:
     rustup component add rustfmt
     # like this we get both the non-zero exit code, and the local code is 
     # formatted.
     cargo fmt --all -- --check || { cargo fmt --all && exit 1; }
 
+# run clippy, in our config
+[group('testing')]
 clippy *args:
   rustup component add clippy
   cargo clippy \
@@ -26,23 +33,30 @@ clippy *args:
     {{ args }} \
     -- -D warnings 
 
+# run clippy --fix
+[group('testing')]
 clippy-fix:
     just clippy --fix --allow-dirty --allow-staged
 
 # run all linters, for local development & CI
+[group('testing')]
 lint: format 
   #!/usr/bin/env bash
+  set -euo pipefail
+
   if [ "$GITHUB_ACTIONS" = "true" ]; then
     just clippy
   else
     just clippy_fix
   fi
 
 
+[group('testing')]
 lint-js *args:
   deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }}
 
-run-gui-tests: ensure_db_and_s3_are_running cli-db-migrate compose-up-web
+[group('testing')]
+run-gui-tests: _ensure_db_and_s3_are_running cli-db-migrate compose-up-web
   just cli-build-update-toolchain
   just cli-build-crate sysinfo 0.23.4
   just cli-build-crate sysinfo 0.23.5
@@ -51,18 +65,35 @@ run-gui-tests: ensure_db_and_s3_are_running cli-db-migrate compose-up-web
 
   just _cli gui_tests 
 
-# build binaries for all tests
-build-tests:
+_build-test-binaries:
+  #!/usr/bin/env bash
+  set -euo pipefail
+
+  export SQLX_OFFLINE=1
   cargo test --no-run --workspace --locked
 
+[group('testing')]
+run-tests: _ensure_db_and_s3_are_running _build-test-binaries
+  #!/usr/bin/env bash
+  set -euo pipefail
+
+  export $(grep -v '^#' .env.tests | xargs)
+  export DOCSRS_INCLUDE_DEFAULT_TARGETS=false
+  export DOCSRS_LOG=docs_rs=debug,rustwide=info
+  export RUST_BACKTRACE=1
 
-run-tests: ensure_db_and_s3_are_running build-tests
   cargo test --workspace --locked --no-fail-fast
 
-run-builder-tests: ensure_db_and_s3_are_running build-tests
-  #!/bin/bash
+[group('testing')]
+[group('builder')]
+run-builder-tests: _ensure_db_and_s3_are_running _build-test-binaries
+  #!/usr/bin/env bash
   set -euo pipefail
 
+  export $(grep -v '^#' .env.tests | xargs)
   export DOCSRS_INCLUDE_DEFAULT_TARGETS=true
+  export DOCSRS_DOCKER_IMAGE=ghcr.io/rust-lang/crates-build-env/linux-micro
+  export DOCSRS_LOG=docs_rs=debug,rustwide=info
+  export RUST_BACKTRACE=1
 
   cargo test --locked -- --ignored --test-threads=1

justfiles/utils.just

@@ -1,5 +1,5 @@
 
-ensure_db_and_s3_are_running: _touch-docker-env
+_ensure_db_and_s3_are_running: _touch-docker-env
   # dependencies in the docker-cli file are ignored 
   # here. Instead we explicitly start any dependent services first.
   docker compose up -d db s3 --wait