From 72f8fb41529e3bf385e37be8638a69be8fcd13bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20A=2E=20Matienzo?= Date: Mon, 24 Nov 2025 12:22:21 -0800 Subject: [PATCH 1/2] AP-501: set up github actions This makes some notable changes to the NARA build in addition to setting up Github Actions: * Switch from previously custom-built MariaDB image (lap/nara-db) to using a stock MariaDB 10 image. * Ship with Docker Compose changes that that allow us to set a MariaDB server-level setting and load an existing dump file to a local development environment. --- .github/workflows/build.yml | 200 +++++++++++++++++++++++ .github/workflows/release.yml | 59 +++++++ .rspec | 4 +- Jenkinsfile | 20 --- README.md | 17 +- docker-compose.ci.yml | 11 ++ docker-compose.dump.yml | 7 + docker-compose.yml | 32 +--- mariadb/conf.d/nara.cnf | 3 + mariadb/docker-entrypoint-initdb.d/.keep | 0 10 files changed, 303 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/release.yml delete mode 100644 Jenkinsfile create mode 100644 docker-compose.ci.yml create mode 100644 docker-compose.dump.yml create mode 100644 mariadb/conf.d/nara.cnf create mode 100644 mariadb/docker-entrypoint-initdb.d/.keep diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..7b2f3a2 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,200 @@ +name: Build / Test / Push + +on: + push: + branches: + - '**' + workflow_dispatch: + +env: + BUILD_SUFFIX: -build-${{ github.run_id }}_${{ github.run_attempt }} + DOCKER_METADATA_SET_OUTPUT_ENV: 'true' + +jobs: + build: + runs-on: ${{ matrix.runner }} + outputs: + image-arm64: ${{ steps.gen-output.outputs.image-arm64 }} + image-x64: ${{ steps.gen-output.outputs.image-x64 }} + strategy: + fail-fast: false + matrix: + runner: + - ubuntu-24.04 + - ubuntu-24.04-arm + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: build-meta + name: Docker meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=sha,suffix=${{ env.BUILD_SUFFIX }} + + # Build cache is shared among all builds of the same architecture + - id: cache-meta + name: Docker meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=raw,value=buildcache-${{ runner.arch }} + + - id: get-registry + name: Get the sanitized registry name + run: | + echo "registry=$(echo '${{ steps.build-meta.outputs.tags }}' | cut -f1 -d:)" | tee -a "$GITHUB_OUTPUT" + + - id: build + name: Build/push the arch-specific image + uses: docker/build-push-action@v6 + with: + cache-from: type=registry,ref=${{ steps.cache-meta.outputs.tags }} + cache-to: type=registry,ref=${{ steps.cache-meta.outputs.tags }},mode=max + labels: ${{ steps.build-meta.outputs.labels }} + provenance: mode=max + sbom: true + tags: ${{ steps.get-registry.outputs.registry }} + outputs: type=image,push-by-digest=true,push=true + + - id: gen-output + name: Write arch-specific image digest to outputs + run: | + echo "image-${RUNNER_ARCH,,}=${{ steps.get-registry.outputs.registry }}@${{ steps.build.outputs.digest }}" | tee -a "$GITHUB_OUTPUT" + + merge: + runs-on: ubuntu-24.04 + needs: build + env: + DOCKER_APP_IMAGE_ARM64: ${{ needs.build.outputs.image-arm64 }} + DOCKER_APP_IMAGE_X64: ${{ needs.build.outputs.image-x64 }} + outputs: + image: ${{ steps.meta.outputs.tags }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: meta + name: Generate tag for the app image + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=sha,suffix=${{ env.BUILD_SUFFIX }} + + - name: Push the multi-platform app image + run: | + docker buildx imagetools create \ + --tag "$DOCKER_METADATA_OUTPUT_TAGS" \ + "$DOCKER_APP_IMAGE_ARM64" "$DOCKER_APP_IMAGE_X64" + + test: + runs-on: ubuntu-24.04 + needs: merge + env: + COMPOSE_FILE: docker-compose.yml:docker-compose.ci.yml + DOCKER_APP_IMAGE: ${{ needs.merge.outputs.image }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Compose + uses: docker/setup-compose-action@v1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup the stack + run: | + docker compose build --quiet + docker compose pull --quiet + docker compose up --wait + docker compose exec app rails assets:precompile + docker compose exec -u root app chown -R nara:nara artifacts + + - name: Run RSpec + if: ${{ always() }} + run: | + docker compose exec -e RAILS_ENV=test app rake check + + - name: Run Rubocop + if: ${{ always() }} + run: | + docker compose exec -e RAILS_ENV=test app rubocop --format progress --format html --out artifacts/rubocop.html + + - name: Run Brakeman + if: ${{ always() }} + run: | + docker compose exec -e RAILS_ENV=test app brakeman -o artifacts/brakeman.html + + - name: Copy out artifacts + if: ${{ always() }} + run: | + docker compose cp app:/opt/app/artifacts ./ + docker compose logs > artifacts/docker-compose-services.log + docker compose config > artifacts/docker-compose.merged.yml + + - name: Upload the test report + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: ruby-nara Build Report (${{ github.run_id }}_${{ github.run_attempt }}) + path: artifacts/* + if-no-files-found: error + + push: + runs-on: ubuntu-24.04 + needs: + - merge + - test + env: + DOCKER_APP_IMAGE: ${{ needs.merge.outputs.image }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Produce permanent image tags + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=sha + type=ref,event=branch + type=raw,value=latest,enable={{is_default_branch}} + + - name: Retag and push the image + run: | + docker pull "$DOCKER_APP_IMAGE" + echo "$DOCKER_METADATA_OUTPUT_TAGS" | tr ' ' '\n' | xargs -n1 docker tag "$DOCKER_APP_IMAGE" + docker push --all-tags "$(echo "$DOCKER_APP_IMAGE" | cut -f1 -d:)" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..5d90890 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,59 @@ +name: Push Release Tags + +on: + push: + tags: + - '**' + workflow_dispatch: + +env: + DOCKER_METADATA_SET_OUTPUT_ENV: 'true' + +jobs: + retag: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Determine the sha-based image tag to retag + id: get-base-image + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: type=sha + + - name: Verify that the image was previously built + env: + BASE_IMAGE: ${{ steps.get-base-image.outputs.tags }} + run: | + docker pull "$BASE_IMAGE" + + - name: Produce release tags + id: tag-meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + flavor: latest=false + tags: | + type=ref,event=tag + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + + - name: Retag the pulled image + env: + BASE_IMAGE: ${{ steps.get-base-image.outputs.tags }} + run: | + echo "$DOCKER_METADATA_OUTPUT_TAGS" | tr ' ' '\n' | xargs -n1 docker tag "$BASE_IMAGE" + docker push --all-tags "$(echo "$BASE_IMAGE" | cut -f1 -d:)" diff --git a/.rspec b/.rspec index 52af74a..7f56466 100644 --- a/.rspec +++ b/.rspec @@ -1,4 +1,4 @@ --require spec_helper --format progress ---format RspecJunitFormatter ---out artifacts/rspec/specs.xml +--format html +--out artifacts/rspec.html diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 2926f7d..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,20 +0,0 @@ -dockerComposePipeline( - stack: [template: 'mariadb'], - commands: [ - [ - [exec: 'rake check RAILS_ENV=test'], - 'rake rubocop', - 'rake brakeman', - 'rake bundle:audit', - ], - ], - artifacts: [ - junit : 'artifacts/rspec/**/*.xml', - html : [ - 'Code Coverage': 'artifacts/rcov', - 'RuboCop' : 'artifacts/rubocop', - 'Brakeman' : 'artifacts/brakeman' - ], - raw : 'artifacts/screenshots/**/*.png' - ] -) diff --git a/README.md b/README.md index 94940d5..84859a7 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,23 @@ # NARA casefiles web database -Rails web app for viewing the NARA case files database. This application runs on VM154 using port 3001. The database itself runs on mysql.lib.berkeley.edu. +Rails web app for viewing the NARA case files database. This application runs on Docker Swarm. -The Nara database app can be found under the existing Chinese immigration website at https://bancroft.berkeley.edu/collections/chinese-immigration-to-the-united-states-1884-1944/ alternatively, you can visit the website at https://nara.lib.berkeley.edu/ +The Nara database app can be found under the archived Chinese immigration website at . Alternatively, you can visit the website at https://nara.lib.berkeley.edu/ + +## Local development + +Local development and CI is handled by Docker Compose. The app requires a custom MariaDB configuration (see [`mariadb/conf.d/nara.cnf`](dmariadb/conf.d/nara.cnf)). This should get automatically loaded as you bring the container up locally, but this setting needs to be deployed to the Swarm configuration. + +By default, the application does not ship with any data, but you can load some if you place a dump file in `mariadb/docker-entrypoint-initdb.d` and bring the containers up with a supplemental Compose file, e.g. + +```bash +cp nara-dump.sql.gz mariadb/docker-entrypoint-initdb.d +docker compose -f docker-compose.yml -f docker-compose.dump.yml up -d +``` ## Testing -Nara uses a combination of Rspec and Capybara, for it's test suites. To run a suite from the command line, run 'rspec' from the projects root directory. +Nara uses a combination of Rspec and Capybara for its test suites. To run a suite from the command line, run 'rspec' from the projects root directory. ## SimpleCov diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 0000000..ab855c9 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,11 @@ +--- + +services: + app: + build: !reset + image: ${DOCKER_APP_IMAGE} + volumes: !override + - artifacts:/opt/app/artifacts + +volumes: + artifacts: diff --git a/docker-compose.dump.yml b/docker-compose.dump.yml new file mode 100644 index 0000000..de896e3 --- /dev/null +++ b/docker-compose.dump.yml @@ -0,0 +1,7 @@ +services: + db: + environment: + - MYSQL_DATABASE=NARA + volumes: + - ./mariadb/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0eeb354..ab78609 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,38 +1,22 @@ -# ###################################################################### -# NOTE -# ###################################################################### -# -# This docker-compose file uses a custom database image from lap/nara-db, -# published in the GitLab container registry (containers.lib.berkeley.edu). -# -# Before running `docker-compose`, use `docker login containers.lib.berkeley.edu` -# to authenticate. -# -# (See https://git.lib.berkeley.edu/help/user/packages/container_registry/index.md#authenticate-with-the-container-registry) - services: -# adminer: -# image: adminer -# restart: always -# ports: -# - 8080:8080 db: - # For local development, set DATABASE_URL=mysql2://root:root@0.0.0.0:33306/NARA - # or use `mysql -u root -proot -h 0.0.0.0 -P 33306 -D NARA` - image: containers.lib.berkeley.edu/lap/nara-db/master:latest + image: mariadb:10 environment: - # Default password for the image is empty, but adminer doesn't support - # that (https://www.adminer.org/en/password/), so we use 'root' + # Default password for the image is empty MYSQL_ROOT_PASSWORD: root restart: always ports: - 3306:3306 + volumes: + - ./mariadb/conf.d:/etc/mysql/conf.d - rails: + app: build: context: . target: development + depends_on: + - db environment: DATABASE_URL: 'mysql2://root:root@db:3306/NARA' ports: @@ -40,5 +24,3 @@ services: restart: always volumes: - ./:/opt/app:delegated - -version: "3.7" diff --git a/mariadb/conf.d/nara.cnf b/mariadb/conf.d/nara.cnf new file mode 100644 index 0000000..efb2d6e --- /dev/null +++ b/mariadb/conf.d/nara.cnf @@ -0,0 +1,3 @@ +[mysqld] + +ft_min_word_len = 3 \ No newline at end of file diff --git a/mariadb/docker-entrypoint-initdb.d/.keep b/mariadb/docker-entrypoint-initdb.d/.keep new file mode 100644 index 0000000..e69de29 From 946dbb5aacd271044667c5303631fd365c8fa3f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20A=2E=20Matienzo?= Date: Mon, 24 Nov 2025 14:10:31 -0800 Subject: [PATCH 2/2] address PR review --- README.md | 11 +++-------- docker-compose.ci.yml | 3 +++ docker-compose.dump.yml | 7 ------- docker-compose.yml | 4 +++- mariadb/conf.d/nara.cnf | 3 --- mariadb/docker-entrypoint-initdb.d/.keep | 0 6 files changed, 9 insertions(+), 19 deletions(-) delete mode 100644 docker-compose.dump.yml delete mode 100644 mariadb/conf.d/nara.cnf delete mode 100644 mariadb/docker-entrypoint-initdb.d/.keep diff --git a/README.md b/README.md index 84859a7..4c00fcd 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,11 @@ Rails web app for viewing the NARA case files database. This application runs on The Nara database app can be found under the archived Chinese immigration website at . Alternatively, you can visit the website at https://nara.lib.berkeley.edu/ -## Local development +## Local development and CI -Local development and CI is handled by Docker Compose. The app requires a custom MariaDB configuration (see [`mariadb/conf.d/nara.cnf`](dmariadb/conf.d/nara.cnf)). This should get automatically loaded as you bring the container up locally, but this setting needs to be deployed to the Swarm configuration. +Local development and CI is handled by Docker Compose. The app requires a custom MariaDB configuration variable (see [`docker-compose.yml`](docker-compose.yml)). This should get automatically loaded as you bring the container up locally, but this setting needs to be deployed to the Swarm configuration. -By default, the application does not ship with any data, but you can load some if you place a dump file in `mariadb/docker-entrypoint-initdb.d` and bring the containers up with a supplemental Compose file, e.g. - -```bash -cp nara-dump.sql.gz mariadb/docker-entrypoint-initdb.d -docker compose -f docker-compose.yml -f docker-compose.dump.yml up -d -``` +By default, Docker Compose will load sample data given a file in `db/dumps`. In CI, no sample data is loaded. ## Testing diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index ab855c9..10ceb7f 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -7,5 +7,8 @@ services: volumes: !override - artifacts:/opt/app/artifacts + db: + volumes: !reset + volumes: artifacts: diff --git a/docker-compose.dump.yml b/docker-compose.dump.yml deleted file mode 100644 index de896e3..0000000 --- a/docker-compose.dump.yml +++ /dev/null @@ -1,7 +0,0 @@ -services: - db: - environment: - - MYSQL_DATABASE=NARA - volumes: - - ./mariadb/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d - \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ab78609..8ebeaae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,11 +5,13 @@ services: environment: # Default password for the image is empty MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: NARA restart: always ports: - 3306:3306 + command: --ft-min-word-len=3 volumes: - - ./mariadb/conf.d:/etc/mysql/conf.d + - ./db/dumps:/docker-entrypoint-initdb.d app: build: diff --git a/mariadb/conf.d/nara.cnf b/mariadb/conf.d/nara.cnf deleted file mode 100644 index efb2d6e..0000000 --- a/mariadb/conf.d/nara.cnf +++ /dev/null @@ -1,3 +0,0 @@ -[mysqld] - -ft_min_word_len = 3 \ No newline at end of file diff --git a/mariadb/docker-entrypoint-initdb.d/.keep b/mariadb/docker-entrypoint-initdb.d/.keep deleted file mode 100644 index e69de29..0000000