diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 867e13132..a2da8e6d5 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -39,12 +39,12 @@ jobs: id: file # Run benchmarks - - name: Run benchmarks - Dataset ${{ github.event.inputs.dataset_name }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} + - name: Run benchmarks - Dataset ${{ github.event.inputs.dataset_name }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }} run: | cd benchmarks cargo bench --bench ${{ github.event.inputs.dataset_name }} -- --save-baseline ${{ steps.file.outputs.basename }} - # Generate critcmpf files + # Generate critcmp files - name: Install critcmp run: cargo install critcmp - name: Export cripcmp file @@ -52,7 +52,7 @@ jobs: critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json # Upload benchmarks - - name: Upload to DO Spaces # DigitalOcean Spaces = S3 + - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3 uses: BetaHuhn/do-spaces-action@v2 with: access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }} @@ -61,3 +61,11 @@ jobs: space_region: ${{ secrets.DO_SPACES_SPACE_REGION }} source: ${{ steps.file.outputs.basename }}.json out_dir: critcmp_results + + # Helper + - name: 'README: compare with another benchmark' + run: | + echo "${{ steps.file.outputs.basename }}.json has just been pushed." + echo 'How to compare this benchmark with another one?' + echo ' - Check the available files with: ./benchmarks/scripts/list.sh' + echo " - Run the following command: ./benchmaks/scipts/compare.sh ${{ steps.file.outputs.basename }}.json " diff --git a/benchmarks/README.md b/benchmarks/README.md index cde4062e5..caa4e163f 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -52,8 +52,12 @@ To trigger the benchmark workflow: This GitHub workflow will run the benchmarks and push the `critcmp` report to a DigitalOcean Space (= S3). +The name of the uploaded file is displayed in the workflow. + _[More about critcmp](https://github.com/BurntSushi/critcmp)._ +💡 To compare the just-uploaded benchmark with another one, check out the [next section](#comparison-between-benchmarks). + ### On your machine To run all the benchmarks (~4h): @@ -85,13 +89,14 @@ The benchmark reports we push are generated with `critcmp`. Thus, we use `critcm We provide a script to download and display the comparison report. Requirements: -- [`s3cmd`](https://github.com/s3tools/s3cmd) and being logged to the DigitalOcean Space "milli-benchmarks". See the [DigitalOcean guide](https://docs.digitalocean.com/products/spaces/resources/s3cmd/) +- `grep` +- `curl` - [`critcmp`](https://github.com/BurntSushi/critcmp) List the available file in the DO Space: ```bash -s3cmd ls s3://milli-benchmarks/critcmp_results/ +./benchmarks/script/list.sh ``` ```bash 2021-05-31 14:40 279890 s3://milli-benchmarks/critcmp_results/songs_main_09a4321.json @@ -101,5 +106,5 @@ s3cmd ls s3://milli-benchmarks/critcmp_results/ Run the comparison script: ```bash -bash benchmarks/scripts/compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json +./benchmarks/scripts/compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json ``` diff --git a/benchmarks/scripts/compare.sh b/benchmarks/scripts/compare.sh index e4231131d..6f8d0c5af 100755 --- a/benchmarks/scripts/compare.sh +++ b/benchmarks/scripts/compare.sh @@ -2,7 +2,7 @@ # Requirements: # - critcmp. See: https://github.com/BurntSushi/critcmp -# - wget +# - curl # Usage # $ bash compare.sh json_file1 json_file1 @@ -17,13 +17,6 @@ if [[ "$?" -ne 0 ]]; then exit 1 fi -# Checking that wget is installed -command -v wget > /dev/null 2>&1 -if [[ "$?" -ne 0 ]]; then - echo 'You must install wget to make this script working.' - exit 1 -fi - if [[ $# -ne 2 ]] then echo 'Need 2 arguments.' @@ -43,9 +36,9 @@ file1_local_path="/tmp/$file1" file2_local_path="/tmp/$file2" if [[ ! -f "$file1_local_path" ]]; then - wget "$file1_s3_url" -O "$file1_local_path" + curl "$file1_s3_url" -O "$file1_local_path" if [[ "$?" -ne 0 ]]; then - echo 'wget command failed. Check your configuration' + echo 'curl command failed. Check your configuration' exit 1 fi else @@ -53,9 +46,9 @@ else fi if [[ ! -f "$file2_local_path" ]]; then - wget "$file2_s3_url" -O "$file2_local_path" + curl "$file2_s3_url" -O "$file2_local_path" if [[ "$?" -ne 0 ]]; then - echo 'wget command failed. Check your configuration' + echo 'curl command failed. Check your configuration' exit 1 fi else