diff --git a/.dockerignore b/.dockerignore index ef2e39bfc..8c6bdbdeb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,4 +2,3 @@ target Dockerfile .dockerignore .gitignore -**/.git diff --git a/.github/workflows/publish-apt-brew-pkg.yml b/.github/workflows/publish-apt-brew-pkg.yml index e24d8ccf1..f7ab8666b 100644 --- a/.github/workflows/publish-apt-brew-pkg.yml +++ b/.github/workflows/publish-apt-brew-pkg.yml @@ -35,7 +35,7 @@ jobs: - name: Build deb package run: cargo deb -p meilisearch -o target/debian/meilisearch.deb - name: Upload debian pkg to release - uses: svenstaro/upload-release-action@2.5.0 + uses: svenstaro/upload-release-action@2.6.1 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/debian/meilisearch.deb diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 76558f3b1..c79176439 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -54,7 +54,7 @@ jobs: # No need to upload binaries for dry run (cron) - name: Upload binaries to release if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.5.0 + uses: svenstaro/upload-release-action@2.6.1 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/release/meilisearch @@ -87,7 +87,7 @@ jobs: # No need to upload binaries for dry run (cron) - name: Upload binaries to release if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.5.0 + uses: svenstaro/upload-release-action@2.6.1 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/release/${{ matrix.artifact_name }} @@ -121,7 +121,7 @@ jobs: - name: Upload the binary to release # No need to upload binaries for dry run (cron) if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.5.0 + uses: svenstaro/upload-release-action@2.6.1 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/${{ matrix.target }}/release/meilisearch @@ -183,7 +183,7 @@ jobs: - name: Upload the binary to release # No need to upload binaries for dry run (cron) if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.5.0 + uses: svenstaro/upload-release-action@2.6.1 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/${{ matrix.target }}/release/meilisearch diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index fa1f145da..9ceeaaaa4 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -58,13 +58,9 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2 - with: - platforms: linux/amd64,linux/arm64 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - with: - platforms: linux/amd64,linux/arm64 - name: Login to Docker Hub uses: docker/login-action@v2 @@ -92,13 +88,10 @@ jobs: push: true platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} - builder: ${{ steps.buildx.outputs.name }} build-args: | COMMIT_SHA=${{ github.sha }} COMMIT_DATE=${{ steps.build-metadata.outputs.date }} GIT_TAG=${{ github.ref_name }} - cache-from: type=gha - cache-to: type=gha,mode=max # /!\ Don't touch this without checking with Cloud team - name: Send CI information to Cloud team diff --git a/.github/workflows/sdks-tests.yml b/.github/workflows/sdks-tests.yml index 617b446d1..fedcb9997 100644 --- a/.github/workflows/sdks-tests.yml +++ b/.github/workflows/sdks-tests.yml @@ -3,6 +3,11 @@ name: SDKs tests on: workflow_dispatch: + inputs: + docker_image: + description: 'The Meilisearch Docker image used' + required: false + default: nightly schedule: - cron: "0 6 * * MON" # Every Monday at 6:00AM @@ -17,7 +22,7 @@ jobs: runs-on: ubuntu-latest services: meilisearch: - image: getmeili/meilisearch:nightly + image: getmeili/meilisearch:${{ github.event.inputs.docker_image }} env: MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }} MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }} @@ -51,7 +56,7 @@ jobs: runs-on: ubuntu-latest services: meilisearch: - image: getmeili/meilisearch:nightly + image: getmeili/meilisearch:${{ github.event.inputs.docker_image }} env: MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }} MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }} @@ -77,7 +82,7 @@ jobs: runs-on: ubuntu-latest services: meilisearch: - image: getmeili/meilisearch:nightly + image: getmeili/meilisearch:${{ github.event.inputs.docker_image }} env: MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }} MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }} @@ -107,7 +112,7 @@ jobs: runs-on: ubuntu-latest services: meilisearch: - image: getmeili/meilisearch:nightly + image: getmeili/meilisearch:${{ github.event.inputs.docker_image }} env: MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }} MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }} @@ -131,7 +136,7 @@ jobs: runs-on: ubuntu-latest services: meilisearch: - image: getmeili/meilisearch:nightly + image: getmeili/meilisearch:${{ github.event.inputs.docker_image }} env: MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }} MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }} @@ -139,7 +144,7 @@ jobs: - '7700:7700' steps: - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: stable - uses: actions/checkout@v3 @@ -160,7 +165,7 @@ jobs: runs-on: ubuntu-latest services: meilisearch: - image: getmeili/meilisearch:nightly + image: getmeili/meilisearch:${{ github.event.inputs.docker_image }} env: MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }} MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }} @@ -184,7 +189,7 @@ jobs: runs-on: ubuntu-latest services: meilisearch: - image: getmeili/meilisearch:nightly + image: getmeili/meilisearch:${{ github.event.inputs.docker_image }} env: MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }} MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }} diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 5a64feda8..e363d36c8 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -43,7 +43,7 @@ jobs: toolchain: nightly override: true - name: Cache dependencies - uses: Swatinem/rust-cache@v2.2.1 + uses: Swatinem/rust-cache@v2.4.0 - name: Run cargo check without any default features uses: actions-rs/cargo@v1 with: @@ -65,7 +65,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Cache dependencies - uses: Swatinem/rust-cache@v2.2.1 + uses: Swatinem/rust-cache@v2.4.0 - name: Run cargo check without any default features uses: actions-rs/cargo@v1 with: @@ -105,6 +105,29 @@ jobs: command: test args: --workspace --locked --release --all-features + test-disabled-tokenization: + name: Test disabled tokenization + runs-on: ubuntu-latest + container: + image: ubuntu:18.04 + if: github.event_name == 'schedule' + steps: + - uses: actions/checkout@v3 + - name: Install needed dependencies + run: | + apt-get update + apt-get install --assume-yes build-essential curl + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Run cargo tree without default features and check lindera is not present + run: | + cargo tree -f '{p} {f}' -e normal --no-default-features | grep lindera -vqz + - name: Run cargo tree with default features and check lindera is pressent + run: | + cargo tree -f '{p} {f}' -e normal | grep lindera -qz + # We run tests in debug also, to make sure that the debug_assertions are hit test-debug: name: Run tests in debug @@ -123,7 +146,7 @@ jobs: toolchain: stable override: true - name: Cache dependencies - uses: Swatinem/rust-cache@v2.2.1 + uses: Swatinem/rust-cache@v2.4.0 - name: Run tests in debug uses: actions-rs/cargo@v1 with: @@ -142,7 +165,7 @@ jobs: override: true components: clippy - name: Cache dependencies - uses: Swatinem/rust-cache@v2.2.1 + uses: Swatinem/rust-cache@v2.4.0 - name: Run cargo clippy uses: actions-rs/cargo@v1 with: @@ -161,7 +184,7 @@ jobs: override: true components: rustfmt - name: Cache dependencies - uses: Swatinem/rust-cache@v2.2.1 + uses: Swatinem/rust-cache@v2.4.0 - name: Run cargo fmt # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file. # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate diff --git a/Dockerfile b/Dockerfile index 0d7593f79..70950f338 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,3 @@ -# syntax=docker/dockerfile:1.4 # Compile FROM rust:alpine3.16 AS compiler @@ -12,7 +11,7 @@ ARG GIT_TAG ENV VERGEN_GIT_SHA=${COMMIT_SHA} VERGEN_GIT_COMMIT_TIMESTAMP=${COMMIT_DATE} VERGEN_GIT_SEMVER_LIGHTWEIGHT=${GIT_TAG} ENV RUSTFLAGS="-C target-feature=-crt-static" -COPY --link . . +COPY . . RUN set -eux; \ apkArch="$(apk --print-arch)"; \ if [ "$apkArch" = "aarch64" ]; then \ @@ -31,7 +30,7 @@ RUN apk update --quiet \ # add meilisearch to the `/bin` so you can run it from anywhere and it's easy # to find. -COPY --from=compiler --link /meilisearch/target/release/meilisearch /bin/meilisearch +COPY --from=compiler /meilisearch/target/release/meilisearch /bin/meilisearch # To stay compatible with the older version of the container (pre v0.27.0) we're # going to symlink the meilisearch binary in the path to `/meilisearch` RUN ln -s /bin/meilisearch /meilisearch diff --git a/config.toml b/config.toml index e5b339ffa..c47989f56 100644 --- a/config.toml +++ b/config.toml @@ -1,131 +1,131 @@ # This file shows the default configuration of Meilisearch. # All variables are defined here: https://www.meilisearch.com/docs/learn/configuration/instance_options#environment-variables -db_path = "./data.ms" # Designates the location where database files will be created and retrieved. # https://www.meilisearch.com/docs/learn/configuration/instance_options#database-path +db_path = "./data.ms" -env = "development" # Configures the instance's environment. Value must be either `production` or `development`. # https://www.meilisearch.com/docs/learn/configuration/instance_options#environment +env = "development" -http_addr = "localhost:7700" # The address on which the HTTP server will listen. +http_addr = "localhost:7700" -# master_key = "YOUR_MASTER_KEY_VALUE" # Sets the instance's master key, automatically protecting all routes except GET /health. # https://www.meilisearch.com/docs/learn/configuration/instance_options#master-key +# master_key = "YOUR_MASTER_KEY_VALUE" -# no_analytics = true # Deactivates Meilisearch's built-in telemetry when provided. # Meilisearch automatically collects data from all instances that do not opt out using this flag. # All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted at any time. # https://www.meilisearch.com/docs/learn/configuration/instance_options#disable-analytics +# no_analytics = true -http_payload_size_limit = "100 MB" # Sets the maximum size of accepted payloads. # https://www.meilisearch.com/docs/learn/configuration/instance_options#payload-limit-size +http_payload_size_limit = "100 MB" -log_level = "INFO" # Defines how much detail should be present in Meilisearch's logs. # Meilisearch currently supports six log levels, listed in order of increasing verbosity: `OFF`, `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE` # https://www.meilisearch.com/docs/learn/configuration/instance_options#log-level +log_level = "INFO" -# max_indexing_memory = "2 GiB" # Sets the maximum amount of RAM Meilisearch can use when indexing. # https://www.meilisearch.com/docs/learn/configuration/instance_options#max-indexing-memory +# max_indexing_memory = "2 GiB" -# max_indexing_threads = 4 # Sets the maximum number of threads Meilisearch can use during indexing. # https://www.meilisearch.com/docs/learn/configuration/instance_options#max-indexing-threads +# max_indexing_threads = 4 ############# ### DUMPS ### ############# -dump_dir = "dumps/" # Sets the directory where Meilisearch will create dump files. # https://www.meilisearch.com/docs/learn/configuration/instance_options#dump-directory +dump_dir = "dumps/" -# import_dump = "./path/to/my/file.dump" # Imports the dump file located at the specified path. Path must point to a .dump file. # https://www.meilisearch.com/docs/learn/configuration/instance_options#import-dump +# import_dump = "./path/to/my/file.dump" -ignore_missing_dump = false # Prevents Meilisearch from throwing an error when `import_dump` does not point to a valid dump file. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-missing-dump +ignore_missing_dump = false -ignore_dump_if_db_exists = false # Prevents a Meilisearch instance with an existing database from throwing an error when using `import_dump`. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-dump-if-db-exists +ignore_dump_if_db_exists = false ################# ### SNAPSHOTS ### ################# -schedule_snapshot = false # Enables scheduled snapshots when true, disable when false (the default). # If the value is given as an integer, then enables the scheduled snapshot with the passed value as the interval # between each snapshot, in seconds. # https://www.meilisearch.com/docs/learn/configuration/instance_options#schedule-snapshot-creation +schedule_snapshot = false -snapshot_dir = "snapshots/" # Sets the directory where Meilisearch will store snapshots. # https://www.meilisearch.com/docs/learn/configuration/instance_options#snapshot-destination +snapshot_dir = "snapshots/" -# import_snapshot = "./path/to/my/snapshot" # Launches Meilisearch after importing a previously-generated snapshot at the given filepath. # https://www.meilisearch.com/docs/learn/configuration/instance_options#import-snapshot +# import_snapshot = "./path/to/my/snapshot" -ignore_missing_snapshot = false # Prevents a Meilisearch instance from throwing an error when `import_snapshot` does not point to a valid snapshot file. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-missing-snapshot +ignore_missing_snapshot = false -ignore_snapshot_if_db_exists = false # Prevents a Meilisearch instance with an existing database from throwing an error when using `import_snapshot`. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-snapshot-if-db-exists +ignore_snapshot_if_db_exists = false ########### ### SSL ### ########### -# ssl_auth_path = "./path/to/root" # Enables client authentication in the specified path. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-authentication-path +# ssl_auth_path = "./path/to/root" -# ssl_cert_path = "./path/to/certfile" # Sets the server's SSL certificates. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-certificates-path +# ssl_cert_path = "./path/to/certfile" -# ssl_key_path = "./path/to/private-key" # Sets the server's SSL key files. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-key-path +# ssl_key_path = "./path/to/private-key" -# ssl_ocsp_path = "./path/to/ocsp-file" # Sets the server's OCSP file. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-ocsp-path +# ssl_ocsp_path = "./path/to/ocsp-file" -ssl_require_auth = false # Makes SSL authentication mandatory. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-require-auth +ssl_require_auth = false -ssl_resumption = false # Activates SSL session resumption. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-resumption +ssl_resumption = false -ssl_tickets = false # Activates SSL tickets. # https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-tickets +ssl_tickets = false ############################# ### Experimental features ### ############################# -experimental_enable_metrics = false # Experimental metrics feature. For more information, see: # Enables the Prometheus metrics on the `GET /metrics` endpoint. +experimental_enable_metrics = false -experimental_reduce_indexing_memory_usage = false # Experimental RAM reduction during indexing, do not use in production, see: +experimental_reduce_indexing_memory_usage = false diff --git a/milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs b/milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs index 1bd132974..01ce523ba 100644 --- a/milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs +++ b/milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs @@ -49,7 +49,7 @@ impl CboRoaringBitmapCodec { } else { // Otherwise, it means we used the classic RoaringBitmapCodec and // that the header takes threshold integers. - RoaringBitmap::deserialize_from(bytes) + RoaringBitmap::deserialize_unchecked_from(bytes) } } @@ -69,7 +69,7 @@ impl CboRoaringBitmapCodec { vec.push(integer); } } else { - roaring |= RoaringBitmap::deserialize_from(bytes.as_ref())?; + roaring |= RoaringBitmap::deserialize_unchecked_from(bytes.as_ref())?; } } diff --git a/milli/src/heed_codec/roaring_bitmap/roaring_bitmap_codec.rs b/milli/src/heed_codec/roaring_bitmap/roaring_bitmap_codec.rs index 8fae9b8fd..6cec0eb44 100644 --- a/milli/src/heed_codec/roaring_bitmap/roaring_bitmap_codec.rs +++ b/milli/src/heed_codec/roaring_bitmap/roaring_bitmap_codec.rs @@ -8,7 +8,7 @@ impl heed::BytesDecode<'_> for RoaringBitmapCodec { type DItem = RoaringBitmap; fn bytes_decode(bytes: &[u8]) -> Option { - RoaringBitmap::deserialize_from(bytes).ok() + RoaringBitmap::deserialize_unchecked_from(bytes).ok() } }