Merge pull request #718 from meilisearch/add-more-analytics-reporting

Add more analytics

CHANGELOG.md

@@ -20,6 +20,7 @@
   - Rename fieldsFrequency into fieldsDistribution in stats (#719)
   - Add support for error code reporting (#703)
   - Allow the dashboard to query private servers (#732)
+  - Add telemetry (#720)
 
 ## v0.10.1
 

README.md

@@ -6,11 +6,11 @@
 <h1 align="center">MeiliSearch</h1>
 
 <h4 align="center">
-  <a href="https://www.meilisearch.com">Website</a> | 
-  <a href="https://blog.meilisearch.com">Blog</a> | 
-  <a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> | 
-  <a href="https://twitter.com/meilisearch">Twitter</a> | 
-  <a href="https://docs.meilisearch.com">Documentation</a> | 
+  <a href="https://www.meilisearch.com">Website</a> |
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> |
+  <a href="https://twitter.com/meilisearch">Twitter</a> |
+  <a href="https://docs.meilisearch.com">Documentation</a> |
   <a href="https://docs.meilisearch.com/faq/">FAQ</a>
 </h4>
 
@@ -122,7 +122,7 @@ curl -i -X POST 'http://127.0.0.1:7700/indexes/movies/documents' \
 
 #### In command line
 
-The search engine is now aware of your documents and can serve those via an HTTP server. 
+The search engine is now aware of your documents and can serve those via an HTTP server.
 
 The [`jq` command-line tool](https://stedolan.github.io/jq/) can greatly help you read the server responses.
 
@@ -167,7 +167,7 @@ You can access the web interface in your web browser at the root of the server.
 
 ### Documentation
 
-Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com). 
+Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com).
 
 
 ## Contributing
@@ -176,8 +176,8 @@ Hey! We're glad you're thinking about contributing to MeiliSearch! If you think
 
 ### Analytic Events
 
-Once a day, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.<br/>
-Only information about the platform on which the server runs is stored. No other information is being sent.<br/>
+Every hour, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.<br/>
+To see what information we're retrieving, please see the complete list [on the dedicated issue](https://github.com/meilisearch/MeiliSearch/issues/720).<br/>
 We also use Sentry to make us crash and error reports. If you want to know more about what Sentry collects, please visit their [privacy policy website](https://sentry.io/privacy/).<br/>
 If this doesn't suit you, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
 

meilisearch-http/src/analytics.rs

@@ -1,20 +1,72 @@
 use std::hash::{Hash, Hasher};
-use std::thread;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::{error, thread};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 
 use log::error;
 use serde::Serialize;
 use serde_qs as qs;
 use siphasher::sip::SipHasher;
+use walkdir::WalkDir;
+
+use crate::Data;
+use crate::Opt;
 
 const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47";
 
+#[derive(Debug, Serialize)]
+struct EventProperties {
+    database_size: u64,
+    last_update_timestamp: Option<i64>, //timestamp
+    number_of_documents: Vec<u64>,
+}
+
+impl EventProperties {
+    fn from(data: Data) -> Result<EventProperties, Box<dyn error::Error>> {
+        let mut index_list = Vec::new();
+
+        let reader = data.db.main_read_txn()?;
+
+        for index_uid in data.db.indexes_uids() {
+            if let Some(index) = data.db.open_index(&index_uid) {
+                let number_of_documents = index.main.number_of_documents(&reader)?;
+                index_list.push(number_of_documents);
+            }
+        }
+
+        let database_size = WalkDir::new(&data.db_path)
+            .into_iter()
+            .filter_map(|entry| entry.ok())
+            .filter_map(|entry| entry.metadata().ok())
+            .filter(|metadata| metadata.is_file())
+            .fold(0, |acc, m| acc + m.len());
+
+        let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp());
+
+        Ok(EventProperties {
+            database_size,
+            last_update_timestamp,
+            number_of_documents: index_list,
+        })
+    }
+}
+
+#[derive(Debug, Serialize)]
+struct UserProperties<'a> {
+    env: &'a str,
+    start_since_days: u64,
+    user_email: Option<String>,
+    server_provider: Option<String>,
+}
+
 #[derive(Debug, Serialize)]
 struct Event<'a> {
     user_id: &'a str,
     event_type: &'a str,
     device_id: &'a str,
     time: u64,
+    app_version: &'a str,
+    user_properties: UserProperties<'a>,
+    event_properties: Option<EventProperties>,
 }
 
 #[derive(Debug, Serialize)]
@@ -23,7 +75,7 @@ struct AmplitudeRequest<'a> {
     event: &'a str,
 }
 
-pub fn analytics_sender() {
+pub fn analytics_sender(data: Data, opt: Opt) {
     let username = whoami::username();
     let hostname = whoami::hostname();
     let platform = whoami::platform();
@@ -36,6 +88,7 @@ pub fn analytics_sender() {
 
     let uid = format!("{:X}", hash);
     let platform = platform.to_string();
+    let first_start = Instant::now();
 
     loop {
         let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
@@ -43,12 +96,27 @@ pub fn analytics_sender() {
         let device_id = &platform;
         let time = n.as_secs();
         let event_type = "runtime_tick";
+        let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day
+        let event_properties = EventProperties::from(data.clone()).ok();
+        let app_version = env!("CARGO_PKG_VERSION").to_string();
+        let app_version = app_version.as_str();
+        let user_email = std::env::var("MEILI_USER_EMAIL").ok();
+        let server_provider = std::env::var("MEILI_SERVER_PROVIDER").ok();
+        let user_properties = UserProperties {
+            env: &opt.env,
+            start_since_days: elapsed_since_start,
+            user_email,
+            server_provider,
+        };
 
         let event = Event {
             user_id,
             event_type,
             device_id,
             time,
+            app_version,
+            user_properties,
+            event_properties
         };
         let event = serde_json::to_string(&event).unwrap();
 
@@ -64,6 +132,6 @@ pub fn analytics_sender() {
             error!("Unsuccessful call to Amplitude: {}", body);
         }
 
-        thread::sleep(Duration::from_secs(86_400)) // one day
+        thread::sleep(Duration::from_secs(3600)) // one hour
     }
 }

meilisearch-http/src/lib.rs

@@ -6,6 +6,7 @@ pub mod helpers;
 pub mod models;
 pub mod option;
 pub mod routes;
+pub mod analytics;
 
 use actix_http::Error;
 use actix_service::ServiceFactory;
@@ -15,6 +16,7 @@ use log::error;
 
 use meilisearch_core::ProcessedUpdateResult;
 
+pub use option::Opt;
 pub use self::data::Data;
 use self::error::{json_error_handler, ResponseError};
 

meilisearch-http/src/main.rs

@@ -3,10 +3,8 @@ use std::{env, thread};
 use actix_cors::Cors;
 use actix_web::{middleware, HttpServer};
 use main_error::MainError;
-use meilisearch_http::data::Data;
 use meilisearch_http::helpers::NormalizePath;
-use meilisearch_http::option::Opt;
-use meilisearch_http::{create_app, index_update_callback};
+use meilisearch_http::{Data, Opt, create_app, index_update_callback};
 use structopt::StructOpt;
 
 mod analytics;
@@ -49,12 +47,16 @@ async fn main() -> Result<(), MainError> {
         _ => unreachable!(),
     }
 
-    if !opt.no_analytics {
-        thread::spawn(analytics::analytics_sender);
-    }
-
     let data = Data::new(opt.clone());
 
+    if !opt.no_analytics {
+        let analytics_data = data.clone();
+        let analytics_opt = opt.clone();
+        thread::spawn(move|| {
+            analytics::analytics_sender(analytics_data, analytics_opt)
+        });
+    }
+
     let data_cloned = data.clone();
     data.db.set_update_callback(Box::new(move |name, status| {
         index_update_callback(name, &data_cloned, status);