diff --git a/doc/decission-support.md b/doc/decission-support.md new file mode 100644 index 000000000..b4406b690 --- /dev/null +++ b/doc/decission-support.md @@ -0,0 +1,80 @@ +# Decission Support + +JULEA can trace backend accesses granularly, +allowing further application analysis and object access pattern comparison across applications. + +Furthermore, this can be used to provide a first rough estimate of which backend combination +is preferable for the application setup. + +## Access Record + +To create an `access-record`, which is simply a CSV file, execute `julea-server` with `JULEA_TRACE=access` and store the output (`stderr`) in a file. +Then run your applications and stop the server when finished. + +```sh +JULEA_TRACE=access julea-server 2> access-record.csv +``` + +Tracing is per default only in debug builds enabled. To build a realease build +with trace funtcionality you need to set `julea_trace` to true + +```sh +meson configure -Djulea_trace=ture --buildtype=release bld +``` + +## Replay + +An `access-record` can be replayed with `julea-access-replay`, which takes only the `access-record` as input. +Replays will execute the same backend operations in the same order as listed in `access-record`. +This allows comparison of backend performance without running the complete application over and over. + +Note that the backends must be in the same state, preferably empty, before every replay. +Also note that the performance of a replay can vary compared to the original run, depending on the device's load. + +```sh +JULEA_TRACE=access julea-access-replay access-record.csv 2> replay-record.csv +``` + +## Setup Testing + +The script `./scripts/decission-support.py` can help to select a good backend combination for a given workload. +It will test different backend configurations and creates a directory `evaluation` with the results. + +Configurations that should be tested must be defined in a `config.json` file. +The expected structure is as follows (an example config can also be found at `example/decission-support-config.json`): + +```json +{ + "object": [{"backend": "posix", "path": "/tmp//posix"}], + "kv": [{"backend": "sqlite", "path": "/tmp//sqlite"}, {"backend": "sqlite", "path": "/mnt/slow//sqlite"}], + "db": [{"backend": "sqlite", "path": ":memory:"}], +} +``` + +Note that `` inside of paths will be replaced with a temporary directory removed after execution. + +The script will replay each access type with all corresponding backends (e.g., all KV accesses are tested on all specified KV configurations). +This will produce `summary.csv`. + +The summary can be evaluated using `./scripts/decision-support.R`. +An example workflow is shown below. + +```sh +JULEA_TRACE=access julea-server 2> access-record.csv +# run application +# stop julea-server (Ctrl+C) +./scripts/decission-support.py config.json acces-record.csv summary.csv +Rscript ./scripts/decission-support.R summary.csv html > summary.html +# or only ci output +Rscript ./scripts/decission-support.R summary.csv +``` + +Important remarks: +* The script assumes an empty backend state on start. +* A MySQL/MariaDB instance must be provided if specified as backend. + * User: `julea_user` + * PW: `julea_pw` + * Access to the existing (and empty) database `julea_db` + * These DB servers must be reset by hand for now. Resets on server startup will be available in the future and the resets will be automated by the script. + + diff --git a/example/decission-support-config.json b/example/decission-support-config.json new file mode 100644 index 000000000..6ccc2a91d --- /dev/null +++ b/example/decission-support-config.json @@ -0,0 +1,56 @@ +{ + "object": [ + { + "backend": "posix", + "path": "/tmp//" + }, + { + "backend": "posix", + "path": "/mnt/stick//" + }, + { + "backend": "gio", + "path": "/tmp//" + } + ], + "db": [ + { + "backend": "sqlite", + "path": "/tmp//sqlite" + }, + { + "backend": "sqlite", + "path": ":memory:" + }, + { + "backend": "sqlite", + "path": "/mnt/stick//sqlite" + }, + { + "backend": "mysql", + "path": "127.0.0.1:julea_db:julea_user:julea_pw" + } + ], + "kv": [ + { + "backend": "leveldb", + "path": "/tmp//" + }, + { + "backend": "lmdb", + "path": "/tmp//" + }, + { + "backend": "sqlite", + "path": "/tmp//sqlite" + }, + { + "backend": "sqlite", + "path": ":memory:" + }, + { + "backend": "rocksdb", + "path": "/tmp//" + } + ] +} diff --git a/include/core/jconfiguration.h b/include/core/jconfiguration.h index f65738e0d..2e4bb69b4 100644 --- a/include/core/jconfiguration.h +++ b/include/core/jconfiguration.h @@ -115,6 +115,7 @@ guint32 j_configuration_get_max_connections(JConfiguration*); guint64 j_configuration_get_stripe_size(JConfiguration*); gchar const* j_configuration_get_checksum(JConfiguration*); +guint32 j_configuration_get_uid(JConfiguration*); G_END_DECLS diff --git a/include/core/jsemantics.h b/include/core/jsemantics.h index 70d55623d..142ef00d6 100644 --- a/include/core/jsemantics.h +++ b/include/core/jsemantics.h @@ -264,7 +264,35 @@ void j_semantics_set(JSemantics* semantics, JSemanticsType key, gint value); * * \return The aspect's value. **/ -gint j_semantics_get(JSemantics* semantics, JSemanticsType key); +gint j_semantics_get(const JSemantics* semantics, JSemanticsType key); + +/** + * Encodes semantics in a guint32 + * + * \code + * JSemantics* semantics; + * ... + * guint32 serial_semantics = j_semantics_serialize(semantics); + * JSemantics* simular_semantics = j_semantics_deserialize(serial_semantics); + * \encode + * + * \param semnatics The Semantics + * + * \return serialized semantics as LE encoded guint32 + * \retval 0 if no semantics are given eg (semantics == NULL) + **/ +guint32 j_semantics_serialize(const JSemantics* semantics); + +/** + * Decode serialized semantics, and create new Semantics + * + * For a example see j_semantics_serialize() + * + * \param serial_semantics serialized semantics, created with j_semantics_serialize() + * + * \return new created JSemantics + **/ +JSemantics* j_semantics_deserialize(guint32 serial_semantics); /** * @} diff --git a/include/core/jtrace.h b/include/core/jtrace.h index b490cc289..f7b73e87c 100644 --- a/include/core/jtrace.h +++ b/include/core/jtrace.h @@ -106,7 +106,7 @@ void j_trace_leave(JTrace* trace); G_DEFINE_AUTOPTR_CLEANUP_FUNC(JTrace, j_trace_leave) -#ifdef JULEA_DEBUG +#ifdef JULEA_TRACE #ifdef __COUNTER__ #define J_TRACE(name, ...) g_autoptr(JTrace) G_PASTE(j_trace, __COUNTER__) G_GNUC_UNUSED = j_trace_enter(name, __VA_ARGS__) #define J_TRACE_FUNCTION(...) g_autoptr(JTrace) G_PASTE(j_trace_function, __COUNTER__) G_GNUC_UNUSED = j_trace_enter(G_STRFUNC, __VA_ARGS__) diff --git a/lib/core/jbackend.c b/lib/core/jbackend.c index 19ba65179..3060d7cad 100644 --- a/lib/core/jbackend.c +++ b/lib/core/jbackend.c @@ -289,7 +289,10 @@ j_backend_object_init(JBackend* backend, gchar const* path) g_return_val_if_fail(path != NULL, FALSE); { - J_TRACE("backend_init", "%s", path); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_init", "%s", path); ret = backend->object.backend_init(path, &(backend->data)); } @@ -305,7 +308,10 @@ j_backend_object_fini(JBackend* backend) g_return_if_fail(backend->type == J_BACKEND_TYPE_OBJECT); { - J_TRACE("backend_fini", NULL); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_fini", NULL); backend->object.backend_fini(backend->data); } } @@ -324,7 +330,10 @@ j_backend_object_create(JBackend* backend, gchar const* namespace, gchar const* g_return_val_if_fail(data != NULL, FALSE); { - J_TRACE("backend_create", "%s, %s, %p", namespace, path, (gpointer)data); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_create", "%s, %s, %p", namespace, path, (gpointer)data); ret = backend->object.backend_create(backend->data, namespace, path, data); } @@ -345,7 +354,10 @@ j_backend_object_open(JBackend* backend, gchar const* namespace, gchar const* pa g_return_val_if_fail(data != NULL, FALSE); { - J_TRACE("backend_open", "%s, %s, %p", namespace, path, (gpointer)data); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_open", "%s, %s, %p", namespace, path, (gpointer)data); ret = backend->object.backend_open(backend->data, namespace, path, data); } @@ -364,7 +376,10 @@ j_backend_object_delete(JBackend* backend, gpointer data) g_return_val_if_fail(data != NULL, FALSE); { - J_TRACE("backend_delete", "%p", data); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_delete", "%p", data); ret = backend->object.backend_delete(backend->data, data); } @@ -384,7 +399,10 @@ j_backend_object_get_all(JBackend* backend, gchar const* namespace, gpointer* it g_return_val_if_fail(iterator != NULL, FALSE); { - J_TRACE("backend_get_all", "%s, %p", namespace, (gpointer)iterator); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_get_all", "%s, %p", namespace, (gpointer)iterator); ret = backend->object.backend_get_all(backend->data, namespace, iterator); } @@ -405,7 +423,10 @@ j_backend_object_get_by_prefix(JBackend* backend, gchar const* namespace, gchar g_return_val_if_fail(iterator != NULL, FALSE); { - J_TRACE("backend_get_by_prefix", "%s, %s, %p", namespace, prefix, (gpointer)iterator); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_get_by_prefix", "%s, %s, %p", namespace, prefix, (gpointer)iterator); ret = backend->object.backend_get_by_prefix(backend->data, namespace, prefix, iterator); } @@ -424,7 +445,10 @@ j_backend_object_iterate(JBackend* backend, gpointer iterator, gchar const** nam g_return_val_if_fail(name != NULL, FALSE); { - J_TRACE("backend_iterate", "%p, %p", iterator, (gpointer)name); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_iterate", "%p, %p", iterator, (gpointer)name); ret = backend->object.backend_iterate(backend->data, iterator, name); } @@ -443,7 +467,10 @@ j_backend_object_close(JBackend* backend, gpointer data) g_return_val_if_fail(data != NULL, FALSE); { - J_TRACE("backend_close", "%p", data); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_close", "%p", data); ret = backend->object.backend_close(backend->data, data); } @@ -464,7 +491,10 @@ j_backend_object_status(JBackend* backend, gpointer data, gint64* modification_t g_return_val_if_fail(size != NULL, FALSE); { - J_TRACE("backend_status", "%p, %p, %p", data, (gpointer)modification_time, (gpointer)size); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_status", "%p, %p, %p", data, (gpointer)modification_time, (gpointer)size); ret = backend->object.backend_status(backend->data, data, modification_time, size); } @@ -483,7 +513,10 @@ j_backend_object_sync(JBackend* backend, gpointer data) g_return_val_if_fail(data != NULL, FALSE); { - J_TRACE("backend_sync", "%p", data); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_sync", "%p", data); ret = backend->object.backend_sync(backend->data, data); } @@ -504,7 +537,10 @@ j_backend_object_read(JBackend* backend, gpointer data, gpointer buffer, guint64 g_return_val_if_fail(bytes_read != NULL, FALSE); { - J_TRACE("backend_read", "%p, %p, %" G_GUINT64_FORMAT ", %" G_GUINT64_FORMAT ", %p", data, buffer, length, offset, (gpointer)bytes_read); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_read", "%p, %p, %" G_GUINT64_FORMAT ", %" G_GUINT64_FORMAT ", %p", data, buffer, length, offset, (gpointer)bytes_read); ret = backend->object.backend_read(backend->data, data, buffer, length, offset, bytes_read); } @@ -525,7 +561,10 @@ j_backend_object_write(JBackend* backend, gpointer data, gconstpointer buffer, g g_return_val_if_fail(bytes_written != NULL, FALSE); { - J_TRACE("backend_write", "%p, %p, %" G_GUINT64_FORMAT ", %" G_GUINT64_FORMAT ", %p", data, buffer, length, offset, (gpointer)bytes_written); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_object_write", "%p, %p, %" G_GUINT64_FORMAT ", %" G_GUINT64_FORMAT ", %p", data, buffer, length, offset, (gpointer)bytes_written); ret = backend->object.backend_write(backend->data, data, buffer, length, offset, bytes_written); } @@ -544,7 +583,10 @@ j_backend_kv_init(JBackend* backend, gchar const* path) g_return_val_if_fail(path != NULL, FALSE); { - J_TRACE("backend_init", "%s", path); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_kv_init", "%s", path); ret = backend->kv.backend_init(path, &(backend->data)); } @@ -560,7 +602,10 @@ j_backend_kv_fini(JBackend* backend) g_return_if_fail(backend->type == J_BACKEND_TYPE_KV); { - J_TRACE("backend_fini", NULL); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_kv_fini", NULL); backend->kv.backend_fini(backend->data); } } @@ -579,7 +624,10 @@ j_backend_kv_batch_start(JBackend* backend, gchar const* namespace, JSemantics* g_return_val_if_fail(batch != NULL, FALSE); { - J_TRACE("backend_batch_start", "%s, %p, %p", namespace, (gpointer)semantics, (gpointer)batch); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_kv_batch_start", "%s, %p, %p", namespace, (gpointer)semantics, (gpointer)batch); ret = backend->kv.backend_batch_start(backend->data, namespace, semantics, batch); } @@ -598,7 +646,10 @@ j_backend_kv_batch_execute(JBackend* backend, gpointer batch) g_return_val_if_fail(batch != NULL, FALSE); { - J_TRACE("backend_batch_execute", "%p", batch); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_kv_batch_execute", "%p", batch); ret = backend->kv.backend_batch_execute(backend->data, batch); } @@ -619,7 +670,10 @@ j_backend_kv_put(JBackend* backend, gpointer batch, gchar const* key, gconstpoin g_return_val_if_fail(value != NULL, FALSE); { - J_TRACE("backend_put", "%p, %s, %p, %u", batch, key, (gconstpointer)value, value_len); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_kv_put", "%p, %s, %p, %u", batch, key, (gconstpointer)value, value_len); ret = backend->kv.backend_put(backend->data, batch, key, value, value_len); } @@ -639,7 +693,10 @@ j_backend_kv_delete(JBackend* backend, gpointer batch, gchar const* key) g_return_val_if_fail(key != NULL, FALSE); { - J_TRACE("backend_delete", "%p, %s", batch, key); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_kv_delete", "%p, %s", batch, key); ret = backend->kv.backend_delete(backend->data, batch, key); } @@ -661,7 +718,10 @@ j_backend_kv_get(JBackend* backend, gpointer batch, gchar const* key, gpointer* g_return_val_if_fail(value_len != NULL, FALSE); { - J_TRACE("backend_get", "%p, %s, %p, %p", batch, key, (gpointer)value, (gpointer)value_len); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_kv_get", "%p, %s, %p, %p", batch, key, (gpointer)value, (gpointer)value_len); ret = backend->kv.backend_get(backend->data, batch, key, value, value_len); } @@ -681,7 +741,10 @@ j_backend_kv_get_all(JBackend* backend, gchar const* namespace, gpointer* iterat g_return_val_if_fail(iterator != NULL, FALSE); { - J_TRACE("backend_get_all", "%s, %p", namespace, (gpointer)iterator); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_kv_get_all", "%s, %p", namespace, (gpointer)iterator); ret = backend->kv.backend_get_all(backend->data, namespace, iterator); } @@ -702,7 +765,10 @@ j_backend_kv_get_by_prefix(JBackend* backend, gchar const* namespace, gchar cons g_return_val_if_fail(iterator != NULL, FALSE); { - J_TRACE("backend_get_by_prefix", "%s, %s, %p", namespace, prefix, (gpointer)iterator); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_kv_get_by_prefix", "%s, %s, %p", namespace, prefix, (gpointer)iterator); ret = backend->kv.backend_get_by_prefix(backend->data, namespace, prefix, iterator); } @@ -723,7 +789,10 @@ j_backend_kv_iterate(JBackend* backend, gpointer iterator, gchar const** key, gc g_return_val_if_fail(value_len != NULL, FALSE); { - J_TRACE("backend_iterate", "%p, %p, %p, %p", iterator, (gpointer)key, (gpointer)value, (gpointer)value_len); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_kv_iterate", "%p, %p, %p, %p", iterator, (gpointer)key, (gpointer)value, (gpointer)value_len); ret = backend->kv.backend_iterate(backend->data, iterator, key, value, value_len); } @@ -742,7 +811,10 @@ j_backend_db_init(JBackend* backend, gchar const* path) g_return_val_if_fail(path != NULL, FALSE); { - J_TRACE("backend_init", "%s", path); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_init", "%s", path); ret = backend->db.backend_init(path, &(backend->data)); } @@ -758,7 +830,10 @@ j_backend_db_fini(JBackend* backend) g_return_if_fail(backend->type == J_BACKEND_TYPE_DB); { - J_TRACE("backend_fini", NULL); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_fini", NULL); backend->db.backend_fini(backend->data); } } @@ -778,7 +853,10 @@ j_backend_db_batch_start(JBackend* backend, gchar const* namespace, JSemantics* g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_batch_start", "%s, %p, %p, %p", namespace, (gpointer)semantics, (gpointer)batch, (gpointer)error); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_batch_start", "%s, %p, %p, %p", namespace, (gpointer)semantics, (gpointer)batch, (gpointer)error); ret = backend->db.backend_batch_start(backend->data, namespace, semantics, batch, error); } @@ -798,7 +876,10 @@ j_backend_db_batch_execute(JBackend* backend, gpointer batch, GError** error) g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_batch_execute", "%p, %p", batch, (gpointer)error); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_batch_execute", "%p, %p", batch, (gpointer)error); ret = backend->db.backend_batch_execute(backend->data, batch, error); } @@ -820,7 +901,10 @@ j_backend_db_schema_create(JBackend* backend, gpointer batch, gchar const* name, g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_schema_create", "%p, %s, %p, %p", batch, name, (gconstpointer)schema, (gpointer)error); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_schema_create", "%p, %s, %u, %p, %p", batch, name, schema->len, (gconstpointer)schema, (gpointer)error); ret = backend->db.backend_schema_create(backend->data, batch, name, schema, error); } @@ -841,7 +925,10 @@ j_backend_db_schema_get(JBackend* backend, gpointer batch, gchar const* name, bs g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_schema_get", "%p, %s, %p, %p", batch, name, (gpointer)schema, (gpointer)error); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_schema_get", "%p, %s, %p, %p", batch, name, (gpointer)schema, (gpointer)error); ret = backend->db.backend_schema_get(backend->data, batch, name, schema, error); } @@ -862,7 +949,10 @@ j_backend_db_schema_delete(JBackend* backend, gpointer batch, gchar const* name, g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_schema_delete", "%p, %s, %p", batch, name, (gpointer)error); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_schema_delete", "%p, %s, %p", batch, name, (gpointer)error); ret = backend->db.backend_schema_delete(backend->data, batch, name, error); } @@ -885,7 +975,10 @@ j_backend_db_insert(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_insert", "%p, %s, %p, %p, %p", batch, name, (gconstpointer)metadata, (gpointer)id, (gpointer)error); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_insert", "%p, %s, %u, %p, %p, %p", batch, name, metadata->len, (gconstpointer)metadata, (gpointer)id, (gpointer)error); ret = backend->db.backend_insert(backend->data, batch, name, metadata, id, error); } @@ -908,7 +1001,10 @@ j_backend_db_update(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_update", "%p, %s, %p, %p, %p", batch, name, (gconstpointer)selector, (gconstpointer)metadata, (gpointer)error); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_update", "%p, %s, %u, %p, %p, %p", batch, name, metadata->len, (gconstpointer)selector, (gconstpointer)metadata, (gpointer)error); ret = backend->db.backend_update(backend->data, batch, name, selector, metadata, error); } @@ -929,7 +1025,10 @@ j_backend_db_delete(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_delete", "%p, %s, %p, %p", batch, name, (gconstpointer)selector, (gpointer)error); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_delete", "%p, %s, %u, %p, %p", batch, name, selector->len, (gconstpointer)selector, (gpointer)error); ret = backend->db.backend_delete(backend->data, batch, name, selector, error); } @@ -951,7 +1050,10 @@ j_backend_db_query(JBackend* backend, gpointer batch, gchar const* name, bson_t g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_query", "%p, %s, %p, %p, %p", batch, name, (gconstpointer)selector, (gpointer)iterator, (gpointer)error); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_query", "%p, %s, %u, %p, %p, %p", batch, name, selector->len, (gconstpointer)selector, (gpointer)iterator, (gpointer)error); ret = backend->db.backend_query(backend->data, batch, name, selector, iterator, error); } @@ -972,7 +1074,10 @@ j_backend_db_iterate(JBackend* backend, gpointer iterator, bson_t* metadata, GEr g_return_val_if_fail(error == NULL || *error == NULL, FALSE); { - J_TRACE("backend_iterate", "%p, %p, %p", iterator, (gpointer)metadata, (gpointer)error); + /** This macro call is used for access recording + * The name and argument order is therfore important! + * \sa j_trace_enter() */ + J_TRACE("backend_db_iterate", "%p, %p, %p", iterator, (gpointer)metadata, (gpointer)error); ret = backend->db.backend_iterate(backend->data, iterator, metadata, error); } diff --git a/lib/core/jconfiguration.c b/lib/core/jconfiguration.c index 3f3faf56a..26b271343 100644 --- a/lib/core/jconfiguration.c +++ b/lib/core/jconfiguration.c @@ -148,6 +148,7 @@ struct JConfiguration guint64 stripe_size; gchar* checksum; + guint32 uid; /** * The reference count. @@ -394,6 +395,12 @@ j_configuration_new_for_data(GKeyFile* key_file) configuration->stripe_size = 4 * 1024 * 1024; } + { // uid should be unique per process in one application + GRand* rand = g_rand_new(); + configuration->uid = g_rand_int(rand); + g_rand_free(rand); + } + key_file_str = g_key_file_to_data(key_file, NULL, NULL); configuration->checksum = g_compute_checksum_for_string(G_CHECKSUM_SHA512, key_file_str, -1); @@ -616,6 +623,16 @@ j_configuration_get_checksum(JConfiguration* configuration) return configuration->checksum; } +guint32 +j_configuration_get_uid(JConfiguration* configuration) +{ + J_TRACE_FUNCTION(NULL); + + g_return_val_if_fail(configuration != NULL, 0); + + return configuration->uid; +} + /** * @} **/ diff --git a/lib/core/jconnection-pool.c b/lib/core/jconnection-pool.c index 67b7e6143..efc98f499 100644 --- a/lib/core/jconnection-pool.c +++ b/lib/core/jconnection-pool.c @@ -67,6 +67,8 @@ typedef struct JConnectionPool JConnectionPool; static JConnectionPool* j_connection_pool = NULL; +extern GPrivate j_trace_thread_default; + void j_connection_pool_init(JConfiguration* configuration) { @@ -167,6 +169,7 @@ j_connection_pool_fini(void) g_slice_free(JConnectionPool, pool); } +extern const char* __progname; static GSocketConnection* j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* server) { @@ -196,6 +199,8 @@ j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* se gchar const* client_checksum; gchar const* server_checksum; + gchar const* client_program_name; + guint32 client_process_uid; guint op_count; client = g_socket_client_new(); @@ -215,9 +220,21 @@ j_connection_pool_pop_internal(GAsyncQueue* queue, guint* count, gchar const* se j_helper_set_nodelay(connection, TRUE); client_checksum = j_configuration_get_checksum(j_configuration()); + client_program_name = g_get_prgname(); + if (client_program_name == NULL) + { + client_program_name = __progname; + } + if (client_program_name == NULL) + { + client_program_name = ""; + } + client_process_uid = j_configuration_get_uid(j_configuration()); message = j_message_new(J_MESSAGE_PING, strlen(client_checksum) + 1); j_message_append_string(message, client_checksum); + j_message_append_string(message, client_program_name); + j_message_append_n(message, &client_process_uid, sizeof(guint32)); j_message_send(message, connection); reply = j_message_new_reply(message); diff --git a/lib/core/jmessage.c b/lib/core/jmessage.c index 247f0c154..f291cde1d 100644 --- a/lib/core/jmessage.c +++ b/lib/core/jmessage.c @@ -42,23 +42,6 @@ * @{ **/ -enum JMessageSemantics -{ - J_MESSAGE_SEMANTICS_ATOMICITY_BATCH = 1 << 0, - J_MESSAGE_SEMANTICS_ATOMICITY_OPERATION = 1 << 1, - J_MESSAGE_SEMANTICS_ATOMICITY_NONE = 1 << 2, - J_MESSAGE_SEMANTICS_CONSISTENCY_IMMEDIATE = 1 << 3, - J_MESSAGE_SEMANTICS_CONSISTENCY_SESSION = 1 << 4, - J_MESSAGE_SEMANTICS_CONSISTENCY_EVENTUAL = 1 << 5, - J_MESSAGE_SEMANTICS_PERSISTENCY_STORAGE = 1 << 6, - J_MESSAGE_SEMANTICS_PERSISTENCY_NETWORK = 1 << 7, - J_MESSAGE_SEMANTICS_PERSISTENCY_NONE = 1 << 8, - J_MESSAGE_SEMANTICS_SECURITY_STRICT = 1 << 9, - J_MESSAGE_SEMANTICS_SECURITY_NONE = 1 << 10 -}; - -typedef enum JMessageSemantics JMessageSemantics; - /** * Additional message data. **/ @@ -756,36 +739,9 @@ j_message_set_semantics(JMessage* message, JSemantics* semantics) { J_TRACE_FUNCTION(NULL); - guint32 serialized_semantics = 0; - g_return_if_fail(message != NULL); - g_return_if_fail(semantics != NULL); - -#define SERIALIZE_SEMANTICS(type, key) \ - { \ - gint tmp; \ - tmp = j_semantics_get(semantics, J_SEMANTICS_##type); \ - if (tmp == J_SEMANTICS_##type##_##key) \ - { \ - serialized_semantics |= J_MESSAGE_SEMANTICS_##type##_##key; \ - } \ - } - SERIALIZE_SEMANTICS(ATOMICITY, BATCH) - SERIALIZE_SEMANTICS(ATOMICITY, OPERATION) - SERIALIZE_SEMANTICS(ATOMICITY, NONE) - SERIALIZE_SEMANTICS(CONSISTENCY, IMMEDIATE) - SERIALIZE_SEMANTICS(CONSISTENCY, SESSION) - SERIALIZE_SEMANTICS(CONSISTENCY, EVENTUAL) - SERIALIZE_SEMANTICS(PERSISTENCY, STORAGE) - SERIALIZE_SEMANTICS(PERSISTENCY, NETWORK) - SERIALIZE_SEMANTICS(PERSISTENCY, NONE) - SERIALIZE_SEMANTICS(SECURITY, STRICT) - SERIALIZE_SEMANTICS(SECURITY, NONE) - -#undef SERIALIZE_SEMANTICS - - message->header.semantics = GUINT32_TO_LE(serialized_semantics); + message->header.semantics = j_semantics_serialize(semantics); } JSemantics* @@ -793,39 +749,9 @@ j_message_get_semantics(JMessage* message) { J_TRACE_FUNCTION(NULL); - JSemantics* semantics; - - guint32 serialized_semantics; - g_return_val_if_fail(message != NULL, NULL); - serialized_semantics = message->header.semantics; - serialized_semantics = GUINT32_FROM_LE(serialized_semantics); - - // If serialized_semantics is 0, we will end up with the default semantics. - semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); - -#define DESERIALIZE_SEMANTICS(type, key) \ - if (serialized_semantics & J_MESSAGE_SEMANTICS_##type##_##key) \ - { \ - j_semantics_set(semantics, J_SEMANTICS_##type, J_SEMANTICS_##type##_##key); \ - } - - DESERIALIZE_SEMANTICS(ATOMICITY, BATCH) - DESERIALIZE_SEMANTICS(ATOMICITY, OPERATION) - DESERIALIZE_SEMANTICS(ATOMICITY, NONE) - DESERIALIZE_SEMANTICS(CONSISTENCY, IMMEDIATE) - DESERIALIZE_SEMANTICS(CONSISTENCY, SESSION) - DESERIALIZE_SEMANTICS(CONSISTENCY, EVENTUAL) - DESERIALIZE_SEMANTICS(PERSISTENCY, STORAGE) - DESERIALIZE_SEMANTICS(PERSISTENCY, NETWORK) - DESERIALIZE_SEMANTICS(PERSISTENCY, NONE) - DESERIALIZE_SEMANTICS(SECURITY, STRICT) - DESERIALIZE_SEMANTICS(SECURITY, NONE) - -#undef DESERIALIZE_SEMANTICS - - return semantics; + return j_semantics_deserialize(message->header.semantics); } /** diff --git a/lib/core/jsemantics.c b/lib/core/jsemantics.c index 7c9cbf924..031e6ba40 100644 --- a/lib/core/jsemantics.c +++ b/lib/core/jsemantics.c @@ -68,6 +68,23 @@ struct JSemantics gint ref_count; }; +enum JSerializedSemantics +{ + J_SERIALIZED_SEMANTICS_ATOMICITY_BATCH = 1 << 0, + J_SERIALIZED_SEMANTICS_ATOMICITY_OPERATION = 1 << 1, + J_SERIALIZED_SEMANTICS_ATOMICITY_NONE = 1 << 2, + J_SERIALIZED_SEMANTICS_CONSISTENCY_IMMEDIATE = 1 << 3, + J_SERIALIZED_SEMANTICS_CONSISTENCY_SESSION = 1 << 4, + J_SERIALIZED_SEMANTICS_CONSISTENCY_EVENTUAL = 1 << 5, + J_SERIALIZED_SEMANTICS_PERSISTENCY_STORAGE = 1 << 6, + J_SERIALIZED_SEMANTICS_PERSISTENCY_NETWORK = 1 << 7, + J_SERIALIZED_SEMANTICS_PERSISTENCY_NONE = 1 << 8, + J_SERIALIZED_SEMANTICS_SECURITY_STRICT = 1 << 9, + J_SERIALIZED_SEMANTICS_SECURITY_NONE = 1 << 10 +}; + +typedef enum JSerializedSemantics JSerializedSemantics; + JSemantics* j_semantics_new(JSemanticsTemplate template) { @@ -290,7 +307,7 @@ j_semantics_set(JSemantics* semantics, JSemanticsType key, gint value) } gint -j_semantics_get(JSemantics* semantics, JSemanticsType key) +j_semantics_get(const JSemantics* semantics, JSemanticsType key) { J_TRACE_FUNCTION(NULL); @@ -311,6 +328,75 @@ j_semantics_get(JSemantics* semantics, JSemanticsType key) } } +guint32 +j_semantics_serialize(const JSemantics* semantics) +{ + J_TRACE_FUNCTION(NULL); + + guint32 serialized_semantics = 0; + + g_return_val_if_fail(semantics != NULL, 0); + +#define SERIALIZE_SEMANTICS(type, key) \ + { \ + gint tmp; \ + tmp = j_semantics_get(semantics, J_SEMANTICS_##type); \ + if (tmp == J_SEMANTICS_##type##_##key) \ + { \ + serialized_semantics |= J_SERIALIZED_SEMANTICS_##type##_##key; \ + } \ + } + + SERIALIZE_SEMANTICS(ATOMICITY, BATCH) + SERIALIZE_SEMANTICS(ATOMICITY, OPERATION) + SERIALIZE_SEMANTICS(ATOMICITY, NONE) + SERIALIZE_SEMANTICS(CONSISTENCY, IMMEDIATE) + SERIALIZE_SEMANTICS(CONSISTENCY, SESSION) + SERIALIZE_SEMANTICS(CONSISTENCY, EVENTUAL) + SERIALIZE_SEMANTICS(PERSISTENCY, STORAGE) + SERIALIZE_SEMANTICS(PERSISTENCY, NETWORK) + SERIALIZE_SEMANTICS(PERSISTENCY, NONE) + SERIALIZE_SEMANTICS(SECURITY, STRICT) + SERIALIZE_SEMANTICS(SECURITY, NONE) + +#undef SERIALIZE_SEMANTICS + return GUINT32_TO_LE(serialized_semantics); +} + +JSemantics* +j_semantics_deserialize(guint32 serial_semantics) +{ + J_TRACE_FUNCTION(NULL); + + JSemantics* semantics; + serial_semantics = GUINT32_FROM_LE(serial_semantics); + + // semantics == 0, will lead to the default template + semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); + +#define DESERIALIZE_SEMANTICS(type, key) \ + if (serial_semantics & J_SERIALIZED_SEMANTICS_##type##_##key) \ + { \ + j_semantics_set(semantics, J_SEMANTICS_##type, J_SEMANTICS_##type##_##key); \ + } + + DESERIALIZE_SEMANTICS(ATOMICITY, BATCH) + DESERIALIZE_SEMANTICS(ATOMICITY, OPERATION) + DESERIALIZE_SEMANTICS(ATOMICITY, NONE) + DESERIALIZE_SEMANTICS(CONSISTENCY, IMMEDIATE) + DESERIALIZE_SEMANTICS(CONSISTENCY, SESSION) + DESERIALIZE_SEMANTICS(CONSISTENCY, EVENTUAL) + DESERIALIZE_SEMANTICS(PERSISTENCY, STORAGE) + DESERIALIZE_SEMANTICS(PERSISTENCY, NETWORK) + DESERIALIZE_SEMANTICS(PERSISTENCY, NONE) + DESERIALIZE_SEMANTICS(SECURITY, STRICT) + DESERIALIZE_SEMANTICS(SECURITY, NONE) + +#undef DESERIALIZE_SEMANTICS + + return semantics; +} + /** * @} **/ diff --git a/lib/core/jtrace.c b/lib/core/jtrace.c index e9f186dc0..09916192e 100644 --- a/lib/core/jtrace.c +++ b/lib/core/jtrace.c @@ -21,6 +21,7 @@ **/ #include +#include #include #include @@ -31,6 +32,8 @@ #include +#include + /** * \addtogroup JTrace * @@ -45,7 +48,8 @@ enum JTraceFlags J_TRACE_OFF = 0, J_TRACE_ECHO = 1 << 0, J_TRACE_OTF = 1 << 1, - J_TRACE_SUMMARY = 1 << 2 + J_TRACE_SUMMARY = 1 << 2, + J_TRACE_ACCESS = 1 << 3 }; typedef enum JTraceFlags JTraceFlags; @@ -66,6 +70,24 @@ struct JTraceTime typedef struct JTraceTime JTraceTime; +struct Access +{ + guint64 timestamp; + guint32 uid; + const char* program_name; + const char* backend; + const char* type; + const char* path; + const char* namespace; + const char* name; + const char* operation; + guint32 semantics; + guint64 size; + guint32 complexity; + const bson_t* bson; +}; +typedef struct Access Access; + /** * A trace thread. **/ @@ -96,6 +118,42 @@ struct JTraceThread guint32 process_id; } otf; #endif + + struct + { + char* program_name; + guint32 uid; + } client; + + struct + { + gboolean inside; + Access row; + const void* utility_ptr; + struct + { + char* type; + char* config_path; + GString* namespace; + guint32 semantic; + } db; + struct + { + char* type; + char* config_path; + GString* namespace; + GString* name; + guint32 semantic; + } kv; + struct + { + char* type; + char* config_path; + GString* namespace; + GString* path; + } object; + + } access; }; typedef struct JTraceThread JTraceThread; @@ -113,6 +171,8 @@ static gint j_trace_thread_id = 1; static GPatternSpec** j_trace_function_patterns = NULL; +static gchar const J_TRACE_ACCESS_PREFIX[] = "backend_"; + #ifdef HAVE_OTF static OTF_FileManager* otf_manager = NULL; static OTF_Writer* otf_writer = NULL; @@ -137,6 +197,11 @@ static GHashTable* j_trace_summary_table = NULL; G_LOCK_DEFINE_STATIC(j_trace_echo); G_LOCK_DEFINE_STATIC(j_trace_summary); +static void +j_trace_access_print_header(void) +{ + g_printerr("time,process_uid,program_name,backend,type,path,namespace,name,operation,semantics,size,complexity,duration,bson\n"); +} /** * Creates a new trace thread. * @@ -160,8 +225,11 @@ j_trace_thread_new(GThread* thread) } trace_thread = g_slice_new(JTraceThread); + memset(trace_thread, 0, sizeof(JTraceThread)); trace_thread->function_depth = 0; trace_thread->stack = g_array_new(FALSE, FALSE, sizeof(JTraceStack)); + trace_thread->client.program_name = NULL; + trace_thread->client.uid = 0; if (thread == NULL) { @@ -186,6 +254,15 @@ j_trace_thread_new(GThread* thread) } #endif + if (j_trace_flags & J_TRACE_ACCESS) + { + trace_thread->access.db.namespace = g_string_new(NULL); + trace_thread->access.kv.namespace = g_string_new(NULL); + trace_thread->access.kv.name = g_string_new(NULL); + trace_thread->access.object.namespace = g_string_new(NULL); + trace_thread->access.object.path = g_string_new(NULL); + } + return trace_thread; } @@ -210,6 +287,21 @@ j_trace_thread_free(JTraceThread* trace_thread) g_return_if_fail(trace_thread != NULL); + if (j_trace_flags & J_TRACE_ACCESS) + { + g_free(trace_thread->access.db.type); + g_free(trace_thread->access.db.config_path); + g_string_free(trace_thread->access.db.namespace, TRUE); + g_free(trace_thread->access.kv.type); + g_free(trace_thread->access.kv.config_path); + g_string_free(trace_thread->access.kv.namespace, TRUE); + g_string_free(trace_thread->access.kv.name, TRUE); + g_free(trace_thread->access.object.type); + g_free(trace_thread->access.object.config_path); + g_string_free(trace_thread->access.object.namespace, TRUE); + g_string_free(trace_thread->access.object.path, TRUE); + } + #ifdef HAVE_OTF if (j_trace_flags & J_TRACE_OTF) { @@ -385,6 +477,10 @@ j_trace_init(gchar const* name) { j_trace_flags |= J_TRACE_SUMMARY; } + else if (g_strcmp0(trace_parts[i], "access") == 0) + { + j_trace_flags |= J_TRACE_ACCESS; + } } if (j_trace_flags == J_TRACE_OFF) @@ -433,6 +529,13 @@ j_trace_init(gchar const* name) j_trace_summary_table = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); } + if (j_trace_flags & J_TRACE_ACCESS) + { + G_LOCK(j_trace_echo); + j_trace_access_print_header(); + G_UNLOCK(j_trace_echo); + } + g_free(j_trace_name); j_trace_name = g_strdup(name); } @@ -499,13 +602,89 @@ j_trace_fini(void) g_free(j_trace_name); } +static void +j_trace_access_print(const Access* row, guint64 duration) +{ + g_printerr("%" G_GUINT64_FORMAT ".%06" G_GUINT64_FORMAT ",%u,%s,%s,%s,%s,%s,%s,%s,%u,%" G_GUINT64_FORMAT ",%u,%" G_GUINT64_FORMAT ".%06" G_GUINT64_FORMAT ",\"%s\"\n", + row->timestamp / G_USEC_PER_SEC, + row->timestamp % G_USEC_PER_SEC, + row->uid, + row->program_name, + row->backend, + row->type, + row->path, + row->namespace, + row->name, + row->operation, + row->semantics, + row->size, + row->complexity, + duration / G_USEC_PER_SEC, + duration % G_USEC_PER_SEC, + row->bson == NULL ? "{}" : bson_as_json(row->bson, NULL)); +} + +static guint32 +_count_keys_recursive(bson_iter_t* itr) +{ + guint32 cnt = 0; + do + { + cnt += 1; + if (bson_iter_type(itr) == BSON_TYPE_ARRAY || bson_iter_type(itr) == BSON_TYPE_DOCUMENT) + { + bson_iter_t child; + bson_iter_recurse(itr, &child); + cnt += _count_keys_recursive(&child); + } + } while (bson_iter_next(itr)); + return cnt; +} + +static guint32 +count_keys_recursive(const bson_t* bson) +{ + bson_iter_t itr; + g_return_val_if_fail(bson_iter_init(&itr, bson), 0); + return _count_keys_recursive(&itr); +} + +static gboolean +parse_backend_operation(const gchar* backend_operation, JBackendType* type, const gchar** operation) +{ + if (strncmp(backend_operation, "kv_", 3) == 0) + { + if (type) + *type = J_BACKEND_TYPE_KV; + if (operation) + *operation = backend_operation + 3; + return TRUE; + } + else if (strncmp(backend_operation, "db_", 3) == 0) + { + if (type) + *type = J_BACKEND_TYPE_DB; + if (operation) + *operation = backend_operation + 3; + return TRUE; + } + else if (strncmp(backend_operation, "object_", 7) == 0) + { + if (type) + *type = J_BACKEND_TYPE_OBJECT; + if (operation) + *operation = backend_operation + 7; + return TRUE; + } + return FALSE; +} + JTrace* j_trace_enter(gchar const* name, gchar const* format, ...) { JTraceThread* trace_thread; JTrace* trace; guint64 timestamp; - va_list args; if (j_trace_flags == J_TRACE_OFF) { @@ -528,8 +707,6 @@ j_trace_enter(gchar const* name, gchar const* format, ...) trace->name = g_strdup(name); trace->enter_time = timestamp; - va_start(args, format); - if (j_trace_flags & J_TRACE_ECHO) { G_LOCK(j_trace_echo); @@ -537,10 +714,13 @@ j_trace_enter(gchar const* name, gchar const* format, ...) if (format != NULL) { + va_list args; g_autofree gchar* arguments = NULL; + va_start(args, format); arguments = g_strdup_vprintf(format, args); g_printerr("ENTER %s (%s)\n", name, arguments); + va_end(args); } else { @@ -550,6 +730,266 @@ j_trace_enter(gchar const* name, gchar const* format, ...) G_UNLOCK(j_trace_echo); } + if (j_trace_flags & J_TRACE_ACCESS) + { + if (strcmp(name, "ping") == 0) + { + va_list args; + va_start(args, format); + g_free(trace_thread->client.program_name); + trace_thread->client.program_name = strdup(va_arg(args, const char*)); + trace_thread->client.uid = va_arg(args, guint32); + va_end(args); + } + // for all traces strating with j_trace_access_prefix create an access row + else if (strncmp(name, J_TRACE_ACCESS_PREFIX, sizeof(J_TRACE_ACCESS_PREFIX) - 1) == 0) + { + gchar const* backend_operation = name + sizeof(J_TRACE_ACCESS_PREFIX) - 1; + JBackendType type = 0; + gchar const* operation = NULL; + + // read config to set type and paths of backends if not already done + if (trace_thread->access.db.type == NULL) + { + JConfiguration* config = j_configuration_new(); + + trace_thread->access.db.type = strdup(j_configuration_get_backend(config, J_BACKEND_TYPE_DB)); + trace_thread->access.db.config_path = strdup(j_configuration_get_backend_path(config, J_BACKEND_TYPE_DB)); + + trace_thread->access.kv.type = strdup(j_configuration_get_backend(config, J_BACKEND_TYPE_KV)); + trace_thread->access.kv.config_path = strdup(j_configuration_get_backend_path(config, J_BACKEND_TYPE_KV)); + + trace_thread->access.object.type = strdup(j_configuration_get_backend(config, J_BACKEND_TYPE_OBJECT)); + trace_thread->access.object.config_path = strdup(j_configuration_get_backend_path(config, J_BACKEND_TYPE_OBJECT)); + + j_configuration_unref(config); + } + + if (!parse_backend_operation(backend_operation, &type, &operation)) + { + /// \TODO with current prefix it is not posible to easly detect unknown backends! + } + else + { + Access* row = &trace_thread->access.row; + va_list args; + memset(row, 0, sizeof(Access)); + + row->uid = trace_thread->client.uid; + row->program_name = trace_thread->client.program_name; + row->timestamp = timestamp; + row->operation = operation; + trace_thread->access.inside = TRUE; + + va_start(args, format); + + if (type == J_BACKEND_TYPE_KV) + { + row->backend = "kv"; + row->namespace = trace_thread->access.kv.namespace->str; + row->type = trace_thread->access.kv.type; + row->path = trace_thread->access.kv.config_path; + row->name = trace_thread->access.kv.name->str; + row->semantics = trace_thread->access.kv.semantic; + if (strcmp(operation, "batch_start") == 0) + { + row->namespace = va_arg(args, const char*); + row->semantics = j_semantics_serialize(va_arg(args, JSemantics*)); + g_string_assign(trace_thread->access.kv.namespace, row->namespace); + g_string_assign(trace_thread->access.kv.name, ""); + trace_thread->access.kv.semantic = row->semantics; + } + else if (strcmp(operation, "put") == 0) + { + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_arg(args, void*); + row->size = va_arg(args, guint32); + } + else if (strcmp(operation, "delete") == 0) + { + va_arg(args, void*); + row->name = va_arg(args, const char*); + } + else if (strcmp(operation, "get") == 0) + { + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_arg(args, void*); + trace_thread->access.utility_ptr = va_arg(args, void*); + } + else if (strcmp(operation, "get_all") == 0) + { + row->namespace = va_arg(args, const char*); + g_string_assign(trace_thread->access.kv.namespace, row->namespace); + g_string_assign(trace_thread->access.kv.name, ""); + } + else if (strcmp(operation, "get_by_prefix") == 0) + { + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); + g_string_assign(trace_thread->access.kv.namespace, row->namespace); + g_string_assign(trace_thread->access.kv.name, row->name); + } + // iterate, init, fini <- no further deatils + // batch_execute <- handled in leave + else if (!( + strcmp(operation, "iterate") == 0 + || strcmp(operation, "init") == 0 + || strcmp(operation, "fini") == 0 + || strcmp(operation, "batch_execute") == 0)) + { + g_warning("Unknown operation '%s' for backend 'kv'", operation); + } + } + else if (type == J_BACKEND_TYPE_DB) + { + row->backend = "db"; + row->type = trace_thread->access.db.type; + row->path = trace_thread->access.db.config_path; + row->namespace = trace_thread->access.db.namespace->str; + row->semantics = trace_thread->access.db.semantic; + row->name = ""; + if (strcmp(operation, "batch_start") == 0) + { + row->namespace = va_arg(args, const char*); + row->semantics = j_semantics_serialize(va_arg(args, const JSemantics*)); + g_string_assign(trace_thread->access.db.namespace, row->namespace); + trace_thread->access.db.semantic = row->semantics; + } + else if (strcmp(operation, "schema_create") == 0) + { + va_arg(args, void*); + row->name = va_arg(args, const char*); + row->size = va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); + } + else if (strcmp(operation, "schema_get") == 0) + { + va_arg(args, void*); + row->name = va_arg(args, const char*); + } + else if (strcmp(operation, "schema_delete") == 0) + { + va_arg(args, void*); + row->name = va_arg(args, const char*); + } + else if (strcmp(operation, "insert") == 0) + { + va_arg(args, void*); + row->name = va_arg(args, const char*); + row->size = va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); + row->complexity = count_keys_recursive(row->bson); + } + else if (strcmp(operation, "update") == 0) + { + static bson_t bson; + const bson_t* selector = NULL; + bson_init(&bson); + va_arg(args, void*); + row->name = va_arg(args, const char*); + row->size = va_arg(args, guint32); + selector = va_arg(args, const bson_t*); + bson_append_document(&bson, "selector", -1, selector); + bson_append_document(&bson, "entry", -1, va_arg(args, const bson_t*)); + row->bson = &bson; + row->complexity = count_keys_recursive(selector); // because the added two top level keys + } + else if (strcmp(operation, "delete") == 0) + { + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); + row->complexity = count_keys_recursive(row->bson); + } + else if (strcmp(operation, "query") == 0) + { + va_arg(args, void*); + row->name = va_arg(args, const char*); + va_arg(args, guint32); + row->bson = va_arg(args, const bson_t*); + row->complexity = count_keys_recursive(row->bson); + } + // iterate, init, fini <- no further details + // batch_execute <- handled in leave + else if (!( + strcmp(operation, "iterate") == 0 + || strcmp(operation, "init") == 0 + || strcmp(operation, "fini") == 0 + || strcmp(operation, "batch_execute") == 0)) + { + g_warning("unknown operation '%s' for backend 'db'", operation); + } + } + else if (type == J_BACKEND_TYPE_OBJECT) + { + row->backend = "object"; + row->namespace = trace_thread->access.object.namespace->str; + row->type = trace_thread->access.object.type; + row->path = trace_thread->access.object.config_path; + row->name = trace_thread->access.object.path->str; + if (strcmp(operation, "create") == 0) + { + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); + g_string_assign(trace_thread->access.object.namespace, row->namespace); + g_string_assign(trace_thread->access.object.path, row->name); + } + else if (strcmp(operation, "open") == 0) + { + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); + g_string_assign(trace_thread->access.object.namespace, row->namespace); + g_string_assign(trace_thread->access.object.path, row->name); + } + else if (strcmp(operation, "get_all") == 0) + { + row->namespace = va_arg(args, const char*); + g_string_assign(trace_thread->access.object.namespace, row->namespace); + g_string_assign(trace_thread->access.object.path, ""); + } + else if (strcmp(operation, "get_by_prefix") == 0) + { + row->namespace = va_arg(args, const char*); + row->name = va_arg(args, const char*); + g_string_assign(trace_thread->access.object.namespace, row->namespace); + g_string_assign(trace_thread->access.object.path, row->name); + } + else if (strcmp(operation, "read") == 0) + { + va_arg(args, void*); + va_arg(args, void*); + row->size = va_arg(args, guint64); + row->complexity = va_arg(args, guint64); + } + else if (strcmp(operation, "write") == 0) + { + va_arg(args, void*); + va_arg(args, void*); + row->size = va_arg(args, guint64); + row->complexity = va_arg(args, guint64); + } + // status, sync, iterate, fini, init <- no further data + // close/delte handled at leave + else if (!( + strcmp(operation, "status") == 0 + || strcmp(operation, "sync") == 0 + || strcmp(operation, "iterate") == 0 + || strcmp(operation, "fini") == 0 + || strcmp(operation, "init") == 0 + || strcmp(operation, "close") == 0 + || strcmp(operation, "delete") == 0)) + { + g_warning("unknown operation '%s' for backend 'object'", operation); + } + } + va_end(args); + } + } + } + #ifdef HAVE_OTF if (j_trace_flags & J_TRACE_OTF) { @@ -596,8 +1036,6 @@ j_trace_enter(gchar const* name, gchar const* format, ...) g_array_append_val(trace_thread->stack, current_stack); } - va_end(args); - trace_thread->function_depth++; return trace; @@ -648,6 +1086,57 @@ j_trace_leave(JTrace* trace) G_UNLOCK(j_trace_echo); } + if (j_trace_flags & J_TRACE_ACCESS) + { + if (trace_thread->access.inside) + { + if (strncmp(trace->name, J_TRACE_ACCESS_PREFIX, sizeof(J_TRACE_ACCESS_PREFIX) - 1) == 0) + { + gchar const* backend_operation = trace->name + sizeof(J_TRACE_ACCESS_PREFIX) - 1; + if (parse_backend_operation(backend_operation, NULL, NULL)) + { + guint64 duration; + Access* row = &trace_thread->access.row; + + duration = timestamp - trace->enter_time; + + if (strcmp(backend_operation, "kv_get") == 0) + { + row->size = *(const guint64*)trace_thread->access.utility_ptr; + } + + G_LOCK(j_trace_echo); + j_trace_access_print(row, duration); + G_UNLOCK(j_trace_echo); + + trace_thread->access.inside = FALSE; + if (strcmp(backend_operation, "kv_batch_execute") == 0) + { + g_string_assign(trace_thread->access.kv.namespace, ""); + g_string_assign(trace_thread->access.kv.name, ""); + } + else if (strcmp(backend_operation, "db_update") == 0) + { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" + bson_destroy((bson_t*)row->bson); +#pragma GCC diagnostic pop + } + else if (strcmp(backend_operation, "db_batch_execute") == 0) + { + g_string_assign(trace_thread->access.db.namespace, ""); + } + else if (strcmp(backend_operation, "object_delete") == 0 + || strcmp(trace->name, "object_close") == 0) + { + g_string_assign(trace_thread->access.object.namespace, ""); + g_string_assign(trace_thread->access.object.path, ""); + } + } + } + } + } + #ifdef HAVE_OTF if (j_trace_flags & J_TRACE_OTF) { diff --git a/lib/db-util/jbson.c b/lib/db-util/jbson.c index 7b156c7d0..6ab3dac98 100644 --- a/lib/db-util/jbson.c +++ b/lib/db-util/jbson.c @@ -341,7 +341,7 @@ j_bson_iter_value(bson_iter_t* iter, JDBType type, JDBTypeValue* value, GError** break; case J_DB_TYPE_SINT64: - if (G_UNLIKELY(!BSON_ITER_HOLDS_INT64(iter))) + if (G_UNLIKELY(!BSON_ITER_HOLDS_INT64(iter) && !BSON_ITER_HOLDS_INT32(iter))) { g_set_error_literal(error, J_BACKEND_BSON_ERROR, J_BACKEND_BSON_ERROR_ITER_INVALID_TYPE, "bson iter invalid type"); goto _error; @@ -349,7 +349,14 @@ j_bson_iter_value(bson_iter_t* iter, JDBType type, JDBTypeValue* value, GError** if (value) { - value->val_sint64 = bson_iter_int64(iter); + if (BSON_ITER_HOLDS_INT64(iter)) + { + value->val_sint64 = bson_iter_int64(iter); + } + else + { + value->val_sint64 = bson_iter_int32(iter); + } } break; diff --git a/meson.build b/meson.build index 58bbd62b0..92bf325c6 100644 --- a/meson.build +++ b/meson.build @@ -326,8 +326,13 @@ julea_conf.set_quoted('JULEA_BACKEND_PATH', get_option('prefix') / get_option('l if get_option('debug') julea_conf.set('_FORTIFY_SOURCE', 2) julea_conf.set('JULEA_DEBUG', 1) + julea_conf.set('JULEA_TRACE', 1) julea_conf.set('GLIB_VERSION_MIN_REQUIRED', 'GLIB_VERSION_@0@'.format(glib_version.underscorify())) #julea_conf.set('GLIB_VERSION_MAX_ALLOWED', 'GLIB_VERSION_@0@'.format(glib_version.underscorify())) +else + if get_option('julea_trace') + julea_conf.set('JULEA_TRACE', 1) + endif endif if hdf_dep.found() @@ -723,6 +728,12 @@ executable('julea-statistics', 'tools/statistics.c', install: true, ) +executable('julea-access-replay', 'tools/access_replay.c', + dependencies: common_deps + [julea_dep], + include_directories: julea_incs, + install: true, +) + if hdf_dep.found() executable('julea-h5migrate', 'tools/h5migrate.c', dependencies: common_deps + [julea_dep] + [hdf_dep], diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 000000000..6503723aa --- /dev/null +++ b/meson_options.txt @@ -0,0 +1 @@ +option('julea_trace', type: 'boolean', value: false) diff --git a/scripts/decission-support.R b/scripts/decission-support.R new file mode 100755 index 000000000..35013246a --- /dev/null +++ b/scripts/decission-support.R @@ -0,0 +1,333 @@ +#!/usr/bin/env Rscript + +# JULEA - Flexible storage framework +# Copyright (C) 2023-2023 Julian Benda +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +packages <- c( + "ggplot2", + "plotly", + "dplyr", + "scales", + "ggpubr", + "knitr") +missing <- packages[!(packages %in% installed.packages()[, "Package"])] +if (length(missing)) { + msg <- paste0( + "Not all requirued packages are installed, you can install them with", + "\n$ R", + "\n> install.packages(c(\"", + paste(missing, collapse = "\", \""), + "\"))", + "\n> q()") + stop(msg) +} + +suppressMessages(library(ggplot2)) +suppressMessages(library(plotly)) +suppressMessages(library(dplyr)) +suppressMessages(library(scales)) +suppressMessages(library(ggpubr)) + +options(knitr.kable.NA = "") +options(warn = 1) + +first_round <- TRUE +last_frame <- NA + +# test if all records of the same backend kind (object,kv,db) +# have executed the same order of commands +test_validity <- function(data) { + res <- TRUE + data <- subset( + data, + select = -c(time, process_uid, program_name, bson, duration)) + for (x in unique(data$backend)) { + first_round <<- TRUE + check <- function(frame, key) { + if (res == FALSE) { + return() + } + frame <- frame %>% subset(select = -c(backend)) + if (first_round != TRUE) { + if (nrow(frame) != nrow(last_frame)) { + warning("number error, the number of accesses differ between runs") + res <<- FALSE + } else if (!Reduce(f = "&", x = frame == last_frame)) { + warning("value error, the access type between runs differ") + res <<- FALSE + } + } + first_round <<- FALSE + last_frame <<- frame + } + data %>% filter(backend == x) %>% group_by(type, path) %>% group_walk(check) + } + return(res) +} + +hovertemplate <- "%{text}" +replace_col_names <- function(dfs, old, new) { + old_new <- data.frame(old, new) + dfs <- lapply(dfs, function(df) { + apply(old_new, 1, function(x) { + names(df)[which(names(df) == x[1])] <<- x[2] + return(df) + }) + return(df) + }) + return(dfs) +} +# create duration plot and table for one backend type +plot_durations <- function(data, key, ..., format = "simple") { + + # generate shorted backend label + # removes path if backend only appears once + confs <- unique(data %>% select("type", "path")) + map <- apply( + confs, 1, + function(x) paste(x[1], if (sum(confs == x[1]) > 1) x[2] else "")) + names(map) <- paste0(confs$type, confs$path) + data$backend <- map[paste0(data$type, data$path)] + + # analyse for runtime per operation type + sepperate_data <- data %>% + group_by(operation, backend) %>% + summarise(duration = sum(duration), cnt = n(), .groups = "drop_last") %>% + group_modify(function(df, keys) { + df$speed_up <- max(df$duration) / df$duration + df$time_diff <- max(df$duration) - df$duration + return(df) + }) + + # check and convert duration unit as sensible + unit <- "s" + if (max(sepperate_data$duration) < 10.) { + sepperate_data$duration <- sepperate_data$duration * 1000. + data$duration <- data$duration * 1000. + unit <- "ms" + } + label <- labs(y = paste0("time in ", unit), fill = "backend", x = "") + + # plotting + # sort categories after largest time save + sepperate_data$backend <- as.factor(sepperate_data$backend) + sepperate_data$operation <- reorder( + factor(sepperate_data$operation), + sepperate_data$time_diff, + FUN = max, decreasing = TRUE, na.rm = TRUE) + off <- 0 + sepperate_data <- sepperate_data %>% + group_modify(function(df, keys) { + df$base <- off + off <<- df$duration + off + return(df) + }) + sepperate_data <- arrange(sepperate_data, operation) + + # static plot + operation_factor <- factor( + sepperate_data$operation, + level = levels(sepperate_data$operation), + labels = paste( + unique(sepperate_data$operation), + "\n#", unique(sepperate_data$cnt))) + seperate <- ggplot(sepperate_data, + aes(y = duration, x = operation_factor, fill = backend)) + + geom_col(position = "dodge") + + theme(axis.title.x = element_blank(), + axis.text.x = element_text(angle = 45)) + + label + + # inteacrtive plotting + text_format <- ~sprintf( + "%s

number: %s
duration: %.2f%s
avg: %.2f%s", + backend, cnt, duration, unit, duration / cnt, unit) + sepperate_ly <- plot_ly( + data = sepperate_data, + x = operation_factor, y = ~duration, + color = ~backend, + colors = hue_pal()(length(unique(sepperate_data$backend))), + text = text_format, textposition = "none", hovertemplate = hovertemplate, + type = "bar") %>% + layout( + xaxis = list( + categoryorder = "array", + categoryarray = levels(operation_factor))) + total_ly <- plot_ly( + data = sepperate_data, + x = ~backend, y = ~duration, base = ~base, + color = ~operation, + colors = ~dichromat_pal("DarkRedtoBlue.12")(length(unique(operation))), + text = text_format, textposition = "none", hovertemplate = hovertemplate, + type = "bar", offsetgroup = 0) %>% + layout( + xaxis = list( + categoryorder = "array", + categoryarray = levels(sepperate_data$backend))) + + # table + sepperate_data <- sepperate_data %>% + group_modify(function(df, keys) { + cnt <- sprintf("#%s", df$cnt[1]) + df$cnt <- sprintf("%.2f", df$duration / df$cnt) + df <- df %>% arrange(duration) %>% add_row(cnt = cnt, .before = 0) + return(df)}) %>% + ungroup() + # clean nonsens values + sepperate_data[! is.na(sepperate_data$duration), ]$operation <- NA + # remove column with plotting only values + sepperate_data <- subset(sepperate_data, select = -c(base)) + + # analyse for total runtime + total_data <- data %>% + group_by(backend) %>% + summarise(duration = sum(duration), .groups = "drop") + total_data <- total_data[order(total_data$backend), ] + # plotting + total <- ggplot(total_data, + aes(y = duration, x = "all operations", fill = backend)) + + geom_col(position = "dodge") + + label + total_data$speed_up <- max(total_data$duration) / total_data$duration + total_data$time_diff <- max(total_data$duration) - total_data$duration + total_data <- arrange(total_data, duration) + + # generate HTML/text table with links + plot_file_name <- paste(key[1], "time.svg", sep = "_") + plot_html_file_name <- paste(key[1], "time.html", sep = "_") + renamed <- replace_col_names( # rename column for table + list(total_data = total_data, sepperate_data = sepperate_data), + c("duration", "time_diff", "speed_up"), + c(sprintf("duration (%s)", unit), + sprintf("time diff (%s)", unit), + "speed up") + ) + table_sepperate_str <- knitr::kable( + renamed$sepperate_data, + format, digits = 2, table.attr = "class=\"fancy\"") + table_total_str <- knitr::kable( + renamed$total_data, format, + digits = 2, table.attr = "class=\"fancy\"") + + if (format == "html") { + cat( + "

", + as.character(key[1]), + "

", sep = "") + cat( + "
", + table_sepperate_str, + "", + table_total_str, + "
", + "", + "", + "
", sep = "") + } else { + cat( + as.character(key[1]), + paste0("plot: ", + plot_file_name), "", + table_total_str, "", + table_sepperate_str, "", + "", sep = "\n") + } + + # combining static plots and store them + plot <- ggarrange(seperate, total, + widths = c(4, 1), ncol = 2, nrow = 1, + common.legend = TRUE, legend = "bottom") + ggsave(plot_file_name, plot, + unit = "in", width = 11, height = 6, bg = "white") + + # combine interactive plots and store them + n <- length(unique(sepperate_data$backend)) - 1 + htmlwidgets::saveWidget( + subplot( + sepperate_ly, + total_ly, + widths = c(0.8, 0.2)) %>% + layout( + font = list(size = 18), + yaxis = list(title = label$y), + hoverlabel = list(bgcolor = "white"), + barmode = "group", + legend = list(traceorder = "normal")) %>% + htmlwidgets::onRender( + sprintf( + read_file("interactive.js"), + n)), + plot_html_file_name, + selfcontained = FALSE, + libdir = "lib") +} + +# main +# setup, load data +args <- commandArgs(trailingOnly = FALSE) +script_dir <- dirname(sub("--file=", "", args[grep("--file=", args)])) +read_file <- function(name) { + return(readLines(paste(script_dir, name, sep = "/"))) +} +args <- commandArgs(trailingOnly = TRUE) +if (length(args) < 1) { + stop( + paste( + "usage:", + "script", # TODO check if script name can be determined + " <- generated by decission-support.sh", + "[(print|html)] <- output as simple print(default) or as html", + sep = "\n"), + call. = FALSE) +} +data <- read.csv(args[1]) +as_html <- length(args) >= 2 & args[2] == "html" + +# check data +res <- test_validity(data) +if (res == FALSE) { + stop( + "Access reports between different configurations are not consistent!", + call. = FALSE) +} + +# print html header +if (as_html) { + cat( + "", + "
", + "
") +} + +data %>% + group_by(backend) %>% + group_walk(plot_durations, format = if (as_html) "html" else "simple") + +# print html closing +if (as_html) { + cat("") +} diff --git a/scripts/decission-support.py b/scripts/decission-support.py new file mode 100755 index 000000000..0b851cebb --- /dev/null +++ b/scripts/decission-support.py @@ -0,0 +1,208 @@ +#!/bin/python + +# JULEA - Flexible storage framework +# Copyright (C) 2023-2023 Julian Benda +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . + +import sys +import inspect +import tempfile +import csv +import json +import socket +import os +import subprocess +from itertools import islice + + +def usage(): + print(inspect.cleandoc(f""" + Usage: {sys.argv[0]} + \tfurther details can be found in doc/decission-support.md + """), file=sys.stderr) + sys.exit(1) + + +def create_config(host, backend, backend_type, path): + return inspect.cleandoc(f""" + [core] + max-operation-size=0 + max-inject-size=0 + port=0 + + [clients] + max-connections=0 + stripe-size=0 + + [servers] + object={host}; + kv={host}; + db={host}; + + [object] + backend={backend_type} + component={'server' if backend == 'object' else 'client'} + path={path} + + [kv] + backend={backend_type} + component={'server' if backend == 'kv' else 'client'} + path={path} + + [db] + backend={backend_type} + component={'server' if backend == 'db' else 'client'} + path={path} + """) + + +# extract all records from one backend type from a acccess-record +def extract(all, record, backend): + header = None + input = csv.reader(all, quotechar='|') + output = csv.writer( + open(record(backend), 'w', newline=None), + delimiter=',', quotechar='|', + quoting=csv.QUOTE_MINIMAL, + lineterminator='\n') + for (i, row) in enumerate(input): + if i == 0: + header = row + output.writerow(row) + elif row[3] == backend: + output.writerow(row) + return header + + +def check_args(argv): + if len(argv) < 3: + print("too view arguments provided", file=sys.stderr) + usage() + + config = None + input = None + output = None + + try: + config = json.load(open(argv[1], 'r', newline=None)) + except IOError as x: + print( + f"failed to readonfig file '{argv[1]}'\nwith:\t{x}", + file=sys.stderr) + sys.exit(1) + except json.decoder.JSONDecodeError as x: + print( + f"failed to parse configuration '{argv[1]}'\nwith:\t{x}", + file=sys.stderr) + sys.exit(1) + + try: + input = open(argv[2], 'r', newline=None) + except IOError as x: + print( + f"failed to read from input file '{argv[2]}'\nwith:\t{x}", + file=sys.stderr) + sys.exit(1) + + try: + output = open(argv[3], 'w', newline=None) + except IOError as x: + print( + f"failed to open output file '{argv[3]}'\nwith\t{x}", + file=sys.stderr) + sys.exit(1) + + return (config, input, output) + + +if __name__ == '__main__': + (config, input, output) = check_args(sys.argv) + host = socket.gethostname() + + backends = ['db', 'kv', 'object'] + env = dict(os.environ) + + with tempfile.TemporaryDirectory() as tmp_dir: + def record(backend): + return f"{tmp_dir}/{backend}.csv" + + header = None + for backend in backends: + input.seek(0) + curr_header = extract(input, record, backend) + if header is None: + header = ','.join(curr_header) + + config_filename = f"{tmp_dir}/config" + tmp_output = f"{tmp_dir}/access-record.csv" + + output.write(header) + output.write('\n') + + env['JULEA_CONFIG'] = config_filename + env['JULEA_TRACE'] = 'access' + + fail = 0 + + for backend in backends: + print(f"start {backend}") + backend_configs = config[backend] + if backend_configs is None: + print(f"no configurations found for backend: '{backend}', will skip this!") + continue + + for entry in backend_configs: + type = entry['backend'] + path = entry['path'] + if type is None or path is None: + print("backend type or path are missing, skip this!") + continue + print(f"\tround {type}:{path}") + + tmp_backend_dir = None + pos = path.find('') + if pos != -1: + tmp_backend_dir = tempfile.TemporaryDirectory( + prefix='julea_tmp', + dir=path[:pos]) + path = path.replace('', tmp_backend_dir.name[pos:]) + + with open(config_filename, 'w', newline=None) as out: + out.write(create_config(host, backend, type, path)) + with open(tmp_output, 'w+', newline=None) as out: + try: + subprocess.run( + ['julea-access-replay', record(backend)], + env=env, check=True, stderr=out) + out.seek(0) + first = True + for row in islice(csv.reader(out), 1, None): + if tmp_backend_dir is not None: + row[5] = row[5].replace( + tmp_backend_dir.name[pos:], '') + output.write(','.join(row)) + output.write('\n') + except subprocess.CalledProcessError: + out.seek(0) + print("\t\texecution failed! with:\n" + + '\n'.join( + islice(out.readlines(), 2, None))) + fail += 1 + if tmp_backend_dir is not None: + tmp_backend_dir.cleanup() + + print( + f"{fail} configurations failed! Other configurations" + + f" results are stored in '{sys.argv[3]}'") diff --git a/scripts/interactive.js b/scripts/interactive.js new file mode 100644 index 000000000..e6f1ae3de --- /dev/null +++ b/scripts/interactive.js @@ -0,0 +1,118 @@ + +function(el) { + axis_adapt = function(axis, original_axis, id, dir) { + range = axis.range + range[1] += dir + + label = original_axis[id] + categoryarray = axis.categoryarray + if(dir < 1) { + categoryarray.splice(categoryarray.indexOf(label), 1) + } else { + i = 0 + while(i < categoryarray.length && original_axis.indexOf(categoryarray[i]) < id) { i += 1 } + categoryarray.splice(i, 0, label) + } + return [range, categoryarray] + } + N=%.0f + el.on('plotly_legendclick', function(d) { + // store original configuration + if(!el.mystore) { + el.mystore = {} + el.mystore.autorange = true + el.mystore.labels0 = JSON.parse(JSON.stringify(d.layout.xaxis.categoryarray)) + el.mystore.labels = JSON.parse(JSON.stringify(d.layout.xaxis2.categoryarray)) + } + // get plotting node + node = d.node; + while(!node.id.startsWith('htmlwidget-')) { + node = node.parentElement; + } + + // check if hiding or showing thing + dir = d.node.style.opacity > 0.5 ? -1 : 1 + layout = {} + console.log('click', d,layout) + + // click at time/operation plot + if (d.curveNumber < N) { + [range, categoryarray] = axis_adapt(d.layout.xaxis2, el.mystore.labels, d.curveNumber, dir) + layout["xaxis2.range"] = range, + layout["xaxis2.categoryarray"] = categoryarray + + if (el.mystore.autorange) { // adapt y axis for plot + // time/backendPlot + max = 0 + categoryarray.forEach(category => { + for (j = N; j < d.data.length; j += 1) { + idx = d.data[j].x.indexOf(category) + y = d.data[j].y[idx] + d.data[j].base[idx] + max = Math.max(max, y) + } + }) + layout["yaxis2.range"] = [-max*0.01, max*1.01] + + // time/operationPlot + max = 0 + for (j = 0; j < N; j += 1) { + if (categoryarray.indexOf(d.data[j].name) == -1) { continue; } + idx = 0 + d.layout.xaxis.categoryarray.forEach(category => { + while(el.mystore.labels0[idx] != category) { idx += 1 } + y = d.data[j].y[idx] + max = Math.max(max, y) + }) + } + layout["yaxis.range"] = [-max*0.01, max*1.01] + } + + Plotly.update(node, {"visible": dir > 0 ? true : "legendonly"}, layout, [d.curveNumber]) + + } else { // click in time/backend plot + [range, categoryarray] = axis_adapt(d.layout.xaxis, el.mystore.labels0, d.curveNumber - N, dir) + layout["xaxis.range"] = range, + layout["xaxis.categoryarray"] = categoryarray + + basis = [] + mod = [] + // move base of bar, to avoid gaps in stacked barPlot + for(i = d.curveNumber; i < d.data.length; i += 1) { + mod.push(i) + basis.push(d.data[i].base.map( + (x, j) => x + dir * d.data[d.curveNumber].y[j])) + } + + if (el.mystore.autorange) { // adapt y-axis forPlot + max = 0 + for(i = N; i < d.data.length; i += 1) { + visible = d.fullData[i].visible != "legendonly" && i != d.curveNumber + if (!visible) { continue } + for(j = 0; j < d.layout.xaxis2.categoryarray.length; j += 1) { + if(d.layout.xaxis2.categoryarray.indexOf(d.data[i].x[j]) != -1) { + if (i >= d.curveNumber) { + max = Math.max(max, basis[i-d.curveNumber][j] + d.data[i].y[j]) + } else { + max = Math.max(max, d.data[i].base[j] + d.data[i].y[j]) + } + } + } + } + layout["yaxis2.range"] = [-max*0.01, max*1.01] + max = 0 + categoryarray.forEach(category => { + for (j = 0; j < N; j += 1) { + if (d.layout.xaxis2.categoryarray.indexOf(d.data[j].name) == -1) { continue; } + idx = d.data[j].x.indexOf(category) + y = d.data[j].y[idx] + max = Math.max(max, y) + } + }) + layout["yaxis.range"] = [-max*0.01, max*1.01] + } + Plotly.update(node, {base: basis}, layout, mod) + Plotly.restyle(node, {visible: dir > 0 ? true : "legendonly"}, [d.curveNumber]) + } + return false; + }); +} diff --git a/scripts/style.css b/scripts/style.css new file mode 100644 index 000000000..ce3c5e30f --- /dev/null +++ b/scripts/style.css @@ -0,0 +1,76 @@ + +body { + padding: 0px; + padding-top:31px; + margin: 0px; +} +h2 { + padding-top: 55px; + padding-left: 1em; + font-size: 3em; +} +nav{ + position:fixed; + top:0; + width:100%; +} + +.fancy { + border-collapse: collapse; + margin: 25px 0; + font-size: 0.9em; + font-family: sans-serif; + min-width: 400px; + box-shadow: 0 0 20px rgba(0, 0, 0, 0.15); +} + +.fancy thead tr { + background-color: #009879; + color: #ffffff; + text-align: left; +} + +.fancy th, +.fancy td { + padding: 6px 15px; +} + +.fancy tbody tr { + border-bottom: 1px solid #dddddd; +} + +.fancy tbody tr:nth-of-type(even) { + background-color: #f3f3f3; +} + +.fancy tbody tr:last-of-type { + border-bottom: 2px solid #009879; +} + +td { + padding: 30px 30px; +} + +ul { + list-style-type: none; + margin: 0; + padding: 0; + overflow: hidden; + background-color: #009879; +} + +li { + float: left; +} + +li a { + display: block; + color: white; + text-align: center; + padding: 16px; + text-decoration: none; +} + +li a:hover { + background-color: #004c3c; +} diff --git a/server/loop.c b/server/loop.c index 427503e12..54fa3f6a7 100644 --- a/server/loop.c +++ b/server/loop.c @@ -458,13 +458,15 @@ jd_handle_message(JMessage* message, GSocketConnection* connection, JMemoryChunk g_autoptr(JMessage) reply = NULL; gchar const* client_checksum; gchar const* server_checksum; + gchar const* client_program_name; + guint32 client_process_uid; guint num; num = g_atomic_int_add(&jd_thread_num, 1); - //g_message("HELLO %d", num); - client_checksum = j_message_get_string(message); + client_program_name = j_message_get_string(message); + client_process_uid = *(guint32*)j_message_get_n(message, sizeof(guint32)); server_checksum = j_configuration_get_checksum(jd_configuration); if (g_strcmp0(client_checksum, server_checksum) != 0) @@ -472,6 +474,12 @@ jd_handle_message(JMessage* message, GSocketConnection* connection, JMemoryChunk g_warning("Client %d uses different configuration than server.", num); } + { + J_TRACE("ping", "%s Thread: %u", client_program_name, client_process_uid); + (void)client_program_name; // avoid warning, iff in non trace mode compiled + (void)client_process_uid; + } + reply = j_message_new_reply(message); j_message_append_string(reply, server_checksum); diff --git a/server/server.c b/server/server.c index d6428b45b..e2adb8695 100644 --- a/server/server.c +++ b/server/server.c @@ -28,6 +28,7 @@ #include #include #include +#include #include diff --git a/tools/access_replay.c b/tools/access_replay.c new file mode 100644 index 000000000..56ad65edb --- /dev/null +++ b/tools/access_replay.c @@ -0,0 +1,664 @@ +/* + * JULEA - Flexible storage framework + * Copyright (C) 2023-2023 Julian Benda + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include +#include +#include + +#include + +JBackend* object_backend = NULL; +JBackend* kv_backend = NULL; +JBackend* db_backend = NULL; + +static gboolean +setup_backend(JConfiguration* configuration, JBackendType type, gchar const* port_str, GModule** module, JBackend** j_backend) +{ + gchar const* backend; + gchar const* component; + g_autofree gchar* path; + + backend = j_configuration_get_backend(configuration, type); + component = j_configuration_get_backend_component(configuration, type); + path = j_helper_str_replace(j_configuration_get_backend_path(configuration, type), "{PORT}", port_str); + + if (strcmp(component, "server") != 0) + { + return TRUE; + } + if (j_backend_load_server(backend, component, type, module, j_backend)) + { + gboolean res = TRUE; + if (j_backend == NULL) + { + g_warning("Failed to load server: %s.", backend); + return FALSE; + } + switch (type) + { + case J_BACKEND_TYPE_OBJECT: + res = j_backend_object_init(*j_backend, path); + break; + case J_BACKEND_TYPE_KV: + res = j_backend_kv_init(*j_backend, path); + break; + case J_BACKEND_TYPE_DB: + res = j_backend_db_init(*j_backend, path); + break; + default: + g_warning("unknown backend type: (%d), unable to setup backend!", type); + res = FALSE; + } + if (!res) + { + g_warning("Failed to initelize backend: %s", backend); + return FALSE; + } + return TRUE; + } + return FALSE; +} + +enum row_fields +{ + TIME, + PROCESS_UID, + PROGRAM_NAME, + BACKEND, + TYPE, + PATH, + NAMESPACE, + NAME, + OPERATION, + SEMANTIC, + SIZE, + COMPLEXITY, + DURATION, + JSON, + ROW_LEN +}; + +static gboolean +split(char* line, char* parts[ROW_LEN]) +{ + int cnt = 0; + char* section; + char* save; + section = strtok_r(line, ",", &save); + while (section != NULL) + { + parts[cnt++] = section; + if (cnt == ROW_LEN - 1) + { + break; + } + while (*save == ',') + { + *save = '\0'; + parts[cnt++] = save++; + } + section = strtok_r(NULL, ",", &save); + } + if (section == NULL) + { + g_warning("to few parts in line"); + return FALSE; + } + while (*++section) + ; + parts[cnt] = section + 1; + parts[JSON] += 1; + { + char* itr = parts[JSON]; + while (*++itr) + ; + itr[-1] = 0; + } + return TRUE; +} + +static guint64 +parse_ul(const char* str) +{ + return strtoul(str, NULL, 10); +} + +static gboolean +replay_object(void* memory_chunk, guint64 memory_chunk_size, char** parts) +{ + static gpointer data = NULL; + const char* op = parts[OPERATION]; + gboolean ret = FALSE; + (void)memory_chunk_size; /// \TODO usage? + if (strcmp(op, "create") == 0) + { + if (data != NULL) + { + g_warning("open new object while still one open"); + } + ret = j_backend_object_create(object_backend, parts[NAMESPACE], parts[NAME], &data); + } + else if (strcmp(op, "open") == 0) + { + if (data != NULL) + { + g_warning("open new object while still one open"); + } + ret = j_backend_object_open(object_backend, parts[NAMESPACE], parts[NAME], &data); + } + else if (strcmp(op, "get_all") == 0) + { + if (data != NULL) + { + g_warning("open new object while still one open"); + } + ret = j_backend_object_get_all(object_backend, parts[NAMESPACE], data); + } + else if (strcmp(op, "ge_by_prefix") == 0) + { + if (data != NULL) + { + g_warning("open new object while still one open"); + } + ret = j_backend_kv_get_by_prefix(object_backend, parts[NAMESPACE], parts[NAME], &data); + } + else if (strcmp(op, "read") == 0) + { + static guint64 bytes_read = 0; + guint64 size = parse_ul(parts[SIZE]); + if (size > memory_chunk_size) + { + g_warning("unable to replay: chunk size to small! %lu vs %lu", size, memory_chunk_size); + } + else + { + ret = j_backend_object_read(object_backend, data, memory_chunk, size, parse_ul(parts[COMPLEXITY]), &bytes_read); + } + } + else if (strcmp(op, "write") == 0) + { + static guint64 bytes_written = 0; + guint64 size = parse_ul(parts[SIZE]); + if (size > memory_chunk_size) + { + g_warning("unable to replay: chunk size to small! %lu vs %lu", size, memory_chunk_size); + } + else + { + ret = j_backend_object_write(object_backend, data, memory_chunk, size, parse_ul(parts[COMPLEXITY]), &bytes_written); + } + } + else if (strcmp(op, "status") == 0) + { + static gint64 modification_time; + static guint64 size; + ret = j_backend_object_status(object_backend, data, &modification_time, &size); + } + else if (strcmp(op, "sync") == 0) + { + ret = j_backend_object_sync(object_backend, data); + } + else if (strcmp(op, "iterate") == 0) + { + static const char* name = NULL; + ret = j_backend_object_iterate(object_backend, data, &name); + if (!ret) + { + data = NULL; + ret = TRUE; + } + } + else if (strcmp(op, "close") == 0) + { + ret = j_backend_object_close(object_backend, data); + data = NULL; + } + else if (strcmp(op, "delete") == 0) + { + ret = j_backend_object_delete(object_backend, data); + data = NULL; + } + else if (strcmp(op, "init") == 0) + { + ret = TRUE; + } + else if (strcmp(op, "fini") == 0) + { + ret = TRUE; + } + else + { + g_warning("unkown object operation: '%s'", op); + } + return ret; +} + +static void +update_semantics(JSemantics** semantics, guint32* serial_semantics, const char* parts_semantics_str) +{ + guint32 parts_semantics = parse_ul(parts_semantics_str); + if (*semantics == NULL || *serial_semantics != parts_semantics) + { + *serial_semantics = parts_semantics; + if (*semantics) + { + j_semantics_unref(*semantics); + } + *semantics = j_semantics_deserialize(*serial_semantics); + } +} + +static gboolean +replay_kv(void* memory_chunk, guint64 memory_chunk_size, char** parts) +{ + gboolean ret = FALSE; + const char* op = parts[OPERATION]; + static gpointer batch = NULL; + static JSemantics* semantics = NULL; + static guint32 serial_semantics; + + if (strcmp(op, "batch_start") == 0) + { + if (batch != NULL) + { + g_warning("starting a new batch, but old one is not finished"); + } + update_semantics(&semantics, &serial_semantics, parts[SEMANTIC]); + ret = j_backend_kv_batch_start(kv_backend, parts[NAMESPACE], semantics, &batch); + } + else if (strcmp(op, "put") == 0) + { + guint64 size = parse_ul(parts[SIZE]); + if (size > memory_chunk_size) + { + g_warning("unable to replay: chunk size to small! %lu vs %lu", size, memory_chunk_size); + } + else + { + ret = j_backend_kv_put(kv_backend, batch, parts[NAME], memory_chunk, size); + } + } + else if (strcmp(op, "delete") == 0) + { + ret = j_backend_kv_delete(kv_backend, batch, parts[NAME]); + } + else if (strcmp(op, "get") == 0) + { + guint32 size; + gpointer value; + ret = j_backend_kv_get(kv_backend, batch, parts[NAME], &value, &size); + } + else if (strcmp(op, "get_all") == 0) + { + if (batch != NULL) + { + g_warning("start iterating with remaining batch or iteration"); + } + ret = j_backend_kv_get_all(kv_backend, parts[NAMESPACE], &batch); + } + else if (strcmp(op, "get_by_prefix") == 0) + { + if (batch != NULL) + { + g_warning("start iterating with remaining batch or iteration"); + } + ret = j_backend_kv_get_by_prefix(kv_backend, parts[NAMESPACE], parts[NAME], &batch); + } + else if (strcmp(op, "iterate") == 0) + { + const char* name; + gconstpointer value; + guint32 len; + ret = j_backend_kv_iterate(kv_backend, batch, &name, &value, &len); + if (!ret) + { + batch = NULL; + ret = TRUE; + } + } + else if (strcmp(op, "init") == 0) + { + ret = TRUE; + } + else if (strcmp(op, "fini") == 0) + { + ret = TRUE; + } + else if (strcmp(op, "batch_execute") == 0) + { + ret = j_backend_kv_batch_execute(kv_backend, batch); + } + else + { + g_warning("unkown operation: '%s'", op); + } + return ret; +} + +struct JSqlBatch +{ + const gchar* namespace; + JSemantics* semantics; + gboolean open; + gboolean aborted; +}; + +static gboolean +replay_db(char** parts) +{ + gboolean ret = FALSE; + static gpointer batch = NULL; + static gpointer itr = NULL; + static gchar* namespace = NULL; + static JSemantics* semantics = NULL; + static guint32 serial_semantics; + GError* error = NULL; + const char* op = parts[OPERATION]; + static GArray* bsons = NULL; + if (bsons == NULL) + { + bsons = g_array_sized_new(FALSE, TRUE, sizeof(bson_t*), 0); + } + if (semantics == NULL) + { + semantics = j_semantics_new(J_SEMANTICS_TEMPLATE_DEFAULT); + } + + if (strcmp(op, "batch_start") == 0) + { + if (batch != NULL) + { + g_warning("starting a new batch, but old one is not finished"); + } + update_semantics(&semantics, &serial_semantics, parts[SEMANTIC]); + namespace = strdup(parts[NAMESPACE]); + ret = j_backend_db_batch_start(db_backend, namespace, semantics, &batch, &error); + } + else if (strcmp(op, "schema_create") == 0) + { + bson_t* schema = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); + ret = j_backend_db_schema_create(db_backend, batch, parts[NAME], schema, &error); + g_array_append_val(bsons, schema); + } + else if (strcmp(op, "schema_get") == 0) + { + bson_t* schema = bson_new(); + ret = j_backend_db_schema_get(db_backend, batch, parts[NAME], schema, &error); + g_array_append_val(bsons, schema); + // schema do not exists + if (error && error->code == 8) + { + g_error_free(error); + error = NULL; + ret = TRUE; + } + } + else if (strcmp(op, "schema_delete") == 0) + { + ret = j_backend_db_schema_delete(db_backend, batch, parts[NAME], &error); + } + else if (strcmp(op, "insert") == 0) + { + bson_t* entry = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); + bson_t* res = bson_new(); + ret = j_backend_db_insert(db_backend, batch, parts[NAME], entry, res, &error); + g_array_append_val(bsons, entry); + g_array_append_val(bsons, res); + } + else if (strcmp(op, "delete") == 0) + { + bson_t* entry = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); + ret = j_backend_db_delete(db_backend, batch, parts[NAME], entry, &error); + g_array_append_val(bsons, entry); + } + else if (strcmp(op, "update") == 0) + { + bson_iter_t bson_iter; + bson_t* selector; + bson_t* entry; + const uint8_t* doc; + uint32_t len; + bson_t* bson = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); + bson_iter_init_find(&bson_iter, bson, "entry"); + bson_iter_document(&bson_iter, &len, &doc); + entry = bson_new_from_data(doc, len); + bson_iter_init_find(&bson_iter, bson, "selector"); + bson_iter_document(&bson_iter, &len, &doc); + selector = bson_new_from_data(doc, len); + ret = j_backend_db_update(db_backend, batch, parts[NAME], selector, entry, &error); + g_array_append_val(bsons, entry); + g_array_append_val(bsons, selector); + g_array_append_val(bsons, bson); + } + else if (strcmp(op, "query") == 0) + { + bson_t* selector = bson_new_from_json((const uint8_t*)parts[JSON], -1, NULL); + if (itr != NULL) + { + g_warning("start new db iteration without finishing old one!"); + } + ret = j_backend_db_query(db_backend, batch, parts[NAME], selector, &itr, &error); + g_array_append_val(bsons, selector); + } + else if (strcmp(op, "iterate") == 0) + { + bson_t* entry = bson_new(); + ret = j_backend_db_iterate(db_backend, itr, entry, &error); + g_array_append_val(bsons, entry); + if (!ret) + { + itr = NULL; + ret = TRUE; + g_error_free(error); + error = NULL; + } + } + else if (strcmp(op, "init") == 0) + { + ret = TRUE; + } + else if (strcmp(op, "fini") == 0) + { + ret = TRUE; + } + else if (strcmp(op, "batch_execute") == 0) + { + ret = j_backend_db_batch_execute(db_backend, batch, &error); + batch = NULL; + for (guint i = 0; i < bsons->len; ++i) + { + bson_destroy(g_array_index(bsons, bson_t*, i)); + } + g_array_remove_range(bsons, 0, bsons->len); + // no operation to do + if (error && error->code == 7) + { + g_error_free(error); + error = NULL; + ret = TRUE; + } + } + else + { + g_warning("unknown operation: '%s'", op); + } + if (error) + { + g_warning("db error(%d): %s", error->code, error->message); + } + return ret && error == NULL; +} + +static gboolean +replay(void* memory_chunk, guint64 memory_chunck_size, char* line) +{ + char* parts[ROW_LEN]; + if (!split(line, parts)) + { + return FALSE; + } + if (strcmp(parts[BACKEND], "object") == 0) + { + if (!replay_object(memory_chunk, memory_chunck_size, parts)) + return FALSE; + } + if (strcmp(parts[BACKEND], "kv") == 0) + { + if (!replay_kv(memory_chunk, memory_chunck_size, parts)) + return FALSE; + } + if (strcmp(parts[BACKEND], "db") == 0) + { + if (!replay_db(parts)) + return FALSE; + } + + return TRUE; +} + +int +main(int argc, char** argv) +{ + JConfiguration* configuration = NULL; + const char* record_file_name = NULL; + FILE* record_file = NULL; + GModule* db_module = NULL; + GModule* kv_module = NULL; + GModule* object_module = NULL; + g_autofree gchar* port_str = NULL; + gint res = 1; + gchar* memory_chunk; + guint64 memory_chunck_size = 0; + + setlocale(LC_ALL, "C.UTF-8"); + + configuration = j_configuration(); + if (configuration == NULL) + { + g_warning("unable to read config"); + goto end; + if (argc < 2) + { + g_warning("useage: %s ", argv[0]); + goto end; + } + } + record_file_name = argv[1]; + if (access(record_file_name, R_OK) != 0) + { + g_warning("file does not exists, or no read permission '%s'", record_file_name); + goto end; + } + record_file = fopen(record_file_name, "r"); + if (record_file == NULL) + { + g_warning("failed to open file '%s'", record_file_name); + goto end; + } + + port_str = g_strdup_printf("%d", j_configuration_get_port(configuration)); + if (!setup_backend(configuration, J_BACKEND_TYPE_OBJECT, port_str, &object_module, &object_backend)) + { + g_warning("failed to initealize object backend"); + goto end; + } + if (!setup_backend(configuration, J_BACKEND_TYPE_KV, port_str, &kv_module, &kv_backend)) + { + g_warning("failed to initealize kv backend"); + goto end; + } + if (!setup_backend(configuration, J_BACKEND_TYPE_DB, port_str, &db_module, &db_backend)) + { + g_warning("failed to initealize db backend"); + goto end; + } + + memory_chunck_size = j_configuration_get_max_operation_size(configuration); + { + // initialize memory_chunk with random values + guint32* memory = malloc(memory_chunck_size + memory_chunck_size % 4); + GRand* rng = g_rand_new(); + for (guint64 i = 0; i < (memory_chunck_size + 3) / 4; i += 1) + { + memory[i] = g_rand_int(rng); + } + + g_rand_free(rng); + memory_chunk = (char*)memory; + } + + { + const char* header = "time,process_uid,program_name,backend,type,path,namespace,name,operation,semantics,size,complexity,duration,bson"; + char* line = NULL; + size_t len = 0; + ssize_t read = 0; + read = getline(&line, &len, record_file); + line[read - 1] = 0; + if (read == -1 || strcmp(line, header) != 0) + { + g_warning("Invalid header in dump!\n'%s'\n'%s'\n", header, line); + goto end; + } + while ((read = getline(&line, &len, record_file)) != -1) + { + line[read - 1] = 0; + if (!replay(memory_chunk, memory_chunck_size, line)) + { + g_warning("failed to replay dump!"); + goto end; + } + } + } + res = 0; +end: + if (record_file) + { + fclose(record_file); + } + if (db_backend != NULL) + { + j_backend_db_fini(db_backend); + } + if (kv_backend != NULL) + { + j_backend_kv_fini(kv_backend); + } + if (object_backend != NULL) + { + j_backend_object_fini(object_backend); + } + if (db_module != NULL) + { + g_module_close(db_module); + } + if (kv_module != NULL) + { + g_module_close(kv_module); + } + if (object_module != NULL) + { + g_module_close(object_module); + } + j_configuration_unref(configuration); + + return res; +}