From 92ee540a19356bbf9d118a33cfc0edeffb58b3da Mon Sep 17 00:00:00 2001 From: Ben Allan Date: Mon, 4 Dec 2023 21:19:03 -0700 Subject: [PATCH] add use_base to dcgm sampler --- ldms/scripts/examples/.canned | 1 + ldms/scripts/examples/dcgm1 | 15 ++ ldms/scripts/examples/dcgm1.1 | 3 + ldms/scripts/examples/dcgm1.2 | 15 ++ ldms/src/sampler/dcgm_sampler/Makefile.am | 25 +- .../dcgm_sampler/Plugin_dcgm_sampler.man | 23 +- ldms/src/sampler/dcgm_sampler/dcgm_sampler.c | 236 +++++++++++++----- .../dcgm_sampler/gen-ldms-dcgm-list-fields | 90 +++++++ 8 files changed, 338 insertions(+), 70 deletions(-) create mode 100644 ldms/scripts/examples/dcgm1 create mode 100644 ldms/scripts/examples/dcgm1.1 create mode 100644 ldms/scripts/examples/dcgm1.2 create mode 100755 ldms/src/sampler/dcgm_sampler/gen-ldms-dcgm-list-fields diff --git a/ldms/scripts/examples/.canned b/ldms/scripts/examples/.canned index 9366398b5..a4a790c3c 100644 --- a/ldms/scripts/examples/.canned +++ b/ldms/scripts/examples/.canned @@ -42,6 +42,7 @@ lustre_mdc dstat.job many conf_csv +dcgm1 # not yet tested rabbitv3 diff --git a/ldms/scripts/examples/dcgm1 b/ldms/scripts/examples/dcgm1 new file mode 100644 index 000000000..1b1007e33 --- /dev/null +++ b/ldms/scripts/examples/dcgm1 @@ -0,0 +1,15 @@ +export plugname=dcgm +portbase=61073 +VGARGS="--leak-check=full --track-origins=yes --trace-children=yes" +JOBDATA $TESTDIR/job.data 1 2 +vgoff +LDMSD -p prolog.jobid 1 2 +vgoff +MESSAGE ldms_ls on host 1: +LDMS_LS 1 -lv +SLEEP 1 +MESSAGE ldms_ls on host 2: +LDMS_LS 2 -l +SLEEP 5 +KILL_LDMSD 1 2 +file_created $STOREDIR/node/$testname diff --git a/ldms/scripts/examples/dcgm1.1 b/ldms/scripts/examples/dcgm1.1 new file mode 100644 index 000000000..b1306e5cb --- /dev/null +++ b/ldms/scripts/examples/dcgm1.1 @@ -0,0 +1,3 @@ +load name=dcgm_sampler +config name=dcgm_sampler producer=localhost${i} schema=${testname} instance=localhost${i}/${testname} component_id=${i} interval=1000000 perm=757 uid=3556 gid=3556 job_set=instance=localhost${i}/job_info use_base=1 +start name=dcgm_sampler interval=1000000 offset=0 diff --git a/ldms/scripts/examples/dcgm1.2 b/ldms/scripts/examples/dcgm1.2 new file mode 100644 index 000000000..05355b528 --- /dev/null +++ b/ldms/scripts/examples/dcgm1.2 @@ -0,0 +1,15 @@ +# cannot load sampler instance on same node. + +load name=store_csv +config name=store_csv path=${STOREDIR} altheader=0 + +prdcr_add name=localhost1 host=${HOST} type=active xprt=${XPRT} port=${port1} interval=2000000 +prdcr_start name=localhost1 + +updtr_add name=allhosts interval=1000000 offset=100000 +updtr_prdcr_add name=allhosts regex=.* +updtr_start name=allhosts + +strgp_add name=store_${testname} plugin=store_csv schema=${testname} container=node +strgp_prdcr_add name=store_${testname} regex=.* +strgp_start name=store_${testname} diff --git a/ldms/src/sampler/dcgm_sampler/Makefile.am b/ldms/src/sampler/dcgm_sampler/Makefile.am index 5f060aa90..af0de9bc9 100644 --- a/ldms/src/sampler/dcgm_sampler/Makefile.am +++ b/ldms/src/sampler/dcgm_sampler/Makefile.am @@ -1,17 +1,34 @@ +bin_PROGRAMS = ldms-dcgm-list-fields + libdcgm_sampler_la_SOURCES = \ - dcgm_sampler.c + dcgm_sampler.c + libdcgm_sampler_la_LIBADD = \ + $(top_builddir)/ldms/src/sampler/libsampler_base.la \ $(top_builddir)/ldms/src/core/libldms.la \ $(top_builddir)/lib/src/coll/libcoll.la \ - $(top_builddir)/ldms/src/sampler/libjobid_helper.la \ + $(top_builddir)/ldms/src/sampler/libjobid_helper.la \ -ldcgm + libdcgm_sampler_la_LDFLAGS = \ -no-undefined \ - -export-symbols-regex 'get_plugin' \ - -version-info 1:0:0 + -export-symbols-regex 'get_plugin' \ + -version-info 1:0:0 libdcgm_sampler_la_CPPFLAGS = \ @OVIS_INCLUDE_ABS@ pkglib_LTLIBRARIES = libdcgm_sampler.la dist_man7_MANS = Plugin_dcgm_sampler.man + +dist_noinst_SCRIPTS = gen-ldms-dcgm-list-fields + +ldms-dcgm-list-fields.c: $(srcdir)/gen-ldms-dcgm-list-fields + $(srcdir)/gen-ldms-dcgm-list-fields > ldms-dcgm-list-fields.c + +ldms_dcgm_list_fields_SOURCES = ldms-dcgm-list-fields.c +ldms_dcgm_list_fields_CPPFLAGS = @OVIS_INCLUDE_ABS@ +ldms_dcgm_list_fields_LDADD = -ldcgm + +clean-local:: + $(RM) $(builddir)/ldms_dcgm_list_fields.c diff --git a/ldms/src/sampler/dcgm_sampler/Plugin_dcgm_sampler.man b/ldms/src/sampler/dcgm_sampler/Plugin_dcgm_sampler.man index 101e7bd8a..0fe3346d7 100644 --- a/ldms/src/sampler/dcgm_sampler/Plugin_dcgm_sampler.man +++ b/ldms/src/sampler/dcgm_sampler/Plugin_dcgm_sampler.man @@ -6,7 +6,7 @@ Plugin_dcgm_sampler - man page for the LDMS dcgm_sampler plugin .SH SYNOPSIS Within ldmsd_controller or a configuration file: .br -config name=dcgm_sampler [ = ] +config name=dcgm_sampler [ = ] [use_base=1] .SH DESCRIPTION With LDMS (Lightweight Distributed Metric Service), plugins for the ldmsd (ldms daemon) are configured via ldmsd_controller @@ -17,7 +17,7 @@ The schema is named "dcgm" by default. .TP .BR config -name= interval= [fields=] [schema=] [job_set=] +name= interval= [fields=] [schema=] [job_set=] [use_base=1 [[uid=] [gid=] [perm=] [instance=] [producer=] [job_id=]] .br configuration line .RS @@ -26,16 +26,24 @@ name= .br This MUST be dcgm_sampler. .TP +use_base=1 +.br +This enables the sampler_base configuration option processing (see ldms_sampler_base(7)). If not given, the options not +listed below are ignored. +.TP interval= .br -The sampling interval. This MUST be set to the same value that is -set on the "start" line, otherwise behavior is undetermined. +The DCGM library sampling interval (dcgmWatchFields() updateFreq). This MUST be set to the same value that is +set on the dcgm_sampler start line, otherwise behavior is undetermined. .TP fields= .br is a comma-separated list of integers representing DCGM field -numebers that the plugin should watch. By default the plugin will -watch fields 150,155. +identifiers that the plugin should watch. By default the plugin will +watch fields 150,155. The field identifier meanings are defined in dcgm_fields.h +and the DCGM Library API Reference Manual and may vary with DCGM release version. +The ldms-dcgm-list-fields command provides a table of fields, subject to hardware +support. .TP schema= .br @@ -59,5 +67,8 @@ config name=dcgm_sampler interval=1000000 fields=150,155,1001,1002,1003 schema=d start name=dcgm_sampler interval=1000000 .fi +.SH NOTES +Multiple instances of the sampler cannot run on the same server. + .SH SEE ALSO ldmsd(8), ldms_quickstart(7), ldmsd_controller(8), ldms_sampler_base(7) diff --git a/ldms/src/sampler/dcgm_sampler/dcgm_sampler.c b/ldms/src/sampler/dcgm_sampler/dcgm_sampler.c index d8482ed37..b1b637fde 100644 --- a/ldms/src/sampler/dcgm_sampler/dcgm_sampler.c +++ b/ldms/src/sampler/dcgm_sampler/dcgm_sampler.c @@ -18,9 +18,14 @@ #include #include "config.h" #include "jobid_helper.h" +#include "sampler_base.h" +#include #define _GNU_SOURCE + +static pthread_mutex_t cfg_lock = PTHREAD_MUTEX_INITIALIZER; + #define SAMP "dcgm_sampler" static unsigned short default_fields[] = { @@ -67,6 +72,9 @@ static ldms_schema_t gpu_schema; /* NOTE: we are assuming here that GPU ids will start at zero and not exceed the DCGM_MAX_NUM_DEVICES count in value */ static ldms_set_t gpu_sets[DCGM_MAX_NUM_DEVICES]; +static int use_base; +static base_data_t base; +static int termed; /* We won't use many of the entries in this array, but DCGM_FI_MAX_FIELDS is is only around 1000. We trade off memory usage to allow quick translation of @@ -235,14 +243,38 @@ static ldms_set_t gpu_metric_set_create(int gpu_id) char instance_name[256]; ovis_log(mylog, OVIS_LDEBUG, "gpu_metric_set_create() (gpu %d)\n", gpu_id); - - snprintf(instance_name, sizeof(instance_name), "%s/gpu_%d", - producer_name, gpu_id); - set = ldms_set_new(instance_name, gpu_schema); - ldms_set_producer_name_set(set, producer_name); - ldms_metric_set_s32(set, gpu_id_metric_index, gpu_id); - ldms_set_publish(set); - ldmsd_set_register(set, SAMP); + if (use_base) { + char *tmp = base->instance_name; + size_t len = strlen(tmp); + base->instance_name = malloc( len + 20); + if (!base->instance_name) { + base->instance_name = tmp; + ovis_log(mylog, OVIS_LERROR, "out of memory\n"); + return NULL; + } + /* append gpu_# to user-defined instance name */ + snprintf(base->instance_name, len + 20, "%s/gpu_%d", tmp, gpu_id); + /* override single set assumed in sampler_base api */ + set = base_set_new(base); + if (!set) { + ovis_log(mylog, OVIS_LERROR, "failed to make %s set for %s\n", + base->instance_name, SAMP); + base->instance_name = tmp; + return set; + } + base_auth_set(&base->auth, set); + base->set = NULL; + free(base->instance_name); + base->instance_name = tmp; + } else { + snprintf(instance_name, sizeof(instance_name), "%s/gpu_%d", + producer_name, gpu_id); + set = ldms_set_new(instance_name, gpu_schema); + ldms_set_producer_name_set(set, producer_name); + ldms_set_publish(set); + ldmsd_set_register(set, SAMP); + } + ldms_metric_set_s32(set, gpu_id_metric_index, gpu_id); return set; } @@ -262,12 +294,18 @@ static int gpu_schema_create() int i; ovis_log(mylog, OVIS_LDEBUG, "gpu_schema_create()\n"); - sch = ldms_schema_new(conf.schema_name); - if (sch == NULL) - goto err1; - rc = jobid_helper_schema_add(sch); - if (rc < 0) - goto err2; + if (!use_base) { + sch = ldms_schema_new(conf.schema_name); + if (sch == NULL) + goto err1; + rc = jobid_helper_schema_add(sch); + if (rc < 0) + goto err2; + } else { + sch = base_schema_new(base); + if (sch == NULL) + goto err1; + } rc = ldms_schema_meta_add(sch, "gpu_id", LDMS_V_S32); if (rc < 0) goto err2; @@ -290,7 +328,10 @@ static int gpu_schema_create() return 0; err2: - ldms_schema_delete(sch); + if (use_base) + base_schema_delete(base); + else + ldms_schema_delete(sch); err1: ovis_log(mylog, OVIS_LERROR, "schema creation failed.\n"); return -1; @@ -298,7 +339,10 @@ static int gpu_schema_create() static void gpu_schema_destroy() { - ldms_schema_delete(gpu_schema); + if (use_base) + base_schema_delete(base); + else + ldms_schema_delete(gpu_schema); gpu_schema = NULL; } @@ -387,37 +431,57 @@ static int config(struct ldmsd_plugin *self, int rc = -1; int i; + pthread_mutex_lock(&cfg_lock); + if (termed) + termed = 0; ovis_log(mylog, OVIS_LDEBUG, "config() called\n"); - - int jc = jobid_helper_config(avl); - if (jc) { - ovis_log(mylog, OVIS_LERROR, SAMP": set name for job_set=" - " is too long.\n"); - rc = jc; - goto err0; + if (dcgm_initialized) { + ovis_log(mylog, OVIS_LERROR, "config() called twice. Stop it first.\n"); + pthread_mutex_unlock(&cfg_lock); + return EINVAL; } - value = av_value(avl, "interval"); - if (value == NULL) { - ovis_log(mylog, OVIS_LERROR, "config() \"interval\" option missing\n"); - goto err0; - } - errno = 0; - conf.interval = strtol(value, NULL, 10); - if (errno != 0) { - ovis_log(mylog, OVIS_LERROR, "config() \"interval\" value conversion error: %d\n", errno); - goto err0; - } - - value = av_value(avl, "schema"); + value = av_value(avl, "use_base"); if (value != NULL) { - conf.schema_name = strdup(value); - } else { - conf.schema_name = strdup("dcgm"); - } - if (conf.schema_name == NULL) { - ovis_log(mylog, OVIS_LERROR, "config() strdup schema failed: %d", errno); - goto err0; - } + use_base = 1; + ovis_log(mylog, OVIS_LDEBUG, "Using sampler_base\n"); + } else { + ovis_log(mylog, OVIS_LDEBUG, "Ignoring sampler_base\n"); + } + + value = av_value(avl, "interval"); + if (value == NULL) { + ovis_log(mylog, OVIS_LERROR, "config() \"interval\" option missing\n"); + goto err0; + } + errno = 0; + conf.interval = strtol(value, NULL, 10); + if (errno != 0) { + ovis_log(mylog, OVIS_LERROR, "config() \"interval\" value conversion error: %d\n", errno); + goto err0; + } + + if (! use_base) { + int jc = jobid_helper_config(avl); + if (jc) { + ovis_log(mylog, OVIS_LERROR, "set name for job_set=" + " is too long.\n"); + rc = jc; + goto err0; + } + value = av_value(avl, "schema"); + if (value != NULL) { + conf.schema_name = strdup(value); + } else { + conf.schema_name = strdup("dcgm"); + } + if (conf.schema_name == NULL) { + ovis_log(mylog, OVIS_LERROR, "config() strdup schema failed: %d", errno); + goto err0; + } + } else { + base_config(avl, SAMP, "dcgm", mylog); + conf.schema_name = strdup(base->schema_name); + } value = av_value(avl, "fields"); if (value != NULL) { @@ -444,19 +508,28 @@ static int config(struct ldmsd_plugin *self, goto err3; for (i = 0; i < gpu_ids_count; i++) { if (gpu_ids[i] > DCGM_MAX_NUM_DEVICES) { - ovis_log(mylog, OVIS_LERROR, "gpu id %d is greater than DCGM_MAX_NUM_DEVICES (%d), will require code fix\n"); + ovis_log(mylog, OVIS_LERROR, + "gpu id %d is greater than DCGM_MAX_NUM_DEVICES (%d), will require code fix\n", + i , DCGM_MAX_NUM_DEVICES); goto err4; } gpu_sets[gpu_ids[i]] = gpu_metric_set_create(gpu_ids[i]); } + pthread_mutex_unlock(&cfg_lock); return 0; err4: for (i = i-1; i >= 0; i--) { gpu_metric_set_destroy(gpu_sets[gpu_ids[i]]); } - gpu_schema_destroy(); + gpu_schema_destroy(); + if (use_base) { + free(base->instance_name); + base->instance_name = NULL; + base_del(base); + base = NULL; + } err3: dcgm_fini(); err2: @@ -468,13 +541,17 @@ static int config(struct ldmsd_plugin *self, free(conf.schema_name); conf.schema_name = NULL; err0: + pthread_mutex_unlock(&cfg_lock); return rc; } static int sample(struct ldmsd_sampler *self) { - ovis_log(mylog, OVIS_LDEBUG, "sample() called\n"); - gpu_sample(); + ovis_log(mylog, OVIS_LDEBUG, SAMP" sample() called\n"); + pthread_mutex_lock(&cfg_lock); + if (!termed) + gpu_sample(); + pthread_mutex_unlock(&cfg_lock); return 0; } @@ -482,20 +559,32 @@ static void term(struct ldmsd_plugin *self) { int i; + ovis_log(mylog, OVIS_LDEBUG, "term() called\n"); + pthread_mutex_lock(&cfg_lock); + gpu_schema_destroy(); + if (use_base) { + free(base->instance_name); + base->instance_name = NULL; + base_del(base); + base = NULL; + } free(conf.schema_name); conf.schema_name = NULL; - free(conf.fields); - conf.fields = NULL; - conf.fields_len = 0; - conf.interval = 0; - for (i = 0; i < gpu_ids_count; i++) { - gpu_metric_set_destroy(gpu_sets[gpu_ids[i]]); - } - gpu_schema_destroy(); - dcgm_fini(); - ovis_log(mylog, OVIS_LDEBUG, "term() called\n"); - if (mylog) + free(conf.fields); + conf.fields = NULL; + conf.fields_len = 0; + conf.interval = 0; + for (i = 0; i < gpu_ids_count; i++) { + gpu_metric_set_destroy(gpu_sets[gpu_ids[i]]); + } + dcgm_fini(); + termed = 1; + use_base = 0; + if (mylog) { ovis_log_destroy(mylog); + mylog = NULL; + } + pthread_mutex_unlock(&cfg_lock); } static ldms_set_t get_set(struct ldmsd_sampler *self) @@ -506,7 +595,34 @@ static ldms_set_t get_set(struct ldmsd_sampler *self) static const char *usage(struct ldmsd_plugin *self) { ovis_log(mylog, OVIS_LDEBUG, "usage() called\n"); - return "config name=" SAMP; + return "config name=" SAMP + " interval= [fields=]\n" + " [schema=] [job_set=]\n" + " [use_base=1\n" + " [[uid=] [gid=] [perm=] [instance=]\n" + " [producer=] [job_id=]\n" + " ]\n" + " name=\n" + " interval= DCGM query interval (microsecond)\n" + " must match dcgm_sampler interval for plugin start\n" + " fields= list of DCGM field_ids\n" + " schema= default " SAMP "\n" + " job_set=\n" + " If use_base=1 is given, the additional parameters are applied\n" + " (see ldms_sampler_base).\n" + " producer A unique name for the host providing the timing data\n" + " (default $HOSTNAME)\n" + " instance A unique name for the timing metric set\n" + " (default $producer/" SAMP "/gpu_X)\n" + " component_id A unique number for the component being monitoring.\n" + " (default 0)\n" + " schema The base name of the port metrics schema.\n" + " (default " SAMP ".\n" + " uid The user-id of the set's owner\n" + " gid The group id of the set's owner\n" + " perm The set's access permissions\n" + " See ldms-dcgm-list-fields for input values to fields\n" + ; } static struct ldmsd_sampler nvidia_dcgm_plugin = { diff --git a/ldms/src/sampler/dcgm_sampler/gen-ldms-dcgm-list-fields b/ldms/src/sampler/dcgm_sampler/gen-ldms-dcgm-list-fields new file mode 100755 index 000000000..c4473ddd3 --- /dev/null +++ b/ldms/src/sampler/dcgm_sampler/gen-ldms-dcgm-list-fields @@ -0,0 +1,90 @@ +#! /bin/bash +cat << EOF +/* generated with gen_ldms_dcgm_list_fields */ +#include +#include +#include +#include +struct fielddef { const char *macro; int field_id; char *tag;}; +#define FIELDDEF(m) { #m, m, NULL }, +struct fielddef all_fields[] = { +EOF +grep DCGM_FI_ /usr/include/dcgm_fields.h |grep '^#define' |grep ' [0-9]'|grep -v + | sed -e 's/#define //' -e 's/ .*//g' -e 's/^/FIELDDEF(/' -e 's/$/)/' +cat << EOF + { NULL, 0, NULL } +}; + + +#define ARRAY_SIZE(a) (sizeof(a) / sizeof(*a)) +#define _GNU_SOURCE + + +static int dcgm_init() +{ + dcgmReturn_t rc; + + rc = dcgmInit(); + if (rc != DCGM_ST_OK) { + return -1; + } + + return 0; +} + +static void dcgm_fini() +{ + dcgmShutdown(); +} + +const char *typeString(int ft) +{ + switch (ft) { + case DCGM_FT_DOUBLE: + return "double"; + case DCGM_FT_INT64: + return "int64_t"; + case DCGM_FT_STRING: + return "string"; + case DCGM_FT_TIMESTAMP: + return "timestamp"; + default: + return "unmapped_type"; + } +} + +static void dump_dcgm_tags() +{ + int i; + printf("field_id\ttag\tmacro\ttype\tunits\n"); + for (i = 0; i < ARRAY_SIZE(all_fields) && + all_fields[i].macro != NULL; i++) { + + dcgm_field_meta_p field_meta; + field_meta = DcgmFieldGetById(all_fields[i].field_id); + if (field_meta) { + switch (field_meta->fieldType) { + case DCGM_FT_DOUBLE: + case DCGM_FT_INT64: + case DCGM_FT_STRING: + case DCGM_FT_TIMESTAMP: + all_fields[i].tag = strdup(field_meta->tag); + printf("%d\t%s\t%s\t%s\t\"%s\"\n", + all_fields[i].field_id, field_meta->tag, + all_fields[i].macro, + typeString(field_meta->fieldType), + field_meta->valueFormat->unit); + break; + default: + break; + } + } + } +} + +int main() +{ + dcgm_init(); + dump_dcgm_tags(); + dcgm_fini(); +} +EOF