[Starpu-devel] Possible bug in performance model file parsing

[Mirko Myllykoski <mirkom@cs.umu.se>]

Hi,

I have encountered a bug with the multiple linear regression performance 
models in the latest StarPU 1.3 nightly snapshot. The bug occurs only 
when a codelet has multiple implementations.

I modified the examples/mlr/mlr.c example program (see the attachment 
mlr.c) so that the bug can be easily replicated. More precisely, I added 
"fake" CUDA implementations to the codelets:

static struct starpu_codelet cl_init =
{
        .cpu_funcs = { cpu_func },
        .cpu_funcs_name = { "cpu_func" },
        .cuda_funcs = { cpu_func }, //<<<<
        .nbuffers = 1,
        .modes = {STARPU_R},
        .specific_nodes = 1,
        .nodes = {STARPU_MAIN_RAM}, //<<<<
        .model = &cl_model_init,    //<<<<
};

static struct starpu_codelet cl_final =
{
        .cpu_funcs = { cpu_func },
        .cpu_funcs_name = { "cpu_func" },
        .cuda_funcs = { cpu_func }, //<<<<
        .nbuffers = 1,
        .modes = {STARPU_R},
        .specific_nodes = 1,
        .nodes = {STARPU_MAIN_RAM}, //<<<<
        .model = &cl_model_final,   //<<<<
};

It appears that the initial calibration run goes through without any 
problems and writes the performance models to the performance model file 
(see the attachment first_run.txt). However, the second run triggers an 
assert at core/perfmodel/perfmodel_history.c:665 (see the attachments 
second_run.txt and gdb_run.txt).

Adding a second CPU implementation also triggers the same assert.

Best Regards,
Mirko Myllykoski

/* StarPU --- Runtime system for heterogeneous multicore architectures.
 *
 * Copyright (C) 2017                                     CNRS
 * Copyright (C) 2016-2017                                Inria
 *
 * StarPU is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or (at
 * your option) any later version.
 *
 * StarPU is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 * See the GNU Lesser General Public License in COPYING.LGPL for more details.
 */

/*
 * This examples demonstrates how to use multiple linear regression
   models.

   First, there is mlr_codelet__init codelet for which we know the
   parameters, but not the their exponents and relations. This tasks
   should be benchmarked and analyzed to find the model, using
   "tools/starpu_mlr_analysis" script as a template.

   For the second (codelet cl_model_final), it is assumed that the
   analysis has already been performed and that the duration of the
   codelet mlr_codelet_final will be computed using the following
   equation:

   T = a + b * (M^2*N) + c * (N^3*K)

   where M, N, K are the parameters of the task, exponents are coming
   from model->combinations[..][..]  and finally a, b, c are
   coefficients which mostly depend on the machine speed.

   These coefficients are going to be automatically computed using
   least square method.

 */

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <starpu.h>

static long sum;

/* Performance function of the task, which is in this case very simple, as the parameter values just need to be written in the array "parameters" */
static void cl_params(struct starpu_task *task, double *parameters)
{
	int m, n, k;
	int* vector_mn;

	vector_mn = (int*)STARPU_VECTOR_GET_PTR(task->interfaces[0]);
	m = vector_mn[0];
	n = vector_mn[1];

	starpu_codelet_unpack_args(task->cl_arg, &k);

	parameters[0] = m;
	parameters[1] = n;
	parameters[2] = k;
}

/* Function of the task that will be executed. In this case running dummy cycles, just to make sure task duration is significant */
void cpu_func(void *buffers[], void *cl_arg)
{
	long i;
	int m,n,k;
	int* vector_mn;

	vector_mn = (int*)STARPU_VECTOR_GET_PTR(buffers[0]);
	m = vector_mn[0];
	n = vector_mn[1];

	starpu_codelet_unpack_args(cl_arg, &k);

	for(i=0; i < (long) (m*m*n); i++)
		sum+=i;

	for(i=0; i < (long) (n*n*n*k); i++)
		sum+=i;
}

/* ############################################ */
/* Start of the part specific to multiple linear regression perfmodels */

/* Defining perfmodel, number of parameters and their names Initially
   application developer only knows these parameters. The execution of
   this codelet will generate traces that can be analyzed using
   "tools/starpu_mlr_analysis" as a template to obtain the parameters
   combinations and exponents.
 */

static const char * parameters_names[]	= {	"M",	"N",	"K", };

static struct starpu_perfmodel cl_model_init =
{
	.type = STARPU_MULTIPLE_REGRESSION_BASED,
	.symbol = "mlr_init",
	.parameters = cl_params,
	.nparameters = 3,
	.parameters_names = parameters_names,
};

/* Defining the equation for modeling duration of the task. The
   parameters combinations and exponents are computed externally
   offline, for example using "tools/starpu_mlr_analysis" tool as a
   template.
 */

static unsigned combi1 [3]		= {	2,	1,	0 };
static unsigned combi2 [3]		= {	0,	3,	1 };

static unsigned *combinations[] = { combi1, combi2 };

static struct starpu_perfmodel cl_model_final =
{
	.type = STARPU_MULTIPLE_REGRESSION_BASED,
	.symbol = "mlr_final",
	.parameters = cl_params,
	.nparameters = 3,
	.parameters_names = parameters_names,
	.ncombinations = 2,
	.combinations = combinations,
};

/* End of the part specific to multiple linear regression perfmodels */
/* ############################################ */

static struct starpu_codelet cl_init =
{
	.cpu_funcs = { cpu_func },
	.cpu_funcs_name = { "cpu_func" },
	.cuda_funcs = { cpu_func },
	.nbuffers = 1,
	.modes = {STARPU_R},
	.specific_nodes = 1,
	.nodes = {STARPU_MAIN_RAM},
	.model = &cl_model_init,
};

static struct starpu_codelet cl_final =
{
	.cpu_funcs = { cpu_func },
	.cpu_funcs_name = { "cpu_func" },
	.cuda_funcs = { cpu_func },
	.nbuffers = 1,
	.modes = {STARPU_R},
	.specific_nodes = 1,
	.nodes = {STARPU_MAIN_RAM},
	.model = &cl_model_final,
};


int main(void)
{
	/* Initialization */
	unsigned i;
	int ret;

	ret = starpu_init(NULL);
	if (ret == -ENODEV)
		return 77;

	sum=0;
	int* vector_mn = calloc(2, sizeof(int));
	starpu_data_handle_t vector_mn_handle;

	starpu_vector_data_register(&vector_mn_handle,
				    STARPU_MAIN_RAM,
				    (uintptr_t)vector_mn, 2,
				    sizeof(int));

	/* Giving pseudo-random values to the M,N,K parameters and inserting tasks */
	for (i = 0; i < 42; i++)
	{
		int j;
		int m,n,k;

		m = (int) ((rand() % 10)+1);
		n = (int) ((rand() % 10)+1);
		k = (int) ((rand() % 10)+1);

		/* To illustrate the usage, M and N are stored in a data handle */
		starpu_data_acquire(vector_mn_handle, STARPU_W);
		vector_mn[0] = m;
		vector_mn[1] = n;
		starpu_data_release(vector_mn_handle);

		for (j = 0; j < 42; j++)
		{
			starpu_insert_task(&cl_init,
					   STARPU_R, vector_mn_handle,
					   STARPU_VALUE, &k, sizeof(int),
					   0);
			starpu_insert_task(&cl_final,
					   STARPU_R, vector_mn_handle,
					   STARPU_VALUE, &k, sizeof(int),
					   0);
		}
	}

	starpu_data_unregister(vector_mn_handle);
	free(vector_mn);
	starpu_shutdown();

	return 0;
}
$ STARPU_SCHED=dm ./a.out 
[starpu][starpu_initialize] Warning: StarPU was configured with 
--enable-verbose, which slows down a bit
[starpu][starpu_initialize] Warning: StarPU was configured with --with-fxt, 
which slows down a bit, limits scalability and makes worker initialization 
sequential
[starpu][starpu_initialize] Warning: StarPU was configured with 
--enable-model-debug, which slows down a bit
[starpu][check_bus_config_file] No performance model for the bus, 
calibrating...
[starpu][_starpu_bus_force_sampling] Force bus sampling ...
[starpu][benchmark_all_gpu_devices] Benchmarking the speed of the bus
[starpu][benchmark_all_gpu_devices] CUDA 0...
[starpu][measure_bandwidth_between_host_and_dev] (      CUDA) BANDWIDTH GPU 0 
NUMA 0 - htod 0.000078 - dtoh 0.000077 - 0.000109
[starpu][benchmark_all_gpu_devices] Benchmarking the speed of the bus is done.
[starpu][write_bus_affinity_file_content] writing affinities to 
/home/mirkom/.starpu/sampling//bus/gandalf.affinity
[starpu][write_bus_latency_file_content] writing latencies to 
/home/mirkom/.starpu/sampling//bus/gandalf.latency
[starpu][write_bus_bandwidth_file_content] writing bandwidth to 
/home/mirkom/.starpu/sampling//bus/gandalf.bandwidth
[starpu][write_bus_config_file_content] writing config to 
/home/mirkom/.starpu/sampling//bus/gandalf.config
[starpu][write_bus_platform_file_content] writing platform to 
/home/mirkom/.starpu/sampling//bus/gandalf.platform.xml
[starpu][check_bus_config_file] ... done
[starpu][check_bus_affinity_file] loading affinities from 
/home/mirkom/.starpu/sampling//bus/gandalf.affinity
[starpu][load_bus_affinity_file_content] loading affinities from 
/home/mirkom/.starpu/sampling//bus/gandalf.affinity
[starpu][load_bus_latency_file_content] loading latencies from 
/home/mirkom/.starpu/sampling//bus/gandalf.latency
[starpu][load_bus_bandwidth_file_content] loading bandwidth from 
/home/mirkom/.starpu/sampling//bus/gandalf.bandwidth
[starpu][_starpu_init_machine_config] Warning: could not find location of 
CUDA0, do you have the hwloc CUDA plugin installed?
[starpu][_starpu_memory_manager_set_global_memory_size] Global size for node 
0 is 67416813568
[starpu][_starpu_init_workers_binding_and_memory] worker 0 type 1 devid 0 
bound to cpu 3, STARPU memory node 1
[starpu][_starpu_init_workers_binding_and_memory] worker 1 type 0 devid 0 
bound to cpu 0, STARPU memory node 0
[starpu][_starpu_init_workers_binding_and_memory] worker 2 type 0 devid 1 
bound to cpu 1, STARPU memory node 0
[starpu][_starpu_init_workers_binding_and_memory] worker 3 type 0 devid 2 
bound to cpu 2, STARPU memory node 0
[starpu][load_sched_policy] Use dm scheduler (performance model)
[starpu][_starpu_launch_drivers] initialising worker 0/4
[starpu][_starpu_launch_drivers] initialising worker 1/4
[starpu][_starpu_driver_start] worker 0x7f5771b27650 0 for dev 0 is ready on 
logical cpu 3
[starpu][_starpu_driver_start] worker 0x7f5771b27650 0 cpuset start at 3
[starpu][_starpu_launch_drivers] initialising worker 2/4
[starpu][_starpu_driver_start] worker 0x7f5771b27af8 1 for dev 0 is ready on 
logical cpu 0
[starpu][_starpu_driver_start] worker 0x7f5771b27af8 1 cpuset start at 0
[starpu][_starpu_launch_drivers] initialising worker 3/4
[starpu][_starpu_driver_start] worker 0x7f5771b27fa0 2 for dev 1 is ready on 
logical cpu 1
[starpu][_starpu_driver_start] worker 0x7f5771b27fa0 2 cpuset start at 1
[starpu][_starpu_driver_start] worker 0x7f5771b28448 3 for dev 2 is ready on 
logical cpu 2
[starpu][_starpu_driver_start] worker 0x7f5771b28448 3 cpuset start at 2
[starpu][_starpu_launch_drivers] waiting for worker 0 initialization
[starpu][_starpu_cuda_limit_gpu_mem_if_needed] CUDA device 0: Wasting 403 MB 
/ Limit 3627 MB / Total 4030 MB / Remains 3627 MB
[starpu][_starpu_memory_manager_set_global_memory_size] Global size for node 
1 is 3803185152
[starpu][_starpu_cuda_driver_init] cuda (GeForce GTX 1050 Ti) dev id 0 worker 
0 thread is ready to run on CPU 3 !
[starpu][_starpu_launch_drivers] waiting for worker 1 initialization
[starpu][_starpu_launch_drivers] waiting for worker 2 initialization
[starpu][_starpu_launch_drivers] waiting for worker 3 initialization
[starpu][_starpu_launch_drivers] finished launching drivers
[starpu][starpu_initialize] Initialisation finished
[starpu][_starpu_load_history_based_model] Opening performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_init.gandalf for model 
mlr_init ...
[starpu][_starpu_load_history_based_model] Performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_init.gandalf does not exist 
or is not readable
[starpu][_starpu_multiple_regression_based_job_expected_perf] Warning: model 
mlr_init is not calibrated enough for cuda0_impl0 (Comb0), forcing 
calibration for this run. Use the STARPU_CALIBRATE environment variable to 
control this.
[starpu][_dm_push_task] Warning: performance model for mlr_init not finished 
calibrating on worker 0, using a dumb scheduling heuristic for now
[starpu][_starpu_load_history_based_model] Opening performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_final.gandalf for model 
mlr_final ...
[starpu][_starpu_load_history_based_model] Performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_final.gandalf does not exist 
or is not readable
[starpu][_starpu_dump_registered_models] DUMP MODELS !
[starpu][save_history_based_model] Opening performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_final.gandalf for model 
mlr_final
[starpu][starpu_validate_mlr] Warning: Constant computed by least square 
method is negative (-0.459867). The model mlr_final is likely to be 
inaccurate.
[starpu][save_history_based_model] Opening performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_init.gandalf for model 
mlr_init
[starpu][_starpu_deinitialize_registered_performance_models] FREE MODELS !
[starpu][_starpu_terminate_workers] wait for worker 0
[starpu][_starpu_terminate_workers] wait for worker 1
[starpu][_starpu_terminate_workers] wait for worker 2
[starpu][_starpu_terminate_workers] wait for worker 3
[starpu][_starpu_barrier_destroy] reached_exit 0
[starpu][_starpu_barrier_destroy] reached_exit 0
[starpu][deinitialize_dmda_policy] dm sched policy (sched_ctx 0): 
total_task_cnt 3528 ready_task_cnt 3528 (100.0%), modelled_task_cnt = 0 
(0.0%) *** Check if performance models are enabled and converging on a 
per-codelet basis, or use an non-modeling scheduling policy. ***
[starpu][_starpu_stop_fxt_profiling] Writing FxT traces into file 
gandalf:/tmp/prof_file_mirkom_0
[starpu][starpu_shutdown] Shutdown finished

$ STARPU_SCHED=dm ./a.out 
[starpu][starpu_initialize] Warning: StarPU was configured with 
--enable-verbose, which slows down a bit
[starpu][starpu_initialize] Warning: StarPU was configured with --with-fxt, 
which slows down a bit, limits scalability and makes worker initialization 
sequential
[starpu][starpu_initialize] Warning: StarPU was configured with 
--enable-model-debug, which slows down a bit
[starpu][check_bus_affinity_file] loading affinities from 
/home/mirkom/.starpu/sampling//bus/gandalf.affinity
[starpu][load_bus_affinity_file_content] loading affinities from 
/home/mirkom/.starpu/sampling//bus/gandalf.affinity
[starpu][load_bus_latency_file_content] loading latencies from 
/home/mirkom/.starpu/sampling//bus/gandalf.latency
[starpu][load_bus_bandwidth_file_content] loading bandwidth from 
/home/mirkom/.starpu/sampling//bus/gandalf.bandwidth
[starpu][_starpu_init_machine_config] Warning: could not find location of 
CUDA0, do you have the hwloc CUDA plugin installed?
[starpu][_starpu_memory_manager_set_global_memory_size] Global size for node 
0 is 67416813568
[starpu][_starpu_init_workers_binding_and_memory] worker 0 type 1 devid 0 
bound to cpu 3, STARPU memory node 1
[starpu][_starpu_init_workers_binding_and_memory] worker 1 type 0 devid 0 
bound to cpu 0, STARPU memory node 0
[starpu][_starpu_init_workers_binding_and_memory] worker 2 type 0 devid 1 
bound to cpu 1, STARPU memory node 0
[starpu][_starpu_init_workers_binding_and_memory] worker 3 type 0 devid 2 
bound to cpu 2, STARPU memory node 0
[starpu][load_sched_policy] Use dm scheduler (performance model)
[starpu][_starpu_launch_drivers] initialising worker 0/4
[starpu][_starpu_launch_drivers] initialising worker 1/4
[starpu][_starpu_driver_start] worker 0x7f57be9d6650 0 for dev 0 is ready on 
logical cpu 3
[starpu][_starpu_driver_start] worker 0x7f57be9d6650 0 cpuset start at 3
[starpu][_starpu_launch_drivers] initialising worker 2/4
[starpu][_starpu_launch_drivers] initialising worker 3/4
[starpu][_starpu_driver_start] worker 0x7f57be9d6af8 1 for dev 0 is ready on 
logical cpu 0
[starpu][_starpu_launch_drivers] waiting for worker 0 initialization
[starpu][_starpu_driver_start] worker 0x7f57be9d6fa0 2 for dev 1 is ready on 
logical cpu 1
[starpu][_starpu_driver_start] worker 0x7f57be9d6af8 1 cpuset start at 0
[starpu][_starpu_driver_start] worker 0x7f57be9d7448 3 for dev 2 is ready on 
logical cpu 2
[starpu][_starpu_driver_start] worker 0x7f57be9d7448 3 cpuset start at 2
[starpu][_starpu_driver_start] worker 0x7f57be9d6fa0 2 cpuset start at 1
[starpu][_starpu_cuda_limit_gpu_mem_if_needed] CUDA device 0: Wasting 403 MB 
/ Limit 3627 MB / Total 4030 MB / Remains 3627 MB
[starpu][_starpu_memory_manager_set_global_memory_size] Global size for node 
1 is 3803185152
[starpu][_starpu_cuda_driver_init] cuda (GeForce GTX 1050 Ti) dev id 0 worker 
0 thread is ready to run on CPU 3 !
[starpu][_starpu_launch_drivers] waiting for worker 1 initialization
[starpu][_starpu_launch_drivers] waiting for worker 2 initialization
[starpu][_starpu_launch_drivers] waiting for worker 3 initialization
[starpu][_starpu_launch_drivers] finished launching drivers
[starpu][starpu_initialize] Initialisation finished
[starpu][_starpu_load_history_based_model] Opening performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_init.gandalf for model 
mlr_init ...
[starpu][_starpu_load_history_based_model] Performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_init.gandalf for model 
mlr_init is loaded
[starpu][_starpu_multiple_regression_based_job_expected_perf] Warning: model 
mlr_init is not calibrated enough for cuda0_impl0 (Comb0), forcing 
calibration for this run. Use the STARPU_CALIBRATE environment variable to 
control this.
[starpu][_dm_push_task] Warning: performance model for mlr_init not finished 
calibrating on worker 0, using a dumb scheduling heuristic for now
[starpu][_starpu_load_history_based_model] Opening performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_final.gandalf for model 
mlr_final ...
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(+0x5eafc)[0x7f57be6c0afc]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(_starpu_load_history_based_model+0x117)[0x7f57be6c2337]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(_starpu_init_and_load_perfmodel+0x8a)[0x7f57be6cce6a]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(+0x39913)[0x7f57be69b913]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(starpu_task_submit+0x8c)[0x7f57be69dc1c]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(+0xcd351)[0x7f57be72f351]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(starpu_insert_task+0x99)[0x7f57be72f969]
./a.out(+0xfd8)[0x55617f0e5fd8]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xe7)[0x7f57be292b97]
./a.out(+0xbba)[0x55617f0e5bba]

[starpu][parse_comb][assert failure] Incorrect performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_final.gandalf

a.out: core/perfmodel/perfmodel_history.c:665: parse_comb: Assertion `ret == 
1' failed.
Aborted (core dumped)
$ STARPU_SCHED=dm gdb ./a.out 
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
Copyright (C) 2018 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from ./a.out...done.
(gdb) run
Starting program: 
/home/mirkom/Downloads/starpu-nightly-latest/starpu-1.3.00/examples/mlr/a.out 
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
[New Thread 0x7fffebbf2700 (LWP 3217)]
[New Thread 0x7fffe93f1700 (LWP 3218)]
[New Thread 0x7fffe6bf0700 (LWP 3219)]
[starpu][starpu_initialize] Warning: StarPU was configured with 
--enable-verbose, which slows down a bit
[starpu][starpu_initialize] Warning: StarPU was configured with --with-fxt, 
which slows down a bit, limits scalability and makes worker initialization 
sequential
[starpu][starpu_initialize] Warning: StarPU was configured with 
--enable-model-debug, which slows down a bit
[New Thread 0x7fffd5601700 (LWP 3221)]
[starpu][check_bus_affinity_file] loading affinities from 
/home/mirkom/.starpu/sampling//bus/gandalf.affinity
[starpu][load_bus_affinity_file_content] loading affinities from 
/home/mirkom/.starpu/sampling//bus/gandalf.affinity
[starpu][load_bus_latency_file_content] loading latencies from 
/home/mirkom/.starpu/sampling//bus/gandalf.latency
[starpu][load_bus_bandwidth_file_content] loading bandwidth from 
/home/mirkom/.starpu/sampling//bus/gandalf.bandwidth
[starpu][_starpu_init_machine_config] Warning: could not find location of 
CUDA0, do you have the hwloc CUDA plugin installed?
[starpu][_starpu_memory_manager_set_global_memory_size] Global size for node 
0 is 67416813568
[starpu][_starpu_init_workers_binding_and_memory] worker 0 type 1 devid 0 
bound to cpu 3, STARPU memory node 1
[starpu][_starpu_init_workers_binding_and_memory] worker 1 type 0 devid 0 
bound to cpu 0, STARPU memory node 0
[starpu][_starpu_init_workers_binding_and_memory] worker 2 type 0 devid 1 
bound to cpu 1, STARPU memory node 0
[starpu][_starpu_init_workers_binding_and_memory] worker 3 type 0 devid 2 
bound to cpu 2, STARPU memory node 0
[starpu][load_sched_policy] Use dm scheduler (performance model)
[starpu][_starpu_launch_drivers] initialising worker 0/4
[New Thread 0x7fffc19ea700 (LWP 3225)]
[starpu][_starpu_launch_drivers] initialising worker 1/4
[starpu][_starpu_driver_start] worker 0x7ffff7daa650 0 for dev 0 is ready on 
logical cpu 3
[starpu][_starpu_driver_start] worker 0x7ffff7daa650 0 cpuset start at 3
[New Thread 0x7fffb91e9700 (LWP 3226)]
[starpu][_starpu_launch_drivers] initialising worker 2/4
[New Thread 0x7fffc11e9700 (LWP 3227)]
[New Thread 0x7fffc09e8700 (LWP 3228)]
[starpu][_starpu_driver_start] worker 0x7ffff7daaaf8 1 for dev 0 is ready on 
logical cpu 0
[starpu][_starpu_driver_start] worker 0x7ffff7daaaf8 1 cpuset start at 0
[starpu][_starpu_launch_drivers] initialising worker 3/4
[starpu][_starpu_driver_start] worker 0x7ffff7daafa0 2 for dev 1 is ready on 
logical cpu 1
[starpu][_starpu_driver_start] worker 0x7ffff7daafa0 2 cpuset start at 1
[New Thread 0x7fffbbfff700 (LWP 3229)]
[starpu][_starpu_launch_drivers] waiting for worker 0 initialization
[starpu][_starpu_driver_start] worker 0x7ffff7dab448 3 for dev 2 is ready on 
logical cpu 2
[starpu][_starpu_driver_start] worker 0x7ffff7dab448 3 cpuset start at 2
[New Thread 0x7fffbb3ff700 (LWP 3230)]
[starpu][_starpu_cuda_limit_gpu_mem_if_needed] CUDA device 0: Wasting 403 MB 
/ Limit 3627 MB / Total 4030 MB / Remains 3627 MB
[starpu][_starpu_memory_manager_set_global_memory_size] Global size for node 
1 is 3803185152
[starpu][_starpu_cuda_driver_init] cuda (GeForce GTX 1050 Ti) dev id 0 worker 
0 thread is ready to run on CPU 3 !
[starpu][_starpu_launch_drivers] waiting for worker 1 initialization
[starpu][_starpu_launch_drivers] waiting for worker 2 initialization
[starpu][_starpu_launch_drivers] waiting for worker 3 initialization
[starpu][_starpu_launch_drivers] finished launching drivers
[starpu][starpu_initialize] Initialisation finished
[starpu][_starpu_load_history_based_model] Opening performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_init.gandalf for model 
mlr_init ...
[starpu][_starpu_load_history_based_model] Performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_init.gandalf for model 
mlr_init is loaded
[starpu][_starpu_multiple_regression_based_job_expected_perf] Warning: model 
mlr_init is not calibrated enough for cuda0_impl0 (Comb0), forcing 
calibration for this run. Use the STARPU_CALIBRATE environment variable to 
control this.
[starpu][_dm_push_task] Warning: performance model for mlr_init not finished 
calibrating on worker 0, using a dumb scheduling heuristic for now
[starpu][_starpu_load_history_based_model] Opening performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_final.gandalf for model 
mlr_final ...
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(+0x5eafc)[0x7ffff7a94afc]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(_starpu_load_history_based_model+0x117)[0x7ffff7a96337]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(_starpu_init_and_load_perfmodel+0x8a)[0x7ffff7aa0e6a]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(+0x39913)[0x7ffff7a6f913]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(starpu_task_submit+0x8c)[0x7ffff7a71c1c]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(+0xcd351)[0x7ffff7b03351]
/home/mirkom/.starpu_install/starpu_latest_debug/lib/libstarpu-1.3.so.0(starpu_insert_task+0x99)[0x7ffff7b03969]
/home/mirkom/Downloads/starpu-nightly-latest/starpu-1.3.00/examples/mlr/a.out(+0xfd8)[0x555555554fd8]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xe7)[0x7ffff7666b97]
/home/mirkom/Downloads/starpu-nightly-latest/starpu-1.3.00/examples/mlr/a.out(+0xbba)[0x555555554bba]

[starpu][parse_comb][assert failure] Incorrect performance model file 
/home/mirkom/.starpu/sampling//codelets/44//mlr_final.gandalf

a.out: core/perfmodel/perfmodel_history.c:665: parse_comb: Assertion `ret == 
1' failed.

Thread 1 "a.out" received signal SIGABRT, Aborted.
__GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
51      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1  0x00007ffff7685801 in __GI_abort () at abort.c:79
#2  0x00007ffff767539a in __assert_fail_base (fmt=0x7ffff77fc7d8 "%s%s%s:%u: 
%s%sAssertion `%s' failed.\n%n", assertion=assertion@entry=0x7ffff7b498a9 
"ret == 1", 
    file=file@entry=0x7ffff7b485a0 "core/perfmodel/perfmodel_history.c", 
line=line@entry=665, function=function@entry=0x7ffff7b4a1c0 
<__PRETTY_FUNCTION__.31408> "parse_comb")
    at assert.c:92
#3  0x00007ffff7675412 in __GI___assert_fail 
(assertion=assertion@entry=0x7ffff7b498a9 "ret == 1", 
file=file@entry=0x7ffff7b485a0 "core/perfmodel/perfmodel_history.c", 
    line=line@entry=665, function=function@entry=0x7ffff7b4a1c0 
<__PRETTY_FUNCTION__.31408> "parse_comb") at assert.c:101
#4  0x00007ffff7a94b50 in parse_comb (comb=1, scan_history=0, 
model=0x5555557560e0 <cl_model_final>, path=0x7fffffffd350 
"/home/mirkom/.starpu/sampling//codelets/44//mlr_final.gandalf", 
    f=0x5555559dd0a0) at core/perfmodel/perfmodel_history.c:665
#5  parse_model_file (f=f@entry=0x5555559dd0a0, 
path=path@entry=0x7fffffffd350 
"/home/mirkom/.starpu/sampling//codelets/44//mlr_final.gandalf", 
    model=model@entry=0x5555557560e0 <cl_model_final>, 
scan_history=scan_history@entry=0) at core/perfmodel/perfmodel_history.c:737
#6  0x00007ffff7a96337 in _starpu_load_history_based_model 
(model=model@entry=0x5555557560e0 <cl_model_final>, 
scan_history=scan_history@entry=0)
    at core/perfmodel/perfmodel_history.c:1235
#7  0x00007ffff7aa0e6a in _starpu_init_and_load_perfmodel 
(model=0x5555557560e0 <cl_model_final>) at core/perfmodel/perfmodel.c:172
#8  0x00007ffff7a6f913 in _starpu_task_submit_head 
(task=task@entry=0x5555559fd190) at core/task.c:626
#9  0x00007ffff7a71c1c in starpu_task_submit (task=task@entry=0x5555559fd190) 
at core/task.c:672
#10 0x00007ffff7b03351 in _starpu_task_insert_v (cl=0x555555756580 
<cl_final>, varg_list=varg_list@entry=0x7fffffffd8f0) at 
util/starpu_task_insert.c:142
#11 0x00007ffff7b03969 in starpu_insert_task (cl=<optimized out>) at 
util/starpu_task_insert.c:175
#12 0x0000555555554fd8 in main () at mlr.c:199