[starpu-devel] [BUG REPORT] Incorrect result when partitioning temporary data asynchronously

[Atte Torri <atte.torri@universite-paris-saclay.fr>]

Hello,

I get incorrect results when doing an asynchronous partitioning on temprary data. I have attached an example file for the problem.

It seems that the partitioned handle and the original handle do not share data correctly.

In the example file the expected result is that the output is the same for the original and partitioned handles (or that an error be raised by starpu).

If you comment line 120 and uncomment line 119 (starpu allocated data to user allocated data), the output of the executable is correct.

I have had similar problems with other partitioners / interfaces as well as with MPI enabled.

Best,

TORRI Atte

/* StarPU --- Runtime system for heterogeneous multicore architectures.
 *
 * Copyright (C) 2015-2023  Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria
 *
 * StarPU is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or (at
 * your option) any later version.
 *
 * StarPU is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *
 * See the GNU Lesser General Public License in COPYING.LGPL for more details.
 */

/*
 * This examplifies how to access the same matrix with different partitioned
 * views, doing the coherency through partition planning.
 *
 * We first run a kernel on the whole matrix to fill it, then check the value
 * in parallel from the whole handle, from the horizontal slices, and from the
 * vertical slices. Then we switch back to the whole matrix to check and scale
 * it. Then we check the result again from the whole handle, the horizontal
 * slices, and the vertical slices. Then we switch to read-write on the
 * horizontal slices to check and scale them. Then we check again from the
 * whole handle, the horizontal slices, and the vertical slices. Eventually we
 * switch back to the whole matrix to check and scale it.
 *
 * Please keep this in sync with fmultiple_submit_readonly.c and fmultiple_submit_readonly_downgrade.c
 */

#include <starpu.h>

#define NX    6
#define NY    6
#define PARTS 2

#define FPRINTF(ofile, fmt, ...) do { if (!getenv("STARPU_SSILENT")) {fprintf(ofile, fmt, ## __VA_ARGS__); }} while(0)

void matrix_fill(void *buffers[], void *cl_arg)
{
	unsigned i, j;
	(void)cl_arg;

	/* length of the matrix */
	unsigned nx = STARPU_MATRIX_GET_NX(buffers[0]);
	unsigned ny = STARPU_MATRIX_GET_NY(buffers[0]);
	unsigned ld = STARPU_MATRIX_GET_LD(buffers[0]);
	int *val = (int *)STARPU_MATRIX_GET_PTR(buffers[0]);

	int factor;
	starpu_codelet_unpack_args(cl_arg, &factor);

	for(j=0; j<ny ; j++)
	{
		for(i=0; i<nx ; i++)
			val[(j*ld)+i] = factor;
	}
}

struct starpu_codelet cl_fill =
{
	.cpu_funcs = {matrix_fill},
	.cpu_funcs_name = {"matrix_fill"},
	.nbuffers = 1,
	.modes = {STARPU_W},
	.name = "matrix_fill"
};

void matrix_print(void *buffers[], void *cl_arg)
{
	unsigned i, j;
	(void)cl_arg;

	/* length of the matrix */
	unsigned nx = STARPU_MATRIX_GET_NX(buffers[0]);
	unsigned ny = STARPU_MATRIX_GET_NY(buffers[0]);
	unsigned ld = STARPU_MATRIX_GET_LD(buffers[0]);
	int *val = (int *)STARPU_MATRIX_GET_PTR(buffers[0]);

	for(j=0; j<ny ; j++)
	{
		for(i=0; i<nx ; i++)
			printf("value [%u,%u] = %d\n", i, j, val[(j*ld)+i]);
	}
}

struct starpu_codelet cl_print =
{
	.cpu_funcs = {matrix_print},
	.cpu_funcs_name = {"matrix_print"},
	.nbuffers = 1,
	.modes = {STARPU_R},
	.name = "matrix_print"
};

int main(void)
{
	int start, factor;
	unsigned n=1;
	int matrix[NX][NY];
	int ret, i;

	/* We haven't taken care otherwise */
	STARPU_ASSERT((NX%PARTS) == 0);
	STARPU_ASSERT((NY%PARTS) == 0);

	starpu_data_handle_t handle;
	starpu_data_handle_t vert_handle[PARTS];
	starpu_data_handle_t horiz_handle[PARTS];

	ret = starpu_init(NULL);
	if (ret == -ENODEV)
		return 77;
	STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");

	/* Declare the whole matrix to StarPU */
	//starpu_matrix_data_register(&handle, STARPU_MAIN_RAM, (uintptr_t)matrix, NX, NX, NY, sizeof(matrix[0][0]));
	starpu_matrix_data_register(&handle, -1, (uintptr_t)NULL, NX, NX, NY, sizeof(matrix[0][0]));

	/* Fill the matrix */
	int value = 10;
	ret = starpu_task_insert(&cl_fill,
				 STARPU_W, handle,
				 STARPU_VALUE, &value, sizeof(value),
				 0);
	if (ret == -ENODEV) goto enodev;
	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");

	printf("Values in original before partitioning\n");
	ret = starpu_task_insert(&cl_print,
				 STARPU_R, handle,
				 0);
	if (ret == -ENODEV) goto enodev;
	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");

	starpu_task_wait_for_all();

	/* Partition the matrix in PARTS vertical slices */
	struct starpu_data_filter f_vert =
	{
		.filter_func = starpu_matrix_filter_block,
		.nchildren = PARTS
	};
	starpu_data_partition_plan(handle, &f_vert, vert_handle);

	/* Check the values of the vertical slices */
	for (i = 0; i < PARTS; i++)
	{
		printf("Values in partition %d before filling\n", i);
		ret = starpu_task_insert(&cl_print,
					 STARPU_R, vert_handle[i],
					 0);
		if (ret == -ENODEV) goto enodev;
		STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");
		starpu_task_wait_for_all();
	}
	for (i = 0; i < PARTS; i++)
	{
		start = i*(NX/PARTS);
		value = 5;
		ret = starpu_task_insert(&cl_fill,
				STARPU_W, vert_handle[i],
				STARPU_VALUE, &value, sizeof(value),
				0);
		if (ret == -ENODEV) goto enodev;
		STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");
	}
	for (i = 0; i < PARTS; i++)
	{
		printf("Values in partition %d after filling\n", i);
		ret = starpu_task_insert(&cl_print,
					 STARPU_R, vert_handle[i],
					 0);
		if (ret == -ENODEV) goto enodev;
		STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");
		starpu_task_wait_for_all();
	}

	/*
	 * Unregister data from StarPU and shutdown.
	 */
	starpu_data_partition_clean(handle, PARTS, vert_handle);

	printf("Values in original after partitioning\n");
	ret = starpu_task_insert(&cl_print,
				 STARPU_R, handle,
				 0);
	if (ret == -ENODEV) goto enodev;
	STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");

	starpu_data_unregister(handle);
	starpu_shutdown();

	return ret;

enodev:
	starpu_shutdown();
	return 77;
}