[starpu-devel] Corrupted large ooc files with unistd backend

[barbier@imacs.polytechnique.fr]

Hello,

write.2 manpage constains this note on Linux:

  On Linux, write() (and similar system calls) will transfer at most
0x7ffff000 (2,147,479,552) bytes, returning the number of bytes actually
transferred. (This is true on both 32-bit and 64-bit systems.)

As a result, files contain only zeros after offset 0x7ffff000.
This is pretty easy to fix in
starpu_unistd_global_read/write with a loop, but I do not see how to deal
with asynchronous IO; I discarded AIO when size is too large in attached
patch.

Regards,
Denis BARBIER
diff --git a/src/core/disk_ops/unistd/disk_unistd_global.c b/src/core/disk_ops/unistd/disk_unistd_global.c
index b43d1870ba..cd66f7dd4e 100644
--- a/src/core/disk_ops/unistd/disk_unistd_global.c
+++ b/src/core/disk_ops/unistd/disk_unistd_global.c
@@ -284,10 +284,20 @@ int starpu_unistd_global_read(void *base STARPU_ATTRIBUTE_UNUSED, void *obj, voi
 	struct starpu_unistd_global_obj *tmp = (struct starpu_unistd_global_obj *) obj;
 	starpu_ssize_t nb;
 	int fd = tmp->descriptor;
+	starpu_ssize_t bytes_to_write = size;
 
 #ifdef HAVE_PREAD
 	if (fd >= 0)
-		nb = pread(fd, buf, size, offset);
+	{
+		while (bytes_to_write > 0)
+		{
+			nb = pread(fd, buf, bytes_to_write, offset);
+			STARPU_ASSERT_MSG(nb >= 0, "Starpu Disk unistd pread failed: size %lu got errno %d", (unsigned long) size, errno);
+			bytes_to_write -= nb;
+			buf += nb;
+			offset += nb;
+		}
+	}
 	else
 #endif
 	{
@@ -299,7 +309,14 @@ int starpu_unistd_global_read(void *base STARPU_ATTRIBUTE_UNUSED, void *obj, voi
 		int res = lseek(fd, offset, SEEK_SET);
 		STARPU_ASSERT_MSG(res >= 0, "Starpu Disk unistd lseek for read failed: offset %lu got errno %d", (unsigned long) offset, errno);
 
-		nb = read(fd, buf, size);
+		while (bytes_to_write > 0)
+		{
+			nb = read(fd, buf, bytes_to_write);
+			STARPU_ASSERT_MSG(nb >= 0, "Starpu Disk unistd read failed: offset %lu got errno %d", (unsigned long) offset, errno);
+			bytes_to_write -= nb;
+			buf += nb;
+			offset += nb;
+		}
 
 		if (tmp->descriptor >= 0)
 			STARPU_PTHREAD_MUTEX_UNLOCK(&tmp->mutex);
@@ -308,9 +325,7 @@ int starpu_unistd_global_read(void *base STARPU_ATTRIBUTE_UNUSED, void *obj, voi
 
 	}
 
-	STARPU_ASSERT_MSG(nb >= 0, "Starpu Disk unistd read failed: size %lu got errno %d", (unsigned long) size, errno);
-
-	return nb;
+	return 0;
 }
 
 #if defined(HAVE_LIBAIO_H)
@@ -413,12 +428,22 @@ int starpu_unistd_global_full_read(void *base STARPU_ATTRIBUTE_UNUSED, void *obj
 int starpu_unistd_global_write(void *base STARPU_ATTRIBUTE_UNUSED, void *obj, const void *buf, off_t offset, size_t size)
 {
 	struct starpu_unistd_global_obj *tmp = (struct starpu_unistd_global_obj *) obj;
-	int res;
+	starpu_ssize_t res;
 	int fd = tmp->descriptor;
+	starpu_ssize_t bytes_to_write = size;
 
 #ifdef HAVE_PWRITE
 	if (fd >= 0)
-		res = pwrite(fd, buf, size, offset);
+	{
+		while (bytes_to_write > 0)
+		{
+			res = pwrite(fd, buf, bytes_to_write, offset);
+			STARPU_ASSERT_MSG(res >= 0, "Starpu Disk unistd pwrite failed: offset %lu got errno %d", (unsigned long) offset, errno);
+			bytes_to_write -= res;
+			buf += res;
+			offset += res;
+		}
+	}
 	else
 #endif
 	{
@@ -430,7 +455,14 @@ int starpu_unistd_global_write(void *base STARPU_ATTRIBUTE_UNUSED, void *obj, co
 		res = lseek(fd, offset, SEEK_SET);
 		STARPU_ASSERT_MSG(res >= 0, "Starpu Disk unistd lseek for write failed: offset %lu got errno %d", (unsigned long) offset, errno);
 
-		res = write(fd, buf, size);
+		while (bytes_to_write > 0)
+		{
+			res = write(fd, buf, bytes_to_write);
+			STARPU_ASSERT_MSG(res >= 0, "Starpu Disk unistd write failed: offset %lu got errno %d", (unsigned long) offset, errno);
+			bytes_to_write -= res;
+			buf += res;
+			offset += res;
+		}
 
 		if (tmp->descriptor >= 0)
 			STARPU_PTHREAD_MUTEX_UNLOCK(&tmp->mutex);
@@ -438,7 +470,6 @@ int starpu_unistd_global_write(void *base STARPU_ATTRIBUTE_UNUSED, void *obj, co
 			_starpu_unistd_reclose(fd);
 	}
 
-	STARPU_ASSERT_MSG(res >= 0, "Starpu Disk unistd write failed: size %lu got errno %d", (unsigned long) size, errno);
 	return 0;
 }
 
@@ -553,6 +584,12 @@ void * starpu_unistd_global_async_full_read (void * base, void * obj, void ** pt
 
 	*size = st.st_size;
 #endif
+#ifdef STARPU_LINUX_SYS
+	/* on Linux, read() (and similar system calls) will transfer at most 0x7ffff000 bytes, see read(2) */
+	if (*size > 0x7ffff000)
+		return NULL;
+#endif
+
 	if (tmp->descriptor < 0)
 		_starpu_unistd_reclose(fd);
 
@@ -565,6 +602,12 @@ void * starpu_unistd_global_async_full_write (void * base, void * obj, void * pt
 {
 	struct starpu_unistd_global_obj *tmp = (struct starpu_unistd_global_obj *) obj;
 
+#ifdef STARPU_LINUX_SYS
+	/* on Linux, write() (and similar system calls) will transfer at most 0x7ffff000 bytes, see write(2) */
+	if (size > 0x7ffff000)
+		return NULL;
+#endif
+
 	/* update file size to realise the next good full_read */
 	if(size != tmp->size)
 	{