Io uring/multiple write modes

HOWTO.rst

@@ -3049,6 +3049,10 @@ with the caveat that when used on the command line, they must come after the
                 **verify**
                         Use Verify commands for write operations
 
+        Multiple modes with mix ratios can be specified using the format
+        ``pct:mode/pct:mode/...``.  Percentages must sum to 100
+        Example: ``60:write/30:zeroes/10:uncor``
+
 .. option:: verify_mode=str : [io_uring_cmd]
 
         Specifies the type of command to be used in the verification phase.  Defaults to 'read'.

engines/io_uring.c

@@ -104,6 +104,14 @@ enum uring_cmd_write_mode {
 	FIO_URING_CMD_WMODE_VERIFY,
 };
 
+#define WMODE_SPLIT_MAX	4
+
+struct wmode_split_entry {
+	uint8_t		opcode;
+	uint32_t	cdw12_flag;
+	unsigned int	perc;
+};
+
 enum uring_cmd_verify_mode {
 	FIO_URING_CMD_VMODE_READ = 1,
 	FIO_URING_CMD_VMODE_COMPARE,
@@ -161,6 +169,7 @@ struct ioring_data {
 	struct nvme_dsm *dsm;
 	uint32_t cdw12_flags[DDIR_RWDIR_CNT];
 	uint8_t write_opcode;
+	struct frand_state wmode_state;
 
 	bool is_uring_cmd_eng;
 
@@ -174,6 +183,8 @@ struct ioring_options {
 	unsigned int writefua;
 	unsigned int deac;
 	unsigned int write_mode;
+	struct wmode_split_entry wmode_split[WMODE_SPLIT_MAX];
+	unsigned int wmode_split_nr;
 	unsigned int verify_mode;
 	struct cmdprio_options cmdprio_options;
 	unsigned int fixedbufs;
@@ -206,6 +217,107 @@ static const int fixed_ddir_to_op[2] = {
 	IORING_OP_WRITE_FIXED
 };
 
+static uint8_t wmode_str_to_opcode(const char *mode)
+{
+	if (!strcmp(mode, "write"))
+		return nvme_cmd_write;
+	if (!strcmp(mode, "uncor"))
+		return nvme_cmd_write_uncor;
+	if (!strcmp(mode, "zeroes"))
+		return nvme_cmd_write_zeroes;
+	if (!strcmp(mode, "verify"))
+		return nvme_cmd_verify;
+	return 0xff;
+}
+
+static int str_write_mode_cb(void *data, const char *str)
+{
+	struct ioring_options *o = data;
+	char *s, *p, *tok;
+	unsigned int total_perc = 0;
+	int i = 0;
+
+	/* Single-value: no '/' means single mode name */
+	if (!strchr(str, '/')) {
+		uint8_t op = wmode_str_to_opcode(str);
+
+		if (op == 0xff) {
+			log_err("fio: invalid write_mode value: %s\n", str);
+			return 1;
+		}
+
+		if (op == nvme_cmd_write_uncor)
+			o->write_mode = FIO_URING_CMD_WMODE_UNCOR;
+		else if (op == nvme_cmd_write_zeroes)
+			o->write_mode = FIO_URING_CMD_WMODE_ZEROES;
+		else if (op == nvme_cmd_verify)
+			o->write_mode = FIO_URING_CMD_WMODE_VERIFY;
+		else
+			o->write_mode = FIO_URING_CMD_WMODE_WRITE;
+		o->wmode_split_nr = 0;
+		return 0;
+	}
+
+	/* Multi-value: e.g., --write_mode=60:write/30:zeroes/10:uncor */
+	s = strdup(str);
+	p = s;
+	while ((tok = strsep(&p, "/")) != NULL) {
+		char *mode_str = strchr(tok, ':');
+		unsigned int perc = 0;
+		uint8_t op;
+
+		if (i >= WMODE_SPLIT_MAX) {
+			log_err("fio: write_mode: too many entries (max %d)\n",
+				WMODE_SPLIT_MAX);
+			free(s);
+			return 1;
+		}
+
+		if (mode_str) {
+			*mode_str++ = '\0';
+			if (*tok)
+				perc = atoi(tok);
+			else {
+				log_err("fio: invalid percentage value\n");
+				free(s);
+				return 1;
+			}
+
+			op = wmode_str_to_opcode(mode_str);
+			if (op == 0xff) {
+				log_err("fio: invalid write_mode value: %s\n", mode_str);
+				free(s);
+				return 1;
+			}
+		} else {
+			log_err("fio: invalid percentage value\n");
+			free(s);
+			return 1;
+		}
+
+		o->wmode_split[i].opcode = op;
+		o->wmode_split[i].cdw12_flag = 0;
+		o->wmode_split[i].perc = perc;
+		total_perc += perc;
+		i++;
+	}
+	free(s);
+
+	if (i < 2) {
+		log_err("fio: write_mode needs at least 2 entries\n");
+		return 1;
+	}
+
+	if (total_perc != 100) {
+		log_err("fio: write_mode percentages should be 100%%\n");
+		return 1;
+	}
+
+	o->wmode_split_nr = i;
+	o->write_mode = FIO_URING_CMD_WMODE_WRITE;
+	return 0;
+}
+
 static int fio_ioring_sqpoll_cb(void *data, unsigned long long *val)
 {
 	struct ioring_options *o = data;
@@ -247,29 +359,12 @@ static struct fio_option options[] = {
 	},
 	{
 		.name	= "write_mode",
-		.lname	= "Additional Write commands support (Write Uncorrectable, Write Zeores)",
+		.lname	= "Write command type(s) with optional mix ratios",
 		.type	= FIO_OPT_STR,
-		.off1	= offsetof(struct ioring_options, write_mode),
-		.help	= "Issue Write Uncorrectable or Zeroes command instead of Write command",
+		.cb	= str_write_mode_cb,
+		.help	= "Single: write|uncor|zeroes|verify. "
+			  "Mixed: pct:mode/pct:mode/... (e.g. 60:write/40:zeroes)",
 		.def	= "write",
-		.posval = {
-			  { .ival = "write",
-			    .oval = FIO_URING_CMD_WMODE_WRITE,
-			    .help = "Issue Write commands for write operations"
-			  },
-			  { .ival = "uncor",
-			    .oval = FIO_URING_CMD_WMODE_UNCOR,
-			    .help = "Issue Write Uncorrectable commands for write operations"
-			  },
-			  { .ival = "zeroes",
-			    .oval = FIO_URING_CMD_WMODE_ZEROES,
-			    .help = "Issue Write Zeroes commands for write operations"
-			  },
-			  { .ival = "verify",
-			    .oval = FIO_URING_CMD_WMODE_VERIFY,
-			    .help = "Issue Verify commands for write operations"
-			  },
-		},
 		.category = FIO_OPT_C_ENGINE,
 		.group	= FIO_OPT_G_IOURING,
 	},
@@ -647,6 +742,33 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u)
 		io_u_set(td, io_u, IO_U_F_VER_IN_DEV);
 	}
 
+	if (o->wmode_split_nr > 1 && io_u->ddir == DDIR_WRITE) {
+		unsigned int rand = rand_between(&ld->wmode_state, 0, 99);
+		unsigned int perc = 0;
+		int i;
+
+		for (i = 0; i < (int)o->wmode_split_nr; i++) {
+			perc += o->wmode_split[i].perc;
+			if (rand < perc) {
+				uint8_t op = o->wmode_split[i].opcode;
+
+				io_u_clear(td, io_u, IO_U_F_TRIMMED | IO_U_F_ZEROED | IO_U_F_ERRORED);
+				if (op == nvme_cmd_write_zeroes) {
+					if (o->deac)
+						io_u_set(td, io_u, IO_U_F_TRIMMED);
+					else
+						io_u_set(td, io_u, IO_U_F_ZEROED);
+				} else if (op == nvme_cmd_write_uncor)
+					io_u_set(td, io_u, IO_U_F_ERRORED);
+
+				return fio_nvme_uring_cmd_prep(cmd, io_u,
+					o->nonvectored ? NULL : &ld->iovecs[io_u->index],
+					dsm, read_opcode, op,
+					o->wmode_split[i].cdw12_flag);
+			}
+		}
+	}
+
 	return fio_nvme_uring_cmd_prep(cmd, io_u,
 			o->nonvectored ? NULL : &ld->iovecs[io_u->index],
 			dsm, read_opcode, ld->write_opcode,
@@ -1437,27 +1559,44 @@ static int fio_ioring_cmd_init(struct thread_data *td, struct ioring_data *ld)
 	struct ioring_options *o = td->eo;
 
 	if (td_write(td)) {
-		switch (o->write_mode) {
-		case FIO_URING_CMD_WMODE_UNCOR:
-			ld->write_opcode = nvme_cmd_write_uncor;
-			break;
-		case FIO_URING_CMD_WMODE_ZEROES:
-			ld->write_opcode = nvme_cmd_write_zeroes;
-			if (o->deac)
-				ld->cdw12_flags[DDIR_WRITE] = 1 << 25;
-			break;
-		case FIO_URING_CMD_WMODE_VERIFY:
-			ld->write_opcode = nvme_cmd_verify;
-			break;
-		default:
-			ld->write_opcode = nvme_cmd_write;
-			break;
+		if (o->wmode_split_nr > 1) {
+			int i;
+
+			init_rand_seed(&ld->wmode_state,
+				       td->rand_seeds[FIO_RAND_WMODE_OFF],
+				       false);
+			for (i = 0; i < (int)o->wmode_split_nr; i++) {
+				struct wmode_split_entry *e = &o->wmode_split[i];
+
+				e->cdw12_flag = 0;
+				if (e->opcode == nvme_cmd_write_zeroes && o->deac)
+					e->cdw12_flag = 1 << 25;
+				else if (e->opcode == nvme_cmd_write && o->writefua)
+					e->cdw12_flag = 1 << 30;
+			}
+		} else {
+			switch (o->write_mode) {
+			case FIO_URING_CMD_WMODE_UNCOR:
+				ld->write_opcode = nvme_cmd_write_uncor;
+				break;
+			case FIO_URING_CMD_WMODE_ZEROES:
+				ld->write_opcode = nvme_cmd_write_zeroes;
+				if (o->deac)
+					ld->cdw12_flags[DDIR_WRITE] = 1 << 25;
+				break;
+			case FIO_URING_CMD_WMODE_VERIFY:
+				ld->write_opcode = nvme_cmd_verify;
+				break;
+			default:
+				ld->write_opcode = nvme_cmd_write;
+				break;
+			}
 		}
 	}
 
 	if (o->readfua)
 		ld->cdw12_flags[DDIR_READ] = 1 << 30;
-	if (o->writefua)
+	if (o->writefua && o->wmode_split_nr <= 1)
 		ld->cdw12_flags[DDIR_WRITE] = 1 << 30;
 
 	return 0;
@@ -1619,6 +1758,16 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
 		io_u->engine_data = pi_data;
 	}
 
+	if (ld->is_uring_cmd_eng && o->wmode_split_nr <= 1) {
+		if (ld->write_opcode == nvme_cmd_write_zeroes) {
+			if (o->deac)
+				io_u_set(td, io_u, IO_U_F_TRIMMED);
+			else
+				io_u_set(td, io_u, IO_U_F_ZEROED);
+		} else if (ld->write_opcode == nvme_cmd_write_uncor)
+			io_u_set(td, io_u, IO_U_F_ERRORED);
+	}
+
 	return 0;
 }
 
@@ -1810,7 +1959,8 @@ static int fio_ioring_open_nvme(struct thread_data *td, struct fio_file *f)
 		return 1;
 	}
 
-	if (o->write_mode != FIO_URING_CMD_WMODE_WRITE && !td_write(td)) {
+	if ((o->write_mode != FIO_URING_CMD_WMODE_WRITE || o->wmode_split_nr > 1) &&
+	    !td_write(td)) {
 		log_err("%s: 'readwrite=|rw=' has no write\n", f->file_name);
 		td_verror(td, EINVAL, "fio_ioring_cmd_open_file");
 		return 1;

fio.h

@@ -156,6 +156,7 @@ enum {
 	FIO_RAND_DEDUPE_WORKING_SET_IX,
 	FIO_RAND_FDP_OFF,
 	FIO_RAND_SPRANDOM_OFF,
+	FIO_RAND_WMODE_OFF,
 	FIO_RAND_NR_OFFS,
 };
 

io_u.c

@@ -2208,9 +2208,16 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,
 				icd->error = ret;
 		}
 	} else if (io_u->error) {
+		if (!((io_u->flags & IO_U_F_ERRORED) &&
+				(io_u->flags & IO_U_F_VER_LIST))) {
 error:
-		icd->error = io_u->error;
-		io_u_log_error(td, io_u);
+			icd->error = io_u->error;
+			io_u_log_error(td, io_u);
+		} else {
+			dprint(FD_IO, "io_completed: errored io_u numberio="
+					"%"PRIu64" for verify (expected)\n",
+					io_u->numberio);
+		}
 	}
 	if (icd->error) {
 		enum error_type_bit eb = td_error_type(ddir, icd->error);

io_u.h

@@ -24,6 +24,8 @@ enum {
 	IO_U_F_PATTERN_DONE	= 1 << 8,
 	IO_U_F_DEVICE_ERROR	= 1 << 9,
 	IO_U_F_VER_IN_DEV	= 1 << 10, /* Verify data in device */
+	IO_U_F_ZEROED		= 1 << 11, /* Zeroed data */
+	IO_U_F_ERRORED		= 1 << 12, /* Errored offset */
 };
 
 /*

iolog.c

@@ -295,6 +295,11 @@ void log_io_piece(struct thread_data *td, struct io_u *io_u)
 
 	io_u->ipo = ipo;
 
+	if (io_u->flags & IO_U_F_ZEROED)
+		ipo->flags |= IP_F_ZEROED;
+	else if (io_u->flags & IO_U_F_ERRORED)
+		ipo->flags |= IP_F_ERRORED;
+
 	if (io_u_should_trim(td, io_u)) {
 		flist_add_tail(&ipo->trim_list, &td->trim_list);
 		td->trim_entries++;

iolog.h

@@ -244,6 +244,8 @@ enum {
 	IP_F_ONLIST	= 2,
 	IP_F_TRIMMED	= 4,
 	IP_F_IN_FLIGHT	= 8,
+	IP_F_ZEROED	= 16,
+	IP_F_ERRORED	= 32,
 };
 
 /*