[KERNEL] Add support for Linux kernels without blk-mq (e.g. CentOS 7)

5 files changed, 259 insertions, 20 deletions

diff --git a/README.md b/README.md
index aaaee32..e71b7dd 100644
--- a/README.md
+++ b/README.md
@@ -16,6 +16,7 @@ The dnbd3 components can be built for the following Linux kernel versions and Un
   - Ubuntu 20.04 with **Linux kernel 5.4.x**
   - Ubuntu 18.04 with **Linux kernel 4.19.x**
   - CentOS 8 with **Linux kernel 4.18.x**
+  - CentOS 7 with **Linux kernel 3.10.x**
   - AlmaLinux 8 with **Linux kernel 4.18.x**
   - FreeBSD 12.x and 13.x (only user space programs, eg. dnbd3-server)
 
@@ -99,6 +100,59 @@ yum install git \
 
 Note that `afl` is not available on CentOS 8 and should be built from the [original sources](https://github.com/google/AFL).
 
+#### CentOS 7 with Linux kernel 3.10.x
+Before any required preliminaries can be installed, enable the `epel` package repository with the following command line calls:
+
+```shell
+yum install epel-release
+yum repolist # refresh epel package repository
+```
+
+The `epel` package repository enables the installation of `cmake3` on CentOS 7 which is later required to build dnbd3 components.
+Then, install the required preliminaries with the following command line call as usual:
+
+```shell
+yum install git \
+            make \
+            cmake3 \
+            gcc \
+            kernel-devel \
+            elfutils-libelf-devel \
+            rpm-build
+```
+
+Note that `afl` is not available on CentOS 7 and should be built from the [original sources](https://github.com/google/AFL).
+
+> **Warning: All dnbd3 components can only be built if a GCC compiler with stdatomic support is used.
+> This feature is available with GCC 4.9 or later as part of the C11 language support.
+> Since CentOS 7 is shipped with GCC 4.8 you have to install a new GCC version greater or equal than GCC 4.9.**
+
+The installation of GCC 7.3 on CentOS requires some additional instructions as follows.
+First, install Software Collections on your system that allows you to build, install, and use multiple versions of GCC on the same system withoutaffecting system-wide installed packages. Software collections is part of the CentOS `extras` repository and can be installed by running the following command:
+
+```shell
+yum install centos-release-scl
+```
+
+After installation of Software Collections, install the Developer Toolset in version 7 and additional packages with the following command line call:
+
+```shell
+yum install devtoolset-7 \
+            devtoolset-7-libatomic-devel \
+            llvm-toolset-7-git-clang-format \
+            fuse-devel \
+            jansson-devel
+```
+
+To access GCC 7.3, you need to launch a new shell instance using the Software Collections `scl` tool:
+
+```shell
+scl enable devtoolset-7 llvm-toolset-7 bash
+```
+
+Now, GCC 7.3 is the default version in your current shell.
+This allows you to build all dnbd3 components on CentOS 7.
+
 #### AlmaLinux 8 with Linux kernel 4.18.x
 ```shell
 yum install git \
diff --git a/src/kernel/blk.c b/src/kernel/blk.c
index d0a3b00..26c0a01 100644
--- a/src/kernel/blk.c
+++ b/src/kernel/blk.c
@@ -27,10 +27,6 @@
 
 #include <linux/pagemap.h>
 
-#define dnbd3_req_read(req) (req_op(req) == REQ_OP_READ)
-#define dnbd3_req_fs(req) (dnbd3_req_read(req) || req_op(req) == REQ_OP_WRITE)
-#define dnbd3_req_special(req) blk_rq_is_private(req)
-
 static int dnbd3_close_device(dnbd3_device_t *dev)
 {
 	int result;
@@ -49,9 +45,13 @@ static int dnbd3_close_device(dnbd3_device_t *dev)
 	/* new requests might have been queued up, */
 	/* but now that imgname is NULL no new ones can show up */
 	dnbd3_blk_fail_all_requests(dev);
+#ifdef DNBD3_BLK_MQ
 	blk_mq_freeze_queue(dev->queue);
+#endif
 	set_capacity(dev->disk, 0);
+#ifdef DNBD3_BLK_MQ
 	blk_mq_unfreeze_queue(dev->queue);
+#endif
 	dev->reported_size = 0;
 	return result;
 }
@@ -118,8 +118,12 @@ static int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int
 			dev->use_server_provided_alts = msg->use_server_provided_alts;
 
 			dev_info(dnbd3_device_to_dev(dev), "opening device.\n");
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
 			if (blk_queue->backing_dev_info != NULL)
 				blk_queue->backing_dev_info->ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+#else
+			blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+#endif
 
 			/* add specified servers to alt server list */
 			for (i = 0; i < NUMBER_SERVERS; i++)
@@ -318,6 +322,10 @@ static const struct block_device_operations dnbd3_blk_ops = {
 	.ioctl = dnbd3_blk_ioctl,
 };
 
+#ifdef DNBD3_BLK_MQ
+/*
+ * Linux kernel blk-mq driver function (entry point) to handle block IO requests
+ */
 static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd)
 {
 	struct request *rq = bd->rq;
@@ -348,6 +356,46 @@ static const struct blk_mq_ops dnbd3_mq_ops = {
 	.queue_rq = dnbd3_queue_rq,
 };
 
+#else /* DNBD3_BLK_MQ */
+/*
+ * Linux kernel blk driver function (entry point) to handle block IO requests
+ */
+static void dnbd3_blk_request(struct request_queue *q)
+{
+	struct request *rq;
+	dnbd3_device_t *dev;
+
+	while ((rq = blk_fetch_request(q)) != NULL) {
+		dev = rq->rq_disk->private_data;
+
+		if (dev->imgname == NULL) {
+			__blk_end_request_all(rq, -EIO);
+			continue;
+		}
+
+		if (!(dnbd3_req_fs(rq))) {
+			__blk_end_request_all(rq, 0);
+			continue;
+		}
+
+		if (PROBE_COUNT_TIMEOUT > 0 && dev->panic_count >= PROBE_COUNT_TIMEOUT) {
+			__blk_end_request_all(rq, -EIO);
+			continue;
+		}
+
+		if (!(dnbd3_req_read(rq))) {
+			__blk_end_request_all(rq, -EACCES);
+			continue;
+		}
+
+		list_add_tail(&rq->queuelist, &dev->request_queue_send);
+		spin_unlock_irq(q->queue_lock);
+		wake_up(&dev->process_queue_send);
+		spin_lock_irq(q->queue_lock);
+	}
+}
+#endif /* DNBD3_BLK_MQ */
+
 int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
 {
 	int ret;
@@ -377,6 +425,7 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
 	// set up spin lock for request queues for send and receive
 	spin_lock_init(&dev->blk_lock);
 
+#ifdef DNBD3_BLK_MQ
 	// set up tag_set for blk-mq
 	dev->tag_set.ops = &dnbd3_mq_ops;
 	dev->tag_set.nr_hw_queues = 1;
@@ -399,12 +448,25 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
 		dev_err(dnbd3_device_to_dev(dev), "blk_mq_init_queue failed\n");
 		goto out_cleanup_tags;
 	}
+#else
+	// set up blk
+	dev->queue = blk_init_queue(&dnbd3_blk_request, &dev->blk_lock);
+	if (!dev->queue) {
+		ret = -ENOMEM;
+		dev_err(dnbd3_device_to_dev(dev), "blk_init_queue failed\n");
+		goto out;
+	}
+#endif /* DNBD3_BLK_MQ */
 	dev->queue->queuedata = dev;
 
 	blk_queue_logical_block_size(dev->queue, DNBD3_BLOCK_SIZE);
 	blk_queue_physical_block_size(dev->queue, DNBD3_BLOCK_SIZE);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0)
 	blk_queue_flag_set(QUEUE_FLAG_NONROT, dev->queue);
 	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, dev->queue);
+#else
+	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, dev->queue);
+#endif
 #define ONE_MEG (1048576)
 	blk_queue_max_segment_size(dev->queue, ONE_MEG);
 	blk_queue_max_segments(dev->queue, 0xffff);
@@ -438,8 +500,10 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
 
 out_cleanup_queue:
 	blk_cleanup_queue(dev->queue);
+#ifdef DNBD3_BLK_MQ
 out_cleanup_tags:
 	blk_mq_free_tag_set(&dev->tag_set);
+#endif
 out:
 	return ret;
 }
@@ -453,7 +517,9 @@ int dnbd3_blk_del_device(dnbd3_device_t *dev)
 	dnbd3_sysfs_exit(dev);
 	del_gendisk(dev->disk);
 	blk_cleanup_queue(dev->queue);
+#ifdef DNBD3_BLK_MQ
 	blk_mq_free_tag_set(&dev->tag_set);
+#endif
 	mutex_destroy(&dev->alt_servers_lock);
 	put_disk(dev->disk);
 	return 0;
@@ -506,7 +572,11 @@ void dnbd3_blk_fail_all_requests(dnbd3_device_t *dev)
 	list_for_each_entry_safe(blk_request, tmp_request, &local_copy, queuelist) {
 		list_del_init(&blk_request->queuelist);
 		if (dnbd3_req_fs(blk_request))
+#ifdef DNBD3_BLK_MQ
 			blk_mq_end_request(blk_request, BLK_STS_IOERR);
+#else
+			blk_end_request_all(blk_request, -EIO);
+#endif
 		else if (dnbd3_req_special(blk_request))
 			kfree(blk_request);
 	}
diff --git a/src/kernel/blk.h b/src/kernel/blk.h
index 2410fe1..2da84bc 100644
--- a/src/kernel/blk.h
+++ b/src/kernel/blk.h
@@ -24,7 +24,96 @@
 
 #include "dnbd3_main.h"
 
-#define REQ_TYPE_SPECIAL REQ_TYPE_DRV_PRIV
+/* define blkdev file system operation type */
+#define DNBD3_REQ_OP_FS           REQ_TYPE_FS
+
+/* define blkdev special operation type */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#define DNBD3_REQ_OP_SPECIAL      REQ_OP_DRV_IN
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) || \
+	RHEL_CHECK_VERSION(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 3))
+#define DNBD3_REQ_OP_SPECIAL      REQ_TYPE_DRV_PRIV
+#else
+#define DNBD3_REQ_OP_SPECIAL      REQ_TYPE_SPECIAL
+#endif
+
+/* define blkdev read operation type */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#define DNBD3_DEV_READ            REQ_OP_READ
+#else
+#define DNBD3_DEV_READ            DNBD3_REQ_OP_FS
+#endif
+
+/* define blkdev write operation type */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#define DNBD3_DEV_WRITE           REQ_OP_WRITE
+#else
+#define DNBD3_DEV_WRITE           DNBD3_REQ_OP_FS
+#endif
+
+/* define command and blkdev operation access macros */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#define DNBD3_REQ_FLAG_BITS       REQ_FLAG_BITS
+/* cmd_flags and cmd_type are merged into cmd_flags now */
+/* sanity check to avoid overriding of request bits */
+#if DNBD3_REQ_FLAG_BITS > 24
+#error "Fix CMD bitshift"
+#endif
+/* pack command into cmd_flags field by shifting CMD_* into unused bits of cmd_flags */
+#define dnbd3_cmd_to_priv(req, cmd) \
+	((req)->cmd_flags = DNBD3_REQ_OP_SPECIAL | ((cmd) << DNBD3_REQ_FLAG_BITS))
+#define dnbd3_priv_to_cmd(req) \
+	((req)->cmd_flags >> DNBD3_REQ_FLAG_BITS)
+#define dnbd3_req_op(req) \
+	req_op(req)
+#else
+/* pack command into cmd_type and cmd_flags field separated */
+#define dnbd3_cmd_to_priv(req, cmd) \
+	do { \
+		(req)->cmd_type = DNBD3_REQ_OP_SPECIAL; \
+		(req)->cmd_flags = (cmd); \
+	} while (0)
+#define dnbd3_priv_to_cmd(req) \
+	((req)->cmd_flags)
+#define dnbd3_req_op(req) \
+	((req)->cmd_type)
+#endif
+
+/* define dnbd3_req_read(req) boolean expression */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#define dnbd3_req_read(req) \
+	(req_op(req) == DNBD3_DEV_READ)
+#else
+#define dnbd3_req_read(req) \
+	(rq_data_dir(req) == READ)
+#endif
+
+/* define dnbd3_req_write(req) boolean expression */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#define dnbd3_req_write(req) \
+	(req_op(req) == DNBD3_DEV_WRITE)
+#else
+#define dnbd3_req_write(req) \
+	(rq_data_dir(req) == WRITE)
+#endif
+
+/* define dnbd3_req_fs(req) boolean expression */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#define dnbd3_req_fs(req) \
+	(dnbd3_req_read(req) || dnbd3_req_write(req))
+#else
+#define dnbd3_req_fs(req) \
+	(dnbd3_req_op(req) == DNBD3_REQ_OP_FS)
+#endif
+
+/* define dnbd3_req_special(req) boolean expression */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+#define dnbd3_req_special(req) \
+	blk_rq_is_private(req)
+#else
+#define dnbd3_req_special(req) \
+	(dnbd3_req_op(req) == DNBD3_REQ_OP_SPECIAL)
+#endif
 
 int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor);
 
diff --git a/src/kernel/dnbd3_main.h b/src/kernel/dnbd3_main.h
index f3bd8fe..efe4a76 100644
--- a/src/kernel/dnbd3_main.h
+++ b/src/kernel/dnbd3_main.h
@@ -26,7 +26,6 @@
 #include <linux/kthread.h>
 #include <linux/module.h>
 #include <linux/blkdev.h>
-#include <linux/blk-mq.h>
 #include <linux/mutex.h>
 #include <net/sock.h>
 
@@ -34,6 +33,26 @@
 #include <dnbd3/types.h>
 #include <dnbd3/shared/serialize.h>
 
+/* define RHEL_CHECK_VERSION macro to check CentOS version */
+#if defined(RHEL_RELEASE_CODE) && defined(RHEL_RELEASE_VERSION)
+#define RHEL_CHECK_VERSION(CONDITION) (CONDITION)
+#else
+#define RHEL_CHECK_VERSION(CONDITION) (0)
+#endif
+
+/* version check to enable/disable blk-mq support */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 18, 0)
+/* enable blk-mq support for Linux kernel 4.18 and later */
+#define DNBD3_BLK_MQ
+#else
+/* disable blk-mq support for Linux kernel prior to 4.18 */
+#undef DNBD3_BLK_MQ
+#endif
+
+#ifdef DNBD3_BLK_MQ
+#include <linux/blk-mq.h>
+#endif
+
 extern int major;
 
 typedef struct {
@@ -47,7 +66,9 @@ typedef struct {
 typedef struct {
 	// block
 	struct gendisk *disk;
+#ifdef DNBD3_BLK_MQ
 	struct blk_mq_tag_set tag_set;
+#endif
 	struct request_queue *queue;
 	spinlock_t blk_lock;
 
diff --git a/src/kernel/net.c b/src/kernel/net.c
index 0eb1d5e..7432fde 100644
--- a/src/kernel/net.c
+++ b/src/kernel/net.c
@@ -23,6 +23,7 @@
 #include "net.h"
 #include "blk.h"
 #include "utils.h"
+#include "dnbd3_main.h"
 
 #include <dnbd3/shared/serialize.h>
 
@@ -61,17 +62,6 @@
 		h.msg_flags = MSG_WAITALL | MSG_NOSIGNAL; \
 	} while (0)
 
-// cmd_flags and cmd_type are merged into cmd_flags now
-#if REQ_FLAG_BITS > 24
-#error "Fix CMD bitshift"
-#endif
-// Pack into cmd_flags field by shifting CMD_* into unused bits of cmd_flags
-#define dnbd3_cmd_to_priv(req, cmd) ((req)->cmd_flags = REQ_OP_DRV_IN | ((cmd) << REQ_FLAG_BITS))
-#define dnbd3_priv_to_cmd(req) ((req)->cmd_flags >> REQ_FLAG_BITS)
-#define dnbd3_req_op(req) req_op(req)
-#define DNBD3_DEV_READ REQ_OP_READ
-#define DNBD3_REQ_OP_SPECIAL REQ_OP_DRV_IN
-
 #define dnbd3_dev_dbg_host(dev, host, fmt, ...) \
 	dev_dbg(dnbd3_device_to_dev(dev), "(%pISpc): " fmt, (host), ##__VA_ARGS__)
 #define dnbd3_dev_err_host(dev, host, fmt, ...) \
@@ -140,8 +130,8 @@ static int dnbd3_net_discover(void *data)
 	dnbd3_alt_server_t *alt;
 	struct sockaddr_storage host_compare, best_server;
 	uint16_t remote_version;
-	ktime_t start = 0, end = 0;
-	unsigned long rtt, best_rtt = 0;
+	ktime_t start = ktime_set(0, 0), end = ktime_set(0, 0);
+	unsigned long rtt = 0, best_rtt = 0;
 	unsigned long irqflags;
 	int i, j, isize, fails, rtt_threshold;
 	int turn = 0;
@@ -593,7 +583,11 @@ static int dnbd3_net_receive(void *data)
 				goto cleanup;
 			}
 			// receive data and answer to block layer
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)
 			rq_for_each_segment(bvec_inst, blk_request, iter) {
+#else
+			rq_for_each_segment(bvec, blk_request, iter) {
+#endif
 				kaddr = kmap(bvec->bv_page) + bvec->bv_offset;
 				iov.iov_base = kaddr;
 				iov.iov_len = bvec->bv_len;
@@ -622,7 +616,11 @@ static int dnbd3_net_receive(void *data)
 					goto cleanup;
 				}
 			}
+#ifdef DNBD3_BLK_MQ
 			blk_mq_end_request(blk_request, BLK_STS_OK);
+#else
+			blk_end_request_all(blk_request, 0);
+#endif
 			continue;
 
 		case CMD_GET_SERVERS:
@@ -753,7 +751,14 @@ static struct socket *dnbd3_connect(dnbd3_device_t *dev, struct sockaddr_storage
 	struct socket *sock;
 	int retries = 4;
 
-	if (sock_create_kern(&init_net, addr->ss_family, SOCK_STREAM, IPPROTO_TCP, &sock) < 0) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)
+	ret = sock_create_kern(&init_net, addr->ss_family, SOCK_STREAM,
+			IPPROTO_TCP, &sock);
+#else
+	ret = sock_create_kern(addr->ss_family, SOCK_STREAM,
+			IPPROTO_TCP, &sock);
+#endif
+	if (ret < 0) {
 		dev_err(dnbd3_device_to_dev(dev), "couldn't create socket\n");
 		return NULL;
 	}