From 0d1e4c5c1632090d86d3b129c50882afdac7ddc0 Mon Sep 17 00:00:00 2001 From: Johann Latocha Date: Tue, 21 Feb 2012 18:57:18 +0100 Subject: [KERNEL] SysFS interface added (/sys/block/dnbd*/net/) --- src/config.h | 2 +- src/kernel/blk.c | 20 +++-- src/kernel/dnbd3.h | 25 +++++-- src/kernel/net.c | 94 +++++++++++++----------- src/kernel/sysfs.c | 212 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/kernel/sysfs.h | 31 ++++++++ 6 files changed, 327 insertions(+), 57 deletions(-) create mode 100644 src/kernel/sysfs.c create mode 100644 src/kernel/sysfs.h (limited to 'src') diff --git a/src/config.h b/src/config.h index 77caf11..8a6cdcf 100644 --- a/src/config.h +++ b/src/config.h @@ -28,7 +28,7 @@ #define SOCKET_TIMEOUT_SERVER 30 #define SOCKET_TIMEOUT_CLIENT_DATA 5 #define SOCKET_TIMEOUT_CLIENT_DISCOVERY 1 -#define TIMER_INTERVAL_HEARTBEAT 2*HZ +#define TIMER_INTERVAL_HEARTBEAT 10*HZ #define TIMER_INTERVAL_PANIC 1*HZ #define NUMBER_SERVERS 8 diff --git a/src/kernel/blk.c b/src/kernel/blk.c index 192c3db..70d4e1f 100644 --- a/src/kernel/blk.c +++ b/src/kernel/blk.c @@ -20,6 +20,7 @@ #include "blk.h" #include "net.h" +#include "sysfs.h" int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor) { @@ -32,10 +33,15 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor) INIT_LIST_HEAD(&dev->request_queue_send); INIT_LIST_HEAD(&dev->request_queue_receive); + memset(dev->cur_server.host, 0, 16); + memset(dev->cur_server.port, 0, 6); + dev->cur_server.rtt = 0; + dev->cur_server.sock = NULL; + dev->vid = 0; dev->rid = 0; - dev->sock = NULL; - dev->num_servers = 0; + dev->alt_servers_num = 0; + memset(dev->alt_servers, 0, sizeof(dnbd3_server_t)*NUMBER_SERVERS); dev->thread_send = NULL; dev->thread_receive = NULL; dev->thread_discover = NULL; @@ -68,12 +74,14 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor) queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue); dev->disk = disk; - add_disk(disk); // must be last + add_disk(disk); + dnbd3_sysfs_init(dev); return 0; } int dnbd3_blk_del_device(dnbd3_device_t *dev) { + dnbd3_sysfs_exit(dev); dnbd3_net_disconnect(dev); del_gendisk(dev->disk); put_disk(dev->disk); @@ -95,8 +103,8 @@ int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, u switch (cmd) { case IOCTL_OPEN: - strcpy(dev->host, msg->host); - strcpy(dev->port, msg->port); + strcpy(dev->cur_server.host, msg->host); + strcpy(dev->cur_server.port, msg->port); dev->vid = msg->vid; dev->rid = msg->rid; dnbd3_net_connect(dev); @@ -110,7 +118,7 @@ int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, u case IOCTL_SWITCH: dnbd3_net_disconnect(dev); - strcpy(dev->host, msg->host); + strcpy(dev->cur_server.host, msg->host); dnbd3_net_connect(dev); break; diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h index 140754c..febd473 100644 --- a/src/kernel/dnbd3.h +++ b/src/kernel/dnbd3.h @@ -31,24 +31,33 @@ extern int major; +typedef struct +{ + char host[16]; + char port[6]; + uint64_t rtt; + struct socket *sock; + struct kobject kobj; +} dnbd3_server_t; + typedef struct { // block struct gendisk *disk; spinlock_t blk_lock; + // sysfs + struct kobject kobj; + // network - char host[16]; - char port[6]; - int vid; - int rid; - struct socket *sock; - struct timer_list hb_timer; - int num_servers; - struct in_addr servers[NUMBER_SERVERS]; + dnbd3_server_t cur_server; + int vid, rid; + int alt_servers_num; + dnbd3_server_t alt_servers[NUMBER_SERVERS]; int discover, panic; // process + struct timer_list hb_timer; struct task_struct *thread_send; struct task_struct *thread_receive; struct task_struct *thread_discover; diff --git a/src/kernel/net.c b/src/kernel/net.c index 8ec34d0..762d0d8 100644 --- a/src/kernel/net.c +++ b/src/kernel/net.c @@ -39,36 +39,36 @@ void dnbd3_net_connect(dnbd3_device_t *dev) printk("ERROR: Kmalloc failed.\n"); return; } - if (!dev->host || !dev->port || (dev->vid == 0)) + if (!dev->cur_server.host || !dev->cur_server.port || (dev->vid == 0)) { printk("ERROR: Host, port or vid not set.\n"); return; } - if (dev->sock) + if (dev->cur_server.sock) { - printk("ERROR: Device %s already connected to %s.\n", dev->disk->disk_name, dev->host); + printk("ERROR: Device %s already connected to %s.\n", dev->disk->disk_name, dev->cur_server.host); return; } - printk("INFO: Connecting device %s to %s\n", dev->disk->disk_name, dev->host); + printk("INFO: Connecting device %s to %s\n", dev->disk->disk_name, dev->cur_server.host); // initialize socket - if (sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &dev->sock) < 0) + if (sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &dev->cur_server.sock) < 0) { printk("ERROR: Couldn't create socket.\n"); - dev->sock = NULL; + dev->cur_server.sock = NULL; return; } - kernel_setsockopt(dev->sock, SOL_SOCKET, SO_SNDTIMEO, (char *) &timeout, sizeof(timeout)); - kernel_setsockopt(dev->sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &timeout, sizeof(timeout)); - dev->sock->sk->sk_allocation = GFP_NOIO; + kernel_setsockopt(dev->cur_server.sock, SOL_SOCKET, SO_SNDTIMEO, (char *) &timeout, sizeof(timeout)); + kernel_setsockopt(dev->cur_server.sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &timeout, sizeof(timeout)); + dev->cur_server.sock->sk->sk_allocation = GFP_NOIO; sin.sin_family = AF_INET; - sin.sin_addr.s_addr = inet_addr(dev->host); - sin.sin_port = htons(simple_strtol(dev->port, NULL, 10)); - if (kernel_connect(dev->sock, (struct sockaddr *) &sin, sizeof(sin), 0) < 0) + sin.sin_addr.s_addr = inet_addr(dev->cur_server.host); + sin.sin_port = htons(simple_strtol(dev->cur_server.port, NULL, 10)); + if (kernel_connect(dev->cur_server.sock, (struct sockaddr *) &sin, sizeof(sin), 0) < 0) { - printk("ERROR: Couldn't connect to host %s:%s\n", dev->host, dev->port); - dev->sock = NULL; + printk("ERROR: Couldn't connect to host %s:%s\n", dev->cur_server.host, dev->cur_server.port); + dev->cur_server.sock = NULL; return; } @@ -121,10 +121,10 @@ void dnbd3_net_disconnect(dnbd3_device_t *dev) kthread_stop(dev->thread_discover); // clear socket - if (dev->sock) + if (dev->cur_server.sock) { - sock_release(dev->sock); - dev->sock = NULL; + sock_release(dev->cur_server.sock); + dev->cur_server.sock = NULL; } } @@ -191,7 +191,7 @@ int dnbd3_net_discover(void *data) if (!&dev->discover) continue; - num = dev->num_servers; + num = dev->alt_servers_num; dev->discover = 0; best_rtt = -1; @@ -206,14 +206,15 @@ int dnbd3_net_discover(void *data) } kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *) &timeout, sizeof(timeout)); kernel_setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &timeout, sizeof(timeout)); - inet_ntoa(dev->servers[i], current_server); + strcpy(current_server, dev->alt_servers[i].host); sock->sk->sk_allocation = GFP_NOIO; sin.sin_family = AF_INET; sin.sin_addr.s_addr = inet_addr(current_server); - sin.sin_port = htons(simple_strtol(dev->port, NULL, 10)); + sin.sin_port = htons(simple_strtol(dev->cur_server.port, NULL, 10)); if (kernel_connect(sock, (struct sockaddr *) &sin, sizeof(sin), 0) < 0) { - printk("ERROR: Couldn't connect to host %s:%s (discover)\n", current_server, dev->port); + printk("ERROR: Couldn't connect to host %s:%s (discover)\n", current_server, dev->cur_server.port); + dev->alt_servers[i].rtt = -1; sock = NULL; continue; } @@ -226,7 +227,7 @@ int dnbd3_net_discover(void *data) kfree(buf); dev->thread_discover = NULL; dnbd3_net_disconnect(dev); - strcpy(dev->host, current_server); + strcpy(dev->cur_server.host, current_server); dnbd3_net_connect(dev); return 0; } @@ -283,9 +284,11 @@ int dnbd3_net_discover(void *data) t1 = (start.tv_sec*1000000ull) + start.tv_usec; t2 = (end.tv_sec*1000000ull) + end.tv_usec; - if ( best_rtt > (t2 - t1) ) + dev->alt_servers[i].rtt = t2 -t1; + + if ( best_rtt > dev->alt_servers[i].rtt ) { - best_rtt = t2 - t1; + best_rtt = dev->alt_servers[i].rtt; strcpy(best_server, current_server); } @@ -299,16 +302,21 @@ int dnbd3_net_discover(void *data) } // take server with lowest rtt - if (num > 1 && strcmp(dev->host, best_server)) + if (num > 1 && strcmp(dev->cur_server.host, best_server)) { printk("INFO: Server %s is faster (%lluus), switching...\n", best_server, best_rtt); kfree(buf); dev->thread_discover = NULL; dnbd3_net_disconnect(dev); - strcpy(dev->host, best_server); + strcpy(dev->cur_server.host, best_server); + dev->cur_server.rtt = best_rtt; dnbd3_net_connect(dev); return 0; } + else + { + dev->cur_server.rtt = best_rtt; + } } kfree(buf); @@ -376,7 +384,7 @@ int dnbd3_net_send(void *data) memcpy(dnbd3_request.handle, &blk_request, sizeof(blk_request)); iov.iov_base = &dnbd3_request; iov.iov_len = sizeof(dnbd3_request); - if (kernel_sendmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_request)) <= 0) + if (kernel_sendmsg(dev->cur_server.sock, &msg, &iov, 1, sizeof(dnbd3_request)) <= 0) goto error; // enqueue request to request_queue_receive @@ -389,9 +397,9 @@ int dnbd3_net_send(void *data) return 0; error: - printk("ERROR: Connection to server %s lost (send)\n", dev->host); - if (dev->sock) - kernel_sock_shutdown(dev->sock, SHUT_RDWR); + printk("ERROR: Connection to server %s lost (send)\n", dev->cur_server.host); + if (dev->cur_server.sock) + kernel_sock_shutdown(dev->cur_server.sock, SHUT_RDWR); dev->thread_send = NULL; dev->panic = 1; return -1; @@ -413,6 +421,7 @@ int dnbd3_net_receive(void *data) unsigned int size, i; uint64_t filesize; + struct in_addr tmp_addr; init_msghdr(msg); set_user_nice(current, -20); @@ -428,7 +437,7 @@ int dnbd3_net_receive(void *data) // receive net replay iov.iov_base = &dnbd3_reply; iov.iov_len = sizeof(dnbd3_reply); - if (kernel_recvmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags) <= 0) + if (kernel_recvmsg(dev->cur_server.sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags) <= 0) goto error; // search for replied request in queue @@ -449,9 +458,9 @@ int dnbd3_net_receive(void *data) list_del_init(&blk_request->queuelist); kthread_stop(dev->thread_send); del_timer(&dev->hb_timer); - sock_release(dev->sock); + sock_release(dev->cur_server.sock); kfree(blk_request); - dev->sock = NULL; + dev->cur_server.sock = NULL; return -1; case ERROR_RELOAD: @@ -478,7 +487,7 @@ int dnbd3_net_receive(void *data) size = bvec->bv_len; iov.iov_base = kaddr; iov.iov_len = size; - if (kernel_recvmsg(dev->sock, &msg, &iov, 1, size, msg.msg_flags) <= 0) + if (kernel_recvmsg(dev->cur_server.sock, &msg, &iov, 1, size, msg.msg_flags) <= 0) { kunmap(bvec->bv_page); goto error; @@ -496,7 +505,7 @@ int dnbd3_net_receive(void *data) case CMD_GET_SIZE: iov.iov_base = &filesize; iov.iov_len = sizeof(uint64_t); - if (kernel_recvmsg(dev->sock, &msg, &iov, 1, dnbd3_reply.size, msg.msg_flags) <= 0) + if (kernel_recvmsg(dev->cur_server.sock, &msg, &iov, 1, dnbd3_reply.size, msg.msg_flags) <= 0) goto error; set_capacity(dev->disk, filesize >> 9); /* 512 Byte blocks */ printk("INFO: Filesize %s: %llu\n", dev->disk->disk_name, filesize); @@ -507,14 +516,15 @@ int dnbd3_net_receive(void *data) continue; case CMD_GET_SERVERS: - dev->num_servers = dnbd3_reply.size / sizeof(struct in_addr); + dev->alt_servers_num = dnbd3_reply.size / sizeof(struct in_addr); size = sizeof(struct in_addr); - for (i = 0; i < dev->num_servers && i < NUMBER_SERVERS; i++) + for (i = 0; i < dev->alt_servers_num && i < NUMBER_SERVERS; i++) { - iov.iov_base = &dev->servers[i]; + iov.iov_base = &tmp_addr; iov.iov_len = size; - if (kernel_recvmsg(dev->sock, &msg, &iov, 1, size, msg.msg_flags) <= 0) + if (kernel_recvmsg(dev->cur_server.sock, &msg, &iov, 1, size, msg.msg_flags) <= 0) goto error; + inet_ntoa(tmp_addr, dev->alt_servers[i].host); } spin_lock_irq(&dev->blk_lock); list_del_init(&blk_request->queuelist); @@ -534,7 +544,7 @@ int dnbd3_net_receive(void *data) return 0; error: - printk("ERROR: Connection to server %s lost (receive)\n", dev->host); + printk("ERROR: Connection to server %s lost (receive)\n", dev->cur_server.host); // move already send requests to request_queue_send again if (!list_empty(&dev->request_queue_receive)) { @@ -547,8 +557,8 @@ int dnbd3_net_receive(void *data) } spin_unlock_irq(&dev->blk_lock); } - if (dev->sock) - kernel_sock_shutdown(dev->sock, SHUT_RDWR); + if (dev->cur_server.sock) + kernel_sock_shutdown(dev->cur_server.sock, SHUT_RDWR); dev->thread_receive = NULL; dev->panic = 1; return -1; diff --git a/src/kernel/sysfs.c b/src/kernel/sysfs.c new file mode 100644 index 0000000..e363824 --- /dev/null +++ b/src/kernel/sysfs.c @@ -0,0 +1,212 @@ +/* + * This file is part of the Distributed Network Block Device 3 + * + * Copyright(c) 2011-2012 Johann Latocha + * + * This file may be licensed under the terms of of the + * GNU General Public License Version 2 (the ``GPL''). + * + * Software distributed under the License is distributed + * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either + * express or implied. See the GPL for the specific language + * governing rights and limitations. + * + * You should have received a copy of the GPL along with this + * program. If not, go to http://www.gnu.org/licenses/gpl.html + * or write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +#include + +#include "sysfs.h" +#include "utils.h" + +typedef struct +{ + struct attribute attr; + ssize_t (*show)(char *, dnbd3_device_t *); + ssize_t (*store)(const char *, size_t, dnbd3_device_t *); +} device_attr_t; + +typedef struct +{ + struct attribute attr; + ssize_t (*show)(char *, dnbd3_server_t *); + ssize_t (*store)(const char *, size_t, dnbd3_server_t *); +} server_attr_t; + + +ssize_t show_cur_server_ip(char *buf, dnbd3_device_t *dev) +{ + return sprintf(buf, "%s\n", dev->cur_server.host); +} + +ssize_t show_cur_server_rtt(char *buf, dnbd3_device_t *dev) +{ + return sprintf(buf, "%lluus\n", dev->cur_server.rtt); +} + +ssize_t show_alt_server_num(char *buf, dnbd3_device_t *dev) +{ + return sprintf(buf, "%d\n", dev->alt_servers_num); +} + +ssize_t show_vid(char *buf, dnbd3_device_t *dev) +{ + return sprintf(buf, "%d\n", dev->vid); +} + +ssize_t show_rid(char *buf, dnbd3_device_t *dev) +{ + return sprintf(buf, "%d\n", dev->rid); +} + +ssize_t show_alt_server_ip(char *buf, dnbd3_server_t *srv) +{ + return sprintf(buf, "%s\n", srv->host); +} + +ssize_t show_alt_server_rtt(char *buf, dnbd3_server_t *srv) +{ + return sprintf(buf, "%lluus\n", srv->rtt); +} + +device_attr_t cur_server_ip = +{ + .attr = {.name = "cur_server_ip", .mode = 0444 }, + .show = show_cur_server_ip, + .store = NULL, +}; + +device_attr_t cur_server_rtt = +{ + .attr = {.name = "cur_server_rtt", .mode = 0444 }, + .show = show_cur_server_rtt, + .store = NULL, +}; + +device_attr_t alt_server_num = +{ + .attr = {.name = "alt_server_num", .mode = 0444 }, + .show = show_alt_server_num, + .store = NULL, +}; + +device_attr_t vid = +{ + .attr = {.name = "vid", .mode = 0444 }, + .show = show_vid, + .store = NULL, +}; + +device_attr_t rid = +{ + .attr = {.name = "rid", .mode = 0444 }, + .show = show_rid, + .store = NULL, +}; + +server_attr_t alt_server_ip = +{ + .attr = {.name = "alt_server_ip", .mode = 0444 }, + .show = show_alt_server_ip, + .store = NULL, +}; + +server_attr_t alt_server_rtt = +{ + .attr = {.name = "alt_server_rtt", .mode = 0444 }, + .show = show_alt_server_rtt, + .store = NULL, +}; + +ssize_t device_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + device_attr_t *device_attr = container_of(attr, device_attr_t, attr); + dnbd3_device_t *dev = container_of(kobj, dnbd3_device_t, kobj); + return device_attr->show(buf, dev); +} + +ssize_t server_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + server_attr_t *server_attr = container_of(attr, server_attr_t, attr); + dnbd3_server_t *srv = container_of(kobj, dnbd3_server_t, kobj); + return server_attr->show(buf, srv); +} + +struct attribute *device_attrs[] = +{ + &cur_server_ip.attr, + &cur_server_rtt.attr, + &alt_server_num.attr, + &vid.attr, + &rid.attr, + NULL, +}; + +struct attribute *server_attrs[] = +{ + &alt_server_ip.attr, + &alt_server_rtt.attr, + NULL, +}; + +struct sysfs_ops device_ops = +{ + .show = device_show, +}; + +struct sysfs_ops server_ops = +{ + .show = server_show, +}; + +void release(struct kobject *kobj) {} + +struct kobj_type device_ktype = +{ + .default_attrs = device_attrs, + .sysfs_ops = &device_ops, + .release = release, +}; + +struct kobj_type server_ktype = +{ + .default_attrs = server_attrs, + .sysfs_ops = &server_ops, + .release = release, +}; + +void dnbd3_sysfs_init(dnbd3_device_t *dev) +{ + int i; + char name[] = "alt_server99"; + struct kobject *kobj = &dev->kobj; + struct kobj_type *ktype = &device_ktype; + struct kobject *parent = &disk_to_dev(dev->disk)->kobj; + + kobject_init_and_add(kobj, ktype, parent, "net"); + + for (i = 0; i < NUMBER_SERVERS; i++) + { + sprintf(name, "alt_server%d", i); + kobj = &dev->alt_servers[i].kobj; + ktype = &server_ktype; + parent = &dev->kobj; + kobject_init_and_add(kobj, ktype, parent, name); + } +} + +void dnbd3_sysfs_exit(dnbd3_device_t *dev) +{ + int i; + + for (i = 0; i < NUMBER_SERVERS; i++) + { + kobject_put(&dev->alt_servers[i].kobj); + } + + kobject_put(&dev->kobj); +} diff --git a/src/kernel/sysfs.h b/src/kernel/sysfs.h new file mode 100644 index 0000000..09a053f --- /dev/null +++ b/src/kernel/sysfs.h @@ -0,0 +1,31 @@ +/* + * This file is part of the Distributed Network Block Device 3 + * + * Copyright(c) 2011-2012 Johann Latocha + * + * This file may be licensed under the terms of of the + * GNU General Public License Version 2 (the ``GPL''). + * + * Software distributed under the License is distributed + * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either + * express or implied. See the GPL for the specific language + * governing rights and limitations. + * + * You should have received a copy of the GPL along with this + * program. If not, go to http://www.gnu.org/licenses/gpl.html + * or write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +#ifndef SYSFS_H_ +#define SYSFS_H_ + +#include "dnbd3.h" + +void dnbd3_sysfs_init(dnbd3_device_t *dev); + +void dnbd3_sysfs_exit(dnbd3_device_t *dev); + + +#endif /* SYSFS_H_ */ -- cgit v1.2.3-55-g7522