[KERNEL] SysFS interface added (/sys/block/dnbd*/net/)

6 files changed, 327 insertions, 57 deletions

diff --git a/src/config.h b/src/config.h
index 77caf11..8a6cdcf 100644
--- a/src/config.h
+++ b/src/config.h
@@ -28,7 +28,7 @@
 #define SOCKET_TIMEOUT_SERVER 30
 #define SOCKET_TIMEOUT_CLIENT_DATA 5
 #define SOCKET_TIMEOUT_CLIENT_DISCOVERY 1
-#define TIMER_INTERVAL_HEARTBEAT 2*HZ
+#define TIMER_INTERVAL_HEARTBEAT 10*HZ
 #define TIMER_INTERVAL_PANIC 1*HZ
 #define NUMBER_SERVERS 8
 
diff --git a/src/kernel/blk.c b/src/kernel/blk.c
index 192c3db..70d4e1f 100644
--- a/src/kernel/blk.c
+++ b/src/kernel/blk.c
@@ -20,6 +20,7 @@
 
 #include "blk.h"
 #include "net.h"
+#include "sysfs.h"
 
 int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
 {
@@ -32,10 +33,15 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
     INIT_LIST_HEAD(&dev->request_queue_send);
     INIT_LIST_HEAD(&dev->request_queue_receive);
 
+    memset(dev->cur_server.host, 0, 16);
+    memset(dev->cur_server.port, 0, 6);
+    dev->cur_server.rtt = 0;
+    dev->cur_server.sock = NULL;
+
     dev->vid = 0;
     dev->rid = 0;
-    dev->sock = NULL;
-    dev->num_servers = 0;
+    dev->alt_servers_num = 0;
+    memset(dev->alt_servers, 0, sizeof(dnbd3_server_t)*NUMBER_SERVERS);
     dev->thread_send = NULL;
     dev->thread_receive = NULL;
     dev->thread_discover = NULL;
@@ -68,12 +74,14 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
     queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue);
     dev->disk = disk;
 
-    add_disk(disk); // must be last
+    add_disk(disk);
+    dnbd3_sysfs_init(dev);
     return 0;
 }
 
 int dnbd3_blk_del_device(dnbd3_device_t *dev)
 {
+	dnbd3_sysfs_exit(dev);
 	dnbd3_net_disconnect(dev);
     del_gendisk(dev->disk);
     put_disk(dev->disk);
@@ -95,8 +103,8 @@ int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, u
     switch (cmd)
     {
     case IOCTL_OPEN:
-        strcpy(dev->host, msg->host);
-        strcpy(dev->port, msg->port);
+        strcpy(dev->cur_server.host, msg->host);
+        strcpy(dev->cur_server.port, msg->port);
         dev->vid = msg->vid;
         dev->rid = msg->rid;
         dnbd3_net_connect(dev);
@@ -110,7 +118,7 @@ int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, u
 
     case IOCTL_SWITCH:
         dnbd3_net_disconnect(dev);
-        strcpy(dev->host, msg->host);
+        strcpy(dev->cur_server.host, msg->host);
         dnbd3_net_connect(dev);
         break;
 
diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h
index 140754c..febd473 100644
--- a/src/kernel/dnbd3.h
+++ b/src/kernel/dnbd3.h
@@ -33,22 +33,31 @@ extern int major;
 
 typedef struct
 {
+    char host[16];
+    char port[6];
+    uint64_t rtt;
+    struct socket *sock;
+    struct kobject kobj;
+} dnbd3_server_t;
+
+typedef struct
+{
     // block
     struct gendisk *disk;
     spinlock_t blk_lock;
 
+    // sysfs
+    struct kobject kobj;
+
     // network
-    char host[16];
-    char port[6];
-    int vid;
-    int rid;
-    struct socket *sock;
-    struct timer_list hb_timer;
-    int num_servers;
-    struct in_addr servers[NUMBER_SERVERS];
+    dnbd3_server_t cur_server;
+    int vid, rid;
+    int alt_servers_num;
+    dnbd3_server_t alt_servers[NUMBER_SERVERS];
     int discover, panic;
 
     // process
+    struct timer_list hb_timer;
     struct task_struct *thread_send;
     struct task_struct *thread_receive;
     struct task_struct *thread_discover;
diff --git a/src/kernel/net.c b/src/kernel/net.c
index 8ec34d0..762d0d8 100644
--- a/src/kernel/net.c
+++ b/src/kernel/net.c
@@ -39,36 +39,36 @@ void dnbd3_net_connect(dnbd3_device_t *dev)
         printk("ERROR: Kmalloc failed.\n");
         return;
     }
-    if (!dev->host || !dev->port || (dev->vid == 0))
+    if (!dev->cur_server.host || !dev->cur_server.port || (dev->vid == 0))
     {
         printk("ERROR: Host, port or vid not set.\n");
         return;
     }
-    if (dev->sock)
+    if (dev->cur_server.sock)
     {
-        printk("ERROR: Device %s already connected to %s.\n", dev->disk->disk_name, dev->host);
+        printk("ERROR: Device %s already connected to %s.\n", dev->disk->disk_name, dev->cur_server.host);
         return;
     }
 
-    printk("INFO: Connecting device %s to %s\n", dev->disk->disk_name, dev->host);
+    printk("INFO: Connecting device %s to %s\n", dev->disk->disk_name, dev->cur_server.host);
 
     // initialize socket
-    if (sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &dev->sock) < 0)
+    if (sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &dev->cur_server.sock) < 0)
     {
         printk("ERROR: Couldn't create socket.\n");
-        dev->sock = NULL;
+        dev->cur_server.sock = NULL;
         return;
     }
-    kernel_setsockopt(dev->sock, SOL_SOCKET, SO_SNDTIMEO, (char *) &timeout, sizeof(timeout));
-    kernel_setsockopt(dev->sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &timeout, sizeof(timeout));
-    dev->sock->sk->sk_allocation = GFP_NOIO;
+    kernel_setsockopt(dev->cur_server.sock, SOL_SOCKET, SO_SNDTIMEO, (char *) &timeout, sizeof(timeout));
+    kernel_setsockopt(dev->cur_server.sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &timeout, sizeof(timeout));
+    dev->cur_server.sock->sk->sk_allocation = GFP_NOIO;
     sin.sin_family = AF_INET;
-    sin.sin_addr.s_addr = inet_addr(dev->host);
-    sin.sin_port = htons(simple_strtol(dev->port, NULL, 10));
-    if (kernel_connect(dev->sock, (struct sockaddr *) &sin, sizeof(sin), 0) < 0)
+    sin.sin_addr.s_addr = inet_addr(dev->cur_server.host);
+    sin.sin_port = htons(simple_strtol(dev->cur_server.port, NULL, 10));
+    if (kernel_connect(dev->cur_server.sock, (struct sockaddr *) &sin, sizeof(sin), 0) < 0)
     {
-        printk("ERROR: Couldn't connect to host %s:%s\n", dev->host, dev->port);
-        dev->sock = NULL;
+        printk("ERROR: Couldn't connect to host %s:%s\n", dev->cur_server.host, dev->cur_server.port);
+        dev->cur_server.sock = NULL;
         return;
     }
 
@@ -121,10 +121,10 @@ void dnbd3_net_disconnect(dnbd3_device_t *dev)
         kthread_stop(dev->thread_discover);
 
     // clear socket
-    if (dev->sock)
+    if (dev->cur_server.sock)
     {
-        sock_release(dev->sock);
-        dev->sock = NULL;
+        sock_release(dev->cur_server.sock);
+        dev->cur_server.sock = NULL;
     }
 }
 
@@ -191,7 +191,7 @@ int dnbd3_net_discover(void *data)
         if (!&dev->discover)
             continue;
 
-        num = dev->num_servers;
+        num = dev->alt_servers_num;
         dev->discover = 0;
         best_rtt = -1;
 
@@ -206,14 +206,15 @@ int dnbd3_net_discover(void *data)
             }
             kernel_setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *) &timeout, sizeof(timeout));
             kernel_setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &timeout, sizeof(timeout));
-            inet_ntoa(dev->servers[i], current_server);
+            strcpy(current_server, dev->alt_servers[i].host);
             sock->sk->sk_allocation = GFP_NOIO;
             sin.sin_family = AF_INET;
             sin.sin_addr.s_addr = inet_addr(current_server);
-            sin.sin_port = htons(simple_strtol(dev->port, NULL, 10));
+            sin.sin_port = htons(simple_strtol(dev->cur_server.port, NULL, 10));
             if (kernel_connect(sock, (struct sockaddr *) &sin, sizeof(sin), 0) < 0)
             {
-                printk("ERROR: Couldn't connect to host %s:%s (discover)\n", current_server, dev->port);
+                printk("ERROR: Couldn't connect to host %s:%s (discover)\n", current_server, dev->cur_server.port);
+                dev->alt_servers[i].rtt = -1;
                 sock = NULL;
                 continue;
             }
@@ -226,7 +227,7 @@ int dnbd3_net_discover(void *data)
                 kfree(buf);
                 dev->thread_discover = NULL;
                 dnbd3_net_disconnect(dev);
-                strcpy(dev->host, current_server);
+                strcpy(dev->cur_server.host, current_server);
                 dnbd3_net_connect(dev);
                 return 0;
             }
@@ -283,9 +284,11 @@ int dnbd3_net_discover(void *data)
 
             t1 = (start.tv_sec*1000000ull) + start.tv_usec;
             t2 = (end.tv_sec*1000000ull) + end.tv_usec;
-            if ( best_rtt > (t2 - t1) )
+            dev->alt_servers[i].rtt = t2 -t1;
+
+            if ( best_rtt > dev->alt_servers[i].rtt )
             {
-            	best_rtt = t2 - t1;
+            	best_rtt = dev->alt_servers[i].rtt;
             	strcpy(best_server, current_server);
             }
 
@@ -299,16 +302,21 @@ int dnbd3_net_discover(void *data)
         }
 
         // take server with lowest rtt
-        if (num > 1 && strcmp(dev->host, best_server))
+        if (num > 1 && strcmp(dev->cur_server.host, best_server))
         {
         	printk("INFO: Server %s is faster (%lluus), switching...\n", best_server, best_rtt);
         	kfree(buf);
             dev->thread_discover = NULL;
             dnbd3_net_disconnect(dev);
-            strcpy(dev->host, best_server);
+            strcpy(dev->cur_server.host, best_server);
+            dev->cur_server.rtt = best_rtt;
             dnbd3_net_connect(dev);
             return 0;
         }
+        else
+        {
+            dev->cur_server.rtt = best_rtt;
+        }
 
     }
     kfree(buf);
@@ -376,7 +384,7 @@ int dnbd3_net_send(void *data)
         memcpy(dnbd3_request.handle, &blk_request, sizeof(blk_request));
         iov.iov_base = &dnbd3_request;
         iov.iov_len = sizeof(dnbd3_request);
-        if (kernel_sendmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_request)) <= 0)
+        if (kernel_sendmsg(dev->cur_server.sock, &msg, &iov, 1, sizeof(dnbd3_request)) <= 0)
             goto error;
 
         // enqueue request to request_queue_receive
@@ -389,9 +397,9 @@ int dnbd3_net_send(void *data)
     return 0;
 
     error:
-        printk("ERROR: Connection to server %s lost (send)\n", dev->host);
-        if (dev->sock)
-            kernel_sock_shutdown(dev->sock, SHUT_RDWR);
+        printk("ERROR: Connection to server %s lost (send)\n", dev->cur_server.host);
+        if (dev->cur_server.sock)
+            kernel_sock_shutdown(dev->cur_server.sock, SHUT_RDWR);
         dev->thread_send = NULL;
         dev->panic = 1;
         return -1;
@@ -413,6 +421,7 @@ int dnbd3_net_receive(void *data)
 
     unsigned int size, i;
     uint64_t filesize;
+    struct in_addr tmp_addr;
 
     init_msghdr(msg);
     set_user_nice(current, -20);
@@ -428,7 +437,7 @@ int dnbd3_net_receive(void *data)
         // receive net replay
         iov.iov_base = &dnbd3_reply;
         iov.iov_len = sizeof(dnbd3_reply);
-        if (kernel_recvmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags) <= 0)
+        if (kernel_recvmsg(dev->cur_server.sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags) <= 0)
             goto error;
 
         // search for replied request in queue
@@ -449,9 +458,9 @@ int dnbd3_net_receive(void *data)
             list_del_init(&blk_request->queuelist);
             kthread_stop(dev->thread_send);
             del_timer(&dev->hb_timer);
-            sock_release(dev->sock);
+            sock_release(dev->cur_server.sock);
             kfree(blk_request);
-            dev->sock = NULL;
+            dev->cur_server.sock = NULL;
             return -1;
 
         case ERROR_RELOAD:
@@ -478,7 +487,7 @@ int dnbd3_net_receive(void *data)
                 size = bvec->bv_len;
                 iov.iov_base = kaddr;
                 iov.iov_len = size;
-                if (kernel_recvmsg(dev->sock, &msg, &iov, 1, size, msg.msg_flags) <= 0)
+                if (kernel_recvmsg(dev->cur_server.sock, &msg, &iov, 1, size, msg.msg_flags) <= 0)
                 {
                     kunmap(bvec->bv_page);
                     goto error;
@@ -496,7 +505,7 @@ int dnbd3_net_receive(void *data)
         case CMD_GET_SIZE:
             iov.iov_base = &filesize;
             iov.iov_len = sizeof(uint64_t);
-            if (kernel_recvmsg(dev->sock, &msg, &iov, 1, dnbd3_reply.size, msg.msg_flags) <= 0)
+            if (kernel_recvmsg(dev->cur_server.sock, &msg, &iov, 1, dnbd3_reply.size, msg.msg_flags) <= 0)
                 goto error;
             set_capacity(dev->disk, filesize >> 9); /* 512 Byte blocks */
             printk("INFO: Filesize %s: %llu\n", dev->disk->disk_name, filesize);
@@ -507,14 +516,15 @@ int dnbd3_net_receive(void *data)
             continue;
 
         case CMD_GET_SERVERS:
-            dev->num_servers = dnbd3_reply.size / sizeof(struct in_addr);
+            dev->alt_servers_num = dnbd3_reply.size / sizeof(struct in_addr);
             size = sizeof(struct in_addr);
-            for (i = 0; i < dev->num_servers && i < NUMBER_SERVERS; i++)
+            for (i = 0; i < dev->alt_servers_num && i < NUMBER_SERVERS; i++)
             {
-                iov.iov_base = &dev->servers[i];
+                iov.iov_base = &tmp_addr;
                 iov.iov_len = size;
-                if (kernel_recvmsg(dev->sock, &msg, &iov, 1, size, msg.msg_flags) <= 0)
+                if (kernel_recvmsg(dev->cur_server.sock, &msg, &iov, 1, size, msg.msg_flags) <= 0)
                     goto error;
+                inet_ntoa(tmp_addr, dev->alt_servers[i].host);
             }
             spin_lock_irq(&dev->blk_lock);
             list_del_init(&blk_request->queuelist);
@@ -534,7 +544,7 @@ int dnbd3_net_receive(void *data)
     return 0;
 
     error:
-        printk("ERROR: Connection to server %s lost (receive)\n", dev->host);
+        printk("ERROR: Connection to server %s lost (receive)\n", dev->cur_server.host);
         // move already send requests to request_queue_send again
         if (!list_empty(&dev->request_queue_receive))
         {
@@ -547,8 +557,8 @@ int dnbd3_net_receive(void *data)
             }
             spin_unlock_irq(&dev->blk_lock);
         }
-        if (dev->sock)
-            kernel_sock_shutdown(dev->sock, SHUT_RDWR);
+        if (dev->cur_server.sock)
+            kernel_sock_shutdown(dev->cur_server.sock, SHUT_RDWR);
         dev->thread_receive = NULL;
         dev->panic = 1;
         return -1;
diff --git a/src/kernel/sysfs.c b/src/kernel/sysfs.c
new file mode 100644
index 0000000..e363824
--- /dev/null
+++ b/src/kernel/sysfs.c
@@ -0,0 +1,212 @@
+/*
+ * This file is part of the Distributed Network Block Device 3
+ *
+ * Copyright(c) 2011-2012 Johann Latocha <johann@latocha.de>
+ *
+ * This file may be licensed under the terms of of the
+ * GNU General Public License Version 2 (the ``GPL'').
+ *
+ * Software distributed under the License is distributed
+ * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
+ * express or implied. See the GPL for the specific language
+ * governing rights and limitations.
+ *
+ * You should have received a copy of the GPL along with this
+ * program. If not, go to http://www.gnu.org/licenses/gpl.html
+ * or write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+#include <linux/kobject.h>
+
+#include "sysfs.h"
+#include "utils.h"
+
+typedef struct
+{
+    struct attribute attr;
+    ssize_t (*show)(char *, dnbd3_device_t *);
+    ssize_t (*store)(const char *, size_t, dnbd3_device_t *);
+} device_attr_t;
+
+typedef struct
+{
+    struct attribute attr;
+    ssize_t (*show)(char *, dnbd3_server_t *);
+    ssize_t (*store)(const char *, size_t, dnbd3_server_t *);
+} server_attr_t;
+
+
+ssize_t show_cur_server_ip(char *buf, dnbd3_device_t *dev)
+{
+    return sprintf(buf, "%s\n", dev->cur_server.host);
+}
+
+ssize_t show_cur_server_rtt(char *buf, dnbd3_device_t *dev)
+{
+    return sprintf(buf, "%lluus\n", dev->cur_server.rtt);
+}
+
+ssize_t show_alt_server_num(char *buf, dnbd3_device_t *dev)
+{
+    return sprintf(buf, "%d\n", dev->alt_servers_num);
+}
+
+ssize_t show_vid(char *buf, dnbd3_device_t *dev)
+{
+    return sprintf(buf, "%d\n", dev->vid);
+}
+
+ssize_t show_rid(char *buf, dnbd3_device_t *dev)
+{
+    return sprintf(buf, "%d\n", dev->rid);
+}
+
+ssize_t show_alt_server_ip(char *buf, dnbd3_server_t *srv)
+{
+    return sprintf(buf, "%s\n", srv->host);
+}
+
+ssize_t show_alt_server_rtt(char *buf, dnbd3_server_t *srv)
+{
+    return sprintf(buf, "%lluus\n", srv->rtt);
+}
+
+device_attr_t cur_server_ip =
+{
+    .attr = {.name = "cur_server_ip", .mode = 0444 },
+    .show   = show_cur_server_ip,
+    .store  = NULL,
+};
+
+device_attr_t cur_server_rtt =
+{
+    .attr = {.name = "cur_server_rtt", .mode = 0444 },
+    .show   = show_cur_server_rtt,
+    .store  = NULL,
+};
+
+device_attr_t alt_server_num =
+{
+    .attr = {.name = "alt_server_num", .mode = 0444 },
+    .show   = show_alt_server_num,
+    .store  = NULL,
+};
+
+device_attr_t vid =
+{
+    .attr = {.name = "vid", .mode = 0444 },
+    .show   = show_vid,
+    .store  = NULL,
+};
+
+device_attr_t rid =
+{
+    .attr = {.name = "rid", .mode = 0444 },
+    .show   = show_rid,
+    .store  = NULL,
+};
+
+server_attr_t alt_server_ip =
+{
+    .attr = {.name = "alt_server_ip", .mode = 0444 },
+    .show   = show_alt_server_ip,
+    .store  = NULL,
+};
+
+server_attr_t alt_server_rtt =
+{
+    .attr = {.name = "alt_server_rtt", .mode = 0444 },
+    .show   = show_alt_server_rtt,
+    .store  = NULL,
+};
+
+ssize_t device_show(struct kobject *kobj, struct attribute *attr, char *buf)
+{
+    device_attr_t *device_attr = container_of(attr, device_attr_t, attr);
+    dnbd3_device_t *dev = container_of(kobj, dnbd3_device_t, kobj);
+    return device_attr->show(buf, dev);
+}
+
+ssize_t server_show(struct kobject *kobj, struct attribute *attr, char *buf)
+{
+    server_attr_t *server_attr = container_of(attr, server_attr_t, attr);
+    dnbd3_server_t *srv = container_of(kobj, dnbd3_server_t, kobj);
+    return server_attr->show(buf, srv);
+}
+
+struct attribute *device_attrs[] =
+{
+    &cur_server_ip.attr,
+    &cur_server_rtt.attr,
+    &alt_server_num.attr,
+    &vid.attr,
+    &rid.attr,
+    NULL,
+};
+
+struct attribute *server_attrs[] =
+{
+    &alt_server_ip.attr,
+    &alt_server_rtt.attr,
+    NULL,
+};
+
+struct sysfs_ops device_ops =
+{
+    .show = device_show,
+};
+
+struct sysfs_ops server_ops =
+{
+    .show = server_show,
+};
+
+void release(struct kobject *kobj) {}
+
+struct kobj_type device_ktype =
+{
+    .default_attrs = device_attrs,
+    .sysfs_ops = &device_ops,
+    .release = release,
+};
+
+struct kobj_type server_ktype =
+{
+    .default_attrs = server_attrs,
+    .sysfs_ops = &server_ops,
+    .release = release,
+};
+
+void dnbd3_sysfs_init(dnbd3_device_t *dev)
+{
+    int i;
+    char name[] = "alt_server99";
+    struct kobject *kobj = &dev->kobj;
+    struct kobj_type *ktype = &device_ktype;
+    struct kobject *parent = &disk_to_dev(dev->disk)->kobj;
+
+    kobject_init_and_add(kobj, ktype, parent, "net");
+
+    for (i = 0; i < NUMBER_SERVERS; i++)
+    {
+        sprintf(name, "alt_server%d", i);
+        kobj = &dev->alt_servers[i].kobj;
+        ktype = &server_ktype;
+        parent = &dev->kobj;
+        kobject_init_and_add(kobj, ktype, parent, name);
+    }
+}
+
+void dnbd3_sysfs_exit(dnbd3_device_t *dev)
+{
+    int i;
+
+    for (i = 0; i < NUMBER_SERVERS; i++)
+    {
+        kobject_put(&dev->alt_servers[i].kobj);
+    }
+
+    kobject_put(&dev->kobj);
+}
diff --git a/src/kernel/sysfs.h b/src/kernel/sysfs.h
new file mode 100644
index 0000000..09a053f
--- /dev/null
+++ b/src/kernel/sysfs.h
@@ -0,0 +1,31 @@
+/*
+ * This file is part of the Distributed Network Block Device 3
+ *
+ * Copyright(c) 2011-2012 Johann Latocha <johann@latocha.de>
+ *
+ * This file may be licensed under the terms of of the
+ * GNU General Public License Version 2 (the ``GPL'').
+ *
+ * Software distributed under the License is distributed
+ * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
+ * express or implied. See the GPL for the specific language
+ * governing rights and limitations.
+ *
+ * You should have received a copy of the GPL along with this
+ * program. If not, go to http://www.gnu.org/licenses/gpl.html
+ * or write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+#ifndef SYSFS_H_
+#define SYSFS_H_
+
+#include "dnbd3.h"
+
+void dnbd3_sysfs_init(dnbd3_device_t *dev);
+
+void dnbd3_sysfs_exit(dnbd3_device_t *dev);
+
+
+#endif /* SYSFS_H_ */