| [vhffs-dev] [1276] The client now ask to server to receive a full view. |
[ Thread Index |
Date Index
| More vhffs.org/vhffs-dev Archives
]
Revision: 1276
Author: gradator
Date: 2008-10-11 23:33:04 +0200 (Sat, 11 Oct 2008)
Log Message:
-----------
The client now ask to server to receive a full view. The server is sending the fullview using a non-blocking recursive loop (implemented by a rollback/recover mechanism).
Modified Paths:
--------------
trunk/vhffs-fssync/vhffsfssync_master.c
trunk/vhffs-fssync/vhffsfssync_slave.c
Modified: trunk/vhffs-fssync/vhffsfssync_master.c
===================================================================
--- trunk/vhffs-fssync/vhffsfssync_master.c 2008-10-11 12:33:44 UTC (rev 1275)
+++ trunk/vhffs-fssync/vhffsfssync_master.c 2008-10-11 21:33:04 UTC (rev 1276)
@@ -2,7 +2,7 @@
#define DEBUG_NET 0
#define DEBUG_INOTIFY 0
-#define DEBUG_EVENTS 1
+#define DEBUG_EVENTS 0
#include <unistd.h>
#include <stdio.h>
@@ -25,6 +25,7 @@
#include <netinet/in.h>
#include <sys/sendfile.h>
+/* TODO: timestamp check */
/* -- inotify stuff -- */
@@ -84,6 +85,10 @@
char *buf_cur;
GList *messages;
+
+ GList *fullviewtree;
+ GList *fullviewcur;
+ int fullviewtimerset;
} vhffsfssync_conn;
@@ -176,11 +181,15 @@
int vhffsfssync_net_send(vhffsfssync_conn *conn);
int vhffsfssync_net_recv_event(vhffsfssync_conn *conn, char *event);
int vhffsfssync_net_parse(vhffsfssync_conn *conn);
+int vhffsfssync_net_fullview(vhffsfssync_conn *conn, char *pathname);
+void vhffsfssync_net_fullview_alarmsignal(int signo);
/* ----------------------------------------- */
+
+/* -- network stuff -- */
void vhffsfssync_net_conn_disable(vhffsfssync_conn *conn) {
- GList *msgs;
+ GList *msgs, *lst, *lst2;
if(conn->fd >= 0) {
#if DEBUG_NET
@@ -194,6 +203,16 @@
while( (msgs = g_list_first(conn->messages)) ) {
vhffsfssync_net_destroy_message(conn, (vhffsfssync_net_message*)msgs->data );
}
+
+ conn->fullviewcur = NULL;
+ conn->fullviewtimerset = 0;
+ while( (lst = g_list_first(conn->fullviewtree)) ) {
+ while( (lst2 = g_list_first(lst->data)) ) {
+ free(lst2->data);
+ lst->data = g_list_delete_link(lst->data, lst2);
+ }
+ conn->fullviewtree = g_list_delete_link(conn->fullviewtree, lst);
+ }
}
@@ -204,7 +223,6 @@
}
-/* -- network stuff -- */
inline vhffsfssync_net_message *vhffsfssync_net_new_message(vhffsfssync_conn *conn, msg_family_t family, uint32_t priority) {
vhffsfssync_net_message *msg;
msg = malloc( sizeof(vhffsfssync_net_message) );
@@ -691,15 +709,20 @@
int vhffsfssync_net_recv_event(vhffsfssync_conn *conn, char *event) {
- char *cur, *args[10];
- int argc;
+ char *cur, **args = NULL;
+ int argalloc = 0, argc = 0;
- argc = 0;
do {
for(cur = event ; *cur++ != '\0' ; );
+
+ if(argc >= argalloc) {
+ argalloc = ( (argc >>8) +1) <<8;
+ args = realloc( args, argalloc * sizeof(char*) );
+ }
+
args[argc++] = event;
event = cur;
- } while(*event && argc < 10);
+ } while(*event);
if(!argc) return -1;
#if DEBUG_EVENTS
@@ -710,9 +733,45 @@
printf("\n");
#endif
- if(!strcmp(args[0], "fulltree")) {
+ if(!strcmp(args[0], "get")) {
+ char *pathname = args[1];
+ struct stat st;
+
+ printf("> %s\n", pathname);
+
+ if(! lstat(pathname, &st) ) {
+
+ if( S_ISREG(st.st_mode) ) {
+ vhffsfssync_net_send_file(conn, pathname);
+ }
+ else if( S_ISDIR(st.st_mode) ) {
+ vhffsfssync_net_send_event(conn, g_strdup_printf("mkdir%c%s%c", '\0', pathname, '\0') , VHFFSFSSYNC_NET_PRIO_MEDIUM);
+ }
+ else if( S_ISLNK(st.st_mode) ) {
+ char *linkto;
+ linkto = malloc(st.st_size +1);
+ if( readlink(pathname, linkto, st.st_size) >= 0 ) {
+ linkto[st.st_size] = '\0';
+ vhffsfssync_net_send_event(conn, g_strdup_printf("symlink%c%s%c%s%c", '\0', pathname, '\0', linkto, '\0') , VHFFSFSSYNC_NET_PRIO_MEDIUM);
+ }
+ free(linkto);
+ }
+ /* we don't need other file types (chr, block, fifo, socket, ...) */
+ }
+ }
+ else if(!strcmp(args[0], "fulltree")) {
// the client requested a full tree of all available files
+ vhffsfssync_net_fullview(conn, ".");
+/* GList *lst;
+ for( lst = g_list_first(conn->fullviewtree) ; lst ; lst = g_list_next(lst) ) {
+ GList *lst2;
+ printf("DEPTH..... N = %d\n", g_list_length(g_list_first(lst->data)));
+ for(lst2 = g_list_first(lst->data) ; lst2 ; lst2 = g_list_next(lst2) ) {
+ printf("DIR........ %s\n", (char*)lst2->data);
+ }
+ }
+*/
}
else if(!strcmp(args[0], "hello")) {
// nice to meet you
@@ -721,6 +780,7 @@
fprintf(stderr, "conn %d, received unhandled event: %s\n", conn->fd, args[0]);
}
+ free(args);
return 0;
}
@@ -767,6 +827,182 @@
}
+int vhffsfssync_net_fullview(vhffsfssync_conn *conn, char *pathname) {
+
+ /*
+ * We need to keep the state of recursivity across runs
+ *
+ * Only directories are interesting, so we store the current
+ * directories tree in linked lists
+ *
+ * The format is as follow:
+ * list[
+ * ["./usr", "./bin", "./boot", "./dev", "./etc", "./home", "./lib", "./home", ...],
+ * ["./usr/include", "./usr/bin", "./usr/sbin", "./usr/lib", "./usr/man", ...],
+ * ["./usr/include/linux", "./usr/include/sys", "./usr/include/bits", "./usr/include/netinet", ...],
+ * ["./usr/include/linux/netfilter", "./usr/include/linux/usb", "./usr/include/linux/sunrpc" , ...]
+ * ];
+ *
+ * The first entry of each list is the current entry, so here it is:
+ * /usr/include/linux/netfilter/
+ *
+ * Entries are deleted when everything is done
+ *
+ * (Yes, I write comments when I am brainstorming)
+ */
+
+ GList *ldirs = NULL;
+
+ if(pathname) {
+ DIR *d;
+
+ //printf("LOOKUP: %s\n", pathname);
+
+ d = opendir(pathname);
+ if(d) {
+ struct dirent *dir;
+ GString *msg = g_string_sized_new(1024);
+ g_string_append_printf(msg, "ls%c%s%c", '\0', pathname, '\0');
+
+ while( (dir = readdir(d)) ) {
+ if( strcmp(dir->d_name, ".") && strcmp(dir->d_name, "..") ) {
+ char *path = g_strdup_printf("%s/%s", pathname, dir->d_name);
+
+ if(dir->d_type == DT_DIR) {
+ // register a new directory
+ ldirs = g_list_append(ldirs, path);
+ g_string_append_printf(msg, "%s%cdir%c0%c0%c", dir->d_name, '\0', '\0', '\0', '\0');
+ }
+ else {
+ // file or link
+ struct stat st;
+ if(! lstat(path, &st) ) {
+ if( S_ISREG(st.st_mode) ) {
+ g_string_append_printf(msg, "%s%cfile%c%lld%c%ld%c", dir->d_name, '\0', '\0', st.st_size, '\0', st.st_mtime, '\0');
+ }
+ else if( S_ISLNK(st.st_mode) ) {
+ g_string_append_printf(msg, "%s%clink%c%lld%c%ld%c", dir->d_name, '\0', '\0', st.st_size, '\0', st.st_mtime, '\0');
+ }
+ /* we don't need other file types (chr, block, fifo, socket, ...) */
+ }
+ else {
+ fprintf(stderr, "cannot lstat() '%s': %s\n", pathname, strerror(errno));
+ }
+ free(path);
+ }
+ }
+ }
+ closedir(d);
+
+ vhffsfssync_net_send_event(conn, g_string_free(msg, FALSE) , VHFFSFSSYNC_NET_PRIO_MEDIUM);
+
+ // any subdirectories ?
+ if(ldirs) {
+ // register the list to the pseudo-tree
+ conn->fullviewtree = g_list_append(conn->fullviewtree, g_list_last(ldirs));
+ //printf("Depth: %d, entries here = %d\n", g_list_length(conn->fullviewtree), g_list_length(ldirs));
+ }
+ }
+ }
+ // recover recursion
+ else {
+ GList *lst;
+ if(!conn->fullviewcur) {
+ lst = g_list_first(conn->fullviewtree);
+ }
+ else {
+ lst = g_list_next(conn->fullviewcur);
+ }
+ if(lst) {
+ ldirs = g_list_first(lst->data);
+ //printf("RECOVERING: %s\n", (char*)ldirs->data);
+ if(g_list_next(lst)) {
+ conn->fullviewcur = lst;
+ }
+ else {
+ //printf("RECOVERED\n");
+ conn->fullviewcur = NULL;
+ conn->fullviewtimerset = 0;
+ }
+ }
+ }
+
+ //printf("Depth: %d, entries here = %d\n", g_list_length(conn->fullviewtree), g_list_length(ldirs));
+
+ // subdirs ?
+ if(ldirs) {
+
+ while( (ldirs = g_list_first(ldirs)) ) {
+ int r;
+ char *path = ldirs->data;
+
+ printf("%s\n", path);
+
+ // we can cancel the recursion here at anytime
+// if( rand() < (RAND_MAX/10)*7 ) {
+ if(conn->fullviewtimerset > 0) {
+ // do nothing, this is here because it matchs most of the time
+ }
+ else if(conn->fullviewtimerset < 0) {
+ printf("CANCELLED\n");
+ conn->fullviewcur = NULL;
+ conn->fullviewtimerset = 0;
+ return 1;
+ }
+ else {
+ struct sigaction act;
+ act.sa_handler = vhffsfssync_net_fullview_alarmsignal;
+ sigemptyset(&act.sa_mask);
+ act.sa_flags = SA_RESETHAND;
+ sigaction(SIGALRM, &act, NULL);
+ alarm(1);
+ conn->fullviewtimerset = 1;
+ }
+
+ //printf("A: %p\n", conn->fullviewcur);
+ if(!conn->fullviewcur) {
+ r = vhffsfssync_net_fullview(conn, path);
+ }
+ else {
+ r = vhffsfssync_net_fullview(conn, NULL);
+ //printf("B: %p\n", conn->fullviewcur);
+ }
+
+ // rollback
+ if(r) {
+ //printf("Depth: %d, entries here = %d\n", g_list_length(conn->fullviewtree), g_list_length(ldirs));
+ //printf("ROLLBACK: %s\n", path);
+ return r;
+ }
+
+ //printf("DELETE %s\n", path);
+ ldirs = g_list_delete_link(ldirs, ldirs);
+ free(path);
+
+ //printf("Depth: %d, entries here = %d\n", g_list_length(conn->fullviewtree), g_list_length(ldirs));
+ }
+ conn->fullviewtree = g_list_delete_link(conn->fullviewtree, g_list_last(conn->fullviewtree) );
+ }
+
+ if(!conn->fullviewtree) {
+ printf("CLEAN UP SUCCESSFULL\n");
+ }
+ return 0;
+}
+
+
+void vhffsfssync_net_fullview_alarmsignal(int signo) {
+
+ GList *conns;
+ for(conns = g_list_first(vhffsfssync_conns) ; conns ; ) {
+ vhffsfssync_conn *conn = conns->data;
+ conns = g_list_next(conns);
+ if( conn->fullviewtimerset > 0)
+ conn->fullviewtimerset = -1;
+ }
+}
+
+
/* -- inotify stuff -- */
int vhffsfssync_add_watch(int inotifyfd, const char *pathname, uint32_t mask) {
@@ -865,19 +1101,17 @@
int wd;
DIR *d;
- struct dirent *dir;
wd = vhffsfssync_add_watch(inotifyfd, pathname, mask);
if(wd < 0) return wd;
d = opendir(pathname);
if(d) {
+ struct dirent *dir;
while( (dir = readdir(d)) ) {
if(dir->d_type == DT_DIR && strcmp(dir->d_name, ".") && strcmp(dir->d_name, "..") ) {
char *path;
- char *sep = "/";
- if( pathname[strlen(pathname)-1] == '/' ) sep = "";
- path = g_strdup_printf("%s%s%s", pathname, sep, dir->d_name);
+ path = g_strdup_printf("%s/%s", pathname, dir->d_name);
wd = vhffsfssync_add_watch_recursively(inotifyfd, path, mask);
free(path);
}
@@ -921,13 +1155,15 @@
#if DEBUG_INOTIFY
printf("==> CREATE %s\n", pathname);
#endif
- vhffsfssync_net_broadcast_event( g_strdup_printf("create%c%s%c", '\0', pathname, '\0') , VHFFSFSSYNC_NET_PRIO_MEDIUM);
- if(sendfile && st.st_size > 0) {
+ if(!sendfile) {
+ vhffsfssync_net_broadcast_event( g_strdup_printf("create%c%s%c", '\0', pathname, '\0') , VHFFSFSSYNC_NET_PRIO_MEDIUM);
+ }
+ else {
vhffsfssync_net_broadcast_file(pathname);
}
}
- if( S_ISDIR(st.st_mode) ) {
+ else if( S_ISDIR(st.st_mode) ) {
#if DEBUG_INOTIFY
printf("==> MKDIR %s\n", pathname);
#endif
@@ -1122,18 +1358,15 @@
int vhffsfssync_fake_events_recursively(int inotifyfd, char *pathname) {
DIR *d;
- struct dirent *dir;
d = opendir(pathname);
if(d) {
+ struct dirent *dir;
while( (dir = readdir(d)) ) {
if( strcmp(dir->d_name, ".") && strcmp(dir->d_name, "..") ) {
char *path;
- char *sep = "/";
+ path = g_strdup_printf("%s/%s", pathname, dir->d_name);
- if( pathname[strlen(pathname)-1] == '/' ) sep = "";
- path = g_strdup_printf("%s%s%s", pathname, sep, dir->d_name);
-
// recursivity is done through vhffsfssync_manage_event_create()
// which calls this function
vhffsfssync_manage_event_create(inotifyfd, path, TRUE);
@@ -1158,17 +1391,24 @@
uint16_t bindport;
struct sockaddr_in src;
+ srand(time(NULL));
/* chdir() to the filesystem to monitor */
root = argv[1];
- if( chdir(root) < 0) {
+ if(!root) return -1;
+ if( root[strlen(root)-1] == '/' ) root[strlen(root)-1] = '\0';
+#if DEBUG_INOTIFY
+ printf("Monitoring %s\n", root);
+#endif
+ if( chdir(root) < 0 ) {
fprintf(stderr, "cannot chdir() to %s: %s\n", root, strerror(errno));
return -1;
}
+ if( chroot(".") < 0 ) {
+ fprintf(stderr, "cannot chroot() to %s: %s\n", root, strerror(errno));
+ //return -1;
+ }
root = ".";
-#if DEBUG_INOTIFY
- printf("Monitoring %s\n", root);
-#endif
/* -- inotify stuff -- */
@@ -1269,7 +1509,7 @@
}
FD_SET(conn->fd, &readfs);
- if(conn->messages) FD_SET(conn->fd, &writefs);
+ if(conn->messages || conn->fullviewtree) FD_SET(conn->fd, &writefs);
if(conn->fd > max_fd) max_fd = conn->fd;
}
@@ -1360,6 +1600,9 @@
conn->buf_len = 0;
conn->buf_cur = conn->buf;
conn->messages = NULL;
+ conn->fullviewtree = NULL;
+ conn->fullviewcur = NULL;
+ conn->fullviewtimerset = 0;
vhffsfssync_conns = g_list_append(vhffsfssync_conns, conn);
#if DEBUG_NET
printf("Welcome %s ! (using fd %d)\n", inet_ntoa(conn->sockaddr.sin_addr), conn->fd);
@@ -1429,6 +1672,11 @@
if(conn->messages && FD_ISSET(conn->fd, &writefs) ) {
vhffsfssync_net_send(conn);
}
+
+ /* continue the fullview if needed */
+ if(conn->fullviewtree && FD_ISSET(conn->fd, &writefs) ) {
+ vhffsfssync_net_fullview(conn, NULL);
+ }
}
}
}
Modified: trunk/vhffs-fssync/vhffsfssync_slave.c
===================================================================
--- trunk/vhffs-fssync/vhffsfssync_slave.c 2008-10-11 12:33:44 UTC (rev 1275)
+++ trunk/vhffs-fssync/vhffsfssync_slave.c 2008-10-11 21:33:04 UTC (rev 1276)
@@ -1,8 +1,8 @@
#define _FILE_OFFSET_BITS 64
#define _ATFILE_SOURCE
-#define DEBUG_NET 1
-#define DEBUG_EVENTS 1
+#define DEBUG_NET 0
+#define DEBUG_EVENTS 0
#include <unistd.h>
#include <stdio.h>
@@ -26,7 +26,8 @@
/* -- network stuff -- */
// huge buffer size reduce syscalls
-#define VHFFSFSSYNC_NET_RECV_BUF_LEN 262144
+/* TODO: fullview require a HUGE buffer, change that to a dynamically allocated buffer */
+#define VHFFSFSSYNC_NET_RECV_BUF_LEN 5242880
typedef struct {
int fd;
@@ -57,6 +58,7 @@
// events protos
int vhffsfssync_remove(char *pathname);
int vhffsfssync_mkdir(char *pathname, mode_t mode);
+int vhffsfssync_checkfile(vhffsfssync_conn *conn, char *path, char *type, int long long size, int long long mtime);
int vhffsfssync_event(vhffsfssync_conn *conn, char *event);
int vhffsfssync_parse(vhffsfssync_conn *conn);
@@ -200,6 +202,7 @@
}
+// the content of pathname is modified
int vhffsfssync_mkdir(char *pathname, mode_t mode) {
char *cur, *dirs[64];
int i, fd, fd_, argc;
@@ -227,20 +230,78 @@
}
}
if(fd >= 0) close(fd);
+ if(i != argc) return -1;
return 0;
}
+int vhffsfssync_checkfile(vhffsfssync_conn *conn, char *path, char *type, int long long size, int long long mtime) {
+ gboolean fetch = FALSE;
+ struct stat st;
+
+ //printf("%s\n", path);
+
+ // file or link
+ if(! lstat(path, &st) ) {
+ if(! S_ISDIR(st.st_mode) ) {
+ char *type_ = "unknown";
+
+ if( S_ISREG(st.st_mode) ) {
+ type_ = "file";
+ }
+
+ else if( S_ISLNK(st.st_mode) ) {
+ type_ = "link";
+ }
+
+ else
+ /* we don't need other file types (chr, block, fifo, socket, ...) */
+
+ if(strcmp(type, type_)) {
+ vhffsfssync_remove(path);
+ fetch = TRUE;
+ }
+ else if(st.st_size != size) {
+ fetch = TRUE;
+ }
+ else if(st.st_mtime < mtime) {
+ fetch = TRUE;
+ }
+ }
+ }
+ else {
+ if(errno == ENOENT) {
+ fetch = TRUE;
+ }
+ else {
+ fprintf(stderr, "cannot lstat() '%s': %s\n", path, strerror(errno));
+ return -1;
+ }
+ }
+
+ if(fetch) {
+ vhffsfssync_net_send_event(conn, g_strdup_printf("get%c%s%c", '\0', path, '\0') );
+ }
+
+ return 0;
+}
+
+
int vhffsfssync_event(vhffsfssync_conn *conn, char *event) {
- char *cur, *args[10];
- int argc;
+ char *cur, **args = NULL;
+ int argalloc = 0, argc = 0;
- argc = 0;
do {
for(cur = event ; *cur++ != '\0' ; );
+
+ if(argc >= argalloc) {
+ argalloc = ( (argc >>8) +1) <<8;
+ args = realloc( args, argalloc * sizeof(char*) );
+ }
+
args[argc++] = event;
event = cur;
- } while(*event && argc < 10);
+ } while(*event);
if(!argc) return -1;
#if DEBUG_EVENTS
@@ -320,6 +381,49 @@
}
}
}
+ else if(!strcmp(args[0], "ls")) {
+ char *root, *root_;
+ root = args[1];
+ root_ = strdup(root);
+
+ if(! vhffsfssync_mkdir(root, 0755) ) {
+ int i;
+ GHashTable *filesindex;
+ DIR *d;
+
+ // build an index with all files
+ filesindex = g_hash_table_new_full(g_str_hash, g_str_equal, NULL, NULL);
+
+ // check each file
+ for( i = 2 ; i < argc ; i+=4 ) {
+ char *path;
+ g_hash_table_insert(filesindex, args[i], args[i]);
+
+ path = g_strdup_printf("%s/%s", root_, args[i]);
+ vhffsfssync_checkfile(conn, path, args[i+1], atoll(args[i+2]), atoll(args[i+3]) );
+ free(path);
+ }
+
+ // check for removed files
+ d = opendir(root_);
+ if(d) {
+ struct dirent *dir;
+ while( (dir = readdir(d)) ) {
+ if(strcmp(dir->d_name, ".") && strcmp(dir->d_name, "..") ) {
+ if(! g_hash_table_lookup(filesindex, dir->d_name) ) {
+ char *path = g_strdup_printf("%s/%s", root_, dir->d_name);
+ //printf("Deleting %s\n", path);
+ vhffsfssync_remove(path);
+ free(path);
+ }
+ }
+ }
+ closedir(d);
+ }
+ g_hash_table_destroy(filesindex);
+ }
+ free(root_);
+ }
else if(!strcmp(args[0], "hello")) {
// nice to meet you
}
@@ -328,6 +432,7 @@
exit(1);
}
+ free(args);
return 0;
}
@@ -420,10 +525,14 @@
/* chdir() to the filesystem to write the data */
root = argv[1];
- if( chdir(root) < 0) {
+ if( chdir(root) < 0 ) {
fprintf(stderr, "cannot chdir() to %s: %s\n", root, strerror(errno));
return -1;
}
+ if( chroot(".") < 0 ) {
+ fprintf(stderr, "cannot chroot() to %s: %s\n", root, strerror(errno));
+ //return -1;
+ }
root = ".";
/* -- network stuff -- */