io: make io_close_taken_fd() unset nonblocking on the fd.

This is what users want, and expect: as demonstrated by the test failure
when not under valgrind!
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

ccan/io/backend.h

@@ -73,6 +73,7 @@ bool add_listener(struct io_listener *l);
 bool add_conn(struct io_conn *c);
 bool add_duplex(struct io_conn *c);
 void del_listener(struct io_listener *l);
+void cleanup_conn_without_close(struct io_conn *c);
 void backend_new_always(struct io_conn *conn);
 void backend_new_plan(struct io_conn *conn);
 void remove_from_always(struct io_conn *conn);

ccan/io/io.c

@@ -421,6 +421,14 @@ struct io_plan *io_close_cb(struct io_conn *conn, void *next_arg)
 	return io_close(conn);
 }
 
+struct io_plan *io_close_taken_fd(struct io_conn *conn)
+{
+	set_blocking(conn->fd.fd, true);
+
+	cleanup_conn_without_close(conn);
+	return io_close(conn);
+}
+
 /* Exit the loop, returning this (non-NULL) arg. */
 void io_break(const void *ret)
 {

ccan/io/io.h

@@ -630,12 +630,14 @@ struct io_plan *io_close_cb(struct io_conn *, void *unused);
 
 /**
  * io_close_taken_fd - close a connection, but remove the filedescriptor first.
- * @conn: the connection to take the file descriptor from and close,
+ * @conn: the connection to take the file descriptor from and close.
  *
  * io_close closes the file descriptor underlying the io_conn; this version does
  * not.  Presumably you have used io_conn_fd() on it beforehand and will take
  * care of the fd yourself.
  *
+ * Note that this also turns off O_NONBLOCK on the fd.
+ *
  * Example:
  * static struct io_plan *steal_fd(struct io_conn *conn, int *fd)
  * {
@@ -664,7 +666,10 @@ void *io_loop(struct timers *timers, struct timer **expired);
  * io_conn_fd - get the fd from a connection.
  * @conn: the connection.
  *
- * Sometimes useful, eg for getsockname().
+ * Sometimes useful, eg for getsockname().  Note that the fd is O_NONBLOCK.
+ *
+ * See Also:
+ *	io_close_taken_fd
  */
 int io_conn_fd(const struct io_conn *conn);
 

ccan/io/poll.c

@@ -187,11 +187,10 @@ bool add_conn(struct io_conn *c)
 	return true;
 }
 
-struct io_plan *io_close_taken_fd(struct io_conn *conn)
+void cleanup_conn_without_close(struct io_conn *conn)
 {
 	tal_del_destructor(conn, destroy_conn_close_fd);
 	destroy_conn(conn, false);
-	return io_close(conn);
 }
 
 static void accept_conn(struct io_listener *l)