MDEV-26574 An improper locking bug due to unreleased lock in the ds_xbstream.cc

release lock in all as cases n xbstream_open, also fix the case where malloc would return NULL.

extra/mariabackup/ds_xbstream.cc

@@ -126,15 +126,19 @@ xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
 	pthread_mutex_lock(&stream_ctxt->mutex);
 	if (stream_ctxt->dest_file == NULL) {
 		stream_ctxt->dest_file = ds_open(dest_ctxt, path, mystat);
-		if (stream_ctxt->dest_file == NULL) {
-			return NULL;
-		}
 	}
 	pthread_mutex_unlock(&stream_ctxt->mutex);
+	if (stream_ctxt->dest_file == NULL) {
+		return NULL;
+	}
 
 	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
 				       sizeof(ds_stream_file_t),
 				       MYF(MY_FAE));
+	if (!file) {
+		msg("my_malloc() failed.");
+		goto err;
+	}
 	stream_file = (ds_stream_file_t *) (file + 1);
 
 	xbstream = stream_ctxt->xbstream;