mem-memcpy.c 9.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 * mem-memcpy.c
 *
 * memcpy: Simple memory copy in various ways
 *
 * Written by Hitoshi Mitake <mitake@dcl.info.waseda.ac.jp>
 */

#include "../perf.h"
#include "../util/util.h"
#include "../util/parse-options.h"
#include "../util/header.h"
13
#include "../util/cloexec.h"
14
#include "bench.h"
15
#include "mem-memcpy-arch.h"
16
#include "mem-memset-arch.h"
17 18 19 20 21 22 23 24 25

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <errno.h>

#define K 1024

26 27
static const char	*length_str	= "1MB";
static const char	*routine	= "default";
28
static int		iterations	= 1;
29 30
static bool		use_cycle;
static int		cycle_fd;
31 32
static bool		only_prefault;
static bool		no_prefault;
33 34 35 36

static const struct option options[] = {
	OPT_STRING('l', "length", &length_str, "1MB",
		    "Specify length of memory to copy. "
37
		    "Available units: B, KB, MB, GB and TB (upper and lower)"),
38 39
	OPT_STRING('r', "routine", &routine, "default",
		    "Specify routine to copy"),
40 41
	OPT_INTEGER('i', "iterations", &iterations,
		    "repeat memcpy() invocation this number of times"),
42
	OPT_BOOLEAN('c', "cycle", &use_cycle,
43
		    "Use cycles event instead of gettimeofday() for measuring"),
44 45 46 47
	OPT_BOOLEAN('o', "only-prefault", &only_prefault,
		    "Show only the result with page faults before memcpy()"),
	OPT_BOOLEAN('n', "no-prefault", &no_prefault,
		    "Show only the result without page faults before memcpy()"),
48 49 50
	OPT_END()
};

51
typedef void *(*memcpy_t)(void *, const void *, size_t);
52
typedef void *(*memset_t)(void *, int, size_t);
53

54 55 56
struct routine {
	const char *name;
	const char *desc;
57 58
	union {
		memcpy_t memcpy;
59
		memset_t memset;
60
	} fn;
61 62
};

63 64 65 66
struct routine memcpy_routines[] = {
	{ .name = "default",
	  .desc = "Default memcpy() provided by glibc",
	  .fn.memcpy = memcpy },
67
#ifdef HAVE_ARCH_X86_64_SUPPORT
68

69
#define MEMCPY_FN(_fn, _name, _desc) {.name = _name, .desc = _desc, .fn.memcpy = _fn},
70 71 72 73 74
#include "mem-memcpy-x86-64-asm-def.h"
#undef MEMCPY_FN

#endif

75 76
	{ NULL,
	  NULL,
77
	  {NULL}   }
78 79 80 81 82 83 84
};

static const char * const bench_mem_memcpy_usage[] = {
	"perf bench mem memcpy <options>",
	NULL
};

85
static struct perf_event_attr cycle_attr = {
86 87
	.type		= PERF_TYPE_HARDWARE,
	.config		= PERF_COUNT_HW_CPU_CYCLES
88 89
};

90
static void init_cycle(void)
91
{
92 93
	cycle_fd = sys_perf_event_open(&cycle_attr, getpid(), -1, -1,
				       perf_event_open_cloexec_flag());
94

95
	if (cycle_fd < 0 && errno == ENOSYS)
96 97
		die("No CONFIG_PERF_EVENTS=y kernel support configured?\n");
	else
98
		BUG_ON(cycle_fd < 0);
99 100
}

101
static u64 get_cycle(void)
102 103 104 105
{
	int ret;
	u64 clk;

106
	ret = read(cycle_fd, &clk, sizeof(u64));
107 108 109 110 111 112 113 114 115 116 117
	BUG_ON(ret != sizeof(u64));

	return clk;
}

static double timeval2double(struct timeval *ts)
{
	return (double)ts->tv_sec +
		(double)ts->tv_usec / (double)1000000;
}

118 119 120 121 122 123 124 125 126 127 128 129 130
#define pf (no_prefault ? 0 : 1)

#define print_bps(x) do {					\
		if (x < K)					\
			printf(" %14lf B/Sec", x);		\
		else if (x < K * K)				\
			printf(" %14lfd KB/Sec", x / K);	\
		else if (x < K * K * K)				\
			printf(" %14lf MB/Sec", x / K / K);	\
		else						\
			printf(" %14lf GB/Sec", x / K / K / K); \
	} while (0)

131 132 133 134 135 136 137 138 139 140
struct bench_mem_info {
	const struct routine *routines;
	u64 (*do_cycle)(const struct routine *r, size_t len, bool prefault);
	double (*do_gettimeofday)(const struct routine *r, size_t len, bool prefault);
	const char *const *usage;
};

static int bench_mem_common(int argc, const char **argv,
		     const char *prefix __maybe_unused,
		     struct bench_mem_info *info)
141 142
{
	int i;
143 144
	size_t len;
	double result_bps[2];
145
	u64 result_cycle[2];
146 147

	argc = parse_options(argc, argv, options,
148
			     info->usage, 0);
149

150 151 152 153 154
	if (no_prefault && only_prefault) {
		fprintf(stderr, "Invalid options: -o and -n are mutually exclusive\n");
		return 1;
	}

155 156
	if (use_cycle)
		init_cycle();
157 158

	len = (size_t)perf_atoll((char *)length_str);
159

160
	result_cycle[0] = result_cycle[1] = 0ULL;
161 162 163
	result_bps[0] = result_bps[1] = 0.0;

	if ((s64)len <= 0) {
164 165 166 167
		fprintf(stderr, "Invalid length:%s\n", length_str);
		return 1;
	}

168 169 170 171
	/* same to without specifying either of prefault and no-prefault */
	if (only_prefault && no_prefault)
		only_prefault = no_prefault = false;

172 173
	for (i = 0; info->routines[i].name; i++) {
		if (!strcmp(info->routines[i].name, routine))
174 175
			break;
	}
176
	if (!info->routines[i].name) {
177 178
		printf("Unknown routine:%s\n", routine);
		printf("Available routines...\n");
179
		for (i = 0; info->routines[i].name; i++) {
180
			printf("\t%s ... %s\n",
181
			       info->routines[i].name, info->routines[i].desc);
182 183 184 185
		}
		return 1;
	}

186 187
	if (bench_format == BENCH_FORMAT_DEFAULT)
		printf("# Copying %s Bytes ...\n\n", length_str);
188

189 190
	if (!only_prefault && !no_prefault) {
		/* show both of results */
191 192
		if (use_cycle) {
			result_cycle[0] =
193
				info->do_cycle(&info->routines[i], len, false);
194
			result_cycle[1] =
195
				info->do_cycle(&info->routines[i], len, true);
196 197
		} else {
			result_bps[0] =
198
				info->do_gettimeofday(&info->routines[i],
199 200
						len, false);
			result_bps[1] =
201
				info->do_gettimeofday(&info->routines[i],
202 203
						len, true);
		}
204
	} else {
205 206
		if (use_cycle) {
			result_cycle[pf] =
207
				info->do_cycle(&info->routines[i],
208 209 210
						len, only_prefault);
		} else {
			result_bps[pf] =
211
				info->do_gettimeofday(&info->routines[i],
212 213
						len, only_prefault);
		}
214 215 216 217
	}

	switch (bench_format) {
	case BENCH_FORMAT_DEFAULT:
218
		if (!only_prefault && !no_prefault) {
219 220 221
			if (use_cycle) {
				printf(" %14lf Cycle/Byte\n",
					(double)result_cycle[0]
222
					/ (double)len);
223 224
				printf(" %14lf Cycle/Byte (with prefault)\n",
					(double)result_cycle[1]
225 226 227 228 229 230
					/ (double)len);
			} else {
				print_bps(result_bps[0]);
				printf("\n");
				print_bps(result_bps[1]);
				printf(" (with prefault)\n");
231
			}
232
		} else {
233 234 235
			if (use_cycle) {
				printf(" %14lf Cycle/Byte",
					(double)result_cycle[pf]
236 237 238 239 240
					/ (double)len);
			} else
				print_bps(result_bps[pf]);

			printf("%s\n", only_prefault ? " (with prefault)" : "");
241 242 243
		}
		break;
	case BENCH_FORMAT_SIMPLE:
244
		if (!only_prefault && !no_prefault) {
245
			if (use_cycle) {
246
				printf("%lf %lf\n",
247 248
					(double)result_cycle[0] / (double)len,
					(double)result_cycle[1] / (double)len);
249 250 251 252 253
			} else {
				printf("%lf %lf\n",
					result_bps[0], result_bps[1]);
			}
		} else {
254 255
			if (use_cycle) {
				printf("%lf\n", (double)result_cycle[pf]
256 257 258 259
					/ (double)len);
			} else
				printf("%lf\n", result_bps[pf]);
		}
260 261
		break;
	default:
262 263
		/* reaching this means there's some disaster: */
		die("unknown format: %d\n", bench_format);
264 265 266 267 268
		break;
	}

	return 0;
}
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341

static void memcpy_alloc_mem(void **dst, void **src, size_t length)
{
	*dst = zalloc(length);
	if (!*dst)
		die("memory allocation failed - maybe length is too large?\n");

	*src = zalloc(length);
	if (!*src)
		die("memory allocation failed - maybe length is too large?\n");
	/* Make sure to always replace the zero pages even if MMAP_THRESH is crossed */
	memset(*src, 0, length);
}

static u64 do_memcpy_cycle(const struct routine *r, size_t len, bool prefault)
{
	u64 cycle_start = 0ULL, cycle_end = 0ULL;
	void *src = NULL, *dst = NULL;
	memcpy_t fn = r->fn.memcpy;
	int i;

	memcpy_alloc_mem(&src, &dst, len);

	if (prefault)
		fn(dst, src, len);

	cycle_start = get_cycle();
	for (i = 0; i < iterations; ++i)
		fn(dst, src, len);
	cycle_end = get_cycle();

	free(src);
	free(dst);
	return cycle_end - cycle_start;
}

static double do_memcpy_gettimeofday(const struct routine *r, size_t len,
				     bool prefault)
{
	struct timeval tv_start, tv_end, tv_diff;
	memcpy_t fn = r->fn.memcpy;
	void *src = NULL, *dst = NULL;
	int i;

	memcpy_alloc_mem(&src, &dst, len);

	if (prefault)
		fn(dst, src, len);

	BUG_ON(gettimeofday(&tv_start, NULL));
	for (i = 0; i < iterations; ++i)
		fn(dst, src, len);
	BUG_ON(gettimeofday(&tv_end, NULL));

	timersub(&tv_end, &tv_start, &tv_diff);

	free(src);
	free(dst);
	return (double)((double)len / timeval2double(&tv_diff));
}

int bench_mem_memcpy(int argc, const char **argv,
		     const char *prefix __maybe_unused)
{
	struct bench_mem_info info = {
		.routines = memcpy_routines,
		.do_cycle = do_memcpy_cycle,
		.do_gettimeofday = do_memcpy_gettimeofday,
		.usage = bench_mem_memcpy_usage,
	};

	return bench_mem_common(argc, argv, prefix, &info);
}
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428

static void memset_alloc_mem(void **dst, size_t length)
{
	*dst = zalloc(length);
	if (!*dst)
		die("memory allocation failed - maybe length is too large?\n");
}

static u64 do_memset_cycle(const struct routine *r, size_t len, bool prefault)
{
	u64 cycle_start = 0ULL, cycle_end = 0ULL;
	memset_t fn = r->fn.memset;
	void *dst = NULL;
	int i;

	memset_alloc_mem(&dst, len);

	if (prefault)
		fn(dst, -1, len);

	cycle_start = get_cycle();
	for (i = 0; i < iterations; ++i)
		fn(dst, i, len);
	cycle_end = get_cycle();

	free(dst);
	return cycle_end - cycle_start;
}

static double do_memset_gettimeofday(const struct routine *r, size_t len,
				     bool prefault)
{
	struct timeval tv_start, tv_end, tv_diff;
	memset_t fn = r->fn.memset;
	void *dst = NULL;
	int i;

	memset_alloc_mem(&dst, len);

	if (prefault)
		fn(dst, -1, len);

	BUG_ON(gettimeofday(&tv_start, NULL));
	for (i = 0; i < iterations; ++i)
		fn(dst, i, len);
	BUG_ON(gettimeofday(&tv_end, NULL));

	timersub(&tv_end, &tv_start, &tv_diff);

	free(dst);
	return (double)((double)len / timeval2double(&tv_diff));
}

static const char * const bench_mem_memset_usage[] = {
	"perf bench mem memset <options>",
	NULL
};

static const struct routine memset_routines[] = {
	{ .name ="default",
	  .desc = "Default memset() provided by glibc",
	  .fn.memset = memset },
#ifdef HAVE_ARCH_X86_64_SUPPORT

#define MEMSET_FN(_fn, _name, _desc) { .name = _name, .desc = _desc, .fn.memset = _fn },
#include "mem-memset-x86-64-asm-def.h"
#undef MEMSET_FN

#endif

	{ .name = NULL,
	  .desc = NULL,
	  .fn.memset = NULL   }
};

int bench_mem_memset(int argc, const char **argv,
		     const char *prefix __maybe_unused)
{
	struct bench_mem_info info = {
		.routines = memset_routines,
		.do_cycle = do_memset_cycle,
		.do_gettimeofday = do_memset_gettimeofday,
		.usage = bench_mem_memset_usage,
	};

	return bench_mem_common(argc, argv, prefix, &info);
}