diff --git a/.github/labeler.yml b/.github/labeler.yml index 6f53a8e376..de46af613f 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,7 +2,7 @@ topic/software-stack: - 'chapters/software-stack/**/*' topic/data: - - 'content/chapters/data/**/*' + - 'chapters/data/**/*' topic/compute: - 'content/chapters/compute/**/*' diff --git a/chapters/data/.gitignore b/chapters/data/.gitignore new file mode 100644 index 0000000000..e7ed234b22 --- /dev/null +++ b/chapters/data/.gitignore @@ -0,0 +1,2 @@ +/_site +/slides.md diff --git a/chapters/data/Makefile b/chapters/data/Makefile new file mode 100644 index 0000000000..12cc2a595f --- /dev/null +++ b/chapters/data/Makefile @@ -0,0 +1,33 @@ +RVMD = reveal-md +MDPP = markdown-pp +FFMPEG = ffmpeg + +SLIDES ?= slides.mdpp +SLIDES_OUT ?= slides.md +MEDIA_DIR ?= media +SITE ?= _site +OPEN ?= xdg-open + +.PHONY: all html clean videos + +all: videos html + +html: $(SITE) + +$(SITE): $(SLIDES) + $(MDPP) $< -o $(SLIDES_OUT) + $(RVMD) $(SLIDES_OUT) --static $@ + +videos: + for TARGET in $(TARGETS); do \ + $(FFMPEG) -framerate 0.5 -f image2 -y \ + -i "$(MEDIA_DIR)/$$TARGET/$$TARGET-%d.svg" -vf format=yuv420p $(MEDIA_DIR)/$$TARGET-generated.gif; \ + done + +open: $(SITE) + $(OPEN) $ +#include +#include +#include +#include +#include + +#include "utils.h" +#include "access_counter.h" + +struct page_info { + int prot; + void *start; +}; + +unsigned long counter; + +#define MAX_PAGES 16 +static struct page_info pages[MAX_PAGES]; +static size_t num_pages; + +/* This is useful for filling the contents when execution rights are provided. */ +static void do_nothing(void) +{ +} + +/* + * The actual SIGSEVG handler. + */ +static void access_handler(int signum, siginfo_t *si, void *arg) +{ + long page_size = sysconf(_SC_PAGESIZE); + void *start; + int rc; + unsigned int i; + + log_debug("Enter handler"); + + counter++; + + if (signum != SIGSEGV) { + fprintf(stderr, "Unable to handle signal %d (%s)\n", signum, strsignal(signum)); + return; + } + + /* TODO 2: Obtain page strart address in start variable. */ + start = (void *) ((unsigned long) si->si_addr & ~0xFFFUL); + log_debug("start: %p", start); + + for (i = 0; i < num_pages; i++) + if (pages[i].start == start) + break; + + if (i >= num_pages && i < MAX_PAGES) { + pages[i].start = start; + pages[i].prot = PROT_NONE; + num_pages += 1; + } + + log_debug("i = %u", i); + + /* TODO 21: Update page proctections with mprotect(). */ + switch (pages[i].prot) { + case PROT_NONE: + rc = mprotect(start, page_size, PROT_READ); + DIE(rc < 0, "mprotect"); + pages[i].prot = PROT_READ; + break; + case PROT_READ: + rc = mprotect(start, page_size, PROT_READ | PROT_WRITE); + DIE(rc < 0, "mprotect"); + pages[i].prot = PROT_WRITE; + break; + case PROT_WRITE: + rc = mprotect(start, page_size, PROT_READ | PROT_WRITE | PROT_EXEC); + DIE(rc < 0, "mprotect"); + /* Optimistically copy 64 bytes of code. Should be more than enough. */ + memcpy(start, do_nothing, 64); + pages[i].prot = PROT_EXEC; + break; + default: + break; + } +} + +void register_sigsegv_handler(void) +{ + struct sigaction sa; + int rc; + + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = access_handler; + sa.sa_flags = SA_SIGINFO; + + rc = sigaction(SIGSEGV, &sa, NULL); + DIE(rc < 0, "sigaction"); +} diff --git a/chapters/data/arena/drills/tasks/access-counter/solution/src/access_counter.h b/chapters/data/arena/drills/tasks/access-counter/solution/src/access_counter.h new file mode 100644 index 0000000000..02c74b9545 --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/solution/src/access_counter.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef ACCESS_PRINTER_H_ +#define ACCESS_PRINTER_H_ 1 + +extern unsigned long counter; +void register_sigsegv_handler(void); + +#endif diff --git a/chapters/data/arena/drills/tasks/access-counter/solution/utils/log/CPPLINT.cfg b/chapters/data/arena/drills/tasks/access-counter/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/arena/drills/tasks/access-counter/solution/utils/log/log.c b/chapters/data/arena/drills/tasks/access-counter/solution/utils/log/log.c new file mode 100644 index 0000000000..a55e75cd10 --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/solution/utils/log/log.c @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char * const level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) +{ + char buf[16]; + +buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) +{ + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) +{ + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) +{ + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) +{ + L.level = level; +} + + +void log_set_quiet(bool enable) +{ + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/arena/drills/tasks/access-counter/solution/utils/log/log.h b/chapters/data/arena/drills/tasks/access-counter/solution/utils/log/log.h new file mode 100644 index 0000000000..9849385b0e --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/solution/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/arena/drills/tasks/access-counter/solution/utils/sock/sock_util.c b/chapters/data/arena/drills/tasks/access-counter/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/access-counter/solution/utils/sock/sock_util.h b/chapters/data/arena/drills/tasks/access-counter/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/arena/drills/tasks/access-counter/solution/utils/utils.h b/chapters/data/arena/drills/tasks/access-counter/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/arena/drills/tasks/access-counter/support/src/.gitignore b/chapters/data/arena/drills/tasks/access-counter/support/src/.gitignore new file mode 100644 index 0000000000..0c00620a8b --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/src/.gitignore @@ -0,0 +1 @@ +/access_counter diff --git a/chapters/data/arena/drills/tasks/access-counter/support/src/Makefile b/chapters/data/arena/drills/tasks/access-counter/support/src/Makefile new file mode 100644 index 0000000000..f467a82d82 --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/src/Makefile @@ -0,0 +1,14 @@ +CFLAGS ?= -Wall -Wextra +CPPFLAGS ?= -I../utils + +.PHONY: all clean + +all: access_counter.o ../utils/log/log.o + +access_counter.o: access_counter.c ../utils/utils.h ../utils/log/log.h + +../utils/log/log.o: ../utils/log/log.c ../utils/log/log.h + +clean: + -rm -f access_counter.o ../utils/log/log.o + -rm -f *~ diff --git a/chapters/data/arena/drills/tasks/access-counter/support/src/access_counter.c b/chapters/data/arena/drills/tasks/access-counter/support/src/access_counter.c new file mode 100644 index 0000000000..7f177fe7ea --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/src/access_counter.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include + +#include "utils.h" +#include "access_counter.h" + +struct page_info { + int prot; + void *start; +}; + +unsigned long counter; + +#define MAX_PAGES 16 +static struct page_info pages[MAX_PAGES]; +static size_t num_pages; + +/* This is useful for filling the contents when execution rights are provided. */ +static void do_nothing(void) +{ +} + +/* + * The actual SIGSEGV handler. + */ +static void access_handler(int signum, siginfo_t *si, void *arg) +{ + long page_size = sysconf(_SC_PAGESIZE); + void *start; + int rc; + unsigned int i; + + log_debug("Enter handler"); + + counter++; + + if (signum != SIGSEGV) { + fprintf(stderr, "Unable to handle signal %d (%s)\n", signum, strsignal(signum)); + return; + } + + /* TODO: Obtain page start address in start variable. */ + + for (i = 0; i < num_pages; i++) + if (pages[i].start == start) + break; + + if (i >= num_pages && i < MAX_PAGES) { + pages[i].start = start; + pages[i].prot = PROT_NONE; + num_pages += 1; + } + + log_debug("i = %u", i); + + /* TODO: Update page protections with mprotect(). */ +} + +void register_sigsegv_handler(void) +{ + struct sigaction sa; + int rc; + + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = access_handler; + sa.sa_flags = SA_SIGINFO; + + rc = sigaction(SIGSEGV, &sa, NULL); + DIE(rc < 0, "sigaction"); +} diff --git a/chapters/data/arena/drills/tasks/access-counter/support/src/access_counter.h b/chapters/data/arena/drills/tasks/access-counter/support/src/access_counter.h new file mode 100644 index 0000000000..02c74b9545 --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/src/access_counter.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef ACCESS_PRINTER_H_ +#define ACCESS_PRINTER_H_ 1 + +extern unsigned long counter; +void register_sigsegv_handler(void); + +#endif diff --git a/chapters/data/arena/drills/tasks/access-counter/support/tests/.gitignore b/chapters/data/arena/drills/tasks/access-counter/support/tests/.gitignore new file mode 100644 index 0000000000..ee4c926823 --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/tests/.gitignore @@ -0,0 +1 @@ +/test diff --git a/chapters/data/arena/drills/tasks/access-counter/support/tests/Makefile b/chapters/data/arena/drills/tasks/access-counter/support/tests/Makefile new file mode 100644 index 0000000000..ee1604b7f3 --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/tests/Makefile @@ -0,0 +1,44 @@ +SRC_PATH ?= ../src +FULL_SRC_PATH = $(realpath $(SRC_PATH)) +CPPFLAGS = -I. -I$(realpath $(SRC_PATH)) -I../utils +CFLAGS = -Wall -Wextra +# Remove the line below to disable debugging support. +CFLAGS += -g -O0 + +SRCS = $(wildcard test*.c) +OBJS = $(patsubst %.c,%.o,$(SRCS)) +EXECS = $(patsubst %.c,%,$(SRCS)) + + +.PHONY: all src check lint clean + +all: src $(EXECS) + +$(EXECS): %:%.o graded_test.o $(SRC_PATH)/access_counter.o ../utils/log/log.o | src + $(CC) $(LDFLAGS) -o $@ $^ $(LDLIBS) + +$(OBJS): %.o:%.c graded_test.h + +graded_test.o: graded_test.c graded_test.h + +../utils/log/log.o: ../utils/log/log.c ../utils/log/log.h + +src: + make -C $(FULL_SRC_PATH) + +check: + make -C $(FULL_SRC_PATH) clean + make clean + make -i SRC_PATH=$(FULL_SRC_PATH) + ./run_all_tests.sh + +lint: + -cd .. && checkpatch.pl -f src/*.c tests/*.c tests/*/*.c + -cd .. && checkpatch.pl -f checker/*.sh tests/*.sh + -cd .. && cpplint --recursive src/ tests/ checker/ + -cd .. && shellcheck checker/*.sh tests/*.sh + +clean: + -rm -f *~ + -rm -f graded_test.o $(OBJS) + -rm -f $(EXECS) diff --git a/chapters/data/arena/drills/tasks/access-counter/support/tests/graded_test.c b/chapters/data/arena/drills/tasks/access-counter/support/tests/graded_test.c new file mode 100644 index 0000000000..1eb03d1001 --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/tests/graded_test.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include + +#include "./graded_test.h" + +static void my_itoa(size_t num, char *a) +{ + unsigned char digit3; + unsigned char digit2; + unsigned char digit1; + + /* Print at most 3 decimal digits. */ + if (num > 999) + num = 999; + + digit1 = num % 10; + num /= 10; + digit2 = num % 10; + num /= 10; + digit3 = num % 10; + + if (digit3 == 0) + a[0] = ' '; + else + a[0] = '0' + digit3; + + if (digit2 == 0) + a[1] = ' '; + else + a[1] = '0' + digit2; + + a[2] = '0' + digit1; +} + +/* + * Print test result. Printed message should fit in 72 characters. + * + * Print format is: + * + * description ...................... passed ... NNN + * description ...................... failed ... NNN + * 32 chars 24 chars 6 3 3 + */ + +static void print_test(const char *description, int result, size_t points) +{ + /* Make these global linkage, so it's only allocated once. */ + static char print_buffer[74]; + static const char failed[] = "failed"; + static const char passed[] = "passed"; + size_t i; + size_t len; + + /* Collect description in print_buffer. */ + len = MIN(strlen(description), 32); + for (i = 0; i < len; i++) + print_buffer[i] = description[i]; + for (i = len; i < 32; i++) + print_buffer[i] = ' '; + print_buffer[32] = ' '; + + /* Collect dots in print_buffer. */ + for (i = 0; i < 24; i++) + print_buffer[33+i] = '.'; + print_buffer[57] = ' '; + + /* Collect passed / failed. */ + for (i = 0; i < 6; i++) { + if (result == 1) + print_buffer[58+i] = passed[i]; + else + print_buffer[58+i] = failed[i]; + } + print_buffer[64] = ' '; + + /* Collect dots in print_buffer. */ + for (i = 0; i < 3; i++) + print_buffer[65+i] = '.'; + print_buffer[68] = ' '; + + /* Collect number. */ + if (result == 1) { + my_itoa(points, &print_buffer[69]); + } else { + print_buffer[69] = ' '; + print_buffer[70] = ' '; + print_buffer[71] = '0'; + } + + /* Collect newline. */ + print_buffer[72] = '\n'; + + write(1, print_buffer, 73); +} + +void run_test(struct graded_test *test) +{ + int res; + + res = test->function(); + print_test(test->description, res, test->points); +#ifdef EXIT_IF_FAIL + exit(EXIT_FAILURE); +#endif +} + +void run_tests(struct graded_test *tests, size_t count) +{ + size_t i; + + for (i = 0; i < count; i++) + run_test(&tests[i]); +} diff --git a/chapters/data/arena/drills/tasks/access-counter/support/tests/graded_test.h b/chapters/data/arena/drills/tasks/access-counter/support/tests/graded_test.h new file mode 100644 index 0000000000..f56f2e07ca --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/tests/graded_test.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef GRADED_TEST_H_ +#define GRADED_TEST_H_ 1 + +/* test function prototype */ +typedef int (*test_f)(void); + +struct graded_test { + test_f function; /* test/evaluation function */ + char *description; /* test description */ + size_t points; /* points for each test */ +}; + +void run_test(struct graded_test *test); +void run_tests(struct graded_test *tests, size_t count); + +#endif /* GRADED_TEST_H_ */ diff --git a/chapters/data/arena/drills/tasks/access-counter/support/tests/run_all_tests.sh b/chapters/data/arena/drills/tasks/access-counter/support/tests/run_all_tests.sh new file mode 100755 index 0000000000..c0228aff91 --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/tests/run_all_tests.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +./test | tee results.txt + +total=$(grep '\( passed \| failed \)' results.txt | rev | cut -d ' ' -f 1 | rev | paste -s -d'+' | bc) +echo "" +echo -n "Total: " +echo -n " " +LC_ALL=C printf "%3d/100\n" "$total" + +rm results.txt diff --git a/chapters/data/arena/drills/tasks/access-counter/support/tests/test.c b/chapters/data/arena/drills/tasks/access-counter/support/tests/test.c new file mode 100644 index 0000000000..ada3c926ad --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/tests/test.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include + +#include "access_counter.h" +#include "utils.h" +#include "graded_test.h" + +static long page_size; + +static void do_read(const void *addr) +{ + char a; + + a = *(char *) addr; +} + +static void do_write(void *addr) +{ + *(char *) addr = 'a'; +} + +static void do_exec(void *addr) +{ + ((void (*)(void)) addr)(); +} + +static int test_acess_read(void) +{ + void *addr; + + addr = mmap(NULL, page_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + DIE(addr == MAP_FAILED, "mmap"); + + counter = 0; + do_read(addr); + + return counter == 1; +} + +static int test_acess_write(void) +{ + void *addr; + + addr = mmap(NULL, page_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + DIE(addr == MAP_FAILED, "mmap"); + + counter = 0; + do_write(addr); + + return counter == 2; +} + +static int test_acess_exec(void) +{ + void *addr; + + addr = mmap(NULL, page_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + DIE(addr == MAP_FAILED, "mmap"); + + counter = 0; + do_exec(addr); + + return counter == 3; +} + +static int test_acess_read_write(void) +{ + void *addr; + + addr = mmap(NULL, page_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + DIE(addr == MAP_FAILED, "mmap"); + + do_read(addr); + counter = 0; + do_write(addr); + + return counter == 1; +} + +static int test_acess_read_exec(void) +{ + void *addr; + + addr = mmap(NULL, page_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + DIE(addr == MAP_FAILED, "mmap"); + + do_read(addr); + counter = 0; + do_exec(addr); + + return counter == 2; +} + +static int test_acess_write_exec(void) +{ + void *addr; + + addr = mmap(NULL, page_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + DIE(addr == MAP_FAILED, "mmap"); + + do_write(addr); + counter = 0; + do_exec(addr); + + return counter == 1; +} + +static int test_acess_exec_read(void) +{ + void *addr; + + addr = mmap(NULL, page_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + DIE(addr == MAP_FAILED, "mmap"); + + do_exec(addr); + counter = 0; + do_read(addr); + + return counter == 0; +} + +static int test_acess_exec_write(void) +{ + void *addr; + + addr = mmap(NULL, page_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + DIE(addr == MAP_FAILED, "mmap"); + + do_exec(addr); + counter = 0; + do_write(addr); + + return counter == 0; +} + +static int test_acess_write_read(void) +{ + void *addr; + + addr = mmap(NULL, page_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + DIE(addr == MAP_FAILED, "mmap"); + + do_write(addr); + counter = 0; + do_read(addr); + + return counter == 0; +} + +static struct graded_test all_tests[] = { + { test_acess_read, "test_acess_read", 9 }, + { test_acess_write, "test_acess_write", 9 }, + { test_acess_exec, "test_acess_exec", 10 }, + { test_acess_read_write, "test_acess_read_write", 12 }, + { test_acess_read_exec, "test_acess_read_exec", 12 }, + { test_acess_write_exec, "test_acess_write_exec", 12 }, + { test_acess_exec_read, "test_acess_exec_read", 12 }, + { test_acess_exec_write, "test_acess_exec_write", 12 }, + { test_acess_write_read, "test_acess_write_read", 12 } +}; + +int main(void) +{ + page_size = sysconf(_SC_PAGESIZE); + + register_sigsegv_handler(); + + /* Update to LOG_DEBUG to print debug messages. */ + log_set_level(LOG_INFO); + + run_tests(all_tests, sizeof(all_tests) / sizeof(all_tests[0])); + + return 0; +} + diff --git a/chapters/data/arena/drills/tasks/access-counter/support/utils/log/CPPLINT.cfg b/chapters/data/arena/drills/tasks/access-counter/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/arena/drills/tasks/access-counter/support/utils/log/log.c b/chapters/data/arena/drills/tasks/access-counter/support/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/arena/drills/tasks/access-counter/support/utils/log/log.h b/chapters/data/arena/drills/tasks/access-counter/support/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/arena/drills/tasks/access-counter/support/utils/sock/sock_util.c b/chapters/data/arena/drills/tasks/access-counter/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/access-counter/support/utils/sock/sock_util.h b/chapters/data/arena/drills/tasks/access-counter/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/arena/drills/tasks/access-counter/support/utils/utils.h b/chapters/data/arena/drills/tasks/access-counter/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/access-counter/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/README.md b/chapters/data/arena/drills/tasks/exec-shellcode/README.md new file mode 100644 index 0000000000..ee81665f87 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/README.md @@ -0,0 +1,35 @@ +## Shellcode Executor + +Navigate to the `support/exec-shellcode/` directory. + +Your goal is to update the `src/exec-shellcode.s` source code file to be able to read and execute shellcodes from a given binary files. +The program thus acts as a shellcode tester. + +A [shellcode](https://cocomelonc.github.io/tutorial/2021/10/09/linux-shellcoding-1.html) is a small program that is commonly used in memory-related security exploits as a form of arbitrary code execution. +It's a binary string consisting of instructions / code to be directly interpreted by the CPU during the execution of the targeted vulnerable program. + +Shellcodes end up in an `exit()` system call to ensure a graceful exit of the program after running the shellcode. +Use `mmap()` to reserve a virtual page. +Use anonymous mapping (i.e. the `MAP_ANONYMOUS`) flag. +Use the proper permissions required to enable the shellcode to be read from the file into memory and then executed. + +To test the implementation, enter the `tests/` directory and run: + +```console +make check +``` + +As an extra item, add a shellcode for the `brk()` system call in the `tests/brk.asm` file. +It should be a simple shellcode that calls `brk(NULL)`, i.e. with the purpose of getting the current program break. + +In case of a correct solution, you will get an output such as: + +```text +./run_all_tests.sh +test_helloworld ........................ passed ... 25 +test_getpid ........................ passed ... 25 +test_openfile ........................ passed ... 25 +test_brk ........................ passed ... 25 + +Total: 100/100 +``` diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/solution/src/.gitignore b/chapters/data/arena/drills/tasks/exec-shellcode/solution/src/.gitignore new file mode 100644 index 0000000000..5b678bc248 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/solution/src/.gitignore @@ -0,0 +1 @@ +/exec_shellcode diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/solution/src/Makefile b/chapters/data/arena/drills/tasks/exec-shellcode/solution/src/Makefile new file mode 100644 index 0000000000..52e057a177 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/solution/src/Makefile @@ -0,0 +1,14 @@ +CFLAGS ?= -Wall -Wextra +CPPFLAGS ?= -I../utils + +.PHONY: all clean + +all: exec_shellcode + +exec_shellcode: exec_shellcode.o ../utils/log/log.o + +../utils/log/log.o: ../utils/log/log.c ../utils/log/log.h + +clean: + -rm -f exec_shellcode exec_shellcode + -rm -f *~ diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/solution/src/exec_shellcode.c b/chapters/data/arena/drills/tasks/exec-shellcode/solution/src/exec_shellcode.c new file mode 100644 index 0000000000..534de35e86 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/solution/src/exec_shellcode.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include + +#include "utils.h" + +static void *shellcode_mapping; + +static void usage(const char * const argv0) +{ + fprintf(stderr, "Usage: %s shellcode_file\n", argv0); +} + +static void create_shellcode_mapping(void) +{ + /* TODO 4: Create mapping to fit the shellcode. */ + shellcode_mapping = mmap(NULL, sysconf(_SC_PAGESIZE), + PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + DIE(shellcode_mapping == MAP_FAILED, "mmap"); +} + +static void read_shellcode(const char * const fname) +{ + /* TODO 8: Read content from file in shellcode. */ + FILE *f; + + f = fopen(fname, "rb"); + DIE(f == NULL, "fopen"); + + fread(shellcode_mapping, sysconf(_SC_PAGESIZE), 1, f); + + fclose(f); +} + +int main(int argc, char **argv) +{ + if (argc != 2) { + usage(argv[0]); + exit(EXIT_FAILURE); + } + + create_shellcode_mapping(); + read_shellcode(argv[1]); + + ((void (*)(void)) shellcode_mapping)(); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/solution/tests/brk.asm b/chapters/data/arena/drills/tasks/exec-shellcode/solution/tests/brk.asm new file mode 100644 index 0000000000..ae15f2cc68 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/solution/tests/brk.asm @@ -0,0 +1,15 @@ +BITS 64 + ; call brk(0) + ; rax <- 12 (__NR_brj) + ; rdi <- 0 + ; TODO 3: Make brk syscall. + mov rax, 12 + xor rdi,rdi + syscall + + ; call exit_group(0) + ; rax <- 231 (__NR_exit_group) + ; rdi <- 0 (exit status) + mov rax, 231 + xor rdi, rdi + syscall diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/log/CPPLINT.cfg b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/log/log.c b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/log/log.h b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.c b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.h b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/utils.h b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/src/.gitignore b/chapters/data/arena/drills/tasks/exec-shellcode/support/src/.gitignore new file mode 100644 index 0000000000..5b678bc248 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/src/.gitignore @@ -0,0 +1 @@ +/exec_shellcode diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/src/Makefile b/chapters/data/arena/drills/tasks/exec-shellcode/support/src/Makefile new file mode 100644 index 0000000000..52e057a177 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/src/Makefile @@ -0,0 +1,14 @@ +CFLAGS ?= -Wall -Wextra +CPPFLAGS ?= -I../utils + +.PHONY: all clean + +all: exec_shellcode + +exec_shellcode: exec_shellcode.o ../utils/log/log.o + +../utils/log/log.o: ../utils/log/log.c ../utils/log/log.h + +clean: + -rm -f exec_shellcode exec_shellcode + -rm -f *~ diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/src/exec_shellcode.c b/chapters/data/arena/drills/tasks/exec-shellcode/support/src/exec_shellcode.c new file mode 100644 index 0000000000..b35d4284fc --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/src/exec_shellcode.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include + +#include "utils.h" + +static void *shellcode_mapping; + +static void usage(const char * const argv0) +{ + fprintf(stderr, "Usage: %s shellcode_file\n", argv0); +} + +static void create_shellcode_mapping(void) +{ + /* TODO: Create mapping to fit the shellcode. */ +} + +static void read_shellcode(const char * const fname) +{ + /* TODO: Read content from file in shellcode. */ +} + +int main(int argc, char **argv) +{ + if (argc != 2) { + usage(argv[0]); + exit(EXIT_FAILURE); + } + + create_shellcode_mapping(); + read_shellcode(argv[1]); + + ((void (*)(void)) shellcode_mapping)(); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/.gitignore b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/.gitignore new file mode 100644 index 0000000000..8c2194af9c --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/.gitignore @@ -0,0 +1,4 @@ +/brk +/getpid +/helloworld +/openfile diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/Makefile b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/Makefile new file mode 100644 index 0000000000..f1602531db --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/Makefile @@ -0,0 +1,34 @@ +SRC_PATH ?= ../src +FULL_SRC_PATH = $(realpath $(SRC_PATH)) +CPPFLAGS = -I. -I$(realpath $(SRC_PATH)) -I../utils +CFLAGS = -Wall -Wextra +# Remove the line below to disable debugging support. +CFLAGS += -g -O0 + +SRCS = $(wildcard *.asm) +SHELLCODES = $(patsubst %.asm,%,$(SRCS)) + +.PHONY: all src check lint clean + +all: $(SHELLCODES) src + +$(SHELLCODES): %:%.asm | src + nasm -o $@ $< + +src: + make -C $(FULL_SRC_PATH) + +check: $(SHELLCODES) + make -C $(FULL_SRC_PATH) clean + make clean + make -i SRC_PATH=$(FULL_SRC_PATH) + ./run_all_tests.sh + +lint: + -cd .. && checkpatch.pl -f src/*.c + -cd .. && checkpatch.pl -f tests/*.sh + -cd .. && cpplint --recursive src/ + -cd .. && shellcheck tests/*.sh + +clean: + -rm -f *~ diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/brk.asm b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/brk.asm new file mode 100644 index 0000000000..824430f8eb --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/brk.asm @@ -0,0 +1,12 @@ +BITS 64 + ; call brk(0) + ; rax <- 12 (__NR_brj) + ; rdi <- 0 + ; TODO 3: Make brk syscall. + + ; call exit_group(0) + ; rax <- 231 (__NR_exit_group) + ; rdi <- 0 (exit status) + mov rax, 231 + xor rdi, rdi + syscall diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/getpid.asm b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/getpid.asm new file mode 100644 index 0000000000..a1b19a1940 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/getpid.asm @@ -0,0 +1,12 @@ +BITS 64 + ; call getpid() + ; rax <- 39 (__NR_write) + mov rax, 39 + syscall + + ; call exit_group(0) + ; rax <- 231 (__NR_exit_group) + ; rdi <- 0 (exit status) + mov rax, 231 + xor rdi, rdi + syscall diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/graded_test.inc.sh b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/graded_test.inc.sh new file mode 100644 index 0000000000..bce1f05bbe --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/graded_test.inc.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +# +# Print test result. Printed message should fit in 72 characters. +# +# Print format is: +# +# description ...................... passed ... NNN +# description ...................... failed ... NNN +# 32 chars 24 chars 6 3 3 +# + +print_test() +{ + func="$1" + result="$2" + points="$3" + + if test "$points" -gt 999; then + points=999 + fi + + printf "%-32s " "${func:0:31}" + printf "........................" + if test "$result" -eq 0; then + printf " passed ... %3d\n" "$points" + else + printf " failed ... 0\n" + fi +} + +run_test() +{ + func="$1" + points="$2" + + # Run in subshell. + (eval "$func") + print_test "$func" "$?" "$points" +} diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/helloworld.asm b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/helloworld.asm new file mode 100644 index 0000000000..626f8f38c9 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/helloworld.asm @@ -0,0 +1,25 @@ +BITS 64 + ; use jmp / call trick to get string address in RCX + jmp hello +back: + ; call write(1, "Hello, World!\n", 14); + ; rax <- 1 (__NR_write) + ; rdi <- 1 (stdout fileno) + ; rsi <- "Hello, World!\n" + ; rdx <- 14 (string length) + mov rax, 1 + mov rdi, 1 + pop rsi + mov rdx, 14 + syscall + + ; call exit_group(0) + ; rax <- 231 (__NR_exit_group) + ; rdi <- 0 (exit status) + mov rax, 231 + xor rdi, rdi + syscall + +hello: + call back + db "Hello, World!", 10, 0 diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/openfile.asm b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/openfile.asm new file mode 100644 index 0000000000..a94813af89 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/openfile.asm @@ -0,0 +1,25 @@ +BITS 64 + ; use jmp / call trick to get filename address in RCX + jmp filename +back: + ; call open("uberfile", O_RDWR | O_TRUNC | O_CREAT, 0644) + ; rax <- 2 (__NR_open) + ; rdi <- "uberfile" + ; rsi <- 578 (O_RDWR | O_TRUNC | O_CREAT - 01102) + ; rdx <- 420 (0644) + mov rax, 2 + pop rdi + mov rsi, 578 + mov rdx, 420 + syscall + + ; call exit_group(0) + ; rax <- 231 (__NR_exit_group) + ; rdi <- 0 (exit status) + mov rax, 231 + xor rdi, rdi + syscall + +filename: + call back + db "uberfile", 0 diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/run_all_tests.sh b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/run_all_tests.sh new file mode 100755 index 0000000000..23f26ec56e --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/run_all_tests.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +if test -z "$SRC_PATH"; then + SRC_PATH=../src +fi + +export SRC_PATH + +( +./test_helloworld.sh +./test_getpid.sh +./test_openfile.sh +./test_brk.sh +) | tee results.txt + +total=$(grep '\( passed \| failed \)' results.txt | rev | cut -d ' ' -f 1 | rev | paste -s -d'+' | bc) +echo "" +echo -n "Total: " +echo -n " " +LC_ALL=C printf "%3d/100\n" "$total" + +rm results.txt diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_brk.sh b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_brk.sh new file mode 100755 index 0000000000..20a70d1d6e --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_brk.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +source graded_test.inc.sh + +shellcode=./brk + +if test -z "$SRC_PATH"; then + SRC_PATH=../src +fi + +test_brk() +{ + if test ! -f "$shellcode"; then + echo "No such file $shellcode" 1>&2 + exit 1 + fi + + objdump -D -M intel -b binary -m i386:x86-64 "$shellcode" > /dev/null 2>&1 + if test $? -ne 0; then + echo "Incorrect shellcode file" 1>&2 + exit 1 + fi + + timeout -k 1 3 "$SRC_PATH"/exec_shellcode "$shellcode" > /dev/null 2>&1 + if test $? -ne 0; then + echo "Program runs unsuccessfully" 1>&2 + exit 1 + fi + + timeout -k 1 3 strace "$SRC_PATH"/exec_shellcode "$shellcode" 2>&1 \ + | grep -A 1 close | tail -2 | grep 'brk(NULL)[ \t]\+= 0x' > /dev/null 2>&1 + if test $? -ne 0; then + echo "brk not called correctly" 1>&2 + exit 1 + fi + exit 0 +} + +run_test test_brk 25 diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_getpid.sh b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_getpid.sh new file mode 100755 index 0000000000..f651b020b8 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_getpid.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +source graded_test.inc.sh + +shellcode=./getpid + +if test -z "$SRC_PATH"; then + SRC_PATH=../src +fi + +test_getpid() +{ + if test ! -f "$shellcode"; then + echo "No such file $shellcode" 1>&2 + exit 1 + fi + + objdump -D -M intel -b binary -m i386:x86-64 "$shellcode" > /dev/null 2>&1 + if test $? -ne 0; then + echo "Incorrect shellcode file" 1>&2 + exit 1 + fi + + timeout -k 1 3 "$SRC_PATH"/exec_shellcode "$shellcode" > /dev/null 2>&1 + if test $? -ne 0; then + echo "Program runs unsuccessfully" 1>&2 + exit 1 + fi + + timeout -k 1 3 strace "$SRC_PATH"/exec_shellcode "$shellcode" 2>&1 \ + | grep -A 1 close | tail -2 | grep 'getpid()[ \t]\+= [0-9]\+' > /dev/null 2>&1 + if test $? -ne 0; then + echo "getpid() not called (successfully)" 1>&2 + exit 1 + fi + exit 0 +} + +run_test test_getpid 25 diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_helloworld.sh b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_helloworld.sh new file mode 100755 index 0000000000..0a0b3d7276 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_helloworld.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +source graded_test.inc.sh + +shellcode=./helloworld + +if test -z "$SRC_PATH"; then + SRC_PATH=../src +fi + +test_helloworld() +{ + if test ! -f "$shellcode"; then + echo "No such file $shellcode" 1>&2 + exit 1 + fi + + objdump -D -M intel -b binary -m i386:x86-64 "$shellcode" > /dev/null 2>&1 + if test $? -ne 0; then + echo "Incorrect shellcode file" 1>&2 + exit 1 + fi + + timeout -k 1 3 "$SRC_PATH"/exec_shellcode "$shellcode" > /dev/null 2>&1 + if test $? -ne 0; then + echo "Program runs unsuccessfully" 1>&2 + exit 1 + fi + + timeout -k 1 3 "$SRC_PATH"/exec_shellcode "$shellcode" | grep 'Hello, World!' > /dev/null 2>&1 + if test $? -ne 0; then + echo "'Hello, World!' not printed" 1>&2 + exit 1 + fi + exit 0 +} + +run_test test_helloworld 25 diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_openfile.sh b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_openfile.sh new file mode 100755 index 0000000000..aeba08f315 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/tests/test_openfile.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +source graded_test.inc.sh + +shellcode=./openfile + +if test -z "$SRC_PATH"; then + SRC_PATH=../src +fi + +test_openfile() +{ + if test ! -f "$shellcode"; then + echo "No such file $shellcode" 1>&2 + exit 1 + fi + + objdump -D -M intel -b binary -m i386:x86-64 "$shellcode" > /dev/null 2>&1 + if test $? -ne 0; then + echo "Incorrect shellcode file" 1>&2 + exit 1 + fi + + timeout -k 1 3 "$SRC_PATH"/exec_shellcode "$shellcode" > /dev/null 2>&1 + if test $? -ne 0; then + echo "Program runs unsuccessfully" 1>&2 + exit 1 + fi + + test -f "uberfile" > /dev/null 2>&1 + if test $? -ne 0; then + echo "File not created" 1>&2 + exit 1 + fi + + test "$(stat --format="%s" "uberfile")" -eq 0 + if test $? -ne 0; then + echo "File is not truncated" 1>&2 + exit 1 + fi + + test "$(stat --format="%a" "uberfile")" = "644" + if test $? -ne 0; then + echo "File permissions not correctly set" 1>&2 + exit 1 + fi + + exit 0 +} + +run_test test_openfile 25 diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/log/CPPLINT.cfg b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/log/log.c b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/log/log.h b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/sock/sock_util.c b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/sock/sock_util.h b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/utils.h b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/exec-shellcode/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/arena/drills/tasks/page-mapper/README.md b/chapters/data/arena/drills/tasks/page-mapper/README.md new file mode 100644 index 0000000000..392b9c487c --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/README.md @@ -0,0 +1,27 @@ +## Page Mapper + +Navigate to the `support/page-mapper/` directory. + +Your goal is to update the `src/page_mapper.c` source code file to reserve virtual pages in the address space of the current process. +Use `mmap()` to reserve virtual pages. +Use anonymous mapping (i.e. the `MAP_ANONYMOUS`) flag. +Use any permissions required. + +Inside the `src/` directory, use `make` to do a quick check of the implementation. +To test it, enter the `tests/` directory and run: + +```console +make check +``` + +In case of a correct solution, you will get an output such as: + +```text +./run_all_tests.sh +test_res_1 ........................ passed ... 25 +test_res_2 ........................ passed ... 25 +test_res_10 ........................ passed ... 25 +test_res_10_res_20 ........................ passed ... 25 + +Total: 100/100 +``` diff --git a/chapters/data/arena/drills/tasks/page-mapper/solution/src/Makefile b/chapters/data/arena/drills/tasks/page-mapper/solution/src/Makefile new file mode 100644 index 0000000000..504e53e866 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/solution/src/Makefile @@ -0,0 +1,20 @@ +CFLAGS ?= -Wall -Wextra +CPPFLAGS ?= -I../utils + +.PHONY: all clean + +all: main + +main: main.o page_mapper.o ../utils/log/log.o + +main.o: main.c page_mapper.h + +page_mapper.o: page_mapper.c page_mapper.h ../utils/utils.h + +../utils/log/log.o: ../utils/log/log.c ../utils/log/log.h + +clean: + -rm -f main + -rm -f main.o + -rm -f page_mapper.o + -rm -f *~ diff --git a/chapters/data/arena/drills/tasks/page-mapper/solution/src/main.c b/chapters/data/arena/drills/tasks/page-mapper/solution/src/main.c new file mode 100644 index 0000000000..8aca384e31 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/solution/src/main.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include + +#include "page_mapper.h" + +int main(void) +{ + do_map(256); + do_map(10); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/page-mapper/solution/src/page_mapper.c b/chapters/data/arena/drills/tasks/page-mapper/solution/src/page_mapper.c new file mode 100644 index 0000000000..68827bf8ca --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/solution/src/page_mapper.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include "page_mapper.h" +#include "utils.h" + +void do_map(unsigned int num_pages) +{ + /* TODO 2: Obtain page size. */ + long page_size = sysconf(_SC_PAGESIZE); + void *ptr; + + /* TODO 2: Map pages in memory using mmap(). */ + ptr = mmap(NULL, num_pages * page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + DIE(ptr == NULL, "mmap"); +} diff --git a/chapters/data/arena/drills/tasks/page-mapper/solution/src/page_mapper.h b/chapters/data/arena/drills/tasks/page-mapper/solution/src/page_mapper.h new file mode 100644 index 0000000000..80bdadb244 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/solution/src/page_mapper.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef ON_COMMAND_H_ +#define ON_COMMAND_H_ 1 + +void do_map(unsigned int num_pages); + +#endif diff --git a/chapters/data/arena/drills/tasks/page-mapper/solution/utils/log/CPPLINT.cfg b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/arena/drills/tasks/page-mapper/solution/utils/log/log.c b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/arena/drills/tasks/page-mapper/solution/utils/log/log.h b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/arena/drills/tasks/page-mapper/solution/utils/sock/sock_util.c b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/page-mapper/solution/utils/sock/sock_util.h b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/arena/drills/tasks/page-mapper/solution/utils/utils.h b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/src/.gitignore b/chapters/data/arena/drills/tasks/page-mapper/support/src/.gitignore new file mode 100644 index 0000000000..01a8302d4e --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/src/.gitignore @@ -0,0 +1 @@ +/page_mapper diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/src/Makefile b/chapters/data/arena/drills/tasks/page-mapper/support/src/Makefile new file mode 100644 index 0000000000..504e53e866 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/src/Makefile @@ -0,0 +1,20 @@ +CFLAGS ?= -Wall -Wextra +CPPFLAGS ?= -I../utils + +.PHONY: all clean + +all: main + +main: main.o page_mapper.o ../utils/log/log.o + +main.o: main.c page_mapper.h + +page_mapper.o: page_mapper.c page_mapper.h ../utils/utils.h + +../utils/log/log.o: ../utils/log/log.c ../utils/log/log.h + +clean: + -rm -f main + -rm -f main.o + -rm -f page_mapper.o + -rm -f *~ diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/src/main b/chapters/data/arena/drills/tasks/page-mapper/support/src/main new file mode 100755 index 0000000000..6fce0852a5 Binary files /dev/null and b/chapters/data/arena/drills/tasks/page-mapper/support/src/main differ diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/src/main.c b/chapters/data/arena/drills/tasks/page-mapper/support/src/main.c new file mode 100644 index 0000000000..bce0bed577 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/src/main.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include "utils.h" +#include "page_mapper.h" + +int main(void) +{ + do_map(256); + do_map(10); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/src/page_mapper.c b/chapters/data/arena/drills/tasks/page-mapper/support/src/page_mapper.c new file mode 100644 index 0000000000..e032d510c1 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/src/page_mapper.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include "utils.h" +#include "page_mapper.h" + + +void do_map(unsigned int num_pages) +{ + /* TODO: Obtain page size. */ + + /* TODO: Map pages in memory using mmap(). */ +} diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/src/page_mapper.h b/chapters/data/arena/drills/tasks/page-mapper/support/src/page_mapper.h new file mode 100644 index 0000000000..80bdadb244 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/src/page_mapper.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef ON_COMMAND_H_ +#define ON_COMMAND_H_ 1 + +void do_map(unsigned int num_pages); + +#endif diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/tests/.gitignore b/chapters/data/arena/drills/tasks/page-mapper/support/tests/.gitignore new file mode 100644 index 0000000000..ee4c926823 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/tests/.gitignore @@ -0,0 +1 @@ +/test diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/tests/Makefile b/chapters/data/arena/drills/tasks/page-mapper/support/tests/Makefile new file mode 100644 index 0000000000..47814932ae --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/tests/Makefile @@ -0,0 +1,44 @@ +SRC_PATH ?= ../src +FULL_SRC_PATH = $(realpath $(SRC_PATH)) +CPPFLAGS = -I. -I$(realpath $(SRC_PATH)) -I../utils +CFLAGS = -Wall -Wextra +# Remove the line below to disable debugging support. +CFLAGS += -g -O0 + +SRCS = $(wildcard test*.c) +OBJS = $(patsubst %.c,%.o,$(SRCS)) +EXECS = $(patsubst %.c,%,$(SRCS)) + + +.PHONY: all src check lint clean + +all: src $(EXECS) + +$(EXECS): %:%.o graded_test.o $(SRC_PATH)/page_mapper.o ../utils/log/log.o | src + $(CC) $(LDFLAGS) -o $@ $^ $(LDLIBS) + +$(OBJS): %.o:%.c graded_test.h + +graded_test.o: graded_test.c graded_test.h + +../utils/log/log.o: ../utils/log/log.c ../utils/log/log.h + +src: + make -C $(FULL_SRC_PATH) + +check: + make -C $(FULL_SRC_PATH) clean + make clean + make -i SRC_PATH=$(FULL_SRC_PATH) + ./run_all_tests.sh + +lint: + -cd .. && checkpatch.pl -f src/*.c tests/*.c tests/*/*.c + -cd .. && checkpatch.pl -f checker/*.sh tests/*.sh + -cd .. && cpplint --recursive src/ tests/ checker/ + -cd .. && shellcheck checker/*.sh tests/*.sh + +clean: + -rm -f *~ + -rm -f graded_test.o $(OBJS) + -rm -f $(EXECS) diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/tests/graded_test.c b/chapters/data/arena/drills/tasks/page-mapper/support/tests/graded_test.c new file mode 100644 index 0000000000..1eb03d1001 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/tests/graded_test.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include + +#include "./graded_test.h" + +static void my_itoa(size_t num, char *a) +{ + unsigned char digit3; + unsigned char digit2; + unsigned char digit1; + + /* Print at most 3 decimal digits. */ + if (num > 999) + num = 999; + + digit1 = num % 10; + num /= 10; + digit2 = num % 10; + num /= 10; + digit3 = num % 10; + + if (digit3 == 0) + a[0] = ' '; + else + a[0] = '0' + digit3; + + if (digit2 == 0) + a[1] = ' '; + else + a[1] = '0' + digit2; + + a[2] = '0' + digit1; +} + +/* + * Print test result. Printed message should fit in 72 characters. + * + * Print format is: + * + * description ...................... passed ... NNN + * description ...................... failed ... NNN + * 32 chars 24 chars 6 3 3 + */ + +static void print_test(const char *description, int result, size_t points) +{ + /* Make these global linkage, so it's only allocated once. */ + static char print_buffer[74]; + static const char failed[] = "failed"; + static const char passed[] = "passed"; + size_t i; + size_t len; + + /* Collect description in print_buffer. */ + len = MIN(strlen(description), 32); + for (i = 0; i < len; i++) + print_buffer[i] = description[i]; + for (i = len; i < 32; i++) + print_buffer[i] = ' '; + print_buffer[32] = ' '; + + /* Collect dots in print_buffer. */ + for (i = 0; i < 24; i++) + print_buffer[33+i] = '.'; + print_buffer[57] = ' '; + + /* Collect passed / failed. */ + for (i = 0; i < 6; i++) { + if (result == 1) + print_buffer[58+i] = passed[i]; + else + print_buffer[58+i] = failed[i]; + } + print_buffer[64] = ' '; + + /* Collect dots in print_buffer. */ + for (i = 0; i < 3; i++) + print_buffer[65+i] = '.'; + print_buffer[68] = ' '; + + /* Collect number. */ + if (result == 1) { + my_itoa(points, &print_buffer[69]); + } else { + print_buffer[69] = ' '; + print_buffer[70] = ' '; + print_buffer[71] = '0'; + } + + /* Collect newline. */ + print_buffer[72] = '\n'; + + write(1, print_buffer, 73); +} + +void run_test(struct graded_test *test) +{ + int res; + + res = test->function(); + print_test(test->description, res, test->points); +#ifdef EXIT_IF_FAIL + exit(EXIT_FAILURE); +#endif +} + +void run_tests(struct graded_test *tests, size_t count) +{ + size_t i; + + for (i = 0; i < count; i++) + run_test(&tests[i]); +} diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/tests/graded_test.h b/chapters/data/arena/drills/tasks/page-mapper/support/tests/graded_test.h new file mode 100644 index 0000000000..f56f2e07ca --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/tests/graded_test.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef GRADED_TEST_H_ +#define GRADED_TEST_H_ 1 + +/* test function prototype */ +typedef int (*test_f)(void); + +struct graded_test { + test_f function; /* test/evaluation function */ + char *description; /* test description */ + size_t points; /* points for each test */ +}; + +void run_test(struct graded_test *test); +void run_tests(struct graded_test *tests, size_t count); + +#endif /* GRADED_TEST_H_ */ diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/tests/run_all_tests.sh b/chapters/data/arena/drills/tasks/page-mapper/support/tests/run_all_tests.sh new file mode 100755 index 0000000000..c0228aff91 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/tests/run_all_tests.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +./test | tee results.txt + +total=$(grep '\( passed \| failed \)' results.txt | rev | cut -d ' ' -f 1 | rev | paste -s -d'+' | bc) +echo "" +echo -n "Total: " +echo -n " " +LC_ALL=C printf "%3d/100\n" "$total" + +rm results.txt diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/tests/test.c b/chapters/data/arena/drills/tasks/page-mapper/support/tests/test.c new file mode 100644 index 0000000000..604dbab71f --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/tests/test.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include + +#include "page_mapper.h" +#include "utils.h" +#include "graded_test.h" + +static long page_size; + +#define test_page(diff, num_pages) \ + ((diff) >= (num_pages) * page_size && \ + (diff) < ((num_pages) + 1) * page_size) + +#define vsz_prologue() \ + long vsz_before, vsz_after, vsz_diff; \ + vsz_before = get_current_vsz() + +#define vsz_epilogue(np) \ + vsz_after = get_current_vsz(); \ + vsz_diff = vsz_after - vsz_before; \ + return test_page(vsz_diff, (np)) + + +static long get_current_vsz(void) +{ + FILE *f; + pid_t pid; + static char proc_buffer[8192]; + char filename[256]; + long size; + char *p; + + pid = getpid(); + sprintf(filename, "/proc/%d/status", pid); + f = fopen(filename, "rt"); + DIE(f == NULL, "fopen"); + fread(proc_buffer, 1, 8192, f); + fclose(f); + + p = strstr(proc_buffer, "VmSize"); + size = strtol(p+7, NULL, 10) * 1024; + log_debug("size: %ld", size); + + return size; +} + +static int test_res_1(void) +{ + vsz_prologue(); + do_map(1); + vsz_epilogue(1); +} + +static int test_res_2(void) +{ + vsz_prologue(); + do_map(2); + vsz_epilogue(2); +} + +static int test_res_10(void) +{ + vsz_prologue(); + do_map(10); + vsz_epilogue(10); +} + +static int test_res_10_res_20(void) +{ + vsz_prologue(); + do_map(20); + do_map(10); + vsz_epilogue(30); +} + +static struct graded_test all_tests[] = { + { test_res_1, "test_res_1", 25 }, + { test_res_2, "test_res_2", 25 }, + { test_res_10, "test_res_10", 25 }, + { test_res_10_res_20, "test_res_10_res_20", 25 }, +}; + +int main(void) +{ + page_size = sysconf(_SC_PAGESIZE); + + /* Change to LOG_DEBUG for more messages. */ + log_set_level(LOG_INFO); + + run_tests(all_tests, sizeof(all_tests) / sizeof(all_tests[0])); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/utils/log/CPPLINT.cfg b/chapters/data/arena/drills/tasks/page-mapper/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/utils/log/log.c b/chapters/data/arena/drills/tasks/page-mapper/support/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/utils/log/log.h b/chapters/data/arena/drills/tasks/page-mapper/support/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/utils/sock/sock_util.c b/chapters/data/arena/drills/tasks/page-mapper/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/utils/sock/sock_util.h b/chapters/data/arena/drills/tasks/page-mapper/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/arena/drills/tasks/page-mapper/support/utils/utils.h b/chapters/data/arena/drills/tasks/page-mapper/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/page-mapper/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/arena/drills/tasks/reference-counting/.gitignore b/chapters/data/arena/drills/tasks/reference-counting/.gitignore new file mode 100644 index 0000000000..207e314ff7 --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/.gitignore @@ -0,0 +1,2 @@ +/operators +/refcount_skel diff --git a/chapters/data/arena/drills/tasks/reference-counting/README.md b/chapters/data/arena/drills/tasks/reference-counting/README.md new file mode 100644 index 0000000000..54d329b2cf --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/README.md @@ -0,0 +1,51 @@ +### Operator overloading + +Navigate to the `support/reference-counting` directory. +Analyze the `operators.d` file. +A `struct` is defined that also implements 4 special functions: a constructor, a copy constructor, an assignment operator and a destructor. +Each of these special functions may be called automatically by the compiler: + +- the constructor is called automatically whenever an object is initialized with a field of a type that corresponds to the constructor parameter type. +- the copy constructor is called automatically when an object is initialized from an object of the same type. +- the assignment operator is called automatically when an object is assigned an object of the same type. +- the destructor is called automatically whenever an object goes out of scope. + +**Note: the difference between initialization and assignment is that the initialization occurs when an object is being declared and occurs a single time (`Obj o1 = 1`), whereas assignment is decoupled from the declaration site and may occur multiple times (provided that the variable is mutable).** + +Compile and run the program in `operators.d`. +Notice how the different special functions are automatically called. +Considering the definition of `Obj` from the file `operators.d`, answer the following [Quiz](../drills/questions/operators.md). + +#### Practice + +Navigate to the `support/reference-counting` directory. +Analyze the `refcount_skel.d`. +A reference counted `int` array is implemented, however, some bits are missing. +Run the code, try to understand what happens. + +The constructor allocates memory for the array, whereas the destructor deallocates it. +Compile and run the code. +Notice how the array's memory is automatically managed. + +1. Uncomment the following line in the `main` function (`//test1()`). + Run the code. + What happens? + Why? + +1. The reference counted array does not implement the copy constructor. + Comment the `version(none)` annotation for the copy constructor and implement the logic so that the reference counted array is correct. + When an object is initialized from another object, we need to appropriately set the fields and then increment the reference count. + Once you have completed this exercise, make sure the output is correct and that the reference counted array is not freed too early. + +1. Uncomment the following line in the `main` function (`//test2()`). + Run the code. + What happens? + Why? + Use GDB to find out. + +1. The reference counted array does not implement the assignment operator. + Comment the `version(none)` annotation for the assignment operator and implement the logic so that the reference counted array is correct. + When an object is assigned to another object, we need to first decrement the count for the object that is being assigned to, then fill the fields similarly to the copy constructor case and lastly increment the count for the assigned object. + After completing the exercise, make sure that the memory is properly managed. + +1. Play with your reference counted array and create different scenarios to test its limits. diff --git a/chapters/data/arena/drills/tasks/reference-counting/support/src/Makefile b/chapters/data/arena/drills/tasks/reference-counting/support/src/Makefile new file mode 100644 index 0000000000..481b0c22fc --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/support/src/Makefile @@ -0,0 +1,14 @@ +DC = gdc + +.PHONY: all clean + +all: operators refcount_skel + +operators: operators.d + $(DC) -o $@ $^ + +refcount_skel: refcount_skel.d + $(DC) -o $@ $^ + +clean: + -rm -f operators refcount_skel diff --git a/chapters/data/arena/drills/tasks/reference-counting/support/src/operators.d b/chapters/data/arena/drills/tasks/reference-counting/support/src/operators.d new file mode 100644 index 0000000000..5193aef564 --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/support/src/operators.d @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-3-Clause + +import std.stdio; +import core.stdc.stdlib; + +struct Obj +{ + + // constructor + this(int a) + { + writeln("---calling constructor"); + } + + // copy constructor + this(ref Obj obj) + { + writeln("---calling copy constructor"); + } + + // assignment operator + void opAssign(ref Obj obj) + { + writeln("---calling assignment operator"); + } + + // destructor + ~this() + { + writeln("---calling destructor"); + } +} + +void fun(Obj o) {} + +void main() +{ + writeln("constructor call"); + Obj o1 = 1; // => rewritten to `Obj o1 = Obj(1);` + + writeln("copy constructor call"); + Obj o2 = o1; // => rewritten to `Obj o2 = Obj(o1);` + + writeln("assignment operator call"); + o2 = o1; // => rewritten to `o2.opAssign(o1)` + + writeln("initializing a function parameter"); + fun(o2); // => rewritten to `fun(Obj(o2))` + + writeln("destroying created objects"); + // compiler inserts `o1.~this()` and `o2.~this()` +} diff --git a/chapters/data/arena/drills/tasks/reference-counting/support/src/refcount_skel.d b/chapters/data/arena/drills/tasks/reference-counting/support/src/refcount_skel.d new file mode 100644 index 0000000000..beaeb854ac --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/support/src/refcount_skel.d @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: BSD-3-Clause + +import core.stdc.stdlib : malloc, free; +import std.stdio : writeln, printf; + +int instance_generator; + +struct RefCountedIntArray +{ + int* ptr; + int* refcount; + int instance_no; + + size_t size; + + this(int[] a) + { + printf("Creating ref counted array (instance: %d) for: ", instance_generator); + writeln(a); + + // allocate memory for array and refcount + ptr = cast(int*)malloc(int.sizeof * a.length); + refcount = cast(int*)malloc(int.sizeof); + + // initialize ref counted array + foreach(i, elem; a) + ptr[i] = elem; + + // initialize size and ref count + size = a.length; + *refcount = 1; + + instance_no = instance_generator; + instance_generator++; + } + + // copy constructor + // `version(none)` functions are ignored. + // uncomment when you want to implement this + version(none) + this(ref RefCountedIntArray src) + { + // TODO - fill in the fields and correctly update the ref count + + } + + // assignment operator + version(none) + void opAssign(ref RefCountedIntArray src) + { + // TODO + } + + void printArray(ref RefCountedIntArray src) + { + printf("array: ["); + int i; + for (i = 0; i < src.size-1; i++) + printf("%d, ", src.ptr[i]); + printf("%d]\n", src.ptr[i]); + } + + // destructor + ~this() + { + printf("Destroying instance %d\n", instance_no); + (*refcount)--; + if (*refcount == 0) + { + printf("Freeing "); + printArray(this); + free(ptr); + free(refcount); + } + } +} + +void doSomething(RefCountedIntArray arr) +{ + writeln("Doing some work"); +} + +RefCountedIntArray fun() +{ + RefCountedIntArray p = RefCountedIntArray([1, 2, 3]); + RefCountedIntArray p2 = p; + return p2; +} + +void test1() +{ + writeln("==== 1 ====="); + RefCountedIntArray p = fun(); + writeln("==== 2 ====="); + // should print `1` + writeln(p.ptr[0]); +} + +RefCountedIntArray gun(RefCountedIntArray p1) +{ + RefCountedIntArray p2 = RefCountedIntArray([1, 1, 1]); + p2 = p1; + return p2; +} + +void test2() +{ + writeln("==== 3 ====="); + RefCountedIntArray p = RefCountedIntArray([5, 5, 5]); + RefCountedIntArray p2 = gun(p); + p2 = p; + writeln("==== 4 ====="); +} + +void main() +{ + RefCountedIntArray arr = RefCountedIntArray([1, 2, 6]); + //test1(); + //test2(); +} diff --git a/chapters/data/arena/drills/tasks/reference-counting/support/utils/log/CPPLINT.cfg b/chapters/data/arena/drills/tasks/reference-counting/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/arena/drills/tasks/reference-counting/support/utils/log/log.c b/chapters/data/arena/drills/tasks/reference-counting/support/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/support/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/arena/drills/tasks/reference-counting/support/utils/log/log.h b/chapters/data/arena/drills/tasks/reference-counting/support/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/support/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/arena/drills/tasks/reference-counting/support/utils/sock/sock_util.c b/chapters/data/arena/drills/tasks/reference-counting/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/arena/drills/tasks/reference-counting/support/utils/sock/sock_util.h b/chapters/data/arena/drills/tasks/reference-counting/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/arena/drills/tasks/reference-counting/support/utils/utils.h b/chapters/data/arena/drills/tasks/reference-counting/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/arena/drills/tasks/reference-counting/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/arena/reading/arena.md b/chapters/data/arena/reading/arena.md new file mode 100644 index 0000000000..e67ceffd7b --- /dev/null +++ b/chapters/data/arena/reading/arena.md @@ -0,0 +1,39 @@ +# Arena + +Go through the practice items to hone your skills in working with data and memory. + + +## Memory Support + +**Manual memory management** (MMM) is one of the most difficult tasks. +Even experienced programmers make mistakes when tackling such a complicated endeavor. +As a consequence, the programming world has been migrating towards languages that offer automatic memory management (AMM). +AMM programming languages typically offer a garbage collector that tracks down the usage of objects and frees memory once no references exist to a given object. +As a consequence, garbage collected programming languages are easier to use and safer. +However, this comes with a cost: the garbage collector, in most cases, requires a significant amount of resources to run. +Therefore, for performance-critical systems, MMM is still the preferred solution. + +A middle-ground between programming languages that have AMM (Java, Python, Swift, D) and those that do not (C, C++) is represented by those languages that do not have built-in AMM but offer the possibility to implement it as a library solution (C++, D). +Concretely, these languages offer lightweight library solutions to optimally track down the lifetime of an object. +This is done by using reference counted objects. + +### Reference Counting + +Reference counting is a technique of tracking the lifetime of an object by counting how many references to an object exist. +As long as at least one reference exists, the object cannot be destroyed. +Once no reference to a given object exists, it can be safely destroyed. +Reference counted is typically implemented by storing a count with the actual payload of the object. +Every time a new reference to the object is created, the reference count is incremented. +Every time a reference expires, the reference is decremented. + +The operations that trigger a reference increment are: + +- initializing an object from another object. +- assigning an object to another object. + +The operations that trigger a reference decrement are: + +- the lifetime of an object expires + +Modern programming languages offer the possibility to specify what code should be run in each of these situations, therefore enabling the implementation of referenced counted data structures. +As such, copy constructors may be used to automatically initialize an object from another object, assignment operators may be used to assign an object to another object and destructors may be used to destroy objects. diff --git a/chapters/data/investigate-memory/drills/questions/memory-leaks.md b/chapters/data/investigate-memory/drills/questions/memory-leaks.md new file mode 100644 index 0000000000..bb18511912 --- /dev/null +++ b/chapters/data/investigate-memory/drills/questions/memory-leaks.md @@ -0,0 +1,20 @@ +# Memory Leaks + +## Question Text + +What happens to the leaked memory when the process finishes execution? + +## Question Answers + +- It will be freed by the OS garbage collector + +- It remains unusable until the first restart + ++ It is freed alongside all memory used by process + +- It will remain reachable untill another process explicitely frees it + +## Feedback + +When a process ends, all its memory zones are freed by the OS to be reused. +Leaking memory becomes a major problem in case of programs that run indefinitely, because the leaked memory will stack, causing a memory outage. diff --git a/chapters/data/investigate-memory/drills/questions/valgrind-leaks.md b/chapters/data/investigate-memory/drills/questions/valgrind-leaks.md new file mode 100644 index 0000000000..6063e8e9bb --- /dev/null +++ b/chapters/data/investigate-memory/drills/questions/valgrind-leaks.md @@ -0,0 +1,33 @@ +# Valgrind Leaks + +## Question Text + +```c +struct student { + char *name; + int age; +} + +struct student *s = malloc(sizeof(*s)); +s->name = strdup("Reginald"); +// ... +free(s); +``` + +What are the leaks in the above c program? + +## Question Answers + +- There are no leaks + +- `s->name` is **definitely lost** + ++ `s->name` is **indirectly lost** + +- `s->name` is **still reachable** + +## Feedback + +`strdup()` allocates memory for a string so the returned pointer must be freed. +Freeing `s` will leave us unable to free `s->name`, so `s->name` is **indirectly lost**. +Find more about valgrind leak categories [here](https://valgrind.org/docs/manual/faq.html#faq.deflost). diff --git a/chapters/data/investigate-memory/guides/app-investigation-deluge/README.md b/chapters/data/investigate-memory/guides/app-investigation-deluge/README.md new file mode 100644 index 0000000000..063f4d7bff --- /dev/null +++ b/chapters/data/investigate-memory/guides/app-investigation-deluge/README.md @@ -0,0 +1,51 @@ +# App Investigation: Deluge + +[Deluge](https://www.deluge-torrent.org/) is a Bittorrent client written in Python. + +We want to locate places that allocate memory in Deluge (in Python). +This generally means locating instantiation of classes. + +Let's clone the [source code](https://github.com/deluge-torrent/deluge): + +```console +student@os:~/.../data/lab/support$ git clone https://github.com/deluge-torrent/deluge +Cloning into 'deluge'... +[...] + +student@os:~/.../data/lab/support$ cd deluge/ + +student@os:~/.../lab/support/deluge$ ls +AUTHORS deluge docs gen_web_gettext.py MANIFEST.in msgfmt.py pyproject.toml requirements-dev.txt requirements.txt setup.py version.py +CHANGELOG.md DEPENDS.md generate_pot.py LICENSE minify_web_js.py packaging README.md requirements-tests.txt setup.cfg tox.ini +``` + +And enter the `deluge/core/` subdirectory: + +```console +student@os:~/.../lab/support/deluge$ cd deluge/core/ + +student@os:~/.../deluge/deluge/core$ ls +alertmanager.py core.py daemon.py filtermanager.py pluginmanager.py rpcserver.py torrent.py +authmanager.py daemon_entry.py eventmanager.py __init__.py preferencesmanager.py torrentmanager.py +``` + +Most files in the subdirectory have a class defined. +We can search for instantiations of that class using `grep`: + +```console +student@os:~/.../deluge/deluge/core$ grep -rn 'Torrent(' +torrentmanager.py:644: torrent = Torrent(handle, options, state, filename, magne + +student@os:~/.../deluge/deluge/core$ grep -rn 'TorrentManager(' +core.py:139: self.torrentmanager = TorrentManager() +torrentmanager.py:135:class TorrentManager(component.Component): +``` + +This gives us an overview of when memory is allocated in Deluge / Python. + +### Practice + +1. Investigate the lines shown to contain instantiations of classes. + Explore the source code and understand their placements in the source code. + +1. Find out other classes and search for their instantiation in the source code. diff --git a/chapters/data/investigate-memory/guides/app-investigation-servo/README.md b/chapters/data/investigate-memory/guides/app-investigation-servo/README.md new file mode 100644 index 0000000000..bcc21d5c28 --- /dev/null +++ b/chapters/data/investigate-memory/guides/app-investigation-servo/README.md @@ -0,0 +1,31 @@ +# App Investigation: Servo + +[Servo](https://servo.org/) is a browser engine written in Rust that provides reusable components to implement web standards. + +We do not clone the repository, since it's very large. + +We find information about allocator used, by accessing the `components/allocator/` in [its source code](https://github.com/servo/servo/tree/master/components/allocator). +In `Cargo.toml` we see that it requires `jemalloc` for non-Windows implementations and the standard Windows API (called `heapapi`) for Windows: + +```text +[...] +[lib] +path = "lib.rs" + +[target.'cfg(not(windows))'.dependencies] +jemalloc-sys = { version = "0.3.2" } + +[target.'cfg(windows)'.dependencies] +winapi = { version = "0.3", features = ["heapapi"] } +ust: https://github.com/servo/servo +``` + +In `lib.rs`, in [`GlobalAlloc:alloc()`](https://github.com/servo/servo/blob/master/components/allocator/lib.rs#L70) we see it is using [the `mallocx` custom function from `jemalloc()`](https://jemalloc.net/jemalloc.3.html). +See [the initialization of `ffi`](https://github.com/servo/servo/blob/master/components/allocator/lib.rs#L17). + +See the use of the allocator in the [`Cargo.toml` file in the `net` component](https://github.com/servo/servo/blob/master/components/net/Cargo.toml). +Search for the `alloc` string. + +### Practice + +Look for uses of the allocator in other components of Servo. diff --git a/chapters/data/investigate-memory/guides/d-allocator/README.md b/chapters/data/investigate-memory/guides/d-allocator/README.md new file mode 100644 index 0000000000..f225cb7a23 --- /dev/null +++ b/chapters/data/investigate-memory/guides/d-allocator/README.md @@ -0,0 +1,38 @@ +# Investigation: Allocator in the D Programming Language + +[Phobos](https://github.com/dlang/phobos) is the standard library that comes with the D programming language compiler. + +Let's clone the source code: + +```console +student@os:~/.../data/lab/support$ git clone https://github.com/dlang/phobos +[...] + +student@os:~/.../data/lab/support$ cd phobos/ + +student@os:~/.../lab/support/phobos$ ls +azure-pipelines.yml changelog CODEOWNERS CONTRIBUTING.md dub.sdl etc index.dd LICENSE_1_0.txt posix.mak project.ddoc README.md std test unittest.d win32.mak win64.mak +``` + +And enter `std/experimental/allocator/` to browse information about the allocator: + +```console +student@os:~/.../lab/support/phobos$ cd std/experimental/allocator/ + +student@os:~/.../std/experimental/allocator$ ls +building_blocks common.d gc_allocator.d mallocator.d mmap_allocator.d package.d showcase.d typed.d +``` + +We then do a search of the `allocate(` string to find instances of allocation calls: + +```console +student@os:~/.../std/experimental/allocator$ grep -r 'allocate(' +[...] +``` + +We see that there are definitions of the function (as expected) as part of `...allocator` files: `mallocator.d`, `gc_allocator.d`, `mmap_allocator.d`. +Browse the functions and look for implementations of the `allocate()` function. + +### Practice + +Do a similar search and then source code browsing for the `deallocate()` function. diff --git a/chapters/data/investigate-memory/guides/git/README.md b/chapters/data/investigate-memory/guides/git/README.md new file mode 100644 index 0000000000..ac9d39644c --- /dev/null +++ b/chapters/data/investigate-memory/guides/git/README.md @@ -0,0 +1,69 @@ +# App Investigation: Git + +[Git](https://git-scm.com/) is among the most used source code management system, powering development infrastructures such as [GitHub](https://github.com/), [GitLab](https://about.gitlab.com/) and [Bitbucket](https://bitbucket.org/). + +Let's clone [the repository](https://github.com/git/git). +Note that it is about 200MB large: + +```console +student@os:~/.../data/lab/support$ git clone https://github.com/git/git +[...] + +student@os:~/.../data/lab/support$ cd git/ +``` + +We look of uses of `malloc()`: + +```console +student@os:~/.../lab/support/git$ grep -r 'malloc(' . +``` + +We see there are multiple calls to the `xmalloc()` function, which is likely a wrapper for `malloc()`. +We search for the definition of `xmalloc()`: + +```console +student@os:~/.../lab/support/git$ grep -rn 'xmalloc(' . | grep -v ';' +./commit.c:188: graft = xmalloc(st_add(sizeof(*graft), +./add-interactive.c:157: list->sorted.items = xmalloc(st_mult(sizeof(*list->sorted.items), +./Documentation/RelNotes/2.24.0.txt:91: * xmalloc() used to have a mechanism to ditch memory and address +./Documentation/RelNotes/2.24.0.txt:210: xmalloc() wrapper, as the rest of the system, for consistency. +./Documentation/RelNotes/2.34.0.txt:230: * mmap() imitation used to call xmalloc() that dies upon malloc() +./Documentation/RelNotes/2.33.1.txt:44: * mmap() imitation used to call xmalloc() that dies upon malloc() +./diffcore-delta.c:56: new_spanhash = xmalloc(st_add(sizeof(*orig), +./diffcore-delta.c:135: hash = xmalloc(st_add(sizeof(*hash), +./kwset.c:41:/* adapter for `xmalloc()`, which takes `size_t`, not `long` */ +./builtin/fast-import.c:461: b = xmalloc(sizeof(struct object_entry_pool) +./hashmap.h:311: * your structure was allocated with xmalloc(), you can just free(3) it, +./xdiff/xdiff.h:122:#define xdl_malloc(x) xmalloc(x) +./wrapper.c:45:static void *do_xmalloc(size_t size, int gentle) +./wrapper.c:70:void *xmalloc(size_t size) +./contrib/credential/wincred/git-credential-wincred.c:26:static void *xmalloc(size_t size) +Binary file ./.git/objects/pack/pack-c587b9f11a82bc4d49848d74132e60ea4dbeb177.pack matches +./git-compat-util.h:1046:# define xalloca(size) (xmalloc(size)) +./git-compat-util.h:1086:#define ALLOC_ARRAY(x, alloc) (x) = xmalloc(st_mult(sizeof(*(x)), (alloc))) +./read-cache.c:3768: ieot = xmalloc(sizeof(struct index_entry_offset_table) +``` + +Line `./wrapper.c:70` is the one with the definition of the `xmalloc()` function. +It makes a call of the [`do_xmalloc()` function](https://github.com/git/git/blob/master/wrapper.c#L45), that makes extra checks. +Also, if the `XMALLOC_POISON` macro is defined, all the allocated data is overwritten with a "poison" value (`0xA5`). +This is useful for early detection of memory-related issues, although, evidently, it adds overhead. + +We can look for parts of the source code with the largest number of uses of `xmalloc()`: + +```console +student@os:~/.../lab/support/git$ grep -rc 'xmalloc(' . | grep -v ':0' | sort -n -t ':' -k 20 +[...] +./compat/mingw.c:6 +./submodule-config.c:6 +./merge-recursive.c:7 +``` + +We can look into the [`merge-recursive.c` file](https://github.com/git/git/blob/master/merge-recursive.c) for uses of the `xmalloc()` function. + +### Practice + +Do the same actions as above for the `mmap()` and `xmmap()` function calls. + +Note that these are not memory allocation calls, since a valid `fd` file argument is passed. +These are file mapping calls, that we will talk more as part of the I/O chapter. diff --git a/chapters/data/investigate-memory/guides/jemalloc/README.md b/chapters/data/investigate-memory/guides/jemalloc/README.md new file mode 100644 index 0000000000..ad7305319c --- /dev/null +++ b/chapters/data/investigate-memory/guides/jemalloc/README.md @@ -0,0 +1,67 @@ +# `jemalloc` + +[jemalloc](http://jemalloc.net/) is a featureful allocator that is intended to replace the standard allocator in the standard C library (libc). +jemalloc provides replacements for the general `malloc()` and `free()` functions, and also provides a custom API targeted for performance tuning. + +As [documented](https://github.com/jemalloc/jemalloc/wiki/Getting-Started), there are multiple ways to use `jemalloc`, the easiest of which is to use the `LD_PRELOAD` environment variable and preload the library and hook into `malloc()` and `free()` function calls. + +First install `jemalloc` on our system. +On your typical Ubuntu / Debian-based system, use `apt`: + +```console +student@os:~/.../data/lab/content$ sudo apt -y install libjemalloc-dev +``` + +Note that this installs the distribution package, not the latest one (that may provide more features). + +With this in place, we can use `jemalloc` against our pre-built executables or system executables (such as `ls`, `ps`). +We can test it against the executable files from `support/memory-leak/`: + +```console +student@os:~/.../lab/support/memory-leak$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so ./memory_leak_malloc +Andrei Popescu is 22 years old and likes Linux +Ioana David is 23 years old and likes macOS +``` + +`jemalloc` can use the `MALLOC_CONF` environment variable for a [diverse set of configurations](https://www.freebsd.org/cgi/man.cgi?query=malloc.conf). +For example, by using `stats_print:true` we print out information regarding the use of the library functions: + +```console +student@os:~/.../lab/support/memory-leak$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so MALLOC_CONF="stats_print:true" ./memory_leak_malloc +Andrei Popescu is 22 years old and likes Linux +Ioana David is 23 years old and likes macOS +___ Begin jemalloc statistics ___ +Version: 3.6.0-11 +Assertions disabled +Run-time option settings: + opt.abort: false + opt.lg_chunk: 22 + opt.dss: "secondary" + opt.narenas: 32 + opt.lg_dirty_mult: 3 + opt.stats_print: true + opt.junk: false + opt.quarantine: 0 + opt.redzone: false +[...] +dirty pages: 26:0 active:dirty, 0 sweeps, 0 madvises, 0 purged + allocated nmalloc ndalloc nrequests +small: 72672 114 0 3 +large: 32768 1 0 1 +total: 105440 115 0 4 +active: 106496 +mapped: 4194304 +[...] +``` + +`jemalloc` doesn't work against system executables using preloading, likely because of security options disabling the use of the library: + +```console +student@os:~/.../lab/support/memory-leak$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so MALLOC_CONF="stats_print:true" /bin/ls +Makefile memory_leak memory_leak.cpp memory_leak_malloc memory_leak_malloc.c memory_leak_malloc.o memory_leak.o + +student@os:~/.../lab/support/memory-leak$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so MALLOC_CONF="stats_print:true" /bin/ps + PID TTY TIME CMD + 1581 pts/22 00:00:00 ps +26732 pts/22 00:00:01 bash +``` diff --git a/chapters/data/investigate-memory/guides/memory-actions/README.md b/chapters/data/investigate-memory/guides/memory-actions/README.md new file mode 100644 index 0000000000..c01d6ea3d8 --- /dev/null +++ b/chapters/data/investigate-memory/guides/memory-actions/README.md @@ -0,0 +1,97 @@ +# Memory Actions (and Leaks) in Existing Programs + +We can use Valgrind to investigate existing programs in the system. +This tells us whether they possess memory leaks: + +```console +student@os:~/.../lab/support/memory-leak$ valgrind ls +==24669== Memcheck, a memory error detector +==24669== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al. +==24669== Using Valgrind-3.13.0 and LibVEX; rerun with -h for copyright info +==24669== Command: ls +==24669== +Makefile memory_leak memory_leak.cpp memory_leak_malloc memory_leak_malloc.c memory_leak_malloc.o memory_leak.o +==24669== +==24669== HEAP SUMMARY: +==24669== in use at exit: 21,696 bytes in 14 blocks +==24669== total heap usage: 51 allocs, 37 frees, 61,331 bytes allocated +==24669== +==24669== LEAK SUMMARY: +==24669== definitely lost: 0 bytes in 0 blocks +==24669== indirectly lost: 0 bytes in 0 blocks +==24669== possibly lost: 0 bytes in 0 blocks +==24669== still reachable: 21,696 bytes in 14 blocks +==24669== suppressed: 0 bytes in 0 blocks +==24669== Rerun with --leak-check=full to see details of leaked memory +==24669== +==24669== For counts of detected and suppressed errors, rerun with: -v +==24669== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0) + +student@os:~/.../lab/support/memory-leak$ valgrind ps +==24671== Memcheck, a memory error detector +==24671== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al. +==24671== Using Valgrind-3.13.0 and LibVEX; rerun with -h for copyright info +==24671== Command: ps +==24671== + PID TTY TIME CMD +24671 pts/22 00:00:00 memcheck-amd64- +26732 pts/22 00:00:01 bash +==24671== +==24671== HEAP SUMMARY: +==24671== in use at exit: 264,929 bytes in 25 blocks +==24671== total heap usage: 692 allocs, 667 frees, 334,268 bytes allocated +==24671== +==24671== LEAK SUMMARY: +==24671== definitely lost: 0 bytes in 0 blocks +==24671== indirectly lost: 0 bytes in 0 blocks +==24671== possibly lost: 0 bytes in 0 blocks +==24671== still reachable: 264,929 bytes in 25 blocks +==24671== suppressed: 0 bytes in 0 blocks +==24671== Rerun with --leak-check=full to see details of leaked memory +==24671== +==24671== For counts of detected and suppressed errors, rerun with: -v +==24671== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0) + +student@os:~/.../lab/support/memory-leak$ valgrind bash -c 'echo "ha"' +==24675== Memcheck, a memory error detector +==24675== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al. +==24675== Using Valgrind-3.13.0 and LibVEX; rerun with -h for copyright info +==24675== Command: bash -c echo\ "ha" +==24675== +ha +==24675== +==24675== HEAP SUMMARY: +==24675== in use at exit: 43,056 bytes in 672 blocks +==24675== total heap usage: 774 allocs, 102 frees, 51,405 bytes allocated +==24675== +==24675== LEAK SUMMARY: +==24675== definitely lost: 12 bytes in 1 blocks +==24675== indirectly lost: 0 bytes in 0 blocks +==24675== possibly lost: 0 bytes in 0 blocks +==24675== still reachable: 43,044 bytes in 671 blocks +==24675== suppressed: 0 bytes in 0 blocks +==24675== Rerun with --leak-check=full to see details of leaked memory +==24675== +==24675== For counts of detected and suppressed errors, rerun with: -v +==24675== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0) +``` + +We can see that `ls` and `ps` don't have memory leaks. +However, the shell (Bash) shows a memory leak of 12 bytes (on the test system). +This may be a false positive or the subject of an actual investigation. + +Note that the `still reachable` section of the output refers to memory that wasn't freed, but still has pointers referring to it. +A true memory leak occurs when no pointers refer any memory area. + +### Practice + +1. Investigate 2-3 other executables in the system using Valgrind. + +1. Use `ltrace` to list `malloc()` and `free()` calls made by the investigated system executables. + +Note that, as explained in the [Software Stack lab](https://open-education-hub.github.io/operating-systems/Lab/Software%20Stack/libcall-syscall), on some systems, `ltrace` does not accurately show the output, due to _now binding_. +Fear not, you can always check the library calls with a more verbose and harder to parse `ltrace` command: + +```console +student@os:~$ ltrace -x "*" +``` diff --git a/chapters/data/investigate-memory/guides/memory-leak/.gitignore b/chapters/data/investigate-memory/guides/memory-leak/.gitignore new file mode 100644 index 0000000000..c868610849 --- /dev/null +++ b/chapters/data/investigate-memory/guides/memory-leak/.gitignore @@ -0,0 +1,3 @@ +/memory_leak +/memory_leak_malloc +/mem.trace diff --git a/chapters/data/investigate-memory/guides/memory-leak/README.md b/chapters/data/investigate-memory/guides/memory-leak/README.md new file mode 100644 index 0000000000..bd154089e8 --- /dev/null +++ b/chapters/data/investigate-memory/guides/memory-leak/README.md @@ -0,0 +1,122 @@ +# Memory Leaks + +A memory leak occurs when we lose reference to a memory area. +That is, a pointer used to point to a memory area. +And then it's pointing to a new memory area and the old memory area is lost. + +Enter the `support/memory-leak/` folder. +It stores two files showing memory leaks: + +- one in C++: `memory_leak.cpp` +- one in C: `memory_leak_malloc` + +Let's build and run the two executables: + +```console +student@os:~/.../lab/support/memory-leak$ make +g++ -c -o memory_leak.o memory_leak.cpp +cc memory_leak.o -lstdc++ -o memory_leak +cc -c -o memory_leak_malloc.o memory_leak_malloc.c +cc memory_leak_malloc.o -lstdc++ -o memory_leak_malloc +``` + +Running them yields similar output: + +```console +student@os:~/.../lab/support/memory-leak$ ./memory_leak +Andrei Popescu is 22 years old and likes Linux +Ioana David is 23 years old and likes macOS +student@os:~/.../lab/support/memory-leak$ ./memory_leak_malloc +Andrei Popescu is 22 years old and likes Linux +Ioana David is 23 years old and likes macOS +``` + +We investigate the memory leaks of the two programs by using [Valgrind](https://valgrind.org/): + +```console +student@os:~/.../lab/support/memory-leak$ valgrind ./memory_leak +==22362== Memcheck, a memory error detector +==22362== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al. +==22362== Using Valgrind-3.13.0 and LibVEX; rerun with -h for copyright info +==22362== Command: ./memory_leak +==22362== +Andrei Popescu is 22 years old and likes Linux +Ioana David is 23 years old and likes macOS +==22362== +==22362== HEAP SUMMARY: +==22362== in use at exit: 72 bytes in 1 blocks +==22362== total heap usage: 4 allocs, 3 frees, 73,872 bytes allocated +==22362== +==22362== LEAK SUMMARY: +==22362== definitely lost: 72 bytes in 1 blocks +==22362== indirectly lost: 0 bytes in 0 blocks +==22362== possibly lost: 0 bytes in 0 blocks +==22362== still reachable: 0 bytes in 0 blocks +==22362== suppressed: 0 bytes in 0 blocks +==22362== Rerun with --leak-check=full to see details of leaked memory +==22362== +==22362== For counts of detected and suppressed errors, rerun with: -v +==22362== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0) + +student@os:~/.../lab/support/memory-leak$ valgrind ./memory_leak_malloc +==22369== Memcheck, a memory error detector +==22369== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al. +==22369== Using Valgrind-3.13.0 and LibVEX; rerun with -h for copyright info +==22369== Command: ./memory_leak_malloc +==22369== +Andrei Popescu is 22 years old and likes Linux +Ioana David is 23 years old and likes macOS +==22369== +==22369== HEAP SUMMARY: +==22369== in use at exit: 148 bytes in 1 blocks +==22369== total heap usage: 3 allocs, 2 frees, 1,320 bytes allocated +==22369== +==22369== LEAK SUMMARY: +==22369== definitely lost: 148 bytes in 1 blocks +==22369== indirectly lost: 0 bytes in 0 blocks +==22369== possibly lost: 0 bytes in 0 blocks +==22369== still reachable: 0 bytes in 0 blocks +==22369== suppressed: 0 bytes in 0 blocks +==22369== Rerun with --leak-check=full to see details of leaked memory +==22369== +==22369== For counts of detected and suppressed errors, rerun with: -v +==22369== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0) +``` + +As we are doing allocations that are not freed, this results in memory leaks. + +For `malloc()`-based programs, we can use [`mtrace()` feature](https://man7.org/linux/man-pages/man3/mtrace.3.html) and [`mtrace` command](https://man7.org/linux/man-pages/man1/mtrace.1.html) to verify proper allocations with `malloc()` and deallocations with `free()`. +We call `mtrace()` in the program (in `memory_leak_malloc.c`) to enable `malloc()` and `free()` checking. + +To use `mtrace()` we define the `MALLOC_TRACE` environment variable. +We probably also require to preload the libc malloc debugging library, so we use `LD_PRELOAD` for that. +Note that the file path used for `LD_PRELOAD` may need to be updated, depending on your distribution: + +```console +student@os:~/.../lab/support/memory-leak$ LD_PRELOAD=/lib/x86_64-linux-gnu/libc_malloc_debug.so.0 MALLOC_TRACE=mem.trace ./memory_leak_malloc +Andrei Popescu is 22 years old and likes Linux +Ioana David is 23 years old and likes macOS +``` + +Subsequently, we use the `mtrace` tool to show information about the leaked data: + +```console +student@os:~/.../lab/support/memory-leak$ mtrace ./memory_leak_malloc mem.trace + +Memory not freed: +----------------- + Address Size Caller +0x000056506d8be6a0 0x94 at 0x56506c3777ec +``` + +The size (`0x94`) is the same value shown by Valgrind (`148`). + +`mtrace` provides an outcome similar to Valgrind. +Valgrind is however more powerful: it works on different types of memory (not only those allocated with `malloc()`) and it doesn't require access to the source code (and the compiler phase). + +### Practice + +1. Print the size of the `Student` class and the `struct student` structure to see if it equates to the leak shown by Valgrind. + +1. Solve the memory leaks in both programs. + Validate with Valgrind. diff --git a/chapters/data/investigate-memory/guides/memory-leak/support/CPPLINT.cfg b/chapters/data/investigate-memory/guides/memory-leak/support/CPPLINT.cfg new file mode 100644 index 0000000000..b21b6b434c --- /dev/null +++ b/chapters/data/investigate-memory/guides/memory-leak/support/CPPLINT.cfg @@ -0,0 +1 @@ +filter=-runtime/printf diff --git a/chapters/data/investigate-memory/guides/memory-leak/support/Makefile b/chapters/data/investigate-memory/guides/memory-leak/support/Makefile new file mode 100644 index 0000000000..439f9b9ed4 --- /dev/null +++ b/chapters/data/investigate-memory/guides/memory-leak/support/Makefile @@ -0,0 +1,18 @@ +LDLIBS = -lstdc++ + +.PHONY: all clean + +all: memory_leak memory_leak_malloc + +memory_leak: memory_leak.o + +memory_leak.o: memory_leak.cpp + +memory_leak_malloc: memory_leak_malloc.o + +memory_leak_malloc.o: memory_leak_malloc.c + +clean: + -rm -f memory_leak memory_leak.o + -rm -f memory_leak_malloc memory_leak_malloc.o + -rm -f *~ diff --git a/chapters/data/investigate-memory/guides/memory-leak/support/memory_leak.cpp b/chapters/data/investigate-memory/guides/memory-leak/support/memory_leak.cpp new file mode 100644 index 0000000000..4b1a6e29c3 --- /dev/null +++ b/chapters/data/investigate-memory/guides/memory-leak/support/memory_leak.cpp @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#include + +class Student { + public: + Student(std::string name, unsigned int age, std::string favorite_os); + void Print() const; + + private: + std::string name_; + unsigned int age_; + std::string favorite_os_; +}; + +Student::Student(std::string name, unsigned int age, std::string favorite_os) + : name_(name), age_(age), favorite_os_(favorite_os) +{ +} + +void Student::Print() const +{ + std::cout << name_ << " is " << age_ << " years old"; + std::cout << " and likes " << favorite_os_ << std::endl; +} + +int main(void) +{ + Student *s; + + s = new Student("Andrei Popescu", 22, "Linux"); + s->Print(); + + s = new Student("Ioana David", 23, "macOS"); + s->Print(); + + delete s; + + return 0; +} diff --git a/chapters/data/investigate-memory/guides/memory-leak/support/memory_leak_malloc.c b/chapters/data/investigate-memory/guides/memory-leak/support/memory_leak_malloc.c new file mode 100644 index 0000000000..3ac639edb5 --- /dev/null +++ b/chapters/data/investigate-memory/guides/memory-leak/support/memory_leak_malloc.c @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#include +#include +#include +#include + +struct student { + char name[128]; + unsigned int age; + char favorite_os[16]; +}; + +static void init_student(struct student *s, const char *name, unsigned int age, const char *os) +{ + strcpy(s->name, name); + s->age = age; + strcpy(s->favorite_os, os); +} + +static void print_student(struct student *s) +{ + printf("%s is %u years old and likes %s\n", + s->name, s->age, s->favorite_os); +} + +int main(void) +{ + struct student *s; + + mtrace(); + + s = malloc(sizeof(*s)); + init_student(s, "Andrei Popescu", 22, "Linux"); + print_student(s); + + s = malloc(sizeof(*s)); + init_student(s, "Ioana David", 23, "macOS"); + print_student(s); + + free(s); + + return 0; +} diff --git a/chapters/data/investigate-memory/reading/investigate-memory.md b/chapters/data/investigate-memory/reading/investigate-memory.md new file mode 100644 index 0000000000..6033092650 --- /dev/null +++ b/chapters/data/investigate-memory/reading/investigate-memory.md @@ -0,0 +1,29 @@ +# Investigate Memory Actions + +Memory actions generally mean: + +- memory access: read, write or execute +- memory allocation +- memory deallocation + +By far, the most important actions are allocation and deallocation. +Because, if not done right, these can get to memory loss and poor memory use. + +Memory loss generally happens in the form of memory leaks. + +## `malloc()` in Musl + +Each libc (or memory allocator such as `jemalloc`) uses their own implementation of `malloc()`, `free()` and other functions. +[Musl libc](https://musl.libc.org/) is a lightweight standard C library that provides compatible features with the more heavyweights [GNU libc](https://www.gnu.org/software/libc/). + +Take a look through implementation of `malloc()` and `free()` in [Musl libc](https://elixir.bootlin.com/musl/v1.2.3/source/src/malloc). +See all three implementations for `malloc()`: + +* the one in `lite_malloc.c` +* the one in `mallocng/malloc.c` +* the one in `oldmalloc/malloc` + +See also [the implementation of `free()`](https://elixir.bootlin.com/musl/v1.2.3/source/src/malloc/mallocng/free.c#L101). +And [the implementation of `calloc()`](https://elixir.bootlin.com/musl/v1.2.3/source/src/malloc/calloc.c#L33). + +You needn't spend too much time browsing the implementation of these functions, just having a broad understanding of how they work. diff --git a/chapters/data/memory-security/reading/memory-security.md b/chapters/data/memory-security/reading/memory-security.md new file mode 100644 index 0000000000..4f34617a08 --- /dev/null +++ b/chapters/data/memory-security/reading/memory-security.md @@ -0,0 +1,180 @@ +# Memory Security + +Memory security is one of the most important aspects in today's computer systems. +Its main objectives are to avoid the development of vulnerable code and prevent attackers from exploiting existing memory vulnerabilities. +Memory protection techniques are implemented at all levels of memory abstraction: hardware, operating system and programming language. +In addition, third-party software tools may be used to statically and dynamically examine a program to identify vulnerabilities when working with memory. + +In this section, we will focus on the main memory vulnerabilities and how to exploit them. +We will use the C programming language for presentation purposes, however, these examples may be reproduced in any language that implements arrays as pointers without bounds and allows the liberal use of pointers. + +## Wild Pointer Arithmetic Info Leak + +In C, once a pointer has been initialized with a valid address it can potentially access any location of the process address space. +This is problematic for situations where the binary contains sensitive information. +Take for example a server that authenticates users: if the password is stored in some buffer, then if a pointer is wrongfully used, it can end up leaking the password. + +Navigate to the `support/memory-security/` directory. +Open and analyze the `buff_leak.c` file. + +[Answer this quiz](../quiz/memory-regions-vars.md) + +### Practice + +The pointer `p` points to the stack, however, we can modify any variable that is declared in the program through `p`. +All we need to know is the offset of the other memory locations that we wish to access. +Run the program and try to input the correct offsets to modify variables from different regions of our program. +Once a correct offset is given as input, the program will output a validation message. + +Note that adding or subtracting user provided values to pointers enables an attacker to observe a program's entire memory! + +## Buffer Overflow Info Leak + +Since arrays decay to pointers in C, the same effect may be obtained by using them. + +Navigate to the `support/memory-security/` directory. +Open and analyze the `array_leak.c` file. + +Compile and run the program. +Can you extract any information from the output? +Can you identify the return address of the main function? + +**Note:** You can use `objdump -d -M intel array_leak` to check the stack layout. +**Note:** Depending on the environment the layout may differ. + +Next, open and analyze the `string_leak.c` file. +Compile and run the program. +To better understand the output, use `xxd` to interpret the output as hexadecimal bytes: + +```console +student@os:~/.../lab/support/memory-security$ ./string_leak | xxd +``` + +**Note:** `73` and `6f` are the ascii values of `s` and `o` + +[Answer this quiz](../quiz/string-buff-over.md) + +### Practice + +In file `string_leak.c` replace the usage of `memcpy` with `strcpy`. +Do not modify anything else (including the size of the buffer). +As the name suggests, `strcpy()` is specialized for string copies, therefore we don need to specify how much we want to copy. +What is the result? +Is the result correct? +Explain the result. + +[Answer this quiz](../quiz/string-strcpy.md) + +## Buffer Overflow Overwrite + +Up until this point we have seen how we can exploit a buffer to leak information. +However, this vulnerability may be used most of the time to overwrite data. + +Take for example the code in `support/memory-security/bo_write.c`. +Compile and run the code. +What happens? +Why? + +[Answer this quiz](../quiz/stack-layout.md) + +### Practice + +Open the `support/memory-security/bo_write_practice.c` file. +Analyze the code, then compile it and run it. + +1. Try to find an input that alters the control flow of the program so that "Comm-link online" is printed. +You are not allowed to modify the source file. + +1. Try to find an input that alters the control flow of the program so that "Channel open." is printed. +You are not allowed to modify the source file. + +**Note:** Addresses are 8 bytes long on 64 bit machines. + +1. Can you think of a different input that results in printing "Comm-link online"? + +## ASLR + +Address Space Layout Randomization (ASLR) is a protection technique employed by operating systems to make it harder for attackers to identify code locations to jump to. +Essentially, every program section (including shared library code) is mapped at a random virtual address every time the program is run. +That way, it is harder for the attacker to exploit a buffer overflow: the address of a potential code target is different with each run. + +To enable randomization, there are 2 steps that need to be taken: + +1. Enable ASLR on the operating system. +This enables the operating system to map library code at different virtual addresses in a program. + +1. Compile our code as Position Independent Code (PIC). +This instructs the compiler to generate code such that it does not use absolute addresses when calling functions or accessing variables. +As a consequence, the code may be loaded at any given address in memory. + +On most Linux systems, ASLR is enabled by default. +Modern compilers generate position independent code by default. + +### Practice + +Use the `Makefile.aslr` file to compile the `support/memory-security/aslr.c` file: + +```console +student@os:~/.../lab/support/memory-security$ make -f Makefile.aslr +``` + +By default, ASLR and PIC are enabled. +Observe the results. +Next, we disable ASLR: + +```console +student@os:~/.../lab/support/memory-security$ echo 0 | sudo tee /proc/sys/kernel/randomize_va_space +``` + +Even though the code is compiled with PIC, both library and user functions have the same address between runs. +Re-enable ASLR: + +```console +student@os:~/.../lab/support/memory-security$ echo 2 | sudo tee /proc/sys/kernel/randomize_va_space +``` + +Disable PIC by uncommenting the `-fno-PIC` and `LDFLAGS` lines. + +We observe that for randomization to work, we need to instruct the OS to randomize the program sections and the compiler to generate code that is position independent. + +[Answer this quiz](../quiz/memory-aslr.md) + +## Stack Protector + +As observed in the previous quiz, ASLR prevents an attacker (most of the times) from discovering a reliable address where to jump to, however, it does not prevent the occurrence of a buffer overflow. +An effective strategy to protect against buffer overflow is represented by the stack protector (or stack canary). +Between the end of the buffer and the return address (below the saved base pointer `rbp`), a random value is placed on the stack. +Before the function returns, the value is checked: if the initial value was modified, then an exception is issued and the program is aborted. + +Stack canaries are enabled by default, however, for learning purposes we have disabled it by passing `-fno-stack-protector` to the compiler. + +### Practice + +Comment the `-fno-stack-protector` switch from the `support/memory-security/Makefile`, recompile and run the `bo_practice_write` executable. +Examine the binary with `objdump` and identify the instructions that set and test the canary. +Observe what happens when a buffer overflow occurs. + +[Answer this quiz](../quiz/memory-stack-protector.md) + +## Bypassing the Stack Protector + +The stack protector is generally placed immediately after the old `rbp`. +With this information we can craft various exploits to bypass it: + +- Leak the canary and do not modify it. +Reading the canary, as long as we do not modify it is not going to cause any disturbances. +If we have the canary, we can just include it in our payload and happily overwrite the return address. + +- If we have access to a pointer that we can modify appropriately, we can just jump over the canary and directly modify the return address. + +- In case we have multiple buffers defined in the same stack frame, we can simply overwrite data in a buffer that is placed above the buffer we are exploiting without the stack protector intervention. + +### Practice + +Inspect the `support/memory-security/stack_protector.c` source file. +Compile the program and examine the object code. +Try to identify the canary value. +Using the `addr` variable, write 2 `scanf` instructions: one that overwrites the canary with the correct value and one that overwrites the return address with the address of function `pawned`. +In case of a successful exploit a video will be offered as reward. + +[Answer this quiz](../quiz/bypass-canary.md) diff --git a/chapters/data/overview/media/data-mem.svg b/chapters/data/overview/media/data-mem.svg new file mode 100644 index 0000000000..6cf01774ab --- /dev/null +++ b/chapters/data/overview/media/data-mem.svg @@ -0,0 +1,4 @@ + + + +
Programmer
Prog...
Data
Data
Program
Program
Data
Data
Input
Input
Output
Output
Memory
Memory
Store
 data
Store...
Retrieve
data
Retrieve...Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/overview/media/data.svg b/chapters/data/overview/media/data.svg new file mode 100644 index 0000000000..3d2d63bc06 --- /dev/null +++ b/chapters/data/overview/media/data.svg @@ -0,0 +1,4 @@ + + + +
Programmer
Prog...
Data
Data
Program
Program
Data
Data
Input
Input
Output
OutputText is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/overview/reading/overview.md b/chapters/data/overview/reading/overview.md new file mode 100644 index 0000000000..05cd991652 --- /dev/null +++ b/chapters/data/overview/reading/overview.md @@ -0,0 +1,21 @@ +# Data + +Data represents information that is to be processed to produce a final result or more data. +Computers store, retrieve and compute data. +This process involves 4 entities: the programmer, the programming language, the operating system and the hardware. + +From a programmer's perspective, data is represented by the variables. +These are declared and utilized depending on the rules of the programming language that is employed. +The programming language analyzes the use of these variables and outputs code that uses an interface provided by the operating system. +This interface offers the possibility to allocate/deallocate different variables in certain memory regions. +Next, the operating system manages the execution of the program and provides the actual physical addresses that are used to interact with the data. + +Moreover, the operating system governs the competing access of multiple programs to memory, ensuring that a program does not have access to a different program's memory. + +## Contents + +1. [Working with Memory](working-memory.md) +1. [Process Memory](process-memory.md) +1. [Investigate Memory](investigate-memory.md) +1. [Memory Security](memory-security.md) +1. [Arena](arena.md) diff --git a/chapters/data/overview/slides/data.md b/chapters/data/overview/slides/data.md new file mode 100644 index 0000000000..7ea2fc70a0 --- /dev/null +++ b/chapters/data/overview/slides/data.md @@ -0,0 +1,45 @@ +--- + +## Data in Software and Computing + +---- + +### What Is Data? + +* Generally: factual information (such as measurements or statistics) used as a basis for reasoning, discussion, or calculation +* Computer science: information in digital form that can be transmitted or processed + +---- + +### Data Examples + +* Anatomy coursework - data regarding the human body +* Census - data regarding population +* Your Facebook profile - data regarding yourself +* Your browser history - data regarding preferences + +--- + +### What Is Data? (Computer Science) + +![Data1](../media/data.svg) + +---- + +### What Is Data? (Computer Science) + +![Data2](../media/data-mem.svg) + +---- + +### What We Want + +* Optimal performance + * retrieve and store should be as fast as possible +* Optimal use of memory space + * once data is not used, memory is freed immediately + * minimize the time when memory is reserved but not used + * data should occupy the minimal required space +* Security + * data correctness + * data isolation diff --git a/chapters/data/perspectives/media/HP.svg b/chapters/data/perspectives/media/HP.svg new file mode 100644 index 0000000000..a4a6a12082 --- /dev/null +++ b/chapters/data/perspectives/media/HP.svg @@ -0,0 +1,4 @@ + + + +
Registers
Registers
Cache memory
Cache memory
Main Memory (RAM)
Main Memory (RAM)
Secondary Memory (Hard-disk, SSD, etc.)
Secondary Memory (Hard-disk, SSD, etc.)
Performance
Performance
Capacity
CapacityText is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/perspectives/media/HP2.svg b/chapters/data/perspectives/media/HP2.svg new file mode 100644 index 0000000000..ace809d12e --- /dev/null +++ b/chapters/data/perspectives/media/HP2.svg @@ -0,0 +1,4 @@ + + + +
M1
M1
M2
M2
CPU
CPU
Cache
Cache
Main Memory
Main Memory
...
...
Cache Miss
Cache Miss
Processing
Unit
Processing...Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/perspectives/media/PL.svg b/chapters/data/perspectives/media/PL.svg new file mode 100644 index 0000000000..683f2f7f38 --- /dev/null +++ b/chapters/data/perspectives/media/PL.svg @@ -0,0 +1,4 @@ + + + +
Compiler
Compiler
int a;
const int b = 7;
char c = 'k';
void main()
{
    int f_a;
    static int f_b = 5;
    char *p = malloc(20);
}
int a;...
c = 'k' , f_b = '5'
c = 'k' , f_b = '5'
.data
.data
a = 0
a = 0
.bss
.bss
b = 7
b = 7
.rodata
.rodata

.txt
.txt
main:
push ebp
mov ebp, esp
        sub esp, 12
        ....
        push 20
        call malloc
         
main:...Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/perspectives/media/SOP.svg b/chapters/data/perspectives/media/SOP.svg new file mode 100644 index 0000000000..500a08e237 --- /dev/null +++ b/chapters/data/perspectives/media/SOP.svg @@ -0,0 +1,4 @@ + + + +
Registers
Registers
Cache memory
Cache memory
Main Memory (RAM)
Main Memory (RAM)
Secondary Memory (Hard-disk, SSD, etc.)
Secondary Memory (Hard-disk, SSD, etc.)
c = 'k' , f_b = '5'
c = 'k' , f_b = '5'
.data
.data
a = 0
a = 0
.bss
.bss
b = 7
b = 7
.rodata
.rodata

.txt
.txt
main:
push ebp
mov ebp, esp
        sub esp, 12
        ....
        push 20
        call malloc
         
main:...
?
?Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/perspectives/media/lvl-data.svg b/chapters/data/perspectives/media/lvl-data.svg new file mode 100644 index 0000000000..a9d83f4fd7 --- /dev/null +++ b/chapters/data/perspectives/media/lvl-data.svg @@ -0,0 +1,4 @@ + + + +
Variable mapping
 Allocation/Deallocation
Variable mapping...
Programming Language
Programming Language
Operating System
Operating System
Hardware
Hardware
Programmer
Pr...
Variables
Variables
Physical Addresses
Physical AddressesText is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/perspectives/media/storing-agency.svg b/chapters/data/perspectives/media/storing-agency.svg new file mode 100644 index 0000000000..7efae1c61f --- /dev/null +++ b/chapters/data/perspectives/media/storing-agency.svg @@ -0,0 +1,4 @@ + + + +
Items
Items
BoxId
BoxIdText is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/perspectives/media/unit-interface.svg b/chapters/data/perspectives/media/unit-interface.svg new file mode 100644 index 0000000000..19669a0151 --- /dev/null +++ b/chapters/data/perspectives/media/unit-interface.svg @@ -0,0 +1,4 @@ + + + +
Users
Users
(UserID, UnitId)
(UserID, UnitId)
BoxId
BoxId
Items
Items
Items
Items
.....
.....
(1, 1) - 37
(1, 1)...
(1, 2) - 29
(1, 2)...
(2, 1) - 57
(2, 1)...Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/perspectives/slides/perspectives-of-data.md b/chapters/data/perspectives/slides/perspectives-of-data.md new file mode 100644 index 0000000000..08fde29264 --- /dev/null +++ b/chapters/data/perspectives/slides/perspectives-of-data.md @@ -0,0 +1,163 @@ +--- + +## Perspectives of Data + +* A Programmer's Perspective on Data +* Programming Language Perspective +* Hardware Perspective +* Operating System Perspective + +---- + +### A Programmer's Perspective on Data + +* Data = variables +* Operations: declare/read/write +* Variables are stored in memory, so depending on the language, you can also: + * allocate memory + * deallocate memory + +---- + +### Example + +`demo/variables/c_vars.c` + +---- + +### Performance: Depends on ... + +* number of memory copies + - `demo/performance/copy.c` +* degree of reuse of memory + - `demo/performance/reuse.c` +* number of memory allocations/deallocations + - `demo/performance/alloc.c` + - `demo/performance/alloc.c` +* [Quiz](quiz/alloc.md) +* cache friendliness of the program's behavior +* hardware specifics + +---- + +### Space usage: Depends on ... + +* how we store data + - use appropriate types for variables + - `demo/space_usage/types.c` + - `#pragma pack(1)` + - `demo/space_usage/pragma.c` + - union + - demo/space_usage/union.c +* how soon memory is freed + +---- + +### Security + +* Buffer overflow + - `demo/security/bo.c` +* NUL-terminated strings + - `demo/security/nts.c` +* Integer overflow + - `demo/security/io.c` + +---- + +### Trade-offs + +* Python - performance for security, ease of use +* D - can switch between both worlds + +---- + +### Example + +* Python + - `demo/variables/d_vars.d` +* D + - `demo/variables/python_vars.d` +* [Quiz](quiz/pl.md) + +---- + +### Who manages memory? + +* You (the programmer) - C/C++, D +* The programming language - Python, Java, D, Rust +* A library implementation - C/C++, D +* The operating system - for all languages + +--- + +### Stages of data + +![Data3](../media/lvl-data.svg) + +--- + +### Programming Language Perspective + +![PL](../media/PL.svg) + +--- + +### Hardware perspective + +![HP](../media/HP.svg) + +---- + +### Hardware Perspective + +![HP2](../media/HP2.svg) + +---- + +### How is it Mapped? + +![SOP](../media/SOP.svg) + +--- + +### Operating System Perspective + +---- + +### Storage Units - v1 + +![STAGENGY](../media/storing-agency.svg) + +---- + +### Storage Units - v1 + +* Each user needs to manually verify that a storage box is free +* The user needs to know the exact ID of the box +* Extra security measures need to be implemented so that a user +does not end up opening another users box +* We expect the users to govern the situation +* Users = programs, storage unit = memory + +---- + +### Storing Units - v2 + +![STAGENGY2](../media/unit-interface.svg) + +---- + +### Storage Units - part 2 + +* An automated system handles the requests and assigns each user a `userId` and a virtual `unitId` +* The system assigns a physical box to each pair of (`userId`, `unitId`) +* Each user will start counting its units from 0 to N +* The user does not handle physical boxes + +---- + +### Storing Memory + +* How do computers use memory? + - v1 or v2? + - `demo/proc_storage/` diff --git a/chapters/data/process-memory/drills/questions/half-page.md b/chapters/data/process-memory/drills/questions/half-page.md new file mode 100644 index 0000000000..8bcdab4b4a --- /dev/null +++ b/chapters/data/process-memory/drills/questions/half-page.md @@ -0,0 +1,25 @@ +# Half Page + +## Question Text + +```c +char *p = malloc(2 * 1024); +``` + +What is a potential problem when allocating half a page? + +## Question Answers + +- It will fragment the virtual memory because users should always request at least one page + +- Asking for less than a page might place the memory on two different pages + ++ Writing to the other half may be allowed + +- Allocations smaller than one page should use the stack + +## Feedback + +The OS allocates memory in chunks of 4 KB (the page size). +If the memory we allocate happens to be placed at the beginning of a page, we have permission to update the second half of this page despite not requesting it. +This might be a problem because buffer overflows can pass unnoticed. diff --git a/chapters/data/process-memory/drills/questions/malloc-brk.md b/chapters/data/process-memory/drills/questions/malloc-brk.md new file mode 100644 index 0000000000..a5d4a70202 --- /dev/null +++ b/chapters/data/process-memory/drills/questions/malloc-brk.md @@ -0,0 +1,20 @@ +# Malloc `brk()` + +## Question Text + +When does `malloc()` use `brk()`? + +## Question Answers + +- `brk()` is outdated, `malloc()` always uses `mmap()` + ++ When it allocates a small chunk of memory + +- When it allocates an array + +- When it's working with dynamic libraries + +## Feedback + +`malloc()` uses both `brk()` and `mmap()`, but preffers `brk()` for small chunks of memory to keep granular allocations in a contiguous area. +This way, `free()` does not necessarily return the memory to the OS as it might only mark the zone as "free" within `libc`'s allocator and reuse it for later allocations. diff --git a/chapters/data/process-memory/drills/questions/malloc-mmap.md b/chapters/data/process-memory/drills/questions/malloc-mmap.md new file mode 100644 index 0000000000..d3eff8797a --- /dev/null +++ b/chapters/data/process-memory/drills/questions/malloc-mmap.md @@ -0,0 +1,21 @@ +# Malloc `mmap()` + +## Question Text + +When does `malloc()` use `mmap()`? + +## Question Answers + +- When it allocates read-only memory + +- When it allocates zeroed memory + ++ When it allocates chunks of memory bigger than an internal threshold + +- When the heap is full + +## Feedback + +`malloc` uses both `brk()` and `mmap()`, but prefers `mmap()` for big chunks of memory (by default larger than 128 KB). +This value can be altered using [`mallopt()`](https://linux.die.net/man/3/mallopt) with the `param` argument set to `M_MMAP_THRESHOLD`. +These memroy blocks are unlikely to be reused so they are not placed on heap to avoid memory fragmentation. diff --git a/chapters/data/process-memory/drills/questions/memory-granularity.md b/chapters/data/process-memory/drills/questions/memory-granularity.md new file mode 100644 index 0000000000..2c4768a0c8 --- /dev/null +++ b/chapters/data/process-memory/drills/questions/memory-granularity.md @@ -0,0 +1,20 @@ +# Memory Granularity + +## Question Text + +What is the granularity of the size of memory sections? + +## Question Answers + +- 4 bytes + +- 4 MB + ++ 4 KB + +- 1 KB + +## Feedback + +All sizes of memory areas are multiple of 4 KB (the page size). +Also, all addresses start at multiple of 4 KB. diff --git a/chapters/data/process-memory/drills/questions/mmap-file.md b/chapters/data/process-memory/drills/questions/mmap-file.md new file mode 100644 index 0000000000..5cfc170b3c --- /dev/null +++ b/chapters/data/process-memory/drills/questions/mmap-file.md @@ -0,0 +1,20 @@ +# `mmap()` file + +## Question Text + +What is one advantage of mapping a file in memory? + +## Question Answers + ++ It reduces interaction with the disk + +- Consumes less memory + +- It is faster because it does not uses the file API + +- Allows all threads to use the same memory area + +## Feedback + +After mapping a file in memory, all changes will be visible only in memory. +When removing the mapping or explicitely calling `msync()` the information from memory will be visible on disk. diff --git a/chapters/data/process-memory/drills/questions/page-allocation.md b/chapters/data/process-memory/drills/questions/page-allocation.md new file mode 100644 index 0000000000..d8dc49a17c --- /dev/null +++ b/chapters/data/process-memory/drills/questions/page-allocation.md @@ -0,0 +1,31 @@ +# Page Allocation + +## Question Text + +```console +student@os:~/.../drills/tasks/static-dynamic/support$ size hello-static +text data bss dec hex filename +893333 20996 7128 921457 e0f71 hello-static +``` + +How many bytes should we add to the `.data` section to make its size `28 KB`, instead of `24 KB`? + +## Question Answers + +- 1 KB + +- 4 KB + +- 3580 bytes + ++ 3581 bytes + +## Feedback + +The total size must be `1` byte over the `24 KB` threshold to cause a new page allocation. +So in order to get that past the current size of `20996`, we need `3581` bytes: + +```console +student@os:~$ echo "24 * 1024 + 1 - 20996" | bc +3581 +``` diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/.gitignore b/chapters/data/process-memory/drills/tasks/alloc-size/.gitignore new file mode 100644 index 0000000000..e352bc56cb --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/alloc-size/.gitignore @@ -0,0 +1 @@ +/alloc_size diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/README.md b/chapters/data/process-memory/drills/tasks/alloc-size/README.md new file mode 100644 index 0000000000..ff43279777 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/alloc-size/README.md @@ -0,0 +1,14 @@ +# Allocating and Deallocating Memory + +## Practice + +1. Use `pmap` to analyze the process address space for `ALLOC_SIZE_KB` initialized to `256`. + Notice the new memory areas and the difference between the use of `mmap` syscall and `brk` syscall. + +1. Use `valgrind` on the resulting executable, and notice there are memory leaks. + They are quite obvious due to the lack of proper freeing. + Solve the leaks. + +1. Use `valgrind` on different executables in the system (in `/bin/`, `/usr/bin/`) and see if they have memory leaks. + +If you're having difficulties solving this exercise, go through [this](../../../reading/process-memory.md) reading material. diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/Makefile b/chapters/data/process-memory/drills/tasks/alloc-size/support/Makefile new file mode 100644 index 0000000000..1c66ec635a --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/alloc-size/support/Makefile @@ -0,0 +1,44 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source and object files for alloc_size +SRC = alloc_size.c +OBJ = $(SRC:.c=.o) + +# Binary name for alloc_size +BINARY = alloc_size + +# Default rule: Build the binary +all: $(BINARY) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile alloc_size object file +$(OBJ): %.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create the alloc_size binary +$(BINARY): $(OBJ) $(LOGGER) + $(CC) $(CFLAGS) $(OBJ) $(LOGGER) -o $(BINARY) $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJ) $(BINARY) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean + diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/alloc_size.c b/chapters/data/process-memory/drills/tasks/alloc-size/support/alloc_size.c new file mode 100644 index 0000000000..41ee007288 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/alloc-size/support/alloc_size.c @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#include +#include +#include "utils/utils.h" + +#define NUM_STEPS 10 +#define ALLOC_SIZE_KB 64 +#define ALLOC_SIZE (ALLOC_SIZE_KB * 1024) + +static void wait_for_input(const char *msg) +{ + char buffer[128]; + + printf("%s ...", msg); + fgets(buffer, 128, stdin); +} + +int main(void) +{ + size_t i; + void *p; + + wait_for_input("Press key to allocate"); + + for (i = 0; i < NUM_STEPS; i++) { + p = malloc(ALLOC_SIZE); + DIE(p == NULL, "malloc"); + printf("New allocation at %p\n", p); + } + + wait_for_input("Press key to deallocate"); + + free(p); + + wait_for_input("Press key to close the program"); + + return 0; +} + diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/CPPLINT.cfg b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/log.c b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/utils.h b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/process-memory/drills/tasks/copy/.gitignore b/chapters/data/process-memory/drills/tasks/copy/.gitignore new file mode 100644 index 0000000000..52d3257e35 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/.gitignore @@ -0,0 +1,4 @@ +/mmap_copy +/read_write_copy +/in.dat +/out.dat diff --git a/chapters/data/process-memory/drills/tasks/copy/README.md b/chapters/data/process-memory/drills/tasks/copy/README.md new file mode 100644 index 0000000000..b8ee01f465 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/README.md @@ -0,0 +1,11 @@ +# Memory Mapping + +## Practice + +1. Use a different value for `BUFSIZE` and see if that affects the comparison between the two executables. + +1. Add a `sleep()` call to the `mmap_copy.c` file **after** the files were mapped. + Rebuild the program and run it. + On a different console, use `pmap` to view the two new memory regions that were added to the process, by mapping the `in.dat` and `out.dat` files. + +If you're having difficulties solving this exercise, go through [this](../../../reading/process-memory.md) reading material. diff --git a/chapters/data/process-memory/drills/tasks/copy/support/Makefile b/chapters/data/process-memory/drills/tasks/copy/support/Makefile new file mode 100644 index 0000000000..ff52cc1b45 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/Makefile @@ -0,0 +1,46 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +SRCS = mmap_copy.c read_write_copy.c +OBJS = $(SRCS:.c=.o) +BINARIES = mmap_copy read_write_copy + +# Default rule: Build everything +all: $(BINARIES) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile object files from source files +%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create mmap_copy binary +mmap_copy: mmap_copy.o $(LOGGER) + $(CC) $(CFLAGS) mmap_copy.o $(LOGGER) -o mmap_copy $(LDFLAGS) + +# Rule to create read_write_copy binary +read_write_copy: read_write_copy.o $(LOGGER) + $(CC) $(CFLAGS) read_write_copy.o $(LOGGER) -o read_write_copy $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean + diff --git a/chapters/data/process-memory/drills/tasks/copy/support/benchmark_cp.sh b/chapters/data/process-memory/drills/tasks/copy/support/benchmark_cp.sh new file mode 100755 index 0000000000..6cbbee88e0 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/benchmark_cp.sh @@ -0,0 +1,21 @@ +#! /bin/bash + +# [WARNING] +# This script should be run from local environment +# The Docker container does not have permission to write to `/proc/sys/vm/_drop_caches` + +# Drop all OS caches: buffer cache, dentry cache, inode cache, page cache. +sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches" + +echo "Benchmarking mmap_copy on in.dat" +# Run the benchmark for `mmap_copy`. +./mmap_copy +echo "" + +# Drop all OS caches: buffer cache, dentry cache, inode cache, page cache. +sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches" + +echo "Benchmarking read_write_copy on in.dat" +# Run the benchmark for `read_write_copy`. +./read_write_copy +echo "" diff --git a/chapters/data/process-memory/drills/tasks/copy/support/generate.sh b/chapters/data/process-memory/drills/tasks/copy/support/generate.sh new file mode 100755 index 0000000000..3eb05c0b98 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/generate.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +dd if=/dev/urandom of=in.dat bs=1M count=32 diff --git a/chapters/data/process-memory/drills/tasks/copy/support/mmap_copy.c b/chapters/data/process-memory/drills/tasks/copy/support/mmap_copy.c new file mode 100644 index 0000000000..2676a5acce --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/mmap_copy.c @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" + +#define SRC_FILENAME "in.dat" +#define DST_FILENAME "out.dat" + +#define diff_us(ta, tb) \ + (((ta).tv_sec - (tb).tv_sec) * 1000 * 1000 + \ + ((ta).tv_nsec - (tb).tv_nsec) / 1000) + +int main(void) +{ + int src_fd, dst_fd; + void *src_p, *dst_p; + struct stat statbuf; + int rc; + struct timespec time_before, time_after; + + src_fd = open(SRC_FILENAME, O_RDONLY); + DIE(src_fd < 0, "open"); + + dst_fd = open(DST_FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0644); + DIE(dst_fd < 0, "open"); + + rc = fstat(src_fd, &statbuf); + DIE(rc < 0, "fstat"); + + /* Truncate destination file. */ + rc = ftruncate(dst_fd, statbuf.st_size); + DIE(rc < 0, "fstat"); + + /* Map files. */ + src_p = mmap(NULL, statbuf.st_size, PROT_READ, MAP_PRIVATE, src_fd, 0); + DIE(src_p == MAP_FAILED, "mmap"); + dst_p = mmap(NULL, statbuf.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, dst_fd, 0); + DIE(dst_p == MAP_FAILED, "mmap"); + + /* Do the copying. */ + clock_gettime(CLOCK_REALTIME, &time_before); + memcpy(dst_p, src_p, statbuf.st_size); + clock_gettime(CLOCK_REALTIME, &time_after); + printf("time passed %ld microseconds\n", diff_us(time_after, time_before)); + + /* Unmap files. */ + munmap(src_p, statbuf.st_size); + munmap(dst_p, statbuf.st_size); + + close(src_fd); + close(dst_fd); + + return 0; +} diff --git a/chapters/data/process-memory/drills/tasks/copy/support/read_write_copy.c b/chapters/data/process-memory/drills/tasks/copy/support/read_write_copy.c new file mode 100644 index 0000000000..048c4c2d28 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/read_write_copy.c @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#include +#include +#include +#include +#include + +#include "utils/utils.h" + +#define SRC_FILENAME "in.dat" +#define DST_FILENAME "out.dat" + +#define diff_us(ta, tb) \ + (((ta).tv_sec - (tb).tv_sec) * 1000 * 1000 + \ + ((ta).tv_nsec - (tb).tv_nsec) / 1000) + +#define BUFSIZE 8192 + +static unsigned char buffer[BUFSIZE]; + +int main(void) +{ + int src_fd, dst_fd; + ssize_t n; + struct timespec time_before, time_after; + + src_fd = open(SRC_FILENAME, O_RDONLY); + DIE(src_fd < 0, "open"); + + dst_fd = open(DST_FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0644); + DIE(dst_fd < 0, "open"); + + clock_gettime(CLOCK_REALTIME, &time_before); + while (1) { + n = read(src_fd, buffer, BUFSIZE); + DIE(n < 0, "read"); + + if (n == 0) + break; + + /* Assume everything is written. */ + n = write(dst_fd, buffer, n); + DIE(n < 0, "write"); + } + clock_gettime(CLOCK_REALTIME, &time_after); + printf("time passed %ld microseconds\n", diff_us(time_after, time_before)); + + close(src_fd); + close(dst_fd); + + return 0; +} diff --git a/chapters/data/process-memory/drills/tasks/copy/support/utils/log/CPPLINT.cfg b/chapters/data/process-memory/drills/tasks/copy/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/process-memory/drills/tasks/copy/support/utils/log/log.c b/chapters/data/process-memory/drills/tasks/copy/support/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/process-memory/drills/tasks/copy/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/copy/support/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/process-memory/drills/tasks/copy/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/copy/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/process-memory/drills/tasks/copy/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/copy/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/process-memory/drills/tasks/copy/support/utils/utils.h b/chapters/data/process-memory/drills/tasks/copy/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/copy/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/.gitignore b/chapters/data/process-memory/drills/tasks/memory-areas/.gitignore new file mode 100644 index 0000000000..e92569d010 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/memory-areas/.gitignore @@ -0,0 +1 @@ +/hello diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/README.md b/chapters/data/process-memory/drills/tasks/memory-areas/README.md new file mode 100644 index 0000000000..2633f76544 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/memory-areas/README.md @@ -0,0 +1,54 @@ +# Memory Regions + +## Practice + +Enter the `drills/tasks/memory-areas/support` directory. +We investigate other programs. + +1. The `hello.c` program prints out a message and then sleeps. + Build it: + + ```console + student@os:~/.../drills/tasks/memory-areas/support$ make + ``` + + then run it (it will block): + + ```console + student@os:~/.../drills/tasks/memory-areas/support$ ./hello + Hello, world! + ``` + + In another terminal, use the command below to show the memory areas of the process: + + ```console + student@os:~/.../drills/tasks/memory-areas/support$ pmap $(pidof hello) + 8220: ./hello + 000055c0bef4b000 8K r-x-- hello + 000055c0bf14c000 4K r---- hello + 000055c0bf14d000 4K rw--- hello + 000055c0bf454000 132K rw--- [ anon ] + 00007f2a9e4a5000 1948K r-x-- libc-2.27.so + 00007f2a9e68c000 2048K ----- libc-2.27.so + 00007f2a9e88c000 16K r---- libc-2.27.so + 00007f2a9e890000 8K rw--- libc-2.27.so + 00007f2a9e892000 16K rw--- [ anon ] + 00007f2a9e896000 164K r-x-- ld-2.27.so + 00007f2a9ea8c000 8K rw--- [ anon ] + 00007f2a9eabf000 4K r---- ld-2.27.so + 00007f2a9eac0000 4K rw--- ld-2.27.so + 00007f2a9eac1000 4K rw--- [ anon ] + 00007ffee6471000 132K rw--- [ stack ] + 00007ffee6596000 12K r---- [ anon ] + 00007ffee6599000 4K r-x-- [ anon ] + ffffffffff600000 4K --x-- [ anon ] + total 4520K + ``` + + The output is similar, but with fewer dynamic libraries than `bash`, since they are not used by the program. + +1. Make a program in another language of your choice that prints `Hello, world!` and sleeps and investigate it with `pmap`. + Note that in the case of interpreted languages (Python, Lua, Perl, Ruby, PHP, JavaScript etc.) you have to investigate the interpreter process. + +If you're having difficulties solving this exercise, go through [this](../../../reading/process-memory.md) reading material. + diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/Makefile b/chapters/data/process-memory/drills/tasks/memory-areas/support/Makefile new file mode 100644 index 0000000000..05f2cb6f6b --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/memory-areas/support/Makefile @@ -0,0 +1,44 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source and object files for alloc_size +SRC = hello.c +OBJ = $(SRC:.c=.o) + +# Binary name for alloc_size +BINARY = hello + +# Default rule: Build the binary +all: $(BINARY) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile alloc_size object file +$(OBJ): %.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create the alloc_size binary +$(BINARY): $(OBJ) $(LOGGER) + $(CC) $(CFLAGS) $(OBJ) $(LOGGER) -o $(BINARY) $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJ) $(BINARY) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean + diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/hello.c b/chapters/data/process-memory/drills/tasks/memory-areas/support/hello.c new file mode 100644 index 0000000000..c19721ba7d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/memory-areas/support/hello.c @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#include +#include + +int main(void) +{ + puts("Hello, world!"); + sleep(10000); + return 0; +} diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/CPPLINT.cfg b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/log.c b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/utils.h b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/.gitignore b/chapters/data/process-memory/drills/tasks/modify-areas/.gitignore new file mode 100644 index 0000000000..e92569d010 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/.gitignore @@ -0,0 +1 @@ +/hello diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/README.md b/chapters/data/process-memory/drills/tasks/modify-areas/README.md new file mode 100644 index 0000000000..9992e5400f --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/README.md @@ -0,0 +1,16 @@ +# Modifying Memory Region Size + +## Practice + +1. Comment out different parts of the `hello.c` program to notice differences in only specific areas (text, data, bss, heap, stack). + +1. Use a different argument (`order`) for the call to the `alloc_stack()` function. + See how it affects the stack size during runtime (investigate with `pmap`). + +1. Do a static build of `hello.c` and check the size of the memory areas, both statically and dynamically. + +1. The `extend_mem_area.py` Python script allocates a new string at each step by merging the two previous versions. + Start the program and investigate the resulting process at each allocation step. + Notice which memory area is updated and explain why. + +If you're having difficulties solving this exercise, go through [this](../../../reading/process-memory.md) reading material. diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/Makefile b/chapters/data/process-memory/drills/tasks/modify-areas/support/Makefile new file mode 100644 index 0000000000..05f2cb6f6b --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/Makefile @@ -0,0 +1,44 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source and object files for alloc_size +SRC = hello.c +OBJ = $(SRC:.c=.o) + +# Binary name for alloc_size +BINARY = hello + +# Default rule: Build the binary +all: $(BINARY) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile alloc_size object file +$(OBJ): %.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create the alloc_size binary +$(BINARY): $(OBJ) $(LOGGER) + $(CC) $(CFLAGS) $(OBJ) $(LOGGER) -o $(BINARY) $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJ) $(BINARY) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean + diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/extend_mem_area.py b/chapters/data/process-memory/drills/tasks/modify-areas/support/extend_mem_area.py new file mode 100755 index 0000000000..31e31e7e6c --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/extend_mem_area.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: BSD-3-Clause + +""" +Demonstrate impact of string (re)allocation at assignment. +""" + +s = "A" * 10000 +input("Press to start: ") + +for i in range(0, 10): + s = s + s + input("Press for next step: ") diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/hello.c b/chapters/data/process-memory/drills/tasks/modify-areas/support/hello.c new file mode 100644 index 0000000000..f97da2ba7c --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/hello.c @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#include +#include +#include + +#define __unused __attribute__((unused)) + +static const char __unused boring[8192] = { 69, }; +static char __unused dull[16384] = { 69, }; +static char __unused nada[32768]; + +static void alloc_heap(void) +{ + malloc(64 * 1024); + malloc(64 * 1024); + malloc(64 * 1024); + malloc(64 * 1024); +} + +static void alloc_stack(size_t order) +{ + char __unused fill[32768]; + + fill[0] = 'a'; + fill[32767] = 'b'; + + if (order == 0) + sleep(1000); + else + alloc_stack(order-1); +} + +int main(void) +{ + puts("Hello, world!"); + alloc_heap(); + alloc_stack(5); + sleep(10000); + + return 0; +} diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/CPPLINT.cfg b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/log.c b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/utils.h b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/.gitignore b/chapters/data/process-memory/drills/tasks/static-dynamic/.gitignore new file mode 100644 index 0000000000..580e551cc4 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/.gitignore @@ -0,0 +1,2 @@ +/hello-static +/hello-dynamic diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/README.md b/chapters/data/process-memory/drills/tasks/static-dynamic/README.md new file mode 100644 index 0000000000..95b8113838 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/README.md @@ -0,0 +1,22 @@ +# Memory Layout of Statically-Linked and Dynamically-Linked Executables + +## Practice + +1. Let's investigate another static executable / process. + + If not already installed, install the `busybox-static` package on your system. + On Debian/Ubuntu systems, use: + + ```console + student@os:~$ sudo apt install busybox-static + ``` + + Start a process using: + + ```console + student@os:~$ busybox sleep 1000 + ``` + + Investigate the process using `pmap` and the executable using `size`. + +If you're having difficulties solving this exercise, go through [this](../../../reading/process-memory.md) reading material. diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/Makefile b/chapters/data/process-memory/drills/tasks/static-dynamic/support/Makefile new file mode 100644 index 0000000000..05f2cb6f6b --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/support/Makefile @@ -0,0 +1,44 @@ +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source and object files for alloc_size +SRC = hello.c +OBJ = $(SRC:.c=.o) + +# Binary name for alloc_size +BINARY = hello + +# Default rule: Build the binary +all: $(BINARY) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile alloc_size object file +$(OBJ): %.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +# Rule to create the alloc_size binary +$(BINARY): $(OBJ) $(LOGGER) + $(CC) $(CFLAGS) $(OBJ) $(LOGGER) -o $(BINARY) $(LDFLAGS) + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJ) $(BINARY) + @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + +.PHONY: all clean + diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/hello.c b/chapters/data/process-memory/drills/tasks/static-dynamic/support/hello.c new file mode 100644 index 0000000000..c19721ba7d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/support/hello.c @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#include +#include + +int main(void) +{ + puts("Hello, world!"); + sleep(10000); + return 0; +} diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/CPPLINT.cfg b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/log.c b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/log.c new file mode 100644 index 0000000000..3aebbfc1a4 --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/log.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct { + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + + +static const char *level_strings[] = { + "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" +}; + +#ifdef LOG_USE_COLOR +static const char *level_colors[] = { + "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" +}; +#endif + + +static void stdout_callback(log_Event *ev) { + char buf[16]; + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf( + ev->udata, "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, level_colors[ev->level], level_strings[ev->level], + ev->file, ev->line); +#else + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void file_callback(log_Event *ev) { + char buf[64]; + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf( + ev->udata, "%s %-5s %s:%d: ", + buf, level_strings[ev->level], ev->file, ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + + +static void lock(void) { + if (L.lock) { L.lock(true, L.udata); } +} + + +static void unlock(void) { + if (L.lock) { L.lock(false, L.udata); } +} + + +const char* log_level_string(int level) { + return level_strings[level]; +} + + +void log_set_lock(log_LockFn fn, void *udata) { + L.lock = fn; + L.udata = udata; +} + + +void log_set_level(int level) { + L.level = level; +} + + +void log_set_quiet(bool enable) { + L.quiet = enable; +} + + +int log_add_callback(log_LogFn fn, void *udata, int level) { + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + + +int log_add_fp(FILE *fp, int level) { + return log_add_callback(file_callback, fp, level); +} + + +static void init_event(log_Event *ev, void *udata) { + if (!ev->time) { + time_t t = time(NULL); + ev->time = localtime(&t); + } + ev->udata = udata; +} + + +void log_log(int level, const char *file, int line, const char *fmt, ...) { + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/log.h b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/log.h new file mode 100644 index 0000000000..1229b481dd --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/log/log.h @@ -0,0 +1,59 @@ +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char* log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.c new file mode 100644 index 0000000000..821874dc3d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.c @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket functions + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "utils/log/log.h" +#include "utils/sock/sock_util.h" + +/* + * Connect to a TCP server identified by name (DNS name or dotted decimal + * string) and port. + */ + +int tcp_connect_to_server(const char *name, unsigned short port) +{ + struct hostent *hent; + struct sockaddr_in server_addr; + int s; + int rc; + + hent = gethostbyname(name); + DIE(hent == NULL, "gethostbyname"); + + s = socket(PF_INET, SOCK_STREAM, 0); + DIE(s < 0, "socket"); + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, + sizeof(server_addr.sin_addr.s_addr)); + + rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); + DIE(rc < 0, "connect"); + + return s; +} + +int tcp_close_connection(int sockfd) +{ + int rc; + + rc = shutdown(sockfd, SHUT_RDWR); + DIE(rc < 0, "shutdown"); + + return close(sockfd); +} + +/* + * Create a server socket. + */ + +int tcp_create_listener(unsigned short port, int backlog) +{ + struct sockaddr_in address; + int listenfd; + int sock_opt; + int rc; + + listenfd = socket(PF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + sock_opt = 1; + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + &sock_opt, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons(port); + address.sin_addr.s_addr = INADDR_ANY; + + rc = bind(listenfd, (SSA *) &address, sizeof(address)); + DIE(rc < 0, "bind"); + + rc = listen(listenfd, backlog); + DIE(rc < 0, "listen"); + + return listenfd; +} + +/* + * Use getpeername(2) to extract remote peer address. Fill buffer with + * address format IP_address:port (e.g. 192.168.0.1:22). + */ + +int get_peer_address(int sockfd, char *buf, size_t len) +{ + struct sockaddr_in addr; + socklen_t addrlen = sizeof(struct sockaddr_in); + + if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) + return -1; + + snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); + + return 0; +} diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.h new file mode 100644 index 0000000000..e2a41e936d --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.h @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * + * Useful socket macros and structures + */ + +#ifndef SOCK_UTIL_H_ +#define SOCK_UTIL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +/* default backlog for listen(2) system call */ +#define DEFAULT_LISTEN_BACKLOG 5 + +/* "shortcut" for struct sockaddr structure */ +#define SSA struct sockaddr + + +int tcp_connect_to_server(const char *name, unsigned short port); +int tcp_close_connection(int s); +int tcp_create_listener(unsigned short port, int backlog); +int get_peer_address(int sockfd, char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/utils.h b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/data/process-memory/reading/process-memory.md b/chapters/data/process-memory/reading/process-memory.md new file mode 100644 index 0000000000..541c305938 --- /dev/null +++ b/chapters/data/process-memory/reading/process-memory.md @@ -0,0 +1,457 @@ +# Process Memory + +## Memory Regions + +To better manage a program's memory, the operating systems creates an address space for each process. +The address space is compartmentalized in multiple areas, each with its own role. +Memory addresses use different permissions to decide what actions are allowed. + +Let's investigate the memory areas of a given process. +We use `pmap` to see the memory layout of a running process. +The command below shows the memory layout of the current shell process: + +```console +student@os:~$ pmap -p $$ +1127: /bin/bash +000055fb4d77d000 1040K r-x-- /bin/bash +000055fb4da80000 16K r---- /bin/bash +000055fb4da84000 36K rw--- /bin/bash +000055fb4da8d000 40K rw--- [ anon ] +000055fb4e9bb000 1604K rw--- [ anon ] +00007f8fcf670000 4480K r---- /usr/lib/locale/locale-archive +00007f8fcfad0000 44K r-x-- /lib/x86_64-linux-gnu/libnss_files-2.27.so +00007f8fcfadb000 2044K ----- /lib/x86_64-linux-gnu/libnss_files-2.27.so +00007f8fcfcda000 4K r---- /lib/x86_64-linux-gnu/libnss_files-2.27.so +00007f8fcfcdb000 4K rw--- /lib/x86_64-linux-gnu/libnss_files-2.27.so +00007f8fcfcdc000 24K rw--- [ anon ] +00007f8fcfce2000 92K r-x-- /lib/x86_64-linux-gnu/libnsl-2.27.so +00007f8fcfcf9000 2044K ----- /lib/x86_64-linux-gnu/libnsl-2.27.so +00007f8fcfef8000 4K r---- /lib/x86_64-linux-gnu/libnsl-2.27.so +00007f8fcfef9000 4K rw--- /lib/x86_64-linux-gnu/libnsl-2.27.so +00007f8fcfefa000 8K rw--- [ anon ] +00007f8fcfefc000 44K r-x-- /lib/x86_64-linux-gnu/libnss_nis-2.27.so +00007f8fcff07000 2044K ----- /lib/x86_64-linux-gnu/libnss_nis-2.27.so +00007f8fd0106000 4K r---- /lib/x86_64-linux-gnu/libnss_nis-2.27.so +00007f8fd0107000 4K rw--- /lib/x86_64-linux-gnu/libnss_nis-2.27.so +00007f8fd0108000 32K r-x-- /lib/x86_64-linux-gnu/libnss_compat-2.27.so +00007f8fd0110000 2048K ----- /lib/x86_64-linux-gnu/libnss_compat-2.27.so +00007f8fd0310000 4K r---- /lib/x86_64-linux-gnu/libnss_compat-2.27.so +00007f8fd0311000 4K rw--- /lib/x86_64-linux-gnu/libnss_compat-2.27.so +00007f8fd0312000 1948K r-x-- /lib/x86_64-linux-gnu/libc-2.27.so +00007f8fd04f9000 2048K ----- /lib/x86_64-linux-gnu/libc-2.27.so +00007f8fd06f9000 16K r---- /lib/x86_64-linux-gnu/libc-2.27.so +00007f8fd06fd000 8K rw--- /lib/x86_64-linux-gnu/libc-2.27.so +00007f8fd06ff000 16K rw--- [ anon ] +00007f8fd0703000 12K r-x-- /lib/x86_64-linux-gnu/libdl-2.27.so +00007f8fd0706000 2044K ----- /lib/x86_64-linux-gnu/libdl-2.27.so +00007f8fd0905000 4K r---- /lib/x86_64-linux-gnu/libdl-2.27.so +00007f8fd0906000 4K rw--- /lib/x86_64-linux-gnu/libdl-2.27.so +00007f8fd0907000 148K r-x-- /lib/x86_64-linux-gnu/libtinfo.so.5.9 +00007f8fd092c000 2048K ----- /lib/x86_64-linux-gnu/libtinfo.so.5.9 +00007f8fd0b2c000 16K r---- /lib/x86_64-linux-gnu/libtinfo.so.5.9 +00007f8fd0b30000 4K rw--- /lib/x86_64-linux-gnu/libtinfo.so.5.9 +00007f8fd0b31000 164K r-x-- /lib/x86_64-linux-gnu/ld-2.27.so +00007f8fd0d24000 20K rw--- [ anon ] +00007f8fd0d53000 28K r--s- /usr/lib/x86_64-linux-gnu/gconv/gconv-modules.cache +00007f8fd0d5a000 4K r---- /lib/x86_64-linux-gnu/ld-2.27.so +00007f8fd0d5b000 4K rw--- /lib/x86_64-linux-gnu/ld-2.27.so +00007f8fd0d5c000 4K rw--- [ anon ] +00007ffff002f000 132K rw--- [ stack ] +00007ffff00c5000 12K r---- [ anon ] +00007ffff00c8000 4K r-x-- [ anon ] +ffffffffff600000 4K --x-- [ anon ] + total 24364K +``` + +Information will differ among different systems. + +See the different regions: + +* the first region, with `r-x` permissions is the `.text` (code) area +* the second region, with `r--` premissions is the `.rodata` area +* the third region, with `rw-` permissions is the `.data` area, for initialized global variables +* the fourth region, with `rw-` permissions is the `.bss` area +* the fifth region, with the `rw-` permissions is the dynamic data memory area, also known as heap +* there are multiple dynamic libraries mapped in the virtual address space of the process, each library with their own regions +* there is a `[stack]` memory region, with `rw-` permissions + +`pmap` also shows the total amount of virtual memory available to the process (`24364K`), as a total of the sizes of the regions. +Note that this is virtual memory, not actual physical memory used by the process. +For the process investigated above (with the `1127` pid) we could use the command below to show the total virtual size and physical size (also called _resident set size_): + +```console +student@os:~$ ps -o pid,rss,vsz -p $$ + PID RSS VSZ + 1127 1968 24364 +``` + +The resident size is `1968K`, much smaller than the virtual size. + +Note how each region has a size multiple of `4K`, this has to do with the memory granularity. +The operating system allocates memory in chunks of a predefined size (in our case `4K`) called pages. + +## Memory Layout of Statically-Linked and Dynamically-Linked Executables + +We want to see the difference in memory layout between the statically-linked and dynamically-linked executables. + +Enter the `drills/tasks/static-dynamic/support` directory and build the statically-linked and dynamically-linked executables `hello-static` and `hello-dynamic`: + +```console +student@os:~/.../drills/tasks/static-dynamic/support$ make +``` + +Now, by running the two programs and inspecting them with `pmap` on another terminal, we get the output: + +```console +student@os:~/.../drills/tasks/static-dynamic/support$ pmap $(pidof hello-static) +9714: ./hello-static +0000000000400000 876K r-x-- hello-static +00000000006db000 24K rw--- hello-static +00000000006e1000 4K rw--- [ anon ] +00000000017b5000 140K rw--- [ anon ] +00007ffc6f1d6000 132K rw--- [ stack ] +00007ffc6f1f9000 12K r---- [ anon ] +00007ffc6f1fc000 4K r-x-- [ anon ] +ffffffffff600000 4K --x-- [ anon ] + total 1196K + +student@os:~/.../drills/tasks/static-dynamic/support$ pmap $(pidof hello-dynamic) +9753: ./hello-dynamic +00005566e757f000 8K r-x-- hello-dynamic +00005566e7780000 4K r---- hello-dynamic +00005566e7781000 4K rw--- hello-dynamic +00005566e8894000 132K rw--- [ anon ] +00007fd434eb8000 1948K r-x-- libc-2.27.so +00007fd43509f000 2048K ----- libc-2.27.so +00007fd43529f000 16K r---- libc-2.27.so +00007fd4352a3000 8K rw--- libc-2.27.so +00007fd4352a5000 16K rw--- [ anon ] +00007fd4352a9000 164K r-x-- ld-2.27.so +00007fd43549f000 8K rw--- [ anon ] +00007fd4354d2000 4K r---- ld-2.27.so +00007fd4354d3000 4K rw--- ld-2.27.so +00007fd4354d4000 4K rw--- [ anon ] +00007ffe497ba000 132K rw--- [ stack ] +00007ffe497e3000 12K r---- [ anon ] +00007ffe497e6000 4K r-x-- [ anon ] +ffffffffff600000 4K --x-- [ anon ] + total 4520K +``` + +For the static executable, we can see there are no areas for dynamic libraries. +And the `.rodata` section has been coalesced in the `.text` area. + +We can see the size of each section in the two executables by using the `size` command: + +```console +student@os:~/.../drills/tasks/static-dynamic/support$ size hello-static +text data bss dec hex filename +893333 20996 7128 921457 e0f71 hello-static + +student@os:~/.../drills/tasks/static-dynamic/support$ size hello-dynamic +text data bss dec hex filename +4598 736 824 6158 180e hello-dynamic +``` + +## Modifying Memory Region Size + +We want to observe the update in size of memory regions for different instructions used in a program. + +Enter the `drills/tasks/modify-areas/support` directory. +Browse the contents of the `hello.c` file; +it is an update to the `hello.c` file in the `memory-areas/` directory. +Build the executable: + +```console +student@os:~/.../drills/tasks/modify-areas/support$ make +``` + +Use `size` to view the difference between the new executable and the one in the `memory-areas/` directory: + +```console +student@os:~/.../drills/tasks/modify-areas/support$ size hello + text data bss dec hex filename + 13131 17128 33592 63851 f96b hello + +student@os:~/.../drills/tasks/modify-areas/support$ size ../memory-areas/hello + text data bss dec hex filename + 4598 736 824 6158 180e ../memory-areas/hello +``` + +Explain the differences. + +Then use the `pmap` to watch the memory areas of the resulting processes from the two different executables. +We will see something like this for the new executable: + +```console +student@os:~/.../drills/tasks/modify-areas/support$ pmap $(pidof hello) +18254: ./hello +000055beff4d0000 16K r-x-- hello +000055beff6d3000 4K r---- hello +000055beff6d4000 20K rw--- hello +000055beff6d9000 32K rw--- [ anon ] +000055beffb99000 324K rw--- [ anon ] +00007f7b6c2e6000 1948K r-x-- libc-2.27.so +00007f7b6c4cd000 2048K ----- libc-2.27.so +00007f7b6c6cd000 16K r---- libc-2.27.so +00007f7b6c6d1000 8K rw--- libc-2.27.so +00007f7b6c6d3000 16K rw--- [ anon ] +00007f7b6c6d7000 164K r-x-- ld-2.27.so +00007f7b6c8cd000 8K rw--- [ anon ] +00007f7b6c900000 4K r---- ld-2.27.so +00007f7b6c901000 4K rw--- ld-2.27.so +00007f7b6c902000 4K rw--- [ anon ] +00007ffe2b196000 204K rw--- [ stack ] +00007ffe2b1d8000 12K r---- [ anon ] +00007ffe2b1db000 4K r-x-- [ anon ] +ffffffffff600000 4K --x-- [ anon ] + total 4840K +``` + +We notice the size increase of text, data, bss, heap and stack sections. + + +## Allocating and Deallocating Memory + +Memory areas in a process address space are static or dynamic. +Static memory areas are known at the beginning of process lifetime (i.e. at load-time), while dynamic memory areas are managed at runtime. + +`.text`, `.rodata`, `.data`, `.bss` are allocated at load-time and have a predefined size. +The stack and the heap and memory mappings are allocated at runtime and have a variable size. +For those, we say we use runtime allocation and deallocation. + +Memory allocation is implicit for the stack and explicit for the heap. +That is, we don't make a particular call to allocate data on the stack; +the compiler generates the code that the operating system uses to increase the stack when required. +For the heap, we use the `malloc()` and `free()` calls to explicitly allocate and deallocate memory. + +Omitting to deallocate memory results in memory leaks that hurt the resource use in the system. +Because of this, some language runtimes employ a garbage collector that automatically frees unused memory areas. +More than that, some languages (think of Python) provide no explicit means to allocate memory: you just define and use data. + +Let's enter the `drills/tasks/alloc_size/support` directory. +Browse the `alloc_size.c` file. +Build it: + +```console +student@os:~/.../drills/tasks/alloc_size/support$ make +``` + +Now see the update in the process layout, by running the program in one console: + +```console +student@os:~/.../drills/tasks/alloc_size/support$ ./alloc_size +Press key to allocate ... +[...] +``` + +And investigating it with `pmap` on another console: + +```console +student@os:~/.../drills/tasks/alloc_size/support$ pmap $(pidof alloc_size) +21107: ./alloc_size +000055de9d173000 8K r-x-- alloc_size +000055de9d374000 4K r---- alloc_size +000055de9d375000 4K rw--- alloc_size +000055de9deea000 132K rw--- [ anon ] +00007f1ea4fd4000 1948K r-x-- libc-2.27.so +00007f1ea51bb000 2048K ----- libc-2.27.so +00007f1ea53bb000 16K r---- libc-2.27.so +00007f1ea53bf000 8K rw--- libc-2.27.so +00007f1ea53c1000 16K rw--- [ anon ] +00007f1ea53c5000 164K r-x-- ld-2.27.so +00007f1ea55bb000 8K rw--- [ anon ] +00007f1ea55ee000 4K r---- ld-2.27.so +00007f1ea55ef000 4K rw--- ld-2.27.so +00007f1ea55f0000 4K rw--- [ anon ] +00007ffcf28e9000 132K rw--- [ stack ] +00007ffcf29be000 12K r---- [ anon ] +00007ffcf29c1000 4K r-x-- [ anon ] +ffffffffff600000 4K --x-- [ anon ] + total 4520K + +student@os:~/.../drills/tasks/alloc_size/support$ pmap $(pidof alloc_size) +21107: ./alloc_size +000055de9d173000 8K r-x-- alloc_size +000055de9d374000 4K r---- alloc_size +000055de9d375000 4K rw--- alloc_size +000055de9deea000 452K rw--- [ anon ] +00007f1ea4fd4000 1948K r-x-- libc-2.27.so +00007f1ea51bb000 2048K ----- libc-2.27.so +00007f1ea53bb000 16K r---- libc-2.27.so +00007f1ea53bf000 8K rw--- libc-2.27.so +00007f1ea53c1000 16K rw--- [ anon ] +00007f1ea53c5000 164K r-x-- ld-2.27.so +00007f1ea55bb000 8K rw--- [ anon ] +00007f1ea55ee000 4K r---- ld-2.27.so +00007f1ea55ef000 4K rw--- ld-2.27.so +00007f1ea55f0000 4K rw--- [ anon ] +00007ffcf28e9000 132K rw--- [ stack ] +00007ffcf29be000 12K r---- [ anon ] +00007ffcf29c1000 4K r-x-- [ anon ] +ffffffffff600000 4K --x-- [ anon ] + total 4840K + +student@os:~/.../drills/tasks/alloc_size/support$ pmap $(pidof alloc_size) +21107: ./alloc_size +000055de9d173000 8K r-x-- alloc_size +000055de9d374000 4K r---- alloc_size +000055de9d375000 4K rw--- alloc_size +000055de9deea000 420K rw--- [ anon ] +00007f1ea4fd4000 1948K r-x-- libc-2.27.so +00007f1ea51bb000 2048K ----- libc-2.27.so +00007f1ea53bb000 16K r---- libc-2.27.so +00007f1ea53bf000 8K rw--- libc-2.27.so +00007f1ea53c1000 16K rw--- [ anon ] +00007f1ea53c5000 164K r-x-- ld-2.27.so +00007f1ea55bb000 8K rw--- [ anon ] +00007f1ea55ee000 4K r---- ld-2.27.so +00007f1ea55ef000 4K rw--- ld-2.27.so +00007f1ea55f0000 4K rw--- [ anon ] +00007ffcf28e9000 132K rw--- [ stack ] +00007ffcf29be000 12K r---- [ anon ] +00007ffcf29c1000 4K r-x-- [ anon ] +ffffffffff600000 4K --x-- [ anon ] + total 4808K +``` + +The three runs above of the `pmap` command occur before the allocation, after allocation and before deallocation and after deallocation. +Notice the update toe the 4th section, the heap. + +Now, let's see what happens behind the scenes. +Run the executable under `ltrace` and `strace`: + +```console +student@os:~/.../drills/tasks/alloc_size/support$ ltrace ./alloc_size +malloc(32768) = 0x55e33f490b10 +printf("New allocation at %p\n", 0x55e33f490b10New allocation at 0x55e33f490b10 +) = 33 +[...] +free(0x55e33f490b10) = +[...] + +student@os:~/.../drills/tasks/alloc_size/support$ strace ./alloc_size +[...] +write(1, "New allocation at 0x55ab98acfaf0"..., 33New allocation at 0x55ab98acfaf0 +) = 33 +write(1, "New allocation at 0x55ab98ad7b00"..., 33New allocation at 0x55ab98ad7b00 +) = 33 +brk(0x55ab98b08000) = 0x55ab98b08000 +write(1, "New allocation at 0x55ab98adfb10"..., 33New allocation at 0x55ab98adfb10 +) = 33 +write(1, "Press key to deallocate ...", 27Press key to deallocate ...) = 27 +read(0, +"\n", 1024) = 1 +brk(0x55ab98b00000) = 0x55ab98b00000 +[...] +``` + +The resulting output above shows us the following: + +* `malloc()` and `free()` library calls both map to the [`brk` syscall](https://man7.org/linux/man-pages/man2/sbrk.2.html), a syscall that updates the end of the heap (called **program break**). +* Multiple `malloc()` calls map to a single `brk` syscall for efficiency. + `brk` is called to preallocate a larger chunk of memory that `malloc` will then use. + +Update the `ALLOC_SIZE_KB` macro in the `alloc_size.c` file to `256`. +Rebuild the program and rerun it under `ltrace` and `strace`: + +```console +student@os:~/.../drills/tasks/alloc_size/support$ ltrace ./alloc_size +[...] +malloc(262144) = 0x7f4c016a9010 +[...] +free(0x7f4c016a9010) = +[...] + +student@os:~/.../drills/tasks/alloc_size/support$ strace ./alloc_size +[...] +mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7feee19f2000 +write(1, "New allocation at 0x7feee19f2010"..., 33New allocation at 0x7feee19f2010 +) = 33 +mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7feee19b1000 +write(1, "New allocation at 0x7feee19b1010"..., 33New allocation at 0x7feee19b1010 +) = 33 +write(1, "Press key to deallocate ...", 27Press key to deallocate ...) = 27 +read(0, +"\n", 1024) = 1 +munmap(0x7feee19b1000, 266240) = 0 +[...] +``` + +For the new allocation size, notice that the remarks above don't hold: + +* `malloc()` now invokes the `mmap` syscall, while `free()` invokes the `munmap` syscall. +* Each `malloc()` calls results in a separate `mmap` syscall. + +This is a behavior of the `malloc()` in libc, documented in the [manual page](https://man7.org/linux/man-pages/man3/malloc.3.html#NOTES). +A variable `MALLOC_THRESHOLD` holds the size after which `mmap` is used, instead of `brk`. +This is based on a heuristic of using the heap or some other area in the process address space. + +## Memory Mapping + +The `mmap` syscall is used to allocate memory as _anonymous mapping_, that is reserving memory in the process address space. +An alternate use is for mapping files in the memory address space. +Mapping of files is done by the loader for executables and libraries. +That is why, in the output of `pmap`, there is a column with a filename. + +Mapping of a file results in getting a pointer to its contents and then using that pointer. +This way, reading and writing to a file is an exercise of pointer copying, instead of the use of `read` / `write`-like system calls. + +In the `drills/tasks/copy/support` folder, there are two source code files and two scripts: + +* `read_write_copy.c` implements copying with `read` / `write` syscalls +* `mmap_copy.c` implements copying using `mmap` +* `generate.sh` script generates the input file `in.dat` +* `benchmark_cp.sh` script runs the two executables `mmap_copy` and `read_write_copy` + +Open the two source code files and investigate them. +You will notice that the `open()` system call has the following prototype `int open(const char *pathname, int flags)`. +The argument `flags` must include one of the following access modes: `O_RDONLY`, `O_WRONLY`, or `O_RDWR` - indicating that the file is opened in read-only, write-only, or read/write mode. +You can add an additional flag - `O_CREAT` - that will create a new file with `pathname` if the file does not already exist. +This is only the case when opening the file for writing (`O_WRONLY` or `O_RDWR`). +If `O_CREAT` is set, a third argument `mode_t mode` is required for the `open()` syscall. +The `mode` argument specifies the permissions of the newly created file. +For example: + +```c +// If DST_FILENAME exists it will be open in read/write mode and truncated to length 0 +// If DST_FILENAME does not exist, a file at the path DST_FILENAME will be create with 644 permissions +dst_fd = open(DST_FILENAME, O_RDWR | O_CREAT | O_TRUNC, 0644); +``` + +Let's generate the input file: + +```console +student@os:~/.../drills/tasks/copy/support$ ./generate.sh +``` + +and let's build the two executable files: + +```console +student@os:~/.../drills/tasks/copy/support$ make +``` + +Run the `benchmark_cp.sh` script: + +Run the script in your local environment, not in the Docker container. +Docker does not have permission to write to `/proc/sys/vm/drop_caches` file. + +```console +student@os:~/.../drills/tasks/copy/support$ ./benchmark_cp.sh +Benchmarking mmap_copy on in.dat +time passed 54015 microseconds + +Benchmarking read_write_copy on in.dat +time passed 42011 microseconds +``` + +Run the script a few more times. +As you can see, there isn't much of a difference between the two approaches. +Although we would have expected the use of multiple system calls to cause overhead, it's too little compared to the memory copying overhead. + +If you inspect `benchmark_cp.sh`, you will notice a weird-looking command `sh -c "sync; echo 3 > /proc/sys/vm/drop_caches"`. +This is used to disable a memory optimization that the kernel does. +It's called "buffer cache" and it's a mechanism by which the kernel caches data blocks from recently accessed files in memory. +You will get more detailed information about this in the I/O chapter. + +Browse the two source code files (`mmap_copy.c` and `read_write_copy.c`) for a glimpse on how the two types of copies are implemented. \ No newline at end of file diff --git a/chapters/data/slides.mdpp b/chapters/data/slides.mdpp new file mode 100644 index 0000000000..d7635f3d29 --- /dev/null +++ b/chapters/data/slides.mdpp @@ -0,0 +1,18 @@ +--- +title: "OS: Data" +revealOptions: + background-color: 'aquamarine' + transition: 'none' + slideNumber: true + autoAnimateDuration: 0.0 +--- + +# Data + +1. [Data in Software and Computing](#data) +1. [Perspectives of Data](#perspectives-of-data) +1. [Virtual Memory](#virtual-memory) + +!INCLUDE "overview/slides/data.md" +!INCLUDE "perspectives/slides/perspectives-of-data.md" +!INCLUDE "virtual-memory/slides/virtual-memory.md" diff --git a/chapters/data/virtual-memory/media/OSVM.svg b/chapters/data/virtual-memory/media/OSVM.svg new file mode 100644 index 0000000000..e035b7b648 --- /dev/null +++ b/chapters/data/virtual-memory/media/OSVM.svg @@ -0,0 +1 @@ +
pid
gid
uid
canary_value
...
mm
...
pid...
task_struct
task_struct
mmap
map_count
...
total_vm
shared_vm
...
mmap...
mm_struct
mm_struct
vm_start
vm_end
...
vm_page_prot
...

vm_start...
vm_area_struct
vm_area_structText is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/media/Swap-arch.svg b/chapters/data/virtual-memory/media/Swap-arch.svg new file mode 100644 index 0000000000..c2a6fc263b --- /dev/null +++ b/chapters/data/virtual-memory/media/Swap-arch.svg @@ -0,0 +1 @@ +
M1
M1
M2
M2
M3
M3
CPU
CPU
Cache
Cache
MMU
MMU
TLB
TLB
Page Table
Page Table
Main Memory
Main Memory
...
...
TLB Miss
TLB Miss
Cache Miss
Cache Miss
Processing
Unit
Processing...
Virtual Address
Virtual Address
Physical
Address
Physical...
Operating System
Operating System
(1)
(1)
(3)
(3)
Page Fault
Page Fault
(4)
(4)
(2)
(2)
(5)
(5)
Disk
Disk
M
M
(3.1) Swap Page M
(3.1) Swap Page M
M
M
(3.2)
(3.2)Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/media/Swap.svg b/chapters/data/virtual-memory/media/Swap.svg new file mode 100644 index 0000000000..78cab23db0 --- /dev/null +++ b/chapters/data/virtual-memory/media/Swap.svg @@ -0,0 +1 @@ +
P1
P1
P2
P2
P3
P3
2GB
2GB
2GB
2GB
2GB
2GB


...
...
Virtual Memory
Virtual Memory
Physical Memory
Physical Memory
Hard Disk
Hard Disk
...
...
...
...
...
...
...
...Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/media/VM-1.svg b/chapters/data/virtual-memory/media/VM-1.svg new file mode 100644 index 0000000000..31900c96a6 --- /dev/null +++ b/chapters/data/virtual-memory/media/VM-1.svg @@ -0,0 +1,4 @@ + + + +
Virtual Memory
Virtual Memory
Physical Memory
Physical Memory
Virtual Address
Virtual Address
Data items
Data items
Physical Address
Physical Address
Data Item
Data ItemText is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/media/VM.svg b/chapters/data/virtual-memory/media/VM.svg new file mode 100644 index 0000000000..0c895253c2 --- /dev/null +++ b/chapters/data/virtual-memory/media/VM.svg @@ -0,0 +1,4 @@ + + + +
M1
M1
M2
M2
M3
M3
CPU
CPU
Cache
Cache
MMU
MMU
TLB
TLB
Page Table
Page Table
Main Memory
Main Memory
...
...
TLB Miss
TLB Miss
Cache Miss
Cache Miss
Processing
Unit
Processing...
Virtual Address
Virtual Address
Physical
Address
Physical...
...
...
Operating System
Operating System
(1)
(1)
(3)
(3)
Page Fault
Page Fault
(4)
(4)
(2)
(2)
(5)
(5)Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/media/pages.svg b/chapters/data/virtual-memory/media/pages.svg new file mode 100644 index 0000000000..9a18723f18 --- /dev/null +++ b/chapters/data/virtual-memory/media/pages.svg @@ -0,0 +1,4 @@ + + + +
Virtual Address
Virtual Address
Data items
Data items
Physical Address
Physical Address
Data Item
Data Item
Virtual Memory
Virtual Memory
Physical Memory
Physical Memory
Pages
Pages
Pages
Pages
Pages
Pages
Pages
Pages
Frames
Frames
Frames
Frames
Frames
Frames
Frames
FramesText is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/media/vm-example1.svg b/chapters/data/virtual-memory/media/vm-example1.svg new file mode 100644 index 0000000000..89b612d13e --- /dev/null +++ b/chapters/data/virtual-memory/media/vm-example1.svg @@ -0,0 +1,4 @@ + + + +
Program - Pid X
Program - Pid X
Virtual Memory
Virtual Memory
Physical Memory
Physical Memory
Pid X
Pid X
int* arr = malloc(2000*sizeof(int));
int* arr = malloc(2000*sizeof(i...
(1)
(1)
K  (U)
K  (U)
K + 4096 (U)  
K + 4096 (U)  Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/media/vm-example2.svg b/chapters/data/virtual-memory/media/vm-example2.svg new file mode 100644 index 0000000000..9924901235 --- /dev/null +++ b/chapters/data/virtual-memory/media/vm-example2.svg @@ -0,0 +1,4 @@ + + + +
Program - Pid X
Program - Pid X
Virtual Memory
Virtual Memory
Physical Memory
Physical Memory
Pid X
Pid X
int* arr = malloc(2000*sizeof(int));
int* arr = malloc(2000*sizeof(i...
arr[1500] = 2;
arr[1500] = 2;
(1)
(1)
(2)
(2)
K  (U)
K  (U)
K + 4096 (U)  
K + 4096 (U)  Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/media/vm-example3.svg b/chapters/data/virtual-memory/media/vm-example3.svg new file mode 100644 index 0000000000..7d105456f6 --- /dev/null +++ b/chapters/data/virtual-memory/media/vm-example3.svg @@ -0,0 +1,4 @@ + + + +
Program - Pid X
Program - Pid X
Virtual Memory
Virtual Memory
Physical Memory
Physical Memory
Pid X
Pid X
int* arr = malloc(2000*sizeof(int));
int* arr = malloc(2000*sizeof(i...
arr[1500] = 2;
arr[1500] = 2;
(1)
(1)
(2)
(2)
K  (U)
K  (U)
K + 4096 (Addr1)  
K + 4096 (Addr1)  
(3)
(3)Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/media/vm-example4.svg b/chapters/data/virtual-memory/media/vm-example4.svg new file mode 100644 index 0000000000..50a506332b --- /dev/null +++ b/chapters/data/virtual-memory/media/vm-example4.svg @@ -0,0 +1,4 @@ + + + +
Program - Pid X
Program - Pid X
Virtual Memory
Virtual Memory
Physical Memory
Physical Memory
Pid X
Pid X
int* arr = malloc(2000*sizeof(int));
int* arr = malloc(2000*sizeof(i...
arr[1500] = 2;
arr[1500] = 2;
(1)
(1)
(2)
(2)
K  (U)
K  (U)
K + 4096 (Addr1)  
K + 4096 (Addr1)  
(3)
(3)
arr[1503] = 2;
arr[1503] = 2;
(4)
(4)
(5)
(5)Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/media/vm-example5.svg b/chapters/data/virtual-memory/media/vm-example5.svg new file mode 100644 index 0000000000..55dc9edc94 --- /dev/null +++ b/chapters/data/virtual-memory/media/vm-example5.svg @@ -0,0 +1,4 @@ + + + +
Program - Pid X
Program - Pid X
Virtual Memory
Virtual Memory
Physical Memory
Physical Memory
Pid X
Pid X
int* arr = malloc(2000*sizeof(int));
int* arr = malloc(2000*sizeof(i...
arr[1500] = 2;
arr[1500] = 2;
(1)
(1)
(2)
(2)
K (Addr2)
K (Addr2)
K + 4096 (Addr1)  
K + 4096 (Addr1)  
(3)
(3)
arr[1503] = 2;
arr[1503] = 2;
(4)
(4)
(5)
(5)
arr[0] = 2;
arr[0] = 2;
(6)
(6)
(6)
(6)Text is not SVG - cannot display \ No newline at end of file diff --git a/chapters/data/virtual-memory/slides/virtual-memory.md b/chapters/data/virtual-memory/slides/virtual-memory.md new file mode 100644 index 0000000000..b34d58fa61 --- /dev/null +++ b/chapters/data/virtual-memory/slides/virtual-memory.md @@ -0,0 +1,290 @@ +--- + +## Virtual Memory + +---- + +### Virtual Memory + +![VM1](../media/VM-1.svg) + +---- + +### Virtual Memory + +* Programs work with virtual memory as if it was physical memory +* Each process can address the entire address space (potentially, more than the size of RAM) => simplicity +* Each process is encapsulated (cannot have access to another process's memory) => security +* Penalties: extra hardware and software needed, extra work is done when accessing memory + +---- + +### How Do We Store Data? + +* When we rent a storage unit, its size is fixed +* Even if we store a coin, we still get a moderately large storage unit +* The same goes for memory + +---- + +### Pages/Frames + +![PG](../media/pages.svg) + +---- + +### Pages/Frames + +* A page is a virtual memory unit +* A frame is a physical memory unit +* A page/frame size is typically 4 KB +* Virtual pages are mapped to physical frames +* The mapping is stored in a page table +* Each page has access rights + +---- + +### An Example - How does it work? + +```c +int *arr = malloc(2000 * sizeof(int)); +arr[1500] = 7; +``` + +---- + +### An Example - How does it work? + +![EX1](../media/vm-example1.svg) + +---- + +### What About Physical Memory? + +* Should we also allocate physical memory upfront? +* What if the program does not end up using that memory? +* Maybe we should allocate only when a memory location is actually used +* We mark allocated pages that don't have a corresponding frame with (U) + +---- + +### An Example - How does it work? + +![EX2](../media/vm-example2.svg) + +---- + +### An Example - How does it work? + +* The requested address is checked to see if it was allocated (virtually) +* If not => _segmentation fault_ +* If yes, then we must retrieve it +* Wait, what about physical frames? + +---- + +### What About Physical Memory? (part 2) + + +![EX3](../media/vm-example3.svg) + +---- + +### What About Physical Memory? (part 2) + +* This strategy is called **demand paging** +* We only allocate physical frames once they are needed +* The pages that have been allocated, along with their mapping status, are stored in a ledger, called the page table +* `demo/alloc_physical/` + +---- + +### An Example - How does it work? + +![EX4](../media/vm-example4.svg) + +---- + +### An Example - How does it work? + +* Subsequent accesses to virtual addresses that have been mapped to a physical frame simply check that the virtual address is valid +* No need to allocate any other physical frames + +---- + +### An Example - How does it work? + +![EX5](../media/vm-example5.svg) + +---- + +### Benefits of Virtual Memory + +* Each process can have a different memory mapping +* Physical RAM can be mapped into multiple processes at once + - shared memory +* Memory regions can have access permissions + - read, write, execute +* [Quiz](quiz/vm-draw.md) + +--- + +### Page Faults + +* Occur when a virtual address access cannot be performed +* Examples: + - A valid virtual address is accessed, but no physical frame is allocated + - An invalid virtual address is accessed + +---- + +### Page Fault Handlers + +* Page faults are handled by pre-registered OS routines: **page fault handlers** +* A hardware unit generates a page fault that is treated by the operating system +* Once a page fault is addressed, the instruction that caused it is re-executed (assuming the OS was able to treat the page fault) + +--- + +### Implementation of Virtual Memory + +![VM](../media/VM.svg) + +---- + +### CPU + +* The CPU always works with virtual addresses +* When a load is made, the MMU intercepts the address + +---- + +### Memory Management Unit (MMU) + +* hardware component that checks whether the virtual address is valid +* stores the mappings (**page tables**) between virtual and physical addresses + +---- + +### The Page Table + +* maps virtual (page) addresses to physical (frame) addresses +* is typically stored in memory (RAM) +* For faster access, modern systems use ... + +---- + +### Translation Lookaside Buffers (TLB) + +* a cache of the page table +* two memory accesses (access to page table and access to actual data) are reduced to a TLB cache access and a memory access + +---- + +### Page Fault Triggers + +* If the virtual address is part of the page table, i.e. there is a physical counterpart address for it, the MMU forwards the physical address to the memory subsystem +* If not, a page fault is triggered + +---- + +### The Operating System + +* had previously registered a page fault handler +* the page fault causes the execution of the page fault handler (part of the operating system) + +---- + +### The Page Fault Handler + +* verifies the virtual address is valid, i.e. it was allocated; issues memory exception (i.e. _segmentation fault_) if not +* if the address is valid, verifies if there is an already allocated physical frame for the virtual page; allocates it if not +* verifies if the access rights allow this access +* updates the page table with the latest mappings +* reruns the instruction that caused the page fault + +---- + +### The Memory Subsystem + +* works only with physical addresses +* the MMU will forward only physical addresses, after it resolves the virtual address mapping +* [Quiz](quiz/swap.md) + +--- + +### Swapping + +* What happens if a process requires more memory than the available physical RAM? +* Frames that haven't been used in a while are evicted on the disk +* Once those frames are required they are brought up in RAM +* However, a large penalty will be incurred +* Typically, systems will freeze once the swap is used + +---- + +### Swapping + +![SWAP](../media/Swap.svg) + +---- + +### Swapping + +![SWAPARCH](../media/Swap-arch.svg) + +* [Quiz](quiz/swap.md) + +--- + +### Case Study: Index Out Of Bounds + +---- + +### What is the result of running the following code? + +```c +void main() +{ + /* some code, may contain allocations */ + + int *arr = malloc(2000 * sizeof(int)); + arr[2005] = 10; +} +``` + +* [Quiz](quiz/index-out-of-bounds.md) + +--- + +### OS Implementation Of Virtual Memory + +---- + +### OS Data Structures + +![OSVM](../media/OSVM.svg) + +* [Quiz](quiz/sdfsd) + +---- + +### OS Interface To Virtual Memory + +* `/proc//mem` - access to virtual memory + - `demo/proc_mem/` +* `/proc//page_map` - access to page mappings + - `demo/proc_pagemap/` +* `/dev/mem` - access to physical memory + - `demo/proc_pagemap/` + +--- + +## Final Quiz + +* [Quiz](quiz/vm-final.md) + +--- + +## The End + diff --git a/chapters/data/working-with-memory/drills/questions/memory-access.md b/chapters/data/working-with-memory/drills/questions/memory-access.md new file mode 100644 index 0000000000..cb7972b15d --- /dev/null +++ b/chapters/data/working-with-memory/drills/questions/memory-access.md @@ -0,0 +1,22 @@ +# Modify String + +## Question Text + +What happens if we introduce the code `cp[2] = 't'` in the program located in the `drills/tasks/memory-access/mem_access.c` file? + +## Question Answers + +- Compile time error because we are trying to modify a `const` pointer. + +- Compile time error because we are trying to modify a `const` value. + ++ Segmentation fault at runtime we are trying to modify read-only memory. + +- Program compiles and runs succesfully. + +## Feedback + +The declaration `char *const cp = "ConstLeString"` actually defines 2 memory locations. +One stores a constant pointer, `cp`, whereas the other stores the actual string. +The compiler thinks that `cp` is able to modify the memory location that it points therefore it passes compilation. +But at runtime a segmentation fault is issued because we are accessing data that is stored in read-only memory. diff --git a/content/chapters/data/lab/support/memory-access/.gitignore b/chapters/data/working-with-memory/drills/tasks/memory-access/.gitignore similarity index 100% rename from content/chapters/data/lab/support/memory-access/.gitignore rename to chapters/data/working-with-memory/drills/tasks/memory-access/.gitignore diff --git a/chapters/data/working-with-memory/drills/tasks/memory-access/README.md b/chapters/data/working-with-memory/drills/tasks/memory-access/README.md new file mode 100644 index 0000000000..07b0cd4fae --- /dev/null +++ b/chapters/data/working-with-memory/drills/tasks/memory-access/README.md @@ -0,0 +1,13 @@ +# Memory Access + +## Practice + +Navigate to the `drills/tasks/memory-access/support/` directory. +Inspect the `mem_access.c` source file. + +1. Describe each variable by completing its **(address, size, access rights)** tuple. + +1. Try to modify the `ca`, `cp` and `cp2` variables by assigning some other value to them. + Explain the behavior. + +If you're having difficulties solving this exercise, go through [this](../../../reading/working-with-memory.md) reading material. diff --git a/chapters/data/working-with-memory/drills/tasks/memory-access/support/Makefile b/chapters/data/working-with-memory/drills/tasks/memory-access/support/Makefile new file mode 100644 index 0000000000..7e4c4194c0 --- /dev/null +++ b/chapters/data/working-with-memory/drills/tasks/memory-access/support/Makefile @@ -0,0 +1,27 @@ +CC = gcc + +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) + +SRC = mem_access.c +OBJ = mem_access.o + +INCLUDES_DIR := $(MAKEFILE_DIR).. +CPPFLAGS += -I$(INCLUDES_DIR) + +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy +BINARY = mem_access + +all: $(BINARY) + +$(BINARY): $(OBJ) + $(CC) $^ $(LDFLAGS) -o $@ + +%.o: %.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + + +clean: + -rm -f $(BINARY) $(OBJ) + +.PHONY: all clean diff --git a/content/chapters/data/lab/support/memory-access/mem_access.c b/chapters/data/working-with-memory/drills/tasks/memory-access/support/mem_access.c similarity index 100% rename from content/chapters/data/lab/support/memory-access/mem_access.c rename to chapters/data/working-with-memory/drills/tasks/memory-access/support/mem_access.c diff --git a/content/chapters/data/lab/support/memory-corruption/.gitignore b/chapters/data/working-with-memory/drills/tasks/memory-corruption/.gitignore similarity index 100% rename from content/chapters/data/lab/support/memory-corruption/.gitignore rename to chapters/data/working-with-memory/drills/tasks/memory-corruption/.gitignore diff --git a/chapters/data/working-with-memory/drills/tasks/memory-corruption/README.md b/chapters/data/working-with-memory/drills/tasks/memory-corruption/README.md new file mode 100644 index 0000000000..543265b538 --- /dev/null +++ b/chapters/data/working-with-memory/drills/tasks/memory-corruption/README.md @@ -0,0 +1,20 @@ +# Memory Corruption + +For this practice item, you will need to identify the programming mistake that makes it possible to corrupt memory. + +Navigate to the 'drills/tasks/memory-corruption/support/` folder. +Inspect the source file `segfault.c`. + +1. What does the program do? (this could be a quiz in the final form) +1. Compile and run it. + What happens? +1. Debug the program and find the line that causes the segfault. + **Note**: Although using `printf()` calls is a viable option, we strongly suggest you use GDB. +1. Fix the program. +1. Analyze the corresponding Python and D implementation. + +What is the expected result in each case? +Why? +Run the programs and see what happens. + +If you're having difficulties solving this exercise, go through [this](../../../reading/working-with-memory.md) reading material. diff --git a/chapters/data/working-with-memory/drills/tasks/memory-corruption/support/Makefile b/chapters/data/working-with-memory/drills/tasks/memory-corruption/support/Makefile new file mode 100644 index 0000000000..e0461f68b8 --- /dev/null +++ b/chapters/data/working-with-memory/drills/tasks/memory-corruption/support/Makefile @@ -0,0 +1,20 @@ +BINARY = c_segfault + + +DC = gdc + +d_segfault: d_segfault.d + $(DC) -o $@ $^ + +clean:: + -rm -f d_segfault + +all: $(BINARY) + +$(BINARY): $(OBJS) $(LOGGER) + $(CC) $^ $(LDFLAGS) -o $@ $(LDLIBS) + +clean:: + -rm -f $(BINARY) + +.PHONY: all clean diff --git a/content/chapters/data/lab/support/memory-corruption/c_segfault.c b/chapters/data/working-with-memory/drills/tasks/memory-corruption/support/c_segfault.c similarity index 100% rename from content/chapters/data/lab/support/memory-corruption/c_segfault.c rename to chapters/data/working-with-memory/drills/tasks/memory-corruption/support/c_segfault.c diff --git a/content/chapters/data/lab/support/memory-corruption/d_segfault.d b/chapters/data/working-with-memory/drills/tasks/memory-corruption/support/d_segfault.d similarity index 100% rename from content/chapters/data/lab/support/memory-corruption/d_segfault.d rename to chapters/data/working-with-memory/drills/tasks/memory-corruption/support/d_segfault.d diff --git a/content/chapters/data/lab/support/memory-corruption/segfault.py b/chapters/data/working-with-memory/drills/tasks/memory-corruption/support/segfault.py similarity index 100% rename from content/chapters/data/lab/support/memory-corruption/segfault.py rename to chapters/data/working-with-memory/drills/tasks/memory-corruption/support/segfault.py diff --git a/content/chapters/data/lab/support/memory-protection/.gitignore b/chapters/data/working-with-memory/drills/tasks/memory-protection/.gitignore similarity index 100% rename from content/chapters/data/lab/support/memory-protection/.gitignore rename to chapters/data/working-with-memory/drills/tasks/memory-protection/.gitignore diff --git a/chapters/data/working-with-memory/drills/tasks/memory-protection/README.md b/chapters/data/working-with-memory/drills/tasks/memory-protection/README.md new file mode 100644 index 0000000000..7e6f05e526 --- /dev/null +++ b/chapters/data/working-with-memory/drills/tasks/memory-protection/README.md @@ -0,0 +1,63 @@ +# Memory Protection + +Let's navigate to the `drills/tasks/memory-protection/support/` directory and inspect the `mem_prot.c` source file. +The file uses different access types for the `data` variable and the `do_nothing` function. + +Build it: + +```console +student@os:~/.../memory-protection/support/$ make +gcc -g -Wall -Wextra -Werror -I../../../../../common/makefile/../utils -I../../../../../common/makefile/../utils/log -c -o mem_prot.o mem_prot.c +gcc mem_prot.o ../../../../../common/makefile/../utils/log/log.o -o mem_prot + +student@os:~/.../memory-protection/support/$ ./mem_prot +reading from .data section +writing to .data section +reading from .text section +executing .text section +``` + +All current actions in the program are valid. + +Let's uncomment each commented line in the program and try again: + +```console +student@os:~/.../memory-protection/support/$ ./mem_prot +reading from .data section +writing to .data section +reading from .text section +executing .text section +executing .data section +Segmentation fault (core dumped) +``` + +We now receive the dreaded _Segmentation fault_ message when we try to access a memory section with wrong permissions. + +Permissions come into play when we control the memory address via pointers. +But even for programming languages that don't offer pointers (such as Python) issues may still arise. + +In the `str.py` file, we look to modify `str[1]`, but this fails: + +```console +student@os:~/.../memory-protection/support/$ ./str.py +n, 110, n +Traceback (most recent call last): + File "./str.py", line 5, in + str[1] = 'z' +TypeError: 'str' object does not support item assignment +``` + +This fails because strings are, in Python, immutable. +Once a string is being created, it can not be modified; +you have to create a new string. + +## Practice + +Go to the `drills/tasks/memory-protection/support/` folder and solve the practice items below. + +1. Add a variable named `ro` that you define as `const`. + The variable will be placed on a read-only section (`.rodata`) such as that write and execution access would result in _Segmentation fault_. + + Access the `ro` variable and show that, indeed, for write and execution access, _Segmentation fault_ is issued. + +If you're having difficulties solving this exercise, go through [this](../../../reading/working-with-memory.md) reading material. diff --git a/chapters/data/working-with-memory/drills/tasks/memory-protection/support/Makefile b/chapters/data/working-with-memory/drills/tasks/memory-protection/support/Makefile new file mode 100644 index 0000000000..1761a0cb73 --- /dev/null +++ b/chapters/data/working-with-memory/drills/tasks/memory-protection/support/Makefile @@ -0,0 +1,24 @@ +CC = gcc + +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) + +SRC = mem_prot.c +OBJ = mem_prot.o + +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +BINARY = mem_prot + +all: $(BINARY) + +$(BINARY): $(OBJ) + $(CC) $^ $(LDFLAGS) -o $@ + +%.o: %.c + $(CC) $(CFLAGS) -c $< -o $@ + +clean: + -rm -f $(BINARY) $(OBJ) + +.PHONY: all clean diff --git a/content/chapters/data/lab/support/memory-protection/mem_prot.c b/chapters/data/working-with-memory/drills/tasks/memory-protection/support/mem_prot.c similarity index 100% rename from content/chapters/data/lab/support/memory-protection/mem_prot.c rename to chapters/data/working-with-memory/drills/tasks/memory-protection/support/mem_prot.c diff --git a/content/chapters/data/lab/support/memory-protection/str.py b/chapters/data/working-with-memory/drills/tasks/memory-protection/support/str.py similarity index 100% rename from content/chapters/data/lab/support/memory-protection/str.py rename to chapters/data/working-with-memory/drills/tasks/memory-protection/support/str.py diff --git a/content/chapters/data/lab/support/memory-alloc/.gitignore b/chapters/data/working-with-memory/guides/memory-alloc/.gitignore similarity index 100% rename from content/chapters/data/lab/support/memory-alloc/.gitignore rename to chapters/data/working-with-memory/guides/memory-alloc/.gitignore diff --git a/chapters/data/working-with-memory/guides/memory-alloc/README.md b/chapters/data/working-with-memory/guides/memory-alloc/README.md new file mode 100644 index 0000000000..8c8ec2d0bb --- /dev/null +++ b/chapters/data/working-with-memory/guides/memory-alloc/README.md @@ -0,0 +1,31 @@ +# Memory Allocation Strategy + +Navigate to the `guides/memory-alloc/support/` directory. +It contains 3 implementations of the same program in different languages: C, Python and D. +The program creates a list of entries, each entry storing a name and an id. +The purpose of this exercise is to present the different strategies that programming languages adopt to manage memory. + +## C + +The C implementation manages the memory manually. +You can observe that all allocations are performed via `malloc()` and the memory is freed using `free()`. +Arrays can be defined as static (on the stack) or dynamic (a pointer to some heap memory). +Stack memory doesn't need to be freed, hence static arrays are automatically deallocated. +Heap memory, however, is managed by the user, therefore it is the burden of the programmer to find the optimal memory strategy. +This offers the advantage that you can fine tune the memory usage depending on your application, but this comes with a cost: more often than not, managing memory is a highly complex error-prone task. + +## Python + +The Python implementation of the program has no notion of memory allocation. +It simply defines variables and the garbage collector takes care of allocating and deallocating memory. +Notice how the destructor is called automatically at some point when the garbage collector deems that the list is not used anymore. +Garbage collection lifts the burden of memory management from the user, however, it may be unsuitable for certain scenarios. +For example, real-time applications that need to take action immediately once a certain event occurs cannot use a garbage collector (GC). +That is because the GC usually stops the application to free dead objects. + +## D + +The previous 2 examples have showcased extreme situations: fully manual vs fully automatic memory management. +In D, both worlds are combined: variables may be allocated manually on the stack/heap or via the garbage collector (for brevity, `malloc()`-based allocation is not presented in this example). +Arrays that are allocated on the stack behave the same as in C, whereas arrays allocated with the garbage collector mimic Python lists. +Classes are also garbage collected. diff --git a/chapters/data/working-with-memory/guides/memory-alloc/support/Makefile b/chapters/data/working-with-memory/guides/memory-alloc/support/Makefile new file mode 100644 index 0000000000..0b3216cd5a --- /dev/null +++ b/chapters/data/working-with-memory/guides/memory-alloc/support/Makefile @@ -0,0 +1,26 @@ +CC = gcc + +SRC = c_memory_alloc.c +OBJ = c_memory_alloc.o +DEP = $(SRC:.c=.d) + +CFLAGS += -g -Wall -Wextra -MMD +LDFLAGS += -z lazy + +BINARY = c_memory_alloc + +all: $(BINARY) + +$(BINARY): $(OBJ) + $(CC) $^ $(LDFLAGS) -o $@ + +# Rule to generate .o and .d files +%.o: %.c + $(CC) $(CFLAGS) -c $< -o $@ + +-include $(DEP) + +clean: + -rm -f $(BINARY) $(OBJ) $(DEP) + +.PHONY: all clean diff --git a/content/chapters/data/lab/support/memory-alloc/alloc_size b/chapters/data/working-with-memory/guides/memory-alloc/support/alloc_size similarity index 100% rename from content/chapters/data/lab/support/memory-alloc/alloc_size rename to chapters/data/working-with-memory/guides/memory-alloc/support/alloc_size diff --git a/content/chapters/data/lab/support/memory-alloc/c_memory_alloc.c b/chapters/data/working-with-memory/guides/memory-alloc/support/c_memory_alloc.c similarity index 100% rename from content/chapters/data/lab/support/memory-alloc/c_memory_alloc.c rename to chapters/data/working-with-memory/guides/memory-alloc/support/c_memory_alloc.c diff --git a/content/chapters/data/lab/support/memory-alloc/d_memory_alloc.d b/chapters/data/working-with-memory/guides/memory-alloc/support/d_memory_alloc.d similarity index 100% rename from content/chapters/data/lab/support/memory-alloc/d_memory_alloc.d rename to chapters/data/working-with-memory/guides/memory-alloc/support/d_memory_alloc.d diff --git a/content/chapters/data/lab/support/memory-alloc/python_memory_alloc.py b/chapters/data/working-with-memory/guides/memory-alloc/support/python_memory_alloc.py similarity index 100% rename from content/chapters/data/lab/support/memory-alloc/python_memory_alloc.py rename to chapters/data/working-with-memory/guides/memory-alloc/support/python_memory_alloc.py diff --git a/content/chapters/data/lab/support/memory-vuln/.gitignore b/chapters/data/working-with-memory/guides/memory-vuln/.gitignore similarity index 100% rename from content/chapters/data/lab/support/memory-vuln/.gitignore rename to chapters/data/working-with-memory/guides/memory-vuln/.gitignore diff --git a/chapters/data/working-with-memory/guides/memory-vuln/README.md b/chapters/data/working-with-memory/guides/memory-vuln/README.md new file mode 100644 index 0000000000..d91338c58c --- /dev/null +++ b/chapters/data/working-with-memory/guides/memory-vuln/README.md @@ -0,0 +1,49 @@ +# Memory Vulnerabilities + +The purpose of this exercise is to provide examples on how memory corruption may occur and what are the safety guards implemented by different programming languages. + +Navigate to the `guides/memory-vuln/support/` directory. +It features 3 files, each showcasing what happens in case of actions that may lead to memory corruption. + +## C + +The C implementation showcases some of the design flaws of the language can lead to memory corruption. + +The first example demonstrates how a pointer to an expired stack frame may be leaked to an outer scope. +The C language does not implement any guards against such behavior, although data flow analysis could be used to detect such cases. + +The second example highlights the fact that C does not check any bounds when performing array operations. +This leads to all sorts of undefined behavior. +In this scenario, some random memory is overwritten with `5`. +The third example exhibits a manifestation of the previous design flaw, where the return address of the `main` function is overwritten with `0`, thus leading to a segmentation fault. + +Although today it seems obvious that such behavior should not be accepted, we should take into account that the context in which the C language was created was entirely different from today. +At that time the resource constraints - DRAM memory was around a few KBs, operating systems were in their infancy, branch predictors did not exist etc. - were overwhelming. +Moreover, security was not a concern because the internet basically did not exist. +As a consequence, the language was not developed with memory safety in mind. + +## Python + +Technically, it is not possible to do any memory corruption in Python (that is, if you avoid calling C functions from it). +Pointers do not formally exist, and any kind of array access is checked to be within its bounds. +The example simply showcases what happens when an out-of-bounds access is performed - an `IndexError` is thrown and execution halts. + +## D + +The D implementation uses almost the same code as the C implementation, but suffers from minor syntax modifications. +In essence, the two implement the same logic. +When compiling this code, it can be observed that the D compiler notices at compile time that an out-of-bounds access is performed. +This makes sense, since a static array cannot modify its length and therefore the compiler has all the information to spot the mistake. +The only way to make the code compile is to comment the faulting lines or to replace the out-of-bounds index with a correct one. +After doing so, the program compiles and we can see that memory corruption occurs. +However, D also has safety checks, however, these are not performed by default. +To enable such checks, the user must annotate a function with the `@safe` keyword: + +```d +int* bad() @safe +``` + +By doing so, the mechanical checks are enabled and a new set of criteria needs to be followed for the code to be accepted. +Taking the address of a local, doing pointer arithmetic, reinterpret casts, calling non-`@safe` functions etc. are not allowed in `@safe` code. +If any of these unsafe features are manually proven to be safe, the `@trusted` keyword may be used to disable the checks but still consider the code `@safe`. +This is to allow writing system code, which by its nature is unsafe. diff --git a/content/chapters/data/lab/support/memory-vuln/Makefile b/chapters/data/working-with-memory/guides/memory-vuln/support/Makefile similarity index 73% rename from content/chapters/data/lab/support/memory-vuln/Makefile rename to chapters/data/working-with-memory/guides/memory-vuln/support/Makefile index df5822f8e9..3dea179333 100644 --- a/content/chapters/data/lab/support/memory-vuln/Makefile +++ b/chapters/data/working-with-memory/guides/memory-vuln/support/Makefile @@ -1,7 +1,5 @@ BINARY = c_memory_vuln CFLAGS = -Wno-unused-variable -include ../../../../../common/makefile/single.mk - DC = gdc d_memory_vuln: d_memory_vuln.d diff --git a/content/chapters/data/lab/support/memory-vuln/c_memory_vuln.c b/chapters/data/working-with-memory/guides/memory-vuln/support/c_memory_vuln.c similarity index 100% rename from content/chapters/data/lab/support/memory-vuln/c_memory_vuln.c rename to chapters/data/working-with-memory/guides/memory-vuln/support/c_memory_vuln.c diff --git a/content/chapters/data/lab/support/memory-vuln/d_memory_vuln.d b/chapters/data/working-with-memory/guides/memory-vuln/support/d_memory_vuln.d similarity index 100% rename from content/chapters/data/lab/support/memory-vuln/d_memory_vuln.d rename to chapters/data/working-with-memory/guides/memory-vuln/support/d_memory_vuln.d diff --git a/content/chapters/data/lab/support/memory-vuln/python_vuln.py b/chapters/data/working-with-memory/guides/memory-vuln/support/python_vuln.py similarity index 100% rename from content/chapters/data/lab/support/memory-vuln/python_vuln.py rename to chapters/data/working-with-memory/guides/memory-vuln/support/python_vuln.py diff --git a/chapters/data/working-with-memory/reading/working-with-memory.md b/chapters/data/working-with-memory/reading/working-with-memory.md new file mode 100644 index 0000000000..cf89e8bf5a --- /dev/null +++ b/chapters/data/working-with-memory/reading/working-with-memory.md @@ -0,0 +1,47 @@ +# Working with Memory + +As previously stated, from a programmer's perspective, memory is abstracted into variables. +This hides most of the lower level abstractions. +Each variable is characterized by an address (or location in memory), type and access rights. +Some languages require that the developer spells out these attributes explicitly (**statically typed languages** - notable examples: C\C++, D, Java) whereas others deduce them by analyzing the context (**dynamically typed languages** - notable examples: Python, JavaScript). +Nevertheless, the language compiler needs to handle this information and, based on it, generate code that manages memory correctly and efficiently. + +## Memory Access + +Accessing memory is defined by reading or writing values to or from a variable. +From a programmer's perspective, this looks pretty straightforward: + +```c +int main(void) +{ + int a; // declare variable + a = 42; // write 42 to variable a + printf("%d\n", a); // read variable a and print its contents + + return 0; +} +``` + +However, from a lower level perspective, there are other attributes that need to be taken care of. +For instance, variable `a` needs to have a correspondent area that is reserved in memory. +That specific chunk of memory is described by an address and a size. +The address for `a` is automatically generated by going through multiple layers of abstractions, but the size is spelled out indirectly by the programmer by using the keyword `int`. +Another aspect is represented by the access rights for a specific memory area. +In our example, `a` is defined as being plain mutable, however, it is possible to declare constant variables which are stored in memory location with no writing rights. + +Using the above information, the compiler and the operating system co-work to allocate memory that can represent the contents of the variable. + +No matter what sort of language you are using, statically or dynamically typed, a variable is always described by the **(address, size, access rights)** triplet. +By using this triplet, the content of a variable is stored, retrieved or rewritten. + +## Memory Protection + +Memory contents (both code and data) are separated into sections or zones. +This makes it easier to manage. +More than that, it allows different zones to have different permissions. +This follows the [principle of least privilege](https://en.wikipedia.org/wiki/Principle_of_least_privilege) where only required permissions are part of a given section. + +Code is usually placed in a section (`.text`) with read and execute permissions; +no write permissions. +Variables are placed in different sections (`.data`, `.bss`, stack, heap) with read and write permissions; +no execute permissions. \ No newline at end of file diff --git a/config.yaml b/config.yaml index 8d08627d01..4de065e4f9 100644 --- a/config.yaml +++ b/config.yaml @@ -176,6 +176,137 @@ docusaurus: path: tasks/ subsections: - Applications/: app-investigation/ + + - Data: + - Slides: /build/embed_reveal/Data/Data.mdx + - Data Perspective: + subsections: + - Overview: + path: chapters/data/overview/ + extra: + - media/ + subsections: + - Reading: reading/overview.md + + - Working With Memory: + subsections: + - Working with Memory: + path: chapters/data/working-with-memory + extra: + - drills/questions/ + - drills/tasks/ + - guides/ + subsections: + - Reading: + path: reading/ + subsections: + - Working with Memory: working-with-memory.md + - Drills: + path: drills/ + subsections: + - Tasks: + path: tasks/ + subsections: + - Memory Access/: memory-access/ + - Memory Corruption/: memory-corruption/ + - Memory Protection/: memory-protection/ + + - Questions: + path: questions/ + subsections: + - Memory Access: memory-access.md + + - Guides: + path: guides/ + subsections: + - Memory Alloc/: memory-alloc/ + - Memory Vuln/: memory-vuln/ + + - Process Memory: + subsections: + - Process Memory: + path: chapters/data/process-memory + extra: + - drills/questions/ + - drills/tasks/ + subsections: + - Reading: + path: reading/ + subsections: + - Process Memory: process-memory.md + - Drills: + path: drills/ + subsections: + - Tasks: + path: tasks/ + subsections: + - Alloc Size/: alloc-size/ + - Copy/: copy/ + - Memory Areas/: memory-areas/ + - Modify Areas/: modify-areas/ + - Static Dinamic/: static-dynamic/ + - Questions: + path: questions/ + subsections: + - Half page: half-page.md + - Malloc Brk: malloc-brk.md + - Malloc Mmap: malloc-mmap.md + - Memory granularity: memory-granularity.md + - Mmap file: mmap-file.md + - Page Allocation: page-allocation.md + + - Investigate Memory: + subsections: + - Investigate Memory: + path: chapters/data/investigate-memory + extra: + - drills/questions/ + - guides/ + subsections: + - Reading: reading/investigate-memory.md + - Drills: + path: drills/ + subsections: + - Questions: + path: questions/ + subsections: + - Memory leaks: memory-leaks.md + - Valgrind leaks: valgrind-leaks.md + - Guides: + path: guides/ + subsections: + - App Investigation Deluge/: app-investigation-deluge/ + - App Investigation Servo/: app-investigation-servo/ + - D allocator/: d-allocator/ + - Investigation Git/: git/ + - Investigation Jemalloc/: jemalloc/ + - Memory Actions/: memory-actions/ + - Memory Leak/: memory-leak/ + + - Data arena: + subsections: + - Arena: + path: chapters/data/arena + extra: + - drills/questions/ + - drills/tasks/ + subsections: + - Reading: reading/arena.md + - Drills: + path: drills/ + subsections: + - Tasks: + path: tasks/ + subsections: + - Exec Shellcode/: exec-shellcode/ + - Access Counter/: access-counter/ + - Reference Counting/: reference-counting/ + - Page Mapper/: page-mapper/ + - Questions: + path: questions/ + subsections: + - Operators: operators.md + - Lecture: path: /build/embed_reveal subsections: diff --git a/content/chapters/data/lab/support/memory-access/Makefile b/content/chapters/data/lab/support/memory-access/Makefile deleted file mode 100644 index 10df9a7cfb..0000000000 --- a/content/chapters/data/lab/support/memory-access/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARY = mem_access -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/data/lab/support/memory-alloc/Makefile b/content/chapters/data/lab/support/memory-alloc/Makefile deleted file mode 100644 index 37767d2a95..0000000000 --- a/content/chapters/data/lab/support/memory-alloc/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -BINARY = alloc_size -include ../../../../../common/makefile/single.mk - -DC = gdc - -d_memory_alloc: d_memory_alloc.d - $(DC) -o $@ $? - -clean:: - -rm -f d_memory_alloc diff --git a/content/chapters/data/lab/support/memory-corruption/Makefile b/content/chapters/data/lab/support/memory-corruption/Makefile deleted file mode 100644 index d587ae2724..0000000000 --- a/content/chapters/data/lab/support/memory-corruption/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -BINARY = c_segfault -include ../../../../../common/makefile/single.mk - -DC = gdc - -d_segfault: d_segfault.d - $(DC) -o $@ $^ - -clean:: - -rm -f d_segfault diff --git a/content/chapters/data/lab/support/memory-protection/Makefile b/content/chapters/data/lab/support/memory-protection/Makefile deleted file mode 100644 index 7bec125691..0000000000 --- a/content/chapters/data/lab/support/memory-protection/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARY = mem_prot -include ../../../../../common/makefile/single.mk