diff --git a/.github/labeler.yml b/.github/labeler.yml index 0b674e7771..62b57aea68 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -8,7 +8,7 @@ topic/compute: - 'chapters/compute/**/*' topic/io: - - 'content/chapters/io/**/*' + - 'chapters/io/**/*' topic/app-interact: - 'content/chapters/app-interact/**/*' diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..97924983a3 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "editor.formatOnSave": false +} \ No newline at end of file diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/README.md b/chapters/compute/copy-on-write/drills/tasks/shared-memory/README.md index ac96f6af78..698dda23ae 100644 --- a/chapters/compute/copy-on-write/drills/tasks/shared-memory/README.md +++ b/chapters/compute/copy-on-write/drills/tasks/shared-memory/README.md @@ -24,10 +24,10 @@ Now let's test this flag, as well as its opposite: `MAP_SHARED`. Compile and run the code in `shared-memory/support/src/shared_memory.c`. 1. See the value read by the parent is different from that written by the child. -Modify the `flags` parameter of `mmap()` so they are the same. + Modify the `flags` parameter of `mmap()` so they are the same. 1. Create a semaphore in the shared page and use it to make the parent signal the child before it can exit. -Use the API defined in [`semaphore.h`](https://man7.org/linux/man-pages/man0/semaphore.h.0p.html). + Use the API defined in [`semaphore.h`](https://man7.org/linux/man-pages/man0/semaphore.h.0p.html). **Be careful!** The value written and read previously by the child and the parent, respectively, must not change. diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/src/utils/sock/sock_util.c b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/src/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/src/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/src/utils/sock/sock_util.h b/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/src/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/copy-on-write/drills/tasks/shared-memory/solution/src/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.c b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.h b/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/copy-on-write/guides/fork-faults/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/processes/drills/tasks/create-process/solution/src/utils/sock/sock_util.c b/chapters/compute/processes/drills/tasks/create-process/solution/src/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/processes/drills/tasks/create-process/solution/src/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/processes/drills/tasks/create-process/solution/src/utils/sock/sock_util.h b/chapters/compute/processes/drills/tasks/create-process/solution/src/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/processes/drills/tasks/create-process/solution/src/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/processes/drills/tasks/mini-shell/solution/src/utils/sock/sock_util.c b/chapters/compute/processes/drills/tasks/mini-shell/solution/src/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/processes/drills/tasks/mini-shell/solution/src/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/processes/drills/tasks/mini-shell/solution/src/utils/sock/sock_util.h b/chapters/compute/processes/drills/tasks/mini-shell/solution/src/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/processes/drills/tasks/mini-shell/solution/src/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/processes/drills/tasks/sleepy/solution/src/utils/sock/sock_util.c b/chapters/compute/processes/drills/tasks/sleepy/solution/src/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/processes/drills/tasks/sleepy/solution/src/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/processes/drills/tasks/sleepy/solution/src/utils/sock/sock_util.h b/chapters/compute/processes/drills/tasks/sleepy/solution/src/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/processes/drills/tasks/sleepy/solution/src/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.c b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.h b/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/processes/guides/sum-array-processes/support/c/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.c b/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.h b/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/processes/guides/system-dissected/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/scheduling/drills/questions/time-slice-value.md b/chapters/compute/scheduling/drills/questions/time-slice-value.md index 1cdf344093..71a3803418 100644 --- a/chapters/compute/scheduling/drills/questions/time-slice-value.md +++ b/chapters/compute/scheduling/drills/questions/time-slice-value.md @@ -18,7 +18,7 @@ Using the [man page](https://man7.org/linux/man-pages/man2/setitimer.2.html), wh The code we're interested in lies in the function `init_profiling_timer()`: -```C +```c const struct itimerval timer = { { 0, 10000 }, { 0, 1 } // arms the timer as soon as possible @@ -27,7 +27,7 @@ const struct itimerval timer = { The [man page](https://man7.org/linux/man-pages/man2/setitimer.2.html) gives the following definition the `struct itimerval`: -```C +```c struct itimerval { struct timeval it_interval; /* Interval for periodic timer */ struct timeval it_value; /* Time until next expiration */ diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.c b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.h b/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/scheduling/drills/tasks/libult/solution/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.c b/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.h b/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/scheduling/drills/tasks/libult/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.c b/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.h b/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/scheduling/guides/libult/solution/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.c b/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.h b/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/scheduling/guides/libult/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/scheduling/reading/scheduling.md b/chapters/compute/scheduling/reading/scheduling.md index 3bfed83c2f..fb62f58cea 100644 --- a/chapters/compute/scheduling/reading/scheduling.md +++ b/chapters/compute/scheduling/reading/scheduling.md @@ -13,9 +13,6 @@ To do this, the scheduler must decide, at given times, to suspend a thread, save This event is called a **context switch**. A context switch means changing the state of one thread (the replaced thread) from RUNNING to WAITING, and the state of the replacement thread from READY / WAITING to RUNNING. - -- Quiz? - ## User-Level vs Kernel-Level Threads There are two types of threads. @@ -36,14 +33,14 @@ In such cases, user-level threads may be useful as context switches bring less o Let's dissect the `threads_create()` function a bit. It first initialises its queues and the timer for preemption. -We'll discuss preemption [in the next section](#scheduling---how-is-it-done). +We'll discuss preemption separately, in a [section of its own](#scheduling---how-is-it-done). After performing initialisations, the function creates a `TCB` object. TCB stands for **Thread Control Block**. During the lecture, you saw that the kernel stores one instance of a [`task_struct`](https://elixir.bootlin.com/linux/v5.19.11/source/include/linux/sched.h#L726) for each thread. Remember that its most important fields are: -```C +```c struct task_struct { unsigned int __state; @@ -150,7 +147,6 @@ If you encounter the following error when running `test_ult`, remember what you > Hint: Use the `LD_LIBRARY_PATH` variable. Notice that the threads run their code and alternatively, because their prints appear interleaved. -[In the next section](#scheduling---how-is-it-done), we'll see how this is done. [Quiz](questions/ult-thread-ids.md) @@ -190,7 +186,7 @@ It is this handler that performs the context switch per se. Look at its code. It first saves the context of the current thread: -```C +```c ucontext_t *stored = &running->context; ucontext_t *updated = (ucontext_t *) context; @@ -203,7 +199,7 @@ stored->uc_sigmask = updated->uc_sigmask; Then it places the current thread in the `ready` queue and replaces it with the first thread in the same queue. This algorithm (that schedules the first thread in the READY queue) is called _Round-Robin_: -```C +```c if (queue_enqueue(ready, running) != 0) { abort(); } @@ -215,7 +211,7 @@ if ((running = queue_dequeue(ready)) == NULL) { The new `running` thread is resumed upon setting the current context to it: -```C +```c if (setcontext(&running->context) == -1) { abort(); } diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.c b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.h b/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/synchronization/drills/tasks/race-condition/solution/c/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.c b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.h b/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/synchronization/drills/tasks/race-condition/support/c/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/synchronization/reading/synchronization.md b/chapters/compute/synchronization/reading/synchronization.md index 14fc357a07..48086c37ec 100644 --- a/chapters/compute/synchronization/reading/synchronization.md +++ b/chapters/compute/synchronization/reading/synchronization.md @@ -108,6 +108,7 @@ The `lock` prefix ensures that the core performing the instruction has exclusive This is how the increment is made into an indivisible unit. For example, `inc dword [x]` can be made atomic, like so: `lock inc dword [x]`. +You can play with the `lock` prefix [in this race conditions task](../drills/tasks/race-condition/README.md). Compilers provide support for such hardware-level atomic operations. GCC exposes [built-ins](https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html) such as `__atomic_load()`, `__atomic_store()`, `__atomic_compare_exchange()` and many others. diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.c b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.h b/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/threads/drills/tasks/multithreaded/solution/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.c b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.h b/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/threads/drills/tasks/sum-array-bugs/support/seg-fault/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.c b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.h b/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/threads/drills/tasks/sum-array/support/c/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.c b/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.h b/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/threads/guides/clone/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/threads/guides/python-gil/README.md b/chapters/compute/threads/guides/python-gil/README.md index 2f53c6b55a..b24d2d812a 100644 --- a/chapters/compute/threads/guides/python-gil/README.md +++ b/chapters/compute/threads/guides/python-gil/README.md @@ -23,7 +23,7 @@ Unlike Bigfoot, or the Loch Ness monster, we have proof that the GIL is real. At first glance, this seems like a huge disadvantage. Why force threads to run sequentially? The answer has to do with memory management. -In the [Data chapter](../../../data), you learned that one way of managing memory is via _garbage collection_ (GC). +In the [Data chapter](../../../../data/process-memory/reading/process-memory.md), you learned that one way of managing memory is via _garbage collection_ (GC). In Python, the GC uses reference counting, i.e. each object also stores the number of live pointers to it (variables that reference it). You can see that this number needs to be modified atomically by the interpreter to avoid race conditions. This involves adding locks to **all** Python data structures. diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.c b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.h b/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/threads/guides/sum-array-threads/support/c/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/threads/guides/wait-for-me-threads/README.md b/chapters/compute/threads/guides/wait-for-me-threads/README.md index 3ff709affa..d60c76c354 100644 --- a/chapters/compute/threads/guides/wait-for-me-threads/README.md +++ b/chapters/compute/threads/guides/wait-for-me-threads/README.md @@ -15,6 +15,6 @@ As you can see, waiting is a very coarse form of synchronization. If we only use waiting, we can expect no speedup as a result of parallelism, because one thread must finish completely before another can continue. We will discuss more fine-grained synchronization mechanisms [later in this lab](reading/synchronization.md). -Also, at this point, you might be wondering why this exercise is written in D, while [the same exercise, but with processes](reading/processes.md#practice-wait-for-me) was written in Python. +Also, at this point, you might be wondering why this exercise is written in D, while [the same exercise, but with processes](reading/processes.md) was written in Python. There is a very good reason for this and has to do with how threads are synchronized by default in Python. -You can find out what this is about [in the Arena section](gil), after you have completed the [Synchronization section](reading/synchronization.md). +You can read about it in this article on [Python Global Interpreter LOCK (GIL)](https://realpython.com/python-gil/), but it will make more sense after you have completed the [Synchronization section](reading/synchronization.md). diff --git a/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.c b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.h b/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/compute/threads/guides/wait-for-me-threads/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/compute/user-level-threads/reading/user-level-threads.md b/chapters/compute/user-level-threads/reading/user-level-threads.md index 7e32c17118..e6232a2028 100644 --- a/chapters/compute/user-level-threads/reading/user-level-threads.md +++ b/chapters/compute/user-level-threads/reading/user-level-threads.md @@ -45,7 +45,7 @@ Answer in this [quiz](../drills/questions/sleeping-on-a-fiber.md). Use `strace` to find calls to `clone()` in the execution of `simple`. Can you find any? Provide your answer in this [quiz](../drills/questions/fiber-strace.md) -Remember that `clone()` is the system call used to create **kernel-level** threads, as pointed out [here](../../processes/guides/clone.md). +`clone` is the system call used to create **kernel-level** threads, as pointed out [in this guide on creating threads and processes](../../threads/guides/clone/README.md). ## Synchronization diff --git a/chapters/data/memory-security/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.c b/chapters/data/memory-security/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/memory-security/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/memory-security/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.h b/chapters/data/memory-security/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/memory-security/drills/tasks/exec-shellcode/solution/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/memory-security/drills/tasks/exec-shellcode/support/utils/sock/sock_util.c b/chapters/data/memory-security/drills/tasks/exec-shellcode/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/memory-security/drills/tasks/exec-shellcode/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/memory-security/drills/tasks/exec-shellcode/support/utils/sock/sock_util.h b/chapters/data/memory-security/drills/tasks/exec-shellcode/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/memory-security/drills/tasks/exec-shellcode/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/process-memory/drills/tasks/alloc-size/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/process-memory/drills/tasks/copy/solution/src/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/copy/solution/src/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/process-memory/drills/tasks/copy/solution/src/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/process-memory/drills/tasks/copy/solution/src/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/copy/solution/src/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/process-memory/drills/tasks/copy/solution/src/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/process-memory/drills/tasks/memory-areas/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/process-memory/drills/tasks/modify-areas/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/process-memory/drills/tasks/page-mapper/solution/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/page-mapper/solution/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/process-memory/drills/tasks/page-mapper/solution/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/process-memory/drills/tasks/page-mapper/solution/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/page-mapper/solution/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/process-memory/drills/tasks/page-mapper/solution/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/process-memory/drills/tasks/page-mapper/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/page-mapper/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/process-memory/drills/tasks/page-mapper/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/process-memory/drills/tasks/page-mapper/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/page-mapper/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/process-memory/drills/tasks/page-mapper/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/process-memory/drills/tasks/reference-counting/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/reference-counting/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/process-memory/drills/tasks/reference-counting/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/process-memory/drills/tasks/reference-counting/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/reference-counting/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/process-memory/drills/tasks/reference-counting/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.c b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.h b/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/process-memory/drills/tasks/static-dynamic/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/working-with-memory/drills/tasks/access-counter/solution/utils/sock/sock_util.c b/chapters/data/working-with-memory/drills/tasks/access-counter/solution/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/working-with-memory/drills/tasks/access-counter/solution/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/working-with-memory/drills/tasks/access-counter/solution/utils/sock/sock_util.h b/chapters/data/working-with-memory/drills/tasks/access-counter/solution/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/working-with-memory/drills/tasks/access-counter/solution/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/data/working-with-memory/drills/tasks/access-counter/support/utils/sock/sock_util.c b/chapters/data/working-with-memory/drills/tasks/access-counter/support/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/data/working-with-memory/drills/tasks/access-counter/support/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/data/working-with-memory/drills/tasks/access-counter/support/utils/sock/sock_util.h b/chapters/data/working-with-memory/drills/tasks/access-counter/support/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/data/working-with-memory/drills/tasks/access-counter/support/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/content/chapters/io/lecture/.gitignore b/chapters/io/.gitignore similarity index 100% rename from content/chapters/io/lecture/.gitignore rename to chapters/io/.gitignore diff --git a/chapters/io/Makefile b/chapters/io/Makefile new file mode 100644 index 0000000000..12cc2a595f --- /dev/null +++ b/chapters/io/Makefile @@ -0,0 +1,33 @@ +RVMD = reveal-md +MDPP = markdown-pp +FFMPEG = ffmpeg + +SLIDES ?= slides.mdpp +SLIDES_OUT ?= slides.md +MEDIA_DIR ?= media +SITE ?= _site +OPEN ?= xdg-open + +.PHONY: all html clean videos + +all: videos html + +html: $(SITE) + +$(SITE): $(SLIDES) + $(MDPP) $< -o $(SLIDES_OUT) + $(RVMD) $(SLIDES_OUT) --static $@ + +videos: + for TARGET in $(TARGETS); do \ + $(FFMPEG) -framerate 0.5 -f image2 -y \ + -i "$(MEDIA_DIR)/$$TARGET/$$TARGET-%d.svg" -vf format=yuv420p $(MEDIA_DIR)/$$TARGET-generated.gif; \ + done + +open: $(SITE) + $(OPEN) $ - ## Question Text -What is the type of the file handler in the C code located in `support/simple-file-operations/file_operations.c`? +What is the type of the file handler in the C code located in `chapters/io/file-descriptors/support/simple-file-operations/file_operations.c`? ## Question Answers @@ -18,7 +16,7 @@ What is the type of the file handler in the C code located in `support/simple-fi - `struct file` -## Feedaback +## Feedback The file is opened using either of the following lines: diff --git a/content/chapters/io/lab/quiz/flush-libc-buffer.md b/chapters/io/file-descriptors/drills/questions/flush-libc-buffer.md similarity index 85% rename from content/chapters/io/lab/quiz/flush-libc-buffer.md rename to chapters/io/file-descriptors/drills/questions/flush-libc-buffer.md index 3ac7371cb1..7413a4b094 100644 --- a/content/chapters/io/lab/quiz/flush-libc-buffer.md +++ b/chapters/io/file-descriptors/drills/questions/flush-libc-buffer.md @@ -2,7 +2,7 @@ ## Question Text -Which of the following is a method of flushing libc's internal buffer? +Which of the following is a method of flushing `libc`'s internal buffer? ## Question Answers diff --git a/content/chapters/io/lab/quiz/fopen-syscall.md b/chapters/io/file-descriptors/drills/questions/fopen-syscall.md similarity index 97% rename from content/chapters/io/lab/quiz/fopen-syscall.md rename to chapters/io/file-descriptors/drills/questions/fopen-syscall.md index 6249ca055f..e9ebfb1033 100644 --- a/content/chapters/io/lab/quiz/fopen-syscall.md +++ b/chapters/io/file-descriptors/drills/questions/fopen-syscall.md @@ -17,7 +17,7 @@ Which one is it? - `fstat()` -## Feedaback +## Feedback ```console student@os:~/.../lab/support/simple-file-handling$ strace ./file_operations diff --git a/content/chapters/io/lab/quiz/local-io-errors.md b/chapters/io/file-descriptors/drills/questions/local-io-errors.md similarity index 87% rename from content/chapters/io/lab/quiz/local-io-errors.md rename to chapters/io/file-descriptors/drills/questions/local-io-errors.md index 332aa00070..a3d02db586 100644 --- a/content/chapters/io/lab/quiz/local-io-errors.md +++ b/chapters/io/file-descriptors/drills/questions/local-io-errors.md @@ -6,7 +6,7 @@ Which of the following types of errors are **unlikely** to occur during an I/O o ## Question Answers -- The current user does not have sufficient permisions to access a given file +- The current user does not have sufficient permissions to access a given file - There is not enough space left on the disk diff --git a/content/chapters/io/lab/quiz/mmap-read-write-benchmark.md b/chapters/io/file-descriptors/drills/questions/mmap-read-write-benchmark.md similarity index 92% rename from content/chapters/io/lab/quiz/mmap-read-write-benchmark.md rename to chapters/io/file-descriptors/drills/questions/mmap-read-write-benchmark.md index 57d9cccad8..618dacbcc1 100644 --- a/content/chapters/io/lab/quiz/mmap-read-write-benchmark.md +++ b/chapters/io/file-descriptors/drills/questions/mmap-read-write-benchmark.md @@ -54,7 +54,7 @@ So the more conservative answer is to say that this depends on external aspects If you want to know why there isn't much of a difference between the 2 implementations, check out [this explanation](https://stackoverflow.com/a/27987994). -However, some newer Linux systems use an updated `cp` that doesn't use `read` and `write` anymore, but instead uses zero-copy, by means of the `copy_file_range()` syscall. +However, some newer Linux systems use an updated `cp` that doesn't use `read` and `write` anymore, but instead uses [zero-copy](../../../optimizations/reading/zero-copy.md), by means of the `copy_file_range()` syscall. This implementation is likely going to be significantly faster than the one using `mmap`, as shown in the snippet below: ```console diff --git a/content/chapters/io/lab/quiz/stderr-fd.md b/chapters/io/file-descriptors/drills/questions/stderr-fd.md similarity index 97% rename from content/chapters/io/lab/quiz/stderr-fd.md rename to chapters/io/file-descriptors/drills/questions/stderr-fd.md index f8d5aa9f2d..813d29b8a6 100644 --- a/content/chapters/io/lab/quiz/stderr-fd.md +++ b/chapters/io/file-descriptors/drills/questions/stderr-fd.md @@ -20,7 +20,7 @@ Which file descriptor is associated by default to `stderr`? - 1 -## Feedaback +## Feedback You would type `ls 2> /dev/null` to ignore `ls`'s errors. This equates to **redirecting** `stderr` to `/dev/null`. diff --git a/content/chapters/io/lab/quiz/strace-printf.md b/chapters/io/file-descriptors/drills/questions/strace-printf.md similarity index 100% rename from content/chapters/io/lab/quiz/strace-printf.md rename to chapters/io/file-descriptors/drills/questions/strace-printf.md diff --git a/content/chapters/io/lab/quiz/syscalls-cp.md b/chapters/io/file-descriptors/drills/questions/syscalls-cp.md similarity index 100% rename from content/chapters/io/lab/quiz/syscalls-cp.md rename to chapters/io/file-descriptors/drills/questions/syscalls-cp.md diff --git a/chapters/io/file-descriptors/drills/tasks/buffering/README.md b/chapters/io/file-descriptors/drills/tasks/buffering/README.md new file mode 100644 index 0000000000..bd3bd62247 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/buffering/README.md @@ -0,0 +1,38 @@ +# Buffering + +Navigate to `chapters/io/drills/tasks/buffering/support` and run `make`. +We'll checkout how effective buffering is in `libc` and then we'll do it ourselves. + +1. First, let's checkout how relevant buffering is. + We'll see how well reading and writing **one byte at a time** performs with `no_buffering` and with `libc` buffering. + Use `benchmark_buffering.sh` script to compare `no_buffering` and `libc_buffering` implementations: + + ```bash + student@os:/.../buffering/support$ ./benchmark_buffering.sh + ======== Testing no_buffering ======== + Testing no_buffering read... + Read 1048576 bytes from test-file.txt in 717 ms + Testing no_buffering write... + Wrote 1048576 bytes to test-file.txt in 19632 ms + ======== Testing libc_buffering ======== + Testing libc_buffering read... + Read 1048576 bytes from test-file.txt in 14 ms + Testing libc_buffering write... + Wrote 1048576 bytes to test-file.txt in 38 ms + ``` + + Buffering achieves dramatic performance gains, reducing read times by **98%** and write times by **99.8%** in this example! + This demonstrates the power of buffering, even though it’s an extreme case. + +1. Complete the TODOs in `diy_buffering.c` to replicate the functionality of `fread()` and `fwrite()`. + The `fread()` function is already complete, so you can use it as a reference for implementing `fwrite()`. + + After implementing `diy_fwrite()`, run `benchmark_buffering.sh` to see if your DIY buffering implementation outperforms `no_buffering`. + Check how close you come to the efficiency of `libc`. + +Keep in mind that I/O buffering is a core optimization in standard libraries like libc, is so effective that the kernel itself uses it. +The kernel’s **double buffering** strategy reads more than requested, storing extra data in a buffer for future use. +In the next section on [Zero-Copy Techniques](../../../../optimizations/reading/zero-copy.md), we’ll look at how to leverage this to further optimize network requests. + +> **Note**: `benchmark_buffering.sh` also includes `echo 3 > /proc/sys/vm/drop_caches` to clear caches before benchmarking, ensuring consistent results. + For a deeper dive into cache behavior, check out [this section in the Arena](../../arena/reading/arena.md#to-drop-or-not-to-drop). diff --git a/content/chapters/io/lab/support/buffering/.gitignore b/chapters/io/file-descriptors/drills/tasks/buffering/support/.gitignore similarity index 100% rename from content/chapters/io/lab/support/buffering/.gitignore rename to chapters/io/file-descriptors/drills/tasks/buffering/support/.gitignore diff --git a/content/chapters/io/lab/solution/mini-shell/Makefile b/chapters/io/file-descriptors/drills/tasks/buffering/support/Makefile similarity index 52% rename from content/chapters/io/lab/solution/mini-shell/Makefile rename to chapters/io/file-descriptors/drills/tasks/buffering/support/Makefile index ae810113b5..e7baf303f5 100644 --- a/content/chapters/io/lab/solution/mini-shell/Makefile +++ b/chapters/io/file-descriptors/drills/tasks/buffering/support/Makefile @@ -1,8 +1,7 @@ -# Get the relative path to the directory of the current makefile. +# Get the relative path to the directory of the current makefile MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) INCLUDES_DIR := $(MAKEFILE_DIR) UTILS_DIR := $(MAKEFILE_DIR)/utils -LOGGER_DIR := $(UTILS_DIR)/log # Compiler and flags CPPFLAGS += -I$(INCLUDES_DIR) @@ -10,32 +9,33 @@ CFLAGS += -g -Wall -Wextra LDFLAGS += -z lazy # Logger object +LOGGER_DIR := $(UTILS_DIR)/log LOGGER_OBJ = log.o LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) # Source files and corresponding binaries -SRCS = mini_shell.c +BINARIES = diy_buffering libc_buffering no_buffering +SRCS = $(BINARIES:=.c) OBJS = $(SRCS:.c=.o) -BINARIES = mini_shell -# Default rule: Build everything +# Default rule: build all binaries all: $(BINARIES) -# Rule to compile the logger -$(LOGGER_OBJ): $(LOGGER_DIR)/log.c - $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) +test-file.txt: + dd if=/dev/urandom of=test-file.txt bs=1024 count=1K -# Rule to compile object files from source files -%.o: %.c - $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ -# Rule to create mini_shell binary -mini_shell: mini_shell.o $(LOGGER) - $(CC) $(CFLAGS) mini_shell.o $(LOGGER) -o mini_shell $(LDFLAGS) +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ # Clean rule: Remove object files and binaries clean: -rm -f $(OBJS) $(BINARIES) - @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + -rm -f $(LOGGER) + -rm -f test-file.txt -.PHONY: all clean \ No newline at end of file +.PHONY: all clean diff --git a/content/chapters/io/lab/support/buffering/benchmark_buffering.sh b/chapters/io/file-descriptors/drills/tasks/buffering/support/benchmark_buffering.sh similarity index 100% rename from content/chapters/io/lab/support/buffering/benchmark_buffering.sh rename to chapters/io/file-descriptors/drills/tasks/buffering/support/benchmark_buffering.sh diff --git a/content/chapters/io/lab/support/buffering/diy_buffering.c b/chapters/io/file-descriptors/drills/tasks/buffering/support/diy_buffering.c similarity index 100% rename from content/chapters/io/lab/support/buffering/diy_buffering.c rename to chapters/io/file-descriptors/drills/tasks/buffering/support/diy_buffering.c diff --git a/content/chapters/io/lab/support/buffering/libc_buffering.c b/chapters/io/file-descriptors/drills/tasks/buffering/support/libc_buffering.c similarity index 100% rename from content/chapters/io/lab/support/buffering/libc_buffering.c rename to chapters/io/file-descriptors/drills/tasks/buffering/support/libc_buffering.c diff --git a/content/chapters/io/lab/support/buffering/no_buffering.c b/chapters/io/file-descriptors/drills/tasks/buffering/support/no_buffering.c similarity index 100% rename from content/chapters/io/lab/support/buffering/no_buffering.c rename to chapters/io/file-descriptors/drills/tasks/buffering/support/no_buffering.c diff --git a/content/chapters/io/lab/solution/mini-shell/utils/log/CPPLINT.cfg b/chapters/io/file-descriptors/drills/tasks/buffering/support/utils/log/CPPLINT.cfg similarity index 100% rename from content/chapters/io/lab/solution/mini-shell/utils/log/CPPLINT.cfg rename to chapters/io/file-descriptors/drills/tasks/buffering/support/utils/log/CPPLINT.cfg diff --git a/content/chapters/io/lab/solution/mini-shell/utils/log/log.c b/chapters/io/file-descriptors/drills/tasks/buffering/support/utils/log/log.c similarity index 100% rename from content/chapters/io/lab/solution/mini-shell/utils/log/log.c rename to chapters/io/file-descriptors/drills/tasks/buffering/support/utils/log/log.c diff --git a/content/chapters/io/lab/solution/mini-shell/utils/log/log.h b/chapters/io/file-descriptors/drills/tasks/buffering/support/utils/log/log.h similarity index 100% rename from content/chapters/io/lab/solution/mini-shell/utils/log/log.h rename to chapters/io/file-descriptors/drills/tasks/buffering/support/utils/log/log.h diff --git a/content/chapters/io/lab/solution/mini-shell/utils/utils.h b/chapters/io/file-descriptors/drills/tasks/buffering/support/utils/utils.h similarity index 100% rename from content/chapters/io/lab/solution/mini-shell/utils/utils.h rename to chapters/io/file-descriptors/drills/tasks/buffering/support/utils/utils.h diff --git a/chapters/io/file-descriptors/drills/tasks/mmap_cp/.gitignore b/chapters/io/file-descriptors/drills/tasks/mmap_cp/.gitignore new file mode 100644 index 0000000000..296b2ccf64 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/mmap_cp/.gitignore @@ -0,0 +1 @@ +support/ diff --git a/chapters/io/file-descriptors/drills/tasks/mmap_cp/Makefile b/chapters/io/file-descriptors/drills/tasks/mmap_cp/Makefile new file mode 100644 index 0000000000..ff78d246ea --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/mmap_cp/Makefile @@ -0,0 +1,9 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + mkdir -p support/ + $(PYTHON) $(SCRIPT) --input ./solution --output ./support + +clean: + rm -rf support/ diff --git a/chapters/io/file-descriptors/drills/tasks/mmap_cp/README.md b/chapters/io/file-descriptors/drills/tasks/mmap_cp/README.md new file mode 100644 index 0000000000..9e71c1e496 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/mmap_cp/README.md @@ -0,0 +1,25 @@ +# Copy a File with `mmap()` + +Navigate to `file-descriptors/drills/tasks/mmap_cp` and run `make` to generate `support`. +As you know `mmap()` can map files in memory, perform operations on them, and then write them back to the disk. +Let's check how well it performs by comparing it to the `cp` command. +The benchmarking is automated by `benchmark_cp.sh` so focus on completing `mmap_cp.c` for now. + +[Quiz: Checkout what syscalls `cp` uses](../../questions/syscalls-cp.md) + +1. Open `mmap_cp.c` and complete the TODOs to map the files in memory and copy the contents. + Do not forget to clean up by unmapping and closing the files. + + To test, run `make test-file` to generate a 1MB file with random data, and then run `mmap_cp test-file output.txt`. + Ensure they have the same content with a simple `diff`: `diff test-file.txt output.txt`. + +1. Compare your implementation to the `cp` command. + Run `make large-file` to generate a 1GB file with random data, and then run `./benchmark_cp.sh`. + + [Quiz: Debunk why `cp` is winning](../../questions/mmap-read-write-benchmark.md) + + If you want a more generic answer, checkout this [guide on `mmap` vs `read()-write()`](../../../guides/file-mappings/README.md). + +1. This demo would not be complete without some live analysis. + Uncomment the calls to `wait_for_input()` and rerun the program. + In another terminal, run `cat /proc/$(pidof mmap_cp)/maps` to see mapped files, and `ps -o pid,vsz,rss ` to see how demand paging happens. diff --git a/chapters/io/file-descriptors/drills/tasks/mmap_cp/generate_skels.py b/chapters/io/file-descriptors/drills/tasks/mmap_cp/generate_skels.py new file mode 100644 index 0000000000..697c9d5b61 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/mmap_cp/generate_skels.py @@ -0,0 +1,151 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + or re.match(r".*\.py$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/content/chapters/io/lab/solution/file-mappings/.gitignore b/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/.gitignore similarity index 100% rename from content/chapters/io/lab/solution/file-mappings/.gitignore rename to chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/.gitignore diff --git a/content/chapters/io/lab/support/mini-shell/Makefile b/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/Makefile similarity index 52% rename from content/chapters/io/lab/support/mini-shell/Makefile rename to chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/Makefile index ae810113b5..a9e7f21f8d 100644 --- a/content/chapters/io/lab/support/mini-shell/Makefile +++ b/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/Makefile @@ -1,8 +1,7 @@ -# Get the relative path to the directory of the current makefile. +# Get the relative path to the directory of the current makefile MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) INCLUDES_DIR := $(MAKEFILE_DIR) UTILS_DIR := $(MAKEFILE_DIR)/utils -LOGGER_DIR := $(UTILS_DIR)/log # Compiler and flags CPPFLAGS += -I$(INCLUDES_DIR) @@ -10,32 +9,36 @@ CFLAGS += -g -Wall -Wextra LDFLAGS += -z lazy # Logger object +LOGGER_DIR := $(UTILS_DIR)/log LOGGER_OBJ = log.o LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) # Source files and corresponding binaries -SRCS = mini_shell.c +BINARIES = mmap_cp +SRCS = $(BINARIES:=.c) OBJS = $(SRCS:.c=.o) -BINARIES = mini_shell -# Default rule: Build everything +# Default rule: build all binaries all: $(BINARIES) -# Rule to compile the logger -$(LOGGER_OBJ): $(LOGGER_DIR)/log.c - $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) +large-file: + dd if=/dev/urandom of=large-file.txt bs=1024 count=1M + +test-file: + dd if=/dev/urandom of=test-file.txt bs=1024 count=1K -# Rule to compile object files from source files -%.o: %.c - $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ -# Rule to create mini_shell binary -mini_shell: mini_shell.o $(LOGGER) - $(CC) $(CFLAGS) mini_shell.o $(LOGGER) -o mini_shell $(LDFLAGS) +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ # Clean rule: Remove object files and binaries clean: -rm -f $(OBJS) $(BINARIES) - @make -C $(LOGGER_DIR) clean # Clean the logger directory as well + -rm -f $(LOGGER) + -rm -f *.txt -.PHONY: all clean \ No newline at end of file +.PHONY: all clean diff --git a/content/chapters/io/lab/support/file-mappings/benchmark_cp.sh b/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/benchmark_cp.sh similarity index 100% rename from content/chapters/io/lab/support/file-mappings/benchmark_cp.sh rename to chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/benchmark_cp.sh diff --git a/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/mmap_cp.c b/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/mmap_cp.c new file mode 100644 index 0000000000..9d306b7a43 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/mmap_cp.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" + +static void wait_for_input(const char *msg) +{ + char buf[32]; + + printf(" * %s\n", msg); + printf(" -- Press ENTER to continue ..."); + fflush(stdout); + fgets(buf, 32, stdin); +} + +int main(int argc, char *argv[]) +{ + int fdin; + int fdout; + int rc; + char *src; + char *dst; + struct stat statbuf; + + DIE(argc != 3, "Usage: ./mmap_cp "); + + printf("PID: %d\n", getpid()); + + /* TODO 2: Open the input file. */ + fdin = open(argv[1], O_RDONLY); + DIE(fdin < 0, "open"); + + /* TODO 2: Open/create the output file. */ + fdout = open(argv[2], O_RDWR | O_CREAT | O_TRUNC, 0644); + DIE(fdout < 0, "open"); + + /** + * The output file is created with a size of 0. We need to set the size + * of the output file to the size of the input file. + */ + + /* TODO 2: Use ftstat() to get the size of the input file. */ + rc = fstat(fdin, &statbuf); + DIE(rc < 0, "fstat"); + + /* TODO 2: Use ftruncate() to set the size of the output file. */ + rc = ftruncate(fdout, statbuf.st_size); + DIE(rc < 0, "ftruncate"); + + /* TODO 2: `mmap()` the input file. */ + src = mmap(NULL, statbuf.st_size, PROT_READ, MAP_PRIVATE, fdin, 0); + DIE(src == MAP_FAILED, "mmap"); + + /* wait_for_input("Mapped input file."); */ + + /* TODO 2: `mmap()` the output file. */ + dst = mmap(NULL, statbuf.st_size, PROT_WRITE, MAP_SHARED, fdout, 0); + DIE(dst == MAP_FAILED, "mmap"); + + /* wait_for_input("Mapped output file."); */ + + /* TODO 1: Copy the contents of the input mapping to the output mapping. */ + memcpy(dst, src, statbuf.st_size); + + /* TODO 10: Clean up. Use `munmap()` to unmap the 2 files. */ + rc = munmap(src, statbuf.st_size); + DIE(rc < 0, "munmap"); + + rc = munmap(dst, statbuf.st_size); + DIE(rc < 0, "munmap"); + + rc = close(fdin); + DIE(rc < 0, "close"); + rc = close(fdout); + DIE(rc < 0, "close"); + + return 0; +} diff --git a/content/chapters/io/lab/support/mini-shell/utils/log/CPPLINT.cfg b/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/utils/log/CPPLINT.cfg similarity index 100% rename from content/chapters/io/lab/support/mini-shell/utils/log/CPPLINT.cfg rename to chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/utils/log/CPPLINT.cfg diff --git a/content/chapters/io/lab/support/mini-shell/utils/log/log.c b/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/utils/log/log.c similarity index 100% rename from content/chapters/io/lab/support/mini-shell/utils/log/log.c rename to chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/utils/log/log.c diff --git a/content/chapters/io/lab/support/mini-shell/utils/log/log.h b/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/utils/log/log.h similarity index 100% rename from content/chapters/io/lab/support/mini-shell/utils/log/log.h rename to chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/utils/log/log.h diff --git a/content/chapters/io/lab/support/mini-shell/utils/utils.h b/chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/utils/utils.h similarity index 100% rename from content/chapters/io/lab/support/mini-shell/utils/utils.h rename to chapters/io/file-descriptors/drills/tasks/mmap_cp/solution/utils/utils.h diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/.gitignore b/chapters/io/file-descriptors/drills/tasks/my-cat/.gitignore new file mode 100644 index 0000000000..296b2ccf64 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/.gitignore @@ -0,0 +1 @@ +support/ diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/Makefile b/chapters/io/file-descriptors/drills/tasks/my-cat/Makefile new file mode 100644 index 0000000000..9a1d5f4058 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/Makefile @@ -0,0 +1,9 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + mkdir -p support/ + $(PYTHON) $(SCRIPT) --input ./solution/src --output ./support/src + +clean: + rm -rf support/ diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/README.md b/chapters/io/file-descriptors/drills/tasks/my-cat/README.md new file mode 100644 index 0000000000..cfc4ba63ca --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/README.md @@ -0,0 +1,23 @@ +# My `cat` + +Navigate to `file-descriptors/drills/tasks/my-cat/support/src` and checkout `my_cat.c`. +We propose to implement the Linux command `cat` that reads one or more files, **concatenates** them (hence the name `cat`), and prints them to standard output. + +1. Implement `rread()` wrapper over `read()`. + + `read()` system call does not guarantee that it will read the requested number of bytes in a single call. + This happens when the file does not have enough bytes, or when `read()` is interrupted by a signal. + `rread()` will handle these situations, ensuring that it reads either `num_bytes` or all available bytes. + +1. Implement `wwrite()` as a wrapper for `write()`. + + The `write()` system call may not write the requested number of bytes in a single call. + This happens if `write()` is interrupted by a signal. + `wwrite()` will guarantee that it wrote the full `num_bytes`, retrying as necessary until all data is successfully written or an error occurs. + +1. Implement `cat()`. + + Use `rread()` to read an entire file and `wwrite()` to write the contents to standard output. + Keep in mind that the buffer size may not fit the entire file at once. + +If you're having difficulties solving this exercise, go through [this reading material](../../../reading/file-descriptors.md). diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/generate_skels.py b/chapters/io/file-descriptors/drills/tasks/my-cat/generate_skels.py new file mode 100644 index 0000000000..697c9d5b61 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/generate_skels.py @@ -0,0 +1,151 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + or re.match(r".*\.py$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/solution/.gitignore b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/.gitignore new file mode 100644 index 0000000000..f8e4a4be65 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/.gitignore @@ -0,0 +1 @@ +my_cat diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/solution/Makefile b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/Makefile new file mode 100644 index 0000000000..cf1cb3df01 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/Makefile @@ -0,0 +1,37 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = my_cat +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + +.PHONY: all clean diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/solution/my_cat.c b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/my_cat.c new file mode 100644 index 0000000000..90f3c4529e --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/my_cat.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include + +#include "utils/utils.h" + +#define BUF_SIZE 128 + +/** + * rread() - Always read the number of bytes we expect or all the bytes available. + * + * Remember that syscalls can fail, so we should always check their return values. + * read() returns the number of bytes read, or -1 if an error occurred. + * Watch out for partial reads, which can happen if the read is interrupted by a signal. + * + * HINT: Use a while loop to keep reading until we have read the number of bytes we expect. + */ +size_t rread(int fd, void *buf, size_t size) +{ + /* TODO 18: Read the entire file and print it to stdout */ + int bytesRead = 0; + size_t totalBytesRead = 0; + + while (true) { + bytesRead = read(fd, buf, size); + DIE(bytesRead < 0, "read"); + + if (bytesRead == 0) + break; + + totalBytesRead += bytesRead; + if (totalBytesRead == size) + break; + } + + return totalBytesRead; + /* REPLACE 1 */ + /* return 0; */ +} + +/** + * wwrite() - Always write the number of bytes we expect. + * + * Remember that syscalls can fail, so we should always check their return values. + * write() returns the number of bytes written, or -1 if an error occurred. + * Watch out for partial writes, which can happen if the write is interrupted by a signal. + * + * HINT: Use a while loop to keep writing until we have written the number of bytes we expect. + */ +size_t wwrite(int fd, const void *buf, size_t size) +{ + /* TODO 14: Read the entire file and print it to stdout */ + int bytesWritten = 0; + size_t totalBytesWritten = 0; + + while (true) { + bytesWritten = write(fd, buf, size); + DIE(bytesWritten < 0, "write"); + + totalBytesWritten += bytesWritten; + if (totalBytesWritten == size) + break; + } + + return totalBytesWritten; + /* REPLACE 1 */ + /* return 0; */ +} + +void cat(const char *filename) +{ + char buf[BUF_SIZE]; + int fd, bytesRead; + int rc; + + fd = open(filename, O_RDONLY); + DIE(fd < 0, "open"); + + /* TODO 8: Read the entire file and print it to stdout */ + while (true) { + bytesRead = rread(fd, buf, sizeof(buf)); + if (bytesRead == 0) + break; + + wwrite(STDOUT_FILENO, buf, bytesRead); + } + + rc = close(fd); + DIE(rc < 0, "close"); +} + +int main(int argc, char *argv[]) +{ + if (argc < 2) { + printf("Usage: %s ... \n", argv[0]); + exit(0); + } + + for (int i = 1; i < argc; i++) + cat(argv[i]); + + return 0; +} diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/log/CPPLINT.cfg b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/log/log.c b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/log/log.h b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/utils.h b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/file-descriptors/drills/tasks/my-cat/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/io/lab/content/local-io-in-action.md b/chapters/io/file-descriptors/guides/deluge-io/README.md similarity index 65% rename from content/chapters/io/lab/content/local-io-in-action.md rename to chapters/io/file-descriptors/guides/deluge-io/README.md index c71025691e..81d0d00ec1 100644 --- a/content/chapters/io/lab/content/local-io-in-action.md +++ b/chapters/io/file-descriptors/guides/deluge-io/README.md @@ -1,49 +1,34 @@ -# Local I/O in Action +# `Deluge` I/O Most of the time, file handling is a simple operation from the perspective of the application. -We've already looked at the [Deluge](https://www.deluge-torrent.org/) Bittorrent client written during the Data lab. -We'll examine it once more, but this time from the perspective of its I/O operations. -Being a Bittorrent client, we expect it to deal with I/O in 4 ways. -There are 2 "local" types of operations: +To see how this looks in practice, we'll examine how the [Deluge Bittorent client](https://www.deluge-torrent.org/) performs I/O operations. -- reading local files from the disk to be sent to other clients +As a BitTorrent client, Deluge manages both local and network-based I/O. +The two "local" operations include **reading files from disk** to share with other clients and **writing downloaded files to disk**. +These are the core tasks we’ll focus on here. -- writing the previously downloaded files to the disk - -Simple, right? -Now we'll only look at the "local" part: reading and writing data to/from the disk. -The remaining 2 actions are about network-related operations. - -- requesting files from other stations on the Web. -Those files are then retained in Deluge's memory while they're being downloaded - -- sending (parts of) files to other clients. -Those files are first read from the disk, retained in memory for a while, and then sent over the internet. - -Networking is complementary to local I/O and we'll discuss it starting from a [future section](./remote-io.md). +Additionally, Deluge handles two [network-related I/O](../../../ipc/reading/network-sockets.md) operations: **requesting files from other peers** on the internet (which are temporarily stored in memory as they download) and **sending file parts** to other clients (involving reading from disk, storing temporarily in memory, and then transmitting online). ## Local I/O in Deluge -If you haven't done so in the Data lab, clone the [Deluge repository](https://github.com/deluge-torrent/deluge). +First of all, clone the [Deluge repository](https://github.com/deluge-torrent/deluge). +Now we are free to explore the source code for actions of interest. ### Writing Data to Files Now find the `write_torrentfile()` function in the repository. It defines an inner function called `write_file()`, which does the actual writing. -Notice that it's very similar to the example we saw in `support/simple-file-operations/file_operations.py`: +Notice that it's very similar to the example we saw in `simple-file-operations/support/file_operations.py`: ```Python with open(filepath, 'wb') as save_file: save_file.write(filedump) -``` - -The code above is the same as: -```Python -save_file = open(filepath, 'wb') -save_file.write(filedump) -save_file.close() +# Same as: +# save_file = open(filepath, 'wb') +# save_file.write(filedump) +# save_file.close() ``` The `with` keyword is just a _context manager_ that makes sure `save_file` is also closed when its "body" finished. @@ -51,7 +36,7 @@ Writing the data is done simply by calling the `write()` function. As you've probably figured out, there's an entire **software stack** beneath this `write()` function that sends the data first to libc and then to the kernel. Furthermore, the kernel itself has its own separate stack for handling I/O requests. -![Software Stacks Everywhere](../media/software-stacks-everywhere.jpeg) +![Software Stacks Everywhere](../../media/software-stacks-everywhere.jpeg) ### Error Handling @@ -75,7 +60,7 @@ Because I/O handling means dealing with peripherals or devices outside the typic The more devices, the higher the complexity. The higher the complexity, the higher the likelihood of errors. -[Quiz](../quiz/local-io-errors.md) +[Quiz](../../drills/questions/local-io-errors.md) ### Reading Data from Files diff --git a/chapters/io/file-descriptors/guides/fd-table/README.md b/chapters/io/file-descriptors/guides/fd-table/README.md new file mode 100644 index 0000000000..294ad1ec30 --- /dev/null +++ b/chapters/io/file-descriptors/guides/fd-table/README.md @@ -0,0 +1,94 @@ +# File Descriptor Table + +Just as each process has its own Virtual Address Space for memory access, it also maintains its own **File Descriptor Table** (FDT) for managing open files. +In this section we will explore how the process structures change when executing syscalls like `open()`, `read()`, `write()`, and `close()`. + +Upon startup, every process has three **file descriptors** that correspond to standard input (`stdin`), standard output (`stdout`), and standard error (`stderr`). +These descriptors are inherited from the parent process and occupy the **first three** entries in the process's **FDT**. + +| fd | Open File Struct | +|-------|--------------------| +| 0 | `stdin` | +| 1 | `stdout` | +| 2 | `stderr` | + +Each entry **points to** an **open file structure** which stores data about the current session: + +- **Permissions**: define how the file can be accessed (read, write, or execute); + these are the options passed to `open()`. +- **Offset**: the current position inside the file from which the next read or write operation will occur. +- **Reference Count**: The number of file descriptors referencing this open file structure. +- **Inode Pointer**: A pointer to the inode structure that contains both the data and metadata associated with the file. + +These **Open File Structures** are held in the **Open File Table (OFT)**, which is global across the system. +Whenever a new file is opened, a new entry is created in this table. + +To illustrate this, let's consider a code snippet and examine how the File Descriptor Table and Open File Table would appear. +We will focus on `fd`, permissions, offset, and reference count, as the inode pointer is not relevant at this moment. +For simplicity, we'll also omit the standard file descriptors, as they remain unchanged. + +```c +int fd = open("file.txt", O_RDONLY); +int fd2 = open("file2.txt", O_WRONLY | O_APPEND); +int fd3 = open("file.txt", O_RDWR); +``` + +| OFT index | Path | Perm | Off | `RefCount` | +|-----------|-------------|------|--------|-------------| +| ... | ... | ... | ... | ... | +| 123 | `file.txt` | r-- | 0 | 1 | +| 140 | `file2.txt` | -w- | 150 | 1 | +| 142 | `file.txt` | rw- | 0 | 1 | + +| fd | Open File Struct (OFT index) | +|-------|-------------------------------| +| 3 | 123 | +| 4 | 140 | +| 5 | 142 | + +Let's discuss the changes from the OFT and FDT to understand what happened: + +- `open("file.txt", O_RDONLY)` created a new **open file structure** in the **Open File Table** for `file.txt`. + The entry has read-only (`O_RDONLY`) permissions and offset `0`, representing the start of the file. + Subsequently, file descriptor `3` was assigned to point to this OFT entry, and the reference counter was set to `1`. +- `open("file2.txt", O_WRONLY)` created a similar structure in the OFT for `file2.txt`, but with write-only (`O_WRONLY`) permissions and an offset of `150`, representing the end of the file (`O_APPEND`). + It then assigned this entry to file descriptor `4`. +- `open("file.txt", O_RDWR)` created a new **open file structure** for `file.txt` and assigned it to file descriptor `5`. + +At this point, one might wonder why the last `open()` call didn't reuse the entry at file descriptor `3` and increase its reference counter instead. +It might seem logical, but doing so would lead to conflicts with the **permissions** and **offset** of the two open file structures. +**Remember:** each `open()` call creates a new **open file structure** in the **Open File Table**. + +This raises the question about the necessity for a reference counter. +The short answer is `dup()` (or `dup2()`) syscall, which duplicates a file descriptor. +Let's continue our previous example with the following snippet: + +```c +// fd = 3 (from the previous snippet) +int fd4 = dup(fd); +``` + +| OFT index | Path | Perm | Offset | `RefCount` | +|-----------|-------------|------|--------|-------------| +| ... | ... | ... | ... | ... | +| 123 | `file.txt` | r-- | 0 | 2 | +| 140 | `file2.txt` | -w- | 150 | 1 | +| 142 | `file.txt` | rw- | 0 | 1 | + +| fd | Open File Struct (OFT index) | +|-------|-------------------------------| +| 3 | 123 | +| 4 | 140 | +| 5 | 142 | +| 6 | 123 | + +The call to `dup(fd)` created a new file descriptor (`6`) that points to the same **open file structure** as its argument `fd` (which equals `3` in our example). +This operation also incremented the reference counter for the entry `123` in the **Open File Table**. + +As a result, operations performed on file descriptor `3` and file descriptor `6` are equivalent. +For instance, `read(3)` and `read(6)` will both increment the shared file offset, while the offset of file descriptor `5` will remain unchanged. +If you want to see a concrete example of when duplicating file descriptors is useful, check out `file-descriptors/guides/fd-table/support/redirect-stdout.c`. + +Now that you know how to create entries in the **File Descriptor Table** and the **Open File Table**, it’s important to understand how to remove them. +Calling `close()` will **always** free a file descriptor, but it will **only decrement** the reference counter of the **open file structure**. +The actual closing of the file occurs when the reference counter reaches `0`. diff --git a/chapters/io/file-descriptors/guides/fd-table/support/.gitignore b/chapters/io/file-descriptors/guides/fd-table/support/.gitignore new file mode 100644 index 0000000000..312038007a --- /dev/null +++ b/chapters/io/file-descriptors/guides/fd-table/support/.gitignore @@ -0,0 +1 @@ +redirect_stdout diff --git a/chapters/io/file-descriptors/guides/fd-table/support/Makefile b/chapters/io/file-descriptors/guides/fd-table/support/Makefile new file mode 100644 index 0000000000..0a5fb64326 --- /dev/null +++ b/chapters/io/file-descriptors/guides/fd-table/support/Makefile @@ -0,0 +1,38 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = redirect_stdout +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + -rm -f file.txt + +.PHONY: all clean diff --git a/chapters/io/file-descriptors/guides/fd-table/support/redirect_stdout.c b/chapters/io/file-descriptors/guides/fd-table/support/redirect_stdout.c new file mode 100644 index 0000000000..fd81e47948 --- /dev/null +++ b/chapters/io/file-descriptors/guides/fd-table/support/redirect_stdout.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include "utils/utils.h" + +#define FILENAME "file.txt" + +/** + * This looks like a good idea, but the change to the file descriptor table is not atomic. + * If another thread is running in parallel, it can call open and get file descriptor 1 before us. + */ +void naive_redirect_stdout(void) +{ + int fd, rc; + + // Free the file descriptor 1. + rc = close(STDOUT_FILENO); + DIE(rc == -1, "close"); + + // Open a file and get **the first** available file descriptor. + fd = open(FILENAME, O_WRONLY | O_CREAT | O_TRUNC, 0644); + DIE(fd == -1, "open"); +} + +/** + * dup2 will close the file descriptor 1 and open a new file on the same file descriptor, atomically. + */ +void redirect_stdout(void) +{ + int fd, rc; + + // Open a file and get **the first** available file descriptor. + fd = open(FILENAME, O_WRONLY | O_CREAT | O_TRUNC, 0644); + DIE(fd == -1, "open"); + + // Close the file descriptor 1 and open a new file on the same file descriptor, atomically. + rc = dup2(fd, STDOUT_FILENO); + DIE(rc == -1, "dup2"); + + // We don't need the old file descriptor anymore. + rc = close(fd); + DIE(rc == -1, "close"); +} + +/** + * We propose to redirect the standard output to a file. + * We will do this without changing the code of the main function. + * + * By default, printf writes to the standard output i.e. the file descriptor 1. + * So, we will close file descriptor 1 and open a new file on the same file descriptor. + */ +int main(void) +{ + redirect_stdout(); + printf("Hello, world!\n"); + + return 0; +} diff --git a/chapters/io/file-descriptors/guides/fd-table/support/utils/log/CPPLINT.cfg b/chapters/io/file-descriptors/guides/fd-table/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/file-descriptors/guides/fd-table/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/file-descriptors/guides/fd-table/support/utils/log/log.c b/chapters/io/file-descriptors/guides/fd-table/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/file-descriptors/guides/fd-table/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/file-descriptors/guides/fd-table/support/utils/log/log.h b/chapters/io/file-descriptors/guides/fd-table/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/file-descriptors/guides/fd-table/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/file-descriptors/guides/fd-table/support/utils/utils.h b/chapters/io/file-descriptors/guides/fd-table/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/file-descriptors/guides/fd-table/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/file-descriptors/guides/file-mappings/README.md b/chapters/io/file-descriptors/guides/file-mappings/README.md new file mode 100644 index 0000000000..5bcbf905a7 --- /dev/null +++ b/chapters/io/file-descriptors/guides/file-mappings/README.md @@ -0,0 +1,74 @@ +# File Mappings + +Mapping a file to the VAS of a process is similar to how shared libraries are loaded into the same VAS. +It's a fancier way of saying that the contents of a file are copied from a given offset within that file to a given address. +What's nice about this is that the OS handles all offsets, addresses and memory allocations on its own, with a single highly versatile syscall: `mmap()`. + +Let's run a `sleep` process and inspect its memory zones: + +```console +student@os:~$ sleep 1000 & # start a `sleep` process in the background +[1] 17579 + +student@os:~$ cat /proc/$(pidof sleep)/maps +55b7b646f000-55b7b6471000 r--p 00000000 103:07 6423964 /usr/bin/sleep +55b7b6471000-55b7b6475000 r-xp 00002000 103:07 6423964 /usr/bin/sleep +55b7b6475000-55b7b6477000 r--p 00006000 103:07 6423964 /usr/bin/sleep +55b7b6478000-55b7b6479000 r--p 00008000 103:07 6423964 /usr/bin/sleep +55b7b6479000-55b7b647a000 rw-p 00009000 103:07 6423964 /usr/bin/sleep +55b7b677c000-55b7b679d000 rw-p 00000000 00:00 0 [heap] +7fe442f61000-7fe44379d000 r--p 00000000 103:07 6423902 /usr/lib/locale/locale-archive +7fe44379d000-7fe4437bf000 r--p 00000000 103:07 6432810 /usr/lib/x86_64-linux-gnu/libc-2.31.so +7fe4437bf000-7fe443937000 r-xp 00022000 103:07 6432810 /usr/lib/x86_64-linux-gnu/libc-2.31.so +7fe443937000-7fe443985000 r--p 0019a000 103:07 6432810 /usr/lib/x86_64-linux-gnu/libc-2.31.so +7fe443985000-7fe443989000 r--p 001e7000 103:07 6432810 /usr/lib/x86_64-linux-gnu/libc-2.31.so +7fe443989000-7fe44398b000 rw-p 001eb000 103:07 6432810 /usr/lib/x86_64-linux-gnu/libc-2.31.so +7fe44398b000-7fe443991000 rw-p 00000000 00:00 0 +7fe4439ad000-7fe4439ae000 r--p 00000000 103:07 6429709 /usr/lib/x86_64-linux-gnu/ld-2.31.so +7fe4439ae000-7fe4439d1000 r-xp 00001000 103:07 6429709 /usr/lib/x86_64-linux-gnu/ld-2.31.so +7fe4439d1000-7fe4439d9000 r--p 00024000 103:07 6429709 /usr/lib/x86_64-linux-gnu/ld-2.31.so +7fe4439da000-7fe4439db000 r--p 0002c000 103:07 6429709 /usr/lib/x86_64-linux-gnu/ld-2.31.so +7fe4439db000-7fe4439dc000 rw-p 0002d000 103:07 6429709 /usr/lib/x86_64-linux-gnu/ld-2.31.so +7fe4439dc000-7fe4439dd000 rw-p 00000000 00:00 0 +7ffd07aeb000-7ffd07b0c000 rw-p 00000000 00:00 0 [stack] +7ffd07b8b000-7ffd07b8e000 r--p 00000000 00:00 0 [vvar] +7ffd07b8e000-7ffd07b8f000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall] +``` + +In the output above, you can see that the `.text`, `.rodata`, and `.data` sections for each dynamic library are mapped into the process’s VAS, along with the sections of the main executable. + +To understand how these mappings are created, let’s explore a simpler example. +Below is an illustration of how `libc` is loaded (or mapped) into the VAS of an `ls` process. + +```console +student@os:~$ strace ls +openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 +[...] +mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fb313c9c000 +mmap(0x7fb313cbe000, 1540096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7fb313cbe000 +mmap(0x7fb313e36000, 319488, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7fb313e36000 +mmap(0x7fb313e84000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7fb313e84000 +``` + +For a quick recap on `mmap(addr, length, prot, flags, fd, offset)`, the fifth argument specifies the **file descriptor** to copy data from, while the sixth is the offset within the file from where to start copying. + +In summary, when an executable runs, the loader uses `mmap()` to reserve memory zones for its shared libraries. +Performance is not affected by this since pages are populated **on-demand**, when they’re accessed for the first time. + +## File I/O vs `mmap()` + +When it comes to dynamic libraries, `mmap` is unmatched. +With a single call, it handles **address mapping**, **permission setting**, and leverages demand paging to populate pages only when accessed. +Additionally, `mmap()` fully supports **copy-on-write (COW)**, allowing libraries to share the same physical frames across multiple processes, which conserves memory and reduces load time. + +In contrast, using `read` and `write` would require loading the entire library into physical memory for each process individually, missing out on both the **copy-on-write** and **demand paging** benefits. + +For regular files, however, the choice isn’t always straightforward. +The main sources of overhead for `mmap()` include managing virtual memory mappings - which can lead to **TLB flushes** - and the cost of page faults due to **demand paging**. + +On the plus side, `mmap()` excels with random access patterns, efficiently reusing mapped pages. +It is also great for operating large amounts of data, as it enables the kernel to automatically unload and reload pages as needed when memory when under memory pressure. + +A concrete scenario where these downsides outweigh the benefits of `mmap()` is one-time, sequential I/O. +If you’re simply planning to read or write a file in one go, `read()` and `write()` are the way to go. diff --git a/chapters/io/file-descriptors/guides/libc-FILE-struct/README.md b/chapters/io/file-descriptors/guides/libc-FILE-struct/README.md new file mode 100644 index 0000000000..ec2c572826 --- /dev/null +++ b/chapters/io/file-descriptors/guides/libc-FILE-struct/README.md @@ -0,0 +1,94 @@ +# libc `FILE` struct + +Now, we will take a short look at how the [file descriptors](../../reading/file-descriptors.md) are handled in libc. +The Software Stack chapter has taught us that applications generally interact with libraries which expose wrappers on top of syscalls. +The most important library in a POSIX system (such as Linux) is libc. +Among many others, it provides higher-level abstractions over I/O-related syscalls. + +**Musl** (read just like "muscle") is a lightweight implementation of libc, which exposes the same API that you have used so far, while also being fit for embedded and OS development. +For example, [Unikraft](https://unikraft.org/) [unikernels](https://unikraft.org/docs/concepts/) may [use musl](https://github.com/unikraft/lib-musl). + +First, it provides a `struct` that groups together multiple data that is necessary when handling files. +We know from the example in `support/simple-file-operations/file_operations.c` that the file handler employed by libc is `FILE *`. +`FILE` is just a `typedef` for [`struct _IO_FILE`](https://elixir.bootlin.com/musl/v1.2.3/source/src/internal/stdio_impl.h#L21). +Here are the most important fields in `struct _IO_FILE`: + +```c +struct _IO_FILE { + int fd; /* File descriptor */ + + unsigned flags; /* Flags with which `open()` was called */ + + int mode; /* File permissions; passed to `open()` */ + + off_t off; /* File offset from where to read / write */ + + /** + * Internal buffer used to make fewer costly `read()`/`write()` + * syscalls. + */ + unsigned char *buf; + size_t buf_size; + + /* Pointers for reading and writing from/to the buffer defined above. */ + unsigned char *rpos, *rend; + unsigned char *wend, *wpos; + + /* Function pointers to syscall wrappers. */ + size_t (*read)(FILE *, unsigned char *, size_t); + size_t (*write)(FILE *, const unsigned char *, size_t); + off_t (*seek)(FILE *, off_t, int); + int (*close)(FILE *); + + /* Lock for concurrent file access. */ + volatile int lock; +}; +``` + +As you might have imagined, this structure contains the underlying file descriptor, the `mode` (read, write, truncate etc.) with which the file was opened, as well as the offset within the file from which the next read / write will start. + +Libc also defines its own wrappers over commonly-used syscalls, such as `read()`, `write()`, `close()` and `lseek()`. +These syscalls themselves need to be implemented by the driver for each file system. +This is done by writing the required functions for each syscall and then populating [this structure](https://elixir.bootlin.com/linux/v6.0.9/source/include/linux/fs.h#L2093) with pointers to them. +You will recognise quite a few syscalls: `open()`, `close()` `read()`, `write()`, `mmap()` etc. + +## `printf()` Buffering + +1. Navigate to `buffering/support/printf_buffering.c`. + Those `printf()` calls obviously end up calling `write()` at some point. + Run the code under `strace`. + + [Quiz: What syscall does `printf` use?](../../drills/questions/strace-printf.md) + + Since there is only one `write()` syscall despite multiple calls to `printf()`, it means that the strings given to `printf()` as arguments are kept _somewhere_ until the syscall is made. + That _somewhere_ is precisely that buffer inside `struct _IO_FILE` that we highlighted above. + Remember that syscalls cause the system to change from user mode to kernel mode, which is time-consuming. + Instead of performing one `write()` syscall per call to `printf()`, it is more efficient to copy the string passed to `printf()` to an **internal buffer** inside libc (the `unsigned char *buf` from above) and then at a given time (like when the buffer is full for example) `write()` the whole buffer. + This results in far fewer `write()` syscalls. + +1. Now, it is interesting to see how we can force libc to dump that internal buffer. + The most direct way is by using the `fflush()` library call, which is made for this exact purpose. + But we can be more subtle. + Add a `\n` in some of the strings printed in `buffering/support/printf_buffering.c`. + Place them wherever you want (at the beginning, at the end, in the middle). + Recompile the code and observe its change in behaviour under `strace`. + + [Quiz: How to get data out of `printf`'s buffer?](../../drills/questions/flush-libc-buffer.md) + + Now we know that I/O buffering **does happen** within libc. + If you need further convincing, check out the Musl implementation of [`fread()`](https://elixir.bootlin.com/musl/v1.2.3/source/src/stdio/fread.c#L6), for example. + It first copies the [data previously saved in the internal buffer](https://elixir.bootlin.com/musl/v1.2.3/source/src/stdio/fread.c#L16): + + ```c + if (f->rpos != f->rend) { + /* First exhaust the buffer. */ + k = MIN(f->rend - f->rpos, l); + memcpy(dest, f->rpos, k); + f->rpos += k; + dest += k; + l -= k; + } + ``` + + Then, if more data is requested and the internal buffer isn't full, it refills it using [the internal `read()` wrapper](https://elixir.bootlin.com/musl/v1.2.3/source/src/stdio/fread.c#L27). + This wrapper also places the data inside the destination buffer. diff --git a/chapters/io/file-descriptors/guides/libc-FILE-struct/support/.gitignore b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/.gitignore new file mode 100644 index 0000000000..274e917e6d --- /dev/null +++ b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/.gitignore @@ -0,0 +1,2 @@ +*buffering +*.txt diff --git a/chapters/io/file-descriptors/guides/libc-FILE-struct/support/Makefile b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/Makefile new file mode 100644 index 0000000000..5b554ce0ab --- /dev/null +++ b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/Makefile @@ -0,0 +1,37 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = printf_buffering +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + +.PHONY: all clean diff --git a/content/chapters/io/lab/support/buffering/printf_buffering.c b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/printf_buffering.c similarity index 100% rename from content/chapters/io/lab/support/buffering/printf_buffering.c rename to chapters/io/file-descriptors/guides/libc-FILE-struct/support/printf_buffering.c diff --git a/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/log/CPPLINT.cfg b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/log/log.c b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/log/log.h b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/utils.h b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/file-descriptors/guides/libc-FILE-struct/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/file-descriptors/guides/reading-linux-dirs/README.md b/chapters/io/file-descriptors/guides/reading-linux-dirs/README.md new file mode 100644 index 0000000000..e5fb5b5ce1 --- /dev/null +++ b/chapters/io/file-descriptors/guides/reading-linux-dirs/README.md @@ -0,0 +1,39 @@ +# Reading Linux Directories + +**Everything in Linux is a file.** +This statement says that the Linux OS treats every entry in a file system (regular file, directory, block device, char device, link, UNIX socket) as a file. +This unified approach simplifies file handling, allowing a single interface to interact with various types of entries. +Let's see how this works in practice: + +1. Navigate to `guides/reading-linux-dirs/support/` and checkout `dir_ops.c`. + This code creates a directory `dir`, if it does not exists, and attempts to open it the same way we would open a regular file. + Compile and run the code. + + ```console + student@os:~/.../reading-linux-dirs/support$ ./dir_ops + 12:45:34 FATAL dir_ops.c:17: fopen: Is a directory + ``` + + The error message is crystal clear: we cannot use `fopen()` on directories. + So the `FILE` structure is unsuited for directories. + Therefore, this handler is not generic enough for a regular Linux filesystem, and we have to use a lower-level function. + + [Quiz - What syscall does fopen() use?](../../drills/questions/fopen-syscall.md) + +1. Now that we know that `fopen()` relies `openat()`, let's try using [`open()`](https://man7.org/linux/man-pages/man2/open.2.html), which wraps `openat()` but offers a simpler interface. + + Inspect, compile and run the code `dir_ops_syscalls.c`. + + ```console + student@os:~/...reading-linux-dirs/support$ ./dir_ops_syscalls + Directory file descriptor is: 3 + ``` + + This output proves that the `open()` syscall is capable of also handling directories, so it's closer to what we want. + + **Note:** that it is rather uncommon to use `open()` for directories. + Most of the time, [`opendir()`](https://man7.org/linux/man-pages/man3/opendir.3.html) is used instead. + +In conclusion, the key difference between `fopen()` and `open()` is in the type of handler they return. +The `FILE` structure from `fopen()` is suited only for regular files, while the **file descriptor** returned by `open()` is more flexible. +The differences between the two handlers are explored in the [file descriptors section](../../../file-descriptors/reading/file-descriptors.md). diff --git a/chapters/io/file-descriptors/guides/reading-linux-dirs/support/.gitignore b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/.gitignore new file mode 100644 index 0000000000..f559ddfc96 --- /dev/null +++ b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/.gitignore @@ -0,0 +1,2 @@ +dir_ops +dir_ops_syscalls diff --git a/chapters/io/file-descriptors/guides/reading-linux-dirs/support/Makefile b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/Makefile new file mode 100644 index 0000000000..9be31f02fe --- /dev/null +++ b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/Makefile @@ -0,0 +1,39 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = dir_ops dir_ops_syscalls +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + @mkdir -p "dir" + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + -rm -rf "dir" + +.PHONY: all clean diff --git a/content/chapters/io/lab/support/simple-file-operations/directory_operations.c b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/dir_ops.c similarity index 72% rename from content/chapters/io/lab/support/simple-file-operations/directory_operations.c rename to chapters/io/file-descriptors/guides/reading-linux-dirs/support/dir_ops.c index fa2b90d303..4d3d9f165d 100644 --- a/content/chapters/io/lab/support/simple-file-operations/directory_operations.c +++ b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/dir_ops.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause #include +#include #include "utils/utils.h" @@ -10,6 +11,8 @@ static void open_directory(const char *dir_name) { FILE *f; + // Always fails because FILE * cannot handle directories. + // Use open() instead. f = fopen(dir_name, "w"); DIE(!f, "fopen"); } diff --git a/content/chapters/io/lab/support/file-descriptors/open_directory.c b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/dir_ops_syscalls.c similarity index 88% rename from content/chapters/io/lab/support/file-descriptors/open_directory.c rename to chapters/io/file-descriptors/guides/reading-linux-dirs/support/dir_ops_syscalls.c index 6e2353f6ca..89edb850e8 100644 --- a/content/chapters/io/lab/support/file-descriptors/open_directory.c +++ b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/dir_ops_syscalls.c @@ -7,7 +7,7 @@ #include "utils/utils.h" -#define DIR_NAME "../simple-file-operations/dir" +#define DIR_NAME "dir" static void open_directory(const char *dir_name) { diff --git a/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/log/CPPLINT.cfg b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/log/log.c b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/log/log.h b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/utils.h b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/file-descriptors/guides/reading-linux-dirs/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/file-descriptors/guides/simple-file-operations/README.md b/chapters/io/file-descriptors/guides/simple-file-operations/README.md new file mode 100644 index 0000000000..e769e93de7 --- /dev/null +++ b/chapters/io/file-descriptors/guides/simple-file-operations/README.md @@ -0,0 +1,31 @@ +# Simple File Operations + +To manipulate the file (read its contents, modify them, change its size etc.), each process must first get a **handler** to this file. +Think of this handler as an object by which the process can identify and refer to the file. + +Now take a look at the code examples in `file-descriptors/guides/simple-file-operations/support`. +Each of them reads the contents of `file.txt`, modifies them, and then reads the previously modified file again. +Use `make` to compile the C code, and `make java-file-operations` to compile the Java code. + +Now run the programs repeatedly in whatever order you wish: + +```console +student@os:~/.../simple-file-operations/support$ python3 file_operations.py +File contents are: OS Rullz! +Wrote new data to file +File contents are: Python was here! + +student@os:~/.../simple-file-operations/support$ ./file_operations # from the C code +File contents are: Python was here! +Wrote new data to file +File contents are: C was here! + +student@os:~/.../simple-file-operations/support$ java FileOperations +File contents are: Python was here! +Wrote new data to file +File contents are: Java was here! +``` + +Note that each piece of code creates a variable, which is then used as a handler. + +[Quiz](../../drills/questions/file-handler-c.md) diff --git a/chapters/io/file-descriptors/guides/simple-file-operations/support/.gitignore b/chapters/io/file-descriptors/guides/simple-file-operations/support/.gitignore new file mode 100644 index 0000000000..93312bb1ab --- /dev/null +++ b/chapters/io/file-descriptors/guides/simple-file-operations/support/.gitignore @@ -0,0 +1 @@ +file_operations diff --git a/content/chapters/io/lab/support/simple-file-operations/FileOperations.java b/chapters/io/file-descriptors/guides/simple-file-operations/support/FileOperations.java similarity index 94% rename from content/chapters/io/lab/support/simple-file-operations/FileOperations.java rename to chapters/io/file-descriptors/guides/simple-file-operations/support/FileOperations.java index 0abe6b68b1..2bffc24a6f 100644 --- a/content/chapters/io/lab/support/simple-file-operations/FileOperations.java +++ b/chapters/io/file-descriptors/guides/simple-file-operations/support/FileOperations.java @@ -1,4 +1,7 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ +/** + * This package provides support for basic file operations. + */ +package com.example.FileOperations; import java.io.File; import java.io.FileNotFoundException; diff --git a/chapters/io/file-descriptors/guides/simple-file-operations/support/Makefile b/chapters/io/file-descriptors/guides/simple-file-operations/support/Makefile new file mode 100644 index 0000000000..f16a0f5d10 --- /dev/null +++ b/chapters/io/file-descriptors/guides/simple-file-operations/support/Makefile @@ -0,0 +1,38 @@ +all: file_operations java_file_operations + +# Get the relative path to the directory of the current makefile. +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils +LOGGER_DIR := $(UTILS_DIR)/log + +# Compiler and flags +CC = gcc +JAVAC = javac +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Rule to compile the logger +$(LOGGER_OBJ): $(LOGGER_DIR)/log.c + $(MAKE) -C $(LOGGER_DIR) $(LOGGER_OBJ) + +# Rule to compile the C file_operations binary +file_operations: file_operations.c $(LOGGER) + $(CC) $(CFLAGS) $^ -o $@ + +# Rule to compile the Java FileOperations class +java_file_operations: FileOperations.java + $(JAVAC) FileOperations.java + +# Clean rule: Remove object files and binaries +clean: + -rm -f file_operations *.o + -rm -f FileOperations.class + -rm -f $(LOGGER) + +.PHONY: all clean diff --git a/chapters/io/file-descriptors/guides/simple-file-operations/support/file.txt b/chapters/io/file-descriptors/guides/simple-file-operations/support/file.txt new file mode 100644 index 0000000000..c2d0be5295 --- /dev/null +++ b/chapters/io/file-descriptors/guides/simple-file-operations/support/file.txt @@ -0,0 +1 @@ +Python was here! \ No newline at end of file diff --git a/content/chapters/io/lab/support/simple-file-operations/file_operations.c b/chapters/io/file-descriptors/guides/simple-file-operations/support/file_operations.c similarity index 100% rename from content/chapters/io/lab/support/simple-file-operations/file_operations.c rename to chapters/io/file-descriptors/guides/simple-file-operations/support/file_operations.c diff --git a/content/chapters/io/lab/support/simple-file-operations/file_operations.py b/chapters/io/file-descriptors/guides/simple-file-operations/support/file_operations.py similarity index 100% rename from content/chapters/io/lab/support/simple-file-operations/file_operations.py rename to chapters/io/file-descriptors/guides/simple-file-operations/support/file_operations.py diff --git a/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/log/CPPLINT.cfg b/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/log/log.c b/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/log/log.h b/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/log/log.h new file mode 100644 index 0000000000..c8d1dee06a --- /dev/null +++ b/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + + typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; + } log_Event; + + typedef void (*log_LogFn)(log_Event *ev); + typedef void (*log_LockFn)(bool lock, void *udata); + + enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL + }; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + + const char *log_level_string(int level); + void log_set_lock(log_LockFn fn, void *udata); + void log_set_level(int level); + void log_set_quiet(bool enable); + int log_add_callback(log_LogFn fn, void *udata, int level); + int log_add_fp(FILE *fp, int level); + + void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/utils.h b/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/file-descriptors/guides/simple-file-operations/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/io/lab/media/file-descriptors.svg b/chapters/io/file-descriptors/media/file-descriptors.svg similarity index 100% rename from content/chapters/io/lab/media/file-descriptors.svg rename to chapters/io/file-descriptors/media/file-descriptors.svg diff --git a/content/chapters/io/lab/media/software-stacks-everywhere.jpeg b/chapters/io/file-descriptors/media/software-stacks-everywhere.jpeg similarity index 100% rename from content/chapters/io/lab/media/software-stacks-everywhere.jpeg rename to chapters/io/file-descriptors/media/software-stacks-everywhere.jpeg diff --git a/chapters/io/file-descriptors/reading/fd-operations.md b/chapters/io/file-descriptors/reading/fd-operations.md new file mode 100644 index 0000000000..5da3b8c072 --- /dev/null +++ b/chapters/io/file-descriptors/reading/fd-operations.md @@ -0,0 +1,70 @@ +# File Descriptor Operations + +File descriptors are the primary means of referencing files in our system. +They are created, deleted, and manipulated through file interface operations, namely `open()`, `close()` `read()`, `write()`, and `lseek()`. +From a programmer's perspective, file descriptors are simply indexes into the process's **File Descriptor Table**, which maintains a list of all currently open files for that process. + +In this section, we will focus on how to utilize file descriptors to perform the same operations that `FILE` allows, and more. +If you want to delve deeper into file descriptors, we recommend exploring [this guide on the **File Descriptor Table**](../guides/fd-table/README.md). + +## `open()` + +All processes start with three default file descriptors, inherited from the process's parent: + +- `stdin` (standard input): 0 +- `stdout` (standard output): 1 +- `stderr` (standard error): 2 + +To create new file descriptors (i.e. open new files), a process can use the [`open()`](https://man7.org/linux/man-pages/man2/open.2.html) system call. +It receives the path to the file, some flags which are akin to the `mode` string passed to `fopen()`. +An optional `mode` parameter that denotes the file's permissions if the `open` must create it can also be provided. +If you use `O_CREAT`, just remember to also pass `0644` (`rw-r--r--` in octal, denoted by the first `0`), or permissions more restrictive. + +Some other useful flags for `open()` are: + +- `O_APPEND`: place file cursor at the end +- `O_CLOEXEC`: close the file descriptor when `exec()` is called. + This is useful because child processes inherit the file descriptors, and this can lead to security problems. +- `O_TRUNC`: truncate the file to length 0. + +## `close()` + +Once you are done with a file descriptor you should call `close()` to free its **open file structure**. +This is similar to how you free memory once you are done with it. + +## `read()` and `write()` + +```c +read_bytes = read(fd, buf, num_bytes); +written_bytes = write(fd, buf, num_bytes); +``` + +As you know, verifying the return code of system calls is the way to go in general. +This is even more apparent when dealing with I/O syscalls, namely `read()` and `write()`, which return the number of bytes read or written. + +Syscalls returning the number of bytes might seem redundant, but once you hear about partial I/O operations, it is of utmost importance. +If your process was interrupted by a signal while reading or writing, it is up to you to continue from where it left off. + +**Remember: It is mandatory that we always use `read()` and `write()` inside `while` loops.** +Higher-level functions like `fread()` and `fwrite()` also use `while` loops when calling `read()` and `write()` respectively. +You can practice this by [implementing your own `cat` command](../drills/tasks/my-cat/README.md). + +In the following sections, we'll use file descriptors and `read()` and `write()` to interact with some inter-process-communication mechanisms, such as pipes. + +## `lseek()` + +As you know, reading or writing from a file always continues from where it left off. +Most of the time you would read from a file monotonically so it makes sense to keep the interface clean and handle bookkeeping in the back. + +For cases when you selectively update the file or jump around fetching data, or making updates, we have [`lseek`](https://man7.org/linux/man-pages/man2/lseek.2.html). + +```c +off_t lseek(int fd, off_t offset, int whence); +``` + +Its parameters are pretty intuitive: `fd` stands for the file descriptor and `offset` stands for the offset. +The `whence` directive explains what `offset` is relative to, and has the following values: + +- `SEEK_SET`: the file offset is set to offset bytes. +- `SEEK_CUR`: The file offset is set to its current location plus offset bytes. +- `SEEK_END`: the file offset is set to the size of the file plus offset bytes. diff --git a/chapters/io/file-descriptors/reading/file-descriptors.md b/chapters/io/file-descriptors/reading/file-descriptors.md new file mode 100644 index 0000000000..6036cee50a --- /dev/null +++ b/chapters/io/file-descriptors/reading/file-descriptors.md @@ -0,0 +1,78 @@ +# File Descriptors + +You've most likely had to deal with files in the past. +The most common command that works with files is `cat`. +For a quick refresher, let's write something to a file, and then read its contents. + +You’ve likely worked with files before; +now it’s time to see what happens behind the scenes. +The most common way to read a file in Linux is by using the `cat` command. +For a quick refresher, let’s do a demo by writing some text to a file and then reading it back. + +```console +student@os:~/$ echo "OS Rullz!" # Print 'OS Rullz!' +OS Rullz! +student@os:~/$ echo "OS Rullz!" > newfile.txt # redirect the output to newfile.txt +# Let's check the contents of newfile.txt +student@os:~/$ cat newfile.txt +OS Rullz! +``` + +If we were to implement this in C, we would use the `FILE` structure and write something like this: + +```c +FILE *f = fopen("newfile.txt", "r"); +if (!f) {...} // handle error + +char buf[1024]; +int rc = fread(buf, 1, sizeof(buf), f); +if (rc < 0) {...} // handle error + +printf("%s\n", buf); +``` + +For a complete example, check out this guide on [file operations in C, Python, and Java](../guides/simple-file-operations/README.md). + +## `FILE` Operations Explained + +The `FILE` structure is not the most straightforward method for performing file operations. +It is part of `libc` and functions as a handler for working with files. +This is not particular to C, as most programming languages offer similar handlers. + +Running `strace cat newfile.txt` reveals that `fopen()` wraps `open()` (or `openat`), `fread()` wraps `read()`, and `fclose()` wraps `close()`. +As you can see, the `FILE`-related functions are just syscalls prefixed with `f-`. + +| `FILE` Operation | Syscall | Description | +|---------------------|-----------------------------------------------------------------------------|-----------------------------------------------| +| `fopen()` | [`open()`](https://man7.org/linux/man-pages/man2/open.2.html) | Opens a file and returns a file pointer. | +| `fclose()` | [`close()`](https://man7.org/linux/man-pages/man2/close.2.html) | Closes the file associated with the pointer. | +| `fread()` | [`read()`](https://man7.org/linux/man-pages/man2/read.2.html) | Reads data from the file into a buffer. | +| `fwrite()` | [`write()`](https://man7.org/linux/man-pages/man2/write.2.html) | Writes data from a buffer to the file. | +| `fseek()` | [`lseek()`](https://man7.org/linux/man-pages/man2/lseek.2.html) | Moves the file position indicator. | +| `truncate()` | [`ftruncate()`](https://man7.org/linux/man-pages/man2/ftruncate.2.html) | Truncates the file to a specified length. | + +The main distinction between `FILE` operations and their corresponding system calls is that the latter use a **file descriptor** to reference a file. +**File descriptors** are simply indexes into the process's **File Descriptor Table**, which is the list of all currently open files for that process. + +This concept is not entirely new, as each process has three default channels: `stdin`, `stdout`, and `stderr`. +These are, in fact, the first three entries in every process’s **File Descriptor Table**. + +[Quiz: Test your intuition by finding the file descriptor of `stderr`](../drills/questions/stderr-fd.md) + +Let's translate our previous example to illustrate how this change affects the implementation: + +```c +int fd = open("newfile.txt", O_RDONLY) +if (fd < 0) {...} // handle error + +char buf[1024]; +int rc = read(fd, buf, sizeof(buf)); // Not complete, should've used a while loop +if (rc < 0) {...} // handle error + +buf[rc] = '\0'; // Null-terminate the buffer +printf("%s\n", buf); +``` + +To better understand the **file descriptor** API, you can either [keep reading about file descriptor operations](../reading/fd-operations.md) or checkout [this guide on reading Linux directories](../guides/reading-linux-dirs/README.md). + +If you're interested in understanding how `libc` utilizes file descriptors to simplify common operations, check out [this guide](../guides/libc-FILE-struct/README.md). diff --git a/content/chapters/io/lab/quiz/anonymous-pipes-limitation.md b/chapters/io/ipc/drills/questions/anonymous-pipes-limitation.md similarity index 100% rename from content/chapters/io/lab/quiz/anonymous-pipes-limitation.md rename to chapters/io/ipc/drills/questions/anonymous-pipes-limitation.md diff --git a/content/chapters/io/lab/quiz/bind-error-cause.md b/chapters/io/ipc/drills/questions/bind-error-cause.md similarity index 100% rename from content/chapters/io/lab/quiz/bind-error-cause.md rename to chapters/io/ipc/drills/questions/bind-error-cause.md diff --git a/content/chapters/io/lab/quiz/client-server-sender-receiver.md b/chapters/io/ipc/drills/questions/client-server-sender-receiver.md similarity index 85% rename from content/chapters/io/lab/quiz/client-server-sender-receiver.md rename to chapters/io/ipc/drills/questions/client-server-sender-receiver.md index f0d650d6a7..834ffad0c6 100644 --- a/content/chapters/io/lab/quiz/client-server-sender-receiver.md +++ b/chapters/io/ipc/drills/questions/client-server-sender-receiver.md @@ -18,4 +18,4 @@ Drawing a parallel from the UDP examples in `support/send-receive/`, which of th `receiver.py` is the passive component. It has to be started first and then waits for `sender.py` (the client) to send data. -Furthermore, you can only have **one** `receiver.py` running at the same time (remember the [multiple `bind()` bug](./bind-error-cause.md)) and multiple `sender.py`s. +Furthermore, you can only have **one** `receiver.py` running at the same time (remember the [multiple `bind()` bug](../questions/bind-error-cause.md)) and multiple `sender.py`s. diff --git a/content/chapters/io/lab/quiz/firefox-tcp-udp.md b/chapters/io/ipc/drills/questions/firefox-tcp-udp.md similarity index 100% rename from content/chapters/io/lab/quiz/firefox-tcp-udp.md rename to chapters/io/ipc/drills/questions/firefox-tcp-udp.md diff --git a/content/chapters/io/lab/quiz/pipe-ends.md b/chapters/io/ipc/drills/questions/pipe-ends.md similarity index 100% rename from content/chapters/io/lab/quiz/pipe-ends.md rename to chapters/io/ipc/drills/questions/pipe-ends.md diff --git a/content/chapters/io/lab/quiz/receiver-socket-fd.md b/chapters/io/ipc/drills/questions/receiver-socket-fd.md similarity index 100% rename from content/chapters/io/lab/quiz/receiver-socket-fd.md rename to chapters/io/ipc/drills/questions/receiver-socket-fd.md diff --git a/chapters/io/ipc/drills/tasks/anon-pipes/.gitignore b/chapters/io/ipc/drills/tasks/anon-pipes/.gitignore new file mode 100644 index 0000000000..296b2ccf64 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/anon-pipes/.gitignore @@ -0,0 +1 @@ +support/ diff --git a/chapters/io/ipc/drills/tasks/anon-pipes/Makefile b/chapters/io/ipc/drills/tasks/anon-pipes/Makefile new file mode 100644 index 0000000000..ff78d246ea --- /dev/null +++ b/chapters/io/ipc/drills/tasks/anon-pipes/Makefile @@ -0,0 +1,9 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + mkdir -p support/ + $(PYTHON) $(SCRIPT) --input ./solution --output ./support + +clean: + rm -rf support/ diff --git a/chapters/io/ipc/drills/tasks/anon-pipes/README.md b/chapters/io/ipc/drills/tasks/anon-pipes/README.md new file mode 100644 index 0000000000..946b91d39e --- /dev/null +++ b/chapters/io/ipc/drills/tasks/anon-pipes/README.md @@ -0,0 +1,22 @@ +# Anonymous Pipes Communication + +Navigate to `chapters/io/file-descriptors/drills/tasks/anon-pipes/support` and run `make`. +In this exercise, you'll implement client-server communication between a parent and a child process using an anonymous pipe. +The parent will act as the sender, while the child acts as the receiver, with both processes sharing messages through the pipe. +Since pipes are unidirectional, each process should close the end of the pipe it does not use. + +1. Use the [`pipe()` syscall](https://man7.org/linux/man-pages/man7/pipe.7.html) to create the pipe. + Remember, the first file descriptor (`fds[0]`) is the read end, and the second (`fds[1]`) is the write end, similar to how `stdin` and `stdout` are represented by file descriptors `0` and `1`. + + **Hint:** Use `exit` to end the program. + + [Quiz: Discover why you cannot use either end of the pipe for reading or writing](../../questions/pipe-ends.md) + +1. Solve the TODOs in `parent_loop` and `child_loop` so that the application stops on `exit`. + Ensure each process closes the its pipe end before exiting to prevent indefinite blocking. + + > Why is closing the pipe ends important? + + The child process checks for the end of communication by reading from the pipe and checking for `EOF`, which occurs when the write end is closed. + Without closing the write end, the child will block indefinitely in `read()`. + As for the parent, it will block indefinitely in `wait()`. diff --git a/chapters/io/ipc/drills/tasks/anon-pipes/generate_skels.py b/chapters/io/ipc/drills/tasks/anon-pipes/generate_skels.py new file mode 100644 index 0000000000..697c9d5b61 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/anon-pipes/generate_skels.py @@ -0,0 +1,151 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + or re.match(r".*\.py$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/content/chapters/io/lab/solution/pipes/.gitignore b/chapters/io/ipc/drills/tasks/anon-pipes/solution/.gitignore similarity index 100% rename from content/chapters/io/lab/solution/pipes/.gitignore rename to chapters/io/ipc/drills/tasks/anon-pipes/solution/.gitignore diff --git a/chapters/io/ipc/drills/tasks/anon-pipes/solution/Makefile b/chapters/io/ipc/drills/tasks/anon-pipes/solution/Makefile new file mode 100644 index 0000000000..b5670d071e --- /dev/null +++ b/chapters/io/ipc/drills/tasks/anon-pipes/solution/Makefile @@ -0,0 +1,37 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = anonymous_pipe +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + +.PHONY: all clean diff --git a/content/chapters/io/lab/support/pipes/anonymous_pipe.c b/chapters/io/ipc/drills/tasks/anon-pipes/solution/anonymous_pipe.c similarity index 59% rename from content/chapters/io/lab/support/pipes/anonymous_pipe.c rename to chapters/io/ipc/drills/tasks/anon-pipes/solution/anonymous_pipe.c index 555b37265e..2a3593c3fb 100644 --- a/content/chapters/io/lab/support/pipes/anonymous_pipe.c +++ b/chapters/io/ipc/drills/tasks/anon-pipes/solution/anonymous_pipe.c @@ -10,33 +10,24 @@ #include "utils/utils.h" #define BUFSIZE 128 -#define EXIT_STR "exit" #define PIPE_READ 0 #define PIPE_WRITE 1 -static bool check_for_exit(const char *input) -{ - if (strcmp(input, EXIT_STR) == 0 || strlen(input) == 0) - { - return true; - } - - return false; -} - static void child_loop(int readfd) { char output[BUFSIZE]; int rc; - while (1) - { + while (1) { rc = read(readfd, output, BUFSIZE); DIE(rc < 0, "read"); + /* TODO 7: Close the pipe head used for reading if the other pipe was closed (HINT: EOF) */ if (rc == 0) { - /* TODO: Close pipe head used for reading. */ + rc = close(readfd); + DIE(rc < 0, "close"); + break; } @@ -52,17 +43,19 @@ static void parent_loop(int writefd) char input[BUFSIZE]; int rc; - while (1) - { + while (1) { memset(input, 0, BUFSIZE); fgets(input, BUFSIZE, stdin); // Remove trailing newline if (input[strlen(input) - 1] == '\n') input[strlen(input) - 1] = '\0'; - if (check_for_exit(input)) + /* TODO 7: Close the pipe head if message is 'exit` */ + if (strcmp(input, "exit") == 0 || strlen(input) == 0) { - /* TODO: Close pipe head used for writing. */ + rc = close(writefd); + DIE(rc < 0, "close"); + break; } @@ -86,11 +79,12 @@ int main(void) { pid_t pid; int rc; + /* TODO 1: Define an array with size 2 to store the pipe heads */ int pipe_des[2]; + /* TODO 3: Create the pipe and print its file descriptors */ rc = pipe(pipe_des); DIE(rc < 0, "pipe"); - printf("pipe_des[0] = %d; pipe_des[1] = %d\n", pipe_des[PIPE_READ], pipe_des[PIPE_WRITE]); wait_for_input("pipe created"); @@ -99,21 +93,32 @@ int main(void) switch (pid) { case -1: /* Fork failed, cleaning up. */ - /* TODO: Close both heads of the pipe. */ + /* TODO 5: Close both heads of the pipe. */ + rc = close(pipe_des[PIPE_READ]); + DIE(rc < 0, "close"); + rc = close(pipe_des[PIPE_WRITE]); + DIE(rc < 0, "close"); + DIE(pid, "fork"); return EXIT_FAILURE; case 0: /* Child process. */ - /* TODO: Close unused pipe head by child. */ + /* TODO 2: Close unused pipe head by child. */ + rc = close(pipe_des[PIPE_WRITE]); + DIE(rc < 0, "close"); - /* TODO: Call child loop and pass pipe head used for reading. */ + /* TODO 1: Call child loop and pass pipe head used for reading. */ + child_loop(pipe_des[0]); break; default: /* Parent process. */ - /* TODO: Close unused pipe head by parent. */ + /* TODO 2: Close unused pipe head by parent. */ + rc = close(pipe_des[PIPE_READ]); + DIE(rc < 0, "close"); - /* TODO: Call parent loop and pass pipe head used for writing. */ + /* TODO 1: Call parent loop and pass pipe head used for writing. */ + parent_loop(pipe_des[PIPE_WRITE]); /* Wait for child process to finish. */ wait(NULL); diff --git a/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/log/CPPLINT.cfg b/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/log/log.c b/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/log/log.h b/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/utils.h b/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/ipc/drills/tasks/anon-pipes/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/ipc/drills/tasks/client-server/.gitignore b/chapters/io/ipc/drills/tasks/client-server/.gitignore new file mode 100644 index 0000000000..296b2ccf64 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/.gitignore @@ -0,0 +1 @@ +support/ diff --git a/chapters/io/ipc/drills/tasks/client-server/Makefile b/chapters/io/ipc/drills/tasks/client-server/Makefile new file mode 100644 index 0000000000..ff78d246ea --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/Makefile @@ -0,0 +1,9 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + mkdir -p support/ + $(PYTHON) $(SCRIPT) --input ./solution --output ./support + +clean: + rm -rf support/ diff --git a/chapters/io/ipc/drills/tasks/client-server/README.md b/chapters/io/ipc/drills/tasks/client-server/README.md new file mode 100644 index 0000000000..48351f5694 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/README.md @@ -0,0 +1,29 @@ +# Ordered Client-Server Communication + +Navigate to `chapters/io/ipc/drills/tasks/client-server/` and run `make` to generate the `support` directory. +This exercise will guide you in creating a basic messaging protocol between a server and a client. +Although in real-world applications a server typically handles multiple connections at once, here we focus on a single connection. +Handling multiple connections is further explored in [I/O multiplexing](../../../../io-multiplexing/reading/io-multiplexing.md). + +Our application protocol is defined as follows: + +- The **server** listens for connections on **localhost** and a specified **port**. +- The **client** connects to `localhost:port`. +- The **client** sends a message, which the **server** then prints, responds to, and the **client** prints the reply. + This sequence repeats in a loop. +- The communication ends when either the **client** or the **server** sends the message `exit`. + +Since we are blocking on `recv()`, the message order is fixed - the client **must** initiate communication. +In real-world applications, this constraint can be avoided with [I/O multiplexing](../../../../io-multiplexing/reading/io-multiplexing.md). + +1. Open `support/client.c` and complete the TODOs to enable message exchange with the server. + Test your client by running `python server.py` in one terminal and then `./client` in another. + If correctly implemented, you should be able to exchange messages as outlined above. + + **Bonus Question:** Why is it OK for the client to be implemented in C while the server is implemented in Python? + +1. Open `support/server.c` and complete the TODOs to enable message exchange with the client. + Test your server by running `./server` in one terminal and then `python client.py` in another. + If implemented correctly, you should be able to exchange messages as outlined above. + +If you're having difficulties solving this exercise, go through [the sockets API](../../../reading/unix-sockets.md) and [the client-server model](../../../reading/client-server-model.md). diff --git a/chapters/io/ipc/drills/tasks/client-server/generate_skels.py b/chapters/io/ipc/drills/tasks/client-server/generate_skels.py new file mode 100644 index 0000000000..697c9d5b61 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/generate_skels.py @@ -0,0 +1,151 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + or re.match(r".*\.py$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/content/chapters/io/lab/support/client-server/.gitignore b/chapters/io/ipc/drills/tasks/client-server/solution/.gitignore similarity index 50% rename from content/chapters/io/lab/support/client-server/.gitignore rename to chapters/io/ipc/drills/tasks/client-server/solution/.gitignore index b051c6c57f..f2ad85300e 100644 --- a/content/chapters/io/lab/support/client-server/.gitignore +++ b/chapters/io/ipc/drills/tasks/client-server/solution/.gitignore @@ -1 +1,2 @@ client +server diff --git a/chapters/io/ipc/drills/tasks/client-server/solution/Makefile b/chapters/io/ipc/drills/tasks/client-server/solution/Makefile new file mode 100644 index 0000000000..8aa3737d14 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/solution/Makefile @@ -0,0 +1,37 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = server client +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + +.PHONY: all clean diff --git a/chapters/io/ipc/drills/tasks/client-server/solution/client.c b/chapters/io/ipc/drills/tasks/client-server/solution/client.c new file mode 100644 index 0000000000..e873d8fa70 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/solution/client.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" + +#ifndef BUFSIZ +#define BUFSIZ 256 +#endif + +static const char IP[] = "127.0.0.1"; +static const int PORT = 5000; + +/** + * Create a sockaddr_in structure with the given IP and port. + */ +struct sockaddr_in get_sockaddr(const char *ip, const int port) +{ + struct sockaddr_in addr; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + + return addr; +} + +/** + * Get user input from stdin and remove trailing newline. + */ +static void get_user_input(char *buf, size_t size) +{ + memset(buf, 0, size); + fgets(buf, size, stdin); + // Remove trailing newline + if (buf[strlen(buf) - 1] == '\n') + buf[strlen(buf) - 1] = '\0'; +} + +/** + * Create a TCP socket and listen for incoming connections. + */ +int main(void) +{ + char buf[BUFSIZ]; + struct sockaddr_in addr = get_sockaddr(IP, PORT); + int sockfd; + int rc; + + /* TODO 2: Create a network socket with SOCK_STREAM type. */ + sockfd = socket(AF_INET, SOCK_STREAM, 0); + DIE(sockfd < 0, "socket"); + + /* TODO 2: Use connect() to connect to the server. */ + rc = connect(sockfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "connect"); + + while (1) { + get_user_input(buf, BUFSIZ); + if (strcmp(buf, "exit") == 0) + break; + + /* TODO 2: Send data to the connected socket */ + rc = send(sockfd, buf, strlen(buf), 0); + DIE(rc < 0, "send"); + + /* TODO 2: Wait for server to respond */ + memset(buf, 0, BUFSIZ); + rc = recv(sockfd, buf, BUFSIZ, 0); + DIE(rc < 0, "recv"); + + /* TODO 2: Stop the loop if recv() receives EOF */ + if (rc == 0) + break; + + printf("[Server %s:%d]: %s\n", IP, PORT, buf); + } + + /* TODO 2: Close sockfd. */ + rc = close(sockfd); + DIE(rc < 0, "close"); + + return 0; +} diff --git a/chapters/io/ipc/drills/tasks/client-server/solution/client.py b/chapters/io/ipc/drills/tasks/client-server/solution/client.py new file mode 100644 index 0000000000..853519520f --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/solution/client.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: BSD-3-Clause + +import socket +import sys + +# Constants +IP = "127.0.0.1" +PORT = 5000 +BUFSIZ = 256 + + +def main(): + # Create a TCP socket + try: + client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + except socket.error as e: + sys.exit(f"Socket creation failed: {e}") + + # Connect to the server + try: + client_socket.connect((IP, PORT)) + print(f"Connected to server at {IP}:{PORT}") + except socket.error as e: + sys.exit(f"Connection failed: {e}") + + while True: + # Get user input + message = input().strip() + if message == "exit": + break + + # Send data to the server + try: + client_socket.send(message.encode()) + except socket.error as e: + sys.exit(f"Send failed: {e}") + + # Receive response from server + try: + data = client_socket.recv(BUFSIZ) + if not data: # Check for EOF + break + print(f"[Server {IP}:{PORT}]: {data.decode()}") + except socket.error as e: + sys.exit(f"Receive failed: {e}") + + # Close the socket + try: + client_socket.close() + except socket.error as e: + sys.exit(f"Close failed: {e}") + + +if __name__ == "__main__": + main() diff --git a/chapters/io/ipc/drills/tasks/client-server/solution/server.c b/chapters/io/ipc/drills/tasks/client-server/solution/server.c new file mode 100644 index 0000000000..e5fe85bdaf --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/solution/server.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" + +#ifndef BUFSIZ +#define BUFSIZ 256 +#endif + +static const char IP[] = "127.0.0.1"; +static const int PORT = 5000; + +/** + * Create a sockaddr_in structure with the given IP and port. + */ +struct sockaddr_in get_sockaddr(const char *ip, const int port) +{ + struct sockaddr_in addr; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + + return addr; +} + +/** + * Get user input from stdin and remove trailing newline. + */ +static void get_user_input(char *buf, size_t size) +{ + memset(buf, 0, size); + fgets(buf, size, stdin); + // Remove trailing newline + if (buf[strlen(buf) - 1] == '\n') + buf[strlen(buf) - 1] = '\0'; +} + +/** + * Create a TCP socket and listen for incoming connections. + */ +int main(void) +{ + char buf[BUFSIZ]; + struct sockaddr_in addr = get_sockaddr(IP, PORT); + struct sockaddr_in cli_addr; // Used to store client address + socklen_t cli_len = sizeof(cli_addr); + int listenfd, connectfd; + int rc; + + /* TODO 2: Create a network socket with SOCK_STREAM type in listenfd. */ + listenfd = socket(AF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + /* TODO 2: Bind the socket to the addr. */ + rc = bind(listenfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "bind"); + + /* TODO 2: Mark socket as passive socket using listen(). */ + rc = listen(listenfd, 1); + DIE(rc < 0, "listen"); + + /* TODO 2: Use accept() to accept a new connection in connectfd. */ + connectfd = accept(listenfd, (struct sockaddr *) &cli_addr, &cli_len); + DIE(connectfd < 0, "accept"); + + printf("[Client %s:%d] connected\n", inet_ntoa(cli_addr.sin_addr), ntohs(cli_addr.sin_port)); + + while (1) { + memset(buf, 0, BUFSIZ); + /* TODO 2: Receive data from the connected socket */ + rc = recv(connectfd, buf, BUFSIZ, 0); + DIE(rc < 0, "recv"); + + /* TODO 2: Stop the loop if recv() receives EOF */ + if (rc == 0) + break; + + char *cli_ip = inet_ntoa(cli_addr.sin_addr); + int cli_port = ntohs(cli_addr.sin_port); + + printf("[Client %s:%d]: %s\n", cli_ip, cli_port, buf); + + get_user_input(buf, BUFSIZ); + if (strcmp(buf, "exit") == 0) + break; + + /* TODO 2: Send the message to the client */ + rc = send(connectfd, buf, strlen(buf), 0); + DIE(rc < 0, "send"); + } + + /* TODO 4: Close connectfd and listenfd. */ + rc = close(connectfd); + DIE(rc < 0, "close"); + rc = close(listenfd); + DIE(rc < 0, "close"); + + return 0; +} diff --git a/chapters/io/ipc/drills/tasks/client-server/solution/server.py b/chapters/io/ipc/drills/tasks/client-server/solution/server.py new file mode 100644 index 0000000000..86edae6293 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/solution/server.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: BSD-3-Clause + +import socket +import sys + +# Constants +IP = "127.0.0.1" +PORT = 5000 +BUFSIZ = 256 + + +def main(): + # Create a TCP socket + try: + listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + except socket.error as e: + sys.exit(f"Socket creation failed: {e}") + + # Bind the socket to the address + try: + listen_socket.bind((IP, PORT)) + except socket.error as e: + sys.exit(f"Bind failed: {e}") + + # Mark socket as passive + try: + listen_socket.listen(1) + except socket.error as e: + sys.exit(f"Listen failed: {e}") + + print(f"Server listening on {IP}:{PORT}") + + # Accept a new connection + try: + connect_socket, client_addr = listen_socket.accept() + print(f"Connected by {client_addr}") + except socket.error as e: + sys.exit(f"Accept failed: {e}") + + while True: + # Receive data from the connected socket + try: + data = connect_socket.recv(BUFSIZ) + if not data: + break # EOF reached + print(f"[Client {client_addr}]: {data.decode()}") + except socket.error as e: + sys.exit(f"Receive failed: {e}") + + # Get user input and send response + message = input().strip() + if message == "exit": + break + + try: + connect_socket.send(message.encode()) + except socket.error as e: + sys.exit(f"Send failed: {e}") + + # Close sockets + try: + connect_socket.close() + listen_socket.close() + except socket.error as e: + sys.exit(f"Close failed: {e}") + + +if __name__ == "__main__": + main() diff --git a/chapters/io/ipc/drills/tasks/client-server/solution/utils/log/CPPLINT.cfg b/chapters/io/ipc/drills/tasks/client-server/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/ipc/drills/tasks/client-server/solution/utils/log/log.c b/chapters/io/ipc/drills/tasks/client-server/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/ipc/drills/tasks/client-server/solution/utils/log/log.h b/chapters/io/ipc/drills/tasks/client-server/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/ipc/drills/tasks/client-server/solution/utils/utils.h b/chapters/io/ipc/drills/tasks/client-server/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/ipc/drills/tasks/client-server/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/ipc/drills/tasks/named-pipes/.gitignore b/chapters/io/ipc/drills/tasks/named-pipes/.gitignore new file mode 100644 index 0000000000..296b2ccf64 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/.gitignore @@ -0,0 +1 @@ +support/ diff --git a/chapters/io/ipc/drills/tasks/named-pipes/Makefile b/chapters/io/ipc/drills/tasks/named-pipes/Makefile new file mode 100644 index 0000000000..ff78d246ea --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/Makefile @@ -0,0 +1,9 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + mkdir -p support/ + $(PYTHON) $(SCRIPT) --input ./solution --output ./support + +clean: + rm -rf support/ diff --git a/chapters/io/ipc/drills/tasks/named-pipes/README.md b/chapters/io/ipc/drills/tasks/named-pipes/README.md new file mode 100644 index 0000000000..18659cd20d --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/README.md @@ -0,0 +1,17 @@ +# Named Pipes Communication + +Navigate to `chapters/io/ipc/drills/tasks/named-pipes` and run `make` to generate the `support` directory. +In this exercise, you'll implement client-server communication between two processes using a named pipe, also called **FIFO**. +Both the sender and receiver are created from the same binary: run without arguments for a receiver, or with `-s` for a sender. + +1. Use the [`mkfifo()` syscall](https://man7.org/linux/man-pages/man3/mkfifo.3.html) to create a named pipe. + If the FIFO already exists, use [`access()`](https://man7.org/linux/man-pages/man2/access.2.html) to check its permissions. + If permissions are incorrect, or if it does not exist, recreate the FIFO. + +1. Complete the TODOs in `receiver_loop()` and `sender_loop()` to enable communication. + Ensure the FIFO is open before reading from or writing to it. + Close the FIFO when you are done. + + **Bonus**: Run two receivers and a single sender in different terminals. + You may notice some "strange" behavior due to how named pipes manage data with multiple readers. + For more on this, see [this Stack Overflow thread](https://stackoverflow.com/a/69325284). diff --git a/chapters/io/ipc/drills/tasks/named-pipes/generate_skels.py b/chapters/io/ipc/drills/tasks/named-pipes/generate_skels.py new file mode 100644 index 0000000000..697c9d5b61 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/generate_skels.py @@ -0,0 +1,151 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + or re.match(r".*\.py$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/chapters/io/ipc/drills/tasks/named-pipes/solution/.gitignore b/chapters/io/ipc/drills/tasks/named-pipes/solution/.gitignore new file mode 100644 index 0000000000..3f666a4fef --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/solution/.gitignore @@ -0,0 +1,2 @@ +named_pipe +my-fifo diff --git a/chapters/io/ipc/drills/tasks/named-pipes/solution/Makefile b/chapters/io/ipc/drills/tasks/named-pipes/solution/Makefile new file mode 100644 index 0000000000..3912beee51 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/solution/Makefile @@ -0,0 +1,38 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = named_pipe +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + -rm -f my-fifo + +.PHONY: all clean diff --git a/chapters/io/ipc/drills/tasks/named-pipes/solution/named_pipe.c b/chapters/io/ipc/drills/tasks/named-pipes/solution/named_pipe.c new file mode 100644 index 0000000000..fcac6907c1 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/solution/named_pipe.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include + +#include "utils/utils.h" + +static const char fifo_path[] = "my-fifo"; + +#ifndef BUFSIZ +#define BUFSIZ 256 +#endif + +void create_fifo_if_needed(void) +{ + /* TODO 9: Use access() to check if the FIFO exists and has the right permissions. + * If it exists but has the wrong permissions, delete it using unlink(). + * If it doesn't exist create it using mkfifo(). + */ + int rc = access(fifo_path, R_OK | W_OK); + + if (rc < 0) { + rc = unlink(fifo_path); + DIE(rc < 0, "unlink"); + + rc = mkfifo(fifo_path, 0666); + DIE(rc < 0, "mkfifo"); + } +} + +static void receiver_loop(void) +{ + char output[BUFSIZ]; + int fd, rc; + + /* TODO 4: Create FIFO if it does not exist, then open it for reading. */ + create_fifo_if_needed(); + + fd = open(fifo_path, O_RDONLY); + DIE(fd < 0, "open"); + + while (1) { + /* TODO 5: Read until EOF. */ + rc = read(fd, output, BUFSIZ); + DIE(rc < 0, "read"); + + if (rc == 0) + break; + + printf("[Receiver]: %s\n", output); + fflush(stdout); + } + + /* TODO 2: Close FIFO. */ + rc = close(fd); + DIE(rc < 0, "close"); +} + +static void sender_loop(void) +{ + char input[BUFSIZ]; + int fd, rc; + + /* TODO 4: Create FIFO if it does not exist, then open it for writing. */ + create_fifo_if_needed(); + + fd = open(fifo_path, O_WRONLY); + DIE(fd < 0, "open"); + + while (1) { + memset(input, 0, BUFSIZ); + fgets(input, BUFSIZ, stdin); + // Remove trailing newline + if (input[strlen(input) - 1] == '\n') + input[strlen(input) - 1] = '\0'; + + if ((strcmp(input, "exit") == 0 || strlen(input) == 0)) + break; + + /* TODO 2: Write to FIFO. */ + rc = write(fd, input, BUFSIZ); + DIE(rc < 0, "write"); + } + + /* TODO 2: Close FIFO. */ + rc = close(fd); + DIE(rc < 0, "close"); +} + +/** + * Simulate a sender-receiver communication using a named pipe. + * Run the program as a receiver by default, or as a sender if the -s or --sender. + */ +int main(int argc, char *argv[]) +{ + if (argc == 1) { // Run as receiver by default + receiver_loop(); + } else if (strcmp(argv[1], "-s") == 0 || strcmp(argv[1], "--sender") == 0) { + sender_loop(); + } else { + fprintf(stderr, "Usage: %s [-s|--sender]\n", argv[0]); + exit(EXIT_FAILURE); + } + + return 0; +} diff --git a/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/log/CPPLINT.cfg b/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/log/log.c b/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/log/log.h b/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/utils.h b/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/ipc/drills/tasks/named-pipes/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/ipc/drills/tasks/network-socket/.gitignore b/chapters/io/ipc/drills/tasks/network-socket/.gitignore new file mode 100644 index 0000000000..296b2ccf64 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/.gitignore @@ -0,0 +1 @@ +support/ diff --git a/chapters/io/ipc/drills/tasks/network-socket/Makefile b/chapters/io/ipc/drills/tasks/network-socket/Makefile new file mode 100644 index 0000000000..ff78d246ea --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/Makefile @@ -0,0 +1,9 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + mkdir -p support/ + $(PYTHON) $(SCRIPT) --input ./solution --output ./support + +clean: + rm -rf support/ diff --git a/chapters/io/ipc/drills/tasks/network-socket/README.md b/chapters/io/ipc/drills/tasks/network-socket/README.md new file mode 100644 index 0000000000..bce4e6633e --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/README.md @@ -0,0 +1,21 @@ +# Network Socket Communication + +Navigate to `chapters/io/ipc/drills/tasks/network-socket` and run `make` to generate the `support` directory. +In this exercise, you'll implement client-server communication between two processes using a **network socket**. +Both the sender and receiver are created from the same binary: run without arguments for a receiver, or with `-s` for a sender. + +1. Complete the TODOs in the `sender_loop()` from `tcp_socket.c`. + You need to verify whether the socket exists i.e. check if the receiver has created it. + Next, **create** your own socket and **connect** to the receiver's socket using its address (**Hint:** use `get_sockaddr(, )` to obtain it). + Once the connection is established, you can send messages using `send()`. + +1. Complete the TODOs in the `receiver_loop()` from `tcp_socket.c`. + Similarly, you will need to **create** a socket and **bind** it to the receiver's address (**Hint:** use `get_sockaddr(, )` for this). + Instead of connecting, you will **listen** for and **accept** incoming connections. + When `accept()` receives a connection request, it will return a new socket file descriptor that you can use to receive messages via `recv()`. + +1. Now we’ll implement the same functionality using datagrams (`SOCK_DGRAM`). + Open `udp_socket.c` and complete the TODOs for `sender_loop()` and `receiver_loop()` functions. + The workflow is similar, but `listen()`, `accept()`, and `connect()` are not required for datagram sockets. + +If you're having difficulties solving this exercise, go through [this reading material](../../../reading/network-sockets.md). diff --git a/chapters/io/ipc/drills/tasks/network-socket/generate_skels.py b/chapters/io/ipc/drills/tasks/network-socket/generate_skels.py new file mode 100644 index 0000000000..697c9d5b61 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/generate_skels.py @@ -0,0 +1,151 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + or re.match(r".*\.py$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/chapters/io/ipc/drills/tasks/network-socket/solution/.gitignore b/chapters/io/ipc/drills/tasks/network-socket/solution/.gitignore new file mode 100644 index 0000000000..1ed5c05bca --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/solution/.gitignore @@ -0,0 +1,2 @@ +tcp_socket +udp_socket diff --git a/chapters/io/ipc/drills/tasks/network-socket/solution/Makefile b/chapters/io/ipc/drills/tasks/network-socket/solution/Makefile new file mode 100644 index 0000000000..f5dec526c7 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/solution/Makefile @@ -0,0 +1,37 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = tcp_socket udp_socket +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + +.PHONY: all clean diff --git a/chapters/io/ipc/drills/tasks/network-socket/solution/tcp_socket.c b/chapters/io/ipc/drills/tasks/network-socket/solution/tcp_socket.c new file mode 100644 index 0000000000..28cc59d784 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/solution/tcp_socket.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" + +#ifndef BUFSIZ +#define BUFSIZ 256 +#endif + +static const char IP[] = "127.0.0.1"; +static const int PORT = 5000; + +/** + * Create a sockaddr_in structure with the given IP and port. + */ +struct sockaddr_in get_sockaddr(const char *ip, const int port) +{ + struct sockaddr_in addr; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + + return addr; +} + +static void receiver_loop(void) +{ + struct sockaddr_in addr = get_sockaddr(IP, PORT); + char output[BUFSIZ]; + int listenfd, connectfd; + int rc; + + /* TODO 2: Create a network socket with SOCK_STREAM type. */ + listenfd = socket(AF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + /* TODO 2: Bind the socket to the addr. */ + rc = bind(listenfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "bind"); + + /* TODO 2: Mark socket as passive socket using listen(). */ + rc = listen(listenfd, 1); + DIE(rc < 0, "listen"); + + /* TODO 2: Use accept() to get the socket for a new connection. */ + connectfd = accept(listenfd, NULL, NULL); + DIE(connectfd < 0, "accept"); + + while (1) { + memset(output, 0, BUFSIZ); + /* TODO 2: Receive data from the connected socket */ + rc = recv(connectfd, output, sizeof(output), 0); + DIE(rc < 0, "recv"); + + if (rc == 0) + break; + + printf("[Receiver]: %s\n", output); + fflush(stdout); + } + + /* TODO 4: Close connectfd and listenfd. */ + rc = close(connectfd); + DIE(rc < 0, "close"); + rc = close(listenfd); + DIE(rc < 0, "close"); +} + +static void sender_loop(void) +{ + struct sockaddr_in addr = get_sockaddr(IP, PORT); + char input[BUFSIZ]; + int sockfd; + int rc; + + /* TODO 2: Create a network socket with SOCK_STREAM type. */ + sockfd = socket(AF_INET, SOCK_STREAM, 0); + DIE(sockfd < 0, "socket"); + + /* TODO 2: Connect to the socket. */ + rc = connect(sockfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "connect"); + + while (1) { + memset(input, 0, BUFSIZ); + fgets(input, BUFSIZ, stdin); + // Remove trailing newline + if (input[strlen(input) - 1] == '\n') + input[strlen(input) - 1] = '\0'; + + printf("[Sender]: %s\n", input); + if ((strcmp(input, "exit") == 0 || strlen(input) == 0)) + break; + + /* TODO 2: Send input to socket. */ + rc = send(sockfd, input, strlen(input), 0); + DIE(rc < 0, "send"); + } + + /* TODO 2: Close socket. */ + rc = close(sockfd); + DIE(rc < 0, "close"); +} + +/** + * Simulate a sender-receiver communication using a named pipe. + * Run the program as a receiver by default, or as a sender if the -s or --sender. + */ +int main(int argc, char *argv[]) +{ + if (argc == 1) { // Run as receiver by default + receiver_loop(); + } else if (strcmp(argv[1], "-s") == 0 || strcmp(argv[1], "--sender") == 0) { + sender_loop(); + } else { + fprintf(stderr, "Usage: %s [-s|--sender]\n", argv[0]); + exit(EXIT_FAILURE); + } + + return 0; +} diff --git a/chapters/io/ipc/drills/tasks/network-socket/solution/udp_socket.c b/chapters/io/ipc/drills/tasks/network-socket/solution/udp_socket.c new file mode 100644 index 0000000000..14587c247a --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/solution/udp_socket.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" + +#ifndef BUFSIZ +#define BUFSIZ 256 +#endif + +static const char IP[] = "127.0.0.1"; +static const int PORT = 5000; + +/** + * Create a sockaddr_in structure with the given IP and port. + */ +struct sockaddr_in get_sockaddr(const char *ip, const int port) +{ + struct sockaddr_in addr; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + + return addr; +} + +static void receiver_loop(void) +{ + struct sockaddr_in addr = get_sockaddr(IP, PORT); + char output[BUFSIZ]; + int listenfd; + int rc; + + /* TODO 2: Create a network socket with SOCK_STREAM type. */ + listenfd = socket(AF_INET, SOCK_DGRAM, 0); + DIE(listenfd < 0, "socket"); + + /* TODO 2: Bind the socket to the addr. */ + rc = bind(listenfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "bind"); + + while (1) { + memset(output, 0, BUFSIZ); + + /* TODO 2: Receive data from the connected socket */ + rc = recvfrom(listenfd, output, BUFSIZ, 0, NULL, NULL); + DIE(rc < 0, "recv"); + + if (strcmp(output, "exit") == 0) + break; + + printf("[Receiver]: %s\n", output); + fflush(stdout); + } + + /* TODO 2: Close listenfd. */ + rc = close(listenfd); + DIE(rc < 0, "close"); +} + +static void sender_loop(void) +{ + struct sockaddr_in addr = get_sockaddr(IP, PORT); + char input[BUFSIZ]; + int sockfd; + int rc; + + /* TODO 2: Create a network socket with SOCK_STREAM type. */ + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + DIE(sockfd < 0, "socket"); + + while (1) { + memset(input, 0, BUFSIZ); + fgets(input, BUFSIZ, stdin); + // Remove trailing newline + if (input[strlen(input) - 1] == '\n') + input[strlen(input) - 1] = '\0'; + + printf("[Sender]: %s\n", input); + + /* TODO 2: Send input to socket. */ + rc = sendto(sockfd, input, strlen(input), 0, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "send"); + + if ((strcmp(input, "exit") == 0 || strlen(input) == 0)) + break; + } + + /* TODO 2: Close socket. */ + rc = close(sockfd); + DIE(rc < 0, "close"); +} + +/** + * Simulate a sender-receiver communication using a named pipe. + * Run the program as a receiver by default, or as a sender if the -s or --sender. + */ +int main(int argc, char *argv[]) +{ + if (argc == 1) { // Run as receiver by default + receiver_loop(); + } else if (strcmp(argv[1], "-s") == 0 || strcmp(argv[1], "--sender") == 0) { + sender_loop(); + } else { + fprintf(stderr, "Usage: %s [-s|--sender]\n", argv[0]); + exit(EXIT_FAILURE); + } + + return 0; +} diff --git a/chapters/io/ipc/drills/tasks/network-socket/solution/utils/log/CPPLINT.cfg b/chapters/io/ipc/drills/tasks/network-socket/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/ipc/drills/tasks/network-socket/solution/utils/log/log.c b/chapters/io/ipc/drills/tasks/network-socket/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/ipc/drills/tasks/network-socket/solution/utils/log/log.h b/chapters/io/ipc/drills/tasks/network-socket/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/ipc/drills/tasks/network-socket/solution/utils/utils.h b/chapters/io/ipc/drills/tasks/network-socket/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/ipc/drills/tasks/network-socket/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/ipc/drills/tasks/receive-challenges/README.md b/chapters/io/ipc/drills/tasks/receive-challenges/README.md new file mode 100644 index 0000000000..3b1137e7d1 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/receive-challenges/README.md @@ -0,0 +1,17 @@ +# Receive Challenges + +Navigate to `chapters/io/ipc/drills/tasks/receive-challenges` and run `make` to generate the `support` directory. +In this task, we will review all the IPC methods we have explored, including [anonymous pipes](../../../reading/pipes.md), [named pipes (FIFOs)](../../../reading/pipes.md), [UNIX sockets](../../../reading/unix-sockets.md), and [network sockets](../../../reading/network-sockets.md). +Each challenge involves building a communication channel using the specified IPC method. + +1. Complete the TODOs in `support/receive_pipe.c`, then compile and run the executable. + If the challenge is completed successfully, you should see the message `Flag is ...`. + +1. Complete the TODOs in `support/receive_fifo.c`, then compile and run the executable. + You will need to run the `send_fifo` executable while your process is running to reveal the flag. + +1. Complete the TODOs in `support/receive_unix_socket.c`, then compile and run the executable. + You will need to run the `send_unix_socket` executable while your process is running to reveal the flag. + +1. Complete the TODOs in `support/receive_net_dgram_socket.c`, then compile and run the executable. + You will need to run the `send_net_dgram_socket` executable while your process is running to reveal the flag. diff --git a/content/chapters/io/lab/support/receive-challenges/.gitignore b/chapters/io/ipc/drills/tasks/receive-challenges/support/.gitignore similarity index 100% rename from content/chapters/io/lab/support/receive-challenges/.gitignore rename to chapters/io/ipc/drills/tasks/receive-challenges/support/.gitignore diff --git a/chapters/io/ipc/drills/tasks/receive-challenges/support/Makefile b/chapters/io/ipc/drills/tasks/receive-challenges/support/Makefile new file mode 100644 index 0000000000..f73f2d585a --- /dev/null +++ b/chapters/io/ipc/drills/tasks/receive-challenges/support/Makefile @@ -0,0 +1,37 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = receive_fifo receive_net_dgram_socket receive_pipe receive_unix_socket send_fd_4 send_fifo send_net_dgram_socket send_unix_socket +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + +.PHONY: all clean diff --git a/content/chapters/io/lab/support/receive-challenges/receive_fifo.c b/chapters/io/ipc/drills/tasks/receive-challenges/support/receive_fifo.c similarity index 100% rename from content/chapters/io/lab/support/receive-challenges/receive_fifo.c rename to chapters/io/ipc/drills/tasks/receive-challenges/support/receive_fifo.c diff --git a/content/chapters/io/lab/support/receive-challenges/receive_net_dgram_socket.c b/chapters/io/ipc/drills/tasks/receive-challenges/support/receive_net_dgram_socket.c similarity index 89% rename from content/chapters/io/lab/support/receive-challenges/receive_net_dgram_socket.c rename to chapters/io/ipc/drills/tasks/receive-challenges/support/receive_net_dgram_socket.c index de929af196..63944831fc 100644 --- a/content/chapters/io/lab/support/receive-challenges/receive_net_dgram_socket.c +++ b/chapters/io/ipc/drills/tasks/receive-challenges/support/receive_net_dgram_socket.c @@ -31,7 +31,7 @@ int main(void) /* TODO: Read flag from socket. */ - + sendto(fd, buffer, BUFSIZ, 0, (struct sockaddr *)&raddr, raddrlen); printf("Flag is: %s\n", buffer); diff --git a/content/chapters/io/lab/support/receive-challenges/receive_pipe.c b/chapters/io/ipc/drills/tasks/receive-challenges/support/receive_pipe.c similarity index 100% rename from content/chapters/io/lab/support/receive-challenges/receive_pipe.c rename to chapters/io/ipc/drills/tasks/receive-challenges/support/receive_pipe.c diff --git a/content/chapters/io/lab/support/receive-challenges/receive_unix_socket.c b/chapters/io/ipc/drills/tasks/receive-challenges/support/receive_unix_socket.c similarity index 100% rename from content/chapters/io/lab/support/receive-challenges/receive_unix_socket.c rename to chapters/io/ipc/drills/tasks/receive-challenges/support/receive_unix_socket.c diff --git a/content/chapters/io/lab/support/receive-challenges/send_fd_4.c b/chapters/io/ipc/drills/tasks/receive-challenges/support/send_fd_4.c similarity index 100% rename from content/chapters/io/lab/support/receive-challenges/send_fd_4.c rename to chapters/io/ipc/drills/tasks/receive-challenges/support/send_fd_4.c diff --git a/content/chapters/io/lab/support/receive-challenges/send_fifo.c b/chapters/io/ipc/drills/tasks/receive-challenges/support/send_fifo.c similarity index 100% rename from content/chapters/io/lab/support/receive-challenges/send_fifo.c rename to chapters/io/ipc/drills/tasks/receive-challenges/support/send_fifo.c diff --git a/content/chapters/io/lab/support/receive-challenges/send_net_dgram_socket.c b/chapters/io/ipc/drills/tasks/receive-challenges/support/send_net_dgram_socket.c similarity index 100% rename from content/chapters/io/lab/support/receive-challenges/send_net_dgram_socket.c rename to chapters/io/ipc/drills/tasks/receive-challenges/support/send_net_dgram_socket.c diff --git a/content/chapters/io/lab/support/receive-challenges/send_unix_socket.c b/chapters/io/ipc/drills/tasks/receive-challenges/support/send_unix_socket.c similarity index 100% rename from content/chapters/io/lab/support/receive-challenges/send_unix_socket.c rename to chapters/io/ipc/drills/tasks/receive-challenges/support/send_unix_socket.c diff --git a/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/log/CPPLINT.cfg b/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/log/log.c b/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/log/log.h b/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/utils.h b/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/ipc/drills/tasks/receive-challenges/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/ipc/drills/tasks/unix-socket/.gitignore b/chapters/io/ipc/drills/tasks/unix-socket/.gitignore new file mode 100644 index 0000000000..296b2ccf64 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/.gitignore @@ -0,0 +1 @@ +support/ diff --git a/chapters/io/ipc/drills/tasks/unix-socket/Makefile b/chapters/io/ipc/drills/tasks/unix-socket/Makefile new file mode 100644 index 0000000000..ff78d246ea --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/Makefile @@ -0,0 +1,9 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + mkdir -p support/ + $(PYTHON) $(SCRIPT) --input ./solution --output ./support + +clean: + rm -rf support/ diff --git a/chapters/io/ipc/drills/tasks/unix-socket/README.md b/chapters/io/ipc/drills/tasks/unix-socket/README.md new file mode 100644 index 0000000000..c7b67d6b52 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/README.md @@ -0,0 +1,17 @@ +# UNIX Socket Communication + +Navigate to `chapters/io/ipc/drills/tasks/unix-socket` and run `make` to generate the `support` directory. +In this exercise, you'll implement client-server communication between two processes using a **UNIX socket**. +Both the sender and receiver are created from the same binary: run without arguments for a receiver, or with `-s` for a sender. + +1. Complete the TODOs in the `sender_loop()`. + You need to verify whether the socket exists i.e. check if the receiver has created it. + Next, **create** your own socket and **connect** to the receiver's socket using its address (**Hint:** use `get_sockaddr(` to obtain it). + Once the connection is established, you can send messages using `send()`. + +1. Complete the TODOs in the `receiver_loop()`. + Similarly, you will need to **create** a socket and **bind** it to the receiver's address (**Hint:** use `get_sockaddr(` for this). + Instead of connecting, you will **listen** for and **accept** incoming connections. + When `accept()` receives a connection request, it will return a new socket file descriptor that you can use to receive messages via `recv()`. + +If you're having difficulties solving this exercise, go through [this reading material](../../../reading/unix-sockets.md). diff --git a/chapters/io/ipc/drills/tasks/unix-socket/generate_skels.py b/chapters/io/ipc/drills/tasks/unix-socket/generate_skels.py new file mode 100644 index 0000000000..697c9d5b61 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/generate_skels.py @@ -0,0 +1,151 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + or re.match(r".*\.py$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/chapters/io/ipc/drills/tasks/unix-socket/solution/.gitignore b/chapters/io/ipc/drills/tasks/unix-socket/solution/.gitignore new file mode 100644 index 0000000000..03ac430b8b --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/solution/.gitignore @@ -0,0 +1,2 @@ +unix_socket +my-socket diff --git a/chapters/io/ipc/drills/tasks/unix-socket/solution/Makefile b/chapters/io/ipc/drills/tasks/unix-socket/solution/Makefile new file mode 100644 index 0000000000..95078566df --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/solution/Makefile @@ -0,0 +1,38 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = unix_socket +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + -rm -f my-socket + +.PHONY: all clean diff --git a/chapters/io/ipc/drills/tasks/unix-socket/solution/unix_socket.c b/chapters/io/ipc/drills/tasks/unix-socket/solution/unix_socket.c new file mode 100644 index 0000000000..4879c8e54f --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/solution/unix_socket.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" + +#ifndef BUFSIZ +#define BUFSIZ 256 +#endif + +static const char socket_path[] = "my-socket"; + +/** + * Create a sockaddr_un structure with family AF_UNIX and the given path. + */ +struct sockaddr_un get_sockaddr(const char *path) +{ + struct sockaddr_un addr; + + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + snprintf(addr.sun_path, sizeof(path), "%s", path); + + return addr; +} + +static void receiver_loop(void) +{ + struct sockaddr_un addr = get_sockaddr(socket_path); + char output[BUFSIZ]; + int listenfd, connectfd; + int rc; + + /* Recreate the socket for each communication. */ + remove(socket_path); + + /* TODO 2: Create a UNIX socket with SOCK_STREAM type. */ + listenfd = socket(PF_UNIX, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + /* TODO 2: Bind the socket to the addr. */ + rc = bind(listenfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "bind"); + + /* TODO 2: Mark socket as passive socket using listen(). */ + rc = listen(listenfd, 1); + DIE(rc < 0, "listen"); + + /* TODO 2: Use accept() to get the socket for a new connection. */ + connectfd = accept(listenfd, NULL, NULL); + DIE(connectfd < 0, "accept"); + + while (1) { + memset(output, 0, BUFSIZ); + /* TODO 2: Receive data from the connected socket */ + rc = recv(connectfd, output, sizeof(output), 0); + DIE(rc < 0, "recv"); + + if (rc == 0) + break; + + printf("[Receiver]: %s\n", output); + fflush(stdout); + } + + /* TODO 4: Close connectfd and listenfd. */ + rc = close(connectfd); + DIE(rc < 0, "close"); + rc = close(listenfd); + DIE(rc < 0, "close"); +} + +static void sender_loop(void) +{ + struct sockaddr_un addr = get_sockaddr(socket_path); + char input[BUFSIZ]; + int sockfd; + int rc; + + /* TODO 2: Check if socket exists. Hint: access(). */ + rc = access(socket_path, R_OK | W_OK); + DIE(rc < 0, "access"); + + /* TODO 2: Create a UNIX socket with SOCK_STREAM type. */ + sockfd = socket(PF_UNIX, SOCK_STREAM, 0); + DIE(sockfd < 0, "socket"); + + /* TODO 2: Connect to the socket. */ + rc = connect(sockfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "connect"); + + while (1) { + memset(input, 0, BUFSIZ); + fgets(input, BUFSIZ, stdin); + // Remove trailing newline + if (input[strlen(input) - 1] == '\n') + input[strlen(input) - 1] = '\0'; + + printf("[Sender]: %s\n", input); + if ((strcmp(input, "exit") == 0 || strlen(input) == 0)) + break; + + /* TODO 2: Send input to socket. */ + rc = send(sockfd, input, strlen(input), 0); + DIE(rc < 0, "send"); + } + + /* TODO 2: Close socket. */ + rc = close(sockfd); + DIE(rc < 0, "close"); +} + +/** + * Simulate a sender-receiver communication using a named pipe. + * Run the program as a receiver by default, or as a sender if the -s or --sender. + */ +int main(int argc, char *argv[]) +{ + if (argc == 1) { // Run as receiver by default + receiver_loop(); + } else if (strcmp(argv[1], "-s") == 0 || strcmp(argv[1], "--sender") == 0) { + sender_loop(); + } else { + fprintf(stderr, "Usage: %s [-s|--sender]\n", argv[0]); + exit(EXIT_FAILURE); + } + + return 0; +} diff --git a/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/log/CPPLINT.cfg b/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/log/log.c b/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/log/log.h b/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/utils.h b/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/ipc/drills/tasks/unix-socket/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/ipc/guides/networking-101/README.md b/chapters/io/ipc/guides/networking-101/README.md new file mode 100644 index 0000000000..8f990d4e78 --- /dev/null +++ b/chapters/io/ipc/guides/networking-101/README.md @@ -0,0 +1,67 @@ +# Networking 101 + +In this section, we will **briefly** explore how networking works in general, from the perspective of the application. +Understanding the details of it, however, is beyond the scope of this course. + +The main **protocols** used by applications are **User Datagram Protocol (UDP)** and **Transmission Control Protocol (TCP)**. + +UDP is the simpler of the two protocols. +It simply sends data to a receiver identified by an IP and port. +It does not care whether the receiver has got all the data, whether it was corrupted or dropped altogether by some router along the way. + +At first glance, UDP might seem useless due to its lack of reliability checks. +However, this simplicity makes UDP **fast**. +As a result, it is often used for **real-time services**, such as video streaming or voice calls, where minor data losses (like dropped frames) are less problematic because they are quickly replaced by new data, masking any errors. + +On the other hand, TCP offers **reliability**. +It ensures that data is received correctly by performing error checks and retransmitting any lost or corrupted packets. +This makes TCP ideal for applications that require guaranteed delivery, such as **web browsing** or **file transfers**, where accuracy and completeness are critical. + +## Local TCP and UDP Services + +To get a full list of all network-handling processes in your system together with the protocols they're using, we can use the `netstat` with the `-tuanp` arguments. +`-tuanp` is short for `-t -u -a -n -p`, which stand for: + +- `-t`: list processes using the TCP protocol +- `-u`: list processes using the UDP protocol +- `-a`: list both servers and clients +- `-n`: list IPs in numeric format +- `-p`: show the PID and name of each program + +```console +student@os:~$ sudo netstat -tunp +Active Internet connections (w/o servers) +Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name +tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 1057/sshd: /usr/sbi +tcp 0 0 127.0.0.1:6463 0.0.0.0:* LISTEN 3261/Discord --type +tcp 0 0 192.168.100.2:51738 162.159.128.235:443 ESTABLISHED 3110/Discord --type +tcp 0 0 192.168.100.2:43694 162.159.129.235:443 ESTABLISHED 3110/Discord --type +tcp 0 0 192.168.100.2:56230 54.230.159.113:443 ESTABLISHED 9154/firefox +tcp 0 0 192.168.100.2:38096 34.107.141.31:443 ESTABLISHED 9154/firefox +tcp 0 0 192.168.100.2:42462 34.117.237.239:443 ESTABLISHED 9154/firefox +tcp 0 0 192.168.100.2:41128 162.159.135.234:443 ESTABLISHED 3110/Discord --type +tcp6 0 0 :::80 :::* LISTEN 1114/apache2 +tcp6 0 0 :::22 :::* LISTEN 1057/sshd: /usr/sbi +tcp6 0 0 2a02:2f0a:c10f:97:55754 2a02:2f0c:dff0:b::1:443 ESTABLISHED 9154/firefox +tcp6 0 0 2a02:2f0a:c10f:97:55750 2a02:2f0c:dff0:b::1:443 ESTABLISHED 9154/firefox +udp 0 0 0.0.0.0:56585 0.0.0.0:* 3261/Discord --type +udp 0 0 0.0.0.0:42629 0.0.0.0:* 3261/Discord --type +udp6 0 0 :::52070 :::* 9154/firefox +udp6 0 0 :::38542 :::* 9154/firefox +``` + +Your output will likely differ from the example above. +Let’s focus on the fourth column, which displays the local address and port in use by each process. +The first 1024 ports are reserved for well-known applications, ensuring consistency across networks. +For example, `SSH` uses port `22` and `Apache2` uses port `80` for both IPv4 and IPv6 addresses (look for rows starting with `tcp` for IPv4 and `tcp6` for IPv6). + +Some user programs, like Firefox, establish multiple connections, often using both IPv4 and IPv6, with each connection assigned a unique port. +Discord is another example, using TCP to handle text messages, images, videos, and other static content, while relying on UDP for real-time voice and video data during calls. + +[Quiz: Why does Firefox uses both TCP and UDP?](../../drills/questions/firefox-tcp-udp.md) + +## Conclusion + +The difference between TCP and UDP can be summarised as follows: + +![TCP vs UDP](../../media/tcp-udp-simplified.png) diff --git a/chapters/io/ipc/guides/redirections/README.md b/chapters/io/ipc/guides/redirections/README.md new file mode 100644 index 0000000000..18357d4b7b --- /dev/null +++ b/chapters/io/ipc/guides/redirections/README.md @@ -0,0 +1,49 @@ +# Redirections + +In the [File Descriptors section](../../../file-descriptors/reading/file-descriptors.md), we mentioned redirections such as `echo "OS Rullz!" > newfile.txt`. +We said `file.txt` has to be opened at some point. +Let’s explore the relevant system calls (`open()`, `openat()`) to see this in action: + +```console +student@os:~/.../guides/redirections$ strace -e trace=open,openat,execve,dup2 -f sh -c "ls > file.txt" +execve("/usr/bin/sh", ["sh", "-c", "ls > file.txt"], 0x7fffe1383e78 /* 36 vars */) = 0 +openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 +openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 +openat(AT_FDCWD, "file.txt", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 3 +dup2(3, 1) = 1 +strace: Process 77547 attached +[pid 77547] execve("/usr/bin/ls", ["ls"], 0x55ebb9b2dbf8 /* 36 vars */) = 0 +[...] +``` + +Notice that we used `sh -c` to run `ls > file.txt`. +Running `strace -e trace=open,openat,execve,dup2 -f ls > file.txt` would instead redirect the `strace` output to `file.txt`, hiding any system calls related to `file.txt`. +This happens because, as we discussed earlier, redirection is transparent for the process being redirected. +The process still writes to its `stdout`, but `stdout` itself is now directed to the specified file. + +Remember how processes are created using `fork()` and `exec()`, as shown in this diagram: + +![Launching a new command in Bash](../../media/fork-exec.svg) + +In our case, the main process is `sh -c "ls > file.txt"`. +In the `strace` output, we see it opens `file.txt` on file descriptor `3`, then uses [`dup2(3, 1)`](https://man7.org/linux/man-pages/man2/dup.2.html) to redirect file descriptor `1` to the same **open file structure**. +It then **forks** a child process and calls `execve()`. + +`execve` replaces the virtual address space (VAS) of the current process but retains the [file descriptor table](../../../file-descriptors/guides/fd-table/README.md). +This preserve the `stdout` of the parent process, thus the redirection to `file.txt` remains effective in the new process as well. + +## `dup()/dup2()` - Atomic IO + +If you're not familiar with the [`dup()` syscall](https://man7.org/linux/man-pages/man2/dup.2.html), it essentially creates a new file descriptor pointing to an existing **open file structure**. +Unlike `open()`, as discussed in the [file descriptor table guide](../../../file-descriptors/guides/fd-table/README.md), `dup()` doesn’t create a fresh open file structure. + +The `dup2(old_fd, new_fd)` variant closes `new_fd` before making it point to the same open file structure as `old_fd`. +While this might seem like a combination of `close(new_fd)` and `open(old_fd)`, `dup2()` is actually atomic, which prevents race conditions. + +To see why atomicity matters, review the code in `support/redirect_parallel.c`, compile it, and run it. + +You’ll find that `redirect_stderr_file.txt` contains `Message for STDOUT`, and `redirect_stdout_file.txt` contains `Message for STDERR`. +Investigate the code to understand where the race condition occurred. + +While a `mutex` around the `close()` and `open()` sequence could fix this, it can make the code cumbersome. +Instead, follow the `FIXME` comments for a more elegant solution using `dup2()`. diff --git a/content/chapters/io/lab/solution/redirect/.gitignore b/chapters/io/ipc/guides/redirections/support/.gitignore similarity index 66% rename from content/chapters/io/lab/solution/redirect/.gitignore rename to chapters/io/ipc/guides/redirections/support/.gitignore index d0f83571d4..239ff6d11e 100644 --- a/content/chapters/io/lab/solution/redirect/.gitignore +++ b/chapters/io/ipc/guides/redirections/support/.gitignore @@ -1,2 +1,2 @@ -redirect redirect_parallel +*.txt diff --git a/chapters/io/ipc/guides/redirections/support/Makefile b/chapters/io/ipc/guides/redirections/support/Makefile new file mode 100644 index 0000000000..400de1730c --- /dev/null +++ b/chapters/io/ipc/guides/redirections/support/Makefile @@ -0,0 +1,38 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = redirect_parallel +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + -rm -f *.txt + +.PHONY: all clean diff --git a/content/chapters/io/lab/support/redirect/redirect_parallel.c b/chapters/io/ipc/guides/redirections/support/redirect_parallel.c similarity index 94% rename from content/chapters/io/lab/support/redirect/redirect_parallel.c rename to chapters/io/ipc/guides/redirections/support/redirect_parallel.c index 32bfe8a820..34ccc16146 100644 --- a/content/chapters/io/lab/support/redirect/redirect_parallel.c +++ b/chapters/io/ipc/guides/redirections/support/redirect_parallel.c @@ -38,7 +38,7 @@ static void do_stdout_redirect() /* Make sure both threads have opened their files. */ pthread_barrier_wait(&barrier); - /* TODO: Use `dup2()` instead of `close()` and `dup()`. */ + /* FIXME: Use `dup2()` instead of `close()` and `dup()`. */ rc = close(STDOUT_FILENO); DIE(rc < 0, "close"); @@ -65,7 +65,7 @@ static void do_stderr_redirect() sleep(1); - /* TODO: Use `dup2()` instead of `close()` and `dup()`. */ + /* FIXME: Use `dup2()` instead of `close()` and `dup()`. */ rc = close(STDERR_FILENO); DIE(rc < 0, "close"); diff --git a/chapters/io/ipc/guides/redirections/support/utils/log/CPPLINT.cfg b/chapters/io/ipc/guides/redirections/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/ipc/guides/redirections/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/ipc/guides/redirections/support/utils/log/log.c b/chapters/io/ipc/guides/redirections/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/ipc/guides/redirections/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/ipc/guides/redirections/support/utils/log/log.h b/chapters/io/ipc/guides/redirections/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/ipc/guides/redirections/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/ipc/guides/redirections/support/utils/utils.h b/chapters/io/ipc/guides/redirections/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/ipc/guides/redirections/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/content/chapters/io/lab/media/fork-exec.svg b/chapters/io/ipc/media/fork-exec.svg similarity index 100% rename from content/chapters/io/lab/media/fork-exec.svg rename to chapters/io/ipc/media/fork-exec.svg diff --git a/content/chapters/io/lab/media/piped-commands.svg b/chapters/io/ipc/media/piped-commands.svg similarity index 100% rename from content/chapters/io/lab/media/piped-commands.svg rename to chapters/io/ipc/media/piped-commands.svg diff --git a/content/chapters/io/lab/media/tcp-udp-simplified.png b/chapters/io/ipc/media/tcp-udp-simplified.png similarity index 100% rename from content/chapters/io/lab/media/tcp-udp-simplified.png rename to chapters/io/ipc/media/tcp-udp-simplified.png diff --git a/chapters/io/ipc/reading/client-server-model.md b/chapters/io/ipc/reading/client-server-model.md new file mode 100644 index 0000000000..a6fd9b838b --- /dev/null +++ b/chapters/io/ipc/reading/client-server-model.md @@ -0,0 +1,66 @@ +# Client-Server Model + +In our previous IPC examples, we used the terms **sender** and **receiver**. +In practice, these are commonly referred to as the **client** (sender) and **server** (receiver). +While the socket API provides the necessary tools for communication, it doesn’t define an actual communication protocol. +This is where an **application protocol** comes in (distinct from transport protocols like `UDP` and `TCP`). +An application protocol defines the rules and structure for how the communication should take place. +For example, in the [Network Sockets Communication task](../drills/tasks/network-socket/README.md), the **server** stops upon receiving the `exit` string from the **client**. + +It’s important to keep in mind how the networking protocol impacts the design of each component: + +- With **UDP** (`SOCK_DGRAM`), there is no active connection. + The server simply waits for incoming messages and handles them as they arrive. + Unlike TCP, UDP **does not guarantee message delivery** and messages may be lost during transmission. + It is up to the **application** to manage these concerns. + For example, a client might resend a request if it does not receive a response within 3 seconds. +- With **TCP** (`SOCK_STREAM`), a **connection is created and maintained** between the client and server. + TCP guarantees that messages arrive in the correct order and will automatically resend data if network issues occur. + +For a dive into how TCP and UDP are used in real-world scenarios checkout the [Networking 101 guide](../guides/networking-101/README.md). + +## Client-Server UDP + +Setting up a UDP client-server communication is straightforward and lightweight. +A typical workflow for a UDP **server** involves: + +- `socket(AF_INET, SOCK_DGRAM, 0)` - creating a UDP network socket. +- `bind(sockfd, &addr, sizeof(addr))` - binding the socket to an address with an `IP` and `port` for network sockets. +- `recvfrom(sockfd, buffer, BUFSIZ, 0, &caddr, &caddrlen);` - waiting for a message from the client. + **Note:** The last two parameters are used to retrieve the client's address information. + +The server requires `bind()` to assign a specific IP and port for listening, so clients know exactly where to connect. +For network clients, `bind()` is optional; +if the IP and port are not specified, they are automatically assigned. + +The typical workflow for a UDP **client** comprises of the following steps: + +- `socket(AF_INET, SOCK_DGRAM, 0)` - creating a UDP network socket +- `sendto(fd, buffer, BUFSIZ, 0, (struct sockaddr *)&svaddr, svaddrlen);` - sending a message to the server. + +## Client-Server TCP + +Setting up a TCP client-server communication involves a few more steps than UDP but remains relatively straightforward. +A typical workflow for a TCP **server** is as follows: + +- `socket(AF_INET, SOCK_STREAM, 0)` - creating a TCP network socket. +- `bind(sockfd, &addr, sizeof(addr))` - binding the socket to an address with an `IP` and `port` for network sockets. +- `listen(sockfd, backlog)` - marking the socket as passive, ready to accept incoming connections. + The `backlog` defines the maximum number of pending connections. + This is usually set to the maximum number of clients you are expecting. +- `accept(sockfd, &client_addr, &client_len)` - accepting a new connection from a client and returning a new socket descriptor for communication. + Keep in mind that the server will block until a connection arrives. +- Once the connection is accepted, you can communicate with the client using `send(sockfd, buffer, BUFSIZ, 0)` and `recv(sockfd, buffer, BUFSIZ, 0)`. + +**Note:** The server requires `bind()` to specify a particular IP and port for listening. +This way, clients can connect to the correct address. +After binding, the server uses `listen()` to prepare for incoming connections and `accept()` to handle them. + +On the **client** side, the typical workflow is: + +- `socket(AF_INET, SOCK_STREAM, 0)` - creating a TCP network socket (also works for Unix sockets). +- `connect(sockfd, (struct sockaddr *)&svaddr, svaddrlen)` - connecting to the server using the server's `IP` and `port`. + Unlike UDP, `connect()` is required to establish a connection to the server. +- Once connected, you can communicate with the server using `send(sockfd, buffer, BUFSIZ, 0)` and `recv(sockfd, buffer, BUFSIZ, 0)`. + +Test your understanding by building a [sequential client-server communication](../drills/tasks/client-server/README.md). diff --git a/chapters/io/ipc/reading/network-sockets.md b/chapters/io/ipc/reading/network-sockets.md new file mode 100644 index 0000000000..31aaa66d87 --- /dev/null +++ b/chapters/io/ipc/reading/network-sockets.md @@ -0,0 +1,51 @@ +# Network Sockets + +Network sockets are an inter-process communication (IPC) method that enables communication between processes on different hosts. +They are managed through the [Berkeley Sockets API](https://en.wikipedia.org/wiki/Berkeley_sockets), which is widely supported across various operating systems and programming languages. +The API is further explored in the [Unix sockets section](../reading/unix-sockets.md). +This section focuses on identifying peer processes and establishing connections between them. + +## Addresses and Ports + +The most crucial aspect of remote I/O is identifying the correct endpoint for communication. +Whether you are connecting to a website or a Minecraft server, you need a reliable way to specify the application you want to interact with. +This is where IP addresses and ports come into play. + +An **IP address** is a unique numerical label assigned to each device on a network. +It functions like a mailing address, ensuring that data packets reach the correct destination. +For example, if you want to access a website, your browser connects to the server's IP address, so that the server knows where to send the requested data. +But what if there is more than one server running at that IP address? + +This is the reason we need ports. +A port is simply a **number** that **uniquely identifies a connection** on a device. +When an application performs remote I/O, it requests a port from the operating system and begins listening for incoming data on that port. +However, how do clients know which port the application is using? +Typically, this information is transmitted by user or established by convention. +For instance, popular applications have predefined ports: `SSH` uses port `22`, `HTTP` operates on port `80`, and `HTTPS` defaults to port `443`. + +**Note**: In most cases, you don’t interact with IP addresses and ports directly. +For example, when you access `https://cs-pub-ro.github.io/operating-systems/`, you don’t see any numbers. +Behind the scenes, the [DNS](https://www.ibm.com/topics/dns) translates the domain name cs-pub-ro.github.io to its corresponding IP address, while the HTTPS protocol automatically uses port 443 by default. + +**Note:** You can use network sockets for communication between local processes. +Each host provides a `localhost` address (commonly `127.0.0.1`) or a loopback address that can be used for this purpose. + +Let's take a coding example to see how addresses and ports are used to identify a process over the network: + +```c +#define PORT 12345 +struct sockaddr_in addr; // Structure to hold the address for the network socket. + +memset(&addr, 0, sizeof(addr)); // Clear the address structure. +addr.sin_family = AF_INET; // Set the address family to IPv4. +server_addr.sin_addr.s_addr = inet_addr("127.0.0.1"); // Set the server's IP address (localhost). +addr.sin_port = htons(PORT); // Set the port number. + +int sockfd = socket(AF_INET, SOCK_STREAM, 0); // Create the socket. +if (sockfd < 0) {...} // handle error + +int rc = bind(sockfd, (struct sockaddr *) &addr, sizeof(addr)); // Bind the socket. +if (rc < 0) {...} // handle error +``` + +You can practice working with network sockets by completing the [Network Sockets Communication task](../drills/tasks/network-socket/README.md). diff --git a/chapters/io/ipc/reading/pipes.md b/chapters/io/ipc/reading/pipes.md new file mode 100644 index 0000000000..a9ab378fbe --- /dev/null +++ b/chapters/io/ipc/reading/pipes.md @@ -0,0 +1,84 @@ +# Pipes + +## Anonymous Pipes + +In this session, we'll explore a new mean of Inter-Process Communication (IPC), namely **the pipes**. +Pipes are by no means something new, and you most probably played with them already in bash: + +```bash +cat 'log_*.csv' | tr -s ' ' | cut -d ',' -f 2 | sort -u | head -n 10 +``` + +Using pipes (denoted as `|` in the above example) enables linking the `stdout` and `stdin` of multiple processes. +The `stdout` of `cat` is the `stdin` of `tr`, whose `stdout` is the `stdin` of `cut` and so on. +This "chain" of commands looks like this: + +![Piped Commands](../media/piped-commands.svg) + +So here we have a **unidirectional** stream of data that starts from `cat`, is modified by each new command, and then is passed to the next one. +We can tell from the image above that the communication channel between any 2 adjacent commands allows one process to write to it while the other reads from it. +For example, there is no need for `cat` to read any of `tr`'s output, only vice versa. + +In UNIX, the need for such a channel is fulfilled by the [`pipe()` syscall](https://man7.org/linux/man-pages/man2/pipe.2.html). +Imagine there's a literal pipe between any 2 adjacent commands in the image above, where data is what flows through this pipe **in only a single way**. + +Such pipes are known as **anonymous pipes** because they don’t have identifiers. +They are created by a parent process, which shares them with its children. +Data written to an anonymous pipe is stored in a kernel-managed circular buffer, where it’s available for related-processes to read. + +The following example showcases a typical workflow with anonymous pipes in Unix: + +```c +#define EXIT_ON_COND(cond) do { if (cond) exit(EXIT_FAILURE); } while (0) + +// pipe_fd[0] -> for reading +// pipe_fd[1] -> for writing +int pipe_fd[2]; + +EXIT_ON_COND(pipe(pipe_fd) < 0); // Create the pipe + +int pid = fork(); // Fork to create a child process +EXIT_ON_COND(pid < 0); // Check for fork() failure + +if (pid == 0) { // Child process + EXIT_ON_COND(close(pipe_fd[0]) != 0); // Close the read end + EXIT_ON_COND(write(pipe_fd[1], "hi", 2) < 0); // Write "hi" to the pipe + EXIT_ON_COND(close(pipe_fd[1]) != 0); // Close the write end +} else { // Parent process + char buf[BUFSIZ]; + + EXIT_ON_COND(close(pipe_fd[1]) != 0); // Close the write end + ssize_t n = read(pipe_fd[0], buf, sizeof(buf)); // Read data from the pipe into buf + EXIT_ON_COND(n < 0); // Check for read() failure + + buf[n] = '\0'; // Null-terminate the string + printf("Received: %s\n", buf); // Output the received message +} +``` + +In summary, the process creates the pipe and then calls `fork()` to create a child process. +By default, the file descriptors created by `pipe()` are shared with the child because the *(file descriptor table)* is copied upon creation. +To better understand how this works, please refer to [this guide on the File Descriptor Table (FDT)](../../file-descriptors/guides/fd-table/README.md). + +You can test your understanding of anonymous pipes by completing the [Anonymous Pipes Communication task](../drills/tasks/anon-pipes/README.md). + +[Check your understanding by identifying the limitations of anonymous pipes](../drills/questions/anonymous-pipes-limitation.md) + +## Named Pipes (FIFOs) + +As we discussed, anonymous pipes are named so because they lack identifiers. +**Named pipes** address this limitation by creating a *special* file on disk that serves as an identifier for the pipe. + +You might think that interacting with a file would result in a performance loss compared to anonymous pipes, but this is not the case. +The FIFO file acts merely as **a handler** within the filesystem, which is used to write data to a buffer inside the kernel. +This buffer is responsible for holding the data that is passed between processes, not the filesystem itself. + +Keep in mind that reading from and writing to a FIFO is not the same as interacting with a regular file - `read()` will block if the pipe is empty and will return `EOF` when the peer closes the pipe. + +You can practice working with named pipes by completing the [Named Pipes Communication task](../drills/tasks/named-pipes/README.md). + +## Redirections + +Although not directly related, redirections (e.g., `ls > file.txt`) operate similarly to pipes. +A process creates a new file descriptor, updates its `stdout`, and then creates the child process. +You can explore the similarities with pipes further in [this guide on redirections](../guides/redirections/README.md). diff --git a/chapters/io/ipc/reading/unix-sockets.md b/chapters/io/ipc/reading/unix-sockets.md new file mode 100644 index 0000000000..a72302bcef --- /dev/null +++ b/chapters/io/ipc/reading/unix-sockets.md @@ -0,0 +1,50 @@ +# Unix Sockets + +Unix sockets are a inter-process communication (IPC) method that addresses some limitations of [pipes](../reading/pipes.md). +Their key characteristics are: + +- **Bidirectional communication**: Allowing both `send` and `receive` operations through the same file descriptor. +- **Data transmission modes**: Supporting both `stream` (continuous data flow) and `datagram` (message-based) models. +- **Connection-based**: Maintaining a connection between processes, so the sender's identity is always known. + +## API - Hail Berkeley Sockets + +Unix sockets are managed through the [Berkeley Sockets API](https://en.wikipedia.org/wiki/Berkeley_sockets), which is widely supported across various operating systems and programming languages. +This API is not limited to **Unix sockets**; it also enables communication with processes on remote systems using [**network sockets**](../../ipc/reading/network-sockets.md) + +The socket interface works similarly to the file interface, offering system calls for creating, reading, and writing data. +It also includes additional calls for setting up addresses, handling connections, and connecting to remote hosts: + +- **[`socket(domain, type, protocol)`](https://man7.org/linux/man-pages/man2/socket.2.html)**: Creates a new socket and returns a file descriptor for further operations. + - The `domain` argument determines whether the socket is intended for **local connections** (Unix socket) or **remote connections** (network socket). + - The `type` argument specifies the communication mode, either `SOCK_STREAM` for stream-oriented communication or `SOCK_DGRAM` for datagram-oriented communication. + - The `protocol` argument indicates the protocol to use, which is often set to `0`, as there is typically only one protocol available for each socket type within a specific protocol family. +- **[`bind()`](https://man7.org/linux/man-pages/man2/bind.2.html)**: Associates an address and port with the socket. + For Unix sockets, `bind()` also creates a file on disk as the socket identifier. +- **[`listen()`](https://man7.org/linux/man-pages/man2/listen.2.html)**: Sets the socket to passive mode, preparing it to accept incoming connections. +- **[`accept()`](https://man7.org/linux/man-pages/man2/accept.2.html)**: Accepts a pending connection and returns a new socket for it, blocking if no connections are pending. +- **[`connect()`](https://man7.org/linux/man-pages/man2/connect.2.html)**: Initiates a connection to a remote socket. +- **[`send()` / `sendto()`](https://man7.org/linux/man-pages/man2/send.2.html)**: Sends data over the socket, similar to [`write()`](https://man7.org/linux/man-pages/man2/write.2.html). +- **[`recv()` / `recvfrom()`](https://man7.org/linux/man-pages/man2/recv.2.html)**: Receives data from the socket, akin to [`read()`](https://man7.org/linux/man-pages/man2/read.2.html). + +Before utilizing the API, it's essential to understand that for two processes to communicate, they need a way to identify the socket. +This identification method is similar to that used with named pipes, relying on a file identifier stored on disk. +If the file identifier does not already exist, it will be created by the `bind()` function. +Below is an example of how to implement this in code: + +```c +char path[] = "my-socket"; +struct sockaddr_un addr; // Structure to hold the address for the UNIX socket. + +memset(&addr, 0, sizeof(addr)); // Clear the address structure. +addr.sun_family = AF_UNIX; // Set the address family. +snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path); // Set the path. + +sockfd = socket(PF_UNIX, SOCK_STREAM, 0); // Create the socket. +if (sockfd < 0) {...} // handle error + +rc = bind(sockfd, (struct sockaddr *) &addr, sizeof(addr)); // Bind the socket. +if (rc < 0) {...} // handle error +``` + +You can practice working with UNIX sockets by completing the [UNIX Sockets Communication task](../drills/tasks/unix-socket/README.md). diff --git a/content/chapters/io/lab/quiz/fewer-than-2-copies.md b/chapters/io/optimizations/drills/questions/fewer-than-2-copies.md similarity index 91% rename from content/chapters/io/lab/quiz/fewer-than-2-copies.md rename to chapters/io/optimizations/drills/questions/fewer-than-2-copies.md index a419ceb702..847c4223f4 100644 --- a/content/chapters/io/lab/quiz/fewer-than-2-copies.md +++ b/chapters/io/optimizations/drills/questions/fewer-than-2-copies.md @@ -4,7 +4,7 @@ Can zero-copy be implemented so as to copy the file fewer than 2 times? -![Zero-Copy](../media/server-copies-zero-copy.svg) +![Zero-Copy](../../media/server-copies-zero-copy.svg) ## Question Answers @@ -21,7 +21,7 @@ Can zero-copy be implemented so as to copy the file fewer than 2 times? The truth is that we can't have fewer copies while using a server with a common PC-like architecture. The disk is not directly connected to the internet, so the file cannot be sent directly from there. The only place from where we can send data to the Web is the NIC. -Then we need the intermediary storage in that kernel buffer because the disk and the NIC aren't dirrectly connected. +Then we need the intermediary storage in that kernel buffer because the disk and the NIC aren't directly connected. They are both connected to the CPU via the motherboard, so it's the CPU's job to do the transfer. For this, it needs a "temporary buffer". Then the NIC needs its own buffer because the speed of the network may be slower than the speed at which it receives data from the kernel, so it needs some memory where to place the "surplus" while waiting for the network to "clear". diff --git a/content/chapters/io/lab/quiz/server-copies.md b/chapters/io/optimizations/drills/questions/server-copies.md similarity index 83% rename from content/chapters/io/lab/quiz/server-copies.md rename to chapters/io/optimizations/drills/questions/server-copies.md index 17de439a9e..3541c2ba3b 100644 --- a/content/chapters/io/lab/quiz/server-copies.md +++ b/chapters/io/optimizations/drills/questions/server-copies.md @@ -5,7 +5,7 @@ The server in the image below uses regular TCP sockets to handle the connection and `send()` to send the data. How many times are the contents of the file copied by the server while being sent to the client? -![Client-Server Steps](../media/client-server-file.svg) +![Client-Server Steps](../../media/client-server-file.svg) ## Question Answers @@ -19,11 +19,11 @@ How many times are the contents of the file copied by the server while being sen ## Feedback -Rembember double buffering! +Remember double buffering! When the app calls `read()`, the server's kernel will first save the file to an internal buffer destined for reading. Then the app will copy the file to its own buffer. Following this step, the app will call `send()`, which will first copy the file to a buffer in the kernel. From this buffer, the kernel itself will copy the file to another buffer on the NIC (Network Interface Card). In total, there the file is copied **4 times**, as outlined in the image below. -![Server Copies - Read-Send](../media/server-copies-normal.svg) +![Server Copies - Read-Send](../../media/server-copies-normal.svg) diff --git a/chapters/io/optimizations/drills/tasks/async-server/.gitignore b/chapters/io/optimizations/drills/tasks/async-server/.gitignore new file mode 100644 index 0000000000..296b2ccf64 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/.gitignore @@ -0,0 +1 @@ +support/ diff --git a/chapters/io/optimizations/drills/tasks/async-server/Makefile b/chapters/io/optimizations/drills/tasks/async-server/Makefile new file mode 100644 index 0000000000..ff78d246ea --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/Makefile @@ -0,0 +1,9 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + mkdir -p support/ + $(PYTHON) $(SCRIPT) --input ./solution --output ./support + +clean: + rm -rf support/ diff --git a/chapters/io/optimizations/drills/tasks/async-server/README.md b/chapters/io/optimizations/drills/tasks/async-server/README.md new file mode 100644 index 0000000000..bdc55347f1 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/README.md @@ -0,0 +1,25 @@ +# Async Server + +Navigate to `chapters/io/optimizations/drills/tasks/async-server` and run `make` to generate the `support` files. +Enter `support` and run `make test-file.txt` to generate the test file. + +This task builds on the previous example of a [multiplexed client-server](../../tasks/multiplexed-client-server/README.md). +The server accepts connections from clients and downloads a file of `1 GB` from each. +After uploading the file, the clients close the connection. + +1. Open `server.c` and complete the TODOs in the main function to setup IO multiplexing using [`epoll`](https://man7.org/linux/man-pages/man7/epoll.7.html). + Use `epoll_create()`, `epoll_wait()`, and the wrappers defined in `w_epoll.h` to handle descriptors without blocking. + **Remember** to remove the client sockets from the `epoll` instance before closing them. + + To test, run `./server` in one terminal and `./client` in another terminal.s + If successful, the clients should print the upload progress. + +1. There is a problem with our current implementation. + Try to start two clients at the same time - the first one will start uploading, and the second one will block at `connect()`. + This happens because, even though we are multiplexing file descriptors on the server-side, it is busy with another client. + To account for this, complete the TODOs in `handle_client()` to serve each client on a different process. + + To test, start `python server.py` in one terminal and run your client implementation in two separate terminals. + If successful, the clients should be able to upload at the same time. + +If you're having difficulties solving this exercise, go through [Async I/O](../../../reading/async-io.md) and [I/O Multiplexing](../../../reading/io-multiplexing.md). diff --git a/chapters/io/optimizations/drills/tasks/async-server/generate_skels.py b/chapters/io/optimizations/drills/tasks/async-server/generate_skels.py new file mode 100644 index 0000000000..697c9d5b61 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/generate_skels.py @@ -0,0 +1,151 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + or re.match(r".*\.py$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/chapters/io/optimizations/drills/tasks/async-server/solution/.gitignore b/chapters/io/optimizations/drills/tasks/async-server/solution/.gitignore new file mode 100644 index 0000000000..8e11b6cc2b --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/solution/.gitignore @@ -0,0 +1,3 @@ +client +server +*.txt diff --git a/chapters/io/optimizations/drills/tasks/async-server/solution/Makefile b/chapters/io/optimizations/drills/tasks/async-server/solution/Makefile new file mode 100644 index 0000000000..a95c0a809d --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/solution/Makefile @@ -0,0 +1,40 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = server client +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + +test-file.txt: + dd if=/dev/urandom of=test-file.txt bs=1024 count=1M + +.PHONY: all clean diff --git a/chapters/io/optimizations/drills/tasks/async-server/solution/client.c b/chapters/io/optimizations/drills/tasks/async-server/solution/client.c new file mode 100644 index 0000000000..37585ff42c --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/solution/client.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "w_epoll.h" + +static const char IP[] = "127.0.0.1"; +static const int PORT = 5000; + +/** + * Create a sockaddr_in structure with the given IP and port. + */ +struct sockaddr_in get_sockaddr(const char *ip, const int port) +{ + struct sockaddr_in addr; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + + return addr; +} + +/** + * Read 1GB file and send it to the server. + */ +static void send_1GB_file(int sockfd, const char *filename) +{ + char buf[BUFSIZ]; + FILE *file; + int rc; + + file = fopen(filename, "r"); + if (file == NULL) { + printf("File not found.\n"); + return; + } + + size_t n, total = 0; + + while (true) { + n = fread(buf, 1, BUFSIZ, file); + if (n == 0) + break; + + rc = send(sockfd, buf, n, 0); + DIE(rc < 0, "send"); + + total += n; + if (total % (10 * 1024 * 1024) == 0) // Print every 10MB + printf("Sent %lu MB\n", total / (1024 * 1024)); + } + + printf("Sent 1GB file to server\n"); + + rc = fclose(file); + DIE(rc < 0, "fclose"); +} + +/** + * Create a TCP socket and listen for incoming connections. + */ +int main(void) +{ + struct sockaddr_in addr = get_sockaddr(IP, PORT); + int sockfd; + int rc; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + DIE(sockfd < 0, "socket"); + + rc = connect(sockfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "connect"); + + send_1GB_file(sockfd, "test-file.txt"); + + rc = close(sockfd); + DIE(rc < 0, "close"); + + return 0; +} diff --git a/chapters/io/optimizations/drills/tasks/async-server/solution/server.c b/chapters/io/optimizations/drills/tasks/async-server/solution/server.c new file mode 100644 index 0000000000..dc7e7f981b --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/solution/server.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "w_epoll.h" + +#define SMALL_BUF 10 // Small buffer size to simulate slow network + +static const char IP[] = "127.0.0.1"; +static const int PORT = 5000; + +/** + * Create a sockaddr_in structure with the given IP and port. + */ +struct sockaddr_in get_sockaddr(const char *ip, const int port) +{ + struct sockaddr_in addr; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + + return addr; +} + +/** + * Get a 1GB file over the network. + * No need to store the data, just receive it to simulate a real-world scenario. + */ +static void receive_1GB_file(int sockfd) +{ + char buf[SMALL_BUF]; + int rc; + + while (true) { // Receive data until the client closes the connection + rc = recv(sockfd, buf, SMALL_BUF, 0); + DIE(rc < 0, "recv"); + + if (rc == 0) + break; + } + + printf("Received 1GB file from [Client %d]\n", sockfd); +} + +/** + * Create a new process to handle the client. + */ +void handle_client(int epollfd, int sockfd) +{ + int pid, rc; + + /* TODO 2: Create a new process to handle the client */ + pid = fork(); + DIE(pid < 0, "fork"); + + /* TODO 5: Print the child process id and handle the client */ + printf("<< Process %d created to handle client %d >>\n", pid, sockfd); + if (pid == 0) { /* Child process */ + receive_1GB_file(sockfd); + exit(EXIT_SUCCESS); + } + + /* REPLACE 2*/ + /* // Remove this after implementing the child process */ + /* receive_1GB_file(sockfd); */ + + /* TODO 2: Remove the client from epoll */ + rc = w_epoll_del_fd(epollfd, sockfd); + DIE(rc < 0, "epoll_ctl"); + + /* TODO 2: Close the client socket */ + rc = close(sockfd); + DIE(rc < 0, "close"); +} + +/** + * Create a TCP socket and listen for incoming connections. + */ +int main(void) +{ + char buf[BUFSIZ]; + struct sockaddr_in addr = get_sockaddr(IP, PORT); + int listenfd, connectfd, epollfd; + int n, rc; + + /* Create a network socket with SOCK_STREAM type in listenfd. */ + listenfd = socket(AF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + /* Set the socket to be reusable. */ + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + /* Bind the socket to the addr. */ + rc = bind(listenfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "bind"); + + /* Mark socket as passive socket using listen(). */ + rc = listen(listenfd, 1); + DIE(rc < 0, "listen"); + + /* TODO 2: Create an epoll instance */ + epollfd = epoll_create1(0); + DIE(epollfd < 0, "epoll_create1"); + + /* TODO 2: Add listenfd to epoll */ + rc = w_epoll_add_fd_in(epollfd, listenfd); + DIE(rc < 0, "epoll_ctl"); + + while (1) { + struct epoll_event events[10]; + + /* TODO 2: Wait for events on the epoll instance */ + n = epoll_wait(epollfd, events, 10, -1); + DIE(n < 0, "epoll_wait"); + + for (int i = 0; i < n; i++) { + if (events[i].data.fd == listenfd) { // New connection + /* TODO 2: Accept a new connection */ + connectfd = accept(listenfd, NULL, NULL); + DIE(connectfd < 0, "accept"); + + printf("<< New client connected on socket %d >>\n", connectfd); + + /* TODO 2: Add the new connection to epoll */ + rc = w_epoll_add_fd_in(epollfd, connectfd); + DIE(rc < 0, "epoll_ctl"); + } else { // A client sent data + memset(buf, 0, BUFSIZ); + + printf("Client %d wants to send a file\n", events[i].data.fd); + handle_client(epollfd, events[i].data.fd); + } + } + } + + /* TODO 4: Close the sockets and the epoll instance */ + rc = close(listenfd); + DIE(rc < 0, "close"); + rc = close(epollfd); + DIE(rc < 0, "close"); + + return 0; +} diff --git a/chapters/io/optimizations/drills/tasks/async-server/solution/utils/log/CPPLINT.cfg b/chapters/io/optimizations/drills/tasks/async-server/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/optimizations/drills/tasks/async-server/solution/utils/log/log.c b/chapters/io/optimizations/drills/tasks/async-server/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/optimizations/drills/tasks/async-server/solution/utils/log/log.h b/chapters/io/optimizations/drills/tasks/async-server/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/optimizations/drills/tasks/async-server/solution/utils/utils.h b/chapters/io/optimizations/drills/tasks/async-server/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/optimizations/drills/tasks/async-server/solution/w_epoll.h b/chapters/io/optimizations/drills/tasks/async-server/solution/w_epoll.h new file mode 100644 index 0000000000..e3acee190f --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/async-server/solution/w_epoll.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef W_EPOLL_H_ +#define W_EPOLL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/** + * Add a file descriptor to the epoll instance. + */ +static inline int w_epoll_add_fd_in(int epollfd, int new_fd) +{ + struct epoll_event ev; // Used to store event information + + /* Set the event type and the file descriptor */ + ev.events = EPOLLIN; + ev.data.fd = new_fd; + + /* Return the result of epoll_ctl ADD operation */ + return epoll_ctl(epollfd, EPOLL_CTL_ADD, new_fd, &ev); +} + +/** + * Remove a file descriptor from the epoll instance. + */ +static inline int w_epoll_del_fd(int epollfd, int fd) +{ + /* Return the result of epoll_ctl DEL operation */ + return epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, NULL); +} + +#ifdef __cplusplus +} +#endif + +#endif /* EPOLL_H_ */ diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/.gitignore b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/.gitignore new file mode 100644 index 0000000000..296b2ccf64 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/.gitignore @@ -0,0 +1 @@ +support/ diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/Makefile b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/Makefile new file mode 100644 index 0000000000..ff78d246ea --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/Makefile @@ -0,0 +1,9 @@ +PYTHON = python3 +SCRIPT = generate_skels.py + +skels: + mkdir -p support/ + $(PYTHON) $(SCRIPT) --input ./solution --output ./support + +clean: + rm -rf support/ diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/README.md b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/README.md new file mode 100644 index 0000000000..1d8b10ccd2 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/README.md @@ -0,0 +1,33 @@ +# Multiplexed Client Server + +Navigate to `chapters/io/optimizations/drills/tasks/client-server-epoll` and run `make` to generate the `support` files. + +This task builds on the previous implementation of a [client-server ordered communication](../../../../ipc/drills/tasks/client-server/README.md). +Previously, the client and server followed a strict, sequential communication pattern: each peer had to send a message and wait for a reply before proceeding. + +We plan to build a group chat where clients can send messages at any time, and each message is broadcast to all other connected clients. +To accomplish this, we’ll implement I/O multiplexing mechanisms that notify us only when data is available on a file descriptor. +This non-blocking approach allows for smooth, unhindered communication between clients. + +1. We’ll use the [`epoll`](https://man7.org/linux/man-pages/man7/epoll.7.html) interface to manage multiple file descriptors. + Begin by opening `w_epoll.h` and completing the TODOs. + We will define wrapper functions to **add** and **remove** file descriptors from the `epoll` instance, making the main code more readable. + **Note:** Ensure that each wrapper returns the result of the `epoll_ctl()` for error handling. + +1. Complete the TODOs in `support/client.c` to enable multiplexing of the available file descriptors. + The file descriptors are `stdin` (for receiving user messages) and `sockfd` (for communication with the server). + Use `epoll_create()`, `epoll_wait()`, and the wrappers defined in `w_epoll.h` to handle these descriptors without blocking. + Remember to close the sockets before exiting. + + To test, start `python server.py` in one terminal and run your client implementation in two separate terminals. + If successful, the clients should be able to communicate through the server. + +1. Complete the TODOs in `support/server.c` to multiplex I/O with clients. + You will need to create an `epoll` instance and dynamically **add** and **remove** clients as they connect and disconnect. + Use `epoll_create()`, `epoll_wait()`, and the wrappers defined in `w_epoll.h` to achieve this functionality. + **Remember** to remove the client sockets from the `epoll` instance before closing them. + + To test your implementation, run `./server` in one terminal and `./client` (or `python client.py`) in two separate terminals. + If everything works correctly, the clients should be able to communicate with each other via the server. + +If you're having difficulties solving this exercise, go through the `epoll` API from [I/O Multiplexing](../../../reading/io-multiplexing.md). diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/generate_skels.py b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/generate_skels.py new file mode 100644 index 0000000000..697c9d5b61 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/generate_skels.py @@ -0,0 +1,151 @@ +#!/usr/bin/python3 -u +# SPDX-License-Identifier: BSD-3-Clause + +import sys +import argparse +import os.path +import re + + +def process_file(src, dst, pattern, replace, remove, replace_pairs, end_string=None): + if not pattern or not replace or not remove: + print( + f"ERROR: The script behaviour is not properly specified for {src}", + file=sys.stderr, + ) + sys.exit(1) + + fin = open(src, "r") + fout = open(dst, "w") + remove_lines = 0 + skip_lines = 0 + uncomment_lines = 0 + end_found = True + + for l in fin.readlines(): + # Skip generation of file. + if "SKIP_GENERATE" in l: + fout.close() + os.remove(dst) + return + + if end_string and end_found == False: + fout.write(l) + if end_string in l: + end_found = True + continue + + if remove_lines > 0: + remove_lines -= 1 + continue + + if skip_lines > 0: + skip_lines -= 1 + m = re.search(pattern, l) + if m: + l = "%s%s\n" % (m.group(1), m.group(3)) + fout.write(l) + continue + + if uncomment_lines > 0: + uncomment_lines -= 1 + for fro, to in replace_pairs: + l = re.sub(fro, to, l) + fout.write(l) + continue + + m = re.search(pattern, l) + if m: + if m.group(2): + skip_lines = int(m.group(2)) + else: + skip_lines = 1 + + if end_string and end_string not in l: + end_found = False + + l = "%s%s\n" % (m.group(1), m.group(3)) + + m = re.search(replace, l) + if m: + if m.group(2): + uncomment_lines = int(m.group(2)) + else: + uncomment_lines = 1 + continue + + m = re.search(remove, l) + if m: + if m.group(2): + remove_lines = int(m.group(2)) + else: + remove_lines = 1 + continue + + fout.write(l) + + fout.close() + + +def main(): + parser = argparse.ArgumentParser( + description="Generate skeletons sources from reference solution sources" + ) + parser.add_argument( + "--input", help="input directory to process files", required=True + ) + parser.add_argument( + "--output", help="output directory to copy processed files", required=True + ) + args = parser.parse_args() + + for root, dirs, files in os.walk(args.input): + new_root = os.path.join(args.output, os.path.relpath(root, args.input)) + for d in dirs: + os.makedirs(os.path.join(new_root, d), exist_ok=True) + + for src in files: + if ( + re.match("Makefile.*$", src) + or re.match(r".*\.sh$", src) + or re.match(r".*\.[sS]$", src) + or re.match(r".*\.py$", src) + ): + pattern = r"(^\s*#\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*#\s*REPLACE)( [0-9]*)" + remove = r"(^\s*#\s*REMOVE)( [0-9]*)" + replace_pairs = [("# ", "")] + end_string = None + elif re.match(r".*\.asm$", src): + pattern = r"(^\s*;\s*TODO)( [0-9]*)(:.*)" + replace = r"(^\s*;\s*REPLACE)( [0-9]*)" + remove = r"(^\s*;\s*REMOVE)( [0-9]*)" + replace_pairs = [("; ", "")] + end_string = None + elif ( + re.match(r".*\.[ch]$", src) + or re.match(r".*\.cpp$", src) + or re.match(r".*\.hpp$", src) + ): + pattern = r"(.*/\*\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*/\*\s*REPLACE)( [0-9]*)" + remove = r"(.*/\*\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"/\* ", ""), (r" \*/", "")] + end_string = "*/" + elif re.match(r".*\.d$", src): + pattern = r"(.*//\s*TODO)([ 0-9]*)(:.*)" + replace = r"(.*//\s*REPLACE)( [0-9]*)" + remove = r"(.*//\s*REMOVE)( [0-9]*)" + replace_pairs = [(r"// ", "")] + end_string = None + else: + continue + + dst = os.path.join(new_root, src) + src = os.path.join(root, src) + print(dst) + process_file(src, dst, pattern, replace, remove, replace_pairs, end_string) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/content/chapters/io/lab/solution/client-server/.gitignore b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/.gitignore similarity index 50% rename from content/chapters/io/lab/solution/client-server/.gitignore rename to chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/.gitignore index b051c6c57f..f2ad85300e 100644 --- a/content/chapters/io/lab/solution/client-server/.gitignore +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/.gitignore @@ -1 +1,2 @@ client +server diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/Makefile b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/Makefile new file mode 100644 index 0000000000..8aa3737d14 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/Makefile @@ -0,0 +1,37 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Source files and corresponding binaries +BINARIES = server client +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + +.PHONY: all clean diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/client.c b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/client.c new file mode 100644 index 0000000000..cbaaf8f264 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/client.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "w_epoll.h" + +#ifndef BUFSIZ +#define BUFSIZ 256 +#endif + +static const char IP[] = "127.0.0.1"; +static const int PORT = 5000; + +/** + * Create a sockaddr_in structure with the given IP and port. + */ +struct sockaddr_in get_sockaddr(const char *ip, const int port) +{ + struct sockaddr_in addr; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + + return addr; +} + +/** + * Get user input from stdin and remove trailing newline. + */ +static void get_user_input(char *buf, size_t size) +{ + memset(buf, 0, size); + fgets(buf, size, stdin); + // Remove trailing newline + if (buf[strlen(buf) - 1] == '\n') + buf[strlen(buf) - 1] = '\0'; +} + +/** + * Create a TCP socket and listen for incoming connections. + */ +int main(void) +{ + char buf[BUFSIZ]; + struct sockaddr_in addr = get_sockaddr(IP, PORT); + int sockfd, epollfd; + int n, rc; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + DIE(sockfd < 0, "socket"); + + rc = connect(sockfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "connect"); + + /* TODO 2: Create an epoll instance */ + epollfd = epoll_create1(0); + DIE(epollfd < 0, "epoll_create1"); + + /* TODO 2: Add sockfd to epoll */ + rc = w_epoll_add_fd_in(epollfd, sockfd); + DIE(rc < 0, "epoll_ctl"); + + /* TODO 2: Add STDIN_FILENO to epoll to be able to read user input */ + rc = w_epoll_add_fd_in(epollfd, STDIN_FILENO); + DIE(rc < 0, "epoll_ctl"); + + while (1) { + struct epoll_event events[10]; + + /* TODO 2: Wait for events on the epoll instance */ + n = epoll_wait(epollfd, events, 10, -1); + DIE(n < 0, "epoll_wait"); + + for (int i = 0; i < n; i++) { + if (events[i].data.fd == STDIN_FILENO) { // Received data from stdin + get_user_input(buf, BUFSIZ); + if (strcmp(buf, "exit") == 0) + break; + + rc = send(sockfd, buf, strlen(buf), 0); + DIE(rc < 0, "send"); + + if (rc == 0) + break; + } else { // Received data from server + memset(buf, 0, BUFSIZ); + rc = recv(sockfd, buf, BUFSIZ, 0); + DIE(rc < 0, "recv"); + + if (rc == 0) { + printf("[SERVER] Connection closed.\n"); + break; + } + + printf("%s\n", buf); + } + } + } + + /* TODO 4: Close the socket and the epoll instance */ + rc = close(sockfd); + DIE(rc < 0, "close"); + rc = close(epollfd); + DIE(rc < 0, "close"); + + return 0; +} diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/client.py b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/client.py new file mode 100644 index 0000000000..991c1432e4 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/client.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: BSD-3-Clause + +import socket +import selectors +import sys + +IP = "127.0.0.1" +PORT = 5000 +BUFSIZ = 256 + + +def main(): + selector = selectors.DefaultSelector() + + # Create a socket and connect to the server + sockfd = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sockfd.connect((IP, PORT)) + sockfd.setblocking(False) + + # Register socket and stdin (standard input) for monitoring with selectors + selector.register(sockfd, selectors.EVENT_READ, handle_server_response) + handle_user_input = lambda stdin, sel: _handle_user_input(stdin, sel, sockfd) + selector.register(sys.stdin, selectors.EVENT_READ, handle_user_input) + + print("Connected to the server. Type 'exit' to quit.") + + try: + while True: + events = selector.select(timeout=None) + for key, _ in events: + callback = key.data + callback(key.fileobj, selector) + except KeyboardInterrupt: + print("Client exiting.") + finally: + # Clean up the connections + selector.unregister(sockfd) + sockfd.close() + selector.close() + + +def handle_server_response(sock: socket.socket, selector: selectors.DefaultSelector): + """Handle receiving data from the server.""" + data = sock.recv(BUFSIZ) + if not data: + print("[SERVER] Connection closed.") + selector.unregister(sock) + sock.close() + sys.exit(0) # Exit if server closes the connection + else: + print(data.decode("utf-8")) + + +def _handle_user_input(stdin, selector: selectors.DefaultSelector, sock: socket.socket): + """Handle user input from stdin and send to server.""" + message = input().strip() + if message.lower() == "exit": + print("Exiting...") + selector.unregister(stdin) + selector.unregister(sock) + sock.close() + sys.exit(0) + else: + sock.sendall(message.encode("utf-8")) + + +if __name__ == "__main__": + main() diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/server.c b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/server.c new file mode 100644 index 0000000000..716799b9eb --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/server.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/utils.h" +#include "w_epoll.h" + +#ifndef BUFSIZ +#define BUFSIZ 256 +#endif + +static const char IP[] = "127.0.0.1"; +static const int PORT = 5000; + +/** + * Create a sockaddr_in structure with the given IP and port. + */ +struct sockaddr_in get_sockaddr(const char *ip, const int port) +{ + struct sockaddr_in addr; + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + addr.sin_addr.s_addr = inet_addr(ip); + + return addr; +} + +/** + * Check if the file descriptor is a socket. + */ +static inline int is_socket(int fd) +{ + struct stat statbuf; + + if (fstat(fd, &statbuf) == -1) // Not a valid file descriptor + return 0; + + return S_ISSOCK(statbuf.st_mode); +} + +/** + * Quick and dirty function to share a message to all connected clients. + * For proper implementation, consider using a list of connected clients. + */ +static inline void share_msg(int senderfd, char *msg) +{ + static const int MAX_FD = 30; // Fails for more than 25 clients + char buf[BUFSIZ]; + int rc; + + sprintf(buf, "[Client %d]: %s", senderfd, msg); + printf("%s\n", buf); + + for (int i = 5; i <= MAX_FD; i++) { + if (is_socket(i) && i != senderfd) { + rc = send(i, buf, strlen(buf), 0); + DIE(rc < 0, "send"); + } + } +} + +/** + * Create a TCP socket and listen for incoming connections. + */ +int main(void) +{ + char buf[BUFSIZ]; + struct sockaddr_in addr = get_sockaddr(IP, PORT); + int listenfd, connectfd, epollfd; + int n, rc; + + /* Create a network socket with SOCK_STREAM type in listenfd. */ + listenfd = socket(AF_INET, SOCK_STREAM, 0); + DIE(listenfd < 0, "socket"); + + /* Set the socket to be reusable. */ + rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int)); + DIE(rc < 0, "setsockopt"); + + /* TODO 2: Bind the socket to the addr. */ + rc = bind(listenfd, (struct sockaddr *) &addr, sizeof(addr)); + DIE(rc < 0, "bind"); + + /* TODO 2: Mark socket as passive socket using listen(). */ + rc = listen(listenfd, 1); + DIE(rc < 0, "listen"); + + /* TODO 2: Create an epoll instance */ + epollfd = epoll_create1(0); + DIE(epollfd < 0, "epoll_create1"); + + /* TODO 2: Add listenfd to epoll */ + rc = w_epoll_add_fd_in(epollfd, listenfd); + DIE(rc < 0, "epoll_ctl"); + + while (1) { + struct epoll_event events[10]; + + /* TODO 2: Wait for events on the epoll instance */ + n = epoll_wait(epollfd, events, 10, -1); + DIE(n < 0, "epoll_wait"); + + for (int i = 0; i < n; i++) { + if (events[i].data.fd == listenfd) { // New connection + /* TODO 2: Accept a new connection */ + connectfd = accept(listenfd, NULL, NULL); + DIE(connectfd < 0, "accept"); + + printf("<< New client connected on socket %d >>\n", connectfd); + + /* TODO 2: Add the new connection to epoll */ + rc = w_epoll_add_fd_in(epollfd, connectfd); + DIE(rc < 0, "epoll_ctl"); + } else { // Received data from a connected client + memset(buf, 0, BUFSIZ); + + /* TODO 2: Receive data from the connected socket */ + rc = recv(events[i].data.fd, buf, BUFSIZ, 0); + DIE(rc < 0, "recv"); + + if (rc == 0) { + printf("<< Client from socket %d disconnected >>\n", events[i].data.fd); + + /* TODO 2: Remove the disconnected client from epoll */ + rc = w_epoll_del_fd(epollfd, events[i].data.fd); + DIE(rc < 0, "epoll_ctl"); + + /* TODO 2: Close the disconnected client socket */ + rc = close(events[i].data.fd); + DIE(rc < 0, "close"); + + continue; + } + + share_msg(events[i].data.fd, buf); + } + } + } + + /* TODO 4: Close the sockets and the epoll instance */ + rc = close(listenfd); + DIE(rc < 0, "close"); + rc = close(epollfd); + DIE(rc < 0, "close"); + + return 0; +} diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/server.py b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/server.py new file mode 100644 index 0000000000..286e901a56 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/server.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: BSD-3-Clause + +import socket +import selectors + + +IP = "127.0.0.1" +PORT = 5000 +BUFSIZ = 1024 +connected_clients = [] # Keep track of connected clients + + +def share_msg(sender: socket.socket, message: str): + """Share a message with all connected clients, except the sender.""" + msg = f"[Client {sender.fileno()}]: {message}".encode("utf-8") + print(msg.decode()) + + for client in connected_clients: + if client != sender: + client.send(msg) + + +def accept_connection(sock: socket.socket, selector: selectors.BaseSelector): + """Accept a new client connection.""" + conn, _ = sock.accept() + conn.setblocking(False) + + print(f"<< New client connected on socket {conn.fileno()} >>") + selector.register(conn, selectors.EVENT_READ, handle_client) + + connected_clients.append(conn) + + +def handle_client(conn: socket.socket, selector: selectors.BaseSelector): + """Handle client data reception and sharing.""" + try: + data = conn.recv(BUFSIZ) + if not data: + print(f"<< Client from socket {conn.fileno()} disconnected >>") + selector.unregister(conn) + conn.close() + connected_clients.remove(conn) + return + except ConnectionResetError: + print(f"<< Client from socket {conn.fileno()} disconnected >>") + selector.unregister(conn) + conn.close() + + share_msg(conn, data.decode("utf-8")) + + +def main(): + # Create and configure listening socket + listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listen_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listen_sock.bind((IP, PORT)) + listen_sock.listen() + listen_sock.setblocking(False) + + selector = selectors.DefaultSelector() + selector.register(listen_sock, selectors.EVENT_READ, accept_connection) + print(f"Server listening on {IP}:{PORT}") + + try: + while True: + events = selector.select(timeout=None) + for key, _ in events: + callback = key.data + callback(key.fileobj, selector) + except KeyboardInterrupt: + print("Server shutting down.") + finally: + selector.close() + listen_sock.close() + + +if __name__ == "__main__": + main() diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/log/CPPLINT.cfg b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/log/log.c b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/log/log.h b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/utils.h b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/w_epoll.h b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/w_epoll.h new file mode 100644 index 0000000000..2e93e851ab --- /dev/null +++ b/chapters/io/optimizations/drills/tasks/multiplexed-client-server/solution/w_epoll.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef W_EPOLL_H_ +#define W_EPOLL_H_ 1 + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/** + * Add a file descriptor to the epoll instance. + */ +static inline int w_epoll_add_fd_in(int epollfd, int new_fd) +{ + struct epoll_event ev; // Used to store event information + + /* TODO 2: Set the event type and the file descriptor */ + ev.events = EPOLLIN; + ev.data.fd = new_fd; + + /* TODO 1: Return the result of epoll_ctl ADD operation */ + return epoll_ctl(epollfd, EPOLL_CTL_ADD, new_fd, &ev); + + /* REPLACE 1 */ + /* return 0; */ +} + +/** + * Remove a file descriptor from the epoll instance. + */ +static inline int w_epoll_del_fd(int epollfd, int fd) +{ + /* TODO 1: Return the result of epoll_ctl DEL operation */ + return epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, NULL); + + /* REPLACE 1 */ + /* return 0; */ +} + +#ifdef __cplusplus +} +#endif + +#endif /* EPOLL_H_ */ diff --git a/chapters/io/optimizations/guides/async/README.md b/chapters/io/optimizations/guides/async/README.md new file mode 100644 index 0000000000..6e59fe8092 --- /dev/null +++ b/chapters/io/optimizations/guides/async/README.md @@ -0,0 +1,58 @@ +# Async + +Enter the `chapters/io/optimizations/guide/async/` folder for some implementations of a simple request-reply server in C. +Here we have the implementation of a server that computes the n-th fibonacci number. +The server serves requests in different ways: + +* **synchronous** server: `server.c` +* **multiprocess** server: `mp_server.c` +* **multithreaded** server: `mt_server.c` + +**Note:** There is no asynchronous C variant, because of the unstable API of [`libaio`](https://pagure.io/libaio) and [`io_uring`](https://unixism.net/loti/what_is_io_uring.html). + +1. Benchmark the synchronous server to have a reference point. + Run `./server 2999` in one terminal and `time ./client_bench.sh 2999` in another. + `client_bench.sh` we'll run `8` instances of `client.py` that make a request. + + ```console + student@os:~/async/support$ time ./client_bench.sh 2000 + [...] + root: Connected to localhost:2000 + root: Sending 30 + function(30): 1346269 + + real 0m1.075s + user 0m0.301s + sys 0m0.029s + ``` + + The value you obtain might be different, but you should observe a speed-up when benchmarking the other two solutions. + +1. Benchmark the multiprocess and multithreaded alternatives and see which one got the best speed increase. + You can obtain more relevant values by tweaking parameters in `client_bench.sh`. + For example, you could increase the **number of clients** or the **fibonacci value to compute**. + +1. Begin by benchmarking the synchronous server to establish a baseline. + Run `./server 2999` in one terminal and `time ./client_bench.sh 2999` in another. + The `client_bench.sh` script will initiate `8` instances of `client.py`, each making a request. + The output might look like this: + + ```console + student@os:~/async/support$ time ./client_bench.sh 2000 + [...] + root: Connected to localhost:2000 + root: Sending 30 + function(30): 1346269 + + real 0m1.075s + user 0m0.301s + sys 0m0.029s + ``` + + Although the actual value may vary, you should observe a noticeable speed-up when testing the other two solutions. + +1. Next, benchmark the multiprocess and multithreaded alternatives to determine which offers the best performance improvement. + To obtain more meaningful results, adjust parameters in `client_bench.sh`, such as increasing the **number of clients** or the **Fibonacci value to compute**. + +If you're having difficulties understanding the support code, go through [this reading material](../../reading/async-io.md). +If you want to practice this yourself, go through the [Async Server task](../../drills/tasks/async-server/README.md). diff --git a/chapters/io/optimizations/guides/async/support/.gitignore b/chapters/io/optimizations/guides/async/support/.gitignore new file mode 100644 index 0000000000..cc9af37561 --- /dev/null +++ b/chapters/io/optimizations/guides/async/support/.gitignore @@ -0,0 +1,3 @@ +server +mp_server +mt_server diff --git a/chapters/io/optimizations/guides/async/support/Makefile b/chapters/io/optimizations/guides/async/support/Makefile new file mode 100644 index 0000000000..bf33a7a528 --- /dev/null +++ b/chapters/io/optimizations/guides/async/support/Makefile @@ -0,0 +1,46 @@ +# Get the relative path to the directory of the current makefile +MAKEFILE_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +INCLUDES_DIR := $(MAKEFILE_DIR) +UTILS_DIR := $(MAKEFILE_DIR)/utils + +# Compiler and flags +CPPFLAGS += -I$(INCLUDES_DIR) +CFLAGS += -g -Wall -Wextra +LDFLAGS += -z lazy + +# Logger object +LOGGER_DIR := $(UTILS_DIR)/log +LOGGER_OBJ = log.o +LOGGER = $(LOGGER_DIR)/$(LOGGER_OBJ) + +# Socket object +SOCKET_DIR := $(UTILS_DIR)/sock +SOCKET_OBJ = sock_util.o +SOCKET = $(SOCKET_DIR)/$(SOCKET_OBJ) + +# Source files and corresponding binaries +BINARIES = server mp_server mt_server +SRCS = $(BINARIES:=.c) +OBJS = $(SRCS:.c=.o) + +# Default rule: build all binaries +all: $(BINARIES) + +# Rule to compile the binaries +$(BINARIES): %: %.o $(LOGGER) $(SOCKET) + $(CC) $(LDFLAGS) $^ -o $@ + +# Rule to compile the logger object +$(LOGGER): $(LOGGER_DIR)/log.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Rule to compile the socket object +$(SOCKET): $(SOCKET_DIR)/sock_util.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@ + +# Clean rule: Remove object files and binaries +clean: + -rm -f $(OBJS) $(BINARIES) + -rm -f $(LOGGER) + +.PHONY: all clean diff --git a/content/chapters/io/lab/support/async/python/client.py b/chapters/io/optimizations/guides/async/support/client.py similarity index 100% rename from content/chapters/io/lab/support/async/python/client.py rename to chapters/io/optimizations/guides/async/support/client.py diff --git a/chapters/io/optimizations/guides/async/support/client_bench.sh b/chapters/io/optimizations/guides/async/support/client_bench.sh new file mode 100755 index 0000000000..d98aa79a05 --- /dev/null +++ b/chapters/io/optimizations/guides/async/support/client_bench.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +NUM_CLIENTS=8 + +for _ in $(seq 1 $NUM_CLIENTS); do + ./client.py localhost "$1" 30 & +done +wait diff --git a/content/chapters/io/lab/support/async/c/mp_server.c b/chapters/io/optimizations/guides/async/support/mp_server.c similarity index 98% rename from content/chapters/io/lab/support/async/c/mp_server.c rename to chapters/io/optimizations/guides/async/support/mp_server.c index dedfa945b5..98ce8e7fcd 100644 --- a/content/chapters/io/lab/support/async/c/mp_server.c +++ b/chapters/io/optimizations/guides/async/support/mp_server.c @@ -1,6 +1,4 @@ -/* - * SPDX-License-Identifier: BSD-3-Clause - */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/support/async/c/mt_server.c b/chapters/io/optimizations/guides/async/support/mt_server.c similarity index 98% rename from content/chapters/io/lab/support/async/c/mt_server.c rename to chapters/io/optimizations/guides/async/support/mt_server.c index 246d0a2233..4182953361 100644 --- a/content/chapters/io/lab/support/async/c/mt_server.c +++ b/chapters/io/optimizations/guides/async/support/mt_server.c @@ -1,6 +1,4 @@ -/* - * SPDX-License-Identifier: BSD-3-Clause - */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/content/chapters/io/lab/support/async/c/server.c b/chapters/io/optimizations/guides/async/support/server.c similarity index 98% rename from content/chapters/io/lab/support/async/c/server.c rename to chapters/io/optimizations/guides/async/support/server.c index 3c7e5708a8..3b06b13ddc 100644 --- a/content/chapters/io/lab/support/async/c/server.c +++ b/chapters/io/optimizations/guides/async/support/server.c @@ -1,6 +1,4 @@ -/* - * SPDX-License-Identifier: BSD-3-Clause - */ +// SPDX-License-Identifier: BSD-3-Clause #include #include diff --git a/chapters/io/optimizations/guides/async/support/utils/log/CPPLINT.cfg b/chapters/io/optimizations/guides/async/support/utils/log/CPPLINT.cfg new file mode 100644 index 0000000000..5aa9cb376c --- /dev/null +++ b/chapters/io/optimizations/guides/async/support/utils/log/CPPLINT.cfg @@ -0,0 +1 @@ +exclude_files=log\.c diff --git a/chapters/io/optimizations/guides/async/support/utils/log/log.c b/chapters/io/optimizations/guides/async/support/utils/log/log.c new file mode 100644 index 0000000000..ac65f4ed33 --- /dev/null +++ b/chapters/io/optimizations/guides/async/support/utils/log/log.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: BSD-3-Clause + +/* + * Copyright (c) 2020 rxi + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#include "log.h" + +#define MAX_CALLBACKS 32 + +typedef struct { + log_LogFn fn; + void *udata; + int level; +} Callback; + +static struct +{ + void *udata; + log_LockFn lock; + int level; + bool quiet; + Callback callbacks[MAX_CALLBACKS]; +} L; + +static const char * const level_strings[] = { "TRACE", "DEBUG", "INFO", + "WARN", "ERROR", "FATAL" }; + +#ifdef LOG_USE_COLOR +static const char * const level_colors[] = { "\x1b[94m", "\x1b[36m", "\x1b[32m", + "\x1b[33m", "\x1b[31m", "\x1b[35m" }; +#endif + +static void +stdout_callback(log_Event *ev) +{ + char buf[16]; + + buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; +#ifdef LOG_USE_COLOR + fprintf(ev->udata, + "%s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", + buf, + level_colors[ev->level], + level_strings[ev->level], + ev->file, + ev->line); +#else + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); +#endif + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +file_callback(log_Event *ev) +{ + char buf[64]; + + buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; + fprintf(ev->udata, + "%s %-5s %s:%d: ", + buf, + level_strings[ev->level], + ev->file, + ev->line); + vfprintf(ev->udata, ev->fmt, ev->ap); + fprintf(ev->udata, "\n"); + fflush(ev->udata); +} + +static void +lock(void) +{ + if (L.lock) + L.lock(true, L.udata); +} + +static void +unlock(void) +{ + if (L.lock) + L.lock(false, L.udata); +} + +const char* +log_level_string(int level) +{ + return level_strings[level]; +} + +void +log_set_lock(log_LockFn fn, void *udata) +{ + L.lock = fn; + L.udata = udata; +} + +void +log_set_level(int level) +{ + L.level = level; +} + +void +log_set_quiet(bool enable) +{ + L.quiet = enable; +} + +int +log_add_callback(log_LogFn fn, void *udata, int level) +{ + for (int i = 0; i < MAX_CALLBACKS; i++) { + if (!L.callbacks[i].fn) { + L.callbacks[i] = (Callback) { fn, udata, level }; + return 0; + } + } + return -1; +} + +int +log_add_fp(FILE *fp, int level) +{ + return log_add_callback(file_callback, fp, level); +} + +static void +init_event(log_Event *ev, void *udata) +{ + if (!ev->time) { + time_t t = time(NULL); + + ev->time = localtime(&t); + } + ev->udata = udata; +} + +void +log_log(int level, const char *file, int line, const char *fmt, ...) +{ + log_Event ev = { + .fmt = fmt, + .file = file, + .line = line, + .level = level, + }; + + lock(); + + if (!L.quiet && level >= L.level) { + init_event(&ev, stderr); + va_start(ev.ap, fmt); + stdout_callback(&ev); + va_end(ev.ap); + } + + for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { + Callback *cb = &L.callbacks[i]; + + if (level >= cb->level) { + init_event(&ev, cb->udata); + va_start(ev.ap, fmt); + cb->fn(&ev); + va_end(ev.ap); + } + } + + unlock(); +} diff --git a/chapters/io/optimizations/guides/async/support/utils/log/log.h b/chapters/io/optimizations/guides/async/support/utils/log/log.h new file mode 100644 index 0000000000..7acb55aab6 --- /dev/null +++ b/chapters/io/optimizations/guides/async/support/utils/log/log.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +/** + * Copyright (c) 2020 rxi + * + * This library is free software; you can redistribute it and/or modify it + * under the terms of the MIT license. See `log.c` for details. + */ + +/* Github link: https://github.com/rxi/log.c */ + +#ifndef LOG_H +#define LOG_H + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + +#define LOG_VERSION "0.1.0" + +typedef struct { + va_list ap; + const char *fmt; + const char *file; + struct tm *time; + void *udata; + int line; + int level; +} log_Event; + +typedef void (*log_LogFn)(log_Event *ev); +typedef void (*log_LockFn)(bool lock, void *udata); + +enum { + LOG_TRACE, + LOG_DEBUG, + LOG_INFO, + LOG_WARN, + LOG_ERROR, + LOG_FATAL +}; + +#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) +#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) +#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) +#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) +#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) +#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) + +const char *log_level_string(int level); +void log_set_lock(log_LockFn fn, void *udata); +void log_set_level(int level); +void log_set_quiet(bool enable); +int log_add_callback(log_LogFn fn, void *udata, int level); +int log_add_fp(FILE *fp, int level); + +void log_log(int level, const char *file, int line, const char *fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* LOG_H */ diff --git a/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/sock/sock_util.c b/chapters/io/optimizations/guides/async/support/utils/sock/sock_util.c similarity index 100% rename from chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/sock/sock_util.c rename to chapters/io/optimizations/guides/async/support/utils/sock/sock_util.c diff --git a/chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/sock/sock_util.h b/chapters/io/optimizations/guides/async/support/utils/sock/sock_util.h similarity index 100% rename from chapters/compute/copy-on-write/drills/tasks/page-faults/support/utils/sock/sock_util.h rename to chapters/io/optimizations/guides/async/support/utils/sock/sock_util.h diff --git a/chapters/io/optimizations/guides/async/support/utils/utils.h b/chapters/io/optimizations/guides/async/support/utils/utils.h new file mode 100644 index 0000000000..efdf6b59dd --- /dev/null +++ b/chapters/io/optimizations/guides/async/support/utils/utils.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ + +#ifndef UTILS_H_ +#define UTILS_H_ 1 + +#include +#include +#include +#include +#include "log/log.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ERR(assertion, call_description) \ + do { \ + if (assertion) \ + log_error("%s: %s", \ + call_description, strerror(errno)); \ + } while (0) + +#define DIE(assertion, call_description) \ + do { \ + if (assertion) { \ + log_fatal("%s: %s", \ + call_description, strerror(errno)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +#ifdef __cplusplus +} +#endif + +#endif /* UTILS_H_ */ diff --git a/chapters/io/optimizations/guides/benchmarking-sendfile/README.md b/chapters/io/optimizations/guides/benchmarking-sendfile/README.md new file mode 100644 index 0000000000..8f38e483cc --- /dev/null +++ b/chapters/io/optimizations/guides/benchmarking-sendfile/README.md @@ -0,0 +1,27 @@ +# Benchmarking `sendfile()` + +Navigate to `chapters/io/optimizations/guides/benchmarking-sendfile/support` and run `make` to prepare the test file. + +`sendfile()` transfers data between two file descriptors directly within the kernel, bypassing user-space buffers. +This process is known as [zero-copy](../../reading/zero-copy.md). +Having established that `sendfile()` is likely faster than traditional I/O operations, it's time to put this theory to the test! + +The code in `server.py` creates two threads that behave nearly identically. +One listens on port `8081` and handles connections using `read()` and `send()`, while the other listens on port `8082` and uses `sendfile()`. +Start the server in one terminal with `python server.py`. +In a second terminal, run `benchmark_client.py read-send` followed by `benchmark_client.py sendfile` to compare the performance. + +The results below are generic, and your outcomes may vary significantly depending on factors such as disk performance, network interface card (NIC), kernel version, Python version, and system load. + +```console +student@os:/.../benchmarking-sendfile/support$ python3 benchmark_client.py read-send +Time taken: 7.175773588009179 seconds + +student@os:/.../benchmarking-sendfile/support$ python3 benchmark_client.py sendfile +Time taken: 3.71454380400246 seconds +``` + +Using `sendfile()` **halves the number of copies** required, reducing it from 4 to 2. +This should translate to a roughly halved running time as well, making `sendfile()` a clear performance improvement over traditional methods. + +You can explore another example of **zero-copy** in practice in this [Quiz: Why is `cp` faster than `mmap()`-based `cp`](../../../file-descriptors/drills/questions/mmap-read-write-benchmark.md). diff --git a/content/chapters/io/lab/support/zero-copy/.gitignore b/chapters/io/optimizations/guides/benchmarking-sendfile/support/.gitignore similarity index 100% rename from content/chapters/io/lab/support/zero-copy/.gitignore rename to chapters/io/optimizations/guides/benchmarking-sendfile/support/.gitignore diff --git a/content/chapters/io/lab/support/zero-copy/Makefile b/chapters/io/optimizations/guides/benchmarking-sendfile/support/Makefile similarity index 100% rename from content/chapters/io/lab/support/zero-copy/Makefile rename to chapters/io/optimizations/guides/benchmarking-sendfile/support/Makefile diff --git a/content/chapters/io/lab/support/zero-copy/benchmark_client.py b/chapters/io/optimizations/guides/benchmarking-sendfile/support/benchmark_client.py similarity index 100% rename from content/chapters/io/lab/support/zero-copy/benchmark_client.py rename to chapters/io/optimizations/guides/benchmarking-sendfile/support/benchmark_client.py diff --git a/content/chapters/io/lab/support/zero-copy/server.py b/chapters/io/optimizations/guides/benchmarking-sendfile/support/server.py similarity index 100% rename from content/chapters/io/lab/support/zero-copy/server.py rename to chapters/io/optimizations/guides/benchmarking-sendfile/support/server.py diff --git a/content/chapters/io/lab/support/file-mappings/Makefile b/chapters/io/optimizations/guides/kernel-caching/Makefile similarity index 56% rename from content/chapters/io/lab/support/file-mappings/Makefile rename to chapters/io/optimizations/guides/kernel-caching/Makefile index 9ef0728333..30d1131313 100644 --- a/content/chapters/io/lab/support/file-mappings/Makefile +++ b/chapters/io/optimizations/guides/kernel-caching/Makefile @@ -1,11 +1,10 @@ -BINARY = mmap_cp -include ../../../../../common/makefile/single.mk - -large-file: +large-file.txt: dd if=/dev/urandom of=large-file.txt bs=1024 count=1M -test-file: +test-file.txt: dd if=/dev/urandom of=test-file.txt bs=1024 count=1K -clean:: +clean: -rm -f *.txt + +.PHONY: clean diff --git a/chapters/io/optimizations/guides/kernel-caching/README.md b/chapters/io/optimizations/guides/kernel-caching/README.md new file mode 100644 index 0000000000..bcd8b2169d --- /dev/null +++ b/chapters/io/optimizations/guides/kernel-caching/README.md @@ -0,0 +1,61 @@ +# Kernel Caching + +I/O is critical to system efficiency, but also often its weakest link. +Techniques to improve I/O performance include [buffering](../../../file-descriptors/guides/libc-FILE-struct/), [zero-copy](../../../optimizations/reading/zero-copy.md), and [async I/O](../../../optimizations/reading/async-io.md). +Among these, buffering is the most common and powerful. + +Remember `buffering/support/benchmark_buffering.sh` or `file-mappings/support/benchmark_cp.sh`. +They both used this line: + +```bash +sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches" +``` + +The line invalidates caches, forcing the OS to perform I/O operations directly from the disk. +This ensures that the scripts benchmark the C code's performance alone, without any speedup from cached data. + +The kernel goes even further with **buffering**. +This time, it’s at a level beyond syscalls like `read()` and `write()`, using a strategy known as **caching**. +While buffering helps with handling the **next data** efficiently by reading in advance or delaying writes, caching is about speeding up repeated access to the **same data**. +Just as your browser caches frequently visited pages or your CPU caches recent addresses, the OS caches files that are accessed often, such as logs or configuration files. + +When the OS encounters a file access, it stores portions of that file in memory so that subsequent requests can read or modify data from RAM rather than waiting on the slower disk. +This makes I/O faster. + +## Caching in action + +Navigate to `chapters/io/optimizations/guides/kernel-caching/support` and run `make` to create a large file that we'll use for benchmarking. +We have two scripts to benchmark the `cp` command with and without caching: + +```console +student@os:/.../kernel-caching/support$ ./benchmark_cp.sh +make: 'large-file.txt' is up to date. +Benchmarking cp on a 1 GB file... + +real 0m1.473s +user 0m0.001s +sys 0m0.985s + +student@os:/.../kernel-caching/support$ ./benchmark_cp_allow_caching.sh +make: 'large-file.txt' is up to date. +Benchmarking cp on a 1 GB file... + +real 0m0.813s +user 0m0.000s +sys 0m0.837s +``` + +Each subsequent benchmark actually reads the data from the caches populated or refreshed by the previous one. +So running the script multiple times might improve the results. + +You can use `free -h` to view how much data your kernel is caching. +Look at the `buff/cache` column. +One possible output is shown below. +It says the OS is caching 7 GB of data. + +```console +student@os:~$ free -h + total used free shared buff/cache available +Mem: 15Gi 8,1Gi 503Mi 691Mi 7,0Gi 6,5Gi +Swap: 7,6Gi 234Mi 7,4Gi +``` diff --git a/chapters/io/optimizations/guides/kernel-caching/benchmark_cp.sh b/chapters/io/optimizations/guides/kernel-caching/benchmark_cp.sh new file mode 100755 index 0000000000..b3aea9b7f8 --- /dev/null +++ b/chapters/io/optimizations/guides/kernel-caching/benchmark_cp.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +[ -f large-file.txt ] || make large-file + +# Drop all OS caches: buffer cache, dentry cache, inode cache, page cache. +sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches" + +echo "Benchmarking cp on a 1 GB file..." +time cp large-file.txt large-file-cp.txt diff --git a/chapters/io/optimizations/guides/kernel-caching/benchmark_cp_allow_caching.sh b/chapters/io/optimizations/guides/kernel-caching/benchmark_cp_allow_caching.sh new file mode 100755 index 0000000000..5d777e8073 --- /dev/null +++ b/chapters/io/optimizations/guides/kernel-caching/benchmark_cp_allow_caching.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause + +[ -f large-file.txt ] || make large-file + +# Drop all OS caches: buffer cache, dentry cache, inode cache, page cache. +# sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches" + +echo "Benchmarking cp on a 1 GB file..." +time cp large-file.txt large-file-cp.txt diff --git a/content/chapters/io/lab/media/client-server-file.svg b/chapters/io/optimizations/media/client-server-file.svg similarity index 100% rename from content/chapters/io/lab/media/client-server-file.svg rename to chapters/io/optimizations/media/client-server-file.svg diff --git a/content/chapters/io/lab/media/server-copies-normal.svg b/chapters/io/optimizations/media/server-copies-normal.svg similarity index 100% rename from content/chapters/io/lab/media/server-copies-normal.svg rename to chapters/io/optimizations/media/server-copies-normal.svg diff --git a/content/chapters/io/lab/media/server-copies-zero-copy.svg b/chapters/io/optimizations/media/server-copies-zero-copy.svg similarity index 100% rename from content/chapters/io/lab/media/server-copies-zero-copy.svg rename to chapters/io/optimizations/media/server-copies-zero-copy.svg diff --git a/chapters/io/optimizations/reading/async-io.md b/chapters/io/optimizations/reading/async-io.md new file mode 100644 index 0000000000..be28aaa3b8 --- /dev/null +++ b/chapters/io/optimizations/reading/async-io.md @@ -0,0 +1,78 @@ +# Asynchronous I/O + +Asynchronous I/O (async I/O) provides an efficient way to handle input/output operations that are typically slower than CPU operations by allowing programs to continue executing while waiting for I/O operations to complete. +Here’s a breakdown of I/O operation types and how asynchronous I/O compares to synchronous operations: + +1. **Synchronous Blocking Operation**: + - This is the simplest and most common form of I/O (e.g., `read()` and `write()`). + - It’s **synchronous**, meaning the program waits for a response to proceed. + - It’s **blocking**, so if the requested data isn’t available (e.g., no data in the buffer for `read()`), the program waits for the operation to finish. + +1. **Synchronous Non-blocking Operation**: + - This gives more control, especially in situations where waiting isn’t ideal. + - Opening a file using `open()` alongside `O_NONBLOCK` flag ensures the operation returns immediately instead of blocking. + - If data isn’t available right away, the operation notifies the program, which can try again later, avoiding unnecessary waiting. + +1. **Asynchronous Operation**: + - Here, the function call returns immediately, allowing the program to continue **without waiting for the result**. + - A notification or callback is sent when the operation completes, or the program can check its status periodically. + +Keep in mind the async I/O is not the same thing as I/O multiplexing. +While both techniques improve I/O efficiency, they’re conceptually different: + +- **Asynchronous I/O** schedules operations concurrently, allowing the program to proceed without blocking. +- [**I/O Multiplexing**](../reading/io-multiplexing.md) (e.g., `select()`, `poll()`, `epoll()`) monitors multiple channels simultaneously and informs the program when a channel has data ready. + Blocking could still occur if the program cannot proceed without data from a channel. + +Think of them as **complementary**: multiplexing helps monitor multiple channels, while async I/O allows the program to do other things while waiting. + +There are several ways asynchronous I/O can be implemented in practice: + +1. **Multiprocess Backend**: Each request runs in a **separate process**, isolating tasks and preventing blocking between them. + + ```c + // Handle requests with processes + void handle_client_proc(int client_sockfd) { + pid_t pid = fork(); + if (pid < 0) // handle error + + if (pid == 0) { // Child process: handle client connection + close(server_sockfd); // Close the server socket in the child + + {...} // compute and send answer + + close(client_sockfd); // Close client socket when done + exit(0); // Exit child process + } + + close(client_sockfd); // close client socket in parent + } + ``` + +1. **Multithreaded Backend**: Each request runs in a **separate thread**, allowing concurrent operations within the same process. + + ```c + // Handle requests with threads + void* handler(void* arg) { + int client_sockfd = *(int*)arg; + + {...} // compute and send answer + close(client_sockfd); // Close client socket when done + + return NULL; + } + + void handle_client_thread(int sockfd) { + int *sockfd_p = malloc(sizeof(int)); // use the heap to pass the address + *sockfd_p = sockfd; + + int rc = pthread_create(&thread_id, NULL, handler, client_sock_ptr); + if (rc < 0) // handle error + pthread_detach(thread_id); // Let the thread clean up after itself + } + ``` + +1. **Event-based Backend**: An action is scheduled with a **callback**, which is invoked upon completion, using event loops to manage tasks. + A callback is simply a function pointer, allowing the system to execute the function later or when a specific event is triggered. + +Test your understanding by solving the [Async Server task](../drills/tasks/async-server/README.md). diff --git a/chapters/io/optimizations/reading/io-multiplexing.md b/chapters/io/optimizations/reading/io-multiplexing.md new file mode 100644 index 0000000000..a45aaa89e5 --- /dev/null +++ b/chapters/io/optimizations/reading/io-multiplexing.md @@ -0,0 +1,69 @@ +# I/O Multiplexing + +I/O multiplexing is the ability to serve multiple I/O channels (or anything that can be referenced via a file descriptor / handle) simultaneously. +If a given application, such a server, has multiple sockets on which it serves connection, it may be the case that operating on one socket blocks the server. +One solution is using asynchronous operations, with different backends. +The other solution is using I/O multiplexing. + +The classical functions for I/O multiplexing are [`select`](https://man7.org/linux/man-pages/man2/select.2.html) and [`poll`](https://man7.org/linux/man-pages/man2/poll.2.html). +Due to several limitations, modern operating systems provide advanced (non-portable) variants to these: + +* Windows provides [I/O completion ports (`IOCP`)](https://learn.microsoft.com/en-us/windows/win32/fileio/i-o-completion-ports). +* BSD provides [`kqueue`](https://www.freebsd.org/cgi/man.cgi?kqueue). +* Linux provides [`epoll()`](https://man7.org/linux/man-pages/man7/epoll.7.html). + +> **Note** that I/O multiplexing is orthogonal to [asynchronous I/O](../reading/async-io.md). + You could tie them together if the completion of the asynchronous operation sends a notification that can be handled via a file descriptor / handle. + This is the case with Windows asynchronous I/O (called [overlapped I/O](https://learn.microsoft.com/en-us/windows/win32/fileio/synchronous-and-asynchronous-i-o)). + +## The `epoll` API + +The `epoll` API allows user-space programs to efficiently monitor multiple file descriptors and be notified when one of them has data to read. +It provides a powerful, event-driven interface concentrated in three primary functions: + +* `int epoll_create1(int flags)`: Creates an `epoll` instance. + The `flags` argument specifies additional options for the instance. + The default value is `0`. +* `int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout)`: Waits for events on the monitored file descriptors. + * `epfd`: The file descriptor returned by `epoll_create1()`. + * `events`: An array of `struct epoll_event` that will store the events that have occurred. + It only contains events that are ready (i.e., received data). + * `maxevents`: The maximum number of events that can be stored in the `events` array. + * `timeout`: The maximum time (in milliseconds) that `epoll_wait()` will block. + A value of `-1` means it will block indefinitely. +* `int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)`: Modifies the set of file descriptors monitored by `epoll`. + * `epfd`: The file descriptor returned by `epoll_create1()`. + * The `op` argument specifies the operation to perform, which can be: + * `EPOLL_CTL_ADD`: Adds a file descriptor to the monitoring list. + * `EPOLL_CTL_MOD`: Modifies an existing file descriptor’s event list. + * `EPOLL_CTL_DEL`: Removes a file descriptor from the monitoring list. + * The `fd` argument is the file descriptor to be added, modified, or removed. + * The `event` argument is a pointer to a `struct epoll_event` that defines the events associated with the file descriptor. + +The [`struct epoll_event`](https://man7.org/linux/man-pages/man3/epoll_event.3type.html) is the core structure used to interact with `epoll`. +It is used to return events to user space after `epoll_wait()` is called and to pass parameters to `epoll_ctl()` when modifying the set of monitored file descriptors. +While the internal workings of `epoll` are complex, understanding how to use these functions and structures will cover most use cases. + +Here is an example demonstrating how to use the `epoll` interface: + +```c +efd = epoll_create1(0) +if (efd < 0) {...} // handle error + +// Add fd to monitored set +struct epoll_event ev; +ev.events = EPOLLIN; // monitor fd for reading +ev.data.fd = fd; +rc = epoll_ctl(efd, EPOLL_CTL_ADD, fd, &ev); +if (rc < 0) {...} // handle error + +struct epoll_event events[10]; +n = epoll_wait(efd, events, 10, -1); // Wait indefinitely +if (n < 0) {...} // handle error + +// Iterate through the events to get active file descriptors +for (int i = 0; i < n; i++) + printf("%d received data\n", events[i].data.fd); +``` + +Test your `epoll` understanding by implementing [I/O multiplexing in a client-server app](../drills/tasks/multiplexed-client-server/README.md). diff --git a/chapters/io/optimizations/reading/zero-copy.md b/chapters/io/optimizations/reading/zero-copy.md new file mode 100644 index 0000000000..221e6a39bc --- /dev/null +++ b/chapters/io/optimizations/reading/zero-copy.md @@ -0,0 +1,58 @@ +# Zero-Copy + +Imagine a server that responds with files that it stores locally. +Its actions would be those highlighted in the image below: + +1. Receive a new request and extract the filename +1. Read the filename from the disk into memory +1. Send the file from memory to the client + +![Client-Server Steps](../media/client-server-file.svg) + +[Quiz: How many copies does the OS make?](../drills/questions/server-copies.md) + +As you might have guessed, 2 of these copies are useless. +Since the app doesn't modify the file, there's no need for it to store the file in its own buffer. +It would be more efficient to use **a single** kernel buffer as intermediate storage between the disk and the NIC (Network Interface Card), as shown in the image below. + +![Server Copies - Zero-Copy](../media/server-copies-zero-copy.svg) + +For an easier comparison with the "classic" `read()` + `send()` model, here's the first version again: + +![Server Copies - Read-Send](../media/server-copies-normal.svg) + +It should be obvious that the former approach is more efficient than the latter. + +[Quiz: Almost zero copies](../drills/questions/fewer-than-2-copies.md) + +These diagrams capture the essence of **zero-copy**: transferring data directly between kernel buffers, avoiding intermediate user-space buffers. +This approach is ideal for serving requests, whether forwarding data between clients or reading from disk. +It relies on the OS to retrieve and send data efficiently without extra copying steps. + +## `sendfile()` + +The syscall with which we can leverage **zero-copy** is called [`sendfile()`](https://man7.org/linux/man-pages/man2/sendfile.2.html). +Here are some practical examples on how to use it: + +```c +// file to file +int in_fd = open("input_file.txt", O_RDONLY); // src +int out_fd = open("output_socket", O_WRONLY); // dst + +ssize_t bytes_sent = sendfile(out_fd, in_fd, &offset, 4096); // Transfer 4096 bytes +if (bytes_sent < 0) {...} // handle error +``` + +```c +// file to network +int in_fd = open("input_file.txt", O_RDONLY); // src +int sockfd = socket(AF_INET, SOCK_STREAM, 0); // dst + +int rc = connect(sock_fd, &server_addr, sizeof(server_addr)); +if (rc < 0) {...} // handle error + +ssize_t bytes_sent = sendfile(out_fd, in_fd, &offset, 4096); // Transfer 4096 bytes +if (bytes_sent < 0) {...} // handle error +``` + +You can read a slightly more detailed article about zero-copy [here](https://developer.ibm.com/articles/j-zerocopy/). diff --git a/content/chapters/io/lab/media/compute-system-oversimplified.svg b/chapters/io/overview/media/compute-system-oversimplified.svg similarity index 100% rename from content/chapters/io/lab/media/compute-system-oversimplified.svg rename to chapters/io/overview/media/compute-system-oversimplified.svg diff --git a/content/chapters/io/lab/media/memory-hierarchy.svg b/chapters/io/overview/media/memory-hierarchy.svg similarity index 100% rename from content/chapters/io/lab/media/memory-hierarchy.svg rename to chapters/io/overview/media/memory-hierarchy.svg diff --git a/content/chapters/io/lab/media/sad-pepe.png b/chapters/io/overview/media/sad-pepe.png similarity index 100% rename from content/chapters/io/lab/media/sad-pepe.png rename to chapters/io/overview/media/sad-pepe.png diff --git a/chapters/io/overview/reading/overview.md b/chapters/io/overview/reading/overview.md new file mode 100644 index 0000000000..c67efa8a97 --- /dev/null +++ b/chapters/io/overview/reading/overview.md @@ -0,0 +1,31 @@ +# I/O + +We know that a compute system is a collection of hardware and software that modifies data. +This data has to come from _somewhere_. +This _somewhere_ is always outside the compute system: files, network packets, radio signals, sensor data. +We'll cover interacting with all of communication channels when discussing **the Linux file interface**. + +![Compute System - Oversimplified](../media/compute-system-oversimplified.svg) + +A compute system without output is essentially useless. +While there are many ways to distribute work across processes and threads, the real benefit comes from efficiently passing data between them. +This is where **Inter-Process Communication** (IPC) methods like pipes, sockets, and file mappings come into play, which we'll explore in detail in the second part. + +You saw this hierarchy during the Data lecture: + +![Memory Hierarchy](../media/memory-hierarchy.svg) + +It says that while the disk can store lots of data, it does so at the cost of speed. +When we say speed, we mean the rate at which we can read/write data from/to the disk, i.e. the maximum number of bytes transferred per unit of time. +This means that `read()` and `write()` syscalls (or their various corresponding library calls) are slow and cause performance bottlenecks. +More often than not, it is the I/O component that drags down the performance of an otherwise fast application. +And what's worse, the further the "destination" of the I/O operation (file on the disk or host on the Web) is, the more time it takes to transfer data to and from it. + +On the other hand, as we've already established, the I/O component defines how we interact with an app. +If we want it to be responsive and to do something useful, most likely, the I/O is the key. +So I/O is crucial for most applications, yet it is also the slowest... + +![Sad Pepe](../media/sad-pepe.png) + +But fear not! +There are countless optimisations out there aimed precisely at bridging the speed gap between the memory and the disk. diff --git a/content/chapters/io/lecture/slides.mdpp b/chapters/io/slides.mdpp similarity index 100% rename from content/chapters/io/lecture/slides.mdpp rename to chapters/io/slides.mdpp diff --git a/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.c b/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.h b/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/chapters/software-stack/libc/drills/tasks/common-functions/solution/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/chapters/software-stack/libc/drills/tasks/libc/README.md b/chapters/software-stack/libc/drills/tasks/libc/README.md index ac3f08e7c4..6c62cd6d77 100644 --- a/chapters/software-stack/libc/drills/tasks/libc/README.md +++ b/chapters/software-stack/libc/drills/tasks/libc/README.md @@ -13,7 +13,7 @@ Now go through the practice items below. Use `strace` to check the system calls invoked by `malloc()` and `free()`. You'll see that, depending on the size, the `brk()` or `mmap()` / `munmap()` system calls are invoked. And for certain calls to `malloc()` / `free()` no syscall is happening. - You'll find more about them in the [Data chapter](../../../data/lab). + You'll find more about them in the [Data chapter](../../../../../data/working-with-memory/reading/working-with-memory.md). [Quiz](../../questions/malloc.md) diff --git a/chapters/software-stack/overview/reading/overview.md b/chapters/software-stack/overview/reading/overview.md index 66b9e565cc..1ce2a80fc6 100644 --- a/chapters/software-stack/overview/reading/overview.md +++ b/chapters/software-stack/overview/reading/overview.md @@ -39,8 +39,6 @@ The last two characteristics rely on two items: ![Software Stack](../media/software-stack.svg) -[Quiz](../drills/software.md) - ## Contents - [Modern Software Stacks](../../modern-software-stacks/reading/modern-sw-stack.md) diff --git a/config.yaml b/config.yaml index 11027fbb80..ee54bdb7a0 100644 --- a/config.yaml +++ b/config.yaml @@ -113,6 +113,44 @@ lab_structure: - reading/scheduling.md - guides/user-level-threads.md - guides/libult.md + - title: Lab 9 - File Descriptors + filename: lab9.md + content: + - tasks/my-cat.md + - tasks/mmap_cp.md + - tasks/anon-pipes.md + - reading/file-descriptors.md + - reading/fd-operations.md + - reading/pipes.md + - guides/simple-file-operations.md + - guides/redirections.md + - guides/fd-table.md + - guides/libc-FILE-struct.md + - guides/file-mappings.md + - guides/reading-linux-dirs.md + - title: Lab 10 - Inter-Process Communication + filename: lab10.md + content: + - tasks/named-pipes.md + - tasks/unix-socket.md + - tasks/network-socket.md + - tasks/receive-challenges.md + - reading/unix-sockets.md + - reading/network-sockets.md + - reading/client-server-model.md + - guides/networking-101.md + - title: Lab 11 - IO Optimizations + filename: lab11.md + content: + - tasks/client-server.md + - tasks/multiplexed-client-server.md + - tasks/async-server.md + - reading/io-multiplexing.md + - reading/async-io.md + - reading/zero-copy.md + - guides/benchmarking-sendfile.md + - guides/kernel-caching.md + - guides/async.md make_assets: @@ -123,7 +161,7 @@ make_assets: - chapters/software-stack - chapters/data - chapters/compute - - content/chapters/io/lecture + - chapters/io - content/chapters/app-interact/lecture args: - all @@ -257,33 +295,41 @@ docusaurus: - Lab 6 - Multiprocess and Multithread: lab6.md - Lab 7 - Copy-on-Write: lab7.md - Lab 8 - Syncronization: lab8.md + - IO: + path: /build/prepare_view/.view + extra: + - media + subsections: + - Slides: /build/embed_reveal/IO/IO.mdx + - Overview: /build/prepare_view/.view/io-overview.md + - Questions: + path: questions + subsections: + - Bind Error Cause: bind-error-cause.md + - Anonymous Pipes Limitation: anonymous-pipes-limitation.md + - Pipe Ends: pipe-ends.md + - Receiver Socket FD: receiver-socket-fd.md + - Client Server Sender Receiver: client-server-sender-receiver.md + - Firefox TCP UDP: firefox-tcp-udp.md + - Server Copies: server-copies.md + - Fewer Than 2 Copies: fewer-than-2-copies.md + - Strace Printf: strace-printf.md + - Mmap Read Write Benchmark: mmap-read-write-benchmark.md + - Syscalls CP: syscalls-cp.md + - File Handler C: file-handler-c.md + - Flush Libc Buffer: flush-libc-buffer.md + - Stderr FD: stderr-fd.md + - Local IO Errors: local-io-errors.md + - Fopen Syscall: fopen-syscall.md + - Lab 9 - File Descriptors: lab9.md + - Lab 10 - Inter-Process Communication: lab10.md + - Lab 11 - IO Optimizations: lab11.md - Lecture: path: /build/embed_reveal subsections: - IO: IO/IO.mdx - Application Interaction: Application-Interaction/Application-Interaction.mdx # - Lab: - # - IO: - # path: content/chapters/io/lab/content - # extra: - # - ../media - # - ../quiz - # subsections: - # - Overview: overview.md - # - File Handlers: file-handlers.md - # - File Descriptors: file-descriptors.md - # - Redirections: redirections.md - # - Pipes: pipes.md - # - Local IO in Action: local-io-in-action.md - # - Remote IO: remote-io.md - # - Networking 101: networking-101.md - # - Client-Server Model: client-server-model.md - # - Beyond Network Sockets: beyond-network-sockets.md - # - File Mappings: file-mappings.md - # - IO Internals: io-internals.md - # - Zero-Copy: zero-copy.md - # - Asynchronous IO: async-io.md - # - IO Multiplexing: io-multiplexing.md # - Application Interaction: # path: content/chapters/app-interact/lab/content # extra: @@ -349,7 +395,7 @@ docusaurus: - slides/Compute: /build/make_assets/chapters/compute/_site - slides/Data: /build/make_assets/chapters/data/_site - slides/Software-Stack: /build/make_assets/chapters/software-stack/_site - - slides/IO: /build/make_assets/content/chapters/io/lecture/_site + - slides/IO: /build/make_assets/chapters/io/_site - slides/Application-Interaction: /build/make_assets/content/chapters/app-interact/lecture/_site config_meta: title: Operating Systems diff --git a/content/assignments/async-web-server/src/http-parser/http_parser.c b/content/assignments/async-web-server/src/http-parser/http_parser.c index 603db9e79f..645357a0ec 100644 --- a/content/assignments/async-web-server/src/http-parser/http_parser.c +++ b/content/assignments/async-web-server/src/http-parser/http_parser.c @@ -244,7 +244,7 @@ enum state , s_chunk_size , s_chunk_parameters , s_chunk_size_almost_done - + , s_headers_almost_done /* Important: 's_headers_almost_done' must be the last 'header' state. All * states beyond this must be 'body' states. It is used for overflow diff --git a/content/assignments/memory-allocator/README.md b/content/assignments/memory-allocator/README.md index 2af8d25c25..6167facbd4 100644 --- a/content/assignments/memory-allocator/README.md +++ b/content/assignments/memory-allocator/README.md @@ -102,7 +102,7 @@ The structure `block_meta` will be used to manage the metadata of a block. Each allocated zone will comprise of a `block_meta` structure placed at the start, followed by data (**payload**). For all functions, the returned address will be that of the **payload** (not of the `block_meta` structure). -```C +```c struct block_meta { size_t size; int status; diff --git a/content/chapters/app-interact/lab/solution/password-cracker/password-cracker-multiprocess.c b/content/chapters/app-interact/lab/solution/password-cracker/password-cracker-multiprocess.c index e187e22a22..505abb913e 100644 --- a/content/chapters/app-interact/lab/solution/password-cracker/password-cracker-multiprocess.c +++ b/content/chapters/app-interact/lab/solution/password-cracker/password-cracker-multiprocess.c @@ -158,7 +158,7 @@ void create_workers(int *request_pipefd, int *result_pipefd) } } -int main() +int main(void) { int i; int request_pipefd[NUM_WORKERS]; diff --git a/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multiprocess.c b/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multiprocess.c index 187456f744..0c6b11a223 100644 --- a/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multiprocess.c +++ b/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multiprocess.c @@ -156,7 +156,7 @@ void create_workers(int *request_pipefd, int *result_pipefd) } } -int main() +int main(void) { int i; int request_pipefd[NUM_WORKERS]; diff --git a/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multithread.c b/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multithread.c index 6dc11f90ee..9da5eac05c 100644 --- a/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multithread.c +++ b/content/chapters/app-interact/lab/support/password-cracker/password-cracker-multithread.c @@ -98,7 +98,7 @@ void create_workers(pthread_t *tids) } } -int main() +int main(void) { pthread_t tids[NUM_WORKERS]; int i; diff --git a/content/chapters/app-interact/lab/support/time-server/server.c b/content/chapters/app-interact/lab/support/time-server/server.c index 696f464cdf..a82a1b250b 100644 --- a/content/chapters/app-interact/lab/support/time-server/server.c +++ b/content/chapters/app-interact/lab/support/time-server/server.c @@ -82,7 +82,7 @@ int handle_client(int sockfd) return 1; } -int main() +int main(void) { int sockfd; diff --git a/content/chapters/io/.gitignore b/content/chapters/io/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/content/chapters/io/lab/content/arena.md b/content/chapters/io/lab/content/arena.md deleted file mode 100644 index 7f69aec7e0..0000000000 --- a/content/chapters/io/lab/content/arena.md +++ /dev/null @@ -1,144 +0,0 @@ -# Arena - -## Open File Structure in the Kernel - -The "open file" `struct` in the Linux kernel is called [`struct file`](https://elixir.bootlin.com/linux/v6.0.9/source/include/linux/fs.h#L940) -Its most important fields are: - -```c -struct file { - struct path f_path; - /* Identifier within the filesystem. */ - struct inode *f_inode; - - /** - * Contains pointers to functions that implement operations that - * correspond to syscalls, such as `read()`, `write()`, `lseek()` etc. - */ - const struct file_operations *f_op; - - /** - * Reference count. A `struct file` is deallocated when this reaches 0, - * i.e. nobody uses it anymore. - */ - atomic_long_t f_count; - - /* Those passed to `open()`. */ - unsigned int f_flags; - fmode_t f_mode; - - /* Cursor from where reads/writes are made */ - loff_t f_pos; - /* To allow `f_pos` to be modified atomically. */ - struct mutex f_pos_lock; -} -``` - -As mentioned above, [`struct file_operations`](https://elixir.bootlin.com/linux/v6.0.9/source/include/linux/fs.h#L2093) contains function pointers that well-known syscalls such as `read()` end up calling. -Each filesystem needs to define its own implementations of these functions. -Some of the most widely known `file_operations` are listed below. -By now, you should recognise most of them: - -```c -struct file_operations { - loff_t (*llseek) (struct file *, loff_t, int); - ssize_t (*read) (struct file *, char __user *, size_t, loff_t *); - ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *); - int (*mmap) (struct file *, struct vm_area_struct *); - unsigned long mmap_supported_flags; - int (*open) (struct inode *, struct file *); - int (*flush) (struct file *, fl_owner_t id); - int (*fsync) (struct file *, loff_t, loff_t, int datasync); - int (*fasync) (int, struct file *, int); -} -``` - -## Mini-shell with Blackjack and Pipes - -OK, there will be no Blackjack... -for now at least. -But there **will** be pipes. -Navigate back to `support/mini-shell/mini_shell.c` and add support for piping 2 commands together like this: - -```console -> cat bosses.txt | head -n 5 -Darkeater Midir -Slave Knight Gael -Nameless King -Dancer Of The Boreal Valley -Ancient Dragon -``` - -## To Drop or Not to Drop? - -Remember `support/buffering/benchmark_buffering.sh` or `support/file-mappings/benchmark_cp.sh`. -They both used this line: - -```bash -sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches" -``` - -Note that `sync` has a [man page](https://linux.die.net/man/8/sync) and it partially explains what's going on: - -> The kernel keeps data in memory to avoid doing (relatively slow) disk reads and writes. This improves performance - -So the kernel does **even more [buffering](./io-internals.md#io-buffering)!** -But this time, it's not at the syscall level, like with `read()` and `write()`. -And it's used a bit differently. - -While buffering is a means of either receiving data in advance (for reading) or committing it retroactively (for writing) to speed up subsequent syscalls that use the **next data**, caching is a means of speeding up calls that use the **same data**. -Just like your browser caches the pages you visit so you refresh them faster or your CPU caches your most recently accessed addresses, so does your OS **with your files**. - -Some files are read more often than others: logs, some configs etc. -Upon encountering a first request (read / write) to a file, the kernel stores chunks of them in its memory so that subsequent requests can receive / modify the data in the RAM rather than waiting for the slow disk. -This makes I/O faster. - -The line from which this discussion started invalidates those caches and forces the OS to perform I/O operations "the slow way" by interrogating the disk. -The scripts use it to benchmark only the C code, not the OS. - -To see just how much faster this type of caching is, navigate to `support/buffering/benchmark_buffering.sh` once again and comment-out the line with `sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches"`. -Now run the script **a few times** and compare the results. -You should see some drastic improvements in the running times, such as: - -```console -student@os:/.../support/file-mappings$ ./benchmark_cp.sh -make: Nothing to be done for 'all'. -Benchmarking mmap_cp on a 1 GB file... - -real 0m13,299s -user 0m0,522s -sys 0m1,695s -Benchmarking cp on a 1 GB file... - -real 0m10,921s -user 0m0,013s -sys 0m1,301s - - -student@os:/.../support/file-mappings$ ./benchmark_cp.sh -make: Nothing to be done for 'all'. -Benchmarking mmap_cp on a 1 GB file... - -real 0m1,286s -user 0m0,174s -sys 0m0,763s -Benchmarking cp on a 1 GB file... - -real 0m5,411s -user 0m0,012s -sys 0m1,201s -``` - -Each subsequent benchmark actually reads the data from the caches populated or refreshed by the previous one. - -You can use `free -h` to view how much data your kernel is caching. -Look at the `buff/cache` column. -One possible output is shown below. -It says the OS is caching 7 GB of data. - -```console -student@os:~$ free -h - total used free shared buff/cache available -Mem: 15Gi 8,1Gi 503Mi 691Mi 7,0Gi 6,5Gi -Swap: 7,6Gi 234Mi 7,4Gi -``` diff --git a/content/chapters/io/lab/content/async-io.md b/content/chapters/io/lab/content/async-io.md deleted file mode 100644 index 35660793a0..0000000000 --- a/content/chapters/io/lab/content/async-io.md +++ /dev/null @@ -1,134 +0,0 @@ -# Asynchronous I/O - -When doing I/O, the major issue we are facing is that I/O operations are typically much slower than CPU operations. -Because of that, in a typical synchronous scenario, a lot of time may be spent waiting for an I/O operation to be complete. - -Because of that, we have other types of I/O operations. - -The simplest type of I/O operating is a synchronous, blocking operation. -In this operation type, we call a given function (such as `read()` or `write()`) and wait until it completes. - -Another type of operation is a non-blocking operation. -In this type of operation, if data isn't available (in case of `read()`) or can not be sent (in case of `write()`), the given call will not block, it will simply return with a mark to try again later on. - -Another type of operation is an asynchronous operation. -In the case of an asynchronous operation, the given function returns immediately and the program continues its execution. -Once the operation succeeds, a notification may be sent out to the program. -Or the program can periodically check the state of the operation. - -Apart from these operating types, there is the option to do I/O multiplexing, i.e. the ability to tackle multiple channels simultaneously. -This is useful in case of servers, that get a large number of requests and have to iterate through them. - -In case of asynchronous I/O, the "backend" used to implement the operations may differ: - -1. It can be a multiprocess backend, where each action / request is passed to a given process. - -1. It can be a multithreaded backend, where each action / request is passed to a given thread. - -1. It can be an event-based backend, where an action is scheduled together with a callback that is invoked when an action ends. - -## Practice - -Enter the `support/async/` folder for some implementations of a simple request-reply server in Python or in C. -The server gets requests and serves them in different ways: synchronous, multiprocess-based, multi-threading-based, asynchronous. - -We use two implementations, in Python and in C. - -1. For the Python implementation, enter the `python/` subdirectory. - Take a look at the implementation of different servers: - - * synchronous server: `server.py` - * multiprocess backend: `mp_server.py` - * multithreaded backend: `mt_server.py` - * asynchronous API backend: `async_server.py` (requires Python >= 3.7) and `async_server_3.6.py` (works for Python 3.6) - - Let's do a benchmarking session of each server type. - For this we will: - - 1. Start the server on one console. - - 1. Use the `client_bench.sh` script to benchmark the server. - `client_bench.sh` is a simple script that starts multiple clients to connect to the servers, by running `client.py`. - It runs `NUM_CLIENT` instances of `client.py` to trigger actions in the remote server. - - **Note** that you may be required to run servers on different ports in case of an `Address already in use error`. - If that is the case, you will also need to run the `client_bench.sh` script. - - To start the server, run each of these commands (one at a time to test the respective server type): - - ```console - student@os:/.../support/async/python$ ./server.py 2999 - - student@os:/.../support/async/python$ ./mp_server.py 2999 - - student@os:/.../support/async/python$ ./mt_server.py 2999 - - student@os:/.../support/async/python$ ./async_server_3.6.py 2999 - ``` - - For each server, in a different console, we can test to see how well it behaves by running: - - ```console - student@os:/.../support/async/python$ time ./client_bench.sh - ``` - - You will see a time duration difference between `mp_server.py` and the others, `mp_server.py` runs requests faster. - This is because the multiprocess model works OK for a CPU-intensive server such as this. - - But not on threading, because threading suffers from the use of [GIL (_Global Interpreter Lock_)](https://realpython.com/python-gil/), that prevents multithreaded programs from running Python code simultaneously. - -1. For the C implementation, enter the `c/` subdirectory. - Take a look at the implementation of different servers: - - * synchronous server: `server.c` - * multiprocess backend: `mp_server.c` - * multithreaded backend: `mt_server.c` - - There is no asynchronous C variant, because of the unstable API of [`libaio`](https://pagure.io/libaio) and [`io_uring`](https://unixism.net/loti/what_is_io_uring.html). - - First, use `make` to build the servers. - Then, go through the same steps as above: - - 1. Start the server on one console. - - 1. Use the `client_bench.sh` script (in the `../python/` directory) to benchmark the server. - - Same as with Python, to start the server, run each of these commands (one at a time to test the respective server type): - - ```console - student@os:/.../support/async/c$ ./server 2999 - - student@os:/.../support/async/c$ ./mp_server 2999 - - student@os:/.../support/async/c$ ./mt_server 2999 - ``` - - For each server, in a different console, we can test to see how well it behaves by running: - - ```console - student@os:/.../support/async/python$ time client_bench.sh - ``` - - We draw 2 conclusions from using the C variant: - - 1. C is faster than Python - 1. The thread variant is indeed parallel, as now the GIL is no longer involved. - -## Remarks - -Asynchronous operations, as with others, provide an API, as is the case with the Python API or the [`libaio`](https://pagure.io/libaio) and [`io_uring`](https://unixism.net/loti/what_is_io_uring.html). -The backend of these operations may be a thread-based one in the library providing the API, or it may rely on support from the operating system. -When aiming for performance, asynchronous I/O operations are part of the game. -And it's very useful having a good understanding of what's happening behind the scenes. - -For example, for the Python `async_server_3.6.py` server, a message `asyncio: Using selector: EpollSelector` is provided. -This means that the backend relies on the use of the [`epoll()` function](https://man7.org/linux/man-pages/man7/epoll.7.html) that's part of the [I/O Multiplexing section](./io-multiplexing.md). - -Also, for Python, the use of the GIL may be an issue when the operations are CPU intensive. - -This last point deserves a discussion. -It's rare that servers or programs using asynchronous operations are CPU intensive. -It's more likely that they are I/O intensive, and the challenge is avoiding blocking points in multiple I/O channels; -not avoiding doing a lot of processing, as is the case with the `fibonacci()` function. -In that particular case, having thread-based asynchronous I/O and the GIL will be a good option, as you rely on the thread scheduler to be able to serve multiple I/O channels simultaneously. -This later approach is called I/O multiplexing, discussed in [the next section](./io-multiplexing.md). diff --git a/content/chapters/io/lab/content/beyond-network-sockets.md b/content/chapters/io/lab/content/beyond-network-sockets.md deleted file mode 100644 index 8eda0aa31b..0000000000 --- a/content/chapters/io/lab/content/beyond-network-sockets.md +++ /dev/null @@ -1,44 +0,0 @@ -# Beyond Network Sockets - -Up until this point, we first learned how to use the [Berkeley Sockets API](./remote-io.md#api---hail-berkeley-sockets), then we learned about the [client-server model](./client-server-model.md), based on this API. -So now we know that sockets offer a ubiquitous interface for inter-process communication, which is great. -A program written in Python can easily send data to another one written in C, D, Java, Haskell, you name it. -However, in the [section dedicated to networking](./networking-101.md), we saw that it takes a whole stack of protocols to send this message from one process to the other. -As you might imagine, this is **much slower even than local I/O using files**. - -So far we've only used sockets for local communication, but in practice it is a bit counterproductive to use network sockets for local IPC due to their high latency. -Wouldn't it be great if we had a way to use the sockets API for local IPC without having to deal with this increased latency? -Well, there is a way and it's called **UNIX sockets**. - -## UNIX Sockets - -UNIX sockets are created using the `socket()` syscall and are bound **TO A FILE** instead of an IP and port using `bind()`. -You may already see a similarity with [named pipes](./pipes.md#named-pipes---mkfifo). -Just like them, UNIX sockets don't work by writing data to the file (that would be slow), but instead the kernel retains the data they send internally so that `send()` and `recv()` can read it from the kernel's storage. -You can use `read()` and `write()` to read/write data from/to both network and UNIX sockets as well, by the way. -The differences between using `send()`/`recv()` or `write()`/`read()` are rather subtle and are described in [this Stack Overflow thread](https://stackoverflow.com/questions/1790750/what-is-the-difference-between-read-and-recv-and-between-send-and-write). - -UNIX sockets are a feature of POSIX-compliant operating systems (e.g. Linux, macOS) and are not available on non-POSIX operating systems, such as Microsoft Windows. -However, there are [third-party libraries](https://crates.io/crates/uds_windows) providing similar features to UNIX sockets in non-POSIX systems, but they might not have the same performance and reliability. - -### Practice: Receive from UNIX Socket - -Navigate to `support/receive-challenges/receive_unix_socket.c`. -Don't write any code yet. -Let's compare UNIX sockets with network sockets first: - -- To create them, give `socket()` the `PF_UNIX`/`AF_UNIX` instead of `PF_INET`/`AF_INET`. -The latter are used to create network sockets. -You can use `PF_*` or `AF_*` interchangeably. -They mean "protocol families" and "address families", respectively, but they are [essentially the same thing](https://stackoverflow.com/a/6737450). -But apart from this, UNIX sockets can be set to work in both TCP (`SOCK_STREAM`) and UDP (`SOCK_DGRAM`) modes. - -- The `bind()` call now takes a path as argument, as specified [in the UNIX socket man page](https://man7.org/linux/man-pages/man7/unix.7.html). - -And this is it. -Everything else is the same. - -Now fill in the `TODO`s in `receive_unix_socket.c`. -Use the example at the bottom of [the UNIX socket man page](https://man7.org/linux/man-pages/man7/unix.7.html) if you get stuck at `bind()`. -When you're done, compile the code and then run `send_unix_socket`. -If you did this task correctly, `receive_unix_socket` should display the flag. diff --git a/content/chapters/io/lab/content/client-server-model.md b/content/chapters/io/lab/content/client-server-model.md deleted file mode 100644 index fcc10db257..0000000000 --- a/content/chapters/io/lab/content/client-server-model.md +++ /dev/null @@ -1,89 +0,0 @@ -# Client-Server Model - -Up to now, we've avoided code snippets using TCP. -Not anymore. -Keep in mind that all socket functions in the Berkeley API have very detailed and informative `man` pages. -For example, here are the `man` pages for [`sendto()`](https://linux.die.net/man/2/sendto) and [`recvfrom()`](https://linux.die.net/man/2/recvfrom) that are used with UDP sockets. - -Going back to our [initial example](./remote-io.md#one-browser---many-connections) with how the browser gets data from , by now we know the preferred method for transferring text is TCP. - -Unlike UDP, TCP is **connection-oriented**. -This is why, in the example with the browser, we kept using the word **connection**. -What's different from UDP is that this connection is **bidirectional**, so we can both [`send()`](https://man7.org/linux/man-pages/man2/send.2.html) and receive ([`recv()`](https://man7.org/linux/man-pages/man2/recv.2.html)) data from it. -Notice that the syscalls have changed. -We were using `sendto()` and `recvfrom()` for UDP, and now we're using `send()` and `recv()` for TCP. -And yes, despite the fact that we're using Python, these are syscalls. -You saw them in C when you solved the [challenge](./remote-io.md#practice-network-sockets-challenge). - -## Server vs Client - -When discussing servers and clients, the server is the passive actor. -It may have some data and wait for clients to ask for this data. -Or it may require some data and wait for clients to send it. -Either way, it is **listening** for connections. - -The client is the active actor, being the one who initiates the connection. - -[Quiz](../quiz/client-server-sender-receiver.md) - -## Establishing the Connection - -There is a series of steps that a client and most importantly a server must take to establish a TCP connection. - -### Steps Taken by the Server - -1. Call `socket()` to create a socket. - -1. `bind()` this socket to an IP and a port. -Up to now, this is nothing new. -However, with TCP sockets, `bind()` doesn't automatically make them listen for connections. - -1. Call [`listen()`](https://man7.org/linux/man-pages/man2/listen.2.html) to make the socket ready to receive connections. - -1. Call [`accept()`](https://man7.org/linux/man-pages/man2/accept.2.html) to set up one connection initiated by a client. -From now, the connection is established. -`accept()` returns a new socket, which will be further used for communication between the server and the client. - -1. Exchange messages with the client. -The server can both read messages from the client and send responses back using the same socket returned by `accept()`. -When done, `close()` the socket returned by accept and repeat from step `4`. - -### Steps Taken by the Client - -1. Call `socket()` to create a socket. - -1. Use [`connect()`](https://man7.org/linux/man-pages/man2/connect.2.html) to... you guessed it: connect to the server. -This step is like an "answer" to step `4` from the server. -We can say that the server `accept()`s a `connect()` request from the client. - -1. Exchange messages with the server. -The client can both send messages to the server and read responses from it using the same socket created during step `1`. - -Below is an image summarising the steps above: - -![Steps to Establish a Connection](../media/connection-establishment.svg) - -### Practice: Client - -Navigate to `support/client-server/`. -Here you will find a minimalistic server implementation in `server.py`. - -1. Read the code and identify the steps outlined above. - -1. Now fill in the code in `client.c` or `client.py` to be able to send messages to the server and receive responses from it. -Run multiple clients. - -1. Run the code, then run `lsof` and `netstat` in other terminals to identify the file descriptor corresponding to the listening file descriptor. - -## Practice: Just a Little Bit More Deluge - -We've already said that Deluge uses an [abstraction over TCP](./networking-101.md#practice-encapsulation-example-deluge-revived) to handle socket operations, so we don't have the luxury of seeing it perform remote I/O "manually". -However, there are a few instances where Deluge uses socket operations itself, mostly for testing purposes. - -Deluge saves its PIDs (it can spawn multiple processes) and ports in a file. -Find the `is_daemon_running()` method. -This method uses the aforementioned file to check if a given process is Deluge or some other process. -To do this, it `connect()`s to that process's socket. -If it can't, then that process is not Deluge. -Otherwise, it is Deluge and that connection is immediately closed :)) -This may sound like a stupid way of checking whether a process is Deluge or not, but _if it's stupid and it works, then it's not stupid!_ diff --git a/content/chapters/io/lab/content/file-descriptors.md b/content/chapters/io/lab/content/file-descriptors.md deleted file mode 100644 index 0424e96a57..0000000000 --- a/content/chapters/io/lab/content/file-descriptors.md +++ /dev/null @@ -1,233 +0,0 @@ -# File Descriptors - -After running the code in the ["File Handlers" section](./file-handlers.md), you saw that `open()` returns a **number**. -This number is a **file descriptor**. -Run the code above multiple times. -You'll always get file descriptor 3. -We can already tell these numbers have some meaning and aren't just some seemingly random numbers, like PIDs were. - -You've most likely used file descriptors before without knowing. -Remember the following concepts: - -- `stdin` (standard input) -- `stdout` (standard output) -- `stderr` (standard error) - -When you wanted to run a shell command and ignore its errors, you wrote something like: - -```console -student@os:~$ ls 2> /dev/null -``` - -The command above uses `2> /dev/null` to **redirect** `stderr` to /dev/null. - -[Quiz](../quiz/stderr-fd.md) - -Good, so we know that `stderr` is file descriptor 2. -The code in `support/file-descriptors/open_directory.c` opened the directory as file descriptor 3. -So what are file descriptors 0 and 1? -They are `stdin` and `stdout`, respectively. - -Now we know that the basic I/O streams of a process correspond to specific file descriptors. -But what is a file descriptor exactly? -From the application's point of view, it is a positive integer acting as a unique identifier for an I/O channel, such as a file. -To the operating system, a file descriptor is an index. -Each process has what's called a **file descriptor table**, which is just a fancy name for an array of pointers. -A file descriptor is an index in this array. -Each element in this array points towards a structure in the kernel's memory that represents an I/O resource. -This structure is called the **open file structure**. - -To illustrate all this information, look at the image below. -All data structures shown below are stored in the Kernel's memory area and are transparent to the user space. -And remember that the file descriptor table is bound to the process, so all its threads share the same file descriptors. - -![File Descriptors](../media/file-descriptors.svg) - -To find out more about the contents of these structures, check out [this section in the Arena](./arena.md#open-file-structure-in-the-kernel) - -## Creating New File Descriptors - -We already know that each process gets 3 file descriptors "by default": - -- `stdin` (standard input): 0 -- `stdout` (standard output): 1 -- `stderr` (standard error): 2 - -To create new file descriptors (i.e. open new files), a process can use the [`open()`](https://man7.org/linux/man-pages/man2/open.2.html) system call. -It receives the path to the file, some flags which are akin to the `mode` string passed to `fopen()`. -An optional `mode` parameter that denotes the file's permissions if the `open` must create it can also be provided. -We'll revisit `open()`'s `mode` parameter in the future. - -From now, you should consult [`open()`'s man page](https://man7.org/linux/man-pages/man2/open.2.html) whenever you encounter a new argument to this syscall. -Navigate to `support/file-descriptors/open_read_write.c`. -The function `open_file_for_reading()` calls `open()` with the `O_RDONLY` flag, which is equivalent to opening the file with `fopen()` and setting `"r"` as the `mode`. -This means read-only. -Note that the file descriptor we get is 3, just like before. - -Then `read_from_file()` reads `bytes_to_read` bytes from this file. -For this, it uses the `read()` syscall. -It returns the number of bytes that were read from the file. - -Notice that the code doesn't simply call `read(fd, buff, bytes_to_read)`. -Instead, it uses a `while` loop to read data from the file. - -The [return value of `read()`](https://man7.org/linux/man-pages/man2/read.2.html#RETURN_VALUE) may be less than `bytes_to_read` if there are not enough bytes available or if the operation is interrupted by a signal. -A return value between **0** and `bytes_to_read` is not enough to decide whether we should stop reading. -To determine this, we make another `read()` call, which will return **0** if the cursor is already at **EOF** (end of file). - -The same goes for `write()`: its return value may differ from the intended number of bytes to write. -Partial writes should also be handled at the application level, and the way to do this is by using loops. - -**Remember:** -**It is mandatory that we always use `read()` and `write()` inside `while` loops.** -Higher-level functions like `fread()` and `fwrite()` also use `while` loops when calling `read()` and `write()` respectively. - -### Practice: Open a File for Writing - -Follow the code in `open_file_for_reading()` and fill in the function `open_file_for_writing()`. -The file `write_file.txt` doesn't exist. -`open()` should create it. -Use the `open()`'s man page to find the flags you require. -Do you require anything _else_? - -At this point, depending on what flags you passed to `open()`, a few things might happen. -Work your way through the errors until successfully create and open the file. -The `open()` syscall should return file descriptor 4. - -Now verify the file. -This part may be different on your system. -Delete `write_file.txt` and rerun `open_read_write` a few times. -Each time, check the permissions of `write_file.txt`. -They may be different between runs, or they may always be the same. -Anyway, they are likely not the default permissions for a regular file (`rw-r--r--`). -It is not mandatory that you get the same output as below. - -```console -student@os:~/.../lab/support/file-descriptors$ ls -l -total 11 -drwxrwxr-x 1 student student 4096 Nov 20 18:26 ./ -drwxrwxr-x 1 student student 0 Nov 20 14:11 ../ --rw-rw-r-- 1 student student 46 Nov 20 17:27 .gitignore --rw-rw-r-- 1 student student 125 Nov 20 18:26 Makefile --rw-rw-r-- 1 student student 396 Nov 20 11:27 open_directory.c --rw-rw-r-- 1 student student 2210 Nov 20 17:24 open_read_write.c --rw-rw-r-- 1 student student 34 Nov 20 18:26 read_file.txt ----------- 1 student student 45 Nov 20 18:26 write_file.txt -``` - -[Quiz](../quiz/write-file-permissions.md) - -**Remember:** -**It is mandatory that we pass a `mode` argument to `open()` when using the `O_CREAT` flag.** - -Now pass some sensible `mode` argument to `open()`, such as `0644` (for `rw-r--r--` permissions). - -### Practice: Write to the File - -Follow the example in `read_from_file()` to fill in `write_to_file()`. -Remember to use a loop to make sure your data is fully written to the file. - -1. Use `cat write_file.txt` to check if the file contains the right data. - -1. Use `open_file_for_reading()` and `read_from_file()` to reopen `write_file.txt` for reading and read the bytes you've just written to it. - -## Replace or Truncate? - -### Practice: Write Once More - -Change the message written to `write_file.txt` by `support/file-descriptors/open_read_write.c` **to a shorter one**. -It is important that the new message be shorter than the first one. -Now recompile the code, then run it, and then inspect the contents of the `write_file.txt` file. - -If the new message were `"Something short"`, the contents of `write_file.txt` should be: - -```console -student@os:~/.../lab/support/file-descriptors$ cat ../../support/file-descriptors/write_file.txt -Something shorte_file.txt: What's up, Doc? -``` - -Note that the final bytes of the previous text remain unchanged. -The new message was simply written **on top** of the old one. - -Now let's do a quick test. -We haven't talked about how redirections work in the terminal (we'll get there, step by step), but you can imagine that if you type `ls > file.txt`, `file.txt` has to be opened at some point. -Let's write data to a file twice and observe the behaviour: - -```console -student@os:~/.../lab/support/file-descriptors$ ls -l > file.txt - -student@os:~/.../lab/support/file-descriptors$ cat file.txt -total 6 --rw-rw-r-- 1 student student 0 Nov 20 21:11 file.txt --rw-rw-r-- 1 student student 125 Nov 20 18:26 Makefile --rw-rw-r-- 1 student student 396 Nov 20 21:10 open_directory.c --rw-rw-r-- 1 student student 2300 Nov 20 20:51 open_read_write.c --rw-rw-r-- 1 student student 34 Nov 20 18:26 read_file.txt --rw-r--r-- 1 student student 45 Nov 20 20:56 write_file.txt - -student@os:~/.../lab/support/file-descriptors$ ls > file.txt - -student@os:~/.../lab/support/file-descriptors$ cat file.txt -file.txt -Makefile -open_directory.c -open_read_write.c -read_file.txt -write_file.txt -``` - -The second output is shorter than the first, yet the first output is no longer present in the file after the second `ls`. -How come? -Well, the reason is another flag being passed to `open()`: `O_TRUNC`. -At this point, you should be accustomed to looking for this flag in `open()`'s man page. -Go ahead and do it. - -[Quiz 1](../quiz/o-trunc.md) - -[Quiz 2](../quiz/fopen-w.md) - -### Practice: Close'em All - -Just like you use `open()` to create new file descriptors, you can use [`close()`](https://man7.org/linux/man-pages/man2/close.2.html) to destroy them. -This clears and frees the open file structure to which that entry in the file descriptor table is pointing. -Use `close()` on the file descriptors you've opened so far in `support/file-descriptors/open_read_write.c`. - -Note that you don't have to close file descriptors 0, 1 and 2 manually. -The standard streams are meant to stay alive throughout the lifetime of the process. -Just like calling `free()` on a `malloc()`-ed pointer, calling `close()` is not really necessary. -When a process terminates, the OS closes all its file descriptors the same way it frees all its memory. - -And keeping this comparison with `malloc()` and `free()`, closing file descriptors is important when they are created inside a loop, as the file descriptor table's size is limited. - -## File Handling Conclusion: libc vs syscalls - -Up to now, we can draw some parallels between `fopen()` and `open()`. -While `fopen()` allows the usage of high-level functions such as `fread()` and `fwrite()`, which, among other things, use `while` loops to ensure they always read the required number of bytes, the libc-specific API is not generic enough. - -In the following sections, we'll use file descriptors and `read()` and `write()` to interact with some inter-process-communication mechanisms, such as pipes. - -The table below shows the higher level API provided by libc and the syscalls it relies on. -As usual, use the `man` pages when in doubt about either of them. - -| libc | syscall | -| :--------: | :-------: | -| `fopen()` | `open()` | -| `fread()` | `read()` | -| `fwrite()` | `write()` | -| `fseek()` | `lseek()` | -| `fclose()` | `close()` | - -So for most equivalents, just remove the leading `f` when moving from the libc function to the underlying syscall. - -For a quick recap of the flags we've discussed so far, take a look at the following table. -But don't bother memorising it. -You can find it any time in by typing [`man fopen`](https://man7.org/linux/man-pages/man3/fopen.3.html) in your terminal. - -| `fopen()` mode | `open()` flag | -| :------------: | :-----------------------------: | -| `"r"` | `O_RDONLY` | -| `"w"` | `O_WRONLY │ O_CREAT │ O_TRUNC` | -| `"a"` | `O_WRONLY │ O_CREAT │ O_APPEND` | -| `"r+"` | `O_RDWR` | -| `"w+"` | `O_RDWR │ O_CREAT │ O_TRUNC` | -| `"a+"` | `O_RDWR │ O_CREAT │ O_APPEND` | diff --git a/content/chapters/io/lab/content/file-handlers.md b/content/chapters/io/lab/content/file-handlers.md deleted file mode 100644 index 8774c29c49..0000000000 --- a/content/chapters/io/lab/content/file-handlers.md +++ /dev/null @@ -1,89 +0,0 @@ -# File Handling - -You've most likely had to deal with files in the past. -Go to `support/simple-file-operations` and run a most basic command: -`cat file.txt`: - -```console -student@os:~/.../lab/support/simple-file-operations$ cat file.txt -OS Rullz! -``` - -But how does `cat` actually "reach" the file, then read its contents, then print them to standard output? -This is part of what we're going to learn. - -## Reaching the File - -To manipulate the file (read its contents, modify them, change its size etc.), each process must first get a **handler** to this file. -Think of this handler as an object by which the process can identify and refer to the file. - -Now take a look at the code examples in `support/simple-file-operations`. -Each of them reads the contents of `file.txt`, modifies them, and then reads the previously modified file again. -Use `make` to compile the C code, and `make java-file-operations` to compile the Java code. - -Now run the programs repeatedly in whatever order you wish: - -```console -student@os:~/.../lab/support/simple-file-operations$ python3 file_operations.py -File contents are: OS Rullz! -Wrote new data to file -File contents are: Python was here! - -student@os:~/.../lab/support/simple-file-operations$ ./file_operations # from the C code -File contents are: Python was here! -Wrote new data to file -File contents are: C was here! - -student@os:~/.../lab/support/simple-file-operations$ java FileOperations -File contents are: Python was here! -Wrote new data to file -File contents are: Java was here! -``` - -Note that each piece of code creates a variable, which is then used as a handler. - -[Quiz](../quiz/file-handler-c.md) - -### Limitations of High-level File Handlers - -**Everything in Linux is a file.** -This statement says that the Linux OS treats every entry in a file system (regular file, directory, block device, char device, link, UNIX socket) as a file. -This is very convenient for creating a unified interface that deals with all these files. -We would like our file handlers to also be able to handle all types of files. - -Let's try this. -Navigate to `support/simple-file-operations/directory_operations.c`. -Read the code. -It does something very simple: -it attempts to open the `dir` directory the same way `file_operations.c` tried to open `file.txt`. -Compile and run the code. - -```console -student@os:~/.../lab/support/simple-file-operations$ ./directory_operations -18:18:11 FATAL directory_operations.c:14: fopen: Is a directory -``` - -The error message is crystal clear: we cannot use `fopen()` on directories. -So the `FILE` structure is unsuited for directories. -Therefore, this handler is not generic enough for a regular Linux filesystem. - -To get to the directory, we need to use a lower-level function. -Let's take a look at the syscall used by `fopen()`. - -[Quiz](../quiz/fopen-syscall.md) - -We will use a simpler syscall, called [`open()`](https://man7.org/linux/man-pages/man2/open.2.html). -In fact, `open()` is a wrapper over `openat()`. -Navigate to `support/file-descriptors/directory_open.c`. -Inspect, compile and run the code. - -```console -student@os:~/.../lab/support/file-descriptors$ ./open_directory -Directory file descriptor is: 3 -``` - -We can now see that the `open()` syscall is capable of also handling directories, so it's closer to what we want. -Note that it is rather uncommon to use `open()` for directories. -Most of the time, [`opendir()`](https://man7.org/linux/man-pages/man3/opendir.3.html) is used instead. -But what does it return? -Find out in [the "File Descriptors" section](./file-descriptors.md). diff --git a/content/chapters/io/lab/content/file-mappings.md b/content/chapters/io/lab/content/file-mappings.md deleted file mode 100644 index 4c71a45ea5..0000000000 --- a/content/chapters/io/lab/content/file-mappings.md +++ /dev/null @@ -1,130 +0,0 @@ -# File Mappings - -Mapping a file to the VAS of a process is similar to how shared libraries are loaded into the same VAS. -It's a fancier way of saying that the contents of a file are copied from a given offset within that file to a given address. -What's nice about this is that the OS handles all offsets, addresses and memory allocations on its own, with a single highly versatile syscall: `mmap()`. - -## The Return of `mmap()` - -Remember that the `.text`, `.rodata` and `.data` sections of libraries are present in the VAS of any Linux process: - -```console -student@os:~$ sleep 1000 & # start a `sleep` process in the background -[1] 17579 - -student@os:~$ cat /proc/$(pidof sleep)/maps -55b7b646f000-55b7b6471000 r--p 00000000 103:07 6423964 /usr/bin/sleep -55b7b6471000-55b7b6475000 r-xp 00002000 103:07 6423964 /usr/bin/sleep -55b7b6475000-55b7b6477000 r--p 00006000 103:07 6423964 /usr/bin/sleep -55b7b6478000-55b7b6479000 r--p 00008000 103:07 6423964 /usr/bin/sleep -55b7b6479000-55b7b647a000 rw-p 00009000 103:07 6423964 /usr/bin/sleep -55b7b677c000-55b7b679d000 rw-p 00000000 00:00 0 [heap] -7fe442f61000-7fe44379d000 r--p 00000000 103:07 6423902 /usr/lib/locale/locale-archive -7fe44379d000-7fe4437bf000 r--p 00000000 103:07 6432810 /usr/lib/x86_64-linux-gnu/libc-2.31.so -7fe4437bf000-7fe443937000 r-xp 00022000 103:07 6432810 /usr/lib/x86_64-linux-gnu/libc-2.31.so -7fe443937000-7fe443985000 r--p 0019a000 103:07 6432810 /usr/lib/x86_64-linux-gnu/libc-2.31.so -7fe443985000-7fe443989000 r--p 001e7000 103:07 6432810 /usr/lib/x86_64-linux-gnu/libc-2.31.so -7fe443989000-7fe44398b000 rw-p 001eb000 103:07 6432810 /usr/lib/x86_64-linux-gnu/libc-2.31.so -7fe44398b000-7fe443991000 rw-p 00000000 00:00 0 -7fe4439ad000-7fe4439ae000 r--p 00000000 103:07 6429709 /usr/lib/x86_64-linux-gnu/ld-2.31.so -7fe4439ae000-7fe4439d1000 r-xp 00001000 103:07 6429709 /usr/lib/x86_64-linux-gnu/ld-2.31.so -7fe4439d1000-7fe4439d9000 r--p 00024000 103:07 6429709 /usr/lib/x86_64-linux-gnu/ld-2.31.so -7fe4439da000-7fe4439db000 r--p 0002c000 103:07 6429709 /usr/lib/x86_64-linux-gnu/ld-2.31.so -7fe4439db000-7fe4439dc000 rw-p 0002d000 103:07 6429709 /usr/lib/x86_64-linux-gnu/ld-2.31.so -7fe4439dc000-7fe4439dd000 rw-p 00000000 00:00 0 -7ffd07aeb000-7ffd07b0c000 rw-p 00000000 00:00 0 [stack] -7ffd07b8b000-7ffd07b8e000 r--p 00000000 00:00 0 [vvar] -7ffd07b8e000-7ffd07b8f000 r-xp 00000000 00:00 0 [vdso] -ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall] -``` - -How does the content of those files get there? -Below you can see how libc is loaded, i.e. mapped into the VAS of an `ls` process. - -```console -student@os:~$ strace ls -openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 -[...] -mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fb313c9c000 -mmap(0x7fb313cbe000, 1540096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7fb313cbe000 -mmap(0x7fb313e36000, 319488, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7fb313e36000 -mmap(0x7fb313e84000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7fb313e84000 -``` - -For a quick reminder about `mmap()`, its 5th argument is the file descriptor from where we want to copy the data to the RAM. -The 6th argument is the offset within the file from where to start copying. - -### What About `read()`? - -Let's get the elephant out of the room: why use `mmap()` instead of `read()` for dynamic libraries? - -The short answer: for efficiency and convenience. - -`read()` is a rather blunt approach for loading libraries into memory, as it does not take advantage of their particularities. -There are two main points to consider: how is memory accessed and when is it modified. -Dynamic libraries are mostly **non-writable** (`.data` section being the exception) and **arbitrarily accessed**. -Reading and storing a library whose content might end up mostly unused does not sound like the right thing to do. - -On the other hand, `mmap()` makes use of `demand paging`. -As a reminder, `demand paging` is a technique where memory is allocated, but each memory page is loaded when needed. -Furthermore, `mmap()` will check if the library was already loaded into memory, and if so, it will not use additional memory, thus reducing the startup time. -As you remember from the Compute chapter, processes sharing memory will lead to [`copy-on-write`](https://open-education-hub.github.io/operating-systems/Lab/Compute/copy-on-write), but this is far better than creating copies of the non-writable sections as `read()` does. - -Even without considering any OS tweaks, using `read()` to load dynamic libraries would require extra work. -This is because allocating memory at the correct address would still require the use of `mmap()`. -Additionally, setting the appropriate permissions to avoid security issues would require the use of `mprotect()`. -Therefore, using `mmap()` alone is a more convenient way for loading dynamic libraries. - -### Practice: Copy a File - -If `mmap()` is good for copying files, let's use it like this. -Navigate to `support/file-mappings/mmap_cp.c`. -It copies the contents of a file to another by `mmap()`-ing each of them to its VAS and then simply copying the bytes from one mapping to another as if copying the contents of 2 arrays. - -1. Fill in the `TODO`s so that the program correctly copies the contents of the input file to the output file. -Use `make test-file` to generate a 1MB file with random data. -You can use this for debugging and `diff` to test whether the input and output files differ (they shouldn't): - -```console -student@os:/.../support/file-mappings$ ./mmap_cp test-file.txt output.txt - -student@os:/.../support/file-mappings$ diff test-file.txt output.txt - -student@os:/.../support/file-mappings$ -``` - - - -2. Uncomment the calls to `wait_for_input()` and rerun the program. -While the program is waiting, open another terminal and run `cat /proc/$(pidof mmap_cp)/maps` and see the mapped files. - - - -### Practice: I Am Speed - -Now we can copy files using `mmap()`. -The code is rather short and convenient to write. -Its disadvantage is that we have to `mmap()` the 2 files entirely of the VAS of the process. -While this may be alright for small files, for larger ones we simply may not have enough RAM. -What if we had to copy a 500GB file? - -Let's look at what the `cp` tool uses for copying. - -[Quiz 1](../quiz/syscalls-cp.md) - -OK, so we have a competition: our `mmap_cp` versus `cp`. -Run the script `benchmark_cp.sh` to measure which of the 2 implementations is faster. - -[Quiz 2](../quiz/mmap-read-write-benchmark.md) - -Now take a look at the `benchmark_cp.sh` script. -You might get a little confused about the following command: `echo 3 > /proc/sys/vm/drop_caches`. -Head over to [this section in the Arena](./arena.md#to-drop-or-not-to-drop) to find out what it's about. - -## Conclusion - -So by using `mmap()`, we pay the price for loading the file into memory **once**, but then all `read()`s, `write()`s and especially `seek()`s will be faster. -This is akin to treating the file as a regular byte array. -So file mappings are especially useful for randomly accessing file data. -Accessing the i-th byte in a file becomes the same as `mapping[i]`, which is obviously more efficient than calling `lseek()`. -File mappings are also useful when we have to overwrite existing data multiple times or when we read certain chunks multiple times. diff --git a/content/chapters/io/lab/content/io-internals.md b/content/chapters/io/lab/content/io-internals.md deleted file mode 100644 index b9d3eded97..0000000000 --- a/content/chapters/io/lab/content/io-internals.md +++ /dev/null @@ -1,189 +0,0 @@ -# I/O Internals - -Now, we will take a short look at how the file descriptors you've just learnt about are handled in libc. -The Software Stack chapter has taught us that applications generally interact with libraries which expose wrappers on top of syscalls. -The most important library in a POSIX system (such as Linux) is libc. -Among many others, it provides higher-level abstractions over I/O-related syscalls. - -**Musl** (read just like "muscle") is a lightweight implementation of libc, which exposes the same API that you have used so far, while also being fit for embedded and OS development. -For example, [Unikraft](https://unikraft.org/) [unikernels](https://unikraft.org/docs/concepts/) may [use musl](https://github.com/unikraft/lib-musl). - -First, it provides a `struct` that groups together multiple data that is necessary when handling files. -We know from the example in `support/simple-file-operations/file_operations.c` that the file handler employed by libc is `FILE *`. -`FILE` is just a `typedef` for [`struct _IO_FILE`](https://elixir.bootlin.com/musl/v1.2.3/source/src/internal/stdio_impl.h#L21). -Here are the most important fields in `struct _IO_FILE`: - -```c -struct _IO_FILE { - int fd; /* File descriptor */ - - unsigned flags; /* Flags with which `open()` was called */ - - int mode; /* File permissions; passed to `open()` */ - - off_t off; /* File offset from where to read / write */ - - /** - * Internal buffer used to make fewer costly `read()`/`write()` - * syscalls. - */ - unsigned char *buf; - size_t buf_size; - - /* Pointers for reading and writing from/to the buffer defined above. */ - unsigned char *rpos, *rend; - unsigned char *wend, *wpos; - - /* Function pointers to syscall wrappers. */ - size_t (*read)(FILE *, unsigned char *, size_t); - size_t (*write)(FILE *, const unsigned char *, size_t); - off_t (*seek)(FILE *, off_t, int); - int (*close)(FILE *); - - /* Lock for concurrent file access. */ - volatile int lock; -}; -``` - -As you might have imagined, this structure contains the underlying file descriptor, the `mode` (read, write, truncate etc.) with which the file was opened, as well as the offset within the file from which the next read / write will start. - -Libc also defines its own wrappers over commonly-used syscalls, such as `read()`, `write()`, `close()` and `lseek()`. -These syscalls themselves need to be implemented by the driver for each file system. -This is done by writing the required functions for each syscall and then populating [this structure](https://elixir.bootlin.com/linux/v6.0.9/source/include/linux/fs.h#L2093) with pointers to them. -You will recognise quite a few syscalls: `open()`, `close()` `read()`, `write()`, `mmap()` etc. - -## IO Optimisations - -You saw this hierarchy during the Data lecture: - -![Memory Hierarchy](../media/memory-hierarchy.svg) - -It says that while the disk can store lots of data, it does so at the cost of speed. -When we say speed, we mean the rate at which we can read/write data from/to the disk, i.e. the maximum number of bytes transferred per unit of time. -This means that `read()` and `write()` syscalls (or their various corresponding library calls) are slow and cause performance bottlenecks. -More often than not, it is the I/O component that drags down the performance of an otherwise fast application. -And what's worse, the further the "destination" of the I/O operation (file on the disk or host on the Web) is, the more time it takes to transfer data to and from it. - -On the other hand, as we've already established, the I/O component defines how we interact with an app. -If we want it to be responsive and to do something useful, most likely, the I/O is the key. - -So I/O is crucial for most applications, yet it is also the slowest... - -![Sad Pepe](../media/sad-pepe.png) - -But fear not! -There are countless optimisations out there aimed precisely at bridging the speed gap between the memory and the disk. - -### I/O Buffering - -Going back to our initial example with [`struct _IO_FILE`](https://elixir.bootlin.com/musl/v1.2.3/source/src/internal/stdio_impl.h#L21) from Musl, we can see some more fields: - -```c -unsigned char *buf; -size_t buf_size; -unsigned char *rpos, *rend; -unsigned char *wend, *wpos; -``` - -Given the number `unsigned char *` fields we have in this structure, it seems there is some heavy _buffering_ going on. -But what is it? - -#### Practice: `printf()` Buffering - -1. Navigate to `support/buffering/printf_buffering.c`. -Those `printf()` calls obviously end up calling `write()` at some point. -Run the code under `strace`. - -[Quiz](../quiz/strace-printf.md) - -Since there is only one `write()` syscall despite multiple calls to `printf()`, it means that the strings given to `printf()` as arguments are kept _somewhere_ until the syscall is made. -That _somewhere_ is precisely that buffer inside `struct _IO_FILE` that we highlighted above. -Remember that syscalls cause the system to change from user mode to kernel mode, which is time-consuming. -Instead of performing one `write()` syscall per call to `printf()`, it is more efficient to copy the string passed to `printf()` to an **internal buffer** inside libc (the `unsigned char *buf` from above) and then at a given time (like when the buffer is full for example) `write()` the whole buffer. -This results in far fewer `write()` syscalls. - - - -2. Now, it is interesting to see how we can force libc to dump that internal buffer. -The most direct way is by using the `fflush()` library call, which is made for this exact purpose. -But we can be more subtle. -Add a `\n` in some of the strings printed in `support/buffering/printf_buffering.c`. -Place them wherever you want (at the beginning, at the end, in the middle). -Recompile the code and observe its change in behaviour under `strace`. - - - -[Quiz](../quiz/flush-libc-buffer.md) - -Now we know that I/O buffering **does happen** within libc. -If you need further convincing, check out the Musl implementation of [`fread()`](https://elixir.bootlin.com/musl/v1.2.3/source/src/stdio/fread.c#L6), for example. -It first copies the [data previously saved in the internal buffer](https://elixir.bootlin.com/musl/v1.2.3/source/src/stdio/fread.c#L16): - -```c -if (f->rpos != f->rend) { - /* First exhaust the buffer. */ - k = MIN(f->rend - f->rpos, l); - memcpy(dest, f->rpos, k); - f->rpos += k; - dest += k; - l -= k; -} -``` - -Then, if more data is requested and the internal buffer isn't full, it refills it using [the internal `read()` wrapper](https://elixir.bootlin.com/musl/v1.2.3/source/src/stdio/fread.c#L27). -This wrapper also places the data inside the destination buffer. - -#### Practice: Buffering Performance - -Up to now, it's pretty clear what I/O buffering is about. -Let's see what kind of a performance increase it brings. -We'll look at an extreme example. -Navigate to `support/buffering/no_buffering.c`. -This code either writes or reads the contents of a file **one byte at a time** using a syscall for each of them. - -Use the `benchmark_buffering.sh` script to compare the `no_buffering` and `libc_buffering` implementations. -Below are some possible results. -Yours are likely going to be different: - -```console -student@os:/.../support/buffering$ -======== Testing no_buffering ======== -Testing no_buffering read... -Read 1048576 bytes from test-file.txt in 717 ms -Testing no_buffering write... -Wrote 1048576 bytes to test-file.txt in 19632 ms -======== Testing libc_buffering ======== -Testing libc_buffering read... -Read 1048576 bytes from test-file.txt in 14 ms -Testing libc_buffering write... -Wrote 1048576 bytes to test-file.txt in 38 ms -``` - - - -So buffering brings a **98%** improvement for reading and a **99.8%** improvement for writing! -This is massive! -Yes, this is an extreme example, but it goes a long way to show how powerful I/O buffering can be. -Now, the question is, "Can YOU do better?" - - - -### Practice: DIY Buffering - -1. Navigate to `support/buffering/diy_buffering.c`. -`diy_fread()` already defines a minimalistic implementation of `fread()`. -Use it as a starting point to implement `diy_write()` as an implementation of `fwrite()`. - -1. Run `benchmark_buffering.sh` to compare the performance of your implementation with that of libc. -Did you beat it? - -## Conclusion - -I/O Buffering is a ubiquitous optimisation in all libc implementations. -It is so powerful that the kernel uses it as well. -This means that the kernel also reads more bytes than it's requested and stores the remainder in an internal buffer, just like libc. -This concept is known as **double buffering**. -In the [following section](./zero-copy.md), we will see how to make use of this internal buffering to optimise network requests. - -Notice that the script in `support/buffering/benchmark_buffering.sh` also uses `echo 3 > /proc/sys/vm/drop_caches`. -That section [in the Arena](./arena.md#to-drop-or-not-to-drop) that we mentioned earlier is becoming even more interesting. diff --git a/content/chapters/io/lab/content/io-multiplexing.md b/content/chapters/io/lab/content/io-multiplexing.md deleted file mode 100644 index 6c9f5780ff..0000000000 --- a/content/chapters/io/lab/content/io-multiplexing.md +++ /dev/null @@ -1,63 +0,0 @@ -# I/O Multiplexing - -I/O multiplexing is the ability to serve multiple I/O channels (or anything that can be referenced via a file descriptor / handle) simultaneously. -If a given application, such a server, has multiple sockets on which it serves connection, it may be the case that operating on one socket blocks the server. -One solution is using asynchronous operations, with different backends. -The other solution is using I/O multiplexing. - -With I/O multiplexing, the API provides a structure / an array to list all used channels. -And then it provides an API to operate on that channel. -And a (blocking) function to retrieve the channel that is ready for interaction (or channels). -So instead of waiting for a given channel (while others may be ready), you now simultaneously wait on all channels, and the ready channel is returned by the function. - -The classical functions for I/O multiplexing are [`select`](https://man7.org/linux/man-pages/man2/select.2.html) and [`poll`](https://man7.org/linux/man-pages/man2/poll.2.html). -Due to several limitations, modern operating systems provide advanced (non-portable) variants to these: - -* Windows provides [I/O completion ports (IOCP)](https://learn.microsoft.com/en-us/windows/win32/fileio/i-o-completion-ports). -* BSD provides [`kqueue`](https://www.freebsd.org/cgi/man.cgi?kqueue). -* Linux provides [`epoll()`](https://man7.org/linux/man-pages/man7/epoll.7.html). - -**Note** that I/O multiplexing is orthogonal to asynchronous operations. -You could tie them together if the completion of the asynchronous operation sends a notification that can be handled via a file descriptor / handle. -This is the case with Windows asynchronous I/O (called [overlapped I/O](https://learn.microsoft.com/en-us/windows/win32/fileio/synchronous-and-asynchronous-i-o)). - -## Practice - -Enter the `multiplex/` directory. -See the implementation of an `epoll()`-based server in `epoll_echo_server.c`. - -1. Build the server: - - ```console - student@os:~/.../lab/support/multiplex$ make - gcc -g -Wall -Wextra -I../../../../../common/makefile/.. -c -o epoll_echo_server.o epoll_echo_server.c - gcc -g -Wall -Wextra -I../../../../../common/makefile/.. -c -o ../../../../../common/makefile/../utils/log/log.o ../../../../../common/makefile/../utils/log/log.c - gcc -g -Wall -Wextra -I../../../../../common/makefile/.. -c -o ../../../../../common/utils/sock/sock_util.o ../../../../../common/utils/sock/sock_util.c - gcc epoll_echo_server.o ../../../../../common/makefile/../utils/log/log.o ../../../../../common/utils/sock/sock_util.o -o epoll_echo_server - ``` - - And run it: - - ```console - student@os:~/.../lab/support/multiplex$ ./epoll_echo_server - 08:41:07 INFO epoll_echo_server.c:252: Server waiting for connections on port 42424 - ``` - - It listens for connection on port `42424`. - - Connect using `netcat` on another console and send messages: - - ```console - $ nc localhost 42424 - aaa - aaa - bbb - bbb - ``` - - Keep the connection open and, on the third console, initiate another `netcat` connection. - The server is now multiplexing both connections. - -1. Create a script and / or a program to exercise the server. - Create many connections to the server and continuously send messages to the server. - See it multiplex the I/O channels (one for each connection - actually two: one for receiving and one for sending). diff --git a/content/chapters/io/lab/content/networking-101.md b/content/chapters/io/lab/content/networking-101.md deleted file mode 100644 index 9d9d9d78c1..0000000000 --- a/content/chapters/io/lab/content/networking-101.md +++ /dev/null @@ -1,126 +0,0 @@ -# Networking 101 - -In this section, we will **briefly** explore how networking works in general, from the perspective of the application. -Understanding the details of it, however, is beyond the scope of this course. - -The main **protocols** used by applications are **User Datagram Protocol (UDP)** and **Transmission Control Protocol (TCP)**. -We can specify this protocol when creating a socket. - -## UDP - -UDP is the simpler of the two protocols above. -It simply states that the sender should pass the data over to the receiver specified by an IP and a port. -The scripts in `support/send-receive/` both used UDP. -You can tell by the fact they both created their sockets like so: - -```Python -sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -``` - - -The `socket.SOCK_DGRAM` argument stands for "datagram" and specifies UDP. - -It doesn't care whether the receiver has got all the data, whether it was corrupted or dropped altogether by some router along the way. - -To prove this, rerun `support/send-receive/receiver.py`, then run `support/send-receive/sender.py`, type `"exit"` and then run `sender.py` again. -You'll see no error from the sender because whether the message reaches its destination or not is not important for UDP. - -So far UDP might seem rather useless. -It doesn't confirm whether a message was received correctly or not, so why use it? -Well, exactly because its mechanism is so simple, UDP is **fast**. -Therefore, it is used by many **real-time services**, such as for streaming or video calls where if a frame drops or is incorrect, it doesn't really matter that much since it will immediately be replaced with the next frame, thus masking the error. - -## TCP - -TCP is the polar opposite of UDP. -TCP makes sure the data is given to the application correctly by performing error checks on the receiving end and then retransmitting any incorrect or missing messages. -For this reason, TCP is good for applications that require precise data, such as banking applications, static images or text. -The cost of correctness, however, is transmission speed. - -### Practice: Encapsulation Example: Deluge Revived - -[Quiz](../quiz/deluge-tcp-udp.md) - -You haven't forgotten about our favourite Bittorrent clint, [Deluge](https://deluge-torrent.org/), have you? -It implements its own protocol for transferring data. -It is built on top of TCP using the [`ITCPTransport` interface](https://twisted.org/documents/18.7.0/api/twisted.internet.interfaces.ITCPTransport.html). - -1. In the Deluge codebase, find the class `DelugeTransferProtocol` and read the docstring comment about the format of its messages. - -1. Follow the code in the `transfer_message()` method of the `DelugeTransferProtocol` class and see the `message` variable conform to the message format you've just read about. -`self.transport` simply refers to the underlying TCP interface. -So `self.transport.write()` invokes the whole socket creation boilerplate necessary to send data via TCP. -We'll discuss it in the [next section](./client-server-model.md#establishing-the-connection). - -This is what protocols are mostly about. -They describe which fields go where within a message and how they should be interpreted. -For example, this protocol says that the `body` (the data itself) should be compressed before being sent. -This makes a lot of sense, as sending less data over the network results in lower latencies. -However, if encryption is too time-consuming, it may not be worth it. -There are compromises to be made everywhere. - -## Local TCP and UDP Services - -To get a full list of all network-handling processes in your system together with the protocols they're using, we can use the `netstat` with the `-tuanp` arguments. -`-tuanp` is short for `-t -u -a -n -p`, which stand for: - -- `-t`: list processes using the TCP protocol -- `-u`: list processes using the UDP protocol -- `-a`: list both servers and clients -- `-n`: list IPs in numeric format -- `-p`: show the PID and name of each program - -```console -student@os:~$ sudo netstat -tunp -Active Internet connections (w/o servers) -Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name -tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 1057/sshd: /usr/sbi -tcp 0 0 127.0.0.1:6463 0.0.0.0:* LISTEN 3261/Discord --type -tcp 0 0 192.168.100.2:51738 162.159.128.235:443 ESTABLISHED 3110/Discord --type -tcp 0 0 192.168.100.2:43694 162.159.129.235:443 ESTABLISHED 3110/Discord --type -tcp 0 0 192.168.100.2:56230 54.230.159.113:443 ESTABLISHED 9154/firefox -tcp 0 0 192.168.100.2:38096 34.107.141.31:443 ESTABLISHED 9154/firefox -tcp 0 0 192.168.100.2:42462 34.117.237.239:443 ESTABLISHED 9154/firefox -tcp 0 0 192.168.100.2:41128 162.159.135.234:443 ESTABLISHED 3110/Discord --type -tcp6 0 0 :::80 :::* LISTEN 1114/apache2 -tcp6 0 0 :::22 :::* LISTEN 1057/sshd: /usr/sbi -tcp6 0 0 2a02:2f0a:c10f:97:55754 2a02:2f0c:dff0:b::1:443 ESTABLISHED 9154/firefox -tcp6 0 0 2a02:2f0a:c10f:97:55750 2a02:2f0c:dff0:b::1:443 ESTABLISHED 9154/firefox -udp 0 0 0.0.0.0:56585 0.0.0.0:* 3261/Discord --type -udp 0 0 0.0.0.0:42629 0.0.0.0:* 3261/Discord --type -udp6 0 0 :::52070 :::* 9154/firefox -udp6 0 0 :::38542 :::* 9154/firefox -``` - -Obviously, your output will differ from the one above. -First look at the fourth column. -It shows the local address and port used by the process. -As we've already established, SSH uses port 22. -Apache2 uses port 80 for both IPv4 and IPv6 addresses (look for rows starting with `tcp` for IPv4 and `tcp6` for IPv6). -Port 80 is used for the HTTP protocol. - -Moving on to some user programs, Firefox runs multiple connections using both IPv4 and IPv6 and a different port for each connection. - -[Quiz](../quiz/firefox-tcp-udp.md) - -Discord does the same things as Firefox. -It uses TCP to send text messages, memes, videos and static content in general. -And at the same time, it uses UDP to exchange voice and video data during calls. - -### Practice: Run `netstat` Yourself - -1. Run `netstat` on your own host machine. -Identify processes that use TCP and UDP and try to figure out what they might be using these protocols for. -Look for browsers (different sites and types of content), servers such as Apache2 or Nginx, online games etc. -Discuss your findings with your classmates and teacher(s). - -1. Now run `receiver.py` and then run `netstat` in another terminal to see your receiver process. -What arguments do you need to pass to `netstat` to see `receiver.py`? -Do you need the whole `-tuapn`? -No. - -### Conclusion - -The difference between TCP and UDP can be summarised as follows: - -![TCP vs UDP](../media/tcp-udp-simplified.png) diff --git a/content/chapters/io/lab/content/overview.md b/content/chapters/io/lab/content/overview.md deleted file mode 100644 index 542329f879..0000000000 --- a/content/chapters/io/lab/content/overview.md +++ /dev/null @@ -1,33 +0,0 @@ -# I/O - -We know that a compute system is a collection of hardware and software that modifies data. -This data has to come from _somewhere_. -This _somewhere_ is always outside the compute system: -files, network packets, radio signals, sensor data. - -A compute system without output is nearly useless. -It will always run the same code on the same data and, thus, produce the same result. -This may be useful in some narrow cases, such as calculating the decimals of Pi. -However, for more real-world-facing applications such as web servers, operating systems and databases inputs and outputs are mandatory. - -The most simplistic representation of a compute system is a black box that receives some input and delivers some output. - -![Compute System - Oversimplified](../media/compute-system-oversimplified.svg) - -In this session, we will look into how a compute system interacts with the outside world to get and produce these inputs and outputs. - -1. [File Handlers](./file-handlers.md) -1. [File Descriptors](./file-descriptors.md) -1. [Redirections](./redirections.md) -1. [Pipes](./pipes.md) -1. [Local I/O in Action](./local-io-in-action.md) -1. [Remote I/O](./remote-io.md) -1. [Networking 101](./networking-101.md) -1. [Client-Server Model](./client-server-model.md) -1. [Beyond Network Sockets](./beyond-network-sockets.md) -1. [File Mappings](./file-mappings.md) -1. [IO Internals](./io-internals.md) -1. [Zero-copy](./zero-copy.md) -1. [Asynchronous I/O](./async-io.md) -1. [I/O Multiplexing](./io-multiplexing.md) -1. [Arena](./arena.md) diff --git a/content/chapters/io/lab/content/pipes.md b/content/chapters/io/lab/content/pipes.md deleted file mode 100644 index 236820bdff..0000000000 --- a/content/chapters/io/lab/content/pipes.md +++ /dev/null @@ -1,179 +0,0 @@ -# Pipes - -When it comes to inter-process communication, so far we know that 2 different processes can `mmap()` the same file and use that as some sort of shared memory, but this requires writing data to the disk which is slow. -Then we know they can `wait()`/`waitpid()` for each other to finish, or better yet, use shared semaphores or mutexes, but these mechanisms aren't good at passing data between processes. -So our goals for this session are to learn how to use an IPC (inter-process communication) mechanism that: - -- allows transfers between processes, not notifications - -- is faster than reading and writing from/to files - -## Anonymous Pipes - `pipe()` - -Have you ever wondered how Bash handles redirecting the `stdout` of a command to the `stdin` of the next command in one-liners such as: - -```bash -cat 'log_*.csv' | tr -s ' ' | cut -d ',' -f 2 | sort -u | head -n 10 -``` - -The `stdout` of `cat` is the `stdin` of `tr`, whose `stdout` is the `stdin` of `cut` and so on. -This "chain" of commands looks like this: - -![Piped Commands](../media/piped-commands.svg) - -So here we have a **unidirectional** stream of data that starts from `cat`, is modified by each new command, and then is passed to the next one. -We can tell from the image above that the communication channel between any 2 adjacent commands allows one process to write to it while the other reads from it. -For example, there is no need for `cat` to read any of `tr`'s output, only vice versa. - -Therefore, this communication channel needs 2 ends: -one for reading (from which commands get their input) and another for writing (to which commands write their output). -In UNIX, the need for such a channel is fulfilled by the [`pipe()` syscall](https://man7.org/linux/man-pages/man2/pipe.2.html). -Imagine there's a literal pipe between any 2 adjacent commands in the image above, where data is what flows through this pipe **in only a single way**. -This is why the `|` operator in Bash is called pipe and why the syscall is also named `pipe()`. - -This type of pipe is also called an **anonymous pipe**, because it cannot be identified using a name (i.e. it is not backed by any file). -The data written to it is kept in a circular buffer inside the kernel from where it can be then read by the child process. -This is faster than writing data to a file, so we achieve both our [initial goals](#pipes). - -## Practice: Find the Right ~~Hole~~ File Descriptor - -Navigate to `support/pipes/anonymous_pipe.c`. -Compile and run the code. -In another terminal, use `lsof` to see: - -- the file descriptors opened by the parent process between the creation of the pipe and the call to `fork()` - -- the file descriptors opened by the child process - -[Quiz](../quiz/pipe-ends.md) - -A simple way to memorise which pipe end is which is to think about `stdin` and `stdout`, respectively. -`stdin` is file descriptor 0 and is mostly for reading and `pipedes[0]` is also for reading. -Conversely, `stdout` is file descriptor 1 and is meant for writing, just like `pipedes[1]`. -Now you won't confuse them. - -### Practice: Inheritance - -An important thing to take note of before we actually use pipes is that file descriptors are **inherited** by the child process from the parent. -So if the parent opens some file descriptors (like, say, for a pipe), the child will also be able to use them. -Don't believe us? - -Modify the code in `support/pipes/anonymous_pipes.c` and call `wait_for_input()` from the child process. -Then use `lsof` again with the PID of the child process to make sure file descriptors 3 and 4 are still open. - -### Practice: Now We Pipe - -Now comes the moment you've most likely been waiting for. -The code in `support/pipes/anonymous_pipes.c` wants to create something like a client-server dynamic between the parent and the child. -The parent reads data from `stdin` and sends it to the child via the pipe they share. -The client (parent) ends communication when you type `"exit"`. - -You can comment out the calls to `wait_for_input()` if you find them annoying. - -Sample output: - -```console -student@os:~/.../support/pipes$ ./anonymous_pipe -pipedes[0] = 3; pipedes[1] = 4 - * pipe created - -- Press ENTER to continue ... -echo -[Child received]: echo -echo -[Child received]: echo -to pipe, or not to pipe -[Child received]: to pipe, or not to pipe -``` - -### Practice: Receive Pipes - -Use your knowledge of pipes to solve a CTF challenge. -Navigate to `support/receive-challenges` and look into the files `receive_pipe.c` and `send_fd_4.c`. -Modify `receive_pipe.c` so that it creates a pipe, then spawns a child process. -The child will redirect file descriptor 4 to `stdout` and then `execlp()` `send_fd_4`. -`send_fd_4` writes the flag to file descriptor 4 (`pipefd[1]`), so the parent process needs to read it from `pipedefd[0]`. - -Once you do this, note that file descriptors are also maintained after calling `exec()` to run a completely new program. - -Now, if you want to use pipes even more, go over to [the Arena](./arena.md#mini-shell-with-blackjack-and-pipes) and add support for pipes to the mini-shell you've previously worked on. - -### Anonymous Pipes: Conclusion - -Anonymous pipes give allow us to efficiently transmit data between 2 processes, as no disk access is required. -However, they still have one major limitation. - -[Quiz](../quiz/anonymous-pipes-limitation.md) - -The answer to this is to employ _some_ filesystem support. - -## Named Pipes - `mkfifo()` - -We will give up on some performance by writing data to a file, but the reading and writing to the file must behave just like using a named pipe: - -- reading doesn't block while there's still data in the pipe -- reading from an empty pipe stalls the current thread until data becomes available. -This is one of the cases where `read()` might not return as many bytes as we requested. -Remember: -always use loops with `read()` and `write()` - -Because this pipe uses a file, which must have a name, it is called a **named pipe**. - -### Practice: From the CLI - -First, let's use named pipes from the terminal. -Use the `mkfifo` command to create one such file: - -```console -student@os:~$ mkfifo fifo - -student@os:~$ ls -l fifo -prw-rw-r-- 1 student student 0 Nov 22 23:20 fifo| -``` - -The fact that pipes are also called FIFOs should come as no surprise. -They act like queues/FIFOs: - -- you add data to one end (push/enqueue) -- you extract data from the other (pop/dequeue) - -Also note the `p` at the beginning of the output above. -It symbolises the type of this file: -a named **pipe**. - -Now let's use it. -Open 2 terminals. - -1. In the first one, use `cat` to read from `fifo`. -Note that the command is blocked because the pipe is empty, so `cat` has nothing to read. -Then, in the second terminal, write some message to `fifo` using `echo`. -In the first terminal, you should see that `cat` has finished, and the message has appeared there. - -1. Now do it the other way around: -first `echo` some string into the pipe and **then** read it with `cat`. -Note that now the `echo` command is blocked. -Now `cat` should end immediately, and the string should appear because we have already placed some data in the pipe. -Also, the previous `echo` should finish now. - -**Remember:** -**Reading from a pipe blocks while the pipe is empty.** -**Writing to a pipe blocks until it is empty.** - -### Practice: From the Code - Receive FIFO - -The libc function with which we can create named pipes is... -[`mkfifo()`](https://man7.org/linux/man-pages/man3/mkfifo.3.html). -You weren't expecting this, were you? -Its underlying syscall is [`mknod()`](https://man7.org/linux/man-pages/man2/mknodat.2.html), which simply creates a file of whatever type we specify, but that's besides the point. - -Navigate to `support/receive-challenges/` and look into `receive_fifo.c` and `send_fifo.c`. -Follow the `TODO`s in the former file to read the flag that the latter writes into the named pipe. -Note that after you create the named pipe, you have to read from it like you would from any regular file. - -### Named Pipes: Conclusion - -It is nice to remember that named pipes sacrifice little to no performance when compared to anonymous pipes. -While it may seem that the data being passed through them is written to the disk, then read and overwritten, this is not the case. -The FIFO file is just a handler within the filesystem that is used to write data to a buffer inside the kernel. -This buffer holds the data that is passed between processes, not the filesystem. -So we still don't break our 2 desires from the beginning of this section: -to allow data transfer and to do so efficiently. diff --git a/content/chapters/io/lab/content/redirections.md b/content/chapters/io/lab/content/redirections.md deleted file mode 100644 index 24465509d3..0000000000 --- a/content/chapters/io/lab/content/redirections.md +++ /dev/null @@ -1,193 +0,0 @@ -# Redirections - -In the [File Descriptors section](./file-descriptors.md), we mentioned redirections such as `ls > file.txt`. -We said `file.txt` has to be opened at some point. -Let's check that. -We'll use `strace`, obviously, to look for `open()` and `openat()` syscalls. - -```console -student@os:~/.../lab/support/simple-file-handling$ strace -e open,openat ls > file.txt -openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 -openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3 -openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 -openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3 -openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3 -openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3 -openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3 -openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3 -openat(AT_FDCWD, ".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3 -``` - -This looks strange. -Where is the call to `openat(AT_FDCWD, "file.txt", ...)`? -Well, if we look at the full `strace` output, the first call is `execve()`: - -```console -student@os:~/.../lab/support/simple-file-handling$ strace ls > file.txt -execve("/usr/bin/ls", ["ls"], 0x7ffe550d59e0 /* 60 vars */) = 0 -[...] -``` - -[Quiz](../quiz/execve.md) - -So the `openat()` syscalls we saw earlier come from the `ls` process. -Remember how launching a command works in Bash: - -1. The Bash process `fork()`s itself -1. The child (still a Bash process) then calls `execve()` -1. The parent (the original Bash process) calls `waitpid()` to wait for the new process to end. - -![Launching a new command in Bash](../media/fork-exec.svg) - -So we can deduce that `file.txt` is opened by the child process **before** calling `execve()`. -Note that despite replacing the VAS of the current process, `execve()` does **NOT** replace its file descriptor table. -So whatever files were opened by the original process remain open within the new one. -This behaviour is the basis of pipes (the `|` that you use in Bash to use the output of a command as the input of another) and redirections (`>`, `<` and `2>`). - -## Practice: Naive Redirection - -But before diving into reimplementing shell functionalities, let's look at a simpler example. -Navigate to `support/redirect/redirect.c`. -The code makes a naive attempt at redirecting the newly opened file to `stdout`. -It simply closes `stdout` first so that when `open()` returns **the lowest available file descriptor**, that value will be 1, which is `STDOUT_FILENO`. - -Note there's a difference between `stdout` and `STDOUT_FILENO`. -While `stdout` is of type `FILE *` and is meant to be used with libc functions such as `fread()`, `STDOUT_FILENO` is the default file descriptor for the standard output, which almost always is 1. -So `STDOUT_FILENO` is an `int` type with the value 1. -Don't confuse them! - -Compile and run the code without modifying it. -It pauses after each file descriptor operation. -In another terminal (or in another `tmux` window), run the following each time you press Enter in the first terminal/window: `lsof -p $(pidof redirect)` or try [`watch`](https://man7.org/linux/man-pages/man1/watch.1.html)-ing it. - -`lsof` displays the files opened by the given process. -From the output below, we see that these files are mainly files (opened as file descriptors) and libraries, which are memory mapped (`mem`). -On the third column, you can see the file descriptor corresponding to each file. - -```console -student@os:~/.../lab/support/redirect$ lsof -w -p $(pidof redirect) # before any file operations -COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME -redirect 44303 student cwd DIR 8,1 4096 299870 /home/student/operating-systems-oer/content/chapters/io/lab/support/redirect -redirect 44303 student rtd DIR 259,6 4096 2 / -redirect 44303 student txt REG 8,1 25848 299929 /home/student/operating-systems-oer/content/chapters/io/lab/support/redirect/redirect -redirect 44303 student mem REG 259,6 2029592 1857435 /usr/lib/x86_64-linux-gnu/libc-2.31.so -redirect 44303 student mem REG 259,6 191504 1835092 /usr/lib/x86_64-linux-gnu/ld-2.31.so -redirect 44303 student 0u CHR 136,0 0t0 3 /dev/pts/0 -redirect 44303 student 1u CHR 136,0 0t0 3 /dev/pts/0 -redirect 44303 student 2u CHR 136,0 0t0 3 /dev/pts/0 -``` - -Notice that all 3 default file descriptors are first liked to `/dev/pts/0`. -It may be different on your machine, but most likely it will be `/dev/pts/`. -This is a [character device](./devices.md) that signifies your current (pseudo)terminal. - -```console -student@os:~/.../lab/support/redirect$ lsof -w -p $(pidof redirect) # after closing `STDOUT_FILENO` -redirect 46906 student 0u CHR 136,0 0t0 3 /dev/pts/0 -redirect 46906 student 2u CHR 136,0 0t0 3 /dev/pts/0 -``` - -See that file descriptor 1 (`stdout`) has "disappeared". -Now the second entry in the process's FD table is free. - -```console -student@os:~/.../lab/support/redirect$ lsof -w -p $(pidof redirect) # after opening `redirect_file.txt` -redirect 46906 student 0u CHR 136,0 0t0 3 /dev/pts/0 -redirect 46906 student 1w REG 8,1 0 299958 /.../lab/support/redirect/redirect_file.txt -redirect 46906 student 2u CHR 136,0 0t0 3 /dev/pts/0 -``` - -`open()` has assigned the newly opened file to the lowest file descriptor available, which is 1, the former `stdout`. -`printf()` writes its output to `stdout`, which in this case is `redirect_file.txt`. -Now inspect the contents of this file to make sure that string was written there. - -[Quiz](../quiz/prints-work-no-stdio.md) - -## Practice: Thread-unsafe Redirection - -This is all fine, but it doesn't allow us to **copy** file descriptors. -We can only replace an existing file descriptor with another one. -To be able to do replacements, we can use the [`dup()`](https://man7.org/linux/man-pages/man2/dup.2.html) syscall. -It simply creates a new file descriptor that refers to the same open file `struct` as the one given to it as an argument. -Both file descriptors remain active after calling `dup()`. - -Change the `do_redirect()` function in `support/redirect/redirect.c` to employ this new logic. -It should follow the steps below. -Track them using `lsof`, just like before. - -Step 1: - -- 0 -> `stdin` -- 1 -> `stdout` -- 2 -> `stderr` - -Step 2 - after `open("redirect_file.txt", ...)`: - -- 0 -> `stdin` -- 1 -> `stdout` -- 2 -> `stderr` -- 3 -> `redirect_file.txt` - -Step 3 - after `close(STDOUT_FILENO)`: - -- 0 -> `stdin` -- 2 -> `stderr` -- 3 -> `redirect_file.txt` - -Step 4 - after `dup(3)`. -Note that now both 1 and 3 are linked to `redirect_file.txt`, so we managed to successfully copy file descriptor 3. - -- 0 -> `stdin` -- 1 -> `redirect_file.txt` -- 2 -> `stderr` -- 3 -> `redirect_file.txt` - -Step 5 - after `close(3)`. -We don't need file descriptor 3 at this point anymore. - -- 0 -> `stdin` -- 1 -> `redirect_file.txt` -- 2 -> `stderr` - -## Practice: Safe Redirection - -`dup()` is all fine and dandy, but what if 2 threads use the steps above concurrently? -Because steps 3 and 4 don't happen atomically, they risk having their results inverted. -Take a look at `support/redirect/redirect_parallel.c`. -Compile and run the code, then inspect the resulting files. -You'll notice they contain opposing strings: - -```console -student@os:~/.../lab/support/redirect$ cat redirect_stderr_file.txt -Message for STDOUT - -student@os:~/.../lab/support/redirect$ cat redirect_stdout_file.txt -Message for STDERR -``` - -What happens is that thread 1 is forced to call `close(STDOUT_FILENO)`, then thread 2 calls `close(STDERR_FILENO)`. -So far, this is not problematic. -But then thread 2 continues to `dup()` `redirect_stderr.txt` into the lowest file descriptor available, which is 1 (`STDOUT_FILENO`). -Then thread 1 resumes to `dup()` `redirect_stdout.txt` into file descriptor 2 (`STDERR_FILENO`). -Thus we end up redirecting `stdout` and `stderr` to the opposite files than those we intended. - -To fix this, we need to call an **atomic** syscall, called [`dup2()`](https://man7.org/linux/man-pages/man2/dup.2.html). -It receives 2 file descriptors (`dup2(src_fd, dst_fd)`) and its actions are equivalent to the following, but performed atomically: - -```c -close(dst_fd); -dup(src_fd); // This places `src_fd` into the previous `dst_fd` -``` - -1. Modify `support/redirect/redirect_parallel.c` and change the calls to `close()` and `dup()` to calls to `dup2()` and check the contents of the resulting files to see they're correct. - -1. Now go back to `support/redirect/redirect.c` and refactor the code in `do_redirect()` to use `dup2()` as well. - -## Practice: Mini-shell Reloaded - -Remember the mini-shell you implemented in the Arena of the previous lab. -It is capable of `fork()`-ing itself and `execvp()`-ing commands, just like Bash. -We can now extend it to allow redirecting `stdout` to a file. - -Use what you've learnt so far in this section to allow this mini-shell to redirect the output of its commands to files. -Redirection is performed just like in bash via `>`. diff --git a/content/chapters/io/lab/content/remote-io.md b/content/chapters/io/lab/content/remote-io.md deleted file mode 100644 index fcb2bc3de4..0000000000 --- a/content/chapters/io/lab/content/remote-io.md +++ /dev/null @@ -1,246 +0,0 @@ -# Remote I/O - -In the previous sections, we started looking into how applications interact with the outside world. -However, so far this "outside world" has only meant **local** files and other **local** processes. -But what about files located on other computers? -What about "talking to" processes running in other parts of the world? - -## One Browser - Many Connections - -What happens when we request a web page? -A simple example is . -When the browser displays this web page, it first **downloads** it and then **renders** it. -The web page comes as a file called `index.html`. -We can roughly redo its steps like this: - -```console -student@os:~$ wget http://example.com/ # download index.html -wget http://example.com/ ---2022-12-02 15:53:31-- http://example.com/ -Resolving example.com (example.com)... 2606:2800:220:1:248:1893:25c8:1946, 93.184.216.34 -Connecting to example.com (example.com)|2606:2800:220:1:248:1893:25c8:1946|:80... connected. -HTTP request sent, awaiting response... 200 OK -Length: 1256 (1,2K) [text/html] -Saving to: ‘index.html’ - -index.html 100%[====================================================================================================================================================================>] 1,23K --.-KB/s in 0s - -2022-12-02 15:53:31 (248 MB/s) - ‘index.html’ saved [1256/1256] -``` - -Then we can view the HTML contents of the file: - -```html -student@os:~$ cat index.html - - - - Example Domain - - - - - - - - -
-

Example Domain

-

This domain is for use in illustrative examples in documents. You may use this - domain in literature without prior coordination or asking for permission.

-

More information...

-
- - -``` - -And, finally, we can render the `index.html` file in the browser like so: - -```console -student@os:~$ xdg-open index.html # xdg-open invokes the OS's default program for opening HTML files -``` - -After running the command above, look at the URL in the browser. -It's not anymore, but the path to your **local** `index.html`. - -So now you've switched from doing **remote** I/O back to **local** I/O. -Deluge does the same thing: it performs **remote** I/O operations to get files locally so that you, the user, can do **local** I/O with it. -Remote and local I/O are not by any means separated. -Rather, they are strongly intertwined. - -## Connection - -Focusing on how the `index.html` file is sent over the Web, we first need to establish the 2 **endpoints**. -We have a **sender**: the server. -Then we have a **receiver**: our host. -How do the 2 endpoints know each other? - -### Recap: IPs - -When someone asks you who you are on the internet, the answer is obvious: your **IP Address**. -IP Addresses are 32-bit (or 128-bit for IPv6) numbers that identify hosts on the web. -To find the IPv4 and IPv6 addresses of a host given by a URL, you can use the `host` command: - -```console -student@os:~$ host example.com -example.com has address 93.184.216.34 -example.com has IPv6 address 2606:2800:220:1:248:1893:25c8:1946 -example.com mail is handled by 0 . -``` - -So we can imagine our browser identifies by its IP address, say `93.184.216.34`. -Similarly, the server also knows our host's IP address. -Each of them uses the other's IP address to locate their peer and communicate with them. - -But what if we shift our example to another site: ? -Let's say we open 2 tabs: - -- one to the [File Descriptors section](https://open-education-hub.github.io/operating-systems/Lab/IO/file-descriptors) of this lab -- another one to the [File Handling section](https://open-education-hub.github.io/operating-systems/Lab/IO/file-handlers) - -Now our browser needs to **know** what to do with data coming from two sources. -In addition, the server also needs to maintain information about our 2 tabs so it can send the correct data to each of them. -Therefore, each tab establishes a different **connection** to the server. -All communication between the tab and the site occurs within this connection. - -Now the question is: _how can we maintain 2 connections between 2 endpoints only identified by IPs?_ -... and the answer is that we can't. -If the browser and the server were to only use IP addresses, they wouldn't be able to differentiate between the 2 connections mentioned above. -We need something more: **ports** - -### Further than IPs: Ports - -A port is simply a **number** assigned to uniquely identify a connection from a host to another and to direct the data that's transferred between them. -This way, we can create multiple connections between the same 2 hosts. -Port numbers are encoded on 16 bits and thus range from $0$ to $2^{16} - 1$, i.e. from $0$ to $65535$. - -The first 1024 ports are reserved for well-known system services, such as SSH (which uses port 22). -These services run using certain **communication protocols**. -A communication protocol is a set of rules that allow 2 or more hosts to communicate a certain way. -These rules include, but are not limited to: - -- the format of each message: fields, sizes, maximum lengths etc. -- the order in which messages are sent -- the behaviour of the endpoints with respect to these messages - -So the correct way of saying it isn't that the SSH process / service runs on port 22, but rather that **the SSH protocol runs on port 22**. - -Our browser also uses ports to distinguish between different connections. -And so does the `github.io` server: it uses one port for sending the "File Descriptors" page and another for the "File Handling" page. -The image below shows how multiple tabs to the same site can be handled. -The port numbers are chosen randomly. -They may have any value higher than 1023. - -![Browser Tabs and Ports](../media/browser-tabs.svg) - -So it should be clear now that a connection is uniquely identified by **an IP and a port**. - -## API - Hail Berkeley Sockets - -Up to now we've described how sites work in general. -Before we can implement something of this sort ourselves, we need to understand the API. -Unlike other APIs such as syscalls, which differ between OSs (Unix vs Windows for example), the one we're about to learn is almost universally adopted across OSs and programming languages. -It's called the [Berkeley Sockets API](https://en.wikipedia.org/wiki/Berkeley_sockets). -And with this, we've just introduced a new word: **socket**. - -![Not this Socket](../media/standard-us-power-outlet.jpg) - -No, not this type of socket... -But our socket is somewhat similar _in concept_. -Just like wall sockets allow us to plug into the electric grid, network sockets allow us to "plug" into the Web. -Remember [file handlers](./file-handlers.md)? -You should. -File handlers are objects with which we can interact with files. -Similarly, sockets are handlers that provide an abstraction for a connection to another process, running either on a remote machine or on the same host. - -### Sender and Receiver - -We'll start with 2 programs: a sender and a receiver. -Navigate to `support/send-receive/` and take a look at both `sender.py` and `receiver.py`. - -The sender reads data from the keyboard and sends it to the receiver. -The receiver reads data continuously. -Upon receiving the message `"exit"`, it closes. -Otherwise, it prints whatever it receives to `stdout`. -This detail about how to handle a message containing `"exit"` may be regarded as a [communication protocol](#further-than-ips-ports) established between the sender and the receiver. - -Now open 2 terminals (or use [tmux](https://tmuxcheatsheet.com/)). -First run `receiver.py` in one terminal. -Looking at its code, the receiver does 2 things. -It creates a socket: - -```Python -sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -``` - -We'll explain the arguments in the [next section](./networking-101.md#udp). -One thing to note here is that **sockets are file descriptors too.** - -The server displays its PID. -Give it as an argument to `lsof`, like you did in [the section on Redirections](./redirections.md), to visualise the file descriptors opened by `receiver.py`. - -[Quiz 1](../quiz/receiver-socket-fd.md) - -After creating the socket, the receiver exposes itself as "listening" for connections on IP `127.0.0.1` and on port 5000. -This means that it is now ready and waiting for other processes to send messages to it. - -[Quiz 2](../quiz/bind-error-cause.md) - -**Remember:** -**`bind()`-ing to a certain port locks (like a mutex) it for the current process.** -**No other socket may `bind()` to this port until the initial socket bound to it is `close()`d.** - -Now run `sender.py` and type some messages. -See them appear in the terminal where `receiver.py` is running. -The sender creates a socket and then sends messages directly to IP `127.0.0.1` and port 5000 using `sendto()`. -Without stopping the first `sender.py` create another one in another terminal. -Type messages to both senders and see all of them displayed by `receiver.py` along with the addresses and ports from where they came. - -In the end, both `sender.py` and `receiver.py` `close()` their sockets. -You can see this in analogy to regular file descriptors. - -So we can use `sendto()` and `recvfrom()` to send and receive messages via sockets. -This is a very simple communication model where the receiver is like a "sink" that receives messages from anywhere. -As you can see, it has no control over who sends data to it. -To get a high-level understanding of how these messages are passed and what other models exist, head over to the [next section](./networking-101.md). - -### Practice - -1. Use the C sockets API to replicate the behavior of `sender.py` and `receiver.py`. - - Start from the skeleton in `support/send-receive`. - The workflow is the same: define the endpoint using IP (`localhost`) and port (`5000`), then communicate using `sendto()` and `recvfrom()`. - We recommend starting with `sender.c` and test it using `receiver.py`, then continue with `receiver.c` and test it using `sender.py`. - If everything is right, each sender should be able to communicate with each receiver. - -1. Use the API you've just learned about to fill in the `TODO`s in `support/receive-challenges/receive_net_dgram_socket.c`. - - This is like `receiver.py`. - For it to run properly, you should compile it using `make`, then run it and after that run `send_net_dgram_socket`. - If you solved the challenge correctly, `receive_net_dgram_socket` should display the flag. diff --git a/content/chapters/io/lab/content/zero-copy.md b/content/chapters/io/lab/content/zero-copy.md deleted file mode 100644 index 14c4a692ca..0000000000 --- a/content/chapters/io/lab/content/zero-copy.md +++ /dev/null @@ -1,62 +0,0 @@ -# Zero-Copy - -Imagine a server that responds with files that it stores locally. -Its actions would be those highlighted in the image below: - -1. Receive a new request and extract the filename -1. Read the filename from the disk into memory -1. Send the file from memory to the client - -![Client-Server Steps](../media/client-server-file.svg) - -**The quiz below is tricky, yet very important.** -**Do NOT skip it in order for this section to make sense!** - -[Quiz](../quiz/server-copies.md) - -As you might have guessed, 2 of these copies are useless. -Since the app doesn't modify the file, there's no need for it to store the file in its own buffer. -It would be more efficient to use **a single** kernel buffer as intermediate storage between the disk and the NIC, as shown in the image below. - -![Server Copies - Zero-Copy](../media/server-copies-zero-copy.svg) - -For an easier comparison with the "classic" `read()` + `send()` model, here's the first version again: - -![Server Copies - Read-Send](../media/server-copies-normal.svg) - -It should be obvious that the former approach is more efficient than the latter. -The syscall with which we can leverage **zero-copy** is called [`sendfile()`](https://man7.org/linux/man-pages/man2/sendfile.2.html). - -## Practice: Measure It - -So we have all the reasons to believe zero-copy is the faster of the two approaches we know. -But belief alone is meaningless. -Let's test it! - -First, look at the code in `support/zero-copy/server.py`. -It spawns 2 threads. -One of them listens on port 8081 and serves connections via `read()` and `send()`. -The other listens on port 8082 and serves connections via `sendfile()`. -As you can see, the difference between them is minimal. - -First generate the test file using the Makefile. -Then start the server in one terminal. -Now, in another one, use `benchmark_client.py` to benchmark both implementations. -Below are some generic results. -Yours might differ by quite a lot, as they depend on your disk, your NIC, your kernel, your Python version, the load on your system etc. - -```console -student@os:/.../support/zero-copy$ python3 benchmark_client.py read-send -Time taken: 7.175773588009179 seconds - -student@os:/.../support/zero-copy$ python3 benchmark_client.py sendfile -Time taken: 3.71454380400246 seconds -``` - -This is quite good! -Using `sendfile()` halves the number of copies needed from 4 to 2. -Thus, it makes sense for the running time to _roughly_ halve as well. - -[Quiz](../quiz/fewer-than-2-copies.md) - -You can read a slightly more detailed article about zero-copy [here](https://developer.ibm.com/articles/j-zerocopy/). diff --git a/content/chapters/io/lab/media/.gitignore b/content/chapters/io/lab/media/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/content/chapters/io/lab/media/browser-tabs.svg b/content/chapters/io/lab/media/browser-tabs.svg deleted file mode 100644 index 022f129c14..0000000000 --- a/content/chapters/io/lab/media/browser-tabs.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Tab 1: File Descriptors
Tab 1: File Descripto...
Tab 1: File Handling
Tab 1: File Handling
Port: 34789
Port: 34789
Port: 25678
Port: 25678
github.io Server
github.io Server
Port: 31234
Port: 31234
Port: 32987
Port: 32987Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lab/media/connection-establishment.svg b/content/chapters/io/lab/media/connection-establishment.svg deleted file mode 100644 index 1c0c5f184d..0000000000 --- a/content/chapters/io/lab/media/connection-establishment.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Server
Server
Client
Client
socket()
socket()
bind()
bind()
listen()
listen()
accept()
accept()
recv()
recv()
send()
send()
close()
close()
socket()
socket()
connect()
connect()
send()
send()
recv()
recv()
close()
close()
Time
Time
Connection Established
Connection...Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lab/media/standard-us-power-outlet.jpg b/content/chapters/io/lab/media/standard-us-power-outlet.jpg deleted file mode 100644 index 926c7b17fb..0000000000 Binary files a/content/chapters/io/lab/media/standard-us-power-outlet.jpg and /dev/null differ diff --git a/content/chapters/io/lab/quiz/deluge-tcp-udp.md b/content/chapters/io/lab/quiz/deluge-tcp-udp.md deleted file mode 100644 index c91c593e32..0000000000 --- a/content/chapters/io/lab/quiz/deluge-tcp-udp.md +++ /dev/null @@ -1,19 +0,0 @@ -# Deluge: TCP or UDP - -## Question Text - -Should Deluge use UDP or TCP to transfer torrent files? - -## Question Answers - -- It should use UDP for faster file transfers - -+ It should use TCP to guarantee the integrity of the transferred files - -## Feedback - -Speed is nice to have. -Correctness is mandatory in most scenarios, including this one. -The only situation when correctness may be overlooked is when some given data will be quckly replaced by some other data. -But files are persistent. -If you download a video game from as a torrent (we've all done that), you want to keep it for a while and first and foremost, it has to work properly, i.e. not be corrupt. diff --git a/content/chapters/io/lab/quiz/execve.md b/content/chapters/io/lab/quiz/execve.md deleted file mode 100644 index ed81d4acfe..0000000000 --- a/content/chapters/io/lab/quiz/execve.md +++ /dev/null @@ -1,26 +0,0 @@ -# Effect of `execve()` Syscall - -## Question Text - -What is the effect of the `execve()` syscall? - -## Question Answers - -- it spawns a new process as the child of the current one - -- it executes a given shell command - -+ it replaces the VAS of the current process with that of the file given as argument - -- it spawns a new shell and executes the given command - -## Feedback - -The [`man` page](https://man7.org/linux/man-pages/man2/execve.2.html) says it all: - -> execve() executes the program referred to by pathname. This -> causes the program that is currently being run by the calling -> process to be replaced with a new program, with newly initialized -> stack, heap, and (initialized and uninitialized) data segments. - -Simply put, we can say that `execve()` replaces the VAS of the current process with that of the program given as argument. diff --git a/content/chapters/io/lab/quiz/fopen-w.md b/content/chapters/io/lab/quiz/fopen-w.md deleted file mode 100644 index bfa4aef332..0000000000 --- a/content/chapters/io/lab/quiz/fopen-w.md +++ /dev/null @@ -1,27 +0,0 @@ -# `open()` equivalent of `fopen(..., "w")` - - - -## Question Text - -Use `strace` on the code in `support/simple-file-operations/file_operations.c` to find the flags used by `openat()` when calling `fopen(file_name, "w")`. -First, try to make an educated guess and only then verify your answer by running `strace`. - -## Question Answers - -+ `O_WRONLY | O_CREAT | O_TRUNC` - -- `O_WRONLY | O_CREAT` - -- `O_WRONLY` - -- `O_WRONLY | O_TRUNC` - -## Feedback - -```console -student@os:~/.../lab/support/simple-file-operations$ strace ./file_operations -[...] -openat(AT_FDCWD, "file.txt", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 3 -[...] -``` diff --git a/content/chapters/io/lab/quiz/o-trunc.md b/content/chapters/io/lab/quiz/o-trunc.md deleted file mode 100644 index 70394adb7e..0000000000 --- a/content/chapters/io/lab/quiz/o-trunc.md +++ /dev/null @@ -1,23 +0,0 @@ -# `O_TRUNC` Flag Behaviour - - - -## Question Text - -How does the `O_TRUNC` flag change the behaviour of `open()`? - -## Question Answers - -+ the previous content of the file is deleted - -- new data will be appended to the file - -- newly written data will be ignored - -## Feedback - -The man page provides us with unlimited wisdon: - -> If the file already exists and is a regular file and the access mode allows writing (i.e., is `O_RDWR` or `O_WRONLY`) it will be truncated to length 0. - -Setting the length of the file to 0 is equivalent to deleting the previous content of the file. diff --git a/content/chapters/io/lab/quiz/prints-work-no-stdio.md b/content/chapters/io/lab/quiz/prints-work-no-stdio.md deleted file mode 100644 index 07f6a52240..0000000000 --- a/content/chapters/io/lab/quiz/prints-work-no-stdio.md +++ /dev/null @@ -1,20 +0,0 @@ -# Prints Working after Closing `stdio` - -## Question Text - -Why does `support/redirect/redirect.c`, still print messages to the console after closing file descriptor 1 (`stdout`)? - -## Question Answers - -+ because `wait_for_input()` calls `fprintf(stderr, ...)`, which prints to `stderr` (file descriptor 2) - -- because the default file descriptors cannot be "truly" closed - -- because the other two default file descriptors are still linked to the console - -- because the `wait_for_input()` function started printed before closing the `stdout` file descriptor - -## Feedback - -If you look at `wait_for_input()` closely, you'll notice it calls `fprintf(stderr, ...)`. -`stderr` is liked to file descriptor 2, which is left unchanged so we can still write data to it. diff --git a/content/chapters/io/lab/quiz/write-file-permissions.md b/content/chapters/io/lab/quiz/write-file-permissions.md deleted file mode 100644 index d42c5d5dc6..0000000000 --- a/content/chapters/io/lab/quiz/write-file-permissions.md +++ /dev/null @@ -1,55 +0,0 @@ -# `write_file.txt` Permissions - - - -## Question Text - -Assume `write_file.txt` is opened like this: -`open("write_file.txt", O_WRONLY | O_CREAT)`. -What might cause it to have different permissions than `read_file.txt`? - -## Question Answers - -- an undefined behaviour in the kernel - -- using the `O_WRONLY` flag - -+ not passing a `mode` argument to `open()`, so a random set of permissions is used. - -- a filesystem error. - -## Feedback - -Quoting from [`open()`'s man page](https://man7.org/linux/man-pages/man2/open.2.html), regarding the `O_CREAT` flag: - -> The mode argument must be supplied if O_CREAT or O_TMPFILE is -> specified in flags; if it is not supplied, some arbitrary -> bytes from the stack will be applied as the file mode. - -The wording "arbitrary bytes from the stack" is a bit obsolete on x86-64 systems. - -The x86 calling convention specifies that arguments are pushed on the stack, and because the `open()` wrapper is a variadic function in `libc` this would result in arbitrary bytes being read from the stack when `mode` is required but not provided. - -On the other hand, the x86-64 calling convention places arguments in registers. - -A correct `open()` call would look something like this: - -```c -open("write_file.txt", O_WRONLY | O_CREAT, 0644). -``` - -`0644` is the octal representation of `rw-r--r--`. -The approximate x86-64 assembly code for this function call would look like this: - -```asm -mov rdi, path_to_file ; first argument: path - -mov rsi, O_WRONLY -or rsi, O_CREAT ; second argument: flags - -mov rdx, 0644 ; third argument: mode - -call open -``` - -When we don't specify a `mode` argument, the `open()` function simply takes this value from `rdx` as it was before the function call, resulting in undefined behavior. diff --git a/content/chapters/io/lab/solution/buffering/.gitignore b/content/chapters/io/lab/solution/buffering/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/content/chapters/io/lab/solution/buffering/Makefile b/content/chapters/io/lab/solution/buffering/Makefile deleted file mode 120000 index 88c55d033b..0000000000 --- a/content/chapters/io/lab/solution/buffering/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../support/buffering/Makefile \ No newline at end of file diff --git a/content/chapters/io/lab/solution/buffering/benchmark_buffering.sh b/content/chapters/io/lab/solution/buffering/benchmark_buffering.sh deleted file mode 120000 index 61bcb5ee7c..0000000000 --- a/content/chapters/io/lab/solution/buffering/benchmark_buffering.sh +++ /dev/null @@ -1 +0,0 @@ -../../support/buffering/benchmark_buffering.sh \ No newline at end of file diff --git a/content/chapters/io/lab/solution/buffering/diy_buffering.c b/content/chapters/io/lab/solution/buffering/diy_buffering.c deleted file mode 100644 index c1f42274f9..0000000000 --- a/content/chapters/io/lab/solution/buffering/diy_buffering.c +++ /dev/null @@ -1,138 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define USEC_PER_SEC 1000000 -#define MSEC_PER_SEC 1000 - -#define BUFSIZE 4096 - -#define MIN(a, b) ((a) < (b) ? (a) : (b)) - -static char buf[BUFSIZE]; -/* Starting and end offsets in `buf`. */ -static ssize_t buf_start, buf_end; - -static unsigned long diff_msec(struct timeval start, struct timeval end) -{ - return (USEC_PER_SEC * (end.tv_sec - start.tv_sec) + end.tv_usec - - start.tv_usec) / MSEC_PER_SEC; -} - -static size_t diy_fread(void *dst, size_t size, size_t nmemb, int fd) -{ - ssize_t to_read = size * nmemb; - size_t dst_pos = 0; - size_t to_copy; - - while (to_read) { - if (buf_start == buf_end) { - /* Buffer is empty. Fill it. */ - buf_end = read(fd, buf, BUFSIZE); - DIE(buf_end == -1, "read"); - - buf_start = 0; - } - - /* Copy data from the buffer to `dst`. */ - to_copy = MIN(to_read, buf_end - buf_start); - memcpy(dst + dst_pos, buf + buf_start, to_copy); - - /* Update all offsets. */ - buf_start += to_copy; - dst_pos += to_copy; - to_read -= to_copy; - } - - return nmemb; -} - -static size_t diy_fwrite(void *src, size_t size, size_t nmemb, int fd) -{ - ssize_t to_write = size * nmemb; - size_t src_pos = 0; - size_t to_copy; - - while (to_write) { - /** - * TODO: write the data first to `buf` and to the file when it - * fills up. - */ - to_copy = MIN(to_write, BUFSIZE - (size_t)buf_end); - memcpy(buf + buf_end, src + src_pos, to_copy); - buf_end += to_copy; - - if (buf_end == BUFSIZE) { - /* Buffer is full. Write it to the file. */ - ssize_t rc = write(fd, buf, BUFSIZE); - DIE(rc == -1, "write"); - - buf_end = 0; - } - - src_pos += to_copy; - to_write -= to_copy; - } - - return nmemb; -} - -int main(int argc, char *argv[]) -{ - char c = 'x'; - int fd; - int rc; - int mode; - long total_bytes = 0; - struct stat statbuf; - struct timeval start, end; - - DIE(argc != 3, "Usage: ./diy_buffering read/write "); - - if (!strcmp(argv[1], "read")) - mode = O_RDONLY; - else if (!strcmp(argv[1], "write")) - mode = O_WRONLY; - else - DIE(1, "Invalid mode"); - - fd = open(argv[2], mode); - DIE(fd == -1, "open"); - - rc = fstat(fd, &statbuf); - DIE(rc == -1, "fstat"); - - gettimeofday(&start, NULL); - - /* Read or write from / to the file 1 byte at a time. */ - while (total_bytes < statbuf.st_size) { - if (mode == O_RDONLY) - diy_fread(&c, 1, 1, fd); - else - diy_fwrite(&c, 1, 1, fd); - - total_bytes++; - } - - gettimeofday(&end, NULL); - - if (mode == O_RDONLY) - printf("Read %zu bytes from %s in %lu ms\n", total_bytes, - argv[2], diff_msec(start, end)); - else - printf("Wrote %zu bytes to %s in %lu ms\n", total_bytes, - argv[2], diff_msec(start, end)); - - rc = close(fd); - DIE(rc == -1, "close"); - - return 0; -} diff --git a/content/chapters/io/lab/solution/client-server/Makefile b/content/chapters/io/lab/solution/client-server/Makefile deleted file mode 100644 index 1fa27e5b49..0000000000 --- a/content/chapters/io/lab/solution/client-server/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARY = client -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/io/lab/solution/client-server/client.c b/content/chapters/io/lab/solution/client-server/client.c deleted file mode 100644 index 7e2059395e..0000000000 --- a/content/chapters/io/lab/solution/client-server/client.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define PORT 5000 -#define BUFSIZE 1024 - -const char *localhost = "127.0.0.1"; - -int read_input_with_prompt(char *buf) -{ - char *ret; - - printf("Message for receiver (type 'exit' to quit): "); - - memset(buf, 0, BUFSIZE); - ret = fgets(buf, BUFSIZE, stdin); - if (!ret) - return -1; - buf[strlen(buf)-1] = '\0'; - - return 0; -} - -void populate_sockaddr(struct sockaddr_in *sockaddr, const char *addr, int port) -{ - int ret; - - sockaddr->sin_family = AF_INET; - sockaddr->sin_port = htons(port); - ret = inet_aton(addr, (struct in_addr *) &sockaddr->sin_addr.s_addr); - DIE(ret == 0, "inet_aton"); -} - -int main(void) -{ - char buf[BUFSIZE]; - int ret, sockfd; - struct sockaddr_in server_addr; - - while (1) { - /* Get user input as a string. */ - ret = read_input_with_prompt(buf); - DIE(ret < 0, "read_input_with_prompt"); - - /* TODO 1: Create a TCP socket (SOCK_STREAM). */ - sockfd = socket(AF_INET, SOCK_STREAM, 0); - DIE(sockfd < 0, "socket"); - - /* TODO 2: Add address and port to server_addr. */ - /* HINT: use populate_sockaddr(). */ - populate_sockaddr(&server_addr, localhost, PORT); - - /* TODO 3: Connect to the server. */ - /* Use DIE to check for errors. */ - ret = connect(sockfd, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(ret < 0, "connect"); - - /* TODO 4: Send message from buf through the socket with send(). */ - ret = send(sockfd, buf, strlen(buf), 0); - DIE(ret < 0, "send"); - - /* If the message is "exit", break out of the loop. */ - if (strncmp(buf, "exit", 4) == 0) - break; - - /* TODO 5: Receive the response from the server with recv(). */ - ret = recv(sockfd, buf, BUFSIZE, 0); - DIE(ret < 0, "recv"); - buf[ret-1] = '\0'; - - /* Print the response. */ - printf("Received from server: %s\n", buf); - - /* TODO 6: Close the socket. */ - ret = close(sockfd); - DIE(ret < 0, "close"); - } - - return 0; -} diff --git a/content/chapters/io/lab/solution/client-server/client.py b/content/chapters/io/lab/solution/client-server/client.py deleted file mode 100644 index c406616c27..0000000000 --- a/content/chapters/io/lab/solution/client-server/client.py +++ /dev/null @@ -1,41 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause - -import socket # Exposes the Berkely Sockets API - -IP = "127.0.0.1" # Loopback IP address (localhost) -MAX_MESSAGE_LENGTH = 1024 -PORT = 5000 # Port to send data to (non-privileged ports are > 1023) - - -def main(): - while True: - # Get user input as a string. - message = input("Message for receiver (type 'exit' to quit): ") - - # TODO: Create a TCP socket (SOCK_STREAM). - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - # TODO: connect to the server. - # `connect()` returns when the server `accept()`s the connection. - sock.connect((IP, PORT)) - - # TODO: send the message to the server. The message must be encoded to - # bytes. - sock.send(message.encode()) - - # TODO: if the message is "exit", break out of the loop. - if message == "exit": - break - - # TODO: receive the response from the server. - response = sock.recv(MAX_MESSAGE_LENGTH).decode() - - # Print the response. - print(f"Received from server: {response}") - - # TODO: close the socket. - sock.close() - - -if __name__ == "__main__": - main() diff --git a/content/chapters/io/lab/solution/file-descriptors/.gitignore b/content/chapters/io/lab/solution/file-descriptors/.gitignore deleted file mode 100644 index 77e9185287..0000000000 --- a/content/chapters/io/lab/solution/file-descriptors/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -open_directory -open_read_write diff --git a/content/chapters/io/lab/solution/file-descriptors/Makefile b/content/chapters/io/lab/solution/file-descriptors/Makefile deleted file mode 100644 index 27eaaae023..0000000000 --- a/content/chapters/io/lab/solution/file-descriptors/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -BINARIES = open_read_write -include ../../../../../common/makefile/multiple.mk - -clean:: - -rm -f ../../support/file-descriptors/write_file.txt diff --git a/content/chapters/io/lab/solution/file-descriptors/file.txt b/content/chapters/io/lab/solution/file-descriptors/file.txt deleted file mode 100644 index f84f3d3917..0000000000 --- a/content/chapters/io/lab/solution/file-descriptors/file.txt +++ /dev/null @@ -1,5 +0,0 @@ -file.txt -Makefile -open_read_write -open_read_write.c -open_read_write.o diff --git a/content/chapters/io/lab/solution/file-descriptors/open_read_write.c b/content/chapters/io/lab/solution/file-descriptors/open_read_write.c deleted file mode 100644 index bda579a53a..0000000000 --- a/content/chapters/io/lab/solution/file-descriptors/open_read_write.c +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifdef BUFSIZ -#undef BUFSIZ -#endif - -#define BUFSIZ 1024 - -#define READ_FILE_NAME "../../support/file-descriptors/read_file.txt" -#define WRITE_FILE_NAME "../../support/file-descriptors/write_file.txt" - -static int open_file_for_reading(const char *file_name) -{ - int fd; - - fd = open(file_name, O_RDONLY); - DIE(fd < 0, "open"); - - printf("Opened file descriptor is for reading: %d\n", fd); - - return fd; -} - -static void read_from_file(char *buff, int fd, int bytes_to_read) -{ - int bytes_read; - int total_read = 0; - - /** - * ALWAYS use `read()` in a loop because it might not read all the - * required bytes. - */ - while (total_read < bytes_to_read) { - /** - * `total_read` bytes have been read so far - * `buff + total_read` is the address of the first byte to be - * read - * `bytes_to_read - total_read` bytes need to be read - */ - bytes_read = read(fd, buff + total_read, - bytes_to_read - total_read); - DIE(bytes_read < 0, "read"); - - /* 0 indicates the end of the file. */ - if (!bytes_read) - break; - - total_read += bytes_read; - } - - printf("Read %d bytes from file descriptor %d. File content is: %s", - total_read, fd, buff); -} - -static int open_file_for_writing(const char *file_name) -{ - int fd = 0; - - /** - * TODO 1: Open the file for writing. The file doesn't exist. `open()` - * must also create it. Remember to use `DIE()` to check for errors. - */ - fd = open(file_name, O_WRONLY | O_CREAT, 0644); - DIE(fd < 0, "open"); - - printf("Opened file descriptor is for writing: %d\n", fd); - - return fd; -} - -static void write_to_file(char *buff, int fd, int bytes_to_write) -{ - int bytes_written; - int total_written = 0; - - /** - * TODO 2: Write the buffer to the file descriptor. - * ALWAYS use `write()` in a loop because it might not read all the - * required bytes. - */ - while (total_written < bytes_to_write) { - /** - * `total_written` bytes have been written so far - * `buff + total_written` is the address of the first byte to be - * written - * `bytes_to_write - total_written` bytes need to be written - */ - bytes_written = write(fd, buff + total_written, - bytes_to_write - total_written); - DIE(bytes_written < 0, "write"); - - total_written += bytes_written; - } - - printf("Wrote %d bytes to file descriptor %d\n", total_written, - fd); -} - -int main(void) -{ - int fd_read; - int fd_write; - int fd_write2; - char buff[BUFSIZ] = { 0 }; - char message[] = "Something short"; - char message2[] = ""; - - fd_read = open_file_for_reading(READ_FILE_NAME); - read_from_file(buff, fd_read, sizeof(buff)); - - fd_write = open_file_for_writing(WRITE_FILE_NAME); - write_to_file(message, fd_write, sizeof(message)); - - /** - * TODO 3: Write to `READ_FILE_NAME` and then read the newly written - * data. Use the functions defined above. - */ - fd_write2 = open_file_for_writing(READ_FILE_NAME); - write_to_file(message, fd_write2, sizeof(message)); - - /* This moves the internal cursor of the file to the beginning. */ - lseek(fd_read, 0, SEEK_SET); - read_from_file(buff, fd_read, sizeof(buff)); - - /* TODO 4: `close()` the file `open()`-ed descriptors. */ - close(fd_read); - close(fd_write); - close(fd_write2); - - return 0; -} diff --git a/content/chapters/io/lab/solution/file-mappings/Makefile b/content/chapters/io/lab/solution/file-mappings/Makefile deleted file mode 120000 index a9cc35d28c..0000000000 --- a/content/chapters/io/lab/solution/file-mappings/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../support/file-mappings/Makefile \ No newline at end of file diff --git a/content/chapters/io/lab/solution/file-mappings/benchmark_cp.sh b/content/chapters/io/lab/solution/file-mappings/benchmark_cp.sh deleted file mode 120000 index 8fe3d6c9be..0000000000 --- a/content/chapters/io/lab/solution/file-mappings/benchmark_cp.sh +++ /dev/null @@ -1 +0,0 @@ -../../support/file-mappings/benchmark_cp.sh \ No newline at end of file diff --git a/content/chapters/io/lab/solution/file-mappings/mmap_cp.c b/content/chapters/io/lab/solution/file-mappings/mmap_cp.c deleted file mode 100644 index 263aca9e87..0000000000 --- a/content/chapters/io/lab/solution/file-mappings/mmap_cp.c +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -static void wait_for_input(const char *msg) -{ - char buf[32]; - - printf(" * %s\n", msg); - printf(" -- Press ENTER to continue ..."); - fflush(stdout); - fgets(buf, 32, stdin); -} - -int main(int argc, char *argv[]) -{ - int fdin; - int fdout; - int rc; - char *src; - char *dst; - struct stat statbuf; - - DIE(argc != 3, "Usage: ./mmap_cp "); - - /* TODO: Open the input file. */ - fdin = open(argv[1], O_RDONLY); - DIE(fdin == -1, "open fdin"); - - /* TODO: Open/create the output file. */ - fdout = open(argv[2], O_RDWR | O_CREAT | O_TRUNC, 0644); - DIE(fdout == -1, "open fdout"); - - /** - * TODO: Truncate the output file to the input file size. Use `fstat()` - * to get the size of the input file and `ftruncate()` to set the size - * of the output file. - */ - rc = fstat(fdin, &statbuf); - DIE(rc == -1, "fstat"); - - rc = ftruncate(fdout, statbuf.st_size); - DIE(rc == -1, "ftruncate"); - - /* TODO: `mmap()` the input file. */ - src = mmap(0, statbuf.st_size, PROT_READ, MAP_SHARED, fdin, 0); - DIE(src == MAP_FAILED, "mmap src"); - - // wait_for_input("Mapped input file."); - - /* TODO: `mmap()` the output file. */ - dst = mmap(0, statbuf.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, - fdout, 0); - DIE(dst == MAP_FAILED, "mmap dst"); - - // wait_for_input("Mapped output file."); - - /** - * TODO: Copy the contents of the input mapping to the output mapping. - */ - memcpy(dst, src, statbuf.st_size); - - /* TODO: Clean up. Use `munmap()` to unmap the 2 files. */ - rc = munmap(src, statbuf.st_size); - DIE(rc == -1, "munmap source"); - - rc = munmap(dst, statbuf.st_size); - DIE(rc == -1, "munmap dest"); - - rc = close(fdin); - DIE(rc == -1, "close source"); - - rc = close(fdout); - DIE(rc == -1, "close dest"); - - return 0; -} diff --git a/content/chapters/io/lab/solution/mini-shell/.gitignore b/content/chapters/io/lab/solution/mini-shell/.gitignore deleted file mode 100644 index fb647f35e4..0000000000 --- a/content/chapters/io/lab/solution/mini-shell/.gitignore +++ /dev/null @@ -1 +0,0 @@ -mini_shell diff --git a/content/chapters/io/lab/solution/mini-shell/mini_shell.c b/content/chapters/io/lab/solution/mini-shell/mini_shell.c deleted file mode 100644 index db972184b6..0000000000 --- a/content/chapters/io/lab/solution/mini-shell/mini_shell.c +++ /dev/null @@ -1,186 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define MAX_LINE_SIZE 256 -#define MAX_ARGS 8 - -#define ERROR 0 -#define SIMPLE 1 -#define REDIRECT 2 -#define PIPE 3 -#define SHELL_EXIT 4 - -static char *verb; -static char **args; -static char *stdin_file; -static char *stdout_file; -static char *stderr_file; - -static int parse_line(char *line); -static void alloc_mem(void); -static void free_mem(void); - -static void do_redirect(int filedes, const char *filename) -{ - int rc; - int fd; - - /* TODO: Redirect `filedes` into `fd` representing `filename`. */ - fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); - DIE(fd < 0, "open"); - - /* `dup2()` closes the old `filedes` and replaces it with `fd`. */ - rc = dup2(fd, filedes); - DIE(rc < 0, "dup2"); - - rc = close(fd); - DIE(rc < 0, "close"); -} - -static void simple_cmd(char **args) -{ - pid_t pid; - pid_t wait_ret; - int status; - - /** - * Create a process to execute the command. Use `execvp` to launch the - * new process. - */ - pid = fork(); - switch (pid) { - case -1: - /* Error */ - DIE(1, "fork"); - break; - - case 0: - /* Child process */ - - /** - * Call `do_redirect()`. Assume the only file descriptor that needs to - * be redirected is `stdout`. Its file descriptor is `STDOUT_FILENO`. - */ - if (stdout_file != NULL) - do_redirect(STDOUT_FILENO, stdout_file); - - execvp(args[0], (char *const *) args); - DIE(1, "execvp"); - break; - - default: - /* Parent process */ - wait_ret = waitpid(pid, &status, 0); - DIE(wait_ret < 0, "waitpid"); - - if (WIFEXITED(status)) - printf("Child process (pid %d) terminated normally, " - "with exit code %d\n", - pid, WEXITSTATUS(status)); - } -} - -int main(void) -{ - int type; - char line[MAX_LINE_SIZE]; - - alloc_mem(); - - while (1) { - printf("> "); - fflush(stdout); - - memset(line, 0, MAX_LINE_SIZE); - - if (fgets(line, sizeof(line), stdin) == NULL) { - free_mem(); - exit(EXIT_SUCCESS); - } - - type = parse_line(line); - - switch (type) { - case SHELL_EXIT: - free_mem(); - exit(EXIT_SUCCESS); - - case SIMPLE: - simple_cmd(args); - break; - } - } - - return 0; -} - -static void alloc_mem(void) -{ - args = malloc(MAX_ARGS * sizeof(char *)); - DIE(args == NULL, "malloc"); -} - -static void free_mem(void) -{ - free(args); -} - -static int parse_line(char *line) -{ - int ret = SIMPLE; - int idx = 0; - char *token; - char *delim = "=\n"; - char *saveptr = NULL; - - stdin_file = NULL; - stdout_file = NULL; - stderr_file = NULL; - - /* Check for exit. */ - if (strncmp("exit", line, strlen("exit")) == 0) - return SHELL_EXIT; - - /* Regular command. */ - delim = " \t\n"; - token = strtok_r(line, delim, &saveptr); - - if (token == NULL) - return ERROR; - - verb = strdup(token); - - /* Copy args. */ - while (token != NULL) { - if (token == NULL) { - printf(" Expansion failed\n"); - return ERROR; - } - - if (strncmp(token, ">", strlen(">")) == 0) { - if (strlen(token) > strlen(">")) { - token++; - stdout_file = strdup(token); - } else { - token = strtok_r(NULL, delim, &saveptr); - stdout_file = strdup(token); - } - } else { - args[idx++] = strdup(token); - } - - token = strtok_r(NULL, delim, &saveptr); - } - - args[idx++] = NULL; - return ret; -} diff --git a/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.c b/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.h b/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/content/chapters/io/lab/solution/mini-shell/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/content/chapters/io/lab/solution/pipes/Makefile b/content/chapters/io/lab/solution/pipes/Makefile deleted file mode 100644 index de4c39c00f..0000000000 --- a/content/chapters/io/lab/solution/pipes/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARY = anonymous_pipe -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/io/lab/solution/pipes/anonymous_pipe.c b/content/chapters/io/lab/solution/pipes/anonymous_pipe.c deleted file mode 100644 index 3918464011..0000000000 --- a/content/chapters/io/lab/solution/pipes/anonymous_pipe.c +++ /dev/null @@ -1,133 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define BUFSIZE 128 -#define EXIT_STR "exit" -#define PIPE_READ 0 -#define PIPE_WRITE 1 - -static bool check_for_exit(const char *input) -{ - if (strcmp(input, EXIT_STR) == 0 || strlen(input) == 0) - { - return true; - } - - return false; -} - -static void child_loop(int readfd) -{ - char output[BUFSIZE]; - int rc; - - while (1) - { - rc = read(readfd, output, BUFSIZE); - DIE(rc < 0, "read"); - - if (rc == 0) - { - /* TODO: Close pipe head used for reading. */ - close(readfd); - break; - } - - printf("[Child received]: %s\n", output); - fflush(stdout); - } - - return; -} - -static void parent_loop(int writefd) -{ - char input[BUFSIZE]; - int rc; - - while (1) - { - memset(input, 0, BUFSIZE); - fgets(input, BUFSIZE, stdin); - // Remove trailing newline - if (input[strlen(input) - 1] == '\n') - input[strlen(input) - 1] = '\0'; - - if (check_for_exit(input)) - { - /* TODO: Close pipe head used for writing. */ - close(writefd); - break; - } - - rc = write(writefd, input, BUFSIZE); - DIE(rc < 0, "write"); - } - - return; -} - -static void wait_for_input(const char *msg) -{ - char buf[32]; - - fprintf(stderr, " * %s\n", msg); - fprintf(stderr, " -- Press ENTER to continue ..."); - fgets(buf, 32, stdin); -} - -int main(void) -{ - pid_t pid; - int rc; - int pipedes[2]; - - rc = pipe(pipedes); - DIE(rc < 0, "pipe"); - - printf("pipedes[0] = %d; pipedes[1] = %d\n", pipedes[PIPE_READ], pipedes[PIPE_WRITE]); - - wait_for_input("pipe created"); - - pid = fork(); - switch (pid) - { - case -1: /* Fork failed, cleaning up. */ - /* TODO: Close both heads of the pipe. */ - close(pipedes[PIPE_READ]); - close(pipedes[PIPE_WRITE]); - DIE(pid, "fork"); - return EXIT_FAILURE; - - case 0: /* Child process. */ - /* TODO: Close unused pipe head by child. */ - close(pipedes[PIPE_WRITE]); - - /* TODO: Call child loop and pass pipe head used for reading. */ - child_loop(pipedes[PIPE_READ]); - - break; - - default: /* Parent process. */ - /* TODO: Close unused pipe head by parent. */ - close(pipedes[PIPE_READ]); - - /* TODO: Call parent loop and pass pipe head used for writing. */ - parent_loop(pipedes[PIPE_WRITE]); - - /* Wait for child process to finish. */ - wait(NULL); - - break; - } - - return 0; -} diff --git a/content/chapters/io/lab/solution/receive-challenges/.gitignore b/content/chapters/io/lab/solution/receive-challenges/.gitignore deleted file mode 100644 index 83d969c8eb..0000000000 --- a/content/chapters/io/lab/solution/receive-challenges/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/receive_pipe -/receive_fifo -/receive_unix_socket -/receive_net_dgram_socket diff --git a/content/chapters/io/lab/solution/receive-challenges/Makefile b/content/chapters/io/lab/solution/receive-challenges/Makefile deleted file mode 100644 index ccced7591a..0000000000 --- a/content/chapters/io/lab/solution/receive-challenges/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARIES = receive_pipe receive_fifo receive_unix_socket receive_net_dgram_socket -include ../../../../../common/makefile/multiple.mk diff --git a/content/chapters/io/lab/solution/receive-challenges/receive_fifo.c b/content/chapters/io/lab/solution/receive-challenges/receive_fifo.c deleted file mode 100644 index 0088000554..0000000000 --- a/content/chapters/io/lab/solution/receive-challenges/receive_fifo.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include - -#include "utils/utils.h" - -static const char fifo_path[] = "../../support/receive-challenges/apollodorus"; - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -int main(void) -{ - int rc; - int fd; - char buffer[BUFSIZ]; - - /** - * Create FIFO if it doesn't exist or if it exists and has incorrect - * permissions. - */ - rc = access(fifo_path, R_OK | W_OK); - if (rc < 0) { - remove(fifo_path); - rc = mkfifo(fifo_path, 0755); - DIE(rc < 0, "mkfifo"); - } - - /* Open FIFO. */ - fd = open(fifo_path, O_RDONLY); - DIE(fd < 0, "open"); - - /* Read flag from FIFO. */ - rc = read(fd, buffer, BUFSIZ); - DIE(rc < 0, "read"); - - printf("Flag is: %s\n", buffer); - - return 0; -} diff --git a/content/chapters/io/lab/solution/receive-challenges/receive_net_dgram_socket.c b/content/chapters/io/lab/solution/receive-challenges/receive_net_dgram_socket.c deleted file mode 100644 index 545333876a..0000000000 --- a/content/chapters/io/lab/solution/receive-challenges/receive_net_dgram_socket.c +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -unsigned short port = 4242; - -int main(void) -{ - int rc; - int fd; - struct sockaddr_in addr; - socklen_t addrlen; - char buffer[BUFSIZ]; - - /* Create socket. */ - fd = socket(PF_INET, SOCK_DGRAM, 0); - DIE(fd < 0, "socket"); - - /* Bind socket to address. */ - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons(port); - addr.sin_addr.s_addr = htonl(INADDR_ANY); - - rc = bind(fd, (struct sockaddr *) &addr, sizeof(addr)); - DIE(rc < 0, "bind"); - - /* Read flag from socket. */ - rc = recvfrom(fd, buffer, BUFSIZ, 0, (struct sockaddr *) &addr, &addrlen); - DIE(rc < 0, "recvfrom"); - - printf("Flag is: %s\n", buffer); - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lab/solution/receive-challenges/receive_pipe.c b/content/chapters/io/lab/solution/receive-challenges/receive_pipe.c deleted file mode 100644 index f0d79e3a1e..0000000000 --- a/content/chapters/io/lab/solution/receive-challenges/receive_pipe.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -static pid_t create_process_with_redirect(const char *path, int new_stdin, - int new_stdout, int new_stderr) -{ - pid_t pid; - int rc; - - pid = fork(); - switch (pid) { - case -1: /* Error */ - DIE(pid < 0, "fork"); - break; - - case 0: /* Child process */ - if (new_stdin != -1) - dup2(new_stdin, 0); - if (new_stdout != -1) - dup2(new_stdout, 1); - if (new_stderr != -1) - dup2(new_stderr, 2); - - rc = execl(path, path, NULL); - DIE(rc < 0, "execl"); - } - - return pid; -} - -int main(void) -{ - int pipefd[2]; - char buf[BUFSIZ]; - int rc; - - rc = pipe(pipefd); - DIE(rc < 0, "pipe"); - - log_info("Created pipe with descriptors %d and %d.\n", pipefd[0], - pipefd[1]); - - /* Create child process. Ignore the PID return value. */ - create_process_with_redirect( - "../../support/receive-challenges/send_fd_4", -1, pipefd[1], - -1); - - /** - * Child process writes to `pipefd[1]`, parent process reads from - * `pipefd[0]`. - */ - close(pipefd[1]); - - rc = read(pipefd[0], buf, BUFSIZ); - DIE(rc < 0, "read"); - - printf("Flag is: %s\n", buf); - - /* Wait for child process. */ - wait(NULL); - - return 0; -} diff --git a/content/chapters/io/lab/solution/receive-challenges/receive_unix_socket.c b/content/chapters/io/lab/solution/receive-challenges/receive_unix_socket.c deleted file mode 100644 index d4f7bae378..0000000000 --- a/content/chapters/io/lab/solution/receive-challenges/receive_unix_socket.c +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -static const char socket_path[] = "../../support/receive-challenges/golden_gate"; - -int main(void) -{ - int rc; - int listenfd, connectfd; - struct sockaddr_un addr, raddr; - socklen_t raddrlen; - char buffer[BUFSIZ]; - - /* Remove socket_path. */ - remove(socket_path); - - /* Create socket. */ - listenfd = socket(PF_UNIX, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - /* Bind socket to path. */ - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strcpy(addr.sun_path, socket_path); - rc = bind(listenfd, (struct sockaddr *) &addr, sizeof(addr)); - DIE(rc < 0, "bind"); - - /* Put in listen mode. */ - rc = listen(listenfd, 10); - DIE(rc < 0, "listen"); - - /* Accept connection. */ - connectfd = accept(listenfd, (struct sockaddr *) &raddr, &raddrlen); - DIE(connectfd < 0, "accept"); - - /* Read flag from socket. */ - rc = read(connectfd, buffer, BUFSIZ); - DIE(rc < 0, "read"); - - printf("Flag is: %s\n", buffer); - - close(connectfd); - close(listenfd); - - return 0; -} diff --git a/content/chapters/io/lab/solution/redirect/Makefile b/content/chapters/io/lab/solution/redirect/Makefile deleted file mode 120000 index 501ac1093d..0000000000 --- a/content/chapters/io/lab/solution/redirect/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../support/redirect/Makefile \ No newline at end of file diff --git a/content/chapters/io/lab/solution/redirect/redirect.c b/content/chapters/io/lab/solution/redirect/redirect.c deleted file mode 100644 index 9e36a25891..0000000000 --- a/content/chapters/io/lab/solution/redirect/redirect.c +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define REDIRECT_FILE_NAME "../../support/redirect/redirect_file.txt" - -static void wait_for_input(const char *msg) -{ - char buf[32]; - - fprintf(stderr, " * %s\n", msg); - fprintf(stderr, " -- Press ENTER to continue ..."); - fgets(buf, 32, stdin); -} - -static void do_redirect(int filedes, const char *filename) -{ - int rc; - int fd; - - /** - * TODO 1: Refactor the code below to use `dup()` instead of - * `close()`-ing `filedes` and `open()`-ing the new file "on top" of - * the old file descriptor. - */ - /** - * fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); - * DIE(fd < 0, "open"); - * - * `close(filedes)` so that the `dup()` below copies `fd` into `filedes`. - * rc = close(filedes); - * DIE(rc < 0, "dup2"); - * - * Copy `fd` into `filedes`. - * rc = dup(fd); - * DIE(rc < 0, "dup"); - * - * Close the old `fd`. We already have it in `filedes`. - * rc = close(fd); - * DIE(rc < 0, "close"); - */ - - /** - * TODO 2: Refactor the code below to use `dup2()` instead of `close()` - * and `dup()`. - */ - - fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); - DIE(fd < 0, "open"); - - /* `dup2()` closes the old `filedes` before replacing it with `fd`. */ - rc = dup2(fd, filedes); - DIE(rc < 0, "dup2"); - - rc = close(fd); - DIE(rc < 0, "close"); -} - -int main(void) -{ - do_redirect(STDOUT_FILENO, REDIRECT_FILE_NAME); - - printf("Where did this message disappear?\n"); - - return 0; -} diff --git a/content/chapters/io/lab/solution/redirect/redirect_parallel b/content/chapters/io/lab/solution/redirect/redirect_parallel deleted file mode 100644 index 374ad66e3b..0000000000 Binary files a/content/chapters/io/lab/solution/redirect/redirect_parallel and /dev/null differ diff --git a/content/chapters/io/lab/solution/redirect/redirect_parallel.c b/content/chapters/io/lab/solution/redirect/redirect_parallel.c deleted file mode 100644 index ed7f69d3b4..0000000000 --- a/content/chapters/io/lab/solution/redirect/redirect_parallel.c +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define NUM_THREADS 2 - -#define REDIRECT_STDOUT_FILE_NAME \ - "../../support/redirect/redirect_stdout_file.txt" -#define REDIRECT_STDERR_FILE_NAME \ - "../../support/redirect/redirect_stderr_file.txt" - -struct redirected_print_args { - int fd; - void (*redirect_func)(); - const char *message; -}; - -static pthread_barrier_t barrier; - -static void do_stdout_redirect() -{ - int rc; - int fd; - - fd = open(REDIRECT_STDOUT_FILE_NAME, O_WRONLY | O_CREAT | O_TRUNC, - 0644); - DIE(fd < 0, "open"); - - /* Make sure both threads have opened their files. */ - pthread_barrier_wait(&barrier); - - /* TODO: Use `dup2()` instead of `close()` and `dup()`. */ - rc = dup2(fd, STDOUT_FILENO); - DIE(rc < 0, "dup2"); - - sleep(3); - - rc = close(fd); - DIE(rc < 0, "close"); -} - -static void do_stderr_redirect() -{ - int rc; - int fd; - - fd = open(REDIRECT_STDERR_FILE_NAME, O_WRONLY | O_CREAT | O_TRUNC, - 0644); - DIE(fd < 0, "open"); - - /* Make sure both threads have opened their files. */ - pthread_barrier_wait(&barrier); - - sleep(1); - - /* TODO: Use `dup2()` instead of `close()` and `dup()`. */ - rc = dup2(fd, STDERR_FILENO); - DIE(rc < 0, "dup2"); - - rc = close(fd); - DIE(rc < 0, "close"); -} - -void *redirected_print(void *arg) -{ - struct redirected_print_args *args = arg; - - args->redirect_func(); - - /* Make sure all redirects are made. */ - pthread_barrier_wait(&barrier); - - write(args->fd, args->message, strlen(args->message)); - - return NULL; -} - -int main(void) -{ - int rc; - size_t i; - pthread_t tids[NUM_THREADS]; - struct redirected_print_args args[NUM_THREADS] = { - { - .fd = STDOUT_FILENO, - .redirect_func = do_stdout_redirect, - .message = "Message for STDOUT\n" - }, - { - .fd = STDERR_FILENO, - .redirect_func = do_stderr_redirect, - .message = "Message for STDERR\n" - } - }; - - pthread_barrier_init(&barrier, NULL, NUM_THREADS); - - for (i = 0; i < NUM_THREADS; i++) { - rc = pthread_create(&tids[i], NULL, redirected_print, &args[i]); - DIE(rc < 0, "pthread_create"); - } - - for (i = 0; i < NUM_THREADS; i++) { - rc = pthread_join(tids[i], NULL); - DIE(rc < 0, "pthread_join"); - } - - pthread_barrier_destroy(&barrier); - - return 0; -} diff --git a/content/chapters/io/lab/solution/send-receive/.gitignore b/content/chapters/io/lab/solution/send-receive/.gitignore deleted file mode 100644 index 5a1c58ff00..0000000000 --- a/content/chapters/io/lab/solution/send-receive/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -sender -receiver diff --git a/content/chapters/io/lab/solution/send-receive/Makefile b/content/chapters/io/lab/solution/send-receive/Makefile deleted file mode 100644 index 6d3aba5b74..0000000000 --- a/content/chapters/io/lab/solution/send-receive/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARIES = receiver sender -include ../../../../../common/makefile/multiple.mk diff --git a/content/chapters/io/lab/solution/send-receive/receiver.c b/content/chapters/io/lab/solution/send-receive/receiver.c deleted file mode 100644 index ffb5affbe5..0000000000 --- a/content/chapters/io/lab/solution/send-receive/receiver.c +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "sock-utils.h" - -const char *localhost = "127.0.0.1"; - -int main(void) -{ - char buf[BUFSIZE]; - int ret, sockfd; - struct sockaddr_in receiver_addr; - struct sockaddr_in sender_addr; - socklen_t sender_addr_len; - - /* Create a socket. */ - sockfd = socket(AF_INET, SOCK_DGRAM, 0); - DIE(sockfd < 0, "socket"); - - /* TODO 1: Add address and port to receiver_addr. */ - /* HINT: use populate_sockaddr(). */ - populate_sockaddr(&receiver_addr, localhost, PORT); - - /* TODO 2: Bind sockfd to receiver_addr. Use DIE to check for errors. */ - ret = bind(sockfd, (struct sockaddr *) &receiver_addr, sizeof(receiver_addr)); - DIE(ret < 0, "bind"); - - printf("Server with PID %d is listening for data on port %d ...\n", getpid(), PORT); - - while (1) { - /* Prepare the buffer */ - memset(buf, 0, BUFSIZE); - - /* TODO 3: Receive at most BUFSIZE bytes through the socket with recvfrom(). */ - /* HINT: Use sender_addr as the fifth arg. You will need to cast it. */ - /* It will be populated with information about the sender. */ - /* Use DIE to check for errors. */ - ret = recvfrom(sockfd, buf, BUFSIZE, 0, (struct sockaddr *) &sender_addr, &sender_addr_len); - DIE(ret < 0, "recvfrom"); - - /* Ensure buf is null-terminated. */ - buf[BUFSIZE-1] = '\0'; - - /* Print the message. */ - printf("Received from IP %s and port %d: %s\n", inet_ntoa(sender_addr.sin_addr), - sender_addr.sin_port, buf); - - if (strncmp(buf, "exit", 4) == 0) - break; - } - - /* Close the socket. */ - ret = close(sockfd); - DIE(ret < 0, "close"); - - return 0; -} diff --git a/content/chapters/io/lab/solution/send-receive/sender.c b/content/chapters/io/lab/solution/send-receive/sender.c deleted file mode 100644 index 49b5a327c5..0000000000 --- a/content/chapters/io/lab/solution/send-receive/sender.c +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "sock-utils.h" - -const char *localhost = "127.0.0.1"; - -int read_input_with_prompt(char *buf) -{ - char *ret; - - printf("Message for receiver (type 'exit' to quit): "); - - memset(buf, 0, BUFSIZE); - ret = fgets(buf, BUFSIZE, stdin); - if (!ret) - return -1; - buf[strlen(buf)-1] = '\0'; - - return 0; -} - -int main(void) -{ - char buf[BUFSIZE]; - int ret, sockfd; - struct sockaddr_in receiver_addr; - - /* Create a socket. */ - sockfd = socket(AF_INET, SOCK_DGRAM, 0); - DIE(sockfd < 0, "socket"); - - /* TODO 1: Add address and port to receiver_addr. */ - /* HINT: use populate_sockaddr(). */ - populate_sockaddr(&receiver_addr, localhost, PORT); - - while (1) { - ret = read_input_with_prompt(buf); - DIE(ret < 0, "read_input_with_prompt"); - - /* TODO 2: Send message from buf through the socket with sendto(). */ - /* HINT: Use receiver_addr as the fifth arg. You will need to cast it. */ - /* Use DIE to check for errors. */ - ret = sendto(sockfd, buf, strlen(buf), 0, (struct sockaddr *) &receiver_addr, sizeof(receiver_addr)); - DIE(ret < 0, "sendto"); - - if (strncmp(buf, "exit", 4) == 0) - break; - } - - /* Close the socket */ - ret = close(sockfd); - DIE(ret < 0, "close"); - - return 0; -} diff --git a/content/chapters/io/lab/solution/send-receive/sock-utils.h b/content/chapters/io/lab/solution/send-receive/sock-utils.h deleted file mode 100644 index 705c3e3135..0000000000 --- a/content/chapters/io/lab/solution/send-receive/sock-utils.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -#pragma once - -#include -#include -#include - -#include "utils/utils.h" - -#define PORT 5000 -#define BUFSIZE 1024 - -void populate_sockaddr(struct sockaddr_in *sockaddr, const char *addr, int port) -{ - int ret; - - sockaddr->sin_family = AF_INET; - sockaddr->sin_port = htons(port); - ret = inet_aton(addr, (struct in_addr *)&sockaddr->sin_addr.s_addr); - DIE(ret == 0, "inet_aton"); -} diff --git a/content/chapters/io/lab/solution/sparse-file/.gitignore b/content/chapters/io/lab/solution/sparse-file/.gitignore deleted file mode 100644 index c1eda30a7d..0000000000 --- a/content/chapters/io/lab/solution/sparse-file/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/solve diff --git a/content/chapters/io/lab/solution/sparse-file/Makefile b/content/chapters/io/lab/solution/sparse-file/Makefile deleted file mode 100644 index 20b642fc25..0000000000 --- a/content/chapters/io/lab/solution/sparse-file/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARIES = generate solve -include ../../../../../common/makefile/multiple.mk diff --git a/content/chapters/io/lab/solution/sparse-file/solve.c b/content/chapters/io/lab/solution/sparse-file/solve.c deleted file mode 100644 index b7eff24c4e..0000000000 --- a/content/chapters/io/lab/solution/sparse-file/solve.c +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define FILENAME "../../support/sparse-file/swiss_cheese.sparse" -#define NUM_SPACES 10 - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -int main(void) -{ - int fd; - size_t i; - char buffer[BUFSIZ]; - - fd = open(FILENAME, O_RDONLY); - DIE(fd < 0, "open"); - - for (i = 0; i < BUFSIZ; i++) { - ssize_t n; - off_t pos; - - n = read(fd, buffer+i, 1); - DIE(n < 0, "read"); - - if (n == 0) - break; - - pos = lseek(fd, NUM_SPACES, SEEK_CUR); - DIE(pos < 0, "lseek"); - } - - printf("flag is %s\n", buffer); - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lab/support/async/c/.gitignore b/content/chapters/io/lab/support/async/c/.gitignore deleted file mode 100644 index 1afff08b8d..0000000000 --- a/content/chapters/io/lab/support/async/c/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/server -/mp_server -/mt_server diff --git a/content/chapters/io/lab/support/async/c/Makefile b/content/chapters/io/lab/support/async/c/Makefile deleted file mode 100644 index 1b7aa9aebe..0000000000 --- a/content/chapters/io/lab/support/async/c/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -BINARIES = server mp_server mt_server - -include ../../../../../../common/makefile/multiple.mk - -server: server.o ../../../../../../common/utils/sock/sock_util.o - -mp_server: mp_server.o ../../../../../../common/utils/sock/sock_util.o - -mt_server: mt_server.o ../../../../../../common/utils/sock/sock_util.o - $(CC) -o $@ $^ -lpthread - -../../../../../../common/utils/sock/sock_util.o: ../../../../../../common/utils/sock/sock_util.c ../../../../../../common/utils/sock/sock_util.h - -clean:: - -rm -f ../../../../../../common/utils/sock/sock_util.o diff --git a/content/chapters/io/lab/support/async/python/async_server.py b/content/chapters/io/lab/support/async/python/async_server.py deleted file mode 100755 index 84c71f26d6..0000000000 --- a/content/chapters/io/lab/support/async/python/async_server.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: BSD-3-Clause - -# https://docs.python.org/3/library/asyncio-stream.html - -import asyncio -import logging -import sys - - -def fibonacci(num): - if num in (0, 1): - return 1 - return fibonacci(num - 1) + fibonacci(num - 2) - - -async def handle(reader, writer): - logging.info("Received connection from %s", writer.get_extra_info("peername")) - response = "" - try: - data = await reader.read(256) - msg = data.decode("utf-8") - logging.debug("Message: %s", msg) - num = int(msg) - if num < 0 or num > 34: - response = "out of range" - response = str(fibonacci(num)) - except ValueError: - response = "error" - - writer.write(response.encode("utf-8")) - await writer.drain() - - writer.close() - - -LISTEN_BACKLOG = 10 - - -async def run_server(port, hostname="0.0.0.0"): - server = await asyncio.start_server(handle, hostname, port) - - async with server: - await server.serve_forever() - - -def main(): - logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") - - if len(sys.argv) != 2: - print("Usage: {} port".format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - port = int(sys.argv[1]) - logging.info("Starting server on port {:d}".format(port)) - - asyncio.run(run_server(port)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lab/support/async/python/async_server_3.6.py b/content/chapters/io/lab/support/async/python/async_server_3.6.py deleted file mode 100755 index 1d494ee9b7..0000000000 --- a/content/chapters/io/lab/support/async/python/async_server_3.6.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: BSD-3-Clause - -# https://docs.python.org/3.6/library/asyncio-stream.html#asyncio.start_server - -import asyncio -import logging -import sys - - -def fibonacci(num): - if num in (0, 1): - return 1 - return fibonacci(num - 1) + fibonacci(num - 2) - - -@asyncio.coroutine -def handle(reader, writer): - logging.info("Received connection from %s", writer.get_extra_info("peername")) - response = "" - try: - data = yield from reader.read(256) - msg = data.decode("utf-8") - logging.debug("Message: %s", msg) - num = int(msg) - if num < 0 or num > 34: - response = "out of range" - response = str(fibonacci(num)) - except ValueError: - response = "error" - - writer.write(response.encode("utf-8")) - yield from writer.drain() - - writer.close() - - -LISTEN_BACKLOG = 10 - - -def run_server(port, hostname="0.0.0.0"): - loop = asyncio.get_event_loop() - coro = asyncio.start_server(handle, hostname, port, loop=loop) - server = loop.run_until_complete(coro) - - try: - loop.run_forever() - except KeyboardInterrupt: - pass - - server.close() - loop.run_until_complete(server.wait_closed()) - loop.close() - - -def main(): - logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") - - if len(sys.argv) != 2: - print("Usage: {} port".format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - port = int(sys.argv[1]) - logging.info("Starting server on port {:d}".format(port)) - run_server(port) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lab/support/async/python/client_bench.sh b/content/chapters/io/lab/support/async/python/client_bench.sh deleted file mode 100755 index 863adaa03f..0000000000 --- a/content/chapters/io/lab/support/async/python/client_bench.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -NUM_CLIENTS=8 - -for _ in $(seq 1 $NUM_CLIENTS); do - ./client.py localhost 2999 30 & -done -wait diff --git a/content/chapters/io/lab/support/async/python/mp_server.py b/content/chapters/io/lab/support/async/python/mp_server.py deleted file mode 100755 index 8b2076c126..0000000000 --- a/content/chapters/io/lab/support/async/python/mp_server.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: BSD-3-Clause - -import logging -import multiprocessing -import socket -import sys - - -def fibonacci(num): - if num in (0, 1): - return 1 - return fibonacci(num - 1) + fibonacci(num - 2) - - -def handle(connection, address): - logging.info("Received connection from %s", address) - response = "" - try: - msg = connection.recv(256).decode("utf-8") - logging.debug("Message: %s", msg) - num = int(msg) - if num < 0 or num > 34: - response = "out of range" - response = str(fibonacci(num)) - except ValueError: - response = "error" - - connection.send(response.encode("utf-8")) - - connection.close() - - -LISTEN_BACKLOG = 10 - - -def run_server(port, hostname="0.0.0.0"): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((hostname, port)) - sock.listen(LISTEN_BACKLOG) - - while True: - conn, address = sock.accept() - process = multiprocessing.Process(target=handle, args=(conn, address)) - process.daemon = True - process.start() - - -def main(): - logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") - - if len(sys.argv) != 2: - print("Usage: {} port".format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - port = int(sys.argv[1]) - logging.info("Starting server on port {:d}".format(port)) - run_server(port) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lab/support/async/python/mt_server.py b/content/chapters/io/lab/support/async/python/mt_server.py deleted file mode 100755 index 36000edcb9..0000000000 --- a/content/chapters/io/lab/support/async/python/mt_server.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: BSD-3-Clause - -import logging -import socket -import sys -import threading - - -def fibonacci(num): - if num in (0, 1): - return 1 - return fibonacci(num - 1) + fibonacci(num - 2) - - -def handle(connection, address): - logging.info("Received connection from %s", address) - response = "" - try: - msg = connection.recv(256).decode("utf-8") - logging.debug("Message: %s", msg) - num = int(msg) - if num < 0 or num > 34: - response = "out of range" - response = str(fibonacci(num)) - except ValueError: - response = "error" - - connection.send(response.encode("utf-8")) - - connection.close() - - -LISTEN_BACKLOG = 10 - - -def run_server(port, hostname="0.0.0.0"): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((hostname, port)) - sock.listen(LISTEN_BACKLOG) - - while True: - conn, address = sock.accept() - thread = threading.Thread(target=handle, args=(conn, address)) - thread.start() - - -def main(): - logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") - - if len(sys.argv) != 2: - print("Usage: {} port".format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - port = int(sys.argv[1]) - logging.info("Starting server on port {:d}".format(port)) - run_server(port) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lab/support/async/python/server.py b/content/chapters/io/lab/support/async/python/server.py deleted file mode 100755 index 4a59e590c2..0000000000 --- a/content/chapters/io/lab/support/async/python/server.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: BSD-3-Clause - -import logging -import socket -import sys - - -def fibonacci(num): - if num in (0, 1): - return 1 - return fibonacci(num - 1) + fibonacci(num - 2) - - -def handle(connection, address): - logging.info("Received connection from %s", address) - response = "" - try: - msg = connection.recv(256).decode("utf-8") - logging.debug("Message: %s", msg) - num = int(msg) - if num < 0 or num > 34: - response = "out of range" - response = str(fibonacci(num)) - except ValueError: - response = "error" - - connection.send(response.encode("utf-8")) - - connection.close() - - -LISTEN_BACKLOG = 10 - - -def run_server(port, hostname="0.0.0.0"): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((hostname, port)) - sock.listen(LISTEN_BACKLOG) - - while True: - conn, address = sock.accept() - handle(conn, address) - - -def main(): - logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") - - if len(sys.argv) != 2: - print("Usage: {} port".format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - port = int(sys.argv[1]) - logging.info("Starting server on port {:d}".format(port)) - run_server(port) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lab/support/buffering/Makefile b/content/chapters/io/lab/support/buffering/Makefile deleted file mode 100644 index 5f2654a627..0000000000 --- a/content/chapters/io/lab/support/buffering/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -BINARIES=printf_buffering no_buffering libc_buffering diy_buffering -include ../../../../../common/makefile/multiple.mk - -test-file: - dd if=/dev/urandom of=test-file.txt bs=1024 count=1K - -clean:: - rm -f *.txt diff --git a/content/chapters/io/lab/support/client-server/Makefile b/content/chapters/io/lab/support/client-server/Makefile deleted file mode 100644 index 1fa27e5b49..0000000000 --- a/content/chapters/io/lab/support/client-server/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARY = client -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/io/lab/support/client-server/client.c b/content/chapters/io/lab/support/client-server/client.c deleted file mode 100644 index 134f688110..0000000000 --- a/content/chapters/io/lab/support/client-server/client.c +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define PORT 5000 -#define BUFSIZE 1024 - -const char *localhost = "127.0.0.1"; - -int read_input_with_prompt(char *buf) -{ - char *ret; - - printf("Message for receiver (type 'exit' to quit): "); - - memset(buf, 0, BUFSIZE); - ret = fgets(buf, BUFSIZE, stdin); - if (!ret) - return -1; - buf[strlen(buf)-1] = '\0'; - - return 0; -} - -void populate_sockaddr(struct sockaddr_in *sockaddr, const char *addr, int port) -{ - int ret; - - sockaddr->sin_family = AF_INET; - sockaddr->sin_port = htons(port); - ret = inet_aton(addr, (struct in_addr *) &sockaddr->sin_addr.s_addr); - DIE(ret == 0, "inet_aton"); -} - -int main(void) -{ - char buf[BUFSIZE]; - int ret, sockfd; - struct sockaddr_in server_addr; - - while (1) { - /* Get user input as a string. */ - ret = read_input_with_prompt(buf); - DIE(ret < 0, "read_input_with_prompt"); - - /* TODO 1: Create a TCP socket (SOCK_STREAM). */ - - /* TODO 2: Add address and port to server_addr. */ - /* HINT: use populate_sockaddr(). */ - - /* TODO 3: Connect to the server. */ - /* Use DIE to check for errors. */ - - /* TODO 4: Send message from buf through the socket with send(). */ - - /* If the message is "exit", break out of the loop. */ - if (strncmp(buf, "exit", 4) == 0) - break; - - /* TODO 5: Receive the response from the server with recv(). */ - - /* Print the response. */ - /* printf("Received from server: %s\n", buf); */ - - /* TODO 6: Close the socket. */ - } - - return 0; -} diff --git a/content/chapters/io/lab/support/client-server/client.py b/content/chapters/io/lab/support/client-server/client.py deleted file mode 100644 index 33e7e7516f..0000000000 --- a/content/chapters/io/lab/support/client-server/client.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause - -import socket # Exposes the Berkely Sockets API - -IP = "127.0.0.1" # Loopback IP address (localhost) -MAX_MESSAGE_LENGTH = 1024 -PORT = 5000 # Port to send data to (non-privileged ports are > 1023) - - -def main(): - while True: - # Get user input as a string. - message = input("Message for receiver (type 'exit' to quit): ") - - # TODO: Create a TCP socket (SOCK_STREAM). - - # TODO: connect to the server. - # `connect()` returns when the server `accept()`s the connection. - - # TODO: send the message to the server. The message must be encoded to - # bytes. - - # TODO: if the message is "exit", break out of the loop. - if message == "exit": - break - - # TODO: receive the response from the server. - response = "TODO" - - # Print the response. - print(f"Received from server: {response}") - - # TODO: close the socket. - - -if __name__ == "__main__": - main() diff --git a/content/chapters/io/lab/support/client-server/server.py b/content/chapters/io/lab/support/client-server/server.py deleted file mode 100644 index 21b7932ed6..0000000000 --- a/content/chapters/io/lab/support/client-server/server.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause - -import socket # Exposes the Berkely Sockets API -from os import getpid - -IP = "127.0.0.1" # Loopback IP address (localhost) -PORT = 5000 # Port to listen for data data on (non-privileged ports are > 1023) -MAX_MESSAGE_LENGTH = 1024 - - -def main(): - # Create a socket. Do not mind the arguments for now. - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - # Bind the socket to the IP and port. But this time, it is no longer - # listening automatically. - sock.bind((IP, PORT)) - - # We must specifically tell the socket to start listening. - # The argument specifies how many connections to queue up before refusing. - sock.listen(socket.SOMAXCONN) - - print(f"Server with PID {getpid()} is listening for data on port {PORT}...") - - while True: - # Create a new socket for the connection. Returns when a client calls - # `connect()` to the server. - connection_socket, (ip, port) = sock.accept() - print(f"Connection established with IP {ip} and port {port}.") - - # Receive at most 1024 bytes data from the socket. - message = connection_socket.recv(MAX_MESSAGE_LENGTH) - - # The sender had to encode the message from string to bytes. Now decode - # it back to a string. - message = message.decode() - - print(f"Received : {message}") - - # If the message is "exit", break out of the loop. - if message == "exit": - break - - # Otherwise, send a response back to the client. - connection_socket.send("Message received.".encode()) - - # Close the connection socket. - connection_socket.close() - - sock.close() - - -if __name__ == "__main__": - main() diff --git a/content/chapters/io/lab/support/file-descriptors/.gitignore b/content/chapters/io/lab/support/file-descriptors/.gitignore deleted file mode 100644 index 5bcad47237..0000000000 --- a/content/chapters/io/lab/support/file-descriptors/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -open_directory -open_read_write -write_file.txt diff --git a/content/chapters/io/lab/support/file-descriptors/Makefile b/content/chapters/io/lab/support/file-descriptors/Makefile deleted file mode 100644 index 46098eab5f..0000000000 --- a/content/chapters/io/lab/support/file-descriptors/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -BINARIES = open_directory open_read_write -include ../../../../../common/makefile/multiple.mk - -clean:: - -rm -f write_file.txt diff --git a/content/chapters/io/lab/support/file-descriptors/open_read_write.c b/content/chapters/io/lab/support/file-descriptors/open_read_write.c deleted file mode 100644 index 5d16df8972..0000000000 --- a/content/chapters/io/lab/support/file-descriptors/open_read_write.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifdef BUFSIZ -#undef BUFSIZ -#endif - -#define BUFSIZ 1024 - -#define READ_FILE_NAME "read_file.txt" -#define WRITE_FILE_NAME "write_file.txt" - -static int open_file_for_reading(const char *file_name) -{ - int fd; - - fd = open(file_name, O_RDONLY); - DIE(fd < 0, "open"); - - printf("Opened file descriptor is for reading: %d\n", fd); - - return fd; -} - -static void read_from_file(char *buff, int fd, int bytes_to_read) -{ - int bytes_read; - int total_read = 0; - - /** - * ALWAYS use `read()` in a loop because it might not read all the - * required bytes. - */ - while (total_read < bytes_to_read) { - /** - * `total_read` bytes have been read so far - * `buff + total_read` is the address of the first byte to be - * read - * `bytes_to_read - total_read` bytes need to be read - */ - bytes_read = read(fd, buff + total_read, - bytes_to_read - total_read); - DIE(bytes_read < 0, "read"); - - /* 0 indicates the end of the file. */ - if (!bytes_read) - break; - - total_read += bytes_read; - } - - printf("Read %d bytes from file descriptor %d:\n%s", total_read, - fd, buff); -} - -static int open_file_for_writing(const char *file_name) -{ - int fd = 0; - - /** - * TODO 1: Open the file for writing. Remember to use `DIE()` to check - * for errors. - */ - - printf("Opened file descriptor is for writing: %d\n", fd); - - return fd; -} - -static void write_to_file(char *buff, int fd, int bytes_to_write) -{ - int bytes_written; - int total_written = 0; - - /** - * TODO 2: Write the buffer to the file descriptor. - * ALWAYS use `write()` in a loop because it might not read all the - * required bytes. - */ - - printf("Wrote %d bytes to file descriptor %d\n", total_written, - fd); -} - -int main(void) -{ - int fd_read; - int fd_write; - char buff[BUFSIZ] = { 0 }; - char message[] = "Message for file2.txt: What's up, Doc?\n"; - - fd_read = open_file_for_reading(READ_FILE_NAME); - read_from_file(buff, fd_read, sizeof(buff)); - - fd_write = open_file_for_writing(WRITE_FILE_NAME); - write_to_file(message, fd_write, sizeof(message)); - - /** - * TODO 3: Write to `READ_FILE_NAME` and then read the newly written - * data. Use the functions defined above. - */ - - /* TODO 4: `close()` the file `open()`-ed descriptors. */ - - return 0; -} diff --git a/content/chapters/io/lab/support/file-descriptors/read_file.txt b/content/chapters/io/lab/support/file-descriptors/read_file.txt deleted file mode 100644 index 45ab396a5a..0000000000 --- a/content/chapters/io/lab/support/file-descriptors/read_file.txt +++ /dev/null @@ -1 +0,0 @@ -What did you expect to find here? diff --git a/content/chapters/io/lab/support/file-mappings/.gitignore b/content/chapters/io/lab/support/file-mappings/.gitignore deleted file mode 100644 index da893e6548..0000000000 --- a/content/chapters/io/lab/support/file-mappings/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -mmap_cp -*.txt diff --git a/content/chapters/io/lab/support/file-mappings/mmap_cp.c b/content/chapters/io/lab/support/file-mappings/mmap_cp.c deleted file mode 100644 index cdf83d33c6..0000000000 --- a/content/chapters/io/lab/support/file-mappings/mmap_cp.c +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -static void wait_for_input(const char *msg) -{ - char buf[32]; - - printf(" * %s\n", msg); - printf(" -- Press ENTER to continue ..."); - fflush(stdout); - fgets(buf, 32, stdin); -} - -int main(int argc, char *argv[]) -{ - int fdin; - int fdout; - int rc; - char *src; - char *dst; - struct stat statbuf; - - DIE(argc != 3, "Usage: ./mmap_cp "); - - /* TODO: Open the input file. */ - - /* TODO: Open/create the output file. */ - - /** - * TODO: Truncate the output file to the input file size. Use `fstat()` - * to get the size of the input file and `ftruncate()` to set the size - * of the output file. - */ - - /* TODO: `mmap()` the input file. */ - - /* wait_for_input("Mapped input file."); */ - - /* TODO: `mmap()` the output file. */ - - /* wait_for_input("Mapped output file."); */ - - /** - * TODO: Copy the contents of the input mapping to the output mapping. - */ - - /* TODO: Clean up. Use `munmap()` to unmap the 2 files. */ - - return 0; -} diff --git a/content/chapters/io/lab/support/mini-shell/.gitignore b/content/chapters/io/lab/support/mini-shell/.gitignore deleted file mode 100644 index fb647f35e4..0000000000 --- a/content/chapters/io/lab/support/mini-shell/.gitignore +++ /dev/null @@ -1 +0,0 @@ -mini_shell diff --git a/content/chapters/io/lab/support/mini-shell/bosses.txt b/content/chapters/io/lab/support/mini-shell/bosses.txt deleted file mode 100644 index a038e9fa83..0000000000 --- a/content/chapters/io/lab/support/mini-shell/bosses.txt +++ /dev/null @@ -1,20 +0,0 @@ -Darkeater Midir -Slave Knight Gael -Nameless King -Dancer Of The Boreal Valley -Ancient Dragon -Black Dragon Kalameet -Lord Gwyn -Ornstein and Smough -The Four Kings -Bed of Chaos -Manus Father of the Abyss -Knight Artorias -Soul of Cinder -Elana the Squalid Queen -Capra Demon -Pontiff Sulyvahn -Sister Friede -Fume Knight -Sir Alonne -Bell Gargoyles diff --git a/content/chapters/io/lab/support/mini-shell/mini_shell.c b/content/chapters/io/lab/support/mini-shell/mini_shell.c deleted file mode 100644 index 41ea11c04b..0000000000 --- a/content/chapters/io/lab/support/mini-shell/mini_shell.c +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define MAX_LINE_SIZE 256 -#define MAX_ARGS 8 - -#define ERROR 0 -#define SIMPLE 1 -#define REDIRECT 2 -#define PIPE 3 -#define SHELL_EXIT 4 - -static char *verb; -static char **args; -static char *stdin_file; -static char *stdout_file; -static char *stderr_file; - -static int parse_line(char *line); -static void alloc_mem(void); -static void free_mem(void); - -static void do_redirect(int filedes, const char *filename) -{ - int rc; - int fd; - - /* TODO: Redirect `filedes` into `fd` representing `filename`. */ -} - -static void simple_cmd(char **args) -{ - pid_t pid; - pid_t wait_ret; - int status; - - /** - * Create a process to execute the command. Use `execvp` to launch the - * new process. - */ - pid = fork(); - switch (pid) { - case -1: - /* Error */ - DIE(1, "fork"); - break; - - case 0: - /* Child process */ - - /** - * Call `do_redirect()`. Assume the only file descriptor that needs to - * be redirected is `stdout`. Its file descriptor is `STDOUT_FILENO`. - */ - if (stdout_file != NULL) - do_redirect(STDOUT_FILENO, stdout_file); - - execvp(args[0], (char *const *) args); - DIE(1, "execvp"); - break; - - default: - /* Parent process */ - wait_ret = waitpid(pid, &status, 0); - DIE(wait_ret < 0, "waitpid"); - - if (WIFEXITED(status)) - printf("Child process (pid %d) terminated normally, " - "with exit code %d\n", - pid, WEXITSTATUS(status)); - } -} - -int main(void) -{ - int type; - char line[MAX_LINE_SIZE]; - - alloc_mem(); - - while (1) { - printf("> "); - fflush(stdout); - - memset(line, 0, MAX_LINE_SIZE); - - if (fgets(line, sizeof(line), stdin) == NULL) { - free_mem(); - exit(EXIT_SUCCESS); - } - - type = parse_line(line); - - switch (type) { - case SHELL_EXIT: - free_mem(); - exit(EXIT_SUCCESS); - - case SIMPLE: - simple_cmd(args); - break; - } - } - - return 0; -} - -static void alloc_mem(void) -{ - args = malloc(MAX_ARGS * sizeof(char *)); - DIE(args == NULL, "malloc"); -} - -static void free_mem(void) -{ - free(args); -} - -static int parse_line(char *line) -{ - int ret = SIMPLE; - int idx = 0; - char *token; - char *delim = "=\n"; - char *saveptr = NULL; - - stdin_file = NULL; - stdout_file = NULL; - stderr_file = NULL; - - /* Check for exit. */ - if (strncmp("exit", line, strlen("exit")) == 0) - return SHELL_EXIT; - - /* Regular command. */ - delim = " \t\n"; - token = strtok_r(line, delim, &saveptr); - - if (token == NULL) - return ERROR; - - verb = strdup(token); - - /* Copy args. */ - while (token != NULL) { - if (token == NULL) { - printf(" Expansion failed\n"); - return ERROR; - } - - if (strncmp(token, ">", strlen(">")) == 0) { - if (strlen(token) > strlen(">")) { - token++; - stdout_file = strdup(token); - } else { - token = strtok_r(NULL, delim, &saveptr); - stdout_file = strdup(token); - } - } else { - args[idx++] = strdup(token); - } - - token = strtok_r(NULL, delim, &saveptr); - } - - args[idx++] = NULL; - return ret; -} diff --git a/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.c b/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.c deleted file mode 100644 index 78581b2239..0000000000 --- a/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Useful socket functions - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" - -/* - * Connect to a TCP server identified by name (DNS name or dotted decimal - * string) and port. - */ - -int tcp_connect_to_server(const char *name, unsigned short port) -{ - struct hostent *hent; - struct sockaddr_in server_addr; - int s; - int rc; - - hent = gethostbyname(name); - DIE(hent == NULL, "gethostbyname"); - - s = socket(PF_INET, SOCK_STREAM, 0); - DIE(s < 0, "socket"); - - memset(&server_addr, 0, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - memcpy(&server_addr.sin_addr.s_addr, hent->h_addr, - sizeof(server_addr.sin_addr.s_addr)); - - rc = connect(s, (struct sockaddr *) &server_addr, sizeof(server_addr)); - DIE(rc < 0, "connect"); - - return s; -} - -int tcp_close_connection(int sockfd) -{ - int rc; - - rc = shutdown(sockfd, SHUT_RDWR); - DIE(rc < 0, "shutdown"); - - return close(sockfd); -} - -/* - * Create a server socket. - */ - -int tcp_create_listener(unsigned short port, int backlog) -{ - struct sockaddr_in address; - int listenfd; - int sock_opt; - int rc; - - listenfd = socket(PF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - sock_opt = 1; - rc = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, - &sock_opt, sizeof(int)); - DIE(rc < 0, "setsockopt"); - - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = htons(port); - address.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (SSA *) &address, sizeof(address)); - DIE(rc < 0, "bind"); - - rc = listen(listenfd, backlog); - DIE(rc < 0, "listen"); - - return listenfd; -} - -/* - * Use getpeername(2) to extract remote peer address. Fill buffer with - * address format IP_address:port (e.g. 192.168.0.1:22). - */ - -int get_peer_address(int sockfd, char *buf, size_t len) -{ - struct sockaddr_in addr; - socklen_t addrlen = sizeof(struct sockaddr_in); - - if (getpeername(sockfd, (SSA *) &addr, &addrlen) < 0) - return -1; - - snprintf(buf, len, "%s:%d", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - return 0; -} diff --git a/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.h b/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.h deleted file mode 100644 index 906976dc94..0000000000 --- a/content/chapters/io/lab/support/mini-shell/utils/sock/sock_util.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -/* - * Useful socket macros and structures - */ - -#ifndef SOCK_UTIL_H_ -#define SOCK_UTIL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -/* default backlog for listen(2) system call */ -#define DEFAULT_LISTEN_BACKLOG 5 - -/* "shortcut" for struct sockaddr structure */ -#define SSA struct sockaddr - - -int tcp_connect_to_server(const char *name, unsigned short port); -int tcp_close_connection(int s); -int tcp_create_listener(unsigned short port, int backlog); -int get_peer_address(int sockfd, char *buf, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/content/chapters/io/lab/support/multiplex/.gitignore b/content/chapters/io/lab/support/multiplex/.gitignore deleted file mode 100644 index 2786e0d0bc..0000000000 --- a/content/chapters/io/lab/support/multiplex/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/epoll_echo_server diff --git a/content/chapters/io/lab/support/multiplex/Makefile b/content/chapters/io/lab/support/multiplex/Makefile deleted file mode 100644 index ce8cfb9002..0000000000 --- a/content/chapters/io/lab/support/multiplex/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -BINARY = epoll_echo_server - -include ../../../../../common/makefile/single.mk - -epoll_echo_server: epoll_echo_server.o ../../../../../common/utils/sock/sock_util.o - -../../../../../common/utils/sock/sock_util.o: ../../../../../common/utils/sock/sock_util.c ../../../../../common/utils/sock/sock_util.h - -clean:: - -rm -f ../../../../../common/utils/sock/sock_util.o diff --git a/content/chapters/io/lab/support/multiplex/README.md b/content/chapters/io/lab/support/multiplex/README.md deleted file mode 100644 index 4569ffddae..0000000000 --- a/content/chapters/io/lab/support/multiplex/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# epoll echo-reply server - -Use `make` to compile the `epoll()`-based echo-reply server: - -```console -student@os:/.../multiplex/c$ make -``` - -Start the server and use `telnet` to connect to it and send messages. -The server will reply with the same message. -The server listens for connections on port `42424`. - -```console -student@os:/.../multiplex/c$ # on one console -student@os:/.../multiplex/c$ ./epoll_echo_server - -student@os:/.../multiplex/c$ # on the second, third, ... console -student@os:/.../multiplex/c$ telnet localhost 42424 -``` - -The server uses `epoll()` for multiplexing connections and receiving notifications (input - `EPOLLIN` and output - `EPOLLOUT`). -A specialized structure (`struct connection`) maintains information regarding each connection. - -Wrappers over `epoll()` are defined in `../../../utils/sock/w_epoll.h`. diff --git a/content/chapters/io/lab/support/multiplex/epoll_echo_server.c b/content/chapters/io/lab/support/multiplex/epoll_echo_server.c deleted file mode 100644 index 471115df25..0000000000 --- a/content/chapters/io/lab/support/multiplex/epoll_echo_server.c +++ /dev/null @@ -1,283 +0,0 @@ -/* - * SPDX-License-Identifier: BSD-3-Clause - * - * epoll-based echo server. Uses epoll(7) to multiplex connections. - * - * TODO: - * - block data receiving when receive buffer is full (use circular buffers) - * - do not copy receive buffer into send buffer when send buffer data is - * still valid - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" -#include "utils/log/log.h" -#include "utils/sock/sock_util.h" -#include "./w_epoll.h" - -#define ECHO_LISTEN_PORT 42424 - - -/* server socket file descriptor */ -static int listenfd; - -/* epoll file descriptor */ -static int epollfd; - -enum connection_state { - STATE_DATA_RECEIVED, - STATE_DATA_SENT, - STATE_CONNECTION_CLOSED -}; - -/* structure acting as a connection handler */ -struct connection { - int sockfd; - /* buffers used for receiving messages and then echoing them back */ - char recv_buffer[BUFSIZ]; - size_t recv_len; - char send_buffer[BUFSIZ]; - size_t send_len; - enum connection_state state; -}; - -/* - * Initialize connection structure on given socket. - */ - -static struct connection *connection_create(int sockfd) -{ - struct connection *conn = malloc(sizeof(*conn)); - - DIE(conn == NULL, "malloc"); - - conn->sockfd = sockfd; - memset(conn->recv_buffer, 0, BUFSIZ); - memset(conn->send_buffer, 0, BUFSIZ); - - return conn; -} - -/* - * Copy receive buffer to send buffer (echo). - */ - -static void connection_copy_buffers(struct connection *conn) -{ - conn->send_len = conn->recv_len; - memcpy(conn->send_buffer, conn->recv_buffer, conn->send_len); -} - -/* - * Remove connection handler. - */ - -static void connection_remove(struct connection *conn) -{ - close(conn->sockfd); - conn->state = STATE_CONNECTION_CLOSED; - free(conn); -} - -/* - * Handle a new connection request on the server socket. - */ - -static void handle_new_connection(void) -{ - static int sockfd; - socklen_t addrlen = sizeof(struct sockaddr_in); - struct sockaddr_in addr; - struct connection *conn; - int rc; - - /* accept new connection */ - sockfd = accept(listenfd, (SSA *) &addr, &addrlen); - DIE(sockfd < 0, "accept"); - - log_error("Accepted connection from: %s:%d", - inet_ntoa(addr.sin_addr), ntohs(addr.sin_port)); - - /* instantiate new connection handler */ - conn = connection_create(sockfd); - - /* add socket to epoll */ - rc = w_epoll_add_ptr_in(epollfd, sockfd, conn); - DIE(rc < 0, "w_epoll_add_in"); -} - -/* - * Receive message on socket. - * Store message in recv_buffer in struct connection. - */ - -static enum connection_state receive_message(struct connection *conn) -{ - ssize_t bytes_recv; - int rc; - char abuffer[64]; - - rc = get_peer_address(conn->sockfd, abuffer, 64); - if (rc < 0) { - log_error("get_peer_address"); - goto remove_connection; - } - - bytes_recv = recv(conn->sockfd, conn->recv_buffer, BUFSIZ, 0); - if (bytes_recv < 0) { /* error in communication */ - log_error("Error in communication from: %s", abuffer); - goto remove_connection; - } - if (bytes_recv == 0) { /* connection closed */ - log_info("Connection closed from: %s", abuffer); - goto remove_connection; - } - - log_debug("Received message from: %s", abuffer); - log_debug("--%s--", conn->recv_buffer); - - conn->recv_len = bytes_recv; - conn->state = STATE_DATA_RECEIVED; - - return STATE_DATA_RECEIVED; - -remove_connection: - rc = w_epoll_remove_ptr(epollfd, conn->sockfd, conn); - DIE(rc < 0, "w_epoll_remove_ptr"); - - /* remove current connection */ - connection_remove(conn); - - return STATE_CONNECTION_CLOSED; -} - -/* - * Send message on socket. - * Store message in send_buffer in struct connection. - */ - -static enum connection_state send_message(struct connection *conn) -{ - ssize_t bytes_sent; - int rc; - char abuffer[64]; - - rc = get_peer_address(conn->sockfd, abuffer, 64); - if (rc < 0) { - log_error("get_peer_address"); - goto remove_connection; - } - - bytes_sent = send(conn->sockfd, conn->send_buffer, conn->send_len, 0); - if (bytes_sent < 0) { /* error in communication */ - log_error("Error in communication to %s", abuffer); - goto remove_connection; - } - if (bytes_sent == 0) { /* connection closed */ - log_info("Connection closed to %s", abuffer); - goto remove_connection; - } - - log_debug("Sending message to %s", abuffer); - log_debug("--%s--", conn->send_buffer); - - /* all done - remove out notification */ - rc = w_epoll_update_ptr_in(epollfd, conn->sockfd, conn); - DIE(rc < 0, "w_epoll_update_ptr_in"); - - conn->state = STATE_DATA_SENT; - - return STATE_DATA_SENT; - -remove_connection: - rc = w_epoll_remove_ptr(epollfd, conn->sockfd, conn); - DIE(rc < 0, "w_epoll_remove_ptr"); - - /* remove current connection */ - connection_remove(conn); - - return STATE_CONNECTION_CLOSED; -} - -/* - * Handle a client request on a client connection. - */ - -static void handle_client_request(struct connection *conn) -{ - int rc; - enum connection_state ret_state; - - ret_state = receive_message(conn); - if (ret_state == STATE_CONNECTION_CLOSED) - return; - - connection_copy_buffers(conn); - - /* add socket to epoll for out events */ - rc = w_epoll_update_ptr_inout(epollfd, conn->sockfd, conn); - DIE(rc < 0, "w_epoll_add_ptr_inout"); -} - -int main(void) -{ - int rc; - - /* init multiplexing */ - epollfd = w_epoll_create(); - DIE(epollfd < 0, "w_epoll_create"); - - /* create server socket */ - listenfd = tcp_create_listener(ECHO_LISTEN_PORT, - DEFAULT_LISTEN_BACKLOG); - DIE(listenfd < 0, "tcp_create_listener"); - - rc = w_epoll_add_fd_in(epollfd, listenfd); - DIE(rc < 0, "w_epoll_add_fd_in"); - - log_info("Server waiting for connections on port %d", - ECHO_LISTEN_PORT); - - /* server main loop */ - while (1) { - struct epoll_event rev; - - /* wait for events */ - rc = w_epoll_wait_infinite(epollfd, &rev); - DIE(rc < 0, "w_epoll_wait_infinite"); - - /* - * switch event types; consider - * - new connection requests (on server socket) - * - socket communication (on connection sockets) - */ - - if (rev.data.fd == listenfd) { - log_debug("New connection"); - if (rev.events & EPOLLIN) - handle_new_connection(); - } else { - if (rev.events & EPOLLIN) { - log_debug("New message"); - handle_client_request(rev.data.ptr); - } - if (rev.events & EPOLLOUT) { - log_debug("Ready to send message"); - send_message(rev.data.ptr); - } - } - } - - return 0; -} diff --git a/content/chapters/io/lab/support/multiplex/w_epoll.h b/content/chapters/io/lab/support/multiplex/w_epoll.h deleted file mode 100644 index 36e9489b1e..0000000000 --- a/content/chapters/io/lab/support/multiplex/w_epoll.h +++ /dev/null @@ -1,181 +0,0 @@ -/* - * SPDX-License-Identifier: BSD-3-Clause - * - * epoll wrapper functions - */ - -#ifndef W_EPOLL_H_ -#define W_EPOLL_H_ 1 - -#ifdef __cplusplus -extern "C" { -#endif - - -#define EPOLL_TIMEOUT_INFINITE -1 - - -static inline int w_epoll_create(void) -{ - return epoll_create(10); -} - -static inline int w_epoll_add_fd_in(int epollfd, int fd) -{ - struct epoll_event ev; - - ev.events = EPOLLIN; - ev.data.fd = fd; - - return epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &ev); -} - -static inline int w_epoll_add_fd_out(int epollfd, int fd) -{ - struct epoll_event ev; - - ev.events = EPOLLOUT; - ev.data.fd = fd; - - return epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &ev); -} - -static inline int w_epoll_add_fd_inout(int epollfd, int fd) -{ - struct epoll_event ev; - - ev.events = EPOLLIN | EPOLLOUT; - ev.data.fd = fd; - - return epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &ev); -} - -static inline int w_epoll_update_fd_in(int epollfd, int fd) -{ - struct epoll_event ev; - - ev.events = EPOLLIN; - ev.data.fd = fd; - - return epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &ev); -} - -static inline int w_epoll_update_fd_out(int epollfd, int fd) -{ - struct epoll_event ev; - - ev.events = EPOLLOUT; - ev.data.fd = fd; - - return epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &ev); -} - -static inline int w_epoll_update_fd_inout(int epollfd, int fd) -{ - struct epoll_event ev; - - ev.events = EPOLLIN | EPOLLOUT; - ev.data.fd = fd; - - return epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &ev); -} - -static inline int w_epoll_remove_fd(int epollfd, int fd) -{ - struct epoll_event ev; - - /* - * ev needn't be initialized - * It has to be passed to epoll as a non-NULL argument for - * pre-2.6.9 kernels. - */ - ev.events = EPOLLIN; - ev.data.fd = fd; - - return epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, &ev); -} - -static inline int w_epoll_add_ptr_in(int epollfd, int fd, void *ptr) -{ - struct epoll_event ev; - - ev.events = EPOLLIN; - ev.data.ptr = ptr; - - return epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &ev); -} - -static inline int w_epoll_add_ptr_out(int epollfd, int fd, void *ptr) -{ - struct epoll_event ev; - - ev.events = EPOLLOUT; - ev.data.ptr = ptr; - - return epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &ev); -} - -static inline int w_epoll_add_ptr_inout(int epollfd, int fd, void *ptr) -{ - struct epoll_event ev; - - ev.events = EPOLLIN | EPOLLOUT; - ev.data.ptr = ptr; - - return epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &ev); -} - -static inline int w_epoll_update_ptr_in(int epollfd, int fd, void *ptr) -{ - struct epoll_event ev; - - ev.events = EPOLLIN; - ev.data.ptr = ptr; - - return epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &ev); -} - -static inline int w_epoll_update_ptr_out(int epollfd, int fd, void *ptr) -{ - struct epoll_event ev; - - ev.events = EPOLLOUT; - ev.data.ptr = ptr; - - return epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &ev); -} - -static inline int w_epoll_update_ptr_inout(int epollfd, int fd, void *ptr) -{ - struct epoll_event ev; - - ev.events = EPOLLIN | EPOLLOUT; - ev.data.ptr = ptr; - - return epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &ev); -} - -static inline int w_epoll_remove_ptr(int epollfd, int fd, void *ptr) -{ - struct epoll_event ev; - - /* - * ev needn't be initialized - * It has to be passed to epoll as a non-NULL argument for - * pre-2.6.9 kernels. - */ - ev.events = EPOLLIN; - ev.data.ptr = ptr; - - return epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, &ev); -} - -static inline int w_epoll_wait_infinite(int epollfd, struct epoll_event *rev) -{ - return epoll_wait(epollfd, rev, 1, EPOLL_TIMEOUT_INFINITE); -} -#ifdef __cplusplus -} -#endif - -#endif /* EPOLL_H_ */ diff --git a/content/chapters/io/lab/support/pipes/.gitignore b/content/chapters/io/lab/support/pipes/.gitignore deleted file mode 100644 index 0c80813751..0000000000 --- a/content/chapters/io/lab/support/pipes/.gitignore +++ /dev/null @@ -1 +0,0 @@ -anonymous_pipe diff --git a/content/chapters/io/lab/support/pipes/Makefile b/content/chapters/io/lab/support/pipes/Makefile deleted file mode 100644 index de4c39c00f..0000000000 --- a/content/chapters/io/lab/support/pipes/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARY = anonymous_pipe -include ../../../../../common/makefile/single.mk diff --git a/content/chapters/io/lab/support/receive-challenges/Makefile b/content/chapters/io/lab/support/receive-challenges/Makefile deleted file mode 100644 index a6f51b69f3..0000000000 --- a/content/chapters/io/lab/support/receive-challenges/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARIES = send_fd_4 receive_pipe send_fifo receive_fifo send_unix_socket receive_unix_socket send_net_dgram_socket receive_net_dgram_socket -include ../../../../../common/makefile/multiple.mk diff --git a/content/chapters/io/lab/support/redirect/.gitignore b/content/chapters/io/lab/support/redirect/.gitignore deleted file mode 100644 index 9fd09cc349..0000000000 --- a/content/chapters/io/lab/support/redirect/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -redirect -redirect_parallel -redirect_file.txt -redirect_std*_file.txt diff --git a/content/chapters/io/lab/support/redirect/Makefile b/content/chapters/io/lab/support/redirect/Makefile deleted file mode 100644 index 3f96da830d..0000000000 --- a/content/chapters/io/lab/support/redirect/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -BINARIES = redirect redirect_parallel -LDLIBS = -lpthread -include ../../../../../common/makefile/multiple.mk - -clean:: - rm -f redirect_*.txt diff --git a/content/chapters/io/lab/support/redirect/redirect.c b/content/chapters/io/lab/support/redirect/redirect.c deleted file mode 100644 index 930ac1024e..0000000000 --- a/content/chapters/io/lab/support/redirect/redirect.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define REDIRECT_FILE_NAME "redirect_file.txt" - -static void wait_for_input(const char *func_name, const char *msg) -{ - char buf[32]; - - fprintf(stderr, " * %s(): %s\n", func_name, msg); - fprintf(stderr, " -- Press ENTER to continue ..."); - fgets(buf, 32, stdin); -} - -static void do_redirect(int file_des, const char *filename) -{ - int rc; - int fd; - - /** - * TODO 1: Refactor the code below to use `dup()` instead of - * `close()`-ing `file_des` and `open()`-ing the new file "on top" of - * the old file descriptor. - */ - - /** - * TODO 2: Refactor the code below to use `dup2()` instead of `close()` - * and `dup()`. - */ - - wait_for_input(__func__, "before closing `file_des`"); - - /** - *`close()` `stdout` so that the file descriptor created below is 1, - * thus "replacing" `stdout`. - */ - rc = close(file_des); - DIE(rc < 0, "close"); - - wait_for_input(__func__, "after closing `file_des`"); - - /* The first available file descriptor is 1 (`stdout`). */ - fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); - DIE(fd < 0, "open"); - - wait_for_input(__func__, "after opening `filename`"); -} - -int main(void) -{ - do_redirect(STDOUT_FILENO, REDIRECT_FILE_NAME); - - printf("Where did this message disappear?\n"); - - return 0; -} diff --git a/content/chapters/io/lab/support/send-receive/.gitignore b/content/chapters/io/lab/support/send-receive/.gitignore deleted file mode 100644 index 5a1c58ff00..0000000000 --- a/content/chapters/io/lab/support/send-receive/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -sender -receiver diff --git a/content/chapters/io/lab/support/send-receive/Makefile b/content/chapters/io/lab/support/send-receive/Makefile deleted file mode 100644 index 6d3aba5b74..0000000000 --- a/content/chapters/io/lab/support/send-receive/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARIES = receiver sender -include ../../../../../common/makefile/multiple.mk diff --git a/content/chapters/io/lab/support/send-receive/receiver.c b/content/chapters/io/lab/support/send-receive/receiver.c deleted file mode 100644 index 253de1880d..0000000000 --- a/content/chapters/io/lab/support/send-receive/receiver.c +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "sock-utils.h" - -const char *localhost = "127.0.0.1"; - -int main(void) -{ - char buf[BUFSIZE]; - int ret, sockfd; - struct sockaddr_in receiver_addr; - struct sockaddr_in sender_addr; - socklen_t sender_addr_len; - - /* Create a socket. */ - sockfd = socket(AF_INET, SOCK_DGRAM, 0); - DIE(sockfd < 0, "socket"); - - /* TODO 1: Add address and port to receiver_addr. */ - /* HINT: use populate_sockaddr(). */ - - /* TODO 2: Bind sockfd to receiver_addr. Use DIE to check for errors. */ - - printf("Server with PID %d is listening for data on port %d ...\n", getpid(), PORT); - - while (1) { - /* TODO 2.1: Remove break. */ - break; - - /* Prepare the buffer */ - memset(buf, 0, BUFSIZE); - - /* TODO 3: Receive at most BUFSIZE bytes through the socket with recvfrom(). */ - /* HINT: Use sender_addr as the fifth arg. You will need to cast it. */ - /* It will be populated with information about the sender. */ - /* Use DIE to check for errors. */ - - /* Ensure buf is null-terminated. */ - buf[BUFSIZE-1] = '\0'; - - /* Print the message. */ - printf("Received from IP %s and port %d: %s\n", inet_ntoa(sender_addr.sin_addr), - sender_addr.sin_port, buf); - - if (strncmp(buf, "exit", 4) == 0) - break; - } - - /* Close the socket. */ - ret = close(sockfd); - DIE(ret < 0, "close"); - - return 0; -} diff --git a/content/chapters/io/lab/support/send-receive/receiver.py b/content/chapters/io/lab/support/send-receive/receiver.py deleted file mode 100644 index f592654841..0000000000 --- a/content/chapters/io/lab/support/send-receive/receiver.py +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause - -import socket # Exposes the Berkely Sockets API -from os import getpid - -IP = "127.0.0.1" # Loopback IP address (localhost) -MAX_MESSAGE_LENGTH = 1024 -PORT = 5000 # Port to listen for data data on (non-privileged ports are > 1023) - - -def main(): - # Create a socket. Do not mind the arguments for now. - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - - # Bind the socket to the IP and port. This registers the current process as - # "listening" for data on the port. - sock.bind((IP, PORT)) - - print(f"Server with PID {getpid()} is listening for data on port {PORT}...") - - while True: - # Receive at most 1024 bytes data from the socket. - message, (ip, port) = sock.recvfrom(MAX_MESSAGE_LENGTH) - - # The sender had to encode the message from string to bytes. Now decode - # it back to a string. - message = message.decode() - - # Print the message. - print(f"Received from IP {ip} and port {port}: {message}") - - # If the message is "exit", break out of the loop. - if message == "exit": - break - - sock.close() - - -if __name__ == "__main__": - main() diff --git a/content/chapters/io/lab/support/send-receive/sender.c b/content/chapters/io/lab/support/send-receive/sender.c deleted file mode 100644 index 54b128b108..0000000000 --- a/content/chapters/io/lab/support/send-receive/sender.c +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "sock-utils.h" - -const char *localhost = "127.0.0.1"; - -int read_input_with_prompt(char *buf) -{ - char *ret; - - printf("Message for receiver (type 'exit' to quit): "); - - memset(buf, 0, BUFSIZE); - ret = fgets(buf, BUFSIZE, stdin); - if (!ret) - return -1; - buf[strlen(buf)-1] = '\0'; - - return 0; -} - -int main(void) -{ - char buf[BUFSIZE]; - int ret, sockfd; - struct sockaddr_in receiver_addr; - - /* Create a socket. */ - sockfd = socket(AF_INET, SOCK_DGRAM, 0); - DIE(sockfd < 0, "socket"); - - /* TODO 1: Add address and port to receiver_addr. */ - /* HINT: use populate_sockaddr(). */ - - while (1) { - ret = read_input_with_prompt(buf); - DIE(ret < 0, "read_input_with_prompt"); - - /* TODO 2: Send message from buf through the socket with sendto(). */ - /* HINT: Use receiver_addr as the fifth arg. You will need to cast it. */ - /* Use DIE to check for errors. */ - - if (strncmp(buf, "exit", 4) == 0) - break; - } - - /* Close the socket */ - ret = close(sockfd); - DIE(ret < 0, "close"); - - return 0; -} diff --git a/content/chapters/io/lab/support/send-receive/sender.py b/content/chapters/io/lab/support/send-receive/sender.py deleted file mode 100644 index e02826c27f..0000000000 --- a/content/chapters/io/lab/support/send-receive/sender.py +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause - -import socket # Exposes the Berkely Sockets API - -IP = "127.0.0.1" # Loopback IP address (localhost) -PORT = 5000 # Port to send data to (non-privileged ports are > 1023) - - -def main(): - # Create a socket. Do not mind the arguments for now. - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - - while True: - # Get user input as a string. - message = input("Message for receiver (type 'exit' to quit): ") - - # Send the message to the server. The message must be encoded to bytes. - sock.sendto(message.encode(), (IP, PORT)) - - # If the message is "exit", break out of the loop. - if message == "exit": - break - - sock.close() - - -if __name__ == "__main__": - main() diff --git a/content/chapters/io/lab/support/send-receive/sock-utils.h b/content/chapters/io/lab/support/send-receive/sock-utils.h deleted file mode 100644 index 705c3e3135..0000000000 --- a/content/chapters/io/lab/support/send-receive/sock-utils.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ - -#pragma once - -#include -#include -#include - -#include "utils/utils.h" - -#define PORT 5000 -#define BUFSIZE 1024 - -void populate_sockaddr(struct sockaddr_in *sockaddr, const char *addr, int port) -{ - int ret; - - sockaddr->sin_family = AF_INET; - sockaddr->sin_port = htons(port); - ret = inet_aton(addr, (struct in_addr *)&sockaddr->sin_addr.s_addr); - DIE(ret == 0, "inet_aton"); -} diff --git a/content/chapters/io/lab/support/simple-file-operations/.gitignore b/content/chapters/io/lab/support/simple-file-operations/.gitignore deleted file mode 100644 index 1101b80169..0000000000 --- a/content/chapters/io/lab/support/simple-file-operations/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -file_operations -directory_operations diff --git a/content/chapters/io/lab/support/simple-file-operations/Makefile b/content/chapters/io/lab/support/simple-file-operations/Makefile deleted file mode 100644 index 0ba5351552..0000000000 --- a/content/chapters/io/lab/support/simple-file-operations/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -BINARIES = file_operations directory_operations -include ../../../../../common/makefile/multiple.mk - -JAVAC = javac -JAVA_FILE_OPERATIONS = FileOperations - -java-file-operations: $(JAVA_FILE_OPERATIONS).java - $(JAVAC) $(JAVA_FILE_OPERATIONS).java - -clean:: - -rm -f $(JAVA_FILE_OPERATIONS).class diff --git a/content/chapters/io/lab/support/simple-file-operations/dir/.gitignore b/content/chapters/io/lab/support/simple-file-operations/dir/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/content/chapters/io/lab/support/simple-file-operations/file.txt b/content/chapters/io/lab/support/simple-file-operations/file.txt deleted file mode 100644 index 3def2e4a62..0000000000 --- a/content/chapters/io/lab/support/simple-file-operations/file.txt +++ /dev/null @@ -1 +0,0 @@ -C was here! \ No newline at end of file diff --git a/content/chapters/io/lab/support/sparse-file/.gitignore b/content/chapters/io/lab/support/sparse-file/.gitignore deleted file mode 100644 index 779a29449b..0000000000 --- a/content/chapters/io/lab/support/sparse-file/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/generate -/solve diff --git a/content/chapters/io/lab/support/sparse-file/Makefile b/content/chapters/io/lab/support/sparse-file/Makefile deleted file mode 100644 index 20b642fc25..0000000000 --- a/content/chapters/io/lab/support/sparse-file/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARIES = generate solve -include ../../../../../common/makefile/multiple.mk diff --git a/content/chapters/io/lab/support/sparse-file/README.md b/content/chapters/io/lab/support/sparse-file/README.md deleted file mode 100644 index 281b4e39c9..0000000000 --- a/content/chapters/io/lab/support/sparse-file/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Sparse File - -The `swiss_cheese.sparse` file is a sparse file. -It stores 1 character of useful data followed by 10 bytes of spaces. -Then another characters, then another 10 bytes of spaces, and so on. - -Fill in the blanks in `solve.c` to read the characters, concatenate them together and get the flag. diff --git a/content/chapters/io/lab/support/sparse-file/generate.c b/content/chapters/io/lab/support/sparse-file/generate.c deleted file mode 100644 index f0494021a2..0000000000 --- a/content/chapters/io/lab/support/sparse-file/generate.c +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define FILENAME "swiss_cheese.sparse" -#define NUM_SPACES 10 - -static const char FLAG[] = "SO{all_for_the_empire}"; - -int main(void) -{ - int fd; - size_t i; - - fd = open(FILENAME, O_WRONLY | O_TRUNC | O_CREAT, 0644); - DIE(fd < 0, "open"); - - for (i = 0; i < sizeof(FLAG); i++) { - ssize_t n; - off_t pos; - - n = write(fd, FLAG+i, 1); - DIE(n < 0, "write"); - - pos = lseek(fd, NUM_SPACES, SEEK_CUR); - DIE(pos < 0, "lseek"); - } - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lab/support/sparse-file/solve.c b/content/chapters/io/lab/support/sparse-file/solve.c deleted file mode 100644 index a7e9df934d..0000000000 --- a/content/chapters/io/lab/support/sparse-file/solve.c +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define FILENAME "../../support/sparse-file/swiss_cheese.sparse" -#define NUM_SPACES 10 - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -int main(void) -{ - int fd; - size_t i; - char buffer[BUFSIZ]; - - fd = open(FILENAME, O_RDONLY); - DIE(fd < 0, "open"); - - for (i = 0; i < BUFSIZ; i++) { - ssize_t n; - off_t pos; - - /* TODO: Read character. */ - - - /* TODO: Stop if no more characters (read returns 0). */ - - - /* TODO: Skep NUM_SPACES blank characters. */ - } - - printf("flag is %s\n", buffer); - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lab/support/sparse-file/swiss_cheese.sparse b/content/chapters/io/lab/support/sparse-file/swiss_cheese.sparse deleted file mode 100644 index 0934038a71..0000000000 Binary files a/content/chapters/io/lab/support/sparse-file/swiss_cheese.sparse and /dev/null differ diff --git a/content/chapters/io/lecture/Makefile b/content/chapters/io/lecture/Makefile deleted file mode 100644 index 8f73d4919a..0000000000 --- a/content/chapters/io/lecture/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -TARGETS = socket-summary -include ../../../common/makefile/slides.mk diff --git a/content/chapters/io/lecture/demo/.gitignore b/content/chapters/io/lecture/demo/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/content/chapters/io/lecture/demo/IPC/.gitignore b/content/chapters/io/lecture/demo/IPC/.gitignore deleted file mode 100644 index 354fffc03d..0000000000 --- a/content/chapters/io/lecture/demo/IPC/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -* -!*.c -!*.md -!.gitignore -!Makefile diff --git a/content/chapters/io/lecture/demo/IPC/Makefile b/content/chapters/io/lecture/demo/IPC/Makefile deleted file mode 100644 index 426cd5ce6d..0000000000 --- a/content/chapters/io/lecture/demo/IPC/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARIES = pipe fifo unix_stream_socket_server unix_stream_socket_client unix_datagram_socket_server unix_datagram_socket_client network_stream_socket_server network_stream_socket_client network_datagram_socket_server network_datagram_socket_client -include ../../../../../common/makefile/multiple.mk diff --git a/content/chapters/io/lecture/demo/IPC/README.md b/content/chapters/io/lecture/demo/IPC/README.md deleted file mode 100644 index d937853832..0000000000 --- a/content/chapters/io/lecture/demo/IPC/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# IPC - -These are some examples on the interprocess communication. - -## Build - -To build the C examples on Linux, run `make`. - -## Running - -### Pipe - -We use the `pipe()` to create a pipe and `fork()` to create another process. -The parent will write at one end of the pipe and the child will read from the other. - -Expected output: - -```console -student@os:~/.../demo/IPC$ ./pipe -I am the one who knocks! -``` - -### FIFO - -We use the `mkfifo()` to create a pipe and then proceed to open it from two processes. -The parent will write at one end of the pipe and the child will read from the other. - -The pipe created with `mkfifo()` is stored on disk as a **special file**. - -```console -student@OS:~$ ls -l my_fifo -prw-r--r-- 1 student student 0 nov 22 18:38 my_fifo -``` - -Expected output: - -```console -student@os:~/.../demo/IPC$ ./fifo -I am the one who knocks! -``` - -### UNIX Sockets - -#### Stream Sockets - -We use **UNIX stream sockets** to send a message to a server and wait for it to response with the exact same string. -This example is best followed using two terminals. - -First we start the server, which will block waiting for a connection. - -```console -student@os:~/.../demo/IPC$ ./unix_stream_socket_server -``` - -We then start the client, which will send the string to the server and will wait for the response. - -```console -student@os:~/.../demo/IPC$ ./unix_stream_socket_client -Received: R2-D2 is underrated -``` - -The server output is also updated, displaying the received string. - -```console -student@os:~/.../demo/IPC$ ./unix_stream_socket_server -Received: R2-D2 is underrated -``` - -#### Datagram Sockets - -We use **UNIX datagram sockets** to send a message to a server and wait for it to response with the exact same string. - -When using **UNIX datagram sockets** we also need to bind the client to a socket path in order to receive a reply. - -The output is similar to [UNIX Stream Sockets](#unix-sockets). - -### Network Sockets - -#### Stream Sockets - -We use **stream sockets** to send a message to a server and wait for it to response with the exact same string. - -The output is similar to [UNIX Sockets](#unix-sockets). - -#### Datagram Sockets - -We use **datagram sockets** to send a message to a server and wait for it to response with the exact same string. - -The output is similar to [UNIX Sockets](#unix-sockets). diff --git a/content/chapters/io/lecture/demo/IPC/fifo.c b/content/chapters/io/lecture/demo/IPC/fifo.c deleted file mode 100644 index 107a105892..0000000000 --- a/content/chapters/io/lecture/demo/IPC/fifo.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define FIFO_NAME "my_fifo" - -static const char *heisenberg = "I am the one who knocks!"; - -int main(void) -{ - pid_t pid; - int fifo_fd; - char buf[32]; - int rc; - - /* Remove old FIFO. */ - remove(FIFO_NAME); - - /* Create FIFO. */ - rc = mkfifo(FIFO_NAME, 0644); - DIE(rc < 0, "mkfifo"); - - pid = fork(); - switch (pid) - { - case -1: /* Error */ - DIE(1, "fork"); - break; - - case 0: /* Child process */ - fifo_fd = open(FIFO_NAME, O_RDONLY); - DIE(fifo_fd < 0, "open"); - - /* Read from pipe and print. */ - rc = read(fifo_fd, buf, 32); - DIE(rc < 0, "read"); - buf[rc] = '\0'; - printf("%s\n", buf); - - close(fifo_fd); - return 42; - - default: /* Parent process */ - fifo_fd = open(FIFO_NAME, O_WRONLY); - DIE(fifo_fd < 0, "open"); - - /* Write to pipe. */ - rc = write(fifo_fd, heisenberg, strlen(heisenberg)); - DIE(rc < 0, "write"); - - close(fifo_fd); - break; - } - - return 0; -} diff --git a/content/chapters/io/lecture/demo/IPC/network_datagram_socket_client.c b/content/chapters/io/lecture/demo/IPC/network_datagram_socket_client.c deleted file mode 100644 index 639f51c71e..0000000000 --- a/content/chapters/io/lecture/demo/IPC/network_datagram_socket_client.c +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -#define IP_ADDR "127.0.0.1" -#define PORT 1234 - -static const char message[] = "R2-D2 is underrated"; - -int main(void) -{ - int rc; - int fd; - struct sockaddr_in addr; - socklen_t addrlen; - char buffer[BUFSIZ]; - - /* Create socket. */ - fd = socket(AF_INET, SOCK_DGRAM, 0); - DIE(fd < 0, "open"); - - /* Fill server info. */ - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons(PORT); - rc = inet_aton(IP_ADDR, &addr.sin_addr); - DIE(rc == 0, "inet_aton"); - - /* Send to server. */ - rc = sendto(fd, message, sizeof(message), 0, (struct sockaddr *) &addr, sizeof(addr)); - DIE(rc < 0, "sendto"); - - /* Wait reply. */ - rc = recvfrom(fd, buffer, BUFSIZ, 0, (struct sockaddr *) &addr, &addrlen); - DIE(rc < 0, "recvfrom"); - - printf("Received: %s\n", buffer); - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/IPC/network_datagram_socket_server.c b/content/chapters/io/lecture/demo/IPC/network_datagram_socket_server.c deleted file mode 100644 index 924d0380f6..0000000000 --- a/content/chapters/io/lecture/demo/IPC/network_datagram_socket_server.c +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -#define PORT 1234 - -int main(void) -{ - int rc; - int fd; - struct sockaddr_in addr, cli_addr; - socklen_t addrlen = sizeof(cli_addr); - char buffer[BUFSIZ]; - - /* Create socket. */ - fd = socket(AF_INET, SOCK_DGRAM, 0); - DIE(fd < 0, "socket"); - - /* Bind socket to address. */ - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons(PORT); - addr.sin_addr.s_addr = INADDR_ANY; - - rc = bind(fd, (struct sockaddr *) &addr, sizeof(addr)); - DIE(rc < 0, "bind"); - - /* Make the server run indefinitely. */ - while(1) { - /* Receive from client. */ - rc = recvfrom(fd, buffer, BUFSIZ, 0, (struct sockaddr *) &cli_addr, &addrlen); - DIE(rc < 0, "recvfrom"); - - printf("Received: %s\n", buffer); - - /* Answer with the same string. */ - addrlen = sizeof(cli_addr); - rc = sendto(fd, buffer, strlen(buffer), 0, (struct sockaddr *)&cli_addr, addrlen); - DIE(rc < 0, "sendto"); - } - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/IPC/network_stream_socket_client.c b/content/chapters/io/lecture/demo/IPC/network_stream_socket_client.c deleted file mode 100644 index 95dbfcdca9..0000000000 --- a/content/chapters/io/lecture/demo/IPC/network_stream_socket_client.c +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -#define IP_ADDR "127.0.0.1" -#define PORT 1234 - -static const char message[] = "R2-D2 is underrated"; - -int main(void) -{ - int rc; - int fd; - struct sockaddr_in addr; - char buffer[BUFSIZ]; - - /* Create socket. */ - fd = socket(AF_INET, SOCK_STREAM, 0); - DIE(fd < 0, "open"); - - /* Connect socket to server. */ - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons(PORT); - rc = inet_aton(IP_ADDR, &addr.sin_addr); - DIE(rc == 0, "inet_aton"); - - rc = connect(fd, (struct sockaddr *) &addr, sizeof(addr)); - DIE(rc < 0, "connect"); - - /* Send to server. */ - rc = write(fd, message, sizeof(message)); - DIE(rc < 0, "write"); - - /* Wait reply. */ - rc = read(fd, buffer, sizeof(buffer)); - DIE(rc < 0, "read"); - - printf("Received: %s\n", buffer); - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/IPC/network_stream_socket_server.c b/content/chapters/io/lecture/demo/IPC/network_stream_socket_server.c deleted file mode 100644 index 307232c059..0000000000 --- a/content/chapters/io/lecture/demo/IPC/network_stream_socket_server.c +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -#define PORT 1234 - -int main(void) -{ - int rc; - int listenfd, connectfd; - struct sockaddr_in addr, cli_addr; - socklen_t cli_addrlen; - char buffer[BUFSIZ]; - - /* Create socket. */ - listenfd = socket(AF_INET, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - /* Bind socket to address. */ - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons(PORT); - addr.sin_addr.s_addr = INADDR_ANY; - - rc = bind(listenfd, (struct sockaddr *) &addr, sizeof(addr)); - DIE(rc < 0, "bind"); - - /* Put in listen mode. */ - rc = listen(listenfd, 10); - DIE(rc < 0, "listen"); - - /* Make the server run indefinitely. */ - while(1) { - /* Accept connection. */ - connectfd = accept(listenfd, (struct sockaddr *) &cli_addr, &cli_addrlen); - DIE(connectfd < 0, "accept"); - - /* Receive from client. */ - rc = read(connectfd, buffer, BUFSIZ); - DIE(rc < 0, "read"); - - printf("Received: %s\n", buffer); - - /* Answer with the same string. */ - rc = write(connectfd, buffer, strlen(buffer)); - DIE(rc < 0, "write"); - } - - close(connectfd); - close(listenfd); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/IPC/pipe.c b/content/chapters/io/lecture/demo/IPC/pipe.c deleted file mode 100644 index 9232b79aaa..0000000000 --- a/content/chapters/io/lecture/demo/IPC/pipe.c +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define PIPE_READ_FD 0 -#define PIPE_WRITE_FD 1 - -static const char *heisenberg = "I am the one who knocks!"; - -int main(void) -{ - pid_t pid; - int rc; - char buf[32]; - int pipe_fd[2]; - - // Create pipe - rc = pipe(pipe_fd); - DIE(rc < 0, "pipe"); - - pid = fork(); - switch (pid) { - case -1: /* Error */ - DIE(1, "fork"); - break; - - case 0: - // Close unused write end - close(pipe_fd[PIPE_WRITE_FD]); - - rc = read(pipe_fd[PIPE_READ_FD], buf, 32); - DIE(rc < 0, "read"); - - buf[rc] = '\0'; - printf("%s\n", buf); - - close(pipe_fd[PIPE_READ_FD]); - return 42; - - default: /* Parent process */ - // Close unused read end - close(pipe_fd[PIPE_READ_FD]); - - rc = write(pipe_fd[PIPE_WRITE_FD], heisenberg, strlen(heisenberg)); - DIE(rc < 0, "write"); - - close(pipe_fd[PIPE_WRITE_FD]); - break; - } - - return 0; -} diff --git a/content/chapters/io/lecture/demo/IPC/unix_datagram_socket_client.c b/content/chapters/io/lecture/demo/IPC/unix_datagram_socket_client.c deleted file mode 100644 index 0e15e8b3eb..0000000000 --- a/content/chapters/io/lecture/demo/IPC/unix_datagram_socket_client.c +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -static const char server_socket_path[] = "tower_bridge"; -static const char client_socket_path[] = "bifrost"; -static const char message[] = "R2-D2 is underrated"; - -int main(void) -{ - int rc; - int fd; - struct sockaddr_un saddr, caddr; - char buffer[BUFSIZ]; - - /* UNIX server socket must exist. It must be created by receiver. */ - rc = access(server_socket_path, R_OK | W_OK); - DIE(rc < 0, "access"); - - /* Create client socket. */ - fd = socket(AF_UNIX, SOCK_DGRAM, 0); - DIE(fd < 0, "open"); - - /* Fill client info. */ - memset(&caddr, 0, sizeof(caddr)); - caddr.sun_family = AF_UNIX; - snprintf(caddr.sun_path, sizeof(client_socket_path)+1, "%s", client_socket_path); - - /* Bind client socket to client info. */ - rc = bind(fd, (struct sockaddr *) &caddr, sizeof(caddr)); - DIE(rc < 0, "bind"); - - /* Fill server info. */ - memset(&saddr, 0, sizeof(saddr)); - saddr.sun_family = AF_UNIX; - snprintf(saddr.sun_path, sizeof(server_socket_path)+1, "%s", server_socket_path); - - /* Send to server. */ - rc = sendto(fd, message, sizeof(message), 0, (struct sockaddr *) &saddr, sizeof(saddr)); - DIE(rc < 0, "sendto"); - - /* Wait reply. */ - rc = recvfrom(fd, buffer, BUFSIZ, 0, NULL, NULL); - DIE(rc < 0, "recvfrom"); - - printf("Received: %s\n", buffer); - - close(fd); - remove(caddr.sun_path); - return 0; -} diff --git a/content/chapters/io/lecture/demo/IPC/unix_datagram_socket_server.c b/content/chapters/io/lecture/demo/IPC/unix_datagram_socket_server.c deleted file mode 100644 index 6de6afcc21..0000000000 --- a/content/chapters/io/lecture/demo/IPC/unix_datagram_socket_server.c +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -static const char socket_path[] = "tower_bridge"; - -int main(void) -{ - int rc; - int fd; - struct sockaddr_un addr, raddr; - socklen_t raddrlen = sizeof(raddr); - char buffer[BUFSIZ]; - - /* Remove socket_path. */ - remove(socket_path); - - /* Create socket. */ - fd = socket(AF_UNIX, SOCK_DGRAM, 0); - DIE(fd < 0, "socket"); - - /* Bind socket to path. */ - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - snprintf(addr.sun_path, strlen(socket_path)+1, "%s", socket_path); - rc = bind(fd, (struct sockaddr *) &addr, sizeof(addr)); - DIE(rc < 0, "bind"); - - /* Make server run indefinitely. */ - while (1) { - /* Receive from client. */ - rc = recvfrom(fd, buffer, BUFSIZ, 0, (struct sockaddr *) &raddr, &raddrlen); - DIE(rc < 0, "recvfrom"); - - printf("Received: %s\n", buffer); - /* Answer with the same string. */ - raddrlen = sizeof(raddr); - rc = sendto(fd, buffer, strlen(buffer), 0, (struct sockaddr *) &raddr, raddrlen); - DIE(rc < 0, "sendto"); - } - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/IPC/unix_stream_socket_client.c b/content/chapters/io/lecture/demo/IPC/unix_stream_socket_client.c deleted file mode 100644 index 21f8b2e0ea..0000000000 --- a/content/chapters/io/lecture/demo/IPC/unix_stream_socket_client.c +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -static const char socket_path[] = "golden_gate"; -static const char message[] = "R2-D2 is underrated"; - -int main(void) -{ - int rc; - int fd; - struct sockaddr_un addr; - char buffer[BUFSIZ]; - - /* UNIX socket must exist. It must be created by receiver. */ - rc = access(socket_path, R_OK | W_OK); - DIE(rc < 0, "access"); - - /* Create socket. */ - fd = socket(PF_UNIX, SOCK_STREAM, 0); - DIE(fd < 0, "open"); - - /* Connect socket to server. */ - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - snprintf(addr.sun_path, sizeof(socket_path), "%s", socket_path); - rc = connect(fd, (struct sockaddr *) &addr, sizeof(addr)); - DIE(rc < 0, "connect"); - - /* Send to server. */ - rc = write(fd, message, sizeof(message)); - DIE(rc < 0, "write"); - - /* Wait reply. */ - rc = read(fd, buffer, sizeof(buffer)); - DIE(rc < 0, "read"); - - printf("Received: %s\n", buffer); - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/IPC/unix_stream_socket_server.c b/content/chapters/io/lecture/demo/IPC/unix_stream_socket_server.c deleted file mode 100644 index ca10d4bd6c..0000000000 --- a/content/chapters/io/lecture/demo/IPC/unix_stream_socket_server.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#ifndef BUFSIZ -#define BUFSIZ 256 -#endif - -static const char socket_path[] = "golden_gate"; - -int main(void) -{ - int rc; - int listenfd, connectfd; - struct sockaddr_un addr, raddr; - socklen_t raddrlen; - char buffer[BUFSIZ]; - - /* Remove socket_path. */ - remove(socket_path); - - /* Create socket. */ - listenfd = socket(AF_UNIX, SOCK_STREAM, 0); - DIE(listenfd < 0, "socket"); - - /* Bind socket to path. */ - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - snprintf(addr.sun_path, strlen(socket_path)+1, "%s", socket_path); - rc = bind(listenfd, (struct sockaddr *) &addr, sizeof(addr)); - DIE(rc < 0, "bind"); - - /* Put in listen mode. */ - rc = listen(listenfd, 10); - DIE(rc < 0, "listen"); - - /* Make server run indefinitely. */ - while(1) { - /* Accept connection. */ - connectfd = accept(listenfd, (struct sockaddr *) &raddr, &raddrlen); - DIE(connectfd < 0, "accept"); - - /* Receive from client. */ - rc = read(connectfd, buffer, BUFSIZ); - DIE(rc < 0, "read"); - - printf("Received: %s\n", buffer); - - /* Answer with the same string. */ - rc = write(connectfd, buffer, strlen(buffer)); - DIE(rc < 0, "write"); - } - - close(connectfd); - close(listenfd); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/devices/.gitignore b/content/chapters/io/lecture/demo/devices/.gitignore deleted file mode 100644 index 0e64b1d7f4..0000000000 --- a/content/chapters/io/lecture/demo/devices/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -* -!*.c -!*.sh -!*.py -!*.md -!.gitignore -!Makefile diff --git a/content/chapters/io/lecture/demo/devices/Makefile b/content/chapters/io/lecture/demo/devices/Makefile deleted file mode 100644 index 6d200384bf..0000000000 --- a/content/chapters/io/lecture/demo/devices/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARIES = hwaddr-ioctl filesystem-size -include ../../../../../common/makefile/multiple.mk diff --git a/content/chapters/io/lecture/demo/devices/README.md b/content/chapters/io/lecture/demo/devices/README.md deleted file mode 100644 index 413c2870e8..0000000000 --- a/content/chapters/io/lecture/demo/devices/README.md +++ /dev/null @@ -1,102 +0,0 @@ -# Devices - -These are some examples on IO devices interaction. - -## Build - -To build the C examples on Linux, run `make`. - -## Running - -### hwaddr ioctl - -We use `ioctl()` to read the `MAC` address for a network interface. -We create a socket and call `ioctl()` using `SIOCGIFHWADDR` as argument to populate the `ifreq structure`. - -```c -// From https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/sockios.h#L80 -#define SIOCGIFHWADDR 0x8927 /* Get hardware address */ -``` - -Expected output: - -```console -student@os:~/.../demo/devices$ # interface name might be different according to personal systems -student@os:~/.../demo/devices$ ./hwaddr-ioctl eth0 -Hardware address for interface eth0 is 00:00:5e:00:53:af -``` - -### Filesystem Size - -We open the filesystem block file from `/dev` and call `ioctl()` using `BLKGETSIZE64` as argument to return the total number of bytes from the filesystem. - -```c -// https://elixir.bootlin.com/linux/v4.7/source/include/uapi/linux/fs.h#L211 -#define BLKGETSIZE64 _IOR(0x12,114,size_t) /* return device size in bytes (u64 *arg) */ -``` - -Expected output: - -```console -student@os:~/.../demo/devices$ # partition name might be different according to personal systems -student@os:~/.../demo/devices$ ./filesystem-size /dev/sda2 -Total space 300.132 GB -``` - -### Read from device - -We will analyze how the output of a device changes based on its type. -We will be using `read-from-device.sh` bash script to read `100 bytes` from a given device with `dd`. - -Block devices guarantee persistance of data and therefore return the same bytes. - -```console -student@os:~/.../demo/devices$ sudo ./read-from-device.sh /dev/sda -Read 100 bytes from /dev/sda: -0000000 63eb d090 00bc 8e7c 8ec0 bed8 7c00 00bf -0000010 b906 0200 f3fc 50a4 1c68 cb06 b9fb 0004 -0000020 bebd 8007 007e 7c00 0f0b 0e85 8301 10c5 -0000030 f1e2 18cd 5688 5500 46c6 0511 46c6 0010 -0000040 41b4 aabb cd55 5d13 0f72 fb81 aa55 0975 -0000050 c1f7 0001 0374 46fe 6610 8000 0001 0000 -0000060 0000 0000 -0000064 - -student@os:~/.../demo/devices$ sudo ./read-from-device.sh /dev/sda -Read 100 bytes from /dev/sda: -0000000 63eb d090 00bc 8e7c 8ec0 bed8 7c00 00bf -0000010 b906 0200 f3fc 50a4 1c68 cb06 b9fb 0004 -0000020 bebd 8007 007e 7c00 0f0b 0e85 8301 10c5 -0000030 f1e2 18cd 5688 5500 46c6 0511 46c6 0010 -0000040 41b4 aabb cd55 5d13 0f72 fb81 aa55 0975 -0000050 c1f7 0001 0374 46fe 6610 8000 0001 0000 -0000060 0000 0000 -0000064 -``` - -Char devices provide information in real-time so we expect a different output if the environment of the device changes. -Data from char devices are also used by the OS to collect randomness. - -```console -student@os:~/.../demo/devices$ ./read-from-device.sh /dev/urandom -Read 100 bytes from /dev/urandom: -0000000 9672 15fc 6631 e9f7 6c6f 99c7 5504 e748 -0000010 6a18 6fa0 ffc1 fded b468 b5e9 8121 1187 -0000020 19f8 e78c 3a34 b051 de0d d0fa b913 0943 -0000030 6bb5 5c77 b763 153e 79d3 b2ca 0715 6fec -0000040 dc27 4acd bf69 1cf8 695e 3c1d a1e2 8234 -0000050 de59 9535 c344 9caa 55e3 a104 6d17 7e47 -0000060 65f3 3ad8 -0000064 - -student@os:~/.../demo/devices$ ./read-from-device.sh /dev/urandom -Read 100 bytes from /dev/urandom: -0000000 773c e9ff ba84 b7e8 9f76 bba9 6394 4023 -0000010 ddc3 4bd5 8777 5a8e 53a5 6532 1fbd 772a -0000020 a8fe 4c76 a1f4 a71e bb0e f525 71bb e8c0 -0000030 a6f6 f92f a906 929d e51d 3b8d 5cbf 2101 -0000040 fe00 6b07 0d6e 5739 32c0 c586 c289 eb27 -0000050 2a4e c817 497a 88c2 d128 9256 8880 4200 -0000060 9e8e 8710 -0000064 -``` diff --git a/content/chapters/io/lecture/demo/devices/filesystem-size.c b/content/chapters/io/lecture/demo/devices/filesystem-size.c deleted file mode 100644 index cc98d5f496..0000000000 --- a/content/chapters/io/lecture/demo/devices/filesystem-size.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Use ioctl to get number of bytes in filesystem. - * - * Some inspiration from Stack Overflow - * https://stackoverflow.com/questions/1960421/how-can-i-determine-the-space-size-of-my-drive-programatically-both-in-linux-an - */ - -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define SIZE_UNIT (1024UL * 1024 * 1024) /* 1GB */ - -int main(int argc, const char *argv[]) -{ - int fd; - unsigned long long numblocks = 0; - int rc; - - if (argc < 2) { - printf("Usage: %s partition_path\n", argv[0]); - return 0; - } - - fd = open(argv[1], O_RDONLY); - DIE(fd < 0, "open"); - - rc = ioctl(fd, BLKGETSIZE64, &numblocks); - DIE(rc < 0, "ioctl"); - - printf("Total space %.3f GB\n", (double)numblocks / SIZE_UNIT); - - close(fd); - return 0; -} - diff --git a/content/chapters/io/lecture/demo/devices/hwaddr-ioctl.c b/content/chapters/io/lecture/demo/devices/hwaddr-ioctl.c deleted file mode 100644 index ae43d67720..0000000000 --- a/content/chapters/io/lecture/demo/devices/hwaddr-ioctl.c +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -/* - * Use ioctl to get MAC address of network interface. - * - * Some inspiration from Stack Overflow - * http://stackoverflow.com/questions/1779715/how-to-get-mac-address-of-your-machine-using-a-c-program - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -int main(int argc, const char *argv[]) -{ - int s; - struct ifreq ifr; - int rc; - size_t i; - - if (argc < 2) { - printf("Usage: %s interface_name\n", argv[0]); - return 0; - } - - // Create socket - s = socket(PF_INET, SOCK_DGRAM, 0); - DIE(s < 0, "socket"); - - // Call ioctl to populate ifr struct - snprintf(ifr.ifr_name, strlen(argv[1])+1, "%s", argv[1]); - rc = ioctl(s, SIOCGIFHWADDR, &ifr); - DIE(rc < 0, "ioctl"); - - // Print result - printf("Hardware address for interface %s is ", argv[1]); - for (i = 0; i < IFHWADDRLEN-1; i++) - printf("%02x:", ((unsigned char *) ifr.ifr_hwaddr.sa_data)[i]); - printf("%02x", ((unsigned char *) ifr.ifr_hwaddr.sa_data)[IFHWADDRLEN-1]); - printf("\n"); - - close(s); - return 0; -} diff --git a/content/chapters/io/lecture/demo/devices/read-from-device.sh b/content/chapters/io/lecture/demo/devices/read-from-device.sh deleted file mode 100755 index 61288dcb50..0000000000 --- a/content/chapters/io/lecture/demo/devices/read-from-device.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [[ $# != 1 ]]; then - echo "usage: $0 " - exit 0 -fi - -BS=1 -COUNT=100 -DEV=$1 - -echo "Read $((COUNT * BS)) bytes from $DEV:" - -dd if="$DEV" bs="$BS" count="$COUNT" status=none | hexdump diff --git a/content/chapters/io/lecture/demo/file-interface/.gitignore b/content/chapters/io/lecture/demo/file-interface/.gitignore deleted file mode 100644 index 9e0058c246..0000000000 --- a/content/chapters/io/lecture/demo/file-interface/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!*.c -!*.py -!*.md -!.gitignore -!Makefile diff --git a/content/chapters/io/lecture/demo/file-interface/Makefile b/content/chapters/io/lecture/demo/file-interface/Makefile deleted file mode 100644 index 0baf887db4..0000000000 --- a/content/chapters/io/lecture/demo/file-interface/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARIES = close-stdout c-file-ops open-vs-dup sparse-file truncate -include ../../../../../common/makefile/multiple.mk diff --git a/content/chapters/io/lecture/demo/file-interface/README.md b/content/chapters/io/lecture/demo/file-interface/README.md deleted file mode 100644 index bc648c6c90..0000000000 --- a/content/chapters/io/lecture/demo/file-interface/README.md +++ /dev/null @@ -1,143 +0,0 @@ -# File Interface - -These are some examples on the IO interface. - -## Build - -To build the C examples on Linux, run `make`. - -## Running - -### Sparse file - -We open a file write some bytes, make a jump using `lseek()` and repeat two times. -Zones between writes are empty, yet the total size of the `sparse.dat` keeps track of them. - -Follow this example using the following command to track `sparse.dat` file size. - -```console -student@os:~/.../demo/file-interface$ watch -n 0.5 ls -l sparse.dat -``` - -Expected output: - -```console -# After the first write --rw-r--r-- 1 student student 1024 nov 19 15:43 sparse.dat -# After the second write --rw-r--r-- 1 student student 3072 nov 19 15:43 sparse.dat -# After the third write --rw-r--r-- 1 student student 5120 nov 19 15:43 sparse.dat -``` - -### Truncate - -We open `fake-gargantua.dat` for write and use `ftruncate()` to set its size to `20GB`. -This only impacts the metadata of the file, so the size (number of used blocks) stays **0**. - -Expected output: - -```console -student@os:~/.../demo/file-interface$ ./truncate - * Open an empty file for writting - -- Press ENTER to continue ... - - * Use truncate to update the file length to 20GB - -- Press ENTER to continue ... - - * Use fstat to read file information - -- Press ENTER to continue ... - -file size is: 21474836480 -number of blocks is: 0 - * Close file - -- Press ENTER to continue ... -``` - -### Python file ops - -We open `f.txt` and write **1024 bytes** to it. -We go backwards **512 bytes**, read **256 bytes**, then truncate it to **256 bytes**. - -Analysing the file offset and file size through the example we obtain: - -```console -operation | offset | size | -*---------------------------------------* -write 1024 | 1024 | 1024 | -lseek -512 | 512 | 1024 | -read 256 | 768 | 1024 | -ftruncate 256 | 768 | 256 | -``` - -**ftruncate** does not modify the file offset, so a future `write()` action would write at offset **768** creating a **sparse file**. - -### C file ops - -**Same as [Python file ops](#python-file-ops)**. - -### open vs dup - -We open a file twice using `open()` and duplicate the first descriptor using `dup()`. -We attempt to read some bytes from each of the three file descriptors. - -The read on the first file descriptor only influenced the duplicated file_descriptor. -They use the same open file strcucture to keep track of the offset in file. - -Follow the list of open files from the `proc` view: - -```console -student@os:~/.../demo/file-interface$ watch -n 0.5 'ls -l /proc/$(pidof open-vs-dup)/fd | sort -nk 9' -``` - -Expected output: - -```console -student@os:~/.../demo/file-interface$ ./open-vs-dup -... - * Read 23 bytes from first file descriptor - -- Press ENTER to continue ... - -A wizard is never late, - - * Read 23 bytes from the second file descriptor - -- Press ENTER to continue ... - -A wizard is never late, - - * Read 23 bytes from the duplicated file descriptor - -- Press ENTER to continue ... - - nor is he early. -``` - -### close stdout - -We open a `my_stdout.txt`, close the `stdout` and then use `dup()` to duplicate the file descriptor. -As a result, the duplicated file descriptor will be assigned value **1 (value of STDOUT_FILENO)** as it is the first unused file descriptor. - -Between the closing of `stdout` and call to `dup()`, the file descriptor 1 does not point to any **Open File Structure** so the message `Duplicate the file descriptor` will fail to print. -After the calling `dup()`, the output will be redirected to `my_stdout.txt`. - -Follow the list of open files from the `proc` view: - -```console -student@os:~/.../demo/file-interface$ watch -n 0.5 'ls -l /proc/$(pidof close-stdout)/fd | sort -nk 9' -``` - -Expected output: - -```console -student@os:~/.../demo/file-interface$ ./close-stdout -* Open file - -- Press ENTER to continue ... - - * Close stdout - -- Press ENTER to continue ... - -# Use cat to display the final output -student@os:~/.../demo/file-interface$ cat my_stdout.txt -This will go to stdout - * Close the file descriptors - -- Press ENTER to continue ... -``` diff --git a/content/chapters/io/lecture/demo/file-interface/c-file-ops.c b/content/chapters/io/lecture/demo/file-interface/c-file-ops.c deleted file mode 100644 index 04dc9a1ff4..0000000000 --- a/content/chapters/io/lecture/demo/file-interface/c-file-ops.c +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define BUFSIZE 1024 - -static char buffer[BUFSIZE] = { '1' }; - -static void -wait_for_input(const char *msg) -{ - char buf[32]; - - printf(" * %s\n", msg); - printf(" -- Press ENTER to continue ...\n"); - fflush(stdout); - fgets(buf, 32, stdin); -} - -int main(void) -{ - int fd; - ssize_t n; - int ret; - - wait_for_input("Open f.txt"); - fd = open("f.txt", O_RDWR | O_CREAT | O_TRUNC, 0644); - DIE(fd < 0, "open"); - - wait_for_input("Write 1024 bytes"); - n = write(fd, buffer, BUFSIZE); - DIE(n < 0, "write"); - - wait_for_input("Go backwards 512 bytes"); - n = lseek(fd, -512, SEEK_CUR); - DIE(n < 0, "lseek"); - - wait_for_input("Read 256 bytes"); - n = read(fd, buffer, 256); - DIE(n < 0, "read"); - - wait_for_input("Truncate to 256 bytes"); - ret = ftruncate(fd, 256); - DIE(ret < 0, "ftruncate"); - - wait_for_input("Close file"); - ret = close(fd); - DIE(ret < 0, "close"); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/file-interface/close-stdout.c b/content/chapters/io/lecture/demo/file-interface/close-stdout.c deleted file mode 100644 index 1d358fa407..0000000000 --- a/content/chapters/io/lecture/demo/file-interface/close-stdout.c +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define FILENAME "my_stdout.txt" - -static void -wait_for_input(const char *msg) -{ - char buf[32]; - - printf(" * %s\n", msg); - printf(" -- Press ENTER to continue ...\n"); - fflush(stdout); - fgets(buf, 32, stdin); -} - -int main(void) -{ - int rc; - int fd, dup_fd; - - wait_for_input("Open file"); - fd = open(FILENAME, O_WRONLY | O_CREAT | O_TRUNC, 0666); - DIE(fd < 0, "open"); - - wait_for_input("Close stdout"); - rc = close(STDOUT_FILENO); - DIE(rc < 0, "close"); - - wait_for_input("Duplicate the file descriptor"); - dup_fd = dup(fd); - DIE(dup_fd < 0, "dup"); - - printf("This will go to stdout\n"); - - wait_for_input("Close the file descriptors"); - close(fd); - // second_fd will be closed automatically at the end of the program - - return 0; -} diff --git a/content/chapters/io/lecture/demo/file-interface/open-vs-dup.c b/content/chapters/io/lecture/demo/file-interface/open-vs-dup.c deleted file mode 100644 index 58d4a3daf5..0000000000 --- a/content/chapters/io/lecture/demo/file-interface/open-vs-dup.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define FILENAME "gandalf.txt" -#define COUNT_BYTES 23 - -static void create_input_file() { - static const char *content = "A wizard is never late, nor is he early."; - - int fd = open(FILENAME, O_WRONLY | O_CREAT | O_TRUNC, 0644); - DIE(fd < 0, "open"); - - int rc = write(fd, content, strlen(content)); - DIE(rc < 0, "write"); - - rc = close(fd); - DIE(rc < 0, "close"); -} - -static void -wait_for_input(const char *msg) -{ - char buf[32]; - - printf(" * %s\n", msg); - printf(" -- Press ENTER to continue ...\n"); - fflush(stdout); - fgets(buf, 32, stdin); -} - -int main(void) -{ - char buf[32]; - int rc, first_fd, second_fd, dup_fd; - - create_input_file(); - - wait_for_input("Open file"); - first_fd = open(FILENAME, O_RDONLY); - DIE(first_fd < 0, "open"); - - wait_for_input("Open file again"); - second_fd = open(FILENAME, O_RDONLY); - DIE(second_fd < 0, "open"); - - wait_for_input("Duplicate the first file descriptor"); - dup_fd = dup(first_fd); - DIE(dup_fd < 0, "dup"); - - wait_for_input("Read 23 bytes from first file descriptor"); - memset(buf, 0, 32); - rc = read(first_fd, buf, COUNT_BYTES); - DIE(rc < 0, "read"); - printf("%s\n\n", buf); - - wait_for_input("Read 23 bytes from the second file descriptor"); - memset(buf, 0, 32); - rc = read(second_fd, buf, COUNT_BYTES); - DIE(rc < 0, "read"); - printf("%s\n\n", buf); - - wait_for_input("Read 23 bytes from the duplicated file descriptor"); - memset(buf, 0, 32); - rc = read(dup_fd, buf, COUNT_BYTES); - DIE(rc < 0, "read"); - printf("%s\n\n", buf); - - wait_for_input("Close file descriptors"); - close(first_fd); - close(second_fd); - close(dup_fd); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/file-interface/py-file-ops.py b/content/chapters/io/lecture/demo/file-interface/py-file-ops.py deleted file mode 100755 index 37c9386686..0000000000 --- a/content/chapters/io/lecture/demo/file-interface/py-file-ops.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python - -# SPDX-License-Identifier: BSD-3-Clause - -import sys - - -def wait_for_input(msg): - print(f" * {msg}") - print(" -- Press ENTER to continue ...") - sys.stdout.flush() - sys.stdin.readline() - - -def main(): - wait_for_input("Open file f.txt") - f = open("f.txt", "w+b") - - wait_for_input("Write 1024 bytes") - for _ in range(64): - f.write(bytes("0123456789012345", "ascii")) - f.flush() - - wait_for_input("Go backwards 512 bytes") - f.seek(-512, 2) - - wait_for_input("Read 256 bytes") - _ = f.read(256) - - wait_for_input("Truncate file to 256 bytes") - f.truncate(256) - - wait_for_input("Close file") - f.close() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lecture/demo/file-interface/sparse-file.c b/content/chapters/io/lecture/demo/file-interface/sparse-file.c deleted file mode 100644 index 52e380a06d..0000000000 --- a/content/chapters/io/lecture/demo/file-interface/sparse-file.c +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define FILENAME "sparse.dat" - -static char buffer[1024]; - -static void -wait_for_input(const char *msg) -{ - char buf[32]; - - printf(" * %s\n", msg); - printf(" -- Press ENTER to continue ...\n"); - fflush(stdout); - fgets(buf, 32, stdin); -} - -int main(void) -{ - int fd; - int ret; - - memset(buffer, 'A', 1024); - - wait_for_input("Open an empty file for writting"); - fd = open(FILENAME, O_WRONLY | O_CREAT | O_TRUNC, 0644); - DIE(fd < 0, "open"); - - wait_for_input("Write 1024 bytes"); - ret = write(fd, buffer, 1024); - DIE(ret < 0, "write"); - - wait_for_input("Move 1024 bytes past the end of the file"); - ret = lseek(fd, 1024, SEEK_END); - DIE(ret < 0, "lseek"); - - wait_for_input("Write 1024 bytes"); - ret = write(fd, buffer, 1024); - DIE(ret < 0, "write"); - - wait_for_input("Move 1024 bytes past the end of the file"); - ret = lseek(fd, 1024, SEEK_END); - DIE(ret < 0, "lseek"); - - wait_for_input("Write 1024 bytes"); - ret = write(fd, buffer, 1024); - DIE(ret < 0, "write"); - - wait_for_input("Close file"); - ret = close(fd); - DIE(ret < 0, "close"); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/file-interface/truncate.c b/content/chapters/io/lecture/demo/file-interface/truncate.c deleted file mode 100644 index 460cc9b806..0000000000 --- a/content/chapters/io/lecture/demo/file-interface/truncate.c +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include -#include -#include - -#include "utils/utils.h" - -#define FILENAME "fake-gargantua.dat" -#define SIZE_TO_TRUNCATE (10*2048UL*1024*1024) /* 20GB */ - -static void -wait_for_input(const char *msg) -{ - char buf[32]; - - printf(" * %s\n", msg); - printf(" -- Press ENTER to continue ...\n"); - fflush(stdout); - fgets(buf, 32, stdin); -} - -int main(void) -{ - int fd; - int ret; - struct stat sbuf; - - wait_for_input("Open an empty file for writting"); - fd = open(FILENAME, O_WRONLY | O_CREAT | O_TRUNC, 0644); - DIE(fd < 0, "open"); - - wait_for_input("Use truncate to update the file length to 20GB"); - ret = ftruncate(fd, SIZE_TO_TRUNCATE); - DIE(ret < 0, "ftruncate"); - - wait_for_input("Use fstat to read file information"); - ret = fstat(fd, &sbuf); - DIE(ret < 0, "ftruncate"); - - printf("file size is: %ld\n", sbuf.st_size); - printf("number of blocks is: %ld\n", sbuf.st_blocks); - - wait_for_input("Close file"); - ret = close(fd); - DIE(ret < 0, "close"); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/optimizations/.gitignore b/content/chapters/io/lecture/demo/optimizations/.gitignore deleted file mode 100644 index 2211df63dd..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.txt diff --git a/content/chapters/io/lecture/demo/optimizations/Makefile b/content/chapters/io/lecture/demo/optimizations/Makefile deleted file mode 100644 index 3e32dbd4b9..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -BINARIES = stdout-vs-stderr fwrite-10000 write-10000 write-1000-unbuf -include ../../../../../common/makefile/multiple.mk diff --git a/content/chapters/io/lecture/demo/optimizations/README.md b/content/chapters/io/lecture/demo/optimizations/README.md deleted file mode 100644 index 2bda7863e3..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/README.md +++ /dev/null @@ -1,171 +0,0 @@ -# Optimizations - -These are some examples on I/O optimizations. - -## Build - -To build the C examples on Linux, run `make`. - -## Running - -### `stdout` vs `stderr` - -We explore the difference between `stdout` and `stderr` when it comes to IO buffering. - -The code prints two interleaved sentences: `"Hello from the other side!"` and `"Join the dark side!"`. -Running the code yields a different result than we would expect. - -```console -student@os:~/.../demo/optimizations$ ./stdout-vs-stderr -Join the dark side! -Hello from the other side! -``` - -Let us investigate using `strace`. - -```console -student@os:~/.../demo/optimizations$ strace --trace=write ./stdout-vs-stderr -write(2, "Join the ", 9Join the ) = 9 -write(2, "dark side!\n", 11dark side! -) = 11 -write(1, "Hello from the other side!", 26Hello from the other side!) = 26 -``` - -We observe that strings meant for `stdout` are collected and printed only once. -This mechanism is called **buffering** and serves to reduce the number of `write()` syscalls. - -Uncomment the `setvbuf()` function to stop the `stdout` buffering. - -```console -student@os:~/.../demo/optimizations$ strace --trace=write ./stdout-vs-stderr -write(1, "Hello ", 6Hello ) = 6 -write(1, "from ", 5from ) = 5 -write(1, "the ", 4the ) = 4 -write(2, "Join the ", 9Join the ) = 9 -write(1, "other ", 6other ) = 6 -write(1, "side!", 5side!) = 5 -write(2, "dark side!\n", 11dark side! -) = 11 -``` - -### `fwrite` 10000 - -We write **10000 bytes** to a file, **1 byte** at a time with `fwrite()` and use `strace` and `time` to benchmark this solution. - -```console -student@os:~/.../demo/optimizations$ strace --trace=write ./fwrite-10000 -write(3, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"..., 4096) = 4096 -write(3, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"..., 4096) = 4096 -write(3, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"..., 1808) = 1808 -``` - -```console -student@os:~/.../demo/optimizations$ time ./fwrite-10000 -real 0m0.001s -user 0m0.001s -sys 0m0.000s -``` - -The default buffer size in `libc` is `4096`, stored in macro `BUFSIZ`. -The `write()` syscalls are triggered when buffer is full, thus reducing the number of syscalls from 10000 to 3. - -### `write` 10000 - -We write **10000 bytes** to a file, **1 byte** at a time with `write()` and use `strace` and `time` to benchmark this solution. - -```console -student@os:~/.../demo/optimizations$ strace -c --trace=write ./write-10000 -% time seconds usecs/call calls errors syscall ------- ----------- ----------- --------- --------- ---------------- -100.00 0.377586 37 10000 write -``` - -```console -student@os:~/.../demo/optimizations$ time ./write-10000 -real 0m0.011s -user 0m0.001s -sys 0m0.011s -``` - -This example serves as a comparison item for `fwrite` and `unbuffered write`. - -### `write` 1000 unbuf - -We write **1000 bytes** to a file, **1 byte** at a time with unbuffered `write()` and use `strace` and `time` to benchmark this solution. - -```console -student@os:~/.../demo/optimizations$ strace -c --trace=write ./write-1000-unbuf -% time seconds usecs/call calls errors syscall ------- ----------- ----------- --------- --------- ---------------- -100.00 0.055044 55 1000 write -``` - -```console -student@os:~/.../demo/optimizations$ time ./write-1000-unbuf -real 0m2.052s -user 0m0.000s -sys 0m0.087s -``` - -The performance drawback of dissabling IO buffering is obvious even when operating **1000 bytes** - **10000 bytes** would have been overkill. - -### client.py - -This is a client that sends an index to a server and expects the fibonacci number with that index. - -The client receives the number as argument or randomly generates it. -It proceeds to encode the number and sends it to the server. -After receiving a response it prints it and stops its execution. - -```console -student@os:~/.../demo/optimizations$ python client.py localhost 1234 6 -root: Connected to localhost:1234 -root: Sending 6 -function(6): 13 -``` - -### server.py - -We use a remote server to compute the nth fibonacci number which is given as parameter by a client. - -After configuring the server we start it in a loop and call `handle()` for each new connection. -`handle()` ensures that received argument is correct and starts computation. -The result is encoded and sent back to the client. - -```console -student@os:~/.../demo/optimizations$ python server.py 1234 -root: Starting server on port 1234 -root: Received connection from ('127.0.0.1', 44834) -root: Message: 6 -``` - -### mp_server.py - -See [server.py](#serverpy). - -### mt_server.py - -See [server.py](#serverpy). - -### async_server.py - -See [server.py](#serverpy). - -### client_bench.sh - -This is a simple bash script that receives the port of a running server as argument and proceeds to run multiple `client.py` instances. - -```console -student@os:~/.../demo/optimizations$ time ./client_bench.sh 1234 -root: Connected to localhost:1234 -root: Sending 30 -root: Connected to localhost:1234 -root: Sending 30 -[...] -function(30): 1346269 -function(30): 1346269 - -real 0m1.318s -user 0m0.225s -sys 0m0.048s -``` diff --git a/content/chapters/io/lecture/demo/optimizations/async_server.py b/content/chapters/io/lecture/demo/optimizations/async_server.py deleted file mode 100755 index 84c71f26d6..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/async_server.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: BSD-3-Clause - -# https://docs.python.org/3/library/asyncio-stream.html - -import asyncio -import logging -import sys - - -def fibonacci(num): - if num in (0, 1): - return 1 - return fibonacci(num - 1) + fibonacci(num - 2) - - -async def handle(reader, writer): - logging.info("Received connection from %s", writer.get_extra_info("peername")) - response = "" - try: - data = await reader.read(256) - msg = data.decode("utf-8") - logging.debug("Message: %s", msg) - num = int(msg) - if num < 0 or num > 34: - response = "out of range" - response = str(fibonacci(num)) - except ValueError: - response = "error" - - writer.write(response.encode("utf-8")) - await writer.drain() - - writer.close() - - -LISTEN_BACKLOG = 10 - - -async def run_server(port, hostname="0.0.0.0"): - server = await asyncio.start_server(handle, hostname, port) - - async with server: - await server.serve_forever() - - -def main(): - logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") - - if len(sys.argv) != 2: - print("Usage: {} port".format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - port = int(sys.argv[1]) - logging.info("Starting server on port {:d}".format(port)) - - asyncio.run(run_server(port)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lecture/demo/optimizations/client.py b/content/chapters/io/lecture/demo/optimizations/client.py deleted file mode 100755 index 697bfd18e2..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/client.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: BSD-3-Clause - -import logging -import random -import socket -import sys - - -def handle(connection, num): - if num == -1: - num = random.randrange(35) - - logging.info("Sending %d", num) - connection.send(str(num).encode("utf-8")) - response = connection.recv(256).decode("utf-8") - print("function({:d}): {}".format(num, response)) - - connection.close() - - -def run_client(server_name, server_port, num=-1): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((server_name, server_port)) - - logging.info("Connected to %s:%d", server_name, server_port) - handle(sock, num) - - -def main(): - logging.basicConfig(level=logging.INFO, format="%(name)s: %(message)s") - random.seed() - - if len(sys.argv) < 3 or len(sys.argv) > 4: - print( - "Usage: {} server_name server_port [num]".format(sys.argv[0]), - file=sys.stderr, - ) - print(" 0 <= num <= 34", file=sys.stderr) - sys.exit(1) - - server_name = sys.argv[1] - server_port = int(sys.argv[2]) - if len(sys.argv) == 4: - num = int(sys.argv[3]) - run_client(server_name, server_port, num) - else: - run_client(server_name, server_port) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lecture/demo/optimizations/client_bench.sh b/content/chapters/io/lecture/demo/optimizations/client_bench.sh deleted file mode 100755 index 44af0a1bfc..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/client_bench.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [[ $# != 1 ]]; then - echo "usage: $0 " - exit 1 -fi - -NUM_CLIENTS=8 -PORT=$1 - -for i in $(seq 1 $NUM_CLIENTS); do - ./client.py localhost "$1" 30 & -done -wait diff --git a/content/chapters/io/lecture/demo/optimizations/fwrite-10000.c b/content/chapters/io/lecture/demo/optimizations/fwrite-10000.c deleted file mode 100644 index fe1ee14600..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/fwrite-10000.c +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include - -#include "utils/utils.h" - -#define COUNT 10000 -#define FILENAME "data.txt" - -int main() -{ - static const char buf[COUNT] = {'a'}; - FILE *fp; - - fp = fopen(FILENAME, "w"); - DIE(fp == NULL, "fopen"); - - for (int i = 0; i < COUNT; i++) - { - fwrite(buf, sizeof(char), 1, fp); - } - - fclose(fp); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/optimizations/mp_server.py b/content/chapters/io/lecture/demo/optimizations/mp_server.py deleted file mode 100755 index 2aee137aae..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/mp_server.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: BSD-3-Clause - -import logging -import multiprocessing -import socket -import sys - - -def fibonacci(num): - if num in (0, 1): - return 1 - return fibonacci(num - 1) + fibonacci(num - 2) - - -def handle(connection, address): - logging.info("Received connection from %s", address) - response = "" - try: - msg = connection.recv(256).decode("utf-8") - logging.debug("Message: %s", msg) - num = int(msg) - if num < 0 or num > 34: - response = "out of range" - response = str(fibonacci(num)) - except ValueError: - response = "error" - - connection.send(response.encode("utf-8")) - - connection.close() - - -LISTEN_BACKLOG = 10 - - -def run_server(port, hostname="0.0.0.0"): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((hostname, port)) - sock.listen(LISTEN_BACKLOG) - - while True: - conn, address = sock.accept() - process = multiprocessing.Process(target=handle, args=(conn, address)) - process.daemon = True - process.start() - - -def main(): - logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") - if len(sys.argv) != 2: - print("Usage: {} port".format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - port = int(sys.argv[1]) - logging.info("Starting server on port {:d}".format(port)) - run_server(port) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lecture/demo/optimizations/mt_server.py b/content/chapters/io/lecture/demo/optimizations/mt_server.py deleted file mode 100755 index 36000edcb9..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/mt_server.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: BSD-3-Clause - -import logging -import socket -import sys -import threading - - -def fibonacci(num): - if num in (0, 1): - return 1 - return fibonacci(num - 1) + fibonacci(num - 2) - - -def handle(connection, address): - logging.info("Received connection from %s", address) - response = "" - try: - msg = connection.recv(256).decode("utf-8") - logging.debug("Message: %s", msg) - num = int(msg) - if num < 0 or num > 34: - response = "out of range" - response = str(fibonacci(num)) - except ValueError: - response = "error" - - connection.send(response.encode("utf-8")) - - connection.close() - - -LISTEN_BACKLOG = 10 - - -def run_server(port, hostname="0.0.0.0"): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((hostname, port)) - sock.listen(LISTEN_BACKLOG) - - while True: - conn, address = sock.accept() - thread = threading.Thread(target=handle, args=(conn, address)) - thread.start() - - -def main(): - logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") - - if len(sys.argv) != 2: - print("Usage: {} port".format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - port = int(sys.argv[1]) - logging.info("Starting server on port {:d}".format(port)) - run_server(port) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lecture/demo/optimizations/server.py b/content/chapters/io/lecture/demo/optimizations/server.py deleted file mode 100755 index 4a59e590c2..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/server.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-License-Identifier: BSD-3-Clause - -import logging -import socket -import sys - - -def fibonacci(num): - if num in (0, 1): - return 1 - return fibonacci(num - 1) + fibonacci(num - 2) - - -def handle(connection, address): - logging.info("Received connection from %s", address) - response = "" - try: - msg = connection.recv(256).decode("utf-8") - logging.debug("Message: %s", msg) - num = int(msg) - if num < 0 or num > 34: - response = "out of range" - response = str(fibonacci(num)) - except ValueError: - response = "error" - - connection.send(response.encode("utf-8")) - - connection.close() - - -LISTEN_BACKLOG = 10 - - -def run_server(port, hostname="0.0.0.0"): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((hostname, port)) - sock.listen(LISTEN_BACKLOG) - - while True: - conn, address = sock.accept() - handle(conn, address) - - -def main(): - logging.basicConfig(level=logging.DEBUG, format="%(name)s: %(message)s") - - if len(sys.argv) != 2: - print("Usage: {} port".format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - port = int(sys.argv[1]) - logging.info("Starting server on port {:d}".format(port)) - run_server(port) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/content/chapters/io/lecture/demo/optimizations/stdout-vs-stderr.c b/content/chapters/io/lecture/demo/optimizations/stdout-vs-stderr.c deleted file mode 100644 index fe56a4cc09..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/stdout-vs-stderr.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include - -int main() -{ - // setvbuf(stdout, NULL, _IONBF, 0); - - fprintf(stdout, "Hello "); - fprintf(stdout, "from "); - fprintf(stdout, "the "); - - fprintf(stderr, "Join the "); - - fprintf(stdout, "other "); - fprintf(stdout, "side!"); - - fprintf(stderr, "dark side!\n"); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/optimizations/write-1000-unbuf.c b/content/chapters/io/lecture/demo/optimizations/write-1000-unbuf.c deleted file mode 100644 index 6a16751851..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/write-1000-unbuf.c +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include - -#include "utils/utils.h" - -#define COUNT 1000 -#define FILENAME "data.txt" - -int main() -{ - static const char buf[COUNT] = {'a'}; - int fd; - - fd = open(FILENAME, O_CREAT | O_TRUNC | O_WRONLY | O_SYNC, 0644); - DIE(fd < 0, "open"); - - for (int i = 0; i < COUNT; i++) - { - DIE((write(fd, buf + i, 1)) < 0, "write"); - } - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lecture/demo/optimizations/write-10000.c b/content/chapters/io/lecture/demo/optimizations/write-10000.c deleted file mode 100644 index 62f0bcba4f..0000000000 --- a/content/chapters/io/lecture/demo/optimizations/write-10000.c +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -#include -#include -#include - -#include "utils/utils.h" - -#define COUNT 10000 -#define FILENAME "data.txt" - -int main() -{ - static const char buf[COUNT] = {'a'}; - int fd; - - fd = open(FILENAME, O_CREAT | O_TRUNC | O_WRONLY, 0644); - DIE(fd < 0, "open"); - - for (int i = 0; i < COUNT; i++) - { - DIE((write(fd, buf + i, 1)) < 0, "write"); - } - - close(fd); - - return 0; -} diff --git a/content/chapters/io/lecture/media/IPC-file-interface.svg b/content/chapters/io/lecture/media/IPC-file-interface.svg deleted file mode 100644 index 4f233d5554..0000000000 --- a/content/chapters/io/lecture/media/IPC-file-interface.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
write()
write()
Proc1
Proc1
Proc2
Proc2
Kernel Space
Kernel...
User Space
User Space
read()
read()
tmp.txt
tmp.txt
Filesystem
Filesy...Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/IPC-pipe.svg b/content/chapters/io/lecture/media/IPC-pipe.svg deleted file mode 100644 index 94c0fb1447..0000000000 --- a/content/chapters/io/lecture/media/IPC-pipe.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
write()
write()
Proc1
Proc1
Proc2
Proc2
Kernel Space
Kernel...
User Space
User Space
read()
read()
pipe
pipeText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/char-block-devices.svg b/content/chapters/io/lecture/media/char-block-devices.svg deleted file mode 100644 index 6a80fd6b6f..0000000000 --- a/content/chapters/io/lecture/media/char-block-devices.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Devices
Devices
Char Devices
Char Devices
Block Devices
Block Devices
Hardware
Hardware
OS
OSFile Interface
char device file
char device file
block device file
block device file
. . .
. . .
read
read
write
write
open
open
seek
seek
close
closeText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/dev-mouse-keyboard.png b/content/chapters/io/lecture/media/dev-mouse-keyboard.png deleted file mode 100644 index aec504ccc9..0000000000 Binary files a/content/chapters/io/lecture/media/dev-mouse-keyboard.png and /dev/null differ diff --git a/content/chapters/io/lecture/media/dev-sensors.png b/content/chapters/io/lecture/media/dev-sensors.png deleted file mode 100644 index 3405134285..0000000000 Binary files a/content/chapters/io/lecture/media/dev-sensors.png and /dev/null differ diff --git a/content/chapters/io/lecture/media/dev-storage.png b/content/chapters/io/lecture/media/dev-storage.png deleted file mode 100644 index 46a44d816c..0000000000 Binary files a/content/chapters/io/lecture/media/dev-storage.png and /dev/null differ diff --git a/content/chapters/io/lecture/media/device-software-stack.svg b/content/chapters/io/lecture/media/device-software-stack.svg deleted file mode 100644 index 54c2b27db9..0000000000 --- a/content/chapters/io/lecture/media/device-software-stack.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -File Interface
read
read
write
write
open
open
seek
seek
close
closeDriver Interface
IOCTL
IOCTL
OS
OS
Hardware
HardwareCommunication Protocol
send_command
send_command
read_data
read_data
read_dev_status
read_dev_status
You are here
You are hereText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/devices.png b/content/chapters/io/lecture/media/devices.png deleted file mode 100644 index 45a0b63ae2..0000000000 Binary files a/content/chapters/io/lecture/media/devices.png and /dev/null differ diff --git a/content/chapters/io/lecture/media/epoll-create.svg b/content/chapters/io/lecture/media/epoll-create.svg deleted file mode 100644 index e009695a03..0000000000 --- a/content/chapters/io/lecture/media/epoll-create.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
User Space
User Space
Application
Application
Kernel Space
Kernel Space
epoll instance
epoll instanceinterest listready_list
epfd = epoll_create(size)
epfd = epoll_create(size)Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/epoll-ctl-1.svg b/content/chapters/io/lecture/media/epoll-ctl-1.svg deleted file mode 100644 index 01840f7048..0000000000 --- a/content/chapters/io/lecture/media/epoll-ctl-1.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
User Space
User Space
Application
Application
Kernel Space
Kernel Space
epoll instance
epoll instanceinterest listready_list
status = epoll_ctl(epfd, op, fd, *event)
status = epoll_ctl(epfd, op, fd, *eve...Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/epoll-ctl-2.svg b/content/chapters/io/lecture/media/epoll-ctl-2.svg deleted file mode 100644 index e84c3c0cdb..0000000000 --- a/content/chapters/io/lecture/media/epoll-ctl-2.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
User Space
User Space
Application
Application
Kernel Space
Kernel Space
epoll instance
epoll instanceinterest listready_list
status = epoll_ctl(epfd, op, fd, *event)
status = epoll_ctl(epfd, op, fd, *eve...
EPOLL_CTL_ADD
EPOLL_CTL_ADD
EPOLL_CTL_MOD
EPOLL_CTL_MOD
EPOLL_CTL_DEL
EPOLL_CTL_DELText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/epoll-ctl-3.svg b/content/chapters/io/lecture/media/epoll-ctl-3.svg deleted file mode 100644 index 1911ea69eb..0000000000 --- a/content/chapters/io/lecture/media/epoll-ctl-3.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
User Space
User Space
Application
Application
Kernel Space
Kernel Space
epoll instance
epoll instanceinterest listready_list
status = epoll_ctl(epfd, op, fd, *event)
status = epoll_ctl(epfd, op, fd, *eve...
EPOLLOUT
EPOLLOUT
EPOLLIN
EPOLLINText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/epoll-ctl-4.svg b/content/chapters/io/lecture/media/epoll-ctl-4.svg deleted file mode 100644 index aef917e668..0000000000 --- a/content/chapters/io/lecture/media/epoll-ctl-4.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
User Space
User Space
Application
Application
Kernel Space
Kernel Space
epoll instance
epoll instanceinterest list
fd
fdready_list
status = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, EPOLLIN)
status = epoll_ctl(epfd, EPOLL_CTL_ADD, fd, EPOLLIN)
R
RText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/epoll-ctl-5.svg b/content/chapters/io/lecture/media/epoll-ctl-5.svg deleted file mode 100644 index 5b766b2439..0000000000 --- a/content/chapters/io/lecture/media/epoll-ctl-5.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
User Space
User Space
Application
Application
Kernel Space
Kernel Space
epoll instance
epoll instanceinterest list
fd
fdready_list
status = epoll_ctl(epfd, EPOLL_CTL_MOD, fd, EPOLLOUT)
status = epoll_ctl(epfd, EPOLL_CTL_MOD, fd, EPOLLOUT)
W
WText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/epoll-ctl-6.svg b/content/chapters/io/lecture/media/epoll-ctl-6.svg deleted file mode 100644 index 9fea5d24e7..0000000000 --- a/content/chapters/io/lecture/media/epoll-ctl-6.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
User Space
User Space
Application
Application
Kernel Space
Kernel Space
epoll instance
epoll instanceinterest list
fd
fdready_list
status = epoll_ctl(epfd, EPOLL_CTL_DEL, fd, NULL)
status = epoll_ctl(epfd, EPOLL_CTL_DEL, fd, NULL)Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/epoll-wait.svg b/content/chapters/io/lecture/media/epoll-wait.svg deleted file mode 100644 index 2c0b0bf18b..0000000000 --- a/content/chapters/io/lecture/media/epoll-wait.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
User Space
User Space
Application
Application
Kernel Space
Kernel Space
epoll instance
epoll instanceinterest list
fd1
fd1
fd2
fd2
fd3
fd3
fd4
fd4
...
...ready_list
fd2
fd2
fd4
fd4
status = epoll_wait(epfd, *events, maxevents, timeout)
status = epoll_wait(epfd, *events, maxevents, timeou...Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/file-descriptor-table.svg b/content/chapters/io/lecture/media/file-descriptor-table.svg deleted file mode 100644 index e792273de1..0000000000 --- a/content/chapters/io/lecture/media/file-descriptor-table.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
PROC
PROCFDT
fd
fd
Open File Struct*
Open File Struct*
0
0
stdin
stdin
1
1
stdout
stdout
2
2
stderr
stderr
3
3
*
*
...
...
Open File Struct
Open File Struct
permissions
permissions
offset
offset
reference count
reference count
inode
inode
keyboard
keyboard
display
displayText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/file-interface-close.svg b/content/chapters/io/lecture/media/file-interface-close.svg deleted file mode 100644 index c09c82a508..0000000000 --- a/content/chapters/io/lecture/media/file-interface-close.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
status = close(handle)
status = close(handle)
OS
OS
IO Session
IO Session
ref_count--
ref_cou...
destroy?
destroy?
syscall
syscall
Filesystem
Filesystem
File
File
pointer
pointer
refer
referText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/file-interface-dup.svg b/content/chapters/io/lecture/media/file-interface-dup.svg deleted file mode 100644 index 4d77b97de7..0000000000 --- a/content/chapters/io/lecture/media/file-interface-dup.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
new_handle = dup(old_handle)
new_handle = dup(old_handle)
OS
OS
IO Session
IO Session
syscall
syscall
Filesystem
Filesystem
File
File
pointer
pointer
refer
refer
refer
referText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/file-interface-ftruncate.svg b/content/chapters/io/lecture/media/file-interface-ftruncate.svg deleted file mode 100644 index 633ecf9aab..0000000000 --- a/content/chapters/io/lecture/media/file-interface-ftruncate.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
status = ftruncate(handle, length)
status = ftruncate(handle, length)
refer
refer
OS
OS
IO Session
IO Session
syscall
syscall
Filesystem
Filesystem
File
File
update size
update size
ISA
ISA
pointer
pointerText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/file-interface-lseek.svg b/content/chapters/io/lecture/media/file-interface-lseek.svg deleted file mode 100644 index 4feb4e2d19..0000000000 --- a/content/chapters/io/lecture/media/file-interface-lseek.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
offset = lseek(handle, offset, whence)
offset = lseek(handle, offset, whenc...
refer
refer
OS
OS
IO Session
IO Session
syscall
syscall
Filesystem
Filesystem
File
File
update offset
update...
pointer
pointerText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/file-interface-open.svg b/content/chapters/io/lecture/media/file-interface-open.svg deleted file mode 100644 index f6fe07df8a..0000000000 --- a/content/chapters/io/lecture/media/file-interface-open.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
handle = open(<filename>, <mode>)
handle = open(<filename>, <mode>)
OS
OS
IO Session
IO Session
syscall
syscall
Filesystem
Filesystem
File
File
create
create
pointer
pointer
returns
returnsText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/file-interface-read-write.svg b/content/chapters/io/lecture/media/file-interface-read-write.svg deleted file mode 100644 index 93d9ae3136..0000000000 --- a/content/chapters/io/lecture/media/file-interface-read-write.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
OS
OS
IO Session
IO Session
syscall
syscall
Filesystem
Filesystem
File
File
pointer
pointer
references
references
count  = read(handle, *buf, num)
count  = read(handle, *buf, num)
count  = write(handle, *buf, num)
count  = write(handle, *buf, num)
bytes
bytesText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/interactive-software.png b/content/chapters/io/lecture/media/interactive-software.png deleted file mode 100644 index bf7ba3e9e7..0000000000 Binary files a/content/chapters/io/lecture/media/interactive-software.png and /dev/null differ diff --git a/content/chapters/io/lecture/media/io-buffering-overview.svg b/content/chapters/io/lecture/media/io-buffering-overview.svg deleted file mode 100644 index a0698aa08d..0000000000 --- a/content/chapters/io/lecture/media/io-buffering-overview.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
fread()
fread()
fwrite()
fwrite()
Application
Application
Disk
Disk
actual
write
actual...
actual
read
actual...
write()
write()
read()
read()
libc
buffer
libc...
kernel
buffer
kernel...
User Space
User Space
Kernel Space
Kernel SpaceText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/pipe-walkthrough-1.svg b/content/chapters/io/lecture/media/pipe-walkthrough-1.svg deleted file mode 100644 index eb7b892659..0000000000 --- a/content/chapters/io/lecture/media/pipe-walkthrough-1.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc1
Proc1FDT
fd
fd
Open File Struct*
Open File Struct*
0
0
stdin
stdin
1
1
stdout
stdout
2
2
stderr
stderr
3
3
4
4
...
...
User Space
User Space
OS
OSText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/pipe-walkthrough-2.svg b/content/chapters/io/lecture/media/pipe-walkthrough-2.svg deleted file mode 100644 index 9ad00213a7..0000000000 --- a/content/chapters/io/lecture/media/pipe-walkthrough-2.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc1
Proc1FDT
fd
fd
Open File Struct*
Open File Struct*
0
0
stdin
stdin
1
1
stdout
stdout
2
2
stderr
stderr
3
3
*
*
4
4
*
*
...
...
pipe
pipe
User Space
User Space
OS
OS
pipe()
pipe()Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/pipe-walkthrough-3.svg b/content/chapters/io/lecture/media/pipe-walkthrough-3.svg deleted file mode 100644 index bbf22c3d8d..0000000000 --- a/content/chapters/io/lecture/media/pipe-walkthrough-3.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc1
Proc1FDT
fd
fd
Open File Struct*
Open File Struct*
0
0
stdin
stdin
1
1
stdout
stdout
2
2
stderr
stderr
3
3
*
*
4
4
*
*
...
...
pipe
pipe
fork()
fork()
Proc2
Proc2FDT
fd
fd
Open File Struct*
Open File Struct*
0
0
stdin
stdin
1
1
stdout
stdout
2
2
stderr
stderr
3
3
*
*
4
4
*
*
...
...
User Space
User Space
OS
OSText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/pipe-walkthrough-4.svg b/content/chapters/io/lecture/media/pipe-walkthrough-4.svg deleted file mode 100644 index 48de0181fe..0000000000 --- a/content/chapters/io/lecture/media/pipe-walkthrough-4.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc1
Proc1FDT
fd
fd
Open File Struct*
Open File Struct*
0
0
stdin
stdin
1
1
stdout
stdout
2
2
stderr
stderr
3
3
4
4
*
*
...
...
pipe
pipe
close(3)
close(3)
Proc2
Proc2FDT
fd
fd
Open File Struct*
Open File Struct*
0
0
stdin
stdin
1
1
stdout
stdout
2
2
stderr
stderr
3
3
*
*
4
4
*
*
...
...
User Space
User Space
OS
OSText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/pipe-walkthrough-5.svg b/content/chapters/io/lecture/media/pipe-walkthrough-5.svg deleted file mode 100644 index 56238b4d52..0000000000 --- a/content/chapters/io/lecture/media/pipe-walkthrough-5.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc1
Proc1FDT
fd
fd
Open File Struct*
Open File Struct*
0
0
stdin
stdin
1
1
stdout
stdout
2
2
stderr
stderr
3
3
4
4
*
*
...
...
pipe
pipe
close(4)
close(4)
Proc2
Proc2FDT
fd
fd
Open File Struct*
Open File Struct*
0
0
stdin
stdin
1
1
stdout
stdout
2
2
stderr
stderr
3
3
*
*
4
4
...
...
User Space
User Space
OS
OSText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/repeated-copy.svg b/content/chapters/io/lecture/media/repeated-copy.svg deleted file mode 100644 index 3da14f039c..0000000000 --- a/content/chapters/io/lecture/media/repeated-copy.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
1
1
Disk
Disk
3
3
Application
Application
2
2
Read buffer
Read buffer
4
4
Socket buffer
Socket buffer
Network
Network
Kernel
Space
Kernel...
User
Space
User...Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/roadmap-Compute.svg b/content/chapters/io/lecture/media/roadmap-Compute.svg deleted file mode 100644 index 31e8d9231a..0000000000 --- a/content/chapters/io/lecture/media/roadmap-Compute.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Data
Store
Data
Store...
Retrieve
Data
Retri...
Programmer
Prog...
Program
Program
Memory
Memory
Data
Data
Input
Input
Data
Data
Output
OutputComputeText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/roadmap-Data.svg b/content/chapters/io/lecture/media/roadmap-Data.svg deleted file mode 100644 index 6fcc5145e3..0000000000 --- a/content/chapters/io/lecture/media/roadmap-Data.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Data
Store
Data
Store...
Retrieve
Data
Retri...
Programmer
Prog...
Program
Program
Memory
Memory
Data
Data
Input
Input
Data
Data
Output
OutputText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/roadmap-IO.svg b/content/chapters/io/lecture/media/roadmap-IO.svg deleted file mode 100644 index 9a68c653f7..0000000000 --- a/content/chapters/io/lecture/media/roadmap-IO.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Data
Store
Data
Store...
Retrieve
Data
Retri...
Programmer
Prog...
Program
Program
Memory
Memory
Data
Data
Input
Input
Data
Data
Output
OutputComputeIOIOText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/roadmap.svg b/content/chapters/io/lecture/media/roadmap.svg deleted file mode 100644 index 60d996467e..0000000000 --- a/content/chapters/io/lecture/media/roadmap.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Programmer
Prog...
Program
Program
Memory
Memory
Data
Data
Input
Input
Data
Data
Output
Output
Store
Data
Store...
Retrieve
Data
Retri...Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/snow-leopard.png b/content/chapters/io/lecture/media/snow-leopard.png deleted file mode 100644 index 5a1a3eb233..0000000000 Binary files a/content/chapters/io/lecture/media/snow-leopard.png and /dev/null differ diff --git a/content/chapters/io/lecture/media/socket-interface-accept.svg b/content/chapters/io/lecture/media/socket-interface-accept.svg deleted file mode 100644 index 610b35c5b6..0000000000 --- a/content/chapters/io/lecture/media/socket-interface-accept.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
sockfd = socket(domain, type, protocol)
sockfd = socket(domain, type, protocol)
syscall
syscall
OS
OS
connection
socket
connection...
socket
socket
create
create
address
address
refer
refer
status = bind(sockfd, addr, addr_len)
status = bind(sockfd, addr, addr_len)
status = listen(sockfd, backlog)
status = listen(sockfd, backlog)
new_ sockfd = accept(sockfd, addr, addr_len)
new_ sockfd = accept(sockfd, addr, addr_len)
refer
referText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-interface-bind.svg b/content/chapters/io/lecture/media/socket-interface-bind.svg deleted file mode 100644 index b78481369b..0000000000 --- a/content/chapters/io/lecture/media/socket-interface-bind.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
sockfd = socket(domain, type, protocol)
sockfd = socket(domain, type, protocol)
syscall
syscall
OS
OS
socket
socket
address
address
refer
refer
status = bind(sockfd, addr, addr_len)
status = bind(sockfd, addr, addr_len)Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-interface-connect.svg b/content/chapters/io/lecture/media/socket-interface-connect.svg deleted file mode 100644 index dffc4b2fb5..0000000000 --- a/content/chapters/io/lecture/media/socket-interface-connect.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
sockfd = socket(domain, type, protocol)
sockfd = socket(domain, type, protocol)
status = connect(sockfd, addr, addr_len)
status = connect(sockfd, addr, addr_len)
syscall
syscall
OS
OS
socket
socket
establish
connection
establ...
refer
referText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-interface-listen.svg b/content/chapters/io/lecture/media/socket-interface-listen.svg deleted file mode 100644 index 4ef24b6438..0000000000 --- a/content/chapters/io/lecture/media/socket-interface-listen.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
sockfd = socket(domain, type, protocol)
sockfd = socket(domain, type, protocol)
syscall
syscall
OS
OS
connection
socket
connection...
mark
passive
mark...
refer
refer
status = bind(sockfd, addr, addr_len)
status = bind(sockfd, addr, addr_len)
status = listen(sockfd, backlog)
status = listen(sockfd, backlog)Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-interface-send-recv.svg b/content/chapters/io/lecture/media/socket-interface-send-recv.svg deleted file mode 100644 index 50156fec3e..0000000000 --- a/content/chapters/io/lecture/media/socket-interface-send-recv.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
count = send(sockfd, *buf, num, flags)
count = send(sockfd, *buf, num, flags)
syscall
syscall
OS
OS
socket
socket
refer
refer
count = recv(sockfd, *buf, num, flags)
count = recv(sockfd, *buf, num, flags)Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-interface-socket.svg b/content/chapters/io/lecture/media/socket-interface-socket.svg deleted file mode 100644 index d19f8c1674..0000000000 --- a/content/chapters/io/lecture/media/socket-interface-socket.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Proc
Proc
sockfd = socket(domain, type, protocol)
sockfd = socket(domain, type, protocol)
syscall
syscall
OS
OS
socket
socket
create
create
refer
referText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-interface.svg b/content/chapters/io/lecture/media/socket-interface.svg deleted file mode 100644 index 7f05934718..0000000000 --- a/content/chapters/io/lecture/media/socket-interface.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
Communication
Channel
Communication...
Proc2
Proc2
Socket
Socket
Protocol
Protocol
User Space
User Spa...
Kernel Space
Kernel Space
Proc1
Proc1
Socket
Socket
Protocol
Protocol
User Space
User Spa...
Kernel Space
Kernel SpaceText is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-1.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-1.svg deleted file mode 100644 index 4024ae60d6..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-1.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -ServerClient \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-10.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-10.svg deleted file mode 100644 index 7cc036d66c..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-10.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()Clientbind()connect()send()accept()optional for SOCK_STREAMsocket()recv() \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-11.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-11.svg deleted file mode 100644 index 6c5cdfa2b7..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-11.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()Clientbind()connect()send()accept()optional for SOCK_STREAMsocket()recv()recv() \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-12.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-12.svg deleted file mode 100644 index 9f746cd27e..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-12.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()send()Clientbind()connect()send()accept()optional for SOCK_STREAMsocket()recv()recv() \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-13.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-13.svg deleted file mode 100644 index 2b6b7b3fe0..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-13.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()send()close()Clientbind()connect()send()accept()optional for SOCK_STREAMsocket()recv()recv() \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-14.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-14.svg deleted file mode 100644 index 76342e9a4b..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-14.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()send()close()Clientbind()connect()send()accept()optional for SOCK_STREAMsocket()recv()recv()close() \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-2.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-2.svg deleted file mode 100644 index 13c70e1f19..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-2.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()Client \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-3.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-3.svg deleted file mode 100644 index ef011922df..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-3.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()Client \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-4.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-4.svg deleted file mode 100644 index fb1e50a278..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-4.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()Client \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-5.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-5.svg deleted file mode 100644 index 272b6edf40..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-5.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()Clientaccept() \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-6.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-6.svg deleted file mode 100644 index 5fff78342e..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-6.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()Clientaccept()socket() \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-7.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-7.svg deleted file mode 100644 index 91cf095a53..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-7.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()Clientbind()accept()optional for SOCK_STREAMsocket() \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-8.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-8.svg deleted file mode 100644 index e4cee63116..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-8.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()Clientbind()connect()accept()optional for SOCK_STREAMsocket() \ No newline at end of file diff --git a/content/chapters/io/lecture/media/socket-summary/socket-summary-9.svg b/content/chapters/io/lecture/media/socket-summary/socket-summary-9.svg deleted file mode 100644 index b0163d79f2..0000000000 --- a/content/chapters/io/lecture/media/socket-summary/socket-summary-9.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -Serversocket()bind()listen()Clientbind()connect()accept()optional for SOCK_STREAMsocket()recv() \ No newline at end of file diff --git a/content/chapters/io/lecture/media/stream-datagram-simplified.png b/content/chapters/io/lecture/media/stream-datagram-simplified.png deleted file mode 100644 index d4de1417d8..0000000000 Binary files a/content/chapters/io/lecture/media/stream-datagram-simplified.png and /dev/null differ diff --git a/content/chapters/io/lecture/media/zero-copy.svg b/content/chapters/io/lecture/media/zero-copy.svg deleted file mode 100644 index 45fb02b050..0000000000 --- a/content/chapters/io/lecture/media/zero-copy.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
1
1
Disk
Disk
Application
Application
2
2
Read buffer
Read buffer
3
3
Socket buffer
Socket buffer
Network
Network
Kernel
Space
Kernel...
User
Space
User...
sendfile()
sendfi...Text is not SVG - cannot display \ No newline at end of file diff --git a/content/chapters/io/lecture/slides/IPC.md b/content/chapters/io/lecture/slides/IPC.md deleted file mode 100644 index 4b21d7d173..0000000000 --- a/content/chapters/io/lecture/slides/IPC.md +++ /dev/null @@ -1,284 +0,0 @@ -### Interprocess Communication - ---- - -### Interprocess Communication - -- A communication channel involves a process and a data endpoint -- We can use persistent data and I/O devices as endpoints with the **file interface** -- Another possible endpoint is a **process** - - [Law of the hammer](https://en.wikipedia.org/wiki/Law_of_the_instrument) - - Socket interface - ----- - -### IPC - File Interface - -![IPC through File Interface](./media/IPC-file-interface.svg) - -- Writing to disk and reading from disk is tedious - ----- - -### IPC - Pipe - -![IPC through pipe](./media/IPC-pipe.svg) - -- Same idea as above - ----- - -- **Proc1** writes at one end -- **Proc2** reads from the other end - - Why is it better? - - The **pipe** is stored in Kernel Space, we no longer use the disk -- Only works for related processes ([fork](https://man7.org/linux/man-pages/man2/fork.2.html)) - ----- - -### Pipe - Walkthrough - -![Pipe walkthrough - 1](./media/pipe-walkthrough-1.svg) - ----- - -### Pipe - Walkthrough - -![Pipe walkthrough - 2](./media/pipe-walkthrough-2.svg) - ----- - -### Pipe - Walkthrough - -![Pipe walkthrough - 3](./media/pipe-walkthrough-3.svg) - ----- - -### Pipe - Walkthrough - -![Pipe walkthrough - 4](./media/pipe-walkthrough-4.svg) - ----- - -### Pipe - Walkthrough - -![Pipe walkthrough - 5](./media/pipe-walkthrough-5.svg) - ----- - -### `pipe()` demo - -- `demo/IPC/pipe.c` - ---- - -### IPC - Named Pipe (FIFO) - -- Bypass unnamed pipes limitations: - - Can be used by more than 2 processes - - Can be used by unrelated processes -- Stored on disk as a **special file** - -```console -student@os:~$ ls -l my_fifo -prw-r--r-- 1 student student 0 nov 22 18:38 my_fifo -``` - ----- - -### `mkfifo()` demo - -- `demo/IPC/fifo.c` - ---- - -### IPC - Socket Interface - -![Socket Interface](./media/socket-interface.svg) - ----- - -### Socket - -- The endpoint in the interprocess communication -- Uniquely identifies the process in the communication -- Supports multiple transmission types - - e.g.: `stream`, `datagram` - ----- - -### Socket Interface - `socket()` - -![socket()](./media/socket-interface-socket.svg) - ----- - -- Creates a socket with given **domain** and **type** -- Returns a **file descriptor** - - Compatible with `read()`/`write()` operations - - Does not support `fseek` - ----- - -### Socket Attributes - -- **Domain** - - `AF_UNIX` for communication on the **same host** - - `AF_INET` for communication over the internet -- **Type** - - **Stream** establishes a reliable connection and ensures all data is transmitted - - **Datagram** sends data faster without checking its integrity - ----- - -### Stream and Datagram - -![Stream and Datagram simplified](./media/stream-datagram-simplified.png) - ---- - -### Client-Server - -- Socket communication uses the **client-server** model -- **Server** - - **Bound** to an **address** and **accepts** connections - - Answers queries from clients -- **Client** - - **Connects** to a server using its **address** - - Sends queries to the server - ---- - -### Server Workflow - ----- - -#### Server - `socket()` - -![socket()](./media/socket-interface-socket.svg) - -- Create socket - ----- - -#### Server - `bind()` - -![bind()](./media/socket-interface-bind.svg) - -- Assign address to the socket - ----- - -#### Server - `listen()` - -![listen()](./media/socket-interface-listen.svg) - -- Mark the socket **passive** - will be used to accept connections - ----- - -#### Server - `accept()` - -![accept()](./media/socket-interface-accept.svg) - -- Wait for a connection -- Accept the connection and create a socket for it - ---- - -### Client Workflow - ----- - -#### Client - `socket()` - -![socket()](./media/socket-interface-socket.svg) - -- Create a socket - ----- - -#### Client - `connect()` - -![TCP socket](./media/socket-interface-connect.svg) - -- Establish connection with server - ---- - -### `send()`/`recv()` - -![Socket send recv](./media/socket-interface-send-recv.svg) - -- Same as `read()`/`write()` - ----- - -#### `send()` - -- Use socket to send bytes -- Return number of sent bytes - - **-1** - send failed - - ** - -- Means for our application to communicate -- Compatibility with various devices -- Responsiveness (Non-blocking I/O) -- Performance (fast transfer, I/O multiplexing) diff --git a/content/chapters/io/lecture/slides/optimizations.md b/content/chapters/io/lecture/slides/optimizations.md deleted file mode 100644 index da433030a8..0000000000 --- a/content/chapters/io/lecture/slides/optimizations.md +++ /dev/null @@ -1,261 +0,0 @@ -## IO Buffering - ---- - -## `libc` Buffering - ----- - -### `stdout` vs `stderr` - -- `demo/optimizations/stdout-vs-stderr.c` - ----- - -### Investigate - -```console -student@os:~/.../demo/optimizations$ ./stdout-vs-stderr -Join the dark side! -Hello from the other side! -``` - -- Not what we expected - -```console -strace --trace=write ./stdout-vs-stderr -``` - ----- - -### Observations - -- Each print to `stderr` results in a `write()` -- Prints to `stdout` result in one single `write()` - ----- - -### Behind The Scenes - -- Printing to `stdout` is **buffered** to avoid multiple context switches -- The same is not true for `stderr` as we want to see errors as they occur - ----- - -### Does It Work? - -- Attempt to write **10000 bytes** to a file, **1 byte** at a time - - `demo/optimizations/fwrite-10000.c` - - `demo/optimizations/write-10000.c` - ----- - -### Can We Do More? - ---- - -## Kernel Buffering - ----- - -### Context - -- `libc` buffering reduces the number of context switches -- But **synchronizing** the disk after every `write()` is a major bottleneck - ----- - -### Kernel Buffer - -- Same as idea as `libc` -- Each `write()` fills a buffer in kernel space -- Transfer data to disk when - - Buffer is full - - `fsynk()` is called - - Enough time has passed since the last `write()` - -```console -student@os:~$ cat /proc/sys/vm/dirty_expire_centisecs -3000 -``` - ----- - -### Overview - -![IO Buffering Overview](./media/io-buffering-overview.svg) - ----- - -### Does It Work? - -- Write **10000 bytes** to disk, **1 byte** at a time - with buffering - - `demo/optimizations/write-10000.c` -- Write **1000 bytes** to disk, **1 byte** at a time - without buffering - - `demo/optimizations/write-1000-unbuf.c` - ---- - -## IO Buffering Drawbacks - -- Transfer of information between kernel buffers requires user space transitions -- Operations might **block** - - Read when buffer is empty - - Write when buffer is full - ---- - -## Kernel Buffers Transfer - -![Repeated Copy](./media/repeated-copy.svg) - ----- - -### Obvious Problems - -- We have an intermediary buffer - - Data from **read buffer** is copied to **application buffer** - - Data from **application buffer** is copied to **socket buffer** -- We perform two context switches - ----- - -### `zero-copy` - -![Zero-copy](./media/zero-copy.svg) - ----- - -- `sendfile()` instructs kernel to copy data from one of its buffers to another -- We perform a single context switch -- Data is copied a single time - ---- - -## Blocking IO - -- Reading from an empty buffer - - The kernel buffer is filled with information from the device - - The library buffer is filled from the kernel buffer - - `read()` operation resumes -- What happens if the device has no information to share? - - `read()` **blocks** - ----- - -### Non-Blocking IO - -- `O_NONBLOCK` makes operations return rather than block - - `SOCK_NONBLOCK` for sockets -- Allows handling of input on multiple file descriptors - - Does not scale with number of file descriptors - - The thread is busy waiting instead of entering WAITING state - - Ugly - ---- - -## `epoll` Interface - -- Linux interface, non-portable -- Kernel keeps an internal structure to monitor file descriptors -- The thread enters WAITING state until new connections emerge -- User updates the interface using the exposed interface - - `epoll_create()` - - `epoll_ctl()` - - `epoll_wait()` - ----- - -### `epoll_create()` - -![epoll_create()](./media/epoll-create.svg) - ---- - -### `epoll_ctl()` - -![epoll_ctl()](./media/epoll-ctl-1.svg) - ----- - -#### `epoll_ctl()` - -![epoll_ctl()](./media/epoll-ctl-2.svg) - ----- - -#### `epoll_ctl()` - -![epoll_ctl()](./media/epoll-ctl-3.svg) - ----- - -#### `epoll_ctl()` - -![epoll_ctl()](./media/epoll-ctl-4.svg) - ----- - -#### `epoll_ctl()` - -![epoll_ctl()](./media/epoll-ctl-5.svg) - ----- - -#### `epoll_ctl()` - -![epoll_ctl()](./media/epoll-ctl-6.svg) - ---- - -### `epoll_wait()` - -![epoll_wait()](./media/epoll-wait.svg) - ---- - -## Asynchronous IO - ---- - -## Client-Server 2.0 - -- Real-life applications use **Client-Server** model -- But they take into consideration scalability -- Let us see what it takes to go from 1 client to **8** - - Use `demo/optimizations/client.py` for the client implementation - - Use `demo/optimizations/client_bench.sh` to compare implementations - ----- - -### Trivial Server - -- Run server in a loop -- `demo/optimizations/server.py` - ----- - -### Good Old Processes - -- Create a process to handle each new client -- `demo/optimizations/mp_server.py` - ----- - -### Good Old Lightweight Processes - -- Create a process to handle each new client -- `demo/optimizations/mt_server.py` - ---- - -### Asynchronous IO - -- `demo/optimizations/async_server.py` - ----- - -- The python asynchronous interface abstracts a lot of the functionality -- The idea behind: - - The server socket is added to an **epoll instance** - - The specified **handle** is called for each event (new connection) diff --git a/gen-view.py b/gen-view.py index 868b64486d..1134d94bf2 100644 --- a/gen-view.py +++ b/gen-view.py @@ -88,7 +88,7 @@ def group_guides(): os.popen(f"cp {os.path.join(root, f)} {guidesDir}/{guideName}.md") -def setup_overview(fileToLab: dict): +def setup_overview(): """ Copy the overview.md file for each chapter to the .view directory. """ @@ -108,12 +108,13 @@ def setup_overview(fileToLab: dict): with open(f"{viewDir}/{c}-overview.md") as f: text = f.read() - text = solve_links(text, fileToLab) with open(f"{viewDir}/{c}-overview.md", "w") as f: f.write(text) + print() # Add a newline for better readability -def solve_links(text: str, fileToLab: dict) -> str: + +def solve_links(filename: str, fileToLab: dict) -> str: """ Make relative links work in the final markdown file. @@ -123,19 +124,29 @@ def solve_links(text: str, fileToLab: dict) -> str: The lab number is determined by the fileToLab dictionary, and the subchapter is the first line of the file. For example, [text](../reading/basic-syscall.md) will become [text](.view/lab1#basic-syscall). """ + with open(filename) as f: + text = f.read() + # Questions from the same chapter are at Questions/, without the .md extension - text = re.sub(r"(\[.*\])\(.*questions/(.*)\.md\)", r"\1(Questions/\2)", text) + text = re.sub(r"(\[.*?\])\(.*?questions/(.*?)\.md\)", r"\1(Questions/\2)", text) # Remove relative links to reading, media, tasks, and guides for section in ["reading", "media", "tasks", "guides"]: + # Questions are placed in a directory of their own, just like media, so we need to go up one level + if "questions" in filename and section == "media": + section = "../" + section text = re.sub( - r"(\[.*\])\(.*" + section + r"/(.*)\)", rf"\1({section}/\2)", text + r"(\[.*?\])\([^\)]*" + section + r"/(.*?)\)", rf"\1({section}/\2)", text ) # Reading links [text](.*/reading/.md) should be replaced with [text](.view/labQ#) # Where Q is the lab number and chapter is the heading of the file - matches = re.findall(r"\[.*\]\((.*\.md)\)", text) + matches = re.findall(r"\[[^\]]*\]\(([^\)]+\.md)\)", text) for sourceFile in matches: + origName = sourceFile # Save the original name for the regex + if sourceFile.endswith("README.md"): + sourceFile = os.path.dirname(sourceFile) + ".md" + filepath = os.path.join(viewDir, sourceFile) # Tasks and guides are prefixed with the section name @@ -152,27 +163,58 @@ def solve_links(text: str, fileToLab: dict) -> str: title = f.readline().strip("#").replace("`", "").replace(":", "") subchapter = prefix + hypenate(title) except: - print(f"Error: Could not solve link to {filepath}") + print(f"Error: Could not solve link to {filepath} for {filename}") continue text = re.sub( - rf"(\[.*\])\({sourceFile}\)", + rf"(\[.*\])\({origName}\)", # Use origName because tasks 'sourceFile' has changed rf"\1({fileToLab[sourceFile]}#{subchapter})", text, ) - return text + with open(filename, "w") as f: + f.write(text) + + +def find_broken_links(): + """ + Find potentially broken links in the markdown file. + """ + prefixes = ["lab", "media", "tasks", "Questions", "reading", "guides", "http"] + + for root, _, files in os.walk(viewDir): + for f in files: + if "lab" in f: # Skip lab files, check source files only + continue + + if f.endswith(".md"): + with open(os.path.join(root, f)) as f: + text = f.read() + + # Find all links that do not point to a markdown file + matches = re.findall(r"\[[^\]]*\]\(([^\)]+)\)", text) + for link in matches: + # Questions media links corner case + isValidQMediaLink = "questions" in root and link.startswith( + "../media/" + ) + + if ( + not any([link.startswith(p) for p in prefixes]) + ) and not isValidQMediaLink: + print(f"Possibly broken link in {f.name}: ({link})") class Lab: def __init__(self, title: str, filename: str, content: List[str]): - self.title = title - self.filename = filename - self.text = f"# {title}\n\n" for file in content: self.process_file(file) + print(f"Generating lab {viewDir}/{filename}") + with open(f"{viewDir}/{filename}", "w") as f: + f.write(self.text) + def process_file(self, filename: str): """ Process a file and add it to the lab text. @@ -195,17 +237,6 @@ def process_file(self, filename: str): filecontent = re.sub(r"^(#+)", r"\1#", filecontent, flags=re.MULTILINE) self.text += filecontent + "\n\n" - def generate(self, fileToLab: dict): - """ - Generate the final markdown file for the lab. - """ - print(f"Generating lab {viewDir}/{self.filename}") - - self.text = solve_links(self.text, fileToLab) - - with open(f"{viewDir}/{self.filename}", "w") as f: - f.write(self.text) - class ConfigParser: def __init__(self, path): @@ -213,11 +244,10 @@ def __init__(self, path): with open(path) as f: self.data = yaml.safe_load(f) - def create_labs(self) -> List[Lab]: - labs = [] + def create_labs(self): for entry in self.data["lab_structure"]: - labs.append(Lab(entry["title"], entry["filename"], entry["content"])) - return labs + Lab(entry["title"], entry["filename"], entry["content"]) + print() # Add a newline for better readability def get_file_to_lab_dict(self) -> dict: """ @@ -254,12 +284,19 @@ def main(): # Parse the config file config = ConfigParser("config.yaml") - labs = config.create_labs() - for lab in labs: - lab.generate(config.get_file_to_lab_dict()) + config.create_labs() # Copy the overview.md file for each chapter to the .view directory - setup_overview(config.get_file_to_lab_dict()) + setup_overview() + + # Solve links recursively in all markdown files + for root, _, files in os.walk(viewDir): + for f in files: + if f.endswith(".md"): + solve_links(os.path.join(root, f), config.get_file_to_lab_dict()) + + # Check for broken links + find_broken_links() if __name__ == "__main__":