diff --git a/.gitignore b/.gitignore index 680d1296e..5c2f91d50 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ tools/_infotojson/infotojson tools/ccanlint/test/run-file_analysis tools/configurator/configurator scores/ +.glimpse_* diff --git a/Makefile-ccan b/Makefile-ccan index c21c953f1..ab97267f2 100644 --- a/Makefile-ccan +++ b/Makefile-ccan @@ -61,6 +61,7 @@ MODS_WITH_SRC := aga \ eratosthenes \ err \ failtest \ + filesize_counter \ foreach \ grab_file \ hash \ diff --git a/ccan/filesize_counter/LICENSE b/ccan/filesize_counter/LICENSE new file mode 100644 index 000000000..4bc044227 --- /dev/null +++ b/ccan/filesize_counter/LICENSE @@ -0,0 +1,9 @@ +Software provided by Dan Good at Dell SecureWorks. For more information on Dell SecureWorks security services please browse to http://www.secureworks.com + +Copyright 2015 Dell SecureWorks + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/ccan/filesize_counter/_info b/ccan/filesize_counter/_info new file mode 100644 index 000000000..9ae7097b1 --- /dev/null +++ b/ccan/filesize_counter/_info @@ -0,0 +1,103 @@ +#include "config.h" +#include +#include + +/** + * filesize_counter - durable counters using sparse files + * + * The filesize_counter (szcnt) module keeps count by incrementing the size of + * a file. These counters are easy to read using lseek(2), stat(2), or ls(1). + * To keep the files small, they are replaced with sparse files at multiples + * of 4 KiB. + * + * The range of the counter is that of off_t. Compile with _FILE_OFFSET_BITS + * set to 64 to count beyond 2 GiB. + * + * Tests suggest this approach performs on par with writing a raw uint every + * time and is 4 to 5 times faster than using ftruncate only. + * + * This is a novel approach, so why do it this way? These counters are simple + * and unambiguous. Picture a large organization, where disparate groups are + * responsible for quality assurance, operations, and health monitoring of + * production systems. Add to that documentation inconsistencies, the need + * to maintain multiple software versions, and a regular influx of new staff, + * or staff turnover. + * + * Consider a counter implemented as a number stored in a file. If an + * operator cats the file and sees 123, is that a decimal number or a hex + * number? If the output appears binary, is it encoded? Is it a raw integer? + * Signed or unsigned? Big or little-endian? If the file is empty, is that + * the same as zero, or is that a bug? + * + * File size, by contrast, is readily understood, and a file always has a + * non-negative size. + * + * A counter implementation can be abstracted behind an API or tool. This way + * may increase the burdens of documentation, deployment, and testing, but + * probably worse is the additional dependency to be tracked and factored + * into release planning. + * + * File size, by contrast, is accessible through standard system calls + * and tools. The intrepid operator can even set the counter at need with + * truncate(1), dd(1), etc. + * + * Example: + * #include + * #include + * #include + * + * int main(int argc, char *argv[]) + * { + * FILE *f; + * struct szcnt *h; + * off_t n, i; + * char buf[BUFSIZ]; + * int ret; + * + * if (argc != 2) return 1; + * f = fopen(argv[1], "r"); + * if (!f) err(1, "fopen"); + * + * h = szcnt_new(); + * if (!h) err(1, "szcnt_new"); + * + * n = szcnt_open(h, "counter"); + * if (n == -1) err(1, "szcnt_open"); + * + * for (i = 0; i < n && !feof(f); i++) { + * // skip lines output last run + * fgets(buf, sizeof(buf), f); + * if (ferror(f)) err(1, "fgets"); + * } + * + * for (;;) { + * fgets(buf, sizeof(buf), f); + * if (ferror(f)) err(1, "fgets"); + * if (feof(f)) break; + * + * fputs(buf, stdout); + * + * ret = szcnt_inc(h); + * if (ret == -1) err(1, "szcnt_inc"); + * } + * + * szcnt_free(h); + * return 0; + * } + * + * License: APACHE-2 + * Author: Dan Good + */ +int main(int argc, char *argv[]) +{ + /* Expect exactly one argument */ + if (argc != 2) + return 1; + + if (strcmp(argv[1], "depends") == 0) { + /* none */ + return 0; + } + + return 1; +} diff --git a/ccan/filesize_counter/filesize_counter.h b/ccan/filesize_counter/filesize_counter.h new file mode 100644 index 000000000..108d12ee8 --- /dev/null +++ b/ccan/filesize_counter/filesize_counter.h @@ -0,0 +1,87 @@ +#ifndef _CNT_H +#define _CNT_H + +#include + +/** + * struct szcnt - counter handle + * @nm: counter file name + * @tmp: temporary file name, + * used when remaking counter as sparse file + * @fd: file descriptor + * @cnt: the count + */ +struct szcnt { + char *nm, *tmp; + int fd; + off_t cnt; +}; + +/** + * szcnt_new() - calloc a new handle + * + * init fd to -1 + * + * Return: ptr to new struct szcnt + */ +struct szcnt *szcnt_new(void); + +/** + * szcnt_free() - free a handle made with szcnt_new() + * @h: handle + * + * returns result of szcnt_close() + */ +int szcnt_free(struct szcnt *h); + +/** + * szcnt_open() - open a counter file on disk + * @h: handle + * @path: file to open + * + * opens and initializes h->cnt with file size; + * mallocs to hold file names + * + * path, if it exists, must be a regular file. + * + * Return: -1 on error, h->cnt on success + */ +off_t szcnt_open(struct szcnt *h, const char *path); + +/** + * szcnt_close() - close file and free file names + * @h: handle + * + * handle is "zeroed" and can be reused with szcnt_open() + * + * Return: -1 if close(2) error, 0 otherwise + */ +int szcnt_close(struct szcnt *); + +/** + * szcnt_zero() - set counter and file to zero + * @h: handle + * + * Return: -1 on error, 0 on success + */ +int szcnt_zero(struct szcnt *); + +/** + * szcnt_sync() - initialize h->cnt with file size + * @h: handle + * + * Return: -1 on error, h->cnt on success + */ +off_t szcnt_sync(struct szcnt *); + +/** + * szcnt_inc() - increment counter + * @h: handle + * + * increment counter and make file sparse on 4 KiB boundaries + * + * Return: -1 on error, 0 on success + */ +int szcnt_inc(struct szcnt *); + +#endif diff --git a/ccan/filesize_counter/szcnt.c b/ccan/filesize_counter/szcnt.c new file mode 100644 index 000000000..1fbc137c6 --- /dev/null +++ b/ccan/filesize_counter/szcnt.c @@ -0,0 +1,167 @@ +#include "filesize_counter.h" +#include +#include +#include +#include +#include +#include +#include + +#define _XOPEN_SOURCE 700 +#include + +struct szcnt *szcnt_new(void) +{ + struct szcnt *h = calloc(1, sizeof(struct szcnt)); + if (h) h->fd = -1; + return h; +} + +int szcnt_free(struct szcnt *h) +{ + int ret; + + assert(h); + ret = szcnt_close(h); + free(h); + return ret; +} + +static int isreg(const char *path) +{ + struct stat s; + int ret = stat(path, &s); + + if (ret == -1 && errno == ENOENT) + return 0; + if (ret == 0 && S_ISREG(s.st_mode)) + return 0; + if (ret == 0) + errno = EINVAL; + return -1; +} + +off_t szcnt_open(struct szcnt *h, const char *path) +{ + int len, i, ret; + + assert(h && path); + + // store original filename and temp filename in one buffer + // temp name is original with a dot preceding basename + len = strlen(path) + 1; + assert(len > 1); + if ((h->nm = malloc(len * 2 + 1)) == NULL) + goto err; + h->tmp = h->nm + len; + + for (i = len; i > 0; i--) + if (path[i] == '/') { + i++; + break; + } + // orig null dirname dot basename + sprintf(h->nm, "%s%c%.*s%c%s", path, '\0', i, path, '.', path + i); + + if (isreg(h->nm) == -1) + goto err; + if (isreg(h->tmp) == -1) + goto err; + + if ((h->fd = open(h->nm, O_WRONLY|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR)) == -1) + goto err; + + if ((ret = szcnt_sync(h)) == -1) + goto err; + + return ret; + +err: + szcnt_close(h); + return -1; +} + +int szcnt_close(struct szcnt *h) +{ + assert(h); + + if (h->fd != -1 && close(h->fd) == -1) + return -1; + h->fd = -1; + + if (h->nm) + free(h->nm); + h->nm = NULL; + h->tmp = NULL; + + h->cnt = 0; + + return 0; +} + +static int szcnt_swap(struct szcnt *h) +{ + int fd; + + assert(h); + + if ((fd = open(h->tmp, O_WRONLY|O_CREAT|O_APPEND|O_TRUNC, S_IRUSR|S_IWUSR)) == -1) + return -1; + + if (ftruncate(fd, h->cnt) == -1) + return -1; + + if (rename(h->tmp, h->nm) == -1) + return -1; + + if (close(h->fd) == -1) + return -1; + + h->fd = fd; + + return 0; +} + +off_t szcnt_sync(struct szcnt *h) +{ + off_t ret; + + assert(h); + + if ((ret = lseek(h->fd, 0, SEEK_END)) == -1) + return -1; + + return h->cnt = ret; +} + +int szcnt_zero(struct szcnt *h) +{ + assert(h); + + if (ftruncate(h->fd, 0) == -1) + return -1; + + return h->cnt = 0; +} + +int szcnt_inc(struct szcnt *h) { + + assert(h); + + if (h->fd == -1) { + errno = EBADF; + return -1; + } + + if (h->cnt + 1 < 0 && szcnt_zero(h) == -1) + return -1; + + if (h->cnt > 0 && h->cnt % 4096 == 0 && szcnt_swap(h) == -1) + return -1; + + if (write(h->fd, "", 1) == -1) + return -1; + else h->cnt++; + + return 0; +} diff --git a/ccan/filesize_counter/test/run.c b/ccan/filesize_counter/test/run.c new file mode 100644 index 000000000..e565e90c1 --- /dev/null +++ b/ccan/filesize_counter/test/run.c @@ -0,0 +1,121 @@ +#include +/* Include the C files directly. */ +#include +#include +#include +#include + +int main(void) +{ + int i; + struct szcnt *h; + struct stat s; + + /* This is how many tests you plan to run */ + plan_tests(35); + + ok1(h = szcnt_new()); + if (!h) exit(1); + + ok1(!h->nm && !h->tmp && h->fd == -1 && !h->cnt); + + if (unlink("t") == -1 && errno != ENOENT) err(1, "unlink"); + if (unlink(".t") == -1 && errno != ENOENT) err(1, "unlink"); + + ok1(szcnt_open(h, "t") == 0); + ok1(h->cnt == 0 && strcmp(h->nm, "t") == 0 && strcmp(h->tmp, ".t") == 0); + int save = h->fd; + + if (stat("t", &s) == -1) err(1, "stat"); + ok1(s.st_size == 0 && s.st_blocks == 0); + + ok1(szcnt_inc(h) == 0); + + if (stat("t", &s) == -1) err(1, "stat"); + ok1(s.st_size == 1 && s.st_blocks == 8); + + for (i = 0; i < 4095 && h->fd == save; i++) + szcnt_inc(h); + + ok1(h->fd == save); + ok1(szcnt_inc(h) == 0); + ok1(h->fd != save); + + if (stat("t", &s) == -1) err(1, "stat"); + ok1(s.st_size == 4097 && s.st_blocks == 8); + + for (i = 0; i < 4096; i++) + szcnt_inc(h); + + if (stat("t", &s) == -1) err(1, "stat"); + ok1(s.st_size == 8193 && s.st_blocks == 8); + + ok1(szcnt_close(h) == 0); + ok1(!h->nm && !h->tmp && h->fd == -1 && !h->cnt); + ok1(szcnt_open(h, "t") == 8193 && h->cnt == 8193); + + ok1(szcnt_zero(h) == 0 && h->cnt == 0); + if (stat("t", &s) == -1) err(1, "stat"); + ok1(s.st_size == 0 && s.st_blocks == 0); + + if (unlink("foo/bar/baz") == -1 && errno != ENOENT) err(1, "unlink"); + if (unlink("foo/bar/.baz") == -1 && errno != ENOENT) err(1, "unlink"); + if (rmdir("foo/bar/.oof") == -1 && errno != ENOENT) err(1, "rmdir"); + if (rmdir("foo/bar") == -1 && errno != ENOENT) err(1, "rmdir"); + if (rmdir("foo") == -1 && errno != ENOENT) err(1, "rmdir"); + if (mkdir("foo", 0777) == -1) err(1, "mkdir"); + if (mkdir("foo/bar", 0777) == -1) err(1, "mkdir"); + if (mkdir("foo/bar/.oof", 0777) == -1) err(1, "mkdir"); + + ok1(szcnt_open(h, "foo/bar/baz") == 0); + ok1(h->cnt == 0 && strcmp(h->nm, "foo/bar/baz") == 0 && strcmp(h->tmp, "foo/bar/.baz") == 0); + ok1(szcnt_close(h) == 0); + + ok1(szcnt_open(h, "foo") == -1 && errno == EINVAL); + ok1(!h->nm && !h->tmp && h->fd == -1 && !h->cnt); + + ok1(szcnt_open(h, "foo/bar/oof") == -1 && errno == EINVAL); + ok1(!h->nm && !h->tmp && h->fd == -1 && !h->cnt); + + if (chmod("t", 0) == -1) err(1, "chmod"); + ok1(szcnt_open(h, "t") == -1 && errno == EACCES); + ok1(!h->nm && !h->tmp && h->fd == -1 && !h->cnt); + + if (chmod("t", 0777) == -1) err(1, "chmod"); + ok1(szcnt_open(h, "t") == 0); + + int fake; + if ((fake = open("t", O_RDONLY)) == -1) err(1, "open"); + save = h->fd; + h->fd = fake; + + ok1(szcnt_inc(h) == -1 && errno == EBADF); + ok1(szcnt_zero(h) == -1 && errno == EINVAL); + char *save_tmp = h->tmp; + h->tmp = (char *) "."; + h->cnt = 4096; + ok1(szcnt_inc(h) == -1 && errno == EISDIR); + h->tmp = save_tmp; + h->cnt = sizeof(off_t) == sizeof(int64_t) ? INT64_MAX : INT32_MAX; + ok1(szcnt_inc(h) == -1 && errno == EINVAL); + h->cnt = 0; + + if (unlink(".t") == -1 && errno != ENOENT) err(1, "unlink"); + if (unlink(".u") == -1 && errno != ENOENT) err(1, "unlink"); + if (symlink(".t", ".u") == -1) err(1, "symlink"); + if (symlink(".u", ".t") == -1) err(1, "symlink"); + ok1(szcnt_swap(h) == -1 && errno == ELOOP); + + close(fake); + ok1(szcnt_close(h) == -1 && errno == EBADF); + + h->fd = save; + ok1(szcnt_free(h) == 0); + + h = szcnt_new(); + ok1(szcnt_inc(h) == -1 && errno == EBADF); + szcnt_close(h); + + /* This exits depending on whether all tests passed */ + return exit_status(); +} diff --git a/ccan/filesize_counter/test/run2.c b/ccan/filesize_counter/test/run2.c new file mode 100644 index 000000000..57fe01d7d --- /dev/null +++ b/ccan/filesize_counter/test/run2.c @@ -0,0 +1,60 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define _XOPEN_SOURCE 700 +#include + +#include + +int fail_calloc = 1, fail_malloc = 1, fail_ftruncate = 1, fail_rename = 1, fail_close = 0, fail_lseek = 1; +#define calloc(x, y) (fail_calloc ? NULL : calloc(x, y)) +#define malloc(x) (fail_malloc ? NULL : malloc(x)) +#define ftruncate(x, y) (fail_ftruncate ? -1 : ftruncate(x, y)) +#define rename(x, y) (fail_rename ? -1 : rename(x, y)) +#define close(x) (fail_close ? -1 : close(x)) +#define lseek(x, y, z) (fail_lseek ? -1 : lseek(x, y, z)) + +#include +#include + +int main(int argc, char *argv[]) +{ + struct szcnt *h; + + plan_no_plan(); + + ok1(szcnt_new() == NULL); + fail_calloc = 0; + ok1(h = szcnt_new()); + + if (unlink("t") == -1 && errno != ENOENT) err(1, "unlink"); + if (unlink(".t") == -1 && errno != ENOENT) err(1, "unlink"); + + ok1(szcnt_open(h, "t") == -1); + fail_malloc = 0; + ok1(szcnt_open(h, "t") == -1); + fail_lseek = 0; + ok1(szcnt_open(h, "t") == 0); + + fail_close = 1; + ok1(szcnt_close(h) == -1); + + h->cnt = 4096; + ok1(szcnt_inc(h) == -1); + fail_ftruncate = 0; + ok1(szcnt_inc(h) == -1); + fail_rename = 0; + ok1(szcnt_inc(h) == -1); + fail_close = 0; + + ok1(szcnt_inc(h) == 0); + + szcnt_free(h); + return exit_status(); +}