Add cnt module - durable counters using sparse files

.gitignore

@@ -20,3 +20,4 @@ tools/_infotojson/infotojson
 tools/ccanlint/test/run-file_analysis
 tools/configurator/configurator
 scores/
+.glimpse_*

Makefile-ccan

@@ -61,6 +61,7 @@ MODS_WITH_SRC := aga \
 	eratosthenes \
 	err \
 	failtest \
+	filesize_counter \
 	foreach \
 	grab_file \
 	hash \

ccan/filesize_counter/LICENSE

@@ -0,0 +1,9 @@
+Software provided by Dan Good at Dell SecureWorks. For more information on Dell SecureWorks security services please browse to http://www.secureworks.com
+
+Copyright 2015 Dell SecureWorks
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

ccan/filesize_counter/_info

@@ -0,0 +1,103 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * filesize_counter - durable counters using sparse files
+ *
+ * The filesize_counter (szcnt) module keeps count by incrementing the size of
+ * a file.  These counters are easy to read using lseek(2), stat(2), or ls(1).
+ * To keep the files small, they are replaced with sparse files at multiples
+ * of 4 KiB.
+ *
+ * The range of the counter is that of off_t.  Compile with _FILE_OFFSET_BITS
+ * set to 64 to count beyond 2 GiB.
+ *
+ * Tests suggest this approach performs on par with writing a raw uint every
+ * time and is 4 to 5 times faster than using ftruncate only.
+ *
+ * This is a novel approach, so why do it this way?  These counters are simple
+ * and unambiguous.  Picture a large organization, where disparate groups are
+ * responsible for quality assurance, operations, and health monitoring of
+ * production systems.  Add to that documentation inconsistencies, the need
+ * to maintain multiple software versions, and a regular influx of new staff,
+ * or staff turnover.
+ *
+ * Consider a counter implemented as a number stored in a file.  If an
+ * operator cats the file and sees 123, is that a decimal number or a hex
+ * number?  If the output appears binary, is it encoded?  Is it a raw integer?
+ * Signed or unsigned?  Big or little-endian?  If the file is empty, is that
+ * the same as zero, or is that a bug?
+ *
+ * File size, by contrast, is readily understood, and a file always has a
+ * non-negative size.
+ *
+ * A counter implementation can be abstracted behind an API or tool.  This way
+ * may increase the burdens of documentation, deployment, and testing, but
+ * probably worse is the additional dependency to be tracked and factored
+ * into release planning.
+ *
+ * File size, by contrast, is accessible through standard system calls
+ * and tools.  The intrepid operator can even set the counter at need with
+ * truncate(1), dd(1), etc.
+ *
+ * Example:
+ *	#include <err.h>
+ *	#include <stdio.h>
+ *	#include <ccan/filesize_counter/filesize_counter.h>
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		FILE *f;
+ *		struct szcnt *h;
+ *		off_t n, i;
+ *		char buf[BUFSIZ];
+ *		int ret;
+ *
+ *		if (argc != 2) return 1;
+ *		f = fopen(argv[1], "r");
+ *		if (!f) err(1, "fopen");
+ *
+ *		h = szcnt_new();
+ *		if (!h) err(1, "szcnt_new");
+ *
+ *		n = szcnt_open(h, "counter");
+ *		if (n == -1) err(1, "szcnt_open");
+ *
+ *		for (i = 0; i < n && !feof(f); i++) {
+ *			// skip lines output last run
+ *			fgets(buf, sizeof(buf), f);
+ *			if (ferror(f)) err(1, "fgets");
+ *		}
+ *
+ *		for (;;) {
+ *			fgets(buf, sizeof(buf), f);
+ *			if (ferror(f)) err(1, "fgets");
+ *			if (feof(f)) break;
+ *
+ *			fputs(buf, stdout);
+ *
+ *			ret = szcnt_inc(h);
+ *			if (ret == -1) err(1, "szcnt_inc");
+ *		}
+ *
+ *		szcnt_free(h);
+ *		return 0;
+ *	}
+ *
+ * License: APACHE-2
+ * Author: Dan Good <[email protected]>
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		/* none */
+		return 0;
+	}
+
+	return 1;
+}

ccan/filesize_counter/filesize_counter.h

@@ -0,0 +1,87 @@
+#ifndef _CNT_H
+#define _CNT_H
+
+#include <sys/types.h>
+
+/**
+ * struct szcnt - counter handle
+ * @nm: counter file name
+ * @tmp: temporary file name,
+ *       used when remaking counter as sparse file
+ * @fd: file descriptor
+ * @cnt: the count
+ */
+struct szcnt {
+	char *nm, *tmp;
+	int fd;
+	off_t cnt;
+};
+
+/**
+ * szcnt_new() - calloc a new handle
+ *
+ * init fd to -1
+ *
+ * Return: ptr to new struct szcnt
+ */
+struct szcnt *szcnt_new(void);
+
+/**
+ * szcnt_free() - free a handle made with szcnt_new()
+ * @h: handle
+ *
+ * returns result of szcnt_close()
+ */
+int szcnt_free(struct szcnt *h);
+
+/**
+ * szcnt_open() - open a counter file on disk
+ * @h: handle
+ * @path: file to open
+ *
+ * opens and initializes h->cnt with file size;
+ * mallocs to hold file names
+ *
+ * path, if it exists, must be a regular file.
+ *
+ * Return: -1 on error, h->cnt on success
+ */
+off_t szcnt_open(struct szcnt *h, const char *path);
+
+/**
+ * szcnt_close() - close file and free file names
+ * @h: handle
+ *
+ * handle is "zeroed" and can be reused with szcnt_open()
+ *
+ * Return: -1 if close(2) error, 0 otherwise
+ */
+int szcnt_close(struct szcnt *);
+
+/**
+ * szcnt_zero() - set counter and file to zero
+ * @h: handle
+ *
+ * Return: -1 on error, 0 on success
+ */
+int szcnt_zero(struct szcnt *);
+
+/**
+ * szcnt_sync() - initialize h->cnt with file size
+ * @h: handle
+ *
+ * Return: -1 on error, h->cnt on success
+ */
+off_t szcnt_sync(struct szcnt *);
+
+/**
+ * szcnt_inc() - increment counter
+ * @h: handle
+ *
+ * increment counter and make file sparse on 4 KiB boundaries
+ *
+ * Return: -1 on error, 0 on success
+ */
+int szcnt_inc(struct szcnt *);
+
+#endif

ccan/filesize_counter/szcnt.c

@@ -0,0 +1,167 @@
+#include "filesize_counter.h"
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#define _XOPEN_SOURCE 700
+#include <unistd.h>
+
+struct szcnt *szcnt_new(void)
+{
+	struct szcnt *h = calloc(1, sizeof(struct szcnt));
+	if (h) h->fd = -1;
+	return h;
+}
+
+int szcnt_free(struct szcnt *h)
+{
+	int ret;
+
+	assert(h);
+	ret = szcnt_close(h);
+	free(h);
+	return ret;
+}
+
+static int isreg(const char *path)
+{
+	struct stat s;
+	int ret = stat(path, &s);
+
+	if (ret == -1 && errno == ENOENT)
+		return 0;
+	if (ret == 0 && S_ISREG(s.st_mode))
+		return 0;
+	if (ret == 0)
+		errno = EINVAL;
+	return -1;
+}
+
+off_t szcnt_open(struct szcnt *h, const char *path)
+{
+	int len, i, ret;
+
+	assert(h && path);
+
+	// store original filename and temp filename in one buffer
+	// temp name is original with a dot preceding basename
+	len = strlen(path) + 1;
+	assert(len > 1);
+	if ((h->nm = malloc(len * 2 + 1)) == NULL)
+		goto err;
+	h->tmp = h->nm + len;
+
+	for (i = len; i > 0; i--)
+		if (path[i] == '/') {
+			i++;
+			break;
+		}
+	//                             orig  null  dirname  dot  basename
+	sprintf(h->nm, "%s%c%.*s%c%s", path, '\0', i, path, '.', path + i);
+
+	if (isreg(h->nm) == -1)
+		goto err;
+	if (isreg(h->tmp) == -1)
+		goto err;
+
+	if ((h->fd = open(h->nm, O_WRONLY|O_CREAT|O_APPEND, S_IRUSR|S_IWUSR)) == -1)
+		goto err;
+
+	if ((ret = szcnt_sync(h)) == -1)
+		goto err;
+
+	return ret;
+
+err:
+	szcnt_close(h);
+	return -1;
+}
+
+int szcnt_close(struct szcnt *h)
+{
+	assert(h);
+
+	if (h->fd != -1 && close(h->fd) == -1)
+		return -1;
+	h->fd = -1;
+
+	if (h->nm)
+		free(h->nm);
+	h->nm  = NULL;
+	h->tmp = NULL;
+
+	h->cnt = 0;
+
+	return 0;
+}
+
+static int szcnt_swap(struct szcnt *h)
+{
+	int fd;
+
+	assert(h);
+
+	if ((fd = open(h->tmp, O_WRONLY|O_CREAT|O_APPEND|O_TRUNC, S_IRUSR|S_IWUSR)) == -1)
+		return -1;
+
+	if (ftruncate(fd, h->cnt) == -1)
+		return -1;
+
+	if (rename(h->tmp, h->nm) == -1)
+		return -1;
+
+	if (close(h->fd) == -1)
+		return -1;
+
+	h->fd = fd;
+
+	return 0;
+}
+
+off_t szcnt_sync(struct szcnt *h)
+{
+	off_t ret;
+
+	assert(h);
+
+	if ((ret = lseek(h->fd, 0, SEEK_END)) == -1)
+		return -1;
+
+	return h->cnt = ret;
+}
+
+int szcnt_zero(struct szcnt *h)
+{
+	assert(h);
+
+	if (ftruncate(h->fd, 0) == -1)
+		return -1;
+
+	return h->cnt = 0;
+}
+
+int szcnt_inc(struct szcnt *h) {
+
+	assert(h);
+
+	if (h->fd == -1) {
+		errno = EBADF;
+		return -1;
+	}
+
+	if (h->cnt + 1 < 0 && szcnt_zero(h) == -1)
+		return -1;
+
+	if (h->cnt > 0 && h->cnt % 4096 == 0 && szcnt_swap(h) == -1)
+		return -1;
+
+	if (write(h->fd, "", 1) == -1)
+		return -1;
+	else h->cnt++;
+
+	return 0;
+}