Welcome to little lamb

Code » limb » commit 7ab8f5e

Add headers cdb.h & cdbmake.h with cdbmaker_sa_*()..

author Olivier Brunel
2023-04-09 14:03:41 UTC
committer Olivier Brunel
2023-05-20 18:06:36 UTC
parent 4ba308eb2874426a00f9dc263fb545dda158b5dd

Add headers cdb.h & cdbmake.h with cdbmaker_sa_*()..

..family of functions, mich luke their cdbmake_*() counterparts (from
skalibs) only the resulting cdb file is stored in memory.

Also add cdb_init_frommem() to use a cdb from a memory area instead of a
file descriptor.

src/doc/cdb.h.0.md +28 -0
src/doc/cdb.h/cdb_init_frommem.3.md +27 -0
src/doc/cdbmake.h.0.md +52 -0
src/doc/cdbmake.h/cdbmaker_sa_start.3.md +68 -0
src/liblimb/cdb.h/cdb_init_frommem.c +11 -0
src/liblimb/cdbmake.h/cdbmaker_sa_add.c +45 -0
src/liblimb/cdbmake.h/cdbmaker_sa_finish.c +74 -0
src/liblimb/cdbmake.h/cdbmaker_sa_free.c +13 -0
src/liblimb/cdbmake.h/cdbmaker_sa_start.c +15 -0
src/liblimb/include/limb/cdb.h +12 -0
src/liblimb/include/limb/cdbmake.h +24 -0

diff --git a/src/doc/cdb.h.0.md b/src/doc/cdb.h.0.md
new file mode 100644
index 0000000..b6e7274
--- /dev/null
+++ b/src/doc/cdb.h.0.md
@@ -0,0 +1,28 @@
+% limb manual
+% cdb.h(0)
+
+# NAME
+
+cdb.h - accessing constant databases
+
+
+# SYNOPSIS
+
+    #include <limb/cdb.h>
+
+
+# DESCRIPTION
+
+The header defines the needed functions to access constant databases, known as
+.cdb files.
+
+! INFO: skalibs
+! This header is a complement to skalibs' own [skalibs/cdb.h](0) and thusly
+! includes said header.
+
+## Functions
+
+The following functions are defined :
+
+: [cdb_init_frommem](3)
+:: To initializes a CDB from a memory area, which must contain a valid CDB file.
diff --git a/src/doc/cdb.h/cdb_init_frommem.3.md b/src/doc/cdb.h/cdb_init_frommem.3.md
new file mode 100644
index 0000000..6000e7c
--- /dev/null
+++ b/src/doc/cdb.h/cdb_init_frommem.3.md
@@ -0,0 +1,27 @@
+% limb manual
+% cdb_init_frommem(3)
+
+# NAME
+
+cdb\_init\_frommem - initialize a cdb from memory area
+
+# SYNOPSIS
+
+    #include <limb/cdb.h>
+
+```pre hl
+void cdb_init_frommem(cdb *<em>c</em>, const char *<em>mem</em>, u32 <em>len</em>)
+```
+
+# DESCRIPTION
+
+The `cdb_init_frommem`() function initializes the cdb pointed to by `c`, which
+must be `CDB_ZERO` before the call, to use the memory pointed by `mem` which
+must hold the content of a valid cdb file, for `len` bytes.
+
+! WARNING:
+! A cdb initialized via this function is just as valid as with one of the
+! functions using a file as backend. However, you should *not* call
+! [cdb_free](3) with a cdb initialized from `cdb_init_frommem`().
+! Instead, it is up to the caller to free the memory pointed to by `mem`
+! however it needs to, once the cdb isn't needed anymore.
diff --git a/src/doc/cdbmake.h.0.md b/src/doc/cdbmake.h.0.md
new file mode 100644
index 0000000..271e455
--- /dev/null
+++ b/src/doc/cdbmake.h.0.md
@@ -0,0 +1,52 @@
+
+% limb manual
+% cdbmake.h(0)
+
+# NAME
+
+cdbmake.h - creating constant databases
+
+
+# SYNOPSIS
+
+    #include <limb/cdbmake.h>
+
+
+# DESCRIPTION
+
+The header defines the needed functions to create constant databases, known as
+.cdb files.
+
+! INFO: skalibs
+! This header is a complement to skalibs' own [skalibs/cdbmake.h](0) and thusly
+! includes said header.
+
+## Structures
+
+The following structures are defined :
+
+: *struct cdbmaker_sa*
+:: Semi-opaque structure used during the creating of a cdb in-memory.
+
+## Types
+
+The followng types are defined :
+
+: *cdbmaker_sa*
+:: Semi-opaque structure used durng the creating of a cdb in-memory.
+
+## Functions
+
+The following functions are defined :
+
+: [cdbmaker_sa_add](3)
+:: Similar to [cdbmake_add](3) but the cdb will be created in-memory.
+
+: [cdbmaker_sa_finish](3)
+:: Similar to [cdbmake_finish](3) but the cdb will be created in-memory.
+
+: [cdbmaker_sa_free](3)
+:: To free memory associated with a semi-opaque *cdbmaker_sa* when done.
+
+: [cdbmaker_sa_start](3)
+:: Similar to [cdbmake_start](3) but the cdb will be created in-memory.
diff --git a/src/doc/cdbmake.h/cdbmaker_sa_start.3.md b/src/doc/cdbmake.h/cdbmaker_sa_start.3.md
new file mode 100644
index 0000000..e232bb9
--- /dev/null
+++ b/src/doc/cdbmake.h/cdbmaker_sa_start.3.md
@@ -0,0 +1,68 @@
+% limb manual
+% cdbmaker_sa_start(3)
+
+# NAME
+
+cdbmaker\_sa\_start, cdbmaker\_sa\_add, cdbmaker\_sa\_finish,
+cdbmaker\_sa\_free - create a cdb in memory
+
+# SYNOPSIS
+
+    #include <limb/cdbmake.h>
+
+```pre hl
+int cdbmaker_sa_finish(cdbmaker_sa *<em>mkr</em>)
+int cdbmaker_sa_add(cdbmaker_sa *<em>mkr</em>, const char *<em>key</em>, u32 <em>klen</em>, const char *<em>data</em>, u32 <em>dlen</em>)
+int cdbmaker_sa_start(cdbmaker_sa *<em>mkr</em>)
+void cdbmaker_sa_free(cdbmaker_sa *<em>mkr</em>)
+```
+
+# DESCRIPTION
+
+The `cdbmaker_sa_start`(), `cdbmaker_sa_add`() and `cdbmaker_sa_finish`()
+functions are similar to [cdbmake_start](3), [cdbmake_add](3) and
+[cdbmake_finish](3) respectively, only the resulting cdb will not be written to
+a file but instead be available in memory.
+
+The `cdbmaker_sa_start`() initializes the semi-opaque structure pointed to by
+`mkr` in order to begin creation of a cdb.
+
+The `cdbmaker_sa_add`() function is to be called for each element to be added to
+the cdb, specifying the key pointed to by `key` of length `klen` bytes, and the
+data pointed to be `data` of length `dlen` bytes. The actual content of both key
+and data will be of course be copied into the cdb.
+
+Elements will be added into the cdb in the order of the calls made to
+`cdbmaker_sa_add`().
+
+Finally, the `cdbmaker_sa_finish`() function must be called after all elements
+have been added via `cdbmaker_sa_add`(), in order for the cdb to be actually
+created.
+
+Specifically, the *cdbmaker_sa* structure pointed to by `mkr` has a member `sa`
+which is a *stralloc*, which - after a successful call to
+`cdbmaker_sa_finish`() - will hold the newly-created cdb (i.e. the content of a
+cdb file), available from `mkr->sa.s` for a length of `mkr->sa.len` bytes.
+
+When done with the cdb, the `cdbmaker_sa_free`() function must be called in
+order to free memory associated with the semi-opaque structure pointed to by
+`mkr`.
+
+# RETURN VALUE
+
+The `cdbmaker_sa_start`(), `cdbmaker_sa_add`() and `cdbmaker_sa_finish`()
+functions return 1 on success. Otherwise, they return 0 and set `errno` to
+indicate the error.
+
+# ERRORS
+
+The `cdbmaker_sa_start`(), `cdbmaker_sa_add`() and `cdbmaker_sa_finish`() may
+fail if :
+
+: *ENOMEM*
+:: Not enough memory to create to cdb.
+
+The `cdbmaker_sa_add`() function may also fail if :
+
+: *EOVERFLOW*
+:: The resulting cdb would overflow, i.e. go over its 4 GiB limit.
diff --git a/src/liblimb/cdb.h/cdb_init_frommem.c b/src/liblimb/cdb.h/cdb_init_frommem.c
new file mode 100644
index 0000000..63b31c3
--- /dev/null
+++ b/src/liblimb/cdb.h/cdb_init_frommem.c
@@ -0,0 +1,11 @@
+/* This file is part of limb                           https://lila.oss/limb
+ * Copyright (C) 2023 Olivier Brunel                          jjk@jjacky.com */
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include <limb/cdb.h>
+
+void
+cdb_init_frommem(cdb *c, const char *mem, u32 len)
+{
+    c->map = mem;
+    c->size = len;
+}
diff --git a/src/liblimb/cdbmake.h/cdbmaker_sa_add.c b/src/liblimb/cdbmake.h/cdbmaker_sa_add.c
new file mode 100644
index 0000000..354e53e
--- /dev/null
+++ b/src/liblimb/cdbmake.h/cdbmaker_sa_add.c
@@ -0,0 +1,45 @@
+/* This file is part of limb                           https://lila.oss/limb
+ * Copyright (C) 2023 Olivier Brunel                          jjk@jjacky.com */
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include <errno.h>
+#include <limb/cdbmake.h>
+#include <limb/u32.h>
+
+#include <skalibs/gccattributes.h>
+extern uint32_t cdb_hash (char const *, uint32_t) gccattr_pure ;
+
+#define diu32   diuint32
+
+#define cms_add(m,d,l)      stralloc_catb(&m->sa, d, l)
+
+static ssize_t
+cms_addbegin(cdbmaker_sa *m, u32 klen, u32 dlen)
+{
+    if (!stralloc_readyplus(&m->sa, 8))
+        return 0;
+    u32_pack(m->sa.s + m->sa.len, klen);
+    m->sa.len += 4;
+    u32_pack(m->sa.s + m->sa.len, dlen);
+    m->sa.len += 4;
+    return 1;
+}
+
+int
+cdbmaker_sa_add(cdbmaker_sa *m, const char *key, u32 klen, const char *data, u32 dlen)
+{
+    int datadone = 0;
+    diu32 b = { .left = cdb_hash(key, klen), .right = m->sa.len };
+    if (!cms_addbegin(m, klen, dlen)
+            || !cms_add(m, key, klen)
+            || !cms_add(m, data, dlen)
+            || !(datadone = 1)
+            || !genalloc_append(diu32, &m->hplist, &b)) {
+        m->sa.len = b.right;
+        if (datadone)
+            m->hplist.len -= sizeof(diu32);
+        return 0;
+    } else if (m->sa.len > UINT32_MAX) {
+        return (errno = EOVERFLOW, 0);
+    }
+    return 1;
+}
diff --git a/src/liblimb/cdbmake.h/cdbmaker_sa_finish.c b/src/liblimb/cdbmake.h/cdbmaker_sa_finish.c
new file mode 100644
index 0000000..84ca8a1
--- /dev/null
+++ b/src/liblimb/cdbmake.h/cdbmaker_sa_finish.c
@@ -0,0 +1,74 @@
+/* This file is part of limb                           https://lila.oss/limb
+ * Copyright (C) 2023 Olivier Brunel                          jjk@jjacky.com */
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include <errno.h>
+#include <limb/cdbmake.h>
+#include <limb/u32.h>
+
+#define diu32   diuint32
+
+int
+cdbmaker_sa_finish(cdbmaker_sa *m)
+{
+    u32 count[256] = { 0 };
+    u32 start[256];
+    unsigned int size = 1;
+    unsigned int n = genalloc_len(diu32, &m->hplist);
+    unsigned int i;
+    diu32 *hp = genalloc_s(diu32, &m->hplist);
+
+    for (i = 0; i < n; ++i)
+        ++count[hp[i].left & 255];
+
+    u32 u = 0;
+    for (i = 0; i < 256; ++i)
+        start[i] = u += count[i];
+    for (i = 0; i < 256; ++i) {
+        u = count[i] << 1;
+        if (u > size)
+            size = u;
+    }
+
+    size += n;
+    u = 0xffffffffUL;
+    u /= sizeof(diu32);
+    if (size > u)
+        return (errno = ENOMEM, 0);
+
+    diu32 split[size];
+    i = n;
+    while (i--)
+        split[--start[hp[i].left & 255]] = hp[i];
+    genalloc_free(diu32, &m->hplist);
+    hp = split + n;
+
+    for (i = 0; i < 256; ++i) {
+        u32 k = count[i];
+        u32 len = k << 1;
+        diu32 *p = split + start[i];
+
+        u32_pack(m->sa.s + (i << 3), m->sa.len);
+        u32_pack(m->sa.s + (i << 3) + 4, len);
+
+        for (u32 j = 0; j < len; ++j)
+            hp[j].left = hp[j].right = 0;
+        for (u32 j = 0; j < k; ++j) {
+            u32 where = (p->left >> 8) % len;
+            while (hp[where].right)
+                if (++where == len)
+                    where = 0;
+            hp[where] = *p++;
+        }
+
+        for (u32 j = 0; j < len; ++j) {
+            if (!stralloc_readyplus(&m->sa, 8))
+                return 0;
+            u32_pack(m->sa.s + m->sa.len, hp[j].left);
+            m->sa.len += 4;
+            u32_pack(m->sa.s + m->sa.len, hp[j].right);
+            m->sa.len += 4;
+        }
+    }
+
+    return 1;
+}
diff --git a/src/liblimb/cdbmake.h/cdbmaker_sa_free.c b/src/liblimb/cdbmake.h/cdbmaker_sa_free.c
new file mode 100644
index 0000000..4b278ec
--- /dev/null
+++ b/src/liblimb/cdbmake.h/cdbmaker_sa_free.c
@@ -0,0 +1,13 @@
+/* This file is part of limb                           https://lila.oss/limb
+ * Copyright (C) 2023 Olivier Brunel                          jjk@jjacky.com */
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include <limb/cdbmake.h>
+
+#define diu32   diuint32
+
+void
+cdbmaker_sa_free(cdbmaker_sa *m)
+{
+    stralloc_free(&m->sa);
+    genalloc_free(diu32, &m->hplist);
+}
diff --git a/src/liblimb/cdbmake.h/cdbmaker_sa_start.c b/src/liblimb/cdbmake.h/cdbmaker_sa_start.c
new file mode 100644
index 0000000..15e36fb
--- /dev/null
+++ b/src/liblimb/cdbmake.h/cdbmaker_sa_start.c
@@ -0,0 +1,15 @@
+/* This file is part of limb                           https://lila.oss/limb
+ * Copyright (C) 2023 Olivier Brunel                          jjk@jjacky.com */
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include <limb/cdbmake.h>
+
+int
+cdbmaker_sa_start(cdbmaker_sa *m)
+{
+    m->sa = stralloc_zero;
+    m->hplist = genalloc_zero;
+    if (!stralloc_readyplus(&m->sa, 2048))
+        return 0;
+    m->sa.len = 2048;
+    return 1;
+}
diff --git a/src/liblimb/include/limb/cdb.h b/src/liblimb/include/limb/cdb.h
new file mode 100644
index 0000000..5ced9b6
--- /dev/null
+++ b/src/liblimb/include/limb/cdb.h
@@ -0,0 +1,12 @@
+/* This file is part of limb                           https://lila.oss/limb
+ * Copyright (C) 2023 Olivier Brunel                          jjk@jjacky.com */
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef LIMB_CDB_H
+#define LIMB_CDB_H
+
+#include <skalibs/cdb.h>
+#include <limb/int.h>
+
+extern void cdb_init_frommem(cdb *c, const char *mem, u32 len);
+
+#endif /* LIMB_CDB_H */
diff --git a/src/liblimb/include/limb/cdbmake.h b/src/liblimb/include/limb/cdbmake.h
new file mode 100644
index 0000000..2447cb8
--- /dev/null
+++ b/src/liblimb/include/limb/cdbmake.h
@@ -0,0 +1,24 @@
+/* This file is part of limb                           https://lila.oss/limb
+ * Copyright (C) 2023 Olivier Brunel                          jjk@jjacky.com */
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef LIMB_CDBMAKE_H
+#define LIMB_CDBMAKE_H
+
+#include <skalibs/cdbmake.h>
+#include <skalibs/stralloc.h>
+#include <skalibs/genalloc.h>
+#include <limb/int.h>
+
+struct cdbmaker_sa
+{
+    stralloc sa;
+    genalloc hplist;
+};
+typedef struct cdbmaker_sa cdbmaker_sa;
+
+extern int cdbmaker_sa_finish(cdbmaker_sa *m);
+extern int cdbmaker_sa_add(cdbmaker_sa *m, const char *key, u32 klen, const char *data, u32 dlen);
+extern int cdbmaker_sa_start(cdbmaker_sa *m);
+extern void cdbmaker_sa_free(cdbmaker_sa *m);
+
+#endif /* LIMB_CDBMAKE_H */