Implement serialization of HSET

README.md

@@ -38,7 +38,7 @@ cd build-opt && ninja dragonfly
 
 for more options, run `./dragonfly --help`
 
-## Milestone Egg 🥚
+## Milestone - Source Available
 
 API 1.0
 - [X] String family
@@ -128,27 +128,8 @@ API 1.0
   - [ ] RANDOMKEY
   - [ ] MOVE
 
-In addition, we want to support efficient expiry (TTL) and cache eviction algorithms.
-We should implement basic memory management support. For Master/Slave replication we should design
-a distributed log format.
-
-### Memchache API
-- [X] set
-- [X] get
-- [X] replace
-- [X] add
-- [X] stats (partial)
-- [x] append
-- [x] prepend
-- [x] delete
-- [x] flush_all
-- [x] incr
-- [x] decr
-- [x] version
-- [x] quit
-
 API 2.0
-- [ ] List Family
+- [X] List Family
   - [X] BLPOP
   - [X] BRPOP
   - [ ] BRPOPLPUSH
@@ -240,6 +221,26 @@ API 2.0
   - [ ] PFCOUNT
   - [ ] PFMERGE
 
+In addition, we want to support efficient expiry (TTL) and cache eviction algorithms.
+We should implement basic memory management support. For Master/Slave replication we should design
+a distributed log format.
+
+### Memchache API
+- [X] set
+- [X] get
+- [X] replace
+- [X] add
+- [X] stats (partial)
+- [x] append
+- [x] prepend
+- [x] delete
+- [x] flush_all
+- [x] incr
+- [x] decr
+- [x] version
+- [x] quit
+
+
 Commands that I prefer avoid implementing before launch:
   - PUNSUBSCRIBE
   - PSUBSCRIBE
@@ -252,22 +253,19 @@ Commands that I prefer avoid implementing before launch:
 Also, I would omit keyspace notifications. For that I would like to deep dive and learn
 exact use-cases for this API.
 
-### Random commands we implemented along the way
+### Random commands we implemented as decorators along the way
 
  - [X] ROLE (2.8) decorator for for master withour replicas
  - [X] UNLINK (4.0) decorator for DEL command
  - [X] BGSAVE
  - [X] FUNCTION FLUSH
-## Milestone Nymph
-API 2,3,4 without cluster support, without modules, without memory inspection commands.
-Without support for keyspace notifications.
-
-Design config support. ~140 commands overall...
-## Milestone Molt
-API 5,6 - without cluster and modules. Streams support. ~80 commands overall.
-## Milestone Adult
-TBD.
 
+## Milestone Stability
+APIs 3,4,5 without cluster support, without modules, without memory introspection commands.
+Without geo commands and without support for keyspace notifications, without streams.
+Design config support. ~10-20 commands overall...
+Probably implement cluster-API decorators to allow cluster-configured clients to connect to a single
+instance.
 
 ## Design decisions along the way
 ### Expiration deadlines with relative accuracy
@@ -280,3 +278,5 @@ expiries of `PEXPIRE key 10010` will expire exactly after 10 seconds and 10ms. H
 
 Such rounding has at most 0.002% error which I hope is acceptable for large ranges.
 If it breaks your use-cases - talk to me or open an issue and explain your case.
+
+For more detailed differences between this and Redis implementations [see here](doc/differences.md).

src/redis/CMakeLists.txt

@@ -10,7 +10,7 @@ endif()
 
 add_library(redis_lib crc64.c crcspeed.c debug.c dict.c endianconv.c intset.c 
             listpack.c mt19937-64.c object.c lzf_c.c lzf_d.c sds.c sha256.c
-            quicklist.c redis_aux.c siphash.c t_hash.c t_zset.c util.c ${ZMALLOC_SRC})
+            quicklist.c redis_aux.c siphash.c t_hash.c t_zset.c util.c ziplist.c ${ZMALLOC_SRC})
 
 cxx_link(redis_lib  ${ZMALLOC_DEPS})
 

src/redis/ziplist.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2009-2012, Pieter Noordhuis <pcnoordhuis at gmail dot com>
+ * Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *   * Neither the name of Redis nor the names of its contributors may be used
+ *     to endorse or promote products derived from this software without
+ *     specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _ZIPLIST_H
+#define _ZIPLIST_H
+
+#define ZIPLIST_HEAD 0
+#define ZIPLIST_TAIL 1
+
+/* Each entry in the ziplist is either a string or an integer. */
+typedef struct {
+    /* When string is used, it is provided with the length (slen). */
+    unsigned char *sval;
+    unsigned int slen;
+    /* When integer is used, 'sval' is NULL, and lval holds the value. */
+    long long lval;
+} ziplistEntry;
+
+unsigned char *ziplistNew(void);
+unsigned char *ziplistMerge(unsigned char **first, unsigned char **second);
+unsigned char *ziplistPush(unsigned char *zl, unsigned char *s, unsigned int slen, int where);
+unsigned char *ziplistIndex(unsigned char *zl, int index);
+unsigned char *ziplistNext(unsigned char *zl, unsigned char *p);
+unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p);
+unsigned int ziplistGet(unsigned char *p, unsigned char **sval, unsigned int *slen, long long *lval);
+unsigned char *ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen);
+unsigned char *ziplistDelete(unsigned char *zl, unsigned char **p);
+unsigned char *ziplistDeleteRange(unsigned char *zl, int index, unsigned int num);
+unsigned char *ziplistReplace(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen);
+unsigned int ziplistCompare(unsigned char *p, unsigned char *s, unsigned int slen);
+unsigned char *ziplistFind(unsigned char *zl, unsigned char *p, unsigned char *vstr, unsigned int vlen, unsigned int skip);
+unsigned int ziplistLen(unsigned char *zl);
+size_t ziplistBlobLen(unsigned char *zl);
+void ziplistRepr(unsigned char *zl);
+typedef int (*ziplistValidateEntryCB)(unsigned char* p, void* userdata);
+int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep,
+                             ziplistValidateEntryCB entry_cb, void *cb_userdata);
+void ziplistRandomPair(unsigned char *zl, unsigned long total_count, ziplistEntry *key, ziplistEntry *val);
+void ziplistRandomPairs(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals);
+unsigned int ziplistRandomPairsUnique(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals);
+int ziplistSafeToAdd(unsigned char* zl, size_t add);
+
+#ifdef REDIS_TEST
+int ziplistTest(int argc, char *argv[], int accurate);
+#endif
+
+#endif /* _ZIPLIST_H */

src/server/rdb_save.cc

@@ -10,8 +10,11 @@
 
 extern "C" {
 #include "redis/intset.h"
+#include "redis/listpack.h"
+#include "redis/ziplist.h"
 #include "redis/rdb.h"
 #include "redis/util.h"
+#include "redis/zmalloc.h"
 }
 
 #include "base/logging.h"
@@ -132,7 +135,7 @@ uint8_t RdbObjectType(const robj* o) {
         return RDB_TYPE_ZSET_2;
       break;
     case OBJ_HASH:
-      if (o->encoding == OBJ_ENCODING_ZIPLIST)
+      if (o->encoding == OBJ_ENCODING_LISTPACK)
         return RDB_TYPE_HASH_ZIPLIST;
       else if (o->encoding == OBJ_ENCODING_HT)
         return RDB_TYPE_HASH;
@@ -206,6 +209,10 @@ error_code RdbSerializer::SaveObject(const robj* o) {
     return SaveSetObject(o);
   }
 
+  if (o->type == OBJ_HASH) {
+    return SaveHSetObject(o);
+  }
+
   LOG(FATAL) << "Not implemented " << o->type;
   return error_code{};
 }
@@ -252,7 +259,7 @@ error_code RdbSerializer::SaveSetObject(const robj* obj) {
 
     dictIterator* di = dictGetIterator(set);
     dictEntry* de;
-    auto key_cleanup = absl::MakeCleanup([di] { dictReleaseIterator(di); });
+    auto cleanup = absl::MakeCleanup([di] { dictReleaseIterator(di); });
 
     while ((de = dictNext(di)) != NULL) {
       sds ele = (sds)de->key;
@@ -270,6 +277,52 @@ error_code RdbSerializer::SaveSetObject(const robj* obj) {
   return error_code{};
 }
 
+error_code RdbSerializer::SaveHSetObject(const robj* obj) {
+  DCHECK_EQ(OBJ_HASH, obj->type);
+  if (obj->encoding == OBJ_ENCODING_HT) {
+    dict* set = (dict*)obj->ptr;
+
+    RETURN_ON_ERR(SaveLen(dictSize(set)));
+
+    dictIterator* di = dictGetIterator(set);
+    dictEntry* de;
+    auto cleanup = absl::MakeCleanup([di] { dictReleaseIterator(di); });
+
+    while ((de = dictNext(di)) != NULL) {
+      sds key = (sds)de->key;
+      sds value = (sds)de->v.val;
+
+      RETURN_ON_ERR(SaveString(string_view{key, sdslen(key)}));
+      RETURN_ON_ERR(SaveString(string_view{value, sdslen(value)}));
+    }
+  } else if (obj->encoding == OBJ_ENCODING_LISTPACK) {
+    // convert to ziplist first.
+    uint8_t* lp = (uint8_t*)obj->ptr;
+
+    size_t lplen = lpLength(lp);
+    CHECK(lplen > 0 && lplen % 2 == 0);  // has (key,value) pairs.
+
+    uint8_t* lpfield = lpFirst(lp);
+    uint8_t* zl = ziplistNew();
+    int64_t entry_len;
+    uint8_t* entry;
+    uint8_t buf[32];
+
+    while (lpfield) {
+      entry = lpGet(lpfield, &entry_len, buf);
+      zl = ziplistPush(zl, entry, entry_len, ZIPLIST_TAIL);
+      lpfield = lpNext(lp, lpfield);
+    }
+    size_t ziplen = ziplistBlobLen(zl);
+    auto cleanup = absl::MakeCleanup([zl] { zfree(zl); });
+    RETURN_ON_ERR(SaveString(string_view{reinterpret_cast<char*>(zl), ziplen}));
+  } else {
+    LOG(FATAL) << "Unknown jset encoding " << obj->encoding;
+  }
+
+  return error_code{};
+}
+
 /* Save a long long value as either an encoded string or a string. */
 error_code RdbSerializer::SaveLongLongAsString(int64_t value) {
   uint8_t buf[32];

src/server/rdb_save.h

@@ -49,6 +49,7 @@ class RdbSerializer {
   std::error_code SaveStringObject(const robj* obj);
   std::error_code SaveListObject(const robj* obj);
   std::error_code SaveSetObject(const robj* obj);
+  std::error_code SaveHSetObject(const robj* obj);
   std::error_code SaveLongLongAsString(int64_t value);
 
   ::io::Sink* sink_ = nullptr;

src/server/rdb_test.cc

@@ -82,9 +82,12 @@ TEST_F(RdbTest, LoadSmall) {
 
 TEST_F(RdbTest, Save) {
   Run({"set", "string_key", "val"});
-  Run({"sadd", "set_key1", "val"});
+  Run({"sadd", "set_key1", "val1", "val2"});
   Run({"sadd", "set_key2", "1", "2", "3"});
-  Run({"rpush", "list_key", "val"});
+
+  // Run({"rpush", "list_key", "val"});  // TODO: invalid encoding when reading by redis 6.
+  // Run({"rpush", "list_key", "val"});
+  Run({"hset", "hset_key", "field1", "val1", "field2", "val2"});
   Run({"save"});
 }