Coverage report: /home/ellis/comp/core/ffi/rocksdb/merge.lisp

Kind	Covered	All	%
expression	119	145	82.1
branch	7	12	58.3

Key

Not instrumented

Conditionalized out

Executed

Not executed

Both branches taken

One branch taken

Neither branch taken

1

;;; rocksdb/merge.lisp --- RocksDB Merge Operators

2

3

;; RocksDB Lisp Merge Operator API

4

5

;;; Commentary:

6

7

;; When to use built-in ROCKSDB-MERGE:

8

9

;; - You have data that needs to be incrementally updated.

10

11

;; - You would usually need to read the data before knowing what the new value would be.

12

13

;; Oterwise as far as the FFI is concerned - which doesn't support

14

;; AssociateMerge, you should use the Generic Merge API.

15

16

;; When to use Associative Merge (unavailable in C/LISP API):

17

18

;; - merge operands are formatted the same as Put values AND

19

20

;; - it is okay to combine multiple operands into one

21

22

;; When to use Generic Merge (this API):

23

24

;; - you are unable to use Associate Merge

25

26

;; - it is possible to combine multiple operands

27

28

;;; Refs:

29

30

;; impl: https://github.com/facebook/rocksdb/wiki/Merge-Operator-Implementation

31

32

;; wiki: https://github.com/facebook/rocksdb/wiki/merge-operator

33

34

;;; Code:

35

(in-package :rocksdb)

36

37

(eval-always

38

(defvar *rocksdb-partial-merge-lambda-list*

39

'((state (* t))

40

(key (* unsigned-char))

41

(klen size-t)

42

(ops (* (* unsigned-char)))

43

(ops-length (* size-t))

44

(num-ops size-t)

45

(success (* unsigned-char))

46

(new-vlen (* size-t))))

47

48

(defvar *rocksdb-full-merge-lambda-list*

49

'((state (* t))

50

(key (* unsigned-char))

51

(klen size-t)

52

(existing-val (* unsigned-char))

53

(existing-vlen size-t)

54

(ops (* (* unsigned-char)))

55

(ops-length (* size-t))

56

(num-ops int)

57

(success (* unsigned-char))

58

(new-vlen (* size-t)))))

59

60

#|

61

Gives the client a way to express the read -> modify -> write semantics

62

key: (IN) The key that's associated with this merge operation.

63

existing: (IN) null indicates that the key does not exist before this op

64

operand_list:(IN) the sequence of merge operations to apply, front() first.

65

new_value: (OUT) Client is responsible for filling the merge result here

66

logger: (IN) Client could use this to log errors during merge.

67

68

Return true on success. Return false failure / error / corruption.

69

|#

70

;; FullMerge() is used when a Put/Delete is the *existing_value (or null)

71

(define-alien-type rocksdb-full-merge-function

72

(function (* char)

73

(* t)

74

(* unsigned-char)

75

size-t

76

(* unsigned-char)

77

size-t

78

(* (* unsigned-char))

79

(* size-t)

80

int

81

(* unsigned-char)

82

(* size-t)))

83

84

#|

85

This function performs merge(left_op, right_op)

86

when both the operands are themselves merge operation types.

87

Save the result in *new_value and return true. If it is impossible

88

or infeasible to combine the two operations, return false instead.

89

|#

90

;; PartialMerge() is used to combine two-merge operands (if possible)

91

(define-alien-type rocksdb-partial-merge-function

92

(function (* char)

93

(* t)

94

(* unsigned-char)

95

size-t

96

(* (* unsigned-char))

97

(* size-t)

98

int

99

(* unsigned-char)

100

(* size-t)))

101

102

(define-alien-type rocksdb-delete-value-function

103

(function void

104

(* unsigned-char)

105

size-t))

106

107

(define-alien-type rocksdb-destructor-function

108

(function void (* t)))

109

110

#|

111

The name of the MergeOperator. Used to check for MergeOperator

112

mismatches (i.e., a DB created with one MergeOperator is

113

accessed using a different MergeOperator)

114

|#

115

(define-alien-type rocksdb-name-function

116

(function c-string))

117

118

;; (sb-alien::define-alien-callable mangle int () 0)

119

120

(defar rocksdb-mergeoperator-create (* rocksdb-mergeoperator)

121

(state (* t))

122

(destructor (* rocksdb-destructor-function))

123

(full-merge (* rocksdb-full-merge-function))

124

(partial-merge (* rocksdb-partial-merge-function))

125

(delete-value (* rocksdb-delete-value-function))

126

(name (* rocksdb-name-function)))

127

128

#| [[file:~/dev/comp/core/c/rocksdb.h::/* Merge Operator */]] |#

129

130

(defar rocksdb-mergeoperator-destroy void (self (* rocksdb-mergeoperator)))

131

132

(define-alien-callable rocksdb-destructor void ((self (* t)))

133

(free-alien self)

134

(values))

135

136

(define-alien-callable rocksdb-name c-string () (make-alien-string #.(symbol-name (gensym "rocksdb:"))))

137

138

;;; Associative Merge Ops

139

;;;; Concat Merge

140

(define-alien-callable rocksdb-concat-merge-name c-string () (make-alien-string "cc:concat"))

141

142

(define-alien-callable rocksdb-concat-full-merge (* char) #.*rocksdb-full-merge-lambda-list*

143

(declare (ignore state))

144

(log:trace!

145

(format nil "Applying CC:CONCAT full merge with ~A operands" num-ops))

146

(log:trace! :key key :klen klen)

147

(let ((len existing-vlen)

148

(opslen (alien-sap ops-length))

149

(ret (make-alien char)))

150

(unless (null-alien existing-val)

151

(loop for i below existing-vlen

152

do (setf (deref ret i) (deref existing-val i))))

153

(unless (zerop num-ops)

154

(loop for i below num-ops

155

with slen = #.(alien-size (* size-t) :bytes)

156

with s = #.(alien-size (* (* unsigned-char)) :bytes)

157

with olen = (deref (sap-alien (sb-alien::sap+ opslen (* slen i)) (* size-t)))

158

do (loop for l below olen

159

do (setf (deref ret (+ len l)) (deref (deref ops i) l)))

160

do (incf len olen)))

161

(setf (deref new-vlen) len

162

(deref success) 1)

163

ret))

164

165

(define-alien-callable rocksdb-concat-partial-merge (* char) #.*rocksdb-partial-merge-lambda-list*

166

(declare (ignore state))

167

(log:trace!

168

"Applying CC:CONCAT partial merge..."

169

(list key klen ops ops-length num-ops success new-vlen))

170

(setf (deref success) 0)

171

nil)

172

173

(define-alien-callable rocksdb-delete-value void

174

((state (* t))

175

(value (* unsigned-char))

176

(value-length size-t))

177

(declare (ignore state value-length))

178

(unless (null-alien value)

179

(setf value nil))

180

(values))

181

182

;;;; Index Merge

183

(define-alien-callable rocksdb-index-merge-name c-string () (make-alien-string "cc:index"))

184

185

(define-alien-callable rocksdb-index-full-merge (* unsigned-char) #.*rocksdb-full-merge-lambda-list*

186

(declare (ignore state))

187

(log:trace! "Applying CC:INDEX full merge with ~A operands" num-ops)

188

(log:trace! :key key :klen klen)

189

(let ((len (if (zerop existing-vlen) 1 existing-vlen))

190

(opslen (alien-sap ops-length))

191

(ret 0))

192

(unless (null-alien existing-val)

193

(incf ret

194

(std:octets-to-integer

195

(coerce

196

(loop for i below existing-vlen

197

collect (deref existing-val i))

198

'std:octet-vector))))

199

(unless (zerop num-ops)

200

(loop for i below num-ops

201

with slen = #.(alien-size (* size-t) :bytes)

202

with s = #.(alien-size (* (* unsigned-char)) :bytes)

203

with olen = (deref (sap-alien (sb-alien::sap+ opslen (* slen i)) (* size-t)))

204

do (incf ret

205

(std:octets-to-integer

206

(coerce

207

(loop for l below olen

208

collect (deref (deref ops i) l))

209

'std:octet-vector)))))

210

(setf (deref new-vlen) len

211

(deref success) 1)

212

(octets-to-alien (std:integer-to-octets ret (* 8 len)))))

213

214

(define-alien-callable rocksdb-index-partial-merge boolean #.*rocksdb-partial-merge-lambda-list*

215

(declare (ignore state key klen ops ops-length num-ops new-vlen))

216

(setf (deref success) 0)

217

0)