1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
/*
Copyright (C) 2009 Red Hat, Inc.
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _H_GLZ_ENCODER_DICTIONARY_PROTECTED
#define _H_GLZ_ENCODER_DICTIONARY_PROTECTED
/* Interface for using the dictionary for encoding.
Data structures are exposed for the encoder for efficiency
purposes. */
typedef struct WindowImage WindowImage;
typedef struct WindowImageSegment WindowImageSegment;
//#define CHAINED_HASH
#ifdef CHAINED_HASH
#define HASH_SIZE_LOG 16
#define HASH_CHAIN_SIZE 4
#else
#define HASH_SIZE_LOG 20
#define HASH_CHAIN_SIZE 1
#endif
#define HASH_SIZE (1 << HASH_SIZE_LOG)
#define HASH_MASK (HASH_SIZE - 1)
typedef struct HashEntry HashEntry;
typedef struct SharedDictionary SharedDictionary;
struct WindowImage {
uint64_t id;
LzImageType type;
int size; // in pixels
uint32_t first_seg;
GlzUsrImageContext *usr_context;
WindowImage* next;
uint8_t is_alive;
};
#define MAX_IMAGE_SEGS_NUM (0xffffffff)
#define NULL_IMAGE_SEG_ID MAX_IMAGE_SEGS_NUM
#define INIT_IMAGE_SEGS_NUM 1000
/* Images can be separated into several chunks. The basic unit of the
dictionary window is one image segment. Each segment is encoded separately.
An encoded match can refer to only one segment.*/
struct WindowImageSegment {
WindowImage *image;
uint8_t *lines;
uint8_t *lines_end;
uint32_t pixels_num; // Number of pixels in the segment
uint64_t pixels_so_far; // Total no. pixels passed through the window till this segment.
// NOTE - never use size delta independently. It should
// always be used with respect to a previous size delta
uint32_t next;
};
struct __attribute__ ((__packed__)) HashEntry {
uint32_t image_seg_idx;
uint32_t ref_pix_idx;
};
struct SharedDictionary {
struct {
/* The segments storage. A dynamic array.
By referring to a segment by its index, instead of address,
we save space in the hash entries (32bit instead of 64bit) */
WindowImageSegment *segs;
uint32_t segs_quota;
/* The window is manged as a linked list rather than as a cyclic
array in order to keep the indices of the segments consistent
after reallocation */
/* the window in a resolution of image segments */
uint32_t used_segs_head; // the latest head
uint32_t used_segs_tail;
uint32_t free_segs_head;
uint32_t *encoders_heads; // Holds for each encoder (by id), the window head when
// it started the encoding.
// The head is NULL_IMAGE_SEG_ID when the encoder is
// not encoding.
/* the window in a resolution of images. But here the head contains the oldest head*/
WindowImage* used_images_tail;
WindowImage* used_images_head;
WindowImage* free_images;
uint64_t pixels_so_far;
uint32_t size_limit; // max number of pixels in a window (per encoder)
} window;
/* Concurrency issues: the reading/writing of each entry field should be atomic.
It is allowed that the reading/writing of the whole entry won't be atomic,
since before we access a reference we check its validity*/
#ifdef CHAINED_HASH
HashEntry htab[HASH_SIZE][HASH_CHAIN_SIZE];
uint8_t htab_counter[HASH_SIZE]; //cyclic counter for the next entry in a chain to be assigned
#else
HashEntry htab[HASH_SIZE];
#endif
uint64_t last_image_id;
uint32_t max_encdoers;
pthread_mutex_t lock;
pthread_rwlock_t rw_alloc_lock;
GlzEncoderUsrContext *cur_usr; // each encoder has other context.
};
/*
Add the image to the tail of the window.
If possible, release images from the head of the window.
Also perform concurrency related operations.
usr_image_context: when an image is released from the window due to capacity overflow,
usr_image_context is given as a parameter to the free_image callback.
image_head_dist : the number of images between the current image and the head of the
window that is associated with the encoder.
*/
WindowImage *glz_dictionary_pre_encode(uint32_t encoder_id, GlzEncoderUsrContext *usr,
SharedDictionary *dict, LzImageType image_type,
int image_width, int image_height, int image_stride,
uint8_t *first_lines, unsigned int num_first_lines,
GlzUsrImageContext *usr_image_context,
uint32_t *image_head_dist);
/*
Performs concurrency related operations.
If possible, release images from the head of the window.
*/
void glz_dictionary_post_encode(uint32_t encoder_id, GlzEncoderUsrContext *usr,
SharedDictionary *dict);
#define IMAGE_SEG_IS_EARLIER(dict, dst_seg, src_seg) ( \
((src_seg) == NULL_IMAGE_SEG_ID) || (((dst_seg) != NULL_IMAGE_SEG_ID) \
&& ((dict)->window.segs[(dst_seg)].pixels_so_far < \
(dict)->window.segs[(src_seg)].pixels_so_far)))
#ifdef CHAINED_HASH
#define UPDATE_HASH(dict, hval, seg, pix) { \
uint8_t tmp_count = (dict)->htab_counter[hval]; \
(dict)->htab[hval][tmp_count].image_seg_idx = seg; \
(dict)->htab[hval][tmp_count].ref_pix_idx = pix; \
tmp_count = ((tmp_count) + 1) & (HASH_CHAIN_SIZE - 1); \
dict->htab_counter[hval] = tmp_count; \
}
#else
#define UPDATE_HASH(dict, hval, seg, pix) { \
(dict)->htab[hval].image_seg_idx = seg; \
(dict)->htab[hval].ref_pix_idx = pix; \
}
#endif
/* checks if the reference segment is located in the range of the window
of the current encoder */
#define REF_SEG_IS_VALID(dict, enc_id, ref_seg, src_seg) ( \
((ref_seg) == (src_seg)) || \
((ref_seg)->image && \
(ref_seg)->image->is_alive && \
(src_seg->image->type == ref_seg->image->type) && \
(ref_seg->pixels_so_far <= src_seg->pixels_so_far) && \
((dict)->window.segs[ \
(dict)->window.encoders_heads[enc_id]].pixels_so_far <= \
ref_seg->pixels_so_far)))
#endif // _H_GLZ_ENCODER_DICTIONARY_PROTECTED
|
