1 package net.obsearch.index;
2
3 import java.io.IOException;
4 import java.lang.reflect.Array;
5 import java.util.Iterator;
6 import java.util.LinkedList;
7 import java.util.List;
8
9 import org.apache.log4j.Logger;
10
11 import net.obsearch.OperationStatus;
12 import net.obsearch.Status;
13 import net.obsearch.exception.AlreadyFrozenException;
14 import net.obsearch.exception.IllegalIdException;
15 import net.obsearch.exception.OBException;
16 import net.obsearch.exception.OBStorageException;
17 import net.obsearch.exception.OutOfRangeException;
18 import net.obsearch.exception.PivotsUnavailableException;
19
20 import net.obsearch.ob.OBLong;
21 import net.obsearch.result.OBPriorityQueueLong;
22 import net.obsearch.result.OBResultLong;
23
24 import static org.junit.Assert.assertEquals;
25 import static org.junit.Assert.assertFalse;
26 import static org.junit.Assert.assertTrue;
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51 /
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75 public TestFrameworkLong(Class<O> type, int dbSize, int querySize, IndexLong<O> index) {
76 this.type = type;
77 queries = createArray(querySize);
78 data = createArray(dbSize);
79 this.index = index;
80 }
81
82
83
84
85
86 public void test() throws Exception {
87 init();
88 search();
89 deletes();
90 init2();
91 search();
92 deletes();
93 init3();
94 search();
95 deletes();
96 close();
97 }
98
99 /
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140 public void search(IndexLong < O > index, long range, byte k)
141 throws Exception {
142
143 index.resetStats();
144
145
146 String re = null;
147 logger.info("Matching begins...");
148
149 int i = 0;
150 long realIndex = index.databaseSize();
151 List < OBPriorityQueueLong < O >> result = new LinkedList < OBPriorityQueueLong < O >>();
152 while (i < this.queries.length) {
153 OBPriorityQueueLong < O > x = new OBPriorityQueueLong < O >(
154 k);
155 if (i % 100 == 0) {
156 logger.info("Matching " + i);
157 }
158
159 O s = queries[i];
160 index.searchOB(s, range, x);
161 result.add(x);
162 i++;
163 }
164 logger.info("Range: " + range + " k " + k + " " + index.getStats().toString());
165
166 int maxQuery = i;
167
168 Iterator < OBPriorityQueueLong < O >> it = result.iterator();
169 i = 0;
170 while (i < queries.length) {
171 if (i % 300 == 0) {
172 logger.info("Validating " + i + " of " + maxQuery);
173 }
174 O s = queries[i];
175 OBPriorityQueueLong < O > x2 = new OBPriorityQueueLong < O >(
176 k);
177 searchSequential( s, x2, index, range);
178 OBPriorityQueueLong < O > x1 = it.next();
179
180 assertEquals("Error in query line: " + i + " " + index.debug(s) + "\n"
181 + debug(x2,index ) + "\n" + debug(x1,index), x2, x1);
182 i++;
183
184
185 }
186
187 logger.info("Finished matching validation.");
188 assertFalse(it.hasNext());
189 }
190
191
192
193
194
195
196
197
198
199 private String debug( OBPriorityQueueLong < O > q, IndexLong<O> index) throws IllegalIdException, OBException, InstantiationException, IllegalAccessException{
200 StringBuilder res = new StringBuilder();
201 Iterator<OBResultLong<O>> it = q.iterator();
202 while(it.hasNext()){
203 OBResultLong<O> r = it.next();
204 res.append(r.getId());
205 res.append(" r: ");
206 res.append(r.getDistance());
207 res.append("\n");
208 res.append(index.debug(index.getObject(r.getId())));
209 res.append("\n");
210 }
211 return res.toString();
212 }
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227 /
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308 private void init2() throws AlreadyFrozenException, IllegalIdException,
309 OutOfRangeException, IOException, IllegalAccessException,
310 InstantiationException, OBException {
311
312
313
314
315 logger.info("Inserting data...");
316 int i = 0;
317 while (i < data.length) {
318 O s = data[i];
319 if(i % 1000 == 0){
320 logger.info("Inserting: " + i);
321 }
322 OperationStatus res = index.insert(s, i);
323 assertTrue("Returned status: " + res.getStatus().toString(), res
324 .getStatus() == Status.OK);
325 assertEquals((long)i, res.getId());
326
327 res = index.insert(s,i);
328 assertTrue(res.getStatus() == Status.EXISTS);
329 assertEquals(res.getId(), (long)i);
330 i++;
331
332 }
333
334
335 logger.info("Checking exists and insert");
336 i = 0;
337 while (i < data.length) {
338
339 O s = data[i];
340 OperationStatus res = index.exists(s);
341 assertTrue("Str: " + s.toString() + " line: " + i,
342 res.getStatus() == Status.EXISTS);
343 assertEquals((long)i, res.getId());
344
345
346 res = index.insert(s,i);
347 assertEquals(res.getId(), (long)i);
348 assertTrue(res.getStatus() == Status.EXISTS);
349 i++;
350
351 if (i % 10000 == 0) {
352 logger.info("Exists/insert : " + i);
353 }
354
355 }
356 assertEquals((long)i, index.databaseSize());
357
358 }
359
360
361 private void init3() throws AlreadyFrozenException, IllegalIdException,
362 OutOfRangeException, IOException, IllegalAccessException,
363 InstantiationException, OBException {
364
365
366
367
368 logger.info("Inserting data (bulk)...");
369 int i = 0;
370 while (i < data.length) {
371 O s = data[i];
372 if(i % 1000 == 0){
373 logger.info("Inserting: " + i);
374 }
375 OperationStatus res = index.insertBulk(s, i);
376 assertTrue("Returned status: " + res.getStatus().toString(), res
377 .getStatus() == Status.OK);
378 assertEquals((long)i, res.getId());
379
380 i++;
381
382 }
383
384
385 logger.info("Checking exists and insert");
386 i = 0;
387 while (i < data.length) {
388
389 O s = data[i];
390 OperationStatus res = index.exists(s);
391 assertTrue("Str: " + s.toString() + " line: " + i,
392 res.getStatus() == Status.EXISTS);
393 assertEquals((long)i, res.getId());
394
395 if (i % 10000 == 0) {
396 logger.info("Exists/insert : " + i);
397 }
398 i++;
399 }
400 assertEquals((long)i, index.databaseSize());
401
402 }
403
404 private void initQueries() {
405 int i = 0;
406 while (i < queries.length) {
407 queries[i] = nextQuery();
408 i++;
409 }
410 }
411
412 private void initData() {
413 int i = 0;
414 while (i < data.length) {
415 data[i] = next();
416 i++;
417 }
418 }
419
420
421
422
423
424
425
426 protected O[] createArray(int size){
427 return (O[])Array.newInstance(type, size);
428 }
429
430
431
432
433
434
435 protected abstract O next();
436
437
438
439
440
441
442 protected O nextQuery() {
443 return next();
444 }
445 }
446