View Javadoc

1   package net.obsearch.index;
2   
3   import java.io.IOException;
4   import java.lang.reflect.Array;
5   import java.util.Iterator;
6   import java.util.LinkedList;
7   import java.util.List;
8   
9   import org.apache.log4j.Logger;
10  
11  import net.obsearch.OperationStatus;
12  import net.obsearch.Status;
13  import net.obsearch.exception.AlreadyFrozenException;
14  import net.obsearch.exception.IllegalIdException;
15  import net.obsearch.exception.OBException;
16  import net.obsearch.exception.OBStorageException;
17  import net.obsearch.exception.OutOfRangeException;
18  import net.obsearch.exception.PivotsUnavailableException;
19  
20  import net.obsearch.ob.OBShort;
21  import net.obsearch.result.OBPriorityQueueShort;
22  import net.obsearch.result.OBResultShort;
23  
24  import static org.junit.Assert.assertEquals;
25  import static org.junit.Assert.assertFalse;
26  import static org.junit.Assert.assertTrue;
27  /*
28  		OBSearch: a distributed similarity search engine This project is to
29   similarity search what 'bit-torrent' is to downloads. 
30      Copyright (C) 2008 Arnoldo Jose Muller Molina
31  
32    	This program is free software: you can redistribute it and/or modify
33      it under the terms of the GNU General Public License as published by
34      the Free Software Foundation, either version 3 of the License, or
35      (at your option) any later version.
36  
37      This program is distributed in the hope that it will be useful,
38      but WITHOUT ANY WARRANTY; without even the implied warranty of
39      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
40      GNU General Public License for more details.
41  
42      You should have received a copy of the GNU General Public License
43      along with this program.  If not, see <http://www.gnu.org/licenses/>.
44  */
45  
46  /** 
47  	*  TestFrameworkShort performs several tests on an index. 
48  	*  This generated file works on shorts.
49    *  @author      Arnoldo Jose Muller Molina    
50    */
51  //*************************************************************************
52  //****** Warning: this is a generated file ********************************
53  //****** The source file is: TestFramework.java    
54  //*************************************************************************
55  public abstract class TestFrameworkShort<O extends OBShort> {
56  
57  	private static transient final Logger logger = Logger
58  			.getLogger(TestFrameworkShort.class);
59  
60  	protected O[] queries;
61  	protected O[] data;
62  
63  	protected IndexShort<O> index;
64  	
65  	private Class<O> type;
66  
67  	/**
68  	 * Create a new test with a DB size of dbSize and a query size of querySize.
69  	 * 
70  	 * @param dbSize
71  	 * @param querySize
72  	 * @param index
73  	 *            The index that will be used for testing.
74  	 */
75  	public TestFrameworkShort(Class<O> type, int dbSize, int querySize, IndexShort<O> index) {
76  		this.type = type;
77  		queries = createArray(querySize);
78  		data = createArray(dbSize);		
79  		this.index = index;
80  	}
81  
82  	/**
83  	 * Perform all the tests and DB creation.
84  	 * @throws Exception 
85  	 */
86  	public void test() throws Exception {		
87  		init();
88  		search();
89  		deletes();
90  		init2();
91  		search();
92  		deletes();
93  		init3();
94  		search();
95  		deletes();
96  		close();
97  	}
98  
99  //*************************************************************************
100 //****** Warning: this is a generated file ********************************
101 //****** The source file is: TestFramework.java    
102 //*************************************************************************
103 	protected void search() throws Exception{
104 		search(index, (short) 3, (byte) 3);
105 		search(index, (short) 7 , (byte) 3);       
106 	}
107 	
108 	protected void deletes() throws Exception{
109 		logger.info("Testing deletes");
110         int i = 0;       
111         while (i < data.length) {
112             O x = data[i];
113             OperationStatus ex = index.exists(x);
114             assertTrue(ex.getStatus() == Status.EXISTS);
115             assertTrue(ex.getId() == i);
116             ex = index.delete(x);
117             assertTrue("Status is: " + ex.getStatus() + " i: " + i , ex.getStatus() == Status.OK);
118             assertEquals((long)i, ex.getId());
119             ex = index.exists(x);            
120             assertTrue( "Exists after delete" + ex.getStatus() + " i " + i, ex.getStatus() == Status.NOT_EXISTS);
121 						if(i % 300 == 0){
122 								logger.info("Deleting... " + i );
123 						}
124             i++;
125         }
126         
127 	}
128 
129 	protected void close()throws Exception{
130 			index.close();
131 	}
132 	
133   	/**
134      * Perform all the searches with
135      * @param x
136      *                the index that will be used
137      * @param range
138      * @param k
139      */
140     public void search(IndexShort < O > index, short range, byte k)
141             throws Exception {
142         //range = (short)Math.min(Short.MAX_VALUE, range);
143         index.resetStats();
144         // it is time to Search
145         
146         String re = null;
147         logger.info("Matching begins...");
148         
149         int i = 0;
150         long realIndex = index.databaseSize();
151         List < OBPriorityQueueShort < O >> result = new LinkedList < OBPriorityQueueShort < O >>();        
152         while (i < this.queries.length) {
153                 OBPriorityQueueShort < O > x = new OBPriorityQueueShort < O >(
154                         k);
155                 if (i % 100 == 0) {
156                     logger.info("Matching " + i);
157                 }
158 
159                 O s = queries[i];                
160                     index.searchOB(s, range, x);
161                     result.add(x);
162                     i++;                                        
163         }
164         logger.info("Range: " + range + " k " + k + " " + index.getStats().toString());
165        
166         int maxQuery = i;
167 
168         Iterator < OBPriorityQueueShort < O >> it = result.iterator();
169         i = 0;
170         while (i < queries.length) {
171         	if (i % 300 == 0) {
172                     logger.info("Validating " + i + " of " + maxQuery);
173         	}
174                 O s = queries[i];
175                     OBPriorityQueueShort < O > x2 = new OBPriorityQueueShort < O >(
176                             k);
177                     searchSequential( s, x2, index, range);
178                     OBPriorityQueueShort < O > x1 = it.next();
179                     
180                     assertEquals("Error in query line: " + i + " " + index.debug(s) + "\n"
181                             + debug(x2,index ) + "\n" + debug(x1,index), x2, x1);
182                     i++;
183                 
184                 
185             }
186                    
187         logger.info("Finished  matching validation.");
188         assertFalse(it.hasNext());
189     }
190 
191 /**
192  * Prints debug info for the given priority queue.	
193  * @return
194  * @throws IllegalAccessException 
195  * @throws InstantiationException 
196  * @throws OBException 
197  * @throws IllegalIdException 
198  */
199 private String debug( OBPriorityQueueShort < O > q, IndexShort<O> index) throws IllegalIdException, OBException, InstantiationException, IllegalAccessException{
200 	StringBuilder res = new StringBuilder();
201 	Iterator<OBResultShort<O>> it = q.iterator();
202 	while(it.hasNext()){
203 		OBResultShort<O> r = it.next();
204 		res.append(r.getId());
205 		res.append(" r: ");
206 		res.append(r.getDistance());
207 		res.append("\n");
208 		res.append(index.debug(index.getObject(r.getId())));
209 		res.append("\n");
210 	}
211 	return res.toString();
212 }
213     
214     /**
215      * Sequential search.   
216      * @param o
217      *                The object to search
218      * @param result
219      *                The queue were the results are stored
220      * @param index
221      *                the index to search
222      * @param range
223      *                The range to employ
224      * @throws Exception
225      *                 If something goes really bad.
226      */
227 //*************************************************************************
228 //****** Warning: this is a generated file ********************************
229 //****** The source file is: TestFramework.java    
230 //*************************************************************************
231     public void searchSequential( O o,
232             OBPriorityQueueShort < O > result,
233             IndexShort < O > index, short range) throws Exception {
234         int i = 0;
235         while (i < data.length) {
236             O obj = data[i];
237             short res = o.distance(obj);
238             if (res <= range) {
239                 result.add(i, obj, res);
240             }
241             i++;
242         }
243     }
244 
245 	private void init() throws AlreadyFrozenException, IllegalIdException,
246 			OutOfRangeException, IOException, IllegalAccessException,
247 			InstantiationException, OBException, PivotsUnavailableException {
248 		// initialize data.
249 		initQueries();
250 		initData();
251 		// insert data into the index.
252 		logger.info("Inserting data...");
253 		int i = 0;
254 		while (i < data.length) {
255 			O s = data[i];
256 			if(i % 1000 == 0){
257 				logger.info("Inserting: " + i);
258 			}
259 			OperationStatus res = index.insert(s);
260 			assertTrue("Returned status: " + res.getStatus().toString(), res
261 					.getStatus() == Status.OK);
262 			assertEquals((long)i, res.getId());
263 			// If we insert before freezing, we should
264 			// be getting a Result.EXISTS if we try to insert
265 			// again!
266 			//assertTrue(!index.isFrozen());
267 			//res = index.insert(s);
268 			//assertTrue(res.getStatus() == Status.EXISTS);
269 			//assertEquals(res.getId(), (long)i);
270 			i++;
271 
272 		}
273 
274 		// "learn the data".
275 		logger.info("freezing");
276 		index.freeze();
277 
278 		logger.info("Checking exists and insert");
279 		i = 0;
280 		while (i < data.length) {
281 
282 			O s = data[i];
283 			OperationStatus res = index.exists(s);
284 			assertTrue("Str: " + s.toString() + " line: " + i,
285 					res.getStatus() == Status.EXISTS);
286 			assertEquals((long)i, res.getId());
287 			// attempt to insert the object again, and get
288 			// the -1
289 			res = index.insert(s);
290 			assertEquals(res.getId(), (long)i);
291 			assertTrue(res.getStatus() == Status.EXISTS);
292 			i++;
293 
294 			if (i % 10000 == 0) {
295 				logger.info("Exists/insert : " + i);
296 			}
297 
298 		}
299 		assertEquals((long)i, index.databaseSize());
300 
301 	}
302 
303 
304 
305 	/**
306    * Test insert(O,long) insertions.
307    */
308 	private void init2() throws AlreadyFrozenException, IllegalIdException,
309 			OutOfRangeException, IOException, IllegalAccessException,
310 			InstantiationException, OBException {
311 
312 			//index.setIdAutoGeneration(false);
313 
314 		// insert data into the index.
315 		logger.info("Inserting data...");
316 		int i = 0;
317 		while (i < data.length) {
318 			O s = data[i];
319 			if(i % 1000 == 0){
320 				logger.info("Inserting: " + i);
321 			}
322 			OperationStatus res = index.insert(s, i);
323 			assertTrue("Returned status: " + res.getStatus().toString(), res
324 					.getStatus() == Status.OK);
325 			assertEquals((long)i, res.getId());
326 
327 			res = index.insert(s,i);
328 			assertTrue(res.getStatus() == Status.EXISTS);
329 			assertEquals(res.getId(), (long)i);
330 			i++;
331 
332 		}
333 
334 
335 		logger.info("Checking exists and insert");
336 		i = 0;
337 		while (i < data.length) {
338 
339 			O s = data[i];
340 			OperationStatus res = index.exists(s);
341 			assertTrue("Str: " + s.toString() + " line: " + i,
342 					res.getStatus() == Status.EXISTS);
343 			assertEquals((long)i, res.getId());
344 			// attempt to insert the object again, and get
345 			// the -1
346 			res = index.insert(s,i);
347 			assertEquals(res.getId(), (long)i);
348 			assertTrue(res.getStatus() == Status.EXISTS);
349 			i++;
350 
351 			if (i % 10000 == 0) {
352 				logger.info("Exists/insert : " + i);
353 			}
354 
355 		}
356 		assertEquals((long)i, index.databaseSize());
357 
358 	}
359 
360 
361 	private void init3() throws AlreadyFrozenException, IllegalIdException,
362 			OutOfRangeException, IOException, IllegalAccessException,
363 			InstantiationException, OBException {
364 
365 			//index.setIdAutoGeneration(false);
366 
367 		// insert data into the index.
368 		logger.info("Inserting data (bulk)...");
369 		int i = 0;
370 		while (i < data.length) {
371 			O s = data[i];
372 			if(i % 1000 == 0){
373 				logger.info("Inserting: " + i);
374 			}
375 			OperationStatus res = index.insertBulk(s, i);
376 			assertTrue("Returned status: " + res.getStatus().toString(), res
377 					.getStatus() == Status.OK);
378 			assertEquals((long)i, res.getId());
379 		
380 			i++;
381 
382 		}
383 
384 
385 		logger.info("Checking exists and insert");
386 		i = 0;
387 		while (i < data.length) {
388 
389 			O s = data[i];
390 			OperationStatus res = index.exists(s);
391 			assertTrue("Str: " + s.toString() + " line: " + i,
392 					res.getStatus() == Status.EXISTS);
393 			assertEquals((long)i, res.getId());			
394 
395 			if (i % 10000 == 0) {
396 				logger.info("Exists/insert : " + i);
397 			}
398 			i++;
399 		}
400 		assertEquals((long)i, index.databaseSize());
401 
402 	}
403 
404 	private void initQueries() {
405 		int i = 0;
406 		while (i < queries.length) {
407 			queries[i] = nextQuery();
408 			i++;
409 		}
410 	}
411 
412 	private void initData() {
413 		int i = 0;
414 		while (i < data.length) {
415 			data[i] = next();
416 			i++;
417 		}
418 	}
419 
420 	/**
421 	 * Create an array of size size.
422 	 * 
423 	 * @param size
424 	 * @return
425 	 */
426 	protected O[] createArray(int size){
427 		return (O[])Array.newInstance(type, size);
428 	}
429 
430 	/**
431 	 * Create a new object.
432 	 * 
433 	 * @return a new O object.
434 	 */
435 	protected abstract O next();
436 
437 	/**
438 	 * Generate an object for the query, defaults {@link #nextO()}.
439 	 * 
440 	 * @return {@link #nextO()}.
441 	 */
442 	protected O nextQuery() {
443 		return next();
444 	}
445 }
446