View Javadoc

1   package net.obsearch.storage;
2   
3   import net.obsearch.utils.bytes.ByteConversion;
4   import java.util.Arrays;
5   import java.util.HashMap;
6   import java.util.Iterator;
7   import java.util.Random;
8   import org.apache.log4j.Logger;
9   
10  import junit.framework.TestCase;
11  
12  import java.nio.ByteBuffer;
13  
14  import net.obsearch.exception.OBStorageException;
15  import net.obsearch.exception.OBException;
16  import net.obsearch.storage.TupleDouble;
17  /*
18  OBSearch: a distributed similarity search engine
19  This project is to similarity search what 'bit-torrent' is to downloads.
20  Copyright (C)  2008 Arnoldo Jose Muller Molina
21  
22  This program is free software: you can redistribute it and/or modify
23  it under the terms of the GNU General Public License as published by
24  the Free Software Foundation, either version 3 of the License, or
25  (at your option) any later version.
26  
27  This program is distributed in the hope that it will be useful,
28  but WITHOUT ANY WARRANTY; without even the implied warranty of
29  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
30  GNU General Public License for more details.
31  
32  You should have received a copy of the GNU General Public License
33  along with this program.  If not, see <http://www.gnu.org/licenses/>.
34  */
35  public class StorageValidationDouble extends TestCase {
36  		
37  		private static final transient Logger logger = Logger
38  			.getLogger(StorageValidationDouble.class);
39      
40      /**
41       * Create a vector of pairs of size {@link #NUM_OF_ITEMS} so that
42       * we can test the storage sub-system. 
43       */
44      public static final int NUM_OF_ITEMS = 1000;
45  
46  		public static final int STORAGE_SIZE = 8;
47            
48      /**
49       * Validates a Storage for shorts. Makes sure that insertions and deletions and
50       * the iterators are working well.      
51       * @param storage
52       * @throws OBStorageException
53       */
54      public static void validate(OBStoreDouble storage) throws OBStorageException, OBException{
55          HashMap<Double, byte[]> testData = new HashMap<Double, byte[]>();
56          Random x = new Random();
57          int i = 0;
58          double min = Double.MAX_VALUE;
59          double max = Double.MIN_VALUE;
60          while(i < NUM_OF_ITEMS){
61  														 double ran = Math.abs(x.nextDouble());
62              if(ran < min){
63                  min = ran;
64              }
65              if(ran > max){
66                  max = ran;
67              }
68  						ByteBuffer buf = ByteConversion.createByteBuffer(8);
69  						buf.putDouble(x.nextDouble());
70              testData.put(ran, buf.array());
71              i++;
72          }
73  
74  				
75  
76          for(double j : testData.keySet()){
77  						//						logger.info(j +  ": " + Arrays.toString(testData.get(j)));
78              storage.put(j, testData.get(j));
79  						//logger.info(Arrays.toString(testData.get(j)));
80          }               
81          // test that all the data is there:
82          for(double j : testData.keySet()){               
83  						assertTrue(Arrays.equals(storage.getValue(j), testData.get(j)));
84          }         
85  
86          
87          // do a range search       
88          CloseIterator<TupleDouble> it = storage.processRange(min, max);
89          i = 0;
90          boolean first = true;
91          double prev = Double.MIN_VALUE;
92          while(it.hasNext()){
93              TupleDouble t = it.next();
94  						assertTrue(testData.get(t.getKey()) != null);
95  						assertTrue("A:" + Arrays.toString(testData.get(t.getKey())) +  "B:" + Arrays.toString(t.getValue()) + " i: " + i + "A-key: " + t.getKey() , Arrays.equals(testData.get(t.getKey()), t.getValue()) );
96              //assertTrue(Arrays.equals(testData.get(t.getKey()), t.getValue().array()));
97              if(first){
98                  prev = t.getKey();
99                  first = false;
100             }else{
101            
102                 assertTrue( "Prev: " + prev + " t: " + t.getKey() + " i: " + i, prev < t.getKey());
103                 prev = t.getKey();
104             }
105             i++;
106         }
107         assertEquals(testData.size(), i);
108 				it.closeCursor();
109 				// do a range search       
110         it = storage.processRangeReverse(min, max);
111         i = 0;
112         first = true;
113         prev = Double.MIN_VALUE;
114         while(it.hasNext()){
115             TupleDouble t = it.next();
116             assertTrue(Arrays.equals(testData.get(t.getKey()), t.getValue()));
117             if(first){
118                 prev = t.getKey();
119                 first = false;
120             }else{
121            
122                 assertTrue( "Prev: " + prev + " t: " + t.getKey()+ " i: " + i, prev > t.getKey());
123                 prev = t.getKey();
124             }
125             i++;
126         }
127 				it.closeCursor();
128         assertEquals(testData.size(), i);
129 
130 
131 				
132         // TODO: add more tests for the iterator
133         // Test updates:
134         for(double j : testData.keySet()){
135             String d = x.nextDouble() + "";
136             testData.put(j, d.getBytes());//:)
137             storage.put(j, d.getBytes());
138         }          
139         // test that all the new  data is there:
140         for(double j : testData.keySet()){               
141 						assertTrue(Arrays.equals(storage.getValue(j), testData.get(j)));
142         }   
143 
144 				// Test deletes:
145         for(double j : testData.keySet()){
146             assertTrue( storage.getValue(j) != null);
147             storage.delete(j);
148             assertTrue(storage.getValue(j) == null);
149         }
150         
151         
152     }
153 
154 
155 		/**
156      * Validates a Storage for shorts. Makes sure that insertions and deletions and
157      * the iterators are working well.      
158      * @param storage
159      * @throws OBStorageException
160      */
161     public static void validateDuplicates(OBStoreDouble storage) throws OBStorageException, OBException{
162         HashMap<Double, byte[][]> testData = new HashMap<Double, byte[][]>();
163         Random x = new Random();
164         int i = 0;
165         double min = Double.MAX_VALUE;
166         double max = Double.MIN_VALUE;
167         while(i < NUM_OF_ITEMS){
168 														 double ran = Math.abs(x.nextDouble());
169             if(ran < min){
170                 min = ran;
171             }
172             if(ran > max){
173                 max = ran;
174             }
175 						byte[][] data = new byte[5][];
176 						int cx = 0;
177 						while(cx < data.length){
178 								ByteBuffer buf = ByteConversion.createByteBuffer(8);
179 								buf.putDouble(x.nextDouble());
180 								data[cx] = buf.array();
181 								cx++;
182 						}
183 						
184             testData.put(ran, data);
185             i++;
186         }
187 
188 				
189 
190         for(double j : testData.keySet()){
191 						//						logger.info(j +  ": " + Arrays.toString(testData.get(j)));
192 						byte[][] data = testData.get(j);
193 						for(byte[] d : data){
194 								storage.put(j, d);
195 						}           
196 						//logger.info(Arrays.toString(testData.get(j)));
197         }               
198         // test that all the data is there:
199 				
200         for(double j : testData.keySet()){   
201 						for(byte[] d : testData.get(j)){
202 								CloseIterator<TupleDouble> it = storage.processRange(j,j);
203 								int found = 0;
204 								while(it.hasNext()){
205 										TupleDouble t = it.next();
206 										if(Arrays.equals(d, t.getValue())){
207 												found++;
208 										}
209 										
210 								}
211 								it.closeCursor();
212 								assertTrue(found == 1);
213 						}
214         }      
215 
216 				// test that the iteration proceeds normally.
217 				CloseIterator<TupleDouble> it = storage.processRange(Double.MIN_VALUE, Double.MAX_VALUE);
218 				int c = 0;
219 				double key = -1;
220 					while(it.hasNext()){
221 							 TupleDouble t = it.next();
222 							 if(t.getKey() != key){
223 									 if(key != -1){
224 											 assert c == 5;
225 									 }
226 									 key = t.getKey();
227 									 c= 1;
228 							 }else{
229 									 c++;
230 							 }
231 					}
232 					it.closeCursor();
233 
234 
235 
236 					// test that the iteration proceeds normally.
237 			 it = storage.processRangeReverse(Double.MIN_VALUE, Double.MAX_VALUE);
238 				 c = 0;
239 				 key = -1;
240 					while(it.hasNext()){
241 							 TupleDouble t = it.next();
242 							 if(t.getKey() != key){
243 									 if(key != -1){
244 											 assert c == 5;
245 									 }
246 									 key = t.getKey();
247 									 c= 1;
248 							 }else{
249 									 c++;
250 							 }
251 					}
252 					it.closeCursor();
253 
254 				// test  deletes:
255 				//logger.info("Testing Deletes");
256 					/*				for(double j : testData.keySet()){   
257 						for(byte[] d : testData.get(j)){
258 								 it = storage.processRange(j,j);
259 								//	logger.info("Begin cycle1");
260 								while(it.hasNext()){
261 										TupleDouble t = it.next();
262 										// delete this guy.
263 										it.remove();
264 										boolean found = false;
265 										CloseIterator<TupleDouble> it2 = storage.processRange(j,j);
266 										//	logger.info("Begin cycle2");
267 										while(it2.hasNext()){
268 												TupleDouble t2 = it2.next();
269 												found = Arrays.equals(t.getValue().array(), t2.getValue().array());
270 												if(found){
271 														// it should have been removed
272 														break;
273 												}
274 										}
275 										assertTrue(! found);
276 										it2.closeCursor();
277 								}
278 								it.closeCursor();
279 								
280 						}
281 						}*/
282         
283         
284         
285         
286     }
287     
288    
289 }