View Javadoc

1   package net.obsearch.example.vectors;
2   
3   import hep.aida.bin.StaticBin1D;
4   
5   import java.io.FileNotFoundException;
6   import java.io.IOException;
7   import java.util.ArrayList;
8   import java.util.Iterator;
9   import java.util.List;
10  import java.util.Random;
11  
12  import net.obsearch.ambient.Ambient;
13  
14  import net.obsearch.ambient.tc.AmbientTC;
15  import net.obsearch.example.AbstractExampleGeneral;
16  import net.obsearch.example.AbstractGHSExample;
17  import net.obsearch.exception.IllegalIdException;
18  import net.obsearch.exception.NotFrozenException;
19  import net.obsearch.exception.OBException;
20  import net.obsearch.exception.OBStorageException;
21  import net.obsearch.exception.PivotsUnavailableException;
22  import net.obsearch.index.IndexUtilsFloat;
23  import net.obsearch.index.OBVectorFloat;
24  import net.obsearch.index.ghs.impl.Sketch64Float;
25  import net.obsearch.pivots.AcceptAll;
26  import net.obsearch.pivots.rf03.RF03PivotSelectorFloat;
27  import net.obsearch.pivots.rf04.RF04PivotSelectorFloat;
28  import net.obsearch.query.OBQueryFloat;
29  import net.obsearch.result.OBPriorityQueueFloat;
30  
31  public class VectorsDemoGHSSISAP extends AbstractGHSExample {
32  	
33  	final static Random r = new Random();
34  	
35  	final static int VEC_SIZE = 100;
36  
37  	public VectorsDemoGHSSISAP(String[] args) throws IOException,
38  			OBStorageException, OBException, IllegalAccessException,
39  			InstantiationException, PivotsUnavailableException {
40  		super(args);
41  		
42  	}
43  
44  	public static OBVectorFloat generateFloatVector() {
45  		float [] vect = new float[VEC_SIZE];
46  		int i = 0;
47  		while(i < VEC_SIZE){
48  			vect[i] = r.nextFloat();
49  			i++;
50  		}
51  		return new OBVectorFloat(vect);
52  	}
53  
54  	@Override
55  	protected void create() throws FileNotFoundException, OBStorageException,
56  			OBException, IOException, IllegalAccessException,
57  			InstantiationException, PivotsUnavailableException {
58  		// TODO Auto-generated method stub
59  		RF04PivotSelectorFloat<OBVectorFloat> sel = new RF04PivotSelectorFloat<OBVectorFloat>(
60  				new AcceptAll<OBVectorFloat>());
61  		sel.setDataSample(400);
62  		//sel.setRepetitions(400);
63  		// sel.setDesiredDistortion(0.10);
64  		// sel.setDesiredSpread(.70);
65  		// make the bit set as short so that m objects can fit in the buckets.
66  		Sketch64Float<OBVectorFloat> index = new Sketch64Float<OBVectorFloat>(OBVectorFloat.class,
67  				sel, 48);
68  		index.setExpectedError(.95f);
69  		index.setSampleSize(100);
70  		index.setKAlpha(alpha);
71  		// select the ks that the user will call.
72  		index.setMaxK(new int[] { 1 });
73  		index.setFixedRecord(true);
74  		index.setFixedRecord(VEC_SIZE * 4);
75  		// Create the ambient that will store the index's data. (NOTE: folder
76  		// name is hardcoded)
77  		// Ambient<L1, Sketch64Short<L1>> a = new AmbientBDBDb<L1,
78  		// Sketch64Short<L1>>( index, INDEX_FOLDER );
79  		// Ambient<L1, Sketch64Short<L1>> a = new AmbientMy<L1,
80  		// Sketch64Short<L1>>( index, INDEX_FOLDER );
81  		Ambient<OBVectorFloat, Sketch64Float<OBVectorFloat>> a = new AmbientTC<OBVectorFloat, Sketch64Float<OBVectorFloat>>(
82  				index, indexFolder);
83  
84  		// Add some random objects to the index:
85  		logger.info("Adding " + super.databaseSize + " objects...");
86  		int i = 0;
87  		while (i < super.databaseSize) {
88  			index.insert(generateFloatVector());
89  			if (i % 100000 == 0) {
90  				logger.info("Loading: " + i);
91  			}
92  			i++;
93  		}
94  
95  		// prepare the index
96  		logger.info("Preparing the index...");
97  		a.freeze();
98  		logger.info("YAY! stats: " + index.getStats());
99  		// add the rest of the objects
100 		/*
101 		 * while(i < DB_SIZE){ index.insert(generateVector()); if(i % 100000 ==
102 		 * 0){ logger.info("Loading: " + i); } i++; }
103 		 */
104 		a.close();
105 	}
106 
107 	
108 	
109 	protected void intrinsic() throws IllegalIdException, IllegalAccessException, InstantiationException, OBException, FileNotFoundException, IOException{
110 		
111 
112 		Ambient<OBVectorFloat, Sketch64Float<OBVectorFloat>> a =  new AmbientTC<OBVectorFloat, Sketch64Float<OBVectorFloat>>(super.indexFolder);
113 		Sketch64Float<OBVectorFloat> index = a.getIndex();
114 		logger
115 				.info("Intrinsic dim: "
116 						+ index.intrinsicDimensionality(1000));
117 		
118 	}
119 
120 	@Override
121 	protected void search() throws FileNotFoundException, OBStorageException,
122 			NotFrozenException, IllegalAccessException, InstantiationException,
123 			OBException, IOException {
124 		// TODO Auto-generated method stub
125 		
126 		Ambient<OBVectorFloat, Sketch64Float<OBVectorFloat>> a =  new AmbientTC<OBVectorFloat, Sketch64Float<OBVectorFloat>>(super.indexFolder);
127 		Sketch64Float<OBVectorFloat> index = a.getIndex();
128 		index.resetStats(); // reset the stats counter
129 		long start = System.currentTimeMillis();
130 		
131 		List<OBQueryFloat> queryResults = new ArrayList<OBQueryFloat>(querySize);
132 		List<OBVectorFloat> queries = new ArrayList<OBVectorFloat>(super.querySize);
133 		int i = 0;
134 		
135 		logger.info("Warming cache...");
136 		index.bucketStats();	
137 		logger.info("Starting search!");
138 		index.resetStats();
139 		index.setKAlpha(1f);
140 		while(i < querySize){
141 			OBVectorFloat q = 	generateFloatVector();	
142 			// query the index with k=1			
143 			OBPriorityQueueFloat<OBVectorFloat> queue = new OBPriorityQueueFloat<OBVectorFloat>(1);			
144 			// perform a query with r=3000000 and k = 1 
145 			index.searchOB(q, Float.MAX_VALUE, queue);
146 			
147 			queryResults.add(new OBQueryFloat(q, queue.peek().getDistance(), queue));
148 			
149 			queries.add(q);
150 			logger.info("Doing query: " + i );
151 			i++;
152 		}
153 		// print the results of the set of queries. 
154 		long elapsed = System.currentTimeMillis() - start;
155 		logger.info("Time per query: " + elapsed / querySize + " millisec.");
156 		
157 		logger.info("Stats follow: (total distances / pivot vectors computed during the experiment)");
158 		logger.info(index.getStats().toString());
159 
160 		
161 		logger.info("Doing CompoundError validation");
162 		StaticBin1D ep = new StaticBin1D();
163 		IndexUtilsFloat.validateResults(queryResults, index);
164 		
165 		
166 		/*
167 		Iterator<OBPriorityQueueFloat<OBVectorFloat>> it1 = queryResults.iterator();
168 		Iterator<OBVectorFloat> it2 = queries.iterator();
169 		StaticBin1D seqTime = new StaticBin1D();
170 		i = 0;
171 		while(it1.hasNext()){
172 			OBPriorityQueueFloat<OBVectorFloat> qu = it1.next();
173 			OBVectorFloat q = it2.next();
174 			Float time = System.currentTimeMillis();
175 			Float[] sortedList = index.fullMatchLite(q, false);
176 			Float el = System.currentTimeMillis() - time;
177 			seqTime.add(el);
178 			logger.info("Elapsed: " + el + " "  + i);
179 			OBQueryFloat<OBVectorFloat> queryObj = new OBQueryFloat<OBVectorFloat	>(q, Float.MAX_VALUE, qu, null);
180 			ep.add(queryObj.ep(sortedList));
181 			i++;
182 		}
183 		
184 		logger.info(ep.toString());
185 		logger.info("Time per seq query: ");
186 		logger.info(seqTime.toString());
187 		*/
188 	}
189 	
190 	
191 	public static void main(String args[]) throws FileNotFoundException,
192 	OBStorageException, NotFrozenException, IllegalAccessException,
193 	InstantiationException, OBException, IOException,
194 	PivotsUnavailableException {
195 
196 		VectorsDemoGHSSISAP s = new VectorsDemoGHSSISAP(args);
197 
198 }
199 
200 }