1 package net.obsearch.example.vectors;
2
3 import hep.aida.bin.StaticBin1D;
4
5 import java.io.FileNotFoundException;
6 import java.io.IOException;
7 import java.util.ArrayList;
8 import java.util.Iterator;
9 import java.util.List;
10 import java.util.Random;
11
12 import net.obsearch.ambient.Ambient;
13
14 import net.obsearch.ambient.tc.AmbientTC;
15 import net.obsearch.example.AbstractExampleGeneral;
16 import net.obsearch.example.AbstractGHSExample;
17 import net.obsearch.exception.IllegalIdException;
18 import net.obsearch.exception.NotFrozenException;
19 import net.obsearch.exception.OBException;
20 import net.obsearch.exception.OBStorageException;
21 import net.obsearch.exception.PivotsUnavailableException;
22 import net.obsearch.index.IndexUtilsFloat;
23 import net.obsearch.index.OBVectorFloat;
24 import net.obsearch.index.ghs.impl.Sketch64Float;
25 import net.obsearch.pivots.AcceptAll;
26 import net.obsearch.pivots.rf03.RF03PivotSelectorFloat;
27 import net.obsearch.pivots.rf04.RF04PivotSelectorFloat;
28 import net.obsearch.query.OBQueryFloat;
29 import net.obsearch.result.OBPriorityQueueFloat;
30
31 public class VectorsDemoGHSSISAP extends AbstractGHSExample {
32
33 final static Random r = new Random();
34
35 final static int VEC_SIZE = 100;
36
37 public VectorsDemoGHSSISAP(String[] args) throws IOException,
38 OBStorageException, OBException, IllegalAccessException,
39 InstantiationException, PivotsUnavailableException {
40 super(args);
41
42 }
43
44 public static OBVectorFloat generateFloatVector() {
45 float [] vect = new float[VEC_SIZE];
46 int i = 0;
47 while(i < VEC_SIZE){
48 vect[i] = r.nextFloat();
49 i++;
50 }
51 return new OBVectorFloat(vect);
52 }
53
54 @Override
55 protected void create() throws FileNotFoundException, OBStorageException,
56 OBException, IOException, IllegalAccessException,
57 InstantiationException, PivotsUnavailableException {
58
59 RF04PivotSelectorFloat<OBVectorFloat> sel = new RF04PivotSelectorFloat<OBVectorFloat>(
60 new AcceptAll<OBVectorFloat>());
61 sel.setDataSample(400);
62
63
64
65
66 Sketch64Float<OBVectorFloat> index = new Sketch64Float<OBVectorFloat>(OBVectorFloat.class,
67 sel, 48);
68 index.setExpectedError(.95f);
69 index.setSampleSize(100);
70 index.setKAlpha(alpha);
71
72 index.setMaxK(new int[] { 1 });
73 index.setFixedRecord(true);
74 index.setFixedRecord(VEC_SIZE * 4);
75
76
77
78
79
80
81 Ambient<OBVectorFloat, Sketch64Float<OBVectorFloat>> a = new AmbientTC<OBVectorFloat, Sketch64Float<OBVectorFloat>>(
82 index, indexFolder);
83
84
85 logger.info("Adding " + super.databaseSize + " objects...");
86 int i = 0;
87 while (i < super.databaseSize) {
88 index.insert(generateFloatVector());
89 if (i % 100000 == 0) {
90 logger.info("Loading: " + i);
91 }
92 i++;
93 }
94
95
96 logger.info("Preparing the index...");
97 a.freeze();
98 logger.info("YAY! stats: " + index.getStats());
99
100
101
102
103
104 a.close();
105 }
106
107
108
109 protected void intrinsic() throws IllegalIdException, IllegalAccessException, InstantiationException, OBException, FileNotFoundException, IOException{
110
111
112 Ambient<OBVectorFloat, Sketch64Float<OBVectorFloat>> a = new AmbientTC<OBVectorFloat, Sketch64Float<OBVectorFloat>>(super.indexFolder);
113 Sketch64Float<OBVectorFloat> index = a.getIndex();
114 logger
115 .info("Intrinsic dim: "
116 + index.intrinsicDimensionality(1000));
117
118 }
119
120 @Override
121 protected void search() throws FileNotFoundException, OBStorageException,
122 NotFrozenException, IllegalAccessException, InstantiationException,
123 OBException, IOException {
124
125
126 Ambient<OBVectorFloat, Sketch64Float<OBVectorFloat>> a = new AmbientTC<OBVectorFloat, Sketch64Float<OBVectorFloat>>(super.indexFolder);
127 Sketch64Float<OBVectorFloat> index = a.getIndex();
128 index.resetStats();
129 long start = System.currentTimeMillis();
130
131 List<OBQueryFloat> queryResults = new ArrayList<OBQueryFloat>(querySize);
132 List<OBVectorFloat> queries = new ArrayList<OBVectorFloat>(super.querySize);
133 int i = 0;
134
135 logger.info("Warming cache...");
136 index.bucketStats();
137 logger.info("Starting search!");
138 index.resetStats();
139 index.setKAlpha(1f);
140 while(i < querySize){
141 OBVectorFloat q = generateFloatVector();
142
143 OBPriorityQueueFloat<OBVectorFloat> queue = new OBPriorityQueueFloat<OBVectorFloat>(1);
144
145 index.searchOB(q, Float.MAX_VALUE, queue);
146
147 queryResults.add(new OBQueryFloat(q, queue.peek().getDistance(), queue));
148
149 queries.add(q);
150 logger.info("Doing query: " + i );
151 i++;
152 }
153
154 long elapsed = System.currentTimeMillis() - start;
155 logger.info("Time per query: " + elapsed / querySize + " millisec.");
156
157 logger.info("Stats follow: (total distances / pivot vectors computed during the experiment)");
158 logger.info(index.getStats().toString());
159
160
161 logger.info("Doing CompoundError validation");
162 StaticBin1D ep = new StaticBin1D();
163 IndexUtilsFloat.validateResults(queryResults, index);
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188 }
189
190
191 public static void main(String args[]) throws FileNotFoundException,
192 OBStorageException, NotFrozenException, IllegalAccessException,
193 InstantiationException, OBException, IOException,
194 PivotsUnavailableException {
195
196 VectorsDemoGHSSISAP s = new VectorsDemoGHSSISAP(args);
197
198 }
199
200 }