1 package net.obsearch.index.pptree.impl;
2
3 import java.io.File;
4 import java.io.IOException;
5 import java.nio.ByteBuffer;
6 import java.util.HashMap;
7 import java.util.Iterator;
8 import java.util.LinkedList;
9 import java.util.List;
10
11 import net.obsearch.Index;
12 import net.obsearch.OperationStatus;
13 import net.obsearch.Status;
14 import net.obsearch.asserts.OBAsserts;
15 import net.obsearch.cache.OBCacheHandlerLong;
16 import net.obsearch.cache.OBCacheLong;
17 import net.obsearch.constants.OBSearchProperties;
18 import net.obsearch.exception.IllegalIdException;
19 import net.obsearch.exception.NotFrozenException;
20 import net.obsearch.exception.OBException;
21 import net.obsearch.exception.OBStorageException;
22 import net.obsearch.exception.OutOfRangeException;
23 import net.obsearch.filter.Filter;
24 import net.obsearch.index.pptree.AbstractPPTree;
25 import net.obsearch.index.pptree.SpaceTreeLeaf;
26 import net.obsearch.pivots.IncrementalPivotSelector;
27 import net.obsearch.storage.CloseIterator;
28 import net.obsearch.utils.bytes.ByteBufferFactoryConversion;
29
30 import net.obsearch.index.IndexShort;
31
32 import net.obsearch.ob.OBShort;
33 import net.obsearch.query.OBQueryShort;
34 import net.obsearch.result.OBPriorityQueueShort;
35 import net.obsearch.result.OBResultShort;
36 import net.obsearch.storage.OBStoreFactory;
37 import net.obsearch.storage.TupleDouble;
38 import org.apache.log4j.Logger;
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71 public class PPTreeShort<O extends OBShort> extends AbstractPPTree<O> implements
72 IndexShort<O> {
73
74
75
76
77 private short minInput;
78
79
80
81
82 private short maxInput;
83
84
85
86
87 private float opt;
88
89
90
91
92 private static final transient Logger logger = Logger
93 .getLogger(PPTreeShort.class);
94
95 private OBCacheLong<double[]> bCache;
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115 public PPTreeShort(Class<O> type,
116 IncrementalPivotSelector<O> pivotSelector, int pivotCount, int od)
117 throws IOException, OBException {
118 this(type, pivotSelector, pivotCount, od, Short.MIN_VALUE,
119 Short.MAX_VALUE);
120 }
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148 public PPTreeShort(Class<O> type,
149 IncrementalPivotSelector<O> pivotSelector, int pivotCount, int od,
150 short minInput, short maxInput) throws IOException, OBException {
151 super(type, pivotSelector, pivotCount, od);
152 OBAsserts.chkAssert(minInput < maxInput,
153 "minInput must be smaller than maxInput");
154 this.minInput = minInput;
155 this.maxInput = maxInput;
156
157
158 this.opt = 1 / ((float) (maxInput - minInput));
159
160 }
161
162 @Override
163 protected double[] extractTuple(ByteBuffer in) throws OutOfRangeException {
164 int i = 0;
165 double[] res = new double[getPivotCount()];
166 while (i < getPivotCount()) {
167 res[i] = normalizeFirstPassAux(in.getShort());
168 i++;
169 }
170 return res;
171 }
172
173 @Override
174 protected ByteBuffer objectToProjectionBytes(O object) throws OBException {
175 short[] t = new short[getPivotCount()];
176 calculatePivotTuple(object, t);
177 ByteBuffer out = ByteBufferFactoryConversion.createByteBuffer(0, super
178 .getPivotCount(), 0, 0, 0, 0);
179 for (short d : t) {
180 out.putShort(d);
181 }
182 return out;
183 }
184
185
186
187
188
189
190 protected int distanceValueSizeInBytes() {
191 return Short.SIZE / 8;
192 }
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209 protected final void generateRectangleFirstPass(short[] t, short r,
210 double[][] q) throws OutOfRangeException {
211
212 int i = 0;
213 while (i < q.length) {
214 q[i][MIN] = normalizeFirstPassAux((short) Math.max(t[i] - r,
215 minInput));
216 q[i][MAX] = normalizeFirstPassAux((short) Math.min(t[i] + r,
217 maxInput));
218 i++;
219 }
220 }
221
222 public boolean intersects(O object, short r, int box)
223 throws NotFrozenException, InstantiationException,
224 IllegalIdException, IllegalAccessException, OutOfRangeException,
225 OBException {
226
227 short[] t = new short[getPivotCount()];
228 calculatePivotTuple(object, t);
229
230 double[][] qrect = new double[getPivotCount()][2];
231 generateRectangleFirstPass(t, r, qrect);
232
233 return super.spaceTreeLeaves[box].intersects(qrect);
234 }
235
236 public Iterator<Long> intersectingBoxes(O object, short r)
237 throws NotFrozenException, InstantiationException,
238 IllegalIdException, IllegalAccessException {
239 throw new UnsupportedOperationException();
240 }
241
242 public void searchOB(O object, short r, OBPriorityQueueShort<O> result)
243 throws NotFrozenException, InstantiationException,
244 IllegalIdException, IllegalAccessException, OutOfRangeException,
245 OBException {
246 OBAsserts.chkPositive(r);
247 short[] t = new short[getPivotCount()];
248
249 calculatePivotTuple(object, t);
250 double[][] qrect = new double[getPivotCount()][2];
251 generateRectangleFirstPass(t, r, qrect);
252 List<SpaceTreeLeaf> hyperRectangles = new LinkedList<SpaceTreeLeaf>();
253
254 double[] center = normalizeFirstPass(t);
255 spaceTree.searchRange(qrect, center, hyperRectangles);
256 searchOBAux(object, r, result, qrect, t, hyperRectangles);
257
258 stats.incQueryCount();
259 stats.incBucketsRead(hyperRectangles.size());
260 }
261
262 public void searchOB(O object, short r, OBPriorityQueueShort<O> result,
263 int[] boxes) throws NotFrozenException, InstantiationException,
264 IllegalIdException, IllegalAccessException, OutOfRangeException,
265 OBException {
266 short[] t = new short[getPivotCount()];
267
268 calculatePivotTuple(object, t);
269 double[][] qrect = new double[getPivotCount()][2];
270 generateRectangleFirstPass(t, r, qrect);
271 List<SpaceTreeLeaf> hyperRectangles = new LinkedList<SpaceTreeLeaf>();
272 int i = 0;
273 int max = boxes.length;
274 while (i < max) {
275 hyperRectangles.add(super.spaceTreeLeaves[boxes[i]]);
276 i++;
277 }
278 searchOBAux(object, r, result, qrect, t, hyperRectangles);
279 }
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314 public void searchOBAux(O object, short r, OBPriorityQueueShort<O> result,
315 double[][] qrect, short[] t, List<SpaceTreeLeaf> hyperRectangles)
316 throws NotFrozenException, InstantiationException,
317 IllegalIdException, IllegalAccessException, OutOfRangeException,
318 OBException {
319
320 assertFrozen();
321
322
323
324
325
326
327
328
329
330
331 int pyramidCount = 2 * getPivotCount();
332
333 short myr = r;
334
335 double[] lowHighResult = new double[2];
336
337 Iterator<SpaceTreeLeaf> it = hyperRectangles.iterator();
338
339 double[][] qw = new double[getPivotCount()][2];
340 double[][] q = new double[getPivotCount()][2];
341 while (it.hasNext()) {
342 SpaceTreeLeaf space = it.next();
343 if (!space.intersects(qrect)) {
344 continue;
345 }
346
347
348 int i = 0;
349
350 space.generateRectangle(qrect, qw);
351 centerQuery(qw);
352 double[] minArray = generateMinArray(qw);
353 while (i < pyramidCount) {
354
355 copyQuery(qw, q);
356 if (intersect(q, i, minArray, lowHighResult)) {
357 int ri = (space.getSNo() * 2 * getPivotCount()) + i;
358
359 stats.incBucketsRead();
360 searchBTreeAndUpdate(object, t, myr, ri
361 + lowHighResult[HLOW], ri + lowHighResult[HHIGH],
362 result);
363
364 short nr = result.updateRange(myr);
365
366 if (nr < myr) {
367 myr = nr;
368
369 generateRectangleFirstPass(t, myr, qrect);
370 space.generateRectangle(qrect, qw);
371 if (!space.intersects(qrect)) {
372 break;
373
374
375
376
377
378 }
379 centerQuery(qw);
380
381 }
382
383 }
384 i++;
385 }
386 }
387
388
389
390 }
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424 private void searchBTreeAndUpdate(final O object, final short[] tuple,
425 final short r, final double hlow, final double hhigh,
426 final OBPriorityQueueShort<O> result)
427 throws IllegalAccessException, InstantiationException,
428 IllegalIdException, OBException {
429
430 CloseIterator<TupleDouble> it = C.processRange(hlow, hhigh);
431 try {
432 short max = Short.MIN_VALUE;
433 short realDistance = Short.MIN_VALUE;
434 while (it.hasNext()) {
435 stats.incSmapCount();
436 TupleDouble tup = it.next();
437 ByteBuffer in = tup.getValue();
438
439 int i = 0;
440 short t;
441 max = Short.MIN_VALUE;
442 while (i < tuple.length) {
443 t = (short) Math.abs(tuple[i] - in.getShort());
444 if (t > max) {
445 max = t;
446 if (t > r) {
447 break;
448
449
450 }
451 }
452 i++;
453 }
454 if (max <= r && result.isCandidate(max)) {
455
456 long id = in.getLong();
457 O toCompare = getObject(id);
458 realDistance = object.distance(toCompare);
459 stats.incDistanceCount();
460 if (realDistance <= r) {
461 result.add(id, toCompare, realDistance);
462 }
463 }
464 }
465 } finally {
466 it.closeCursor();
467 }
468
469 }
470
471 protected net.obsearch.OperationStatus insertAux(long id, O object)
472 throws OBStorageException, OBException, IllegalAccessException,
473 InstantiationException {
474 short[] t = new short[getPivotCount()];
475 calculatePivotTuple(object, t);
476 double[] first = normalizeFirstPass(t);
477 double ppTreeValue = super.ppvalue(first);
478 ByteBuffer out = ByteBufferFactoryConversion.createByteBuffer(0, super
479 .getPivotCount(), 0, 1, 0, 0);
480
481 for (short d : t) {
482 out.putShort(d);
483 }
484
485 out.putLong(id);
486 C.put(ppTreeValue, out);
487 OperationStatus result = new OperationStatus();
488 result.setStatus(Status.OK);
489 result.setId(id);
490 return result;
491 }
492
493 protected net.obsearch.OperationStatus insertAuxBulk(long id, O object)
494 throws OBStorageException, OBException, IllegalAccessException,
495 InstantiationException {
496 return insertAux(id, object);
497 }
498
499
500
501
502
503
504
505
506
507
508
509 protected double[] normalizeFirstPass(short[] t) throws OutOfRangeException {
510 assert t.length == getPivotCount();
511 double[] res = new double[getPivotCount()];
512
513 int i = 0;
514 while (i < t.length) {
515 res[i] = normalizeFirstPassAux(t[i]);
516 i++;
517 }
518 return res;
519 }
520
521 public void init(OBStoreFactory fact) throws OBStorageException,
522 OBException, NotFrozenException, IllegalAccessException,
523 InstantiationException, OBException {
524 super.init(fact);
525 bCache = new OBCacheLong<double[]>(new BLoader(), OBSearchProperties
526 .getBCacheSize());
527 }
528
529 private class BLoader implements OBCacheHandlerLong<double[]> {
530
531 public long getDBSize() throws OBStorageException {
532 return B.size();
533 }
534
535 public double[] loadObject(long id) throws OutOfRangeException,
536 OBException, InstantiationException, IllegalAccessException {
537 double[] tempTuple = new double[getPivotCount()];
538 ByteBuffer in = B.getValue(id);
539 int i = 0;
540 while (i < getPivotCount()) {
541 tempTuple[i] = normalizeFirstPassAux(in.getShort());
542 i++;
543 }
544 return tempTuple;
545 }
546
547 @Override
548 public void store(long key, double[] object) throws OBException {
549
550 }
551
552 }
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568 @Override
569 protected final double[] readFromB(long id) throws OutOfRangeException,
570 OBException {
571 double[] res;
572 try {
573 res = this.bCache.get(id);
574 } catch (Exception e) {
575 throw new OBException(e);
576 }
577 return res;
578 }
579
580
581
582
583
584
585
586
587
588
589
590
591 protected double normalizeFirstPassAux(short x) throws OutOfRangeException {
592 if (x < minInput || x > maxInput) {
593 throw new OutOfRangeException(minInput + "", maxInput + "", "" + x);
594 }
595 return ((double) (x - minInput)) * opt;
596 }
597
598
599
600
601
602
603
604
605
606 protected void calculatePivotTuple(final O obj, short[] tuple)
607 throws OBException {
608 assert tuple.length == this.getPivotCount();
609 int i = 0;
610 while (i < tuple.length) {
611 tuple[i] = obj.distance(this.pivots[i]);
612 i++;
613 }
614 }
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636 protected long equalTuples(short[] tuple, ByteBuffer in) {
637
638 int i = 0;
639 while (i < tuple.length) {
640 if (tuple[i] != in.getShort()) {
641 return -1;
642 }
643 i++;
644 }
645 return in.getLong();
646 }
647
648 public OperationStatus exists(O object) throws OBException,
649 IllegalAccessException, InstantiationException {
650 OperationStatus res = new OperationStatus(Status.NOT_EXISTS);
651 OBPriorityQueueShort<O> result = new OBPriorityQueueShort<O>((byte) 1);
652 searchOB(object, (short) 1, result);
653
654 if (result.getSize() == 1) {
655 OBResultShort<O> r = result.iterator().next();
656 if (r.getObject().equals(object)) {
657 res.setStatus(Status.EXISTS);
658 res.setId(r.getId());
659 }
660 }
661 return res;
662 }
663
664 @Override
665 protected OperationStatus deleteAux(final O object) throws OBException,
666 IllegalAccessException, InstantiationException {
667 long resId = -1;
668 OperationStatus res = new OperationStatus(Status.NOT_EXISTS);
669 short[] tuple = new short[getPivotCount()];
670
671 calculatePivotTuple(object, tuple);
672
673 double[] first = normalizeFirstPass(tuple);
674 double ppTreeValue = super.ppvalue(first);
675
676 CloseIterator<TupleDouble> it = C
677 .processRange(ppTreeValue, ppTreeValue);
678 try {
679 short max = Short.MIN_VALUE;
680 while (it.hasNext()) {
681 TupleDouble tup = it.next();
682 ByteBuffer in = tup.getValue();
683
684 int i = 0;
685 short t;
686 max = Short.MIN_VALUE;
687
688 while (i < tuple.length) {
689 t = (short) Math.abs(tuple[i] - in.getShort());
690 if (t > max) {
691 max = t;
692 if (t != 0) {
693 break;
694
695
696 }
697 }
698 i++;
699 }
700
701 if (max == 0) {
702
703 long id = in.getLong();
704 O toCompare = super.getObject(id);
705 if (object.equals(toCompare)) {
706 resId = id;
707 res = new OperationStatus(Status.OK);
708 res.setId(resId);
709 it.remove();
710 break;
711 }
712
713 }
714
715 }
716 } catch (Exception e) {
717 throw new OBException(e);
718 } finally {
719 it.closeCursor();
720 }
721 return res;
722 }
723
724 @Override
725 public void searchOB(O object, short r, Filter<O> filter,
726 OBPriorityQueueShort<O> result) throws NotFrozenException,
727 InstantiationException, IllegalIdException, IllegalAccessException,
728 OutOfRangeException, OBException {
729
730 throw new UnsupportedOperationException();
731 }
732
733 public double distance(O a, O b) throws OBException {
734 short result = ((OBShort) a).distance((OBShort) b);
735 return normalizeFirstPassAux(result);
736 }
737
738 }