1 package net.obsearch.index.knngraph;
2
3 import java.io.File;
4 import java.io.IOException;
5 import java.nio.ByteBuffer;
6
7 import org.apache.log4j.Logger;
8 import org.neo4j.api.core.EmbeddedNeo;
9 import org.neo4j.api.core.NeoService;
10 import org.neo4j.api.core.Node;
11 import org.neo4j.api.core.RelationshipType;
12 import org.neo4j.api.core.Transaction;
13
14 import net.obsearch.OB;
15 import net.obsearch.OperationStatus;
16 import net.obsearch.Status;
17 import net.obsearch.constants.ByteConstants;
18 import net.obsearch.exception.AlreadyFrozenException;
19 import net.obsearch.exception.IllegalIdException;
20 import net.obsearch.exception.OBException;
21 import net.obsearch.exception.OBStorageException;
22 import net.obsearch.exception.OutOfRangeException;
23 import net.obsearch.index.bucket.AbstractBucketIndex;
24 import net.obsearch.index.bucket.BucketContainer;
25 import net.obsearch.index.bucket.BucketObject;
26 import net.obsearch.pivots.IncrementalPivotSelector;
27 import net.obsearch.storage.CloseIterator;
28 import net.obsearch.storage.OBStorageConfig;
29 import net.obsearch.storage.OBStoreFactory;
30 import net.obsearch.storage.TupleBytes;
31 import net.obsearch.storage.TupleLong;
32 import net.obsearch.utils.bytes.ByteConversion;
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48 public abstract class AbstractKnnGraph<O extends OB, B extends BucketObject, Q, BC extends BucketContainer<O, B, Q>>
49 extends AbstractBucketIndex<O, B, Q, BC> {
50
51 protected static final String PROP_IDS = "i";
52 protected static final String PROP_SMAP = "s";
53 protected static final String PROP_VAL = "d";
54
55 private static transient final Logger logger = Logger
56 .getLogger(AbstractKnnGraph.class);
57
58
59
60
61 protected int localk = 15;
62
63 protected float t = 1.4f;
64
65 protected transient NeoService neo;
66 protected int seeds = 14;
67
68 public void setT(float t){
69 this.t = t;
70 }
71
72 protected enum RelTypes implements RelationshipType
73
74 {
75 NN
76 }
77
78 public AbstractKnnGraph(Class<O> type,
79 IncrementalPivotSelector<O> pivotSelector, int pivotCount,
80 int localk) throws OBStorageException, OBException {
81 super(type, pivotSelector, pivotCount);
82 this.localk = localk;
83 }
84
85
86
87
88
89 public void setSeeds(int value){
90 this.seeds = value;
91 }
92
93
94
95
96
97 protected void initByteArrayBuckets() throws OBException {
98 OBStorageConfig conf = new OBStorageConfig();
99 conf.setTemp(false);
100 conf.setDuplicates(false);
101 conf.setBulkMode(!isFrozen());
102 this.Buckets = fact.createOBStore("Buckets", conf);
103 }
104
105
106
107
108
109
110
111
112
113
114
115 protected void fillNode(Node n, B bucket) throws OBException {
116
117
118 long[] idsToStore = null;
119 if (n.hasProperty(PROP_IDS)) {
120 long[] ids = (long[]) n.getProperty(PROP_IDS);
121 idsToStore = new long[ids.length + 1];
122 System.arraycopy(ids, 0, idsToStore, 0, ids.length);
123 } else {
124 idsToStore = new long[1];
125 }
126 idsToStore[idsToStore.length - 1] = bucket.getId();
127 n.setProperty(PROP_IDS, idsToStore);
128
129
130 fillNodeAux(n, bucket);
131 }
132
133 protected abstract void fillNodeAux(Node n, B Bucket) throws OBException;
134
135
136
137
138
139
140
141
142
143 protected long getNodeId(byte[] grayCode) throws IllegalArgumentException, OBStorageException{
144 ByteBuffer b = Buckets.getValue(grayCode);
145 if(b == null){
146 return -1;
147 }
148 return b.getLong();
149 }
150
151
152
153
154 protected OperationStatus insertBucket(B b, O object)
155 throws OBStorageException, IllegalIdException,
156 IllegalAccessException, InstantiationException,
157 OutOfRangeException, OBException {
158 OperationStatus res = exists(object);
159 if(res.getStatus() == Status.NOT_EXISTS){
160 return insertBucketBulk(b, object);
161 }else{
162 return res;
163 }
164 }
165
166
167
168 @Override
169 public void freeze() throws IOException, AlreadyFrozenException,
170 IllegalIdException, IllegalAccessException, InstantiationException,
171 OBStorageException, OutOfRangeException, OBException {
172
173 super.freeze();
174 CloseIterator<TupleLong> it = A.processAll();
175 int i = 0;
176 while(it.hasNext()){
177 TupleLong t = it.next();
178 O o = super.bytesToObject(t.getValue());
179 B b = getBucket(o);
180 b.setId(t.getKey());
181 if(i % 100 == 0){
182 logger.info("Bulk insert: " + i);
183 }
184 insertBucketBulk(b,o);
185 i ++;
186 }
187 it.closeCursor();
188
189 logger.debug("Gray size: " + Buckets.size());
190 }
191
192
193
194
195 @Override
196 public void init(OBStoreFactory fact) throws OBStorageException,
197 OBException, InstantiationException, IllegalAccessException {
198
199 super.init(fact);
200
201 neo = new EmbeddedNeo((new File(fact.getFactoryLocation(), "neo")).getAbsolutePath() );
202 }
203
204
205
206
207
208
209
210
211
212 @Override
213 public void close() throws OBException {
214 super.close();
215 neo.shutdown();
216 }
217
218
219
220
221
222 }