View Javadoc

1   package net.obsearch.index.knngraph;
2   
3   import java.io.File;
4   import java.io.IOException;
5   import java.nio.ByteBuffer;
6   
7   import org.apache.log4j.Logger;
8   import org.neo4j.api.core.EmbeddedNeo;
9   import org.neo4j.api.core.NeoService;
10  import org.neo4j.api.core.Node;
11  import org.neo4j.api.core.RelationshipType;
12  import org.neo4j.api.core.Transaction;
13  
14  import net.obsearch.OB;
15  import net.obsearch.OperationStatus;
16  import net.obsearch.Status;
17  import net.obsearch.constants.ByteConstants;
18  import net.obsearch.exception.AlreadyFrozenException;
19  import net.obsearch.exception.IllegalIdException;
20  import net.obsearch.exception.OBException;
21  import net.obsearch.exception.OBStorageException;
22  import net.obsearch.exception.OutOfRangeException;
23  import net.obsearch.index.bucket.AbstractBucketIndex;
24  import net.obsearch.index.bucket.BucketContainer;
25  import net.obsearch.index.bucket.BucketObject;
26  import net.obsearch.pivots.IncrementalPivotSelector;
27  import net.obsearch.storage.CloseIterator;
28  import net.obsearch.storage.OBStorageConfig;
29  import net.obsearch.storage.OBStoreFactory;
30  import net.obsearch.storage.TupleBytes;
31  import net.obsearch.storage.TupleLong;
32  import net.obsearch.utils.bytes.ByteConversion;
33  
34  /**
35   * Since neo does not allow to define ids, then we have to control two types of
36   * ids: 1) neo ids 2) OBSearch ids.
37   * 
38   * The ub-tree will hold ids of type 1). Each node in the graph will hold ids of
39   * type 2.
40   * 
41   * @author Arnoldo Jose Muller-Molina
42   * 
43   * @param <O>
44   * @param <B>
45   * @param <Q>
46   * @param <BC>
47   */
48  public abstract class AbstractKnnGraph<O extends OB, B extends BucketObject, Q, BC extends BucketContainer<O, B, Q>>
49  		extends AbstractBucketIndex<O, B, Q, BC> {
50  
51  	protected static final String PROP_IDS = "i";
52  	protected static final String PROP_SMAP = "s";
53  	protected static final String PROP_VAL = "d";
54  	
55  	private static transient final Logger logger = Logger
56      .getLogger(AbstractKnnGraph.class);
57  
58  	/**
59  	 * Number of connections per node.
60  	 */
61  	protected int localk = 15;
62  	
63  	protected float t = 1.4f;
64  
65  	protected transient NeoService neo;
66  	protected int seeds = 14;
67  	
68  	public void setT(float t){
69  		this.t = t;
70  	}
71  
72  	protected enum RelTypes implements RelationshipType
73  
74  	{
75  		NN
76  	}
77  
78  	public AbstractKnnGraph(Class<O> type,
79  			IncrementalPivotSelector<O> pivotSelector, int pivotCount,
80  			int localk) throws OBStorageException, OBException {
81  		super(type, pivotSelector, pivotCount);
82  		this.localk = localk;
83  	}
84  	
85  	/**
86  	 * Set the seeds to be used in search as value.
87  	 * @param value
88  	 */
89  	public void setSeeds(int value){
90  		this.seeds = value;
91  	}
92  	
93  
94  	/**
95  	 * Buckets do not have duplicates to preserve space.
96  	 */
97  	protected void initByteArrayBuckets() throws OBException {
98  		OBStorageConfig conf = new OBStorageConfig();
99  		conf.setTemp(false);
100 		conf.setDuplicates(false);
101 		conf.setBulkMode(!isFrozen());
102 		this.Buckets = fact.createOBStore("Buckets", conf);
103 	}
104 
105 	/**
106 	 * Fill data into node n from bucket. Node n will be modified.
107 	 * 
108 	 * @param n
109 	 *            Node where to fill the data in.
110 	 * @param bucket
111 	 *            The bucket where we will take the data.
112 	 * @return
113 	 * @throws OBException 
114 	 */
115 	protected void fillNode(Node n, B bucket) throws OBException {
116 		// add the bucket id to the given node.
117 		// all the objects here have the same smap vector.
118 		long[] idsToStore = null;
119 		if (n.hasProperty(PROP_IDS)) {
120 			long[] ids = (long[]) n.getProperty(PROP_IDS);
121 			idsToStore = new long[ids.length + 1];
122 			System.arraycopy(ids, 0, idsToStore, 0, ids.length);
123 		} else {
124 			idsToStore = new long[1];
125 		}
126 		idsToStore[idsToStore.length - 1] = bucket.getId();
127 		n.setProperty(PROP_IDS, idsToStore);
128 
129 		// fill the remaining type specific stuff
130 		fillNodeAux(n, bucket);
131 	}
132 
133 	protected abstract void fillNodeAux(Node n, B Bucket) throws OBException;
134 
135 	
136 	/**
137 	 * Return a node id from a gray code
138 	 * @param grayCode
139 	 * @return -1 if the code was not found.
140 	 * @throws OBStorageException 
141 	 * @throws IllegalArgumentException 
142 	 */
143 	protected long getNodeId(byte[] grayCode) throws IllegalArgumentException, OBStorageException{
144 		ByteBuffer b = Buckets.getValue(grayCode);
145 		if(b == null){
146 			return -1;
147 		}
148 		return b.getLong();
149 	}
150 	
151 	/**
152 	 * TODO: test if the element exists in the DB. Put this method as insertBucketBulk
153 	 */
154 	protected OperationStatus insertBucket(B b, O object)
155 			throws OBStorageException, IllegalIdException,
156 			IllegalAccessException, InstantiationException,
157 			OutOfRangeException, OBException {
158 		OperationStatus res = exists(object);
159 		if(res.getStatus() == Status.NOT_EXISTS){
160 			return insertBucketBulk(b, object);
161 		}else{
162 			return res;
163 		}
164 	}
165 	
166 	
167 	
168 	@Override
169 	public void freeze() throws IOException, AlreadyFrozenException,
170 			IllegalIdException, IllegalAccessException, InstantiationException,
171 			OBStorageException, OutOfRangeException, OBException {
172 
173 		super.freeze();
174 		CloseIterator<TupleLong> it = A.processAll();
175 		int i = 0;
176 		while(it.hasNext()){
177 			TupleLong t = it.next();
178 			O o = super.bytesToObject(t.getValue());
179 			B b = getBucket(o);
180 			b.setId(t.getKey());
181 			if(i % 100 == 0){
182 				logger.info("Bulk insert: " + i);
183 			}
184 			insertBucketBulk(b,o);
185 			i ++;
186 		}
187 		it.closeCursor();
188 		
189 		logger.debug("Gray size: " + Buckets.size());
190 	}
191 	
192 	
193 	
194 
195 	@Override
196 	public void init(OBStoreFactory fact) throws OBStorageException,
197 			OBException, InstantiationException, IllegalAccessException {
198 		// TODO Auto-generated method stub
199 		super.init(fact);
200 		
201 		neo = new EmbeddedNeo((new File(fact.getFactoryLocation(), "neo")).getAbsolutePath() );
202 	}
203 	
204 	
205 
206 	
207 	
208 	
209 	
210 	
211 	
212 	@Override
213 	public void close() throws OBException {
214 		super.close();
215 		neo.shutdown();
216 	}
217 
218 	
219 
220 	
221 
222 }