r9625 - helma/helma/branches/lazy_collections/src/helma/objectmodel/db
[email protected] Fri, 17 Apr 2009 16:16:06 +0200 (CEST)
| Newsgroups | gmane.comp.java.helma.cvs |
|---|---|
| Message-ID | <20090417141606.D68813D0D6@mia> |
Author: hannes
Date: 2009-04-17 16:16:06 +0200 (Fri, 17 Apr 2009)
New Revision: 9625
Modified:
helma/helma/branches/lazy_collections/src/helma/objectmodel/db/Node.java
helma/helma/branches/lazy_collections/src/helma/objectmodel/db/SegmentedSubnodeList.java
helma/helma/branches/lazy_collections/src/helma/objectmodel/db/SubnodeList.java
Log:
Implement segmented subnode index loading.
Details at http://dev.helma.org/trac/helma/changeset/9625
Modified: helma/helma/branches/lazy_collections/src/helma/objectmodel/db/Node.java
===================================================================
--- helma/helma/branches/lazy_collections/src/helma/objectmodel/db/Node.java 2009-04-16 22:42:32 UTC (rev 9624)
+++ helma/helma/branches/lazy_collections/src/helma/objectmodel/db/Node.java 2009-04-17 14:16:06 UTC (rev 9625)
@@ -645,8 +645,7 @@
DbMapping smap = (dbmap == null) ? null : dbmap.getSubnodeMapping();
if (subnodes != null && smap != null && smap.isRelational()) {
- subnodes.subnodeCount = -1;
- subnodes.lastSubnodeFetch = -1;
+ subnodes = null;
}
}
@@ -1539,7 +1538,7 @@
* @return List an empty List of the type used by this Node
*/
public SubnodeList createSubnodeList() {
- subnodes = new SubnodeList(this);
+ subnodes = new SegmentedSubnodeList(this);
return subnodes;
}
Modified: helma/helma/branches/lazy_collections/src/helma/objectmodel/db/SegmentedSubnodeList.java
===================================================================
--- helma/helma/branches/lazy_collections/src/helma/objectmodel/db/SegmentedSubnodeList.java 2009-04-16 22:42:32 UTC (rev 9624)
+++ helma/helma/branches/lazy_collections/src/helma/objectmodel/db/SegmentedSubnodeList.java 2009-04-17 14:16:06 UTC (rev 9625)
@@ -11,15 +11,16 @@
package helma.objectmodel.db;
-import java.util.BitSet;
-import java.util.List;
+import java.util.*;
public class SegmentedSubnodeList extends SubnodeList {
- transient BitSet loadmap;
- transient int loadstatus;
- transient List[] keys;
+ transient Segment[] segments;
+ static int SEGLENGTH = 10000;
+ transient long lastSubnodeCount = 0;
+ transient int subnodeCount = -1;
+
/**
* Creates a new subnode list
* @param node the node we belong to
@@ -28,30 +29,201 @@
super(node);
}
- public int size() {
- // If the subnodes are loaded aggressively, we really just
- // do a count statement, otherwise we just return the size of the id index.
- // (after loading it, if it's coming from a relational data source).
- DbMapping dbmap = getSubnodeMapping();
- Relation rel = getSubnodeRelation();
+ /**
+ * Adds the specified object to this list performing
+ * custom ordering
+ *
+ * @param obj element to be inserted.
+ */
+ public synchronized boolean add(Object obj) {
+ subnodeCount++;
+ return list.add(obj);
+ }
+ /**
+ * Adds the specified object to the list at the given position
+ * @param index the index to insert the element at
+ * @param obj the object t add
+ */
+ public synchronized void add(int index, Object obj) {
+ if (!hasRelationalNodes() || segments == null) {
+ super.add(index, obj);
+ return;
+ }
+ list.add(index, obj);
+ // shift segment indices by one
+ int s = getSegment(index);
+ segments[s].length += 1;
+ for (int i = s + 1; i < segments.length; i++) {
+ segments[i].startIndex += 1;
+ }
+ }
- if (dbmap == null || !dbmap.isRelational() || rel.getGroup() != null) {
- return super.size();
+ public Object get(int index) {
+ if (!hasRelationalNodes() || segments == null) {
+ return super.get(index);
}
+ if (index < 0 || index >= subnodeCount) {
+ return null;
+ }
+ loadSegment(getSegment(index), false);
+ return list.get(index);
+ }
- if (node.getSubnodeRelation() == null &&
- (node.getState() == Node.TRANSIENT || node.getState() == Node.NEW)) {
+ public synchronized boolean contains(Object object) {
+ if (!hasRelationalNodes() || segments == null) {
+ return super.contains(object);
+ }
+ if (list.contains(object)) {
+ return true;
+ }
+ for (int i = 0; i < segments.length; i++) {
+ if (loadSegment(i, false).contains(object)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public synchronized int indexOf(Object object) {
+ if (!hasRelationalNodes() || segments == null) {
+ return super.indexOf(object);
+ }
+ int index;
+ if ((index = list.indexOf(object)) > -1) {
+ return index;
+ }
+ for (int i = 0; i < segments.length; i++) {
+ if ((index = loadSegment(i, false).indexOf(object)) > -1) {
+ return segments[i].startIndex + index;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * remove the object specified by the given index-position
+ * @param index the index-position of the NodeHandle to remove
+ */
+ public synchronized Object remove(int index) {
+ if (!hasRelationalNodes() || segments == null) {
+ return super.remove(index);
+ }
+ Object removed = list.remove(index);
+ int s = getSegment(index);
+ segments[s].length -= 1;
+ for (int i = s + 1; i < segments.length; i++) {
+ segments[i].startIndex -= 1;
+ }
+ return removed;
+ }
+
+ /**
+ * remove the given Object from this List
+ * @param object the NodeHandle to remove
+ */
+ public synchronized boolean remove(Object object) {
+ if (!hasRelationalNodes() || segments == null) {
+ return super.remove(object);
+ }
+ int index = indexOf(object);
+ if (index > -1) {
+ list.remove(object);
+ int s = getSegment(index);
+ segments[s].length -= 1;
+ for (int i = s + 1; i < segments.length; i++) {
+ segments[i].startIndex -= 1;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ public synchronized Object[] toArray() {
+ if (!hasRelationalNodes() || segments == null) {
+ return super.toArray();
+ }
+ node.nmgr.logEvent("Warning: toArray() called on large segmented collection: " + node);
+ for (int i = 0; i < segments.length; i++) {
+ loadSegment(i, false);
+ }
+ return list.toArray();
+ }
+
+ private int getSegment(int index) {
+ for (int i = 1; i < segments.length; i++) {
+ if (index < segments[i].startIndex) {
+ return i - 1;
+ }
+ }
+ return segments.length - 1;
+ }
+
+ private List loadSegment(int seg, boolean deep) {
+ Segment segment = segments[seg];
+ if (segment != null && !segment.loaded) {
+ Relation rel = getSubnodeRelation().getClone();
+ rel.offset = segment.startIndex;
+ int expectedSize = rel.maxSize = segment.length;
+ List seglist = deep ?
+ node.nmgr.getNodes(node, rel) :
+ node.nmgr.getNodeIDs(node, rel);
+ int actualSize = seglist.size();
+ if (actualSize != expectedSize) {
+ node.nmgr.logEvent("Inconsistent segment size in " + node + ": " + segment);
+ }
+ int listSize = list.size();
+ for (int i = 0; i < actualSize; i++) {
+ if (segment.startIndex + i < listSize) {
+ list.set(segment.startIndex + i, seglist.get(i));
+ } else {
+ list.add(seglist.get(i));
+ }
+ // FIXME how to handle inconsistencies?
+ }
+ segment.loaded = true;
+ return seglist;
+ }
+ return Collections.EMPTY_LIST;
+ }
+
+ protected synchronized void update() {
+ if (!hasRelationalNodes()) {
+ super.update();
+ }
+ // also reload if the type mapping has changed.
+ long lastChange = getLastSubnodeChange();
+ if (lastChange != lastSubnodeFetch) {
+ float size = size();
+ if (size > SEGLENGTH) {
+ int nsegments = (int) Math.ceil(size / SEGLENGTH);
+ int remainder = (int) size % SEGLENGTH;
+ segments = new Segment[nsegments];
+ for (int s = 0; s < nsegments; s++) {
+ int length = (s == nsegments - 1 && remainder > 0) ?
+ remainder : SEGLENGTH;
+ segments[s] = new Segment(s * SEGLENGTH, length);
+ }
+ list = new ArrayList((int) size + 5);
+ for (int i = 0; i < size; i++) {
+ list.add(null);
+ }
+ } else {
+ segments = null;
+ super.update();
+ }
+ lastSubnodeFetch = lastChange;
+ }
+ }
+
+ public int size() {
+ if (!hasRelationalNodes()) {
return super.size();
}
- // we don't want to load *all* nodes if we just want to count them
+ Relation rel = getSubnodeRelation();
long lastChange = getLastSubnodeChange();
- if (lastChange == lastSubnodeFetch) {
- // we can use the nodes vector to determine number of subnodes
- subnodeCount = list.size();
- lastSubnodeCount = lastChange;
- } else if (lastChange != lastSubnodeCount || subnodeCount < 0) {
+ if (lastChange != lastSubnodeCount || subnodeCount < 0) {
// count nodes in db without fetching anything
subnodeCount = node.nmgr.countNodes(node, rel);
lastSubnodeCount = lastChange;
@@ -59,5 +231,25 @@
return subnodeCount;
}
+ class Segment {
+
+ int startIndex, length;
+ boolean loaded;
+
+ Segment(int startIndex, int length) {
+ this.startIndex = startIndex;
+ this.length = length;
+ this.loaded = false;
+ }
+
+ int endIndex() {
+ return startIndex + length;
+ }
+
+ public String toString() {
+ return "Segment{startIndex: " + startIndex + ", length: " + length + "}";
+ }
+ }
+
}
Modified: helma/helma/branches/lazy_collections/src/helma/objectmodel/db/SubnodeList.java
===================================================================
--- helma/helma/branches/lazy_collections/src/helma/objectmodel/db/SubnodeList.java 2009-04-16 22:42:32 UTC (rev 9624)
+++ helma/helma/branches/lazy_collections/src/helma/objectmodel/db/SubnodeList.java 2009-04-17 14:16:06 UTC (rev 9625)
@@ -30,8 +30,6 @@
transient long lastSubnodeFetch = 0;
transient long lastSubnodeChange = 0;
- transient long lastSubnodeCount = 0;
- transient int subnodeCount = -1;
/**
@@ -79,7 +77,6 @@
if (handle != null) {
retval = handle.getNode(node.nmgr);
-
// Legacy alarm!
if ((retval != null) && (retval.parentHandle == null) &&
!node.nmgr.isRootNode(retval)) {
@@ -130,16 +127,13 @@
protected void update() {
// also reload if the type mapping has changed.
long lastChange = getLastSubnodeChange();
-
- if (node.getSubnodeRelation() != null) System.err.println(" *** *** *** " + lastChange + "/ " + lastSubnodeFetch);
- Relation rel = getSubnodeRelation();
if (lastChange != lastSubnodeFetch) {
+ Relation rel = getSubnodeRelation();
if (rel.aggressiveLoading && rel.groupby == null) {
list = node.nmgr.getNodes(node, rel);
} else {
list = node.nmgr.getNodeIDs(node, rel);
}
-
lastSubnodeFetch = lastChange;
}
}
@@ -160,15 +154,6 @@
if (!dbmap.isRelational() || rel.getGroup() != null) {
return;
}
-
- // this is the code we're going to use for segmented key loading!
- /* if (start > 0 || length > 0 && length < size()) {
- rel = new Relation(rel);
- rel.offset = start;
- rel.maxSize = length < 0 ? size() : length;
- }
- node.nmgr.getNodes(node, rel); */
-
node.nmgr.prefetchNodes(node, rel, this, start, length);
}
@@ -176,7 +161,7 @@
* Compute a serial number indicating the last change in subnode collection
* @return a serial number that increases with each subnode change
*/
- long getLastSubnodeChange() {
+ protected long getLastSubnodeChange() {
// include dbmap.getLastTypeChange to also reload if the type mapping has changed.
long checkSum = lastSubnodeChange + node.dbmap.getLastTypeChange();
Relation rel = getSubnodeRelation();
@@ -184,6 +169,13 @@
checkSum : checkSum + rel.otherType.getLastDataChange();
}
+ protected boolean hasRelationalNodes() {
+ DbMapping dbmap = getSubnodeMapping();
+ return (dbmap != null && dbmap.isRelational()
+ && ((node.getState() != Node.TRANSIENT && node.getState() != Node.NEW)
+ || node.getSubnodeRelation() != null));
+ }
+
protected DbMapping getSubnodeMapping() {
return node.dbmap == null ? null : node.dbmap.getSubnodeMapping();
}