diff --git a/src/java/com/threerings/io/FieldMarshaller.java b/src/java/com/threerings/io/FieldMarshaller.java index eac1ad5fc..f55df54b4 100644 --- a/src/java/com/threerings/io/FieldMarshaller.java +++ b/src/java/com/threerings/io/FieldMarshaller.java @@ -90,6 +90,11 @@ public abstract class FieldMarshaller ftype = Object.class; } + // use the intern marshaller for pooled strings + if (ftype == String.class && field.isAnnotationPresent(Intern.class)) { + return _internMarshaller; + } + // if we have an exact match, use that FieldMarshaller fm = _marshallers.get(ftype); @@ -356,11 +361,28 @@ public abstract class FieldMarshaller _marshallers.put(BasicStreamers.BSTREAMER_TYPES[ii], new StreamerMarshaller(BasicStreamers.BSTREAMER_INSTANCES[ii])); } + + // create the field marshaller for pooled strings + _internMarshaller = new FieldMarshaller() { + @Override + public void readField (Field field, Object target, ObjectInputStream in) + throws Exception { + field.set(target, in.readIntern()); + } + @Override + public void writeField (Field field, Object source, ObjectOutputStream out) + throws Exception { + out.writeIntern((String)field.get(source)); + } + }; } /** Contains a mapping from field type to field marshaller instance for that type. */ protected static HashMap, FieldMarshaller> _marshallers; + /** The field marshaller for pooled strings. */ + protected static FieldMarshaller _internMarshaller; + /** Defines the signature to a custom field reader method. */ protected static final Class[] READER_ARGS = { ObjectInputStream.class }; diff --git a/src/java/com/threerings/io/Intern.java b/src/java/com/threerings/io/Intern.java new file mode 100644 index 000000000..1f339f3aa --- /dev/null +++ b/src/java/com/threerings/io/Intern.java @@ -0,0 +1,38 @@ +// +// $Id$ +// +// Narya library - tools for developing networked games +// Copyright (C) 2002-2008 Three Rings Design, Inc., All Rights Reserved +// http://www.threerings.net/code/narya/ +// +// This library is free software; you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License as published +// by the Free Software Foundation; either version 2.1 of the License, or +// (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package com.threerings.io; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Flags a {@link String} field in a streamable object as being a member of the global string pool + * accessed using {@link String#intern}. Each unique value will be streamed only once, with + * further instances replaced by a compact reference. + */ +@Target({ ElementType.FIELD }) +@Retention(RetentionPolicy.RUNTIME) +public @interface Intern +{ +} diff --git a/src/java/com/threerings/io/ObjectInputStream.java b/src/java/com/threerings/io/ObjectInputStream.java index 2c9df5435..8f1e6aa56 100644 --- a/src/java/com/threerings/io/ObjectInputStream.java +++ b/src/java/com/threerings/io/ObjectInputStream.java @@ -100,6 +100,64 @@ public class ObjectInputStream extends DataInputStream } } + /** + * Reads a pooled string value from the input stream. + */ + public String readIntern () + throws IOException + { + // create our intern map if necessary + if (_internmap == null) { + _internmap = new ArrayList(); + // insert a zeroth element + _internmap.add(null); + } + + // read in the intern code for this instance + short code = readShort(); + + // a zero code indicates a null value + if (code == 0) { + return null; + + // if the code is negative, that means that we've never seen if before and value follows + } else if (code < 0) { + // first swap the code into positive-land + code *= -1; + + // read in the value + String value = readUTF().intern(); + + // create the mapping and return the value + mapIntern(code, value); + return value; + + } else { + String value = (code < _internmap.size()) ? _internmap.get(code) : null; + + // sanity check + if (value == null) { + // this will help with debugging + log.warning("Internal stream error, no intern value", "code", code, + "ois", this, new Exception()); + log.warning("ObjectInputStream mappings", "map", _internmap); + String errmsg = "Read intern code for which we have no registered value " + + "metadata [code=" + code + "]"; + throw new RuntimeException(errmsg); + } + return value; + } + } + + /** + * Adds the intern mapping for the specified code and value. + */ + protected void mapIntern (short code, String value) + throws IOException + { + _internmap.add(code, value); + } + /** * Reads a class mapping from the stream. * @@ -253,6 +311,9 @@ public class ObjectInputStream extends DataInputStream * them. */ protected ArrayList _classmap; + /** Maps numeric codes to pooled strings. */ + protected ArrayList _internmap; + /** The object currently being read from the stream. */ protected Object _current; diff --git a/src/java/com/threerings/io/ObjectOutputStream.java b/src/java/com/threerings/io/ObjectOutputStream.java index a91834f70..8bfb0a054 100644 --- a/src/java/com/threerings/io/ObjectOutputStream.java +++ b/src/java/com/threerings/io/ObjectOutputStream.java @@ -77,6 +77,83 @@ public class ObjectOutputStream extends DataOutputStream writeBareObject(object, cmap.streamer, true); } + /** + * Writes a pooled string value to the output stream. + */ + public void writeIntern (String value) + throws IOException + { + // if the value to be written is null, simply write a zero + if (value == null) { + writeShort(0); + return; + } + + // create our intern map if necessary + if (_internmap == null) { + _internmap = new HashMap(); + } + + // look up the intern mapping record + Short code = _internmap.get(value); + + // create a mapping for the value if we've not got one + if (code == null) { + if (ObjectInputStream.STREAM_DEBUG) { + log.info(hashCode() + ": Creating intern mapping", "code", _nextInternCode, + "value", value); + } + code = createInternMapping(_nextInternCode++); + _internmap.put(value.intern(), code); + + // make sure we didn't blow past our maximum intern count + if (_nextInternCode <= 0) { + throw new RuntimeException("Too many unique interns written to ObjectOutputStream"); + } + writeNewInternMapping(code, value); + + } else { + writeExistingInternMapping(code, value); + } + } + + /** + * Creates and returns a new intern mapping. + */ + protected Short createInternMapping (short code) + { + return code; + } + + /** + * Writes a new intern mapping to the stream. + */ + protected void writeNewInternMapping (short code, String value) + throws IOException + { + // writing a negative code indicates that the value will follow + writeInternMapping(-code, value); + } + + /** + * Writes an existing intern mapping to the stream. + */ + protected void writeExistingInternMapping (short code, String value) + throws IOException + { + writeShort(code); + } + + /** + * Writes out the mapping for an intern. + */ + protected void writeInternMapping (int code, String value) + throws IOException + { + writeShort(code); + writeUTF(value); + } + /** * Retrieves or creates the class mapping for the supplied class, writes it out to the stream, * and returns a reference to it. @@ -97,17 +174,17 @@ public class ObjectOutputStream extends DataOutputStream // create a streamer instance and assign a code to this class Streamer streamer = Streamer.getStreamer(sclass); // we specifically do not inline the getStreamer() call into the ClassMapping - // constructor because we want to be sure not to call _nextCode++ if getStreamer() + // constructor because we want to be sure not to call _nextClassCode++ if getStreamer() // throws an exception if (ObjectInputStream.STREAM_DEBUG) { - log.info(hashCode() + ": Creating class mapping", "code", _nextCode, + log.info(hashCode() + ": Creating class mapping", "code", _nextClassCode, "class", sclass.getName()); } - cmap = createClassMapping(_nextCode++, sclass, streamer); + cmap = createClassMapping(_nextClassCode++, sclass, streamer); _classmap.put(sclass, cmap); // make sure we didn't blow past our maximum class count - if (_nextCode <= 0) { + if (_nextClassCode <= 0) { throw new RuntimeException("Too many unique classes written to ObjectOutputStream"); } writeNewClassMapping(cmap); @@ -215,8 +292,14 @@ public class ObjectOutputStream extends DataOutputStream * them. */ protected Map, ClassMapping> _classmap; + /** Used to map pooled strings to numeric codes. */ + protected Map _internmap; + /** A counter used to assign codes to streamed classes. */ - protected short _nextCode = 1; + protected short _nextClassCode = 1; + + /** A counter used to assign codes to pooled strings. */ + protected short _nextInternCode = 1; /** The object currently being written to the stream. */ protected Object _current; diff --git a/src/java/com/threerings/io/UnreliableObjectInputStream.java b/src/java/com/threerings/io/UnreliableObjectInputStream.java index bf773b5d4..594ca6a9e 100644 --- a/src/java/com/threerings/io/UnreliableObjectInputStream.java +++ b/src/java/com/threerings/io/UnreliableObjectInputStream.java @@ -43,4 +43,25 @@ public class UnreliableObjectInputStream extends ObjectInputStream _classmap.set(code, cmap); return cmap; } + + @Override // documentation inherited + protected void mapIntern (short code, String value) + { + // see if we already have a mapping + String ovalue = (code < _internmap.size()) ? _internmap.get(code) : null; + if (ovalue != null) { + // sanity check + if (!ovalue.equals(value)) { + throw new RuntimeException( + "Received mapping for intern that conflicts with existing mapping " + + "[code=" + code + ", ovalue=" + ovalue + ", nvalue=" + value + "]"); + } + return; + } + // insert null entries for missing mappings + for (int ii = 0, nn = (code + 1) - _internmap.size(); ii < nn; ii++) { + _internmap.add(null); + } + _internmap.set(code, value); + } } diff --git a/src/java/com/threerings/io/UnreliableObjectOutputStream.java b/src/java/com/threerings/io/UnreliableObjectOutputStream.java index 8b2c1b3c8..2a403d365 100644 --- a/src/java/com/threerings/io/UnreliableObjectOutputStream.java +++ b/src/java/com/threerings/io/UnreliableObjectOutputStream.java @@ -43,10 +43,27 @@ public class UnreliableObjectOutputStream extends ObjectOutputStream } /** - * Notes that the receiver has received the mappings for a group of classes, and thus that from - * now on, only the class codes need be sent. + * Sets the reference to the set that will hold the pooled strings for which mappings have been + * written. */ - public void noteMappingsReceived (Collection> sclasses) + public void setMappedInterns (Set mappedInterns) + { + _mappedInterns = mappedInterns; + } + + /** + * Returns a reference to the set of pooled strings for which mappings have been written. + */ + public Set getMappedInterns () + { + return _mappedInterns; + } + + /** + * Notes that the receiver has received the mappings for a group of classes and thus that from + * now on, only the codes need be sent. + */ + public void noteClassMappingsReceived (Collection> sclasses) { // sanity check if (_classmap == null) { @@ -63,6 +80,74 @@ public class UnreliableObjectOutputStream extends ObjectOutputStream } } + /** + * Notes that the receiver has received the mappings for a group of interns and thus that from + * now on, only the codes need be sent. + */ + public void noteInternMappingsReceived (Collection sinterns) + { + // sanity check + if (_internmap == null) { + throw new RuntimeException("Missing intern map"); + } + + // make each intern's code positive to signify that we no longer need to send metadata + for (String sintern : sinterns) { + Short code = _internmap.get(sintern); + if (code == null) { + throw new RuntimeException("No intern mapping for " + sintern); + } + short scode = code; + if (scode < 0) { + _internmap.put(sintern, (short)-code); + } + } + } + + @Override // documentation inherited + protected Short createInternMapping (short code) + { + // the negative intern code indicates that we must rewrite the metadata for the first + // instance in each go-round; when we are notified that the other side has cached the + // mapping, we can simply write the (positive) code + return (short)-code; + } + + @Override // documentation inherited + protected void writeNewInternMapping (short code, String value) + throws IOException + { + writeInternMapping(code, value); + } + + @Override // documentation inherited + protected void writeExistingInternMapping (short code, String value) + throws IOException + { + // if the other side has received the mapping, we need only send the reference + if (code > 0) { + writeShort(code); + + // likewise if we've written the value once this go-round + } else if (_mappedInterns.contains(value)) { + writeShort(-code); + + // otherwise, we must retransmit the mapping + } else { + writeInternMapping(code, value); + } + } + + @Override // documentation inherited + protected void writeInternMapping (int code, String value) + throws IOException + { + super.writeInternMapping(code, value); + + // note that we've written the mapping + _mappedInterns.add(value); + } + @Override // documentation inherited protected ClassMapping createClassMapping (short code, Class sclass, Streamer streamer) { @@ -109,4 +194,7 @@ public class UnreliableObjectOutputStream extends ObjectOutputStream /** The set of classes for which we have written mappings. */ protected Set> _mappedClasses = new HashSet>(); + + /** The set of pooled strings for which we have written mappings. */ + protected Set _mappedInterns = new HashSet(); } diff --git a/src/java/com/threerings/presents/util/DatagramSequencer.java b/src/java/com/threerings/presents/util/DatagramSequencer.java index cae4702f1..2f088f12f 100644 --- a/src/java/com/threerings/presents/util/DatagramSequencer.java +++ b/src/java/com/threerings/presents/util/DatagramSequencer.java @@ -58,9 +58,11 @@ public class DatagramSequencer _uout.writeInt(++_lastNumber); _uout.writeInt(_lastReceived); - // make sure the mapped class set is clear + // make sure the mapped sets are clear Set> mappedClasses = _uout.getMappedClasses(); mappedClasses.clear(); + Set mappedInterns = _uout.getMappedInterns(); + mappedInterns.clear(); // write the object _uout.writeObject(datagram); @@ -72,8 +74,15 @@ public class DatagramSequencer _uout.setMappedClasses(new HashSet>()); } + // likewise with the intern mappings + if (mappedInterns.isEmpty()) { + mappedInterns = null; + } else { + _uout.setMappedInterns(new HashSet()); + } + // record the transmission - _sendrecs.add(new SendRecord(_lastNumber, mappedClasses)); + _sendrecs.add(new SendRecord(_lastNumber, mappedClasses, mappedInterns)); } /** @@ -101,8 +110,13 @@ public class DatagramSequencer break; } remove++; - if (sendrec.number == received && sendrec.mappedClasses != null) { - _uout.noteMappingsReceived(sendrec.mappedClasses); + if (sendrec.number == received) { + if (sendrec.mappedClasses != null) { + _uout.noteClassMappingsReceived(sendrec.mappedClasses); + } + if (sendrec.mappedInterns != null) { + _uout.noteInternMappingsReceived(sendrec.mappedInterns); + } } } _sendrecs.subList(0, remove).clear(); @@ -125,10 +139,15 @@ public class DatagramSequencer * null for none). */ public Set> mappedClasses; - public SendRecord (int number, Set> mappedClasses) + /** The set of interns for which mappings were included in the datagram (or + * null for none). */ + public Set mappedInterns; + + public SendRecord (int number, Set> mappedClasses, Set mappedInterns) { this.number = number; this.mappedClasses = mappedClasses; + this.mappedInterns = mappedInterns; } }