AfflictionNetworks/narya
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added methods and @Intern annotation to stream pooled strings (a

Browse Source 
la String.intern) as we do class names, by sending the value the 
first time and a short code thereafter (with special handling for 
datagrams, where me must continue sending the mapping until we 
know they've been received).  This is mostly useful for the 
lengthy config names and parameter names of Project X, but it 
could conceivably also be used for things like NamedEvents, which 
stream the same object field names many times in the course of a 
stream.


git-svn-id: svn+ssh://src.earth.threerings.net/narya/trunk@5797 542714f4-19e9-0310-aa3c-eee0fc999fb1
This commit is contained in:
Andrzej Kapolka
2009-05-22 22:43:19 +00:00
parent e5172053c8
commit 1bdb2cae16
7 changed files with 345 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/java/com/threerings/io/FieldMarshaller.java
+22
View File
@@ -90,6 +90,11 @@ public abstract class FieldMarshaller
ftype = Object.class;
}
// use the intern marshaller for pooled strings
if (ftype == String.class && field.isAnnotationPresent(Intern.class)) {
return _internMarshaller;
}
// if we have an exact match, use that
FieldMarshaller fm = _marshallers.get(ftype);
@@ -356,11 +361,28 @@ public abstract class FieldMarshaller
_marshallers.put(BasicStreamers.BSTREAMER_TYPES[ii],
new StreamerMarshaller(BasicStreamers.BSTREAMER_INSTANCES[ii]));
}
// create the field marshaller for pooled strings
_internMarshaller = new FieldMarshaller() {
@Override
public void readField (Field field, Object target, ObjectInputStream in)
throws Exception {
field.set(target, in.readIntern());
}
@Override
public void writeField (Field field, Object source, ObjectOutputStream out)
throws Exception {
out.writeIntern((String)field.get(source));
}
};
}
/** Contains a mapping from field type to field marshaller instance for that type. */
protected static HashMap<Class<?>, FieldMarshaller> _marshallers;
/** The field marshaller for pooled strings. */
protected static FieldMarshaller _internMarshaller;
/** Defines the signature to a custom field reader method. */
protected static final Class<?>[] READER_ARGS = { ObjectInputStream.class };
src/java/com/threerings/io/Intern.java
+38
View File
@@ -0,0 +1,38 @@
//
// $Id$
//
// Narya library - tools for developing networked games
// Copyright (C) 2002-2008 Three Rings Design, Inc., All Rights Reserved
// http://www.threerings.net/code/narya/
//
// This library is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation; either version 2.1 of the License, or
// (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package com.threerings.io;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Flags a {@link String} field in a streamable object as being a member of the global string pool
* accessed using {@link String#intern}. Each unique value will be streamed only once, with
* further instances replaced by a compact reference.
*/
@Target({ ElementType.FIELD })
@Retention(RetentionPolicy.RUNTIME)
public @interface Intern
{
}
src/java/com/threerings/io/ObjectInputStream.java
+61
View File
@@ -100,6 +100,64 @@ public class ObjectInputStream extends DataInputStream
}
}
/**
* Reads a pooled string value from the input stream.
*/
public String readIntern ()
throws IOException
{
// create our intern map if necessary
if (_internmap == null) {
_internmap = new ArrayList<String>();
// insert a zeroth element
_internmap.add(null);
}
// read in the intern code for this instance
short code = readShort();
// a zero code indicates a null value
if (code == 0) {
return null;
// if the code is negative, that means that we've never seen if before and value follows
} else if (code < 0) {
// first swap the code into positive-land
code *= -1;
// read in the value
String value = readUTF().intern();
// create the mapping and return the value
mapIntern(code, value);
return value;
} else {
String value = (code < _internmap.size()) ? _internmap.get(code) : null;
// sanity check
if (value == null) {
// this will help with debugging
log.warning("Internal stream error, no intern value", "code", code,
"ois", this, new Exception());
log.warning("ObjectInputStream mappings", "map", _internmap);
String errmsg = "Read intern code for which we have no registered value " +
"metadata [code=" + code + "]";
throw new RuntimeException(errmsg);
}
return value;
}
}
/**
* Adds the intern mapping for the specified code and value.
*/
protected void mapIntern (short code, String value)
throws IOException
{
_internmap.add(code, value);
}
/**
* Reads a class mapping from the stream.
*
@@ -253,6 +311,9 @@ public class ObjectInputStream extends DataInputStream
* them. */
protected ArrayList<ClassMapping> _classmap;
/** Maps numeric codes to pooled strings. */
protected ArrayList<String> _internmap;
/** The object currently being read from the stream. */
protected Object _current;
src/java/com/threerings/io/ObjectOutputStream.java
+88 -5
View File
@@ -77,6 +77,83 @@ public class ObjectOutputStream extends DataOutputStream
writeBareObject(object, cmap.streamer, true);
}
/**
* Writes a pooled string value to the output stream.
*/
public void writeIntern (String value)
throws IOException
{
// if the value to be written is null, simply write a zero
if (value == null) {
writeShort(0);
return;
}
// create our intern map if necessary
if (_internmap == null) {
_internmap = new HashMap<String, Short>();
}
// look up the intern mapping record
Short code = _internmap.get(value);
// create a mapping for the value if we've not got one
if (code == null) {
if (ObjectInputStream.STREAM_DEBUG) {
log.info(hashCode() + ": Creating intern mapping", "code", _nextInternCode,
"value", value);
}
code = createInternMapping(_nextInternCode++);
_internmap.put(value.intern(), code);
// make sure we didn't blow past our maximum intern count
if (_nextInternCode <= 0) {
throw new RuntimeException("Too many unique interns written to ObjectOutputStream");
}
writeNewInternMapping(code, value);
} else {
writeExistingInternMapping(code, value);
}
}
/**
* Creates and returns a new intern mapping.
*/
protected Short createInternMapping (short code)
{
return code;
}
/**
* Writes a new intern mapping to the stream.
*/
protected void writeNewInternMapping (short code, String value)
throws IOException
{
// writing a negative code indicates that the value will follow
writeInternMapping(-code, value);
}
/**
* Writes an existing intern mapping to the stream.
*/
protected void writeExistingInternMapping (short code, String value)
throws IOException
{
writeShort(code);
}
/**
* Writes out the mapping for an intern.
*/
protected void writeInternMapping (int code, String value)
throws IOException
{
writeShort(code);
writeUTF(value);
}
/**
* Retrieves or creates the class mapping for the supplied class, writes it out to the stream,
* and returns a reference to it.
@@ -97,17 +174,17 @@ public class ObjectOutputStream extends DataOutputStream
// create a streamer instance and assign a code to this class
Streamer streamer = Streamer.getStreamer(sclass);
// we specifically do not inline the getStreamer() call into the ClassMapping
// constructor because we want to be sure not to call _nextCode++ if getStreamer()
// constructor because we want to be sure not to call _nextClassCode++ if getStreamer()
// throws an exception
if (ObjectInputStream.STREAM_DEBUG) {
log.info(hashCode() + ": Creating class mapping", "code", _nextCode,
log.info(hashCode() + ": Creating class mapping", "code", _nextClassCode,
"class", sclass.getName());
}
cmap = createClassMapping(_nextCode++, sclass, streamer);
cmap = createClassMapping(_nextClassCode++, sclass, streamer);
_classmap.put(sclass, cmap);
// make sure we didn't blow past our maximum class count
if (_nextCode <= 0) {
if (_nextClassCode <= 0) {
throw new RuntimeException("Too many unique classes written to ObjectOutputStream");
}
writeNewClassMapping(cmap);
@@ -215,8 +292,14 @@ public class ObjectOutputStream extends DataOutputStream
* them. */
protected Map<Class<?>, ClassMapping> _classmap;
/** Used to map pooled strings to numeric codes. */
protected Map<String, Short> _internmap;
/** A counter used to assign codes to streamed classes. */
protected short _nextCode = 1;
protected short _nextClassCode = 1;
/** A counter used to assign codes to pooled strings. */
protected short _nextInternCode = 1;
/** The object currently being written to the stream. */
protected Object _current;
src/java/com/threerings/io/UnreliableObjectInputStream.java
+21
View File
@@ -43,4 +43,25 @@ public class UnreliableObjectInputStream extends ObjectInputStream
_classmap.set(code, cmap);
return cmap;
}
@Override // documentation inherited
protected void mapIntern (short code, String value)
{
// see if we already have a mapping
String ovalue = (code < _internmap.size()) ? _internmap.get(code) : null;
if (ovalue != null) {
// sanity check
if (!ovalue.equals(value)) {
throw new RuntimeException(
"Received mapping for intern that conflicts with existing mapping " +
"[code=" + code + ", ovalue=" + ovalue + ", nvalue=" + value + "]");
}
return;
}
// insert null entries for missing mappings
for (int ii = 0, nn = (code + 1) - _internmap.size(); ii < nn; ii++) {
_internmap.add(null);
}
_internmap.set(code, value);
}
}
src/java/com/threerings/io/UnreliableObjectOutputStream.java
+91 -3
View File
@@ -43,10 +43,27 @@ public class UnreliableObjectOutputStream extends ObjectOutputStream
}
/**
* Notes that the receiver has received the mappings for a group of classes, and thus that from
* now on, only the class codes need be sent.
* Sets the reference to the set that will hold the pooled strings for which mappings have been
* written.
*/
public void noteMappingsReceived (Collection<Class<?>> sclasses)
public void setMappedInterns (Set<String> mappedInterns)
{
_mappedInterns = mappedInterns;
}
/**
* Returns a reference to the set of pooled strings for which mappings have been written.
*/
public Set<String> getMappedInterns ()
{
return _mappedInterns;
}
/**
* Notes that the receiver has received the mappings for a group of classes and thus that from
* now on, only the codes need be sent.
*/
public void noteClassMappingsReceived (Collection<Class<?>> sclasses)
{
// sanity check
if (_classmap == null) {
@@ -63,6 +80,74 @@ public class UnreliableObjectOutputStream extends ObjectOutputStream
}
}
/**
* Notes that the receiver has received the mappings for a group of interns and thus that from
* now on, only the codes need be sent.
*/
public void noteInternMappingsReceived (Collection<String> sinterns)
{
// sanity check
if (_internmap == null) {
throw new RuntimeException("Missing intern map");
}
// make each intern's code positive to signify that we no longer need to send metadata
for (String sintern : sinterns) {
Short code = _internmap.get(sintern);
if (code == null) {
throw new RuntimeException("No intern mapping for " + sintern);
}
short scode = code;
if (scode < 0) {
_internmap.put(sintern, (short)-code);
}
}
}
@Override // documentation inherited
protected Short createInternMapping (short code)
{
// the negative intern code indicates that we must rewrite the metadata for the first
// instance in each go-round; when we are notified that the other side has cached the
// mapping, we can simply write the (positive) code
return (short)-code;
}
@Override // documentation inherited
protected void writeNewInternMapping (short code, String value)
throws IOException
{
writeInternMapping(code, value);
}
@Override // documentation inherited
protected void writeExistingInternMapping (short code, String value)
throws IOException
{
// if the other side has received the mapping, we need only send the reference
if (code > 0) {
writeShort(code);
// likewise if we've written the value once this go-round
} else if (_mappedInterns.contains(value)) {
writeShort(-code);
// otherwise, we must retransmit the mapping
} else {
writeInternMapping(code, value);
}
}
@Override // documentation inherited
protected void writeInternMapping (int code, String value)
throws IOException
{
super.writeInternMapping(code, value);
// note that we've written the mapping
_mappedInterns.add(value);
}
@Override // documentation inherited
protected ClassMapping createClassMapping (short code, Class<?> sclass, Streamer streamer)
{
@@ -109,4 +194,7 @@ public class UnreliableObjectOutputStream extends ObjectOutputStream
/** The set of classes for which we have written mappings. */
protected Set<Class<?>> _mappedClasses = new HashSet<Class<?>>();
/** The set of pooled strings for which we have written mappings. */
protected Set<String> _mappedInterns = new HashSet<String>();
}
src/java/com/threerings/presents/util/DatagramSequencer.java
+24 -5
View File
@@ -58,9 +58,11 @@ public class DatagramSequencer
_uout.writeInt(++_lastNumber);
_uout.writeInt(_lastReceived);
// make sure the mapped class set is clear
// make sure the mapped sets are clear
Set<Class<?>> mappedClasses = _uout.getMappedClasses();
mappedClasses.clear();
Set<String> mappedInterns = _uout.getMappedInterns();
mappedInterns.clear();
// write the object
_uout.writeObject(datagram);
@@ -72,8 +74,15 @@ public class DatagramSequencer
_uout.setMappedClasses(new HashSet<Class<?>>());
}
// likewise with the intern mappings
if (mappedInterns.isEmpty()) {
mappedInterns = null;
} else {
_uout.setMappedInterns(new HashSet<String>());
}
// record the transmission
_sendrecs.add(new SendRecord(_lastNumber, mappedClasses));
_sendrecs.add(new SendRecord(_lastNumber, mappedClasses, mappedInterns));
}
/**
@@ -101,8 +110,13 @@ public class DatagramSequencer
break;
}
remove++;
if (sendrec.number == received && sendrec.mappedClasses != null) {
_uout.noteMappingsReceived(sendrec.mappedClasses);
if (sendrec.number == received) {
if (sendrec.mappedClasses != null) {
_uout.noteClassMappingsReceived(sendrec.mappedClasses);
}
if (sendrec.mappedInterns != null) {
_uout.noteInternMappingsReceived(sendrec.mappedInterns);
}
}
}
_sendrecs.subList(0, remove).clear();
@@ -125,10 +139,15 @@ public class DatagramSequencer
* <code>null</code> for none). */
public Set<Class<?>> mappedClasses;
public SendRecord (int number, Set<Class<?>> mappedClasses)
/** The set of interns for which mappings were included in the datagram (or
* <code>null</code> for none). */
public Set<String> mappedInterns;
public SendRecord (int number, Set<Class<?>> mappedClasses, Set<String> mappedInterns)
{
this.number = number;
this.mappedClasses = mappedClasses;
this.mappedInterns = mappedInterns;
}
}